chore: revamp migration system

scripts/build.sh

@@ -49,4 +49,4 @@ cd "../$base_or_head"
 
 isready
 initdb
-ls ./schema/ | xargs -I{} bash -c "$(declare -f query_file); query_file \"$url/dnim\" ./schema/{}"
+ls ./schema/ | xargs -I{} bash -c "set -e; $(declare -f query_file); query_file \"$url/dnim\" ./schema/{}"

scripts/diff.sh

@@ -23,18 +23,12 @@ if [[ -z "$base" ]] || [[ -z "$head" ]]; then
 fi
 
 if [[ ! -f "$migration" ]]; then
-  ./scripts/build.sh base "$base" 1>&2
-  ./scripts/build.sh head "$head" 1>&2
+  ./scripts/build.sh base "$base" 1>&2 || (echo "base failed to build" && exit 1)
+  ./scripts/build.sh head "$head" 1>&2 || (echo "head failed to build" && exit 1)
 
   until pg_isready -p 5432 1>/dev/null && pg_isready -p 5433 1>/dev/null; do true; done;
 
-  migra --unsafe "$base_url" "$head_url" > "$migration" \
-    || echo "migra exited with code $?. this is /probably/ fine" 1>&2
-
-  cd ./migrations
-  git add --all
-  git commit -m "$migration"
-  cd ../
+  migra --unsafe "$base_url" "$head_url" || echo "migra exited with code $?. this is /probably/ fine" 1>&2
+else
+  cat "$migration"
 fi
-
-echo "$migration"

scripts/gen_migrations.sh

@@ -0,0 +1,42 @@
+#! /bin/bash
+
+set -e
+
+source ./scripts/common.sh
+
+revs=$(git log --format=%h | tac)
+first_commit="7371374"
+first_commit_n=$(echo "$revs" | grep -n "$first_commit" | awk -F: '{print $1}')
+revs=$(echo "empty" && (echo "$revs" | tail --lines "+$first_commit_n"))
+revs_ct=$(echo "$revs" | wc -l)
+
+do_i() {
+  base=$(echo "$revs" | tail --lines "+$1" | head -n 1)
+  if [[ -f "./migrations/${base}_skipped.sql" ]]; then
+    return
+  fi
+
+  for j in $(seq "$(($1 + 1))" "$revs_ct"); do
+    head=$(echo "$revs" | tail --lines "+$j" | head -n 1)
+    migration="./migrations/${base}_to_${head}.sql"
+    set +e
+    out=$(./scripts/diff.sh "$base" "$head")
+    status="$?"
+    set -e
+
+    if [[ "$status" = "1" && "$out" = *"head failed to build"* ]]; then
+      echo "" > "./migrations/${head}_skipped.sql"
+      continue
+    elif [[ "$status" = "1" ]]; then
+      echo out
+      exit 1
+    else
+      echo "$out" > "$migration"
+      break
+    fi
+  done
+}
+
+for i in $(seq 1 "$revs_ct"); do
+  do_i "$i"
+done

scripts/migrate.sh

@@ -5,15 +5,10 @@ set -e
 source ./scripts/common.sh
 source ./scripts/env.sh ./.env
 
-if [[ -n $(git status --porcelain) ]]; then
-  echo "git working tree dirty" 1>&2;
-  exit 1;
-fi
-
-to_tag="$1"
-
 get_dnim_database_count="copy (select count(*) from pg_database where datname = 'dnim') to stdout with null as '';"
-dnim_database_count=$(query "$POSTGRES_URI/postgres" "$get_dnim_database_count")
+dnim_database_count=$(query "$POSTGRES_URI/postgres" "$get_dnim_database_count" 2>&1)
+
+git submodule update --init migrations
 
 if [[ "$dnim_database_count" = "0" ]]; then
   echo "fresh database"
@@ -23,18 +18,43 @@ if [[ "$dnim_database_count" = "0" ]]; then
   script=''
 else
   get_rev_last="copy (select to_revision from migration order by performed_on desc limit 1) to stdout with null as '';"
-  rev_last=$(query "$POSTGRES_URI/dnim" "$get_rev_last")
-  migration=$(./scripts/diff.sh "$rev_last" "$to_tag")
+  rev_last=$(psql "$POSTGRES_URI/dnim" -c "$get_rev_last")
 
-  if [[ "$2" = "--greenlight" ]]; then
-    query_file "$POSTGRES_URI/dnim" "$migration"
-  else
-    echo "migration available at $migration"
-    echo "review and rerun with --greenlight to apply"
-    exit
+  log=$(git log --format=%h | tac)
+  first_commit="7371374"
+  first_commit_n=$(echo "$log" | grep -n "$first_commit" | awk -F: '{print $1}')
+  rev_most_recent=$(echo "$log" | tail -n 1)
+  log=$(echo "empty" && (echo "$log" | tail --lines "+$first_commit_n" | head --lines "-1"))
+
+  revs_to_run="$log"
+
+  if [[ "$rev_last" = "$rev_most_recent" ]]; then
+    echo "all caught up! :)"
+    exit 0
+  elif [[ "$rev_last" != "empty" ]]; then
+    rev_n=$(echo "$log" | grep -n "$rev_last" | awk -F: '{print $1}')
+    revs_to_run=$(echo "$log" | tail --lines "+$rev_n")
   fi
-fi
 
-insert_migration="insert into migration (from_revision, to_revision) values ('$rev_last', '$to_tag');"
-query "$POSTGRES_URI/dnim" "$insert_migration"
-echo "inserted migration"
+  revs_to_run_ct=$(echo "$revs_to_run" | wc -l)
+
+  for i in $(seq 1 "$revs_to_run_ct"); do
+    base=$(echo "$revs_to_run" | tail --lines "+$i" | head -n 1)
+    set +e
+    mig=$(ls migrations | grep "${base}_to")
+    set -e
+
+    if [[ -f "./migrations/${base}_skipped.sql" ]]; then
+      echo "${base} (skipped)"
+      continue
+    elif [[ -f "./migrations/$mig" ]]; then
+      head=$(echo $mig | awk -F'[_.]' '{print $3}')
+      echo "${base} -> ${head}"
+      query_file "$POSTGRES_URI/dnim" "./migrations/$mig"
+      query "$POSTGRES_URI/dnim" "insert into migration (from_revision, to_revision) values ('$base', '$head');"
+    else
+      echo "cant find migration for $base" 1>&2
+      exit 1
+    fi
+  done;
+fi