Merge pull request #15 from vulcanize/compare-migration

Compare Up and Down Migration.

.github/workflows/on-pr.yaml

@@ -21,7 +21,8 @@ jobs:
       - name: Verify schema is latest
         run: |
           PGPASSWORD="password" pg_dump -h localhost -p 8066 -U vdbm vulcanize_testing --no-owner --schema-only > ./db/migration_schema.sql
-          ./scripts/check_diff.sh
+          ./scripts/check_diff.sh ./db/migration_schema.sql db/schema.sql
+
   incremental_migration_diff:
     name: Compare conscise migration schema with incremental migration.
     runs-on: ubuntu-latest
@@ -38,4 +39,25 @@ jobs:
       - name: Verify schema is latest
         run: |
           PGPASSWORD="password" pg_dump -h localhost -p 8066 -U vdbm vulcanize_testing --no-owner --schema-only > ./db/migration_schema.sql
-          ./scripts/check_diff.sh
+          ./scripts/check_diff.sh db/schema.sql ./db/migration_schema.sql
+
+  migration:
+    name: Compare up and down migration
+    env:
+      GOPATH: /tmp/go
+    strategy:
+      matrix:
+        go-version: [ 1.16.x ]
+        os: [ ubuntu-latest ]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Create GOPATH
+        run: mkdir -p /tmp/go
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go-version }}
+      - uses: actions/checkout@v2
+      - name: Test migration
+        run: |
+          timeout 5m make test-migrations

Makefile

@@ -87,4 +87,8 @@ docker-build:
 ## Build docker image for migration
 .PHONY: docker-concise-migration-build
 docker-concise-migration-build:
-	docker build -t vulcanize/concise-migration-build -f ./db/Dockerfile .
+	docker build -t vulcanize/concise-migration-build -f ./db/Dockerfile .
+
+.PHONY: test-migrations
+test-migrations: $(GOOSE)
+	./scripts/test_migration.sh

db/migrations/00011_create_postgraphile_comments.sql

@@ -4,3 +4,11 @@ COMMENT ON TABLE eth.transaction_cids IS E'@name EthTransactionCids';
 COMMENT ON TABLE eth.header_cids IS E'@name EthHeaderCids';
 COMMENT ON COLUMN public.nodes.node_id IS E'@name ChainNodeID';
 COMMENT ON COLUMN eth.header_cids.node_id IS E'@name EthNodeID';
+
+-- +goose Down
+
+COMMENT ON TABLE public.nodes IS NULL;
+COMMENT ON TABLE eth.transaction_cids IS NULL;
+COMMENT ON TABLE eth.header_cids IS NULL;
+COMMENT ON COLUMN public.nodes.node_id IS NULL;
+COMMENT ON COLUMN eth.header_cids.node_id IS NULL;

docker-compose.yml

@@ -7,7 +7,7 @@ services:
     build: .
     environment:
       POSTGRES_USER: "vdbm"
-      POSTGRES_DB: "vulcanize_public"
+      POSTGRES_DB: "vulcanize_testing"
       POSTGRES_PASSWORD: "password"
       hostname: db
     ports:

scripts/check_diff.sh

@@ -1,7 +1,10 @@
-lines=$( git diff --no-index ./db/migration_schema.sql schema.sql | wc -l )
-if [ $lines -gt 0 ]; then
+file1=$1
+file2=$2
+
+lines=$( git diff --no-index "$file1" "$file2" | wc -l )
+if [ "$lines" -gt 0 ]; then
     echo "There are differences in schema"
-    git diff --no-index ./db/migration_schema.sql schema.sql
+    git diff --no-index "$file1" "$file2"
     exit 1
 fi
 echo "Schema is latest"

scripts/test_migration.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+docker rm -f $(docker ps -a -q)
+
+docker volume rm $(docker volume ls -q)
+
+docker-compose -f docker-compose.test.yml up -d test-db
+sleep 5s
+
+export HOST_NAME=localhost
+export PORT=8066
+export USER=vdbm
+export TEST_DB=vulcanize_testing
+export TEST_CONNECT_STRING=postgresql://$USER@$HOST_NAME:$PORT/$TEST_DB?sslmode=disable
+export PGPASSWORD=password
+
+# Get count of total number of migrations
+Count=$(find ./db/migrations -name "*sql" -type f | wc -l | awk '{print $1}')
+
+goose -dir ./db/migrations postgres "$TEST_CONNECT_STRING" status
+
+clean_up () {
+    rm schema*.sql
+}
+trap clean_up EXIT
+
+while true;
+do
+    pg_dump -h localhost -p $PORT -U $USER $TEST_DB --no-owner --schema-only > schema1.sql
+
+    # take action on each file. $f store current file name
+    goose -dir ./db/migrations postgres "$TEST_CONNECT_STRING" up-by-one
+
+    goose -dir ./db/migrations postgres "$TEST_CONNECT_STRING" down
+
+    pg_dump -h localhost -p $PORT -U $USER $TEST_DB  --no-owner --schema-only > schema2.sql
+
+    if ! ./scripts/check_diff.sh schema1.sql schema2.sql &> /dev/null;
+    then
+        # Column names are reordered when they are added back.
+        sed "s/\,//" schema1.sql | sort > schema1-sorted.sql
+        sed "s/\,//" schema2.sql | sort > schema2-sorted.sql
+        if ! ./scripts/check_diff.sh schema1-sorted.sql schema2-sorted.sql &> /dev/null;
+        then
+            echo "Up and Down migrations doesn't match for this migrations"
+            exit 1
+        fi
+    fi
+
+    goose -dir ./db/migrations postgres "$TEST_CONNECT_STRING" up-by-one
+
+    Version=$(goose -dir ./db/migrations postgres "$TEST_CONNECT_STRING" version 2>&1 | sed 's/.*version //')
+    if [ "$Count" = "$Version" ]
+    then
+        exit 0
+    fi
+done