Merge pull request #14 from vulcanize/fix-dir

Fix migration directory in Makefile.

.github/workflows/on-pr.yaml

@@ -21,7 +21,8 @@ jobs:
       - name: Verify schema is latest
         run: |
           PGPASSWORD="password" pg_dump -h localhost -p 8066 -U vdbm vulcanize_testing --no-owner --schema-only > ./db/migration_schema.sql
-          ./scripts/check_diff.sh
+          ./scripts/check_diff.sh ./db/migration_schema.sql db/schema.sql
+
   incremental_migration_diff:
     name: Compare conscise migration schema with incremental migration.
     runs-on: ubuntu-latest
@@ -38,4 +39,25 @@ jobs:
       - name: Verify schema is latest
         run: |
           PGPASSWORD="password" pg_dump -h localhost -p 8066 -U vdbm vulcanize_testing --no-owner --schema-only > ./db/migration_schema.sql
-          ./scripts/check_diff.sh
+          ./scripts/check_diff.sh db/schema.sql ./db/migration_schema.sql
+
+  migration:
+    name: Compare up and down migration
+    env:
+      GOPATH: /tmp/go
+    strategy:
+      matrix:
+        go-version: [ 1.16.x ]
+        os: [ ubuntu-latest ]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Create GOPATH
+        run: mkdir -p /tmp/go
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go-version }}
+      - uses: actions/checkout@v2
+      - name: Test migration
+        run: |
+          timeout 5m make test-migrations

Makefile

@@ -41,36 +41,36 @@ checkmigname:
 ## Rollback the last migration
 .PHONY: rollback
 rollback: $(GOOSE) checkdbvars
-	$(GOOSE) -dir migrations postgres "$(CONNECT_STRING)" down
+	$(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" down
 	pg_dump -O -s $(CONNECT_STRING) > schema.sql
 
 
 ## Rollback to a select migration (id/timestamp)
 .PHONY: rollback_to
 rollback_to: $(GOOSE) checkmigration checkdbvars
-	$(GOOSE) -dir migrations postgres "$(CONNECT_STRING)" down-to "$(MIGRATION)"
+	$(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" down-to "$(MIGRATION)"
 
 ## Apply all migrations not already run
 .PHONY: migrate
 migrate: $(GOOSE) checkdbvars
-	$(GOOSE) -dir migrations postgres "$(CONNECT_STRING)" up
+	$(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" up
 	pg_dump -O -s $(CONNECT_STRING) > schema.sql
 
 ## Create a new migration file
 .PHONY: new_migration
 new_migration: $(GOOSE) checkmigname
-	$(GOOSE) -dir migrations create $(NAME) sql
+	$(GOOSE) -dir db/migrations create $(NAME) sql
 
 ## Check which migrations are applied at the moment
 .PHONY: migration_status
 migration_status: $(GOOSE) checkdbvars
-	$(GOOSE) -dir migrations postgres "$(CONNECT_STRING)" status
+	$(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" status
 
 # Convert timestamped migrations to versioned (to be run in CI);
 # merge timestamped files to prevent conflict
 .PHONY: version_migrations
 version_migrations:
-	$(GOOSE) -dir migrations fix
+	$(GOOSE) -dir db/migrations fix
 
 # Import a psql schema to the database
 .PHONY: import
@@ -87,4 +87,8 @@ docker-build:
 ## Build docker image for migration
 .PHONY: docker-concise-migration-build
 docker-concise-migration-build:
-	docker build -t vulcanize/concise-migration-build -f ./db/Dockerfile .
+	docker build -t vulcanize/concise-migration-build -f ./db/Dockerfile .
+
+.PHONY: test-migrations
+test-migrations: $(GOOSE)
+	./scripts/test_migration.sh

db/migrations/00011_create_postgraphile_comments.sql

@@ -4,3 +4,11 @@ COMMENT ON TABLE eth.transaction_cids IS E'@name EthTransactionCids';
 COMMENT ON TABLE eth.header_cids IS E'@name EthHeaderCids';
 COMMENT ON COLUMN public.nodes.node_id IS E'@name ChainNodeID';
 COMMENT ON COLUMN eth.header_cids.node_id IS E'@name EthNodeID';
+
+-- +goose Down
+
+COMMENT ON TABLE public.nodes IS NULL;
+COMMENT ON TABLE eth.transaction_cids IS NULL;
+COMMENT ON TABLE eth.header_cids IS NULL;
+COMMENT ON COLUMN public.nodes.node_id IS NULL;
+COMMENT ON COLUMN eth.header_cids.node_id IS NULL;

docker-compose.yml

@@ -7,7 +7,7 @@ services:
     build: .
     environment:
       POSTGRES_USER: "vdbm"
-      POSTGRES_DB: "vulcanize_public"
+      POSTGRES_DB: "vulcanize_testing"
       POSTGRES_PASSWORD: "password"
       hostname: db
     ports:

scripts/check_diff.sh

@@ -1,7 +1,10 @@
-lines=$( git diff --no-index ./db/migration_schema.sql schema.sql | wc -l )
-if [ $lines -gt 0 ]; then
+file1=$1
+file2=$2
+
+lines=$( git diff --no-index "$file1" "$file2" | wc -l )
+if [ "$lines" -gt 0 ]; then
     echo "There are differences in schema"
-    git diff --no-index ./db/migration_schema.sql schema.sql
+    git diff --no-index "$file1" "$file2"
     exit 1
 fi
 echo "Schema is latest"

scripts/test_migration.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+docker rm -f $(docker ps -a -q)
+
+docker volume rm $(docker volume ls -q)
+
+docker-compose -f docker-compose.test.yml up -d test-db
+sleep 5s
+
+export HOST_NAME=localhost
+export PORT=8066
+export USER=vdbm
+export TEST_DB=vulcanize_testing
+export TEST_CONNECT_STRING=postgresql://$USER@$HOST_NAME:$PORT/$TEST_DB?sslmode=disable
+export PGPASSWORD=password
+
+# Get count of total number of migrations
+Count=$(find ./db/migrations -name "*sql" -type f | wc -l | awk '{print $1}')
+
+goose -dir ./db/migrations postgres "$TEST_CONNECT_STRING" status
+
+clean_up () {
+    rm schema*.sql
+}
+trap clean_up EXIT
+
+while true;
+do
+    pg_dump -h localhost -p $PORT -U $USER $TEST_DB --no-owner --schema-only > schema1.sql
+
+    # take action on each file. $f store current file name
+    goose -dir ./db/migrations postgres "$TEST_CONNECT_STRING" up-by-one
+
+    goose -dir ./db/migrations postgres "$TEST_CONNECT_STRING" down
+
+    pg_dump -h localhost -p $PORT -U $USER $TEST_DB  --no-owner --schema-only > schema2.sql
+
+    if ! ./scripts/check_diff.sh schema1.sql schema2.sql &> /dev/null;
+    then
+        # Column names are reordered when they are added back.
+        sed "s/\,//" schema1.sql | sort > schema1-sorted.sql
+        sed "s/\,//" schema2.sql | sort > schema2-sorted.sql
+        if ! ./scripts/check_diff.sh schema1-sorted.sql schema2-sorted.sql &> /dev/null;
+        then
+            echo "Up and Down migrations doesn't match for this migrations"
+            exit 1
+        fi
+    fi
+
+    goose -dir ./db/migrations postgres "$TEST_CONNECT_STRING" up-by-one
+
+    Version=$(goose -dir ./db/migrations postgres "$TEST_CONNECT_STRING" version 2>&1 | sed 's/.*version //')
+    if [ "$Count" = "$Version" ]
+    then
+        exit 0
+    fi
+done