diff --git a/.github/workflows/on-pr.yaml b/.github/workflows/on-pr.yaml index 4888e38..1177683 100644 --- a/.github/workflows/on-pr.yaml +++ b/.github/workflows/on-pr.yaml @@ -21,7 +21,8 @@ jobs: - name: Verify schema is latest run: | PGPASSWORD="password" pg_dump -h localhost -p 8066 -U vdbm vulcanize_testing --no-owner --schema-only > ./db/migration_schema.sql - ./scripts/check_diff.sh + ./scripts/check_diff.sh ./db/migration_schema.sql db/schema.sql + incremental_migration_diff: name: Compare conscise migration schema with incremental migration. runs-on: ubuntu-latest @@ -38,4 +39,25 @@ jobs: - name: Verify schema is latest run: | PGPASSWORD="password" pg_dump -h localhost -p 8066 -U vdbm vulcanize_testing --no-owner --schema-only > ./db/migration_schema.sql - ./scripts/check_diff.sh + ./scripts/check_diff.sh db/schema.sql ./db/migration_schema.sql + + migration: + name: Compare up and down migration + env: + GOPATH: /tmp/go + strategy: + matrix: + go-version: [ 1.16.x ] + os: [ ubuntu-latest ] + runs-on: ${{ matrix.os }} + steps: + - name: Create GOPATH + run: mkdir -p /tmp/go + - name: Install Go + uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go-version }} + - uses: actions/checkout@v2 + - name: Test migration + run: | + timeout 5m make test-migrations \ No newline at end of file diff --git a/Makefile b/Makefile index 884eb96..d79caf1 100644 --- a/Makefile +++ b/Makefile @@ -87,4 +87,8 @@ docker-build: ## Build docker image for migration .PHONY: docker-concise-migration-build docker-concise-migration-build: - docker build -t vulcanize/concise-migration-build -f ./db/Dockerfile . \ No newline at end of file + docker build -t vulcanize/concise-migration-build -f ./db/Dockerfile . + +.PHONY: test-migrations +test-migrations: $(GOOSE) + ./scripts/test_migration.sh \ No newline at end of file diff --git a/db/migrations/00011_create_postgraphile_comments.sql b/db/migrations/00011_create_postgraphile_comments.sql index 16a051f..f3ce2e6 100644 --- a/db/migrations/00011_create_postgraphile_comments.sql +++ b/db/migrations/00011_create_postgraphile_comments.sql @@ -4,3 +4,11 @@ COMMENT ON TABLE eth.transaction_cids IS E'@name EthTransactionCids'; COMMENT ON TABLE eth.header_cids IS E'@name EthHeaderCids'; COMMENT ON COLUMN public.nodes.node_id IS E'@name ChainNodeID'; COMMENT ON COLUMN eth.header_cids.node_id IS E'@name EthNodeID'; + +-- +goose Down + +COMMENT ON TABLE public.nodes IS NULL; +COMMENT ON TABLE eth.transaction_cids IS NULL; +COMMENT ON TABLE eth.header_cids IS NULL; +COMMENT ON COLUMN public.nodes.node_id IS NULL; +COMMENT ON COLUMN eth.header_cids.node_id IS NULL; \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 16ab291..dd120cf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,7 +7,7 @@ services: build: . environment: POSTGRES_USER: "vdbm" - POSTGRES_DB: "vulcanize_public" + POSTGRES_DB: "vulcanize_testing" POSTGRES_PASSWORD: "password" hostname: db ports: diff --git a/scripts/check_diff.sh b/scripts/check_diff.sh index 3eaa9b8..6cee816 100755 --- a/scripts/check_diff.sh +++ b/scripts/check_diff.sh @@ -1,7 +1,10 @@ -lines=$( git diff --no-index ./db/migration_schema.sql schema.sql | wc -l ) -if [ $lines -gt 0 ]; then +file1=$1 +file2=$2 + +lines=$( git diff --no-index "$file1" "$file2" | wc -l ) +if [ "$lines" -gt 0 ]; then echo "There are differences in schema" - git diff --no-index ./db/migration_schema.sql schema.sql + git diff --no-index "$file1" "$file2" exit 1 fi echo "Schema is latest" \ No newline at end of file diff --git a/scripts/test_migration.sh b/scripts/test_migration.sh new file mode 100755 index 0000000..fe60b33 --- /dev/null +++ b/scripts/test_migration.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +docker rm -f $(docker ps -a -q) + +docker volume rm $(docker volume ls -q) + +docker-compose -f docker-compose.test.yml up -d test-db +sleep 5s + +export HOST_NAME=localhost +export PORT=8066 +export USER=vdbm +export TEST_DB=vulcanize_testing +export TEST_CONNECT_STRING=postgresql://$USER@$HOST_NAME:$PORT/$TEST_DB?sslmode=disable +export PGPASSWORD=password + +# Get count of total number of migrations +Count=$(find ./db/migrations -name "*sql" -type f | wc -l | awk '{print $1}') + +goose -dir ./db/migrations postgres "$TEST_CONNECT_STRING" status + +clean_up () { + rm schema*.sql +} +trap clean_up EXIT + +while true; +do + pg_dump -h localhost -p $PORT -U $USER $TEST_DB --no-owner --schema-only > schema1.sql + + # take action on each file. $f store current file name + goose -dir ./db/migrations postgres "$TEST_CONNECT_STRING" up-by-one + + goose -dir ./db/migrations postgres "$TEST_CONNECT_STRING" down + + pg_dump -h localhost -p $PORT -U $USER $TEST_DB --no-owner --schema-only > schema2.sql + + if ! ./scripts/check_diff.sh schema1.sql schema2.sql &> /dev/null; + then + # Column names are reordered when they are added back. + sed "s/\,//" schema1.sql | sort > schema1-sorted.sql + sed "s/\,//" schema2.sql | sort > schema2-sorted.sql + if ! ./scripts/check_diff.sh schema1-sorted.sql schema2-sorted.sql &> /dev/null; + then + echo "Up and Down migrations doesn't match for this migrations" + exit 1 + fi + fi + + goose -dir ./db/migrations postgres "$TEST_CONNECT_STRING" up-by-one + + Version=$(goose -dir ./db/migrations postgres "$TEST_CONNECT_STRING" version 2>&1 | sed 's/.*version //') + if [ "$Count" = "$Version" ] + then + exit 0 + fi +done \ No newline at end of file