Merge pull request #60 from vulcanize/release-v2.1.1

Release v2.1.1

.github/workflows/on-main.yaml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - main
+      - main-v2
 
 jobs:
   build:

.github/workflows/on-pr.yaml

@@ -21,7 +21,8 @@ jobs:
       - name: Verify schema is latest
         run: |
           PGPASSWORD="password" pg_dump -h localhost -p 8066 -U vdbm vulcanize_testing --no-owner --schema-only > ./db/migration_schema.sql
-          ./scripts/check_diff.sh
+          ./scripts/check_diff.sh ./db/migration_schema.sql db/schema.sql
+
   incremental_migration_diff:
     name: Compare conscise migration schema with incremental migration.
     runs-on: ubuntu-latest
@@ -38,4 +39,25 @@ jobs:
       - name: Verify schema is latest
         run: |
           PGPASSWORD="password" pg_dump -h localhost -p 8066 -U vdbm vulcanize_testing --no-owner --schema-only > ./db/migration_schema.sql
-          ./scripts/check_diff.sh
+          ./scripts/check_diff.sh db/schema.sql ./db/migration_schema.sql
+
+  migration:
+    name: Compare up and down migration
+    env:
+      GOPATH: /tmp/go
+    strategy:
+      matrix:
+        go-version: [ 1.16.x ]
+        os: [ ubuntu-latest ]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Create GOPATH
+        run: mkdir -p /tmp/go
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go-version }}
+      - uses: actions/checkout@v2
+      - name: Test migration
+        run: |
+          timeout 5m make test-migrations

Dockerfile

@@ -1 +1,3 @@
-FROM postgres:12-alpine
+FROM postgres:12-alpine
+
+COPY ./schema.sql /docker-entrypoint-initdb.d/init.sql

Makefile

@@ -41,36 +41,36 @@ checkmigname:
 ## Rollback the last migration
 .PHONY: rollback
 rollback: $(GOOSE) checkdbvars
-	$(GOOSE) -dir migrations postgres "$(CONNECT_STRING)" down
+	$(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" down
 	pg_dump -O -s $(CONNECT_STRING) > schema.sql
 
 
 ## Rollback to a select migration (id/timestamp)
 .PHONY: rollback_to
 rollback_to: $(GOOSE) checkmigration checkdbvars
-	$(GOOSE) -dir migrations postgres "$(CONNECT_STRING)" down-to "$(MIGRATION)"
+	$(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" down-to "$(MIGRATION)"
 
 ## Apply all migrations not already run
 .PHONY: migrate
 migrate: $(GOOSE) checkdbvars
-	$(GOOSE) -dir migrations postgres "$(CONNECT_STRING)" up
+	$(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" up
 	pg_dump -O -s $(CONNECT_STRING) > schema.sql
 
 ## Create a new migration file
 .PHONY: new_migration
 new_migration: $(GOOSE) checkmigname
-	$(GOOSE) -dir migrations create $(NAME) sql
+	$(GOOSE) -dir db/migrations create $(NAME) sql
 
 ## Check which migrations are applied at the moment
 .PHONY: migration_status
 migration_status: $(GOOSE) checkdbvars
-	$(GOOSE) -dir migrations postgres "$(CONNECT_STRING)" status
+	$(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" status
 
 # Convert timestamped migrations to versioned (to be run in CI);
 # merge timestamped files to prevent conflict
 .PHONY: version_migrations
 version_migrations:
-	$(GOOSE) -dir migrations fix
+	$(GOOSE) -dir db/migrations fix
 
 # Import a psql schema to the database
 .PHONY: import
@@ -87,4 +87,8 @@ docker-build:
 ## Build docker image for migration
 .PHONY: docker-concise-migration-build
 docker-concise-migration-build:
-	docker build -t vulcanize/concise-migration-build -f ./db/Dockerfile .
+	docker build -t vulcanize/concise-migration-build -f ./db/Dockerfile .
+
+.PHONY: test-migrations
+test-migrations: $(GOOSE)
+	./scripts/test_migration.sh

db/migrations/00011_create_postgraphile_comments.sql

@@ -4,3 +4,11 @@ COMMENT ON TABLE eth.transaction_cids IS E'@name EthTransactionCids';
 COMMENT ON TABLE eth.header_cids IS E'@name EthHeaderCids';
 COMMENT ON COLUMN public.nodes.node_id IS E'@name ChainNodeID';
 COMMENT ON COLUMN eth.header_cids.node_id IS E'@name EthNodeID';
+
+-- +goose Down
+
+COMMENT ON TABLE public.nodes IS NULL;
+COMMENT ON TABLE eth.transaction_cids IS NULL;
+COMMENT ON TABLE eth.header_cids IS NULL;
+COMMENT ON COLUMN public.nodes.node_id IS NULL;
+COMMENT ON COLUMN eth.header_cids.node_id IS NULL;

db/migrations/00017_create_eth_meta_schema.sql

@@ -0,0 +1,5 @@
+-- +goose Up
+CREATE SCHEMA eth_meta;
+
+-- +goose Down
+DROP SCHEMA eth_meta;

db/migrations/00018_create_watched_addresses_table.sql

@@ -0,0 +1,10 @@
+-- +goose Up
+CREATE TABLE eth_meta.watched_addresses (
+    address                 VARCHAR(66) PRIMARY KEY,
+    created_at              BIGINT NOT NULL,
+    watched_at              BIGINT NOT NULL,
+    last_filled_at          BIGINT NOT NULL DEFAULT 0
+);
+
+-- +goose Down
+DROP TABLE eth_meta.watched_addresses;

docker-compose.yml

@@ -7,10 +7,8 @@ services:
     build: .
     environment:
       POSTGRES_USER: "vdbm"
-      POSTGRES_DB: "vulcanize_public"
+      POSTGRES_DB: "vulcanize_testing"
       POSTGRES_PASSWORD: "password"
       hostname: db
-    volumes:
-      - ./schema.sql:/docker-entrypoint-initdb.d/init.sql
     ports:
       - "127.0.0.1:8077:5432"

schema.sql

@@ -23,6 +23,13 @@ SET row_security = off;
 CREATE SCHEMA eth;
 
 
+--
+-- Name: eth_meta; Type: SCHEMA; Schema: -; Owner: -
+--
+
+CREATE SCHEMA eth_meta;
+
+
 SET default_tablespace = '';
 
 --
@@ -552,6 +559,18 @@ CREATE SEQUENCE eth.uncle_cids_id_seq
 ALTER SEQUENCE eth.uncle_cids_id_seq OWNED BY eth.uncle_cids.id;
 
 
+--
+-- Name: watched_addresses; Type: TABLE; Schema: eth_meta; Owner: -
+--
+
+CREATE TABLE eth_meta.watched_addresses (
+    address character varying(66) NOT NULL,
+    created_at bigint NOT NULL,
+    watched_at bigint NOT NULL,
+    last_filled_at bigint DEFAULT 0 NOT NULL
+);
+
+
 --
 -- Name: blocks; Type: TABLE; Schema: public; Owner: -
 --
@@ -863,6 +882,14 @@ ALTER TABLE ONLY eth.uncle_cids
     ADD CONSTRAINT uncle_cids_pkey PRIMARY KEY (id);
 
 
+--
+-- Name: watched_addresses watched_addresses_pkey; Type: CONSTRAINT; Schema: eth_meta; Owner: -
+--
+
+ALTER TABLE ONLY eth_meta.watched_addresses
+    ADD CONSTRAINT watched_addresses_pkey PRIMARY KEY (address);
+
+
 --
 -- Name: blocks blocks_key_key; Type: CONSTRAINT; Schema: public; Owner: -
 --

scripts/check_diff.sh

@@ -1,7 +1,10 @@
-lines=$( git diff --no-index ./db/migration_schema.sql schema.sql | wc -l )
+file1=$1
-if [ $lines -gt 0 ]; then
+file2=$2
+
+lines=$( git diff --no-index "$file1" "$file2" | wc -l )
+if [ "$lines" -gt 0 ]; then
     echo "There are differences in schema"
-    git diff --no-index ./db/migration_schema.sql schema.sql
+    git diff --no-index "$file1" "$file2"
     exit 1
 fi
 echo "Schema is latest"

scripts/test_migration.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+docker rm -f $(docker ps -a -q)
+
+docker volume rm $(docker volume ls -q)
+
+docker-compose -f docker-compose.test.yml up -d test-db
+sleep 5s
+
+export HOST_NAME=localhost
+export PORT=8066
+export USER=vdbm
+export TEST_DB=vulcanize_testing
+export TEST_CONNECT_STRING=postgresql://$USER@$HOST_NAME:$PORT/$TEST_DB?sslmode=disable
+export PGPASSWORD=password
+
+# Get count of total number of migrations
+Count=$(find ./db/migrations -name "*sql" -type f | wc -l | awk '{print $1}')
+
+goose -dir ./db/migrations postgres "$TEST_CONNECT_STRING" status
+
+clean_up () {
+    rm schema*.sql
+}
+trap clean_up EXIT
+
+while true;
+do
+    pg_dump -h localhost -p $PORT -U $USER $TEST_DB --no-owner --schema-only > schema1.sql
+
+    # take action on each file. $f store current file name
+    goose -dir ./db/migrations postgres "$TEST_CONNECT_STRING" up-by-one
+
+    goose -dir ./db/migrations postgres "$TEST_CONNECT_STRING" down
+
+    pg_dump -h localhost -p $PORT -U $USER $TEST_DB  --no-owner --schema-only > schema2.sql
+
+    if ! ./scripts/check_diff.sh schema1.sql schema2.sql &> /dev/null;
+    then
+        # Column names are reordered when they are added back.
+        sed "s/\,//" schema1.sql | sort > schema1-sorted.sql
+        sed "s/\,//" schema2.sql | sort > schema2-sorted.sql
+        if ! ./scripts/check_diff.sh schema1-sorted.sql schema2-sorted.sql &> /dev/null;
+        then
+            echo "Up and Down migrations doesn't match for this migrations"
+            exit 1
+        fi
+    fi
+
+    goose -dir ./db/migrations postgres "$TEST_CONNECT_STRING" up-by-one
+
+    Version=$(goose -dir ./db/migrations postgres "$TEST_CONNECT_STRING" version 2>&1 | sed 's/.*version //')
+    if [ "$Count" = "$Version" ]
+    then
+        exit 0
+    fi
+done