Create a docker image with postgres DB and concise migration.

.github/workflow/on-master.yaml

@@ -0,0 +1,25 @@
+name: Docker Compose Build
+
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  build:
+    name: Run docker build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Get the version
+        id: vars
+        run: echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
+      - name: Run docker build
+        run: make docker-build
+      - name: Tag docker image
+        run: docker tag vulcanize/ipld-eth-db docker.pkg.github.com/vulcanize/ipl-eth-db/:${{steps.vars.outputs.sha}}
+      - name: Docker Login
+        run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
+      - name: Docker Push
+        run: docker push docker.pkg.github.com/vulcanize/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.sha}}
+

.github/workflow/on-pr.yaml

@@ -0,0 +1,40 @@
+name: Docker Build
+
+on: [pull_request]
+
+jobs:
+  concise_migration_diff:
+    name: Verify concise migration and generated schema
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Run docker concise migration build
+        run: make docker-concise-migration-build
+      - name: Run database
+        run: docker-compose -f docker-compose.test.yml up -d test-db
+      - name: Test concise migration
+        run: |
+          sleep 10
+          docker run --rm --network host -e DATABASE_USER=vdbm -e DATABASE_PASSWORD=password \
+          -e DATABASE_HOSTNAME=127.0.0.1 -e DATABASE_PORT=8066 -e DATABASE_NAME=vulcanize_testing \
+          vulcanize/docker-concise-migration-build
+      - name: Verify schema is latest
+        run: |
+          PGPASSWORD="password" pg_dump -h localhost -p 8066 -U vdbm vulcanize_testing --no-owner --schema-only > ./db/migration_schema.sql
+          ./scripts/check_diff.sh
+  incremental_migration_diff:
+    name: Compare conscise migration schema with incremental migration.
+    runs-on: ubuntu-latest
+    steps:
+      - name: Run database
+        run: docker-compose -f docker-compose.test.yml up -d test-db statediff-migrations
+      - name: Test incremental migration
+        run: |
+          sleep 10
+          docker run --rm --network host -e DATABASE_USER=vdbm -e DATABASE_PASSWORD=password \
+          -e DATABASE_HOSTNAME=127.0.0.1 -e DATABASE_PORT=8066 -e DATABASE_NAME=vulcanize_testing \
+          vulcanize/statediff-migrations:v0.8.0
+      - name: Verify schema is latest
+        run: |
+          PGPASSWORD="password" pg_dump -h localhost -p 8066 -U vdbm vulcanize_testing --no-owner --schema-only > ./db/migration_schema.sql
+          ./scripts/check_diff.sh

.github/workflow/publish.yaml

@@ -0,0 +1,24 @@
+name: Publish Docker image
+on:
+  release:
+    types: [published]
+jobs:
+  push_to_registries:
+    name: Push Docker image to Docker Hub
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get the version
+        id: vars
+        run: |
+          echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
+          echo ::set-output name=tag::$(echo ${GITHUB_REF#refs/tags/})
+      - name: Docker Login to Github Registry
+        run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
+      - name: Docker Pull
+        run: docker pull docker.pkg.github.com/vulcanize/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.sha}}
+      - name: Docker Login to Docker Registry
+        run: echo ${{ secrets.VULCANIZEJENKINS_PAT }} | docker login -u vulcanizejenkins --password-stdin
+      - name: Tag docker image
+        run: docker tag docker.pkg.github.com/vulcanize/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.sha}} vulcanize/ipld-eth-db:${{steps.vars.outputs.tag}}
+      - name: Docker Push to Docker Hub
+        run: docker push vulcanize/ipld-eth-db:${{steps.vars.outputs.tag}}

Dockerfile

@@ -0,0 +1 @@
+FROM postgres:12-alpine

Makefile

@@ -77,3 +77,14 @@ version_migrations:
 import:
 	test -n "$(NAME)" # $$NAME
 	psql $(NAME) < schema.sql
+
+
+## Build docker image with schema
+.PHONY: docker-build
+docker-build:
+	docker-compose build
+
+## Build docker image for migration
+.PHONY: docker-concise-migration-build
+docker-concise-migration-build:
+	docker build -t vulcanize/docker-concise-migration-build -f ./db/Dockerfile .

db/Dockerfile

@@ -0,0 +1,23 @@
+FROM golang:1.16-alpine as builder
+
+RUN apk --update --no-cache add make git g++ linux-headers
+
+ADD . /go/src/github.com/vulcanize/ipld-eth-db
+
+# Build migration tool
+WORKDIR /go/src/github.com/pressly
+RUN git clone https://github.com/pressly/goose.git
+WORKDIR /go/src/github.com/pressly/goose/cmd/goose
+RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -tags='no_mysql no_sqlite' -o goose .
+
+# app container
+FROM alpine
+
+WORKDIR /app
+
+COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-db/scripts/startup_script.sh .
+
+COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose goose
+COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-db/db/migrations migrations/vulcanizedb
+
+ENTRYPOINT ["/app/startup_script.sh"]

db/migrations/00007_create_eth_receipt_cids_table.sql

@@ -2,8 +2,8 @@
 CREATE TABLE eth.receipt_cids (
   id                    SERIAL PRIMARY KEY,
   tx_id                 INTEGER NOT NULL REFERENCES eth.transaction_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  cid                   TEXT NOT NULL,
-  mh_key                TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
+  leaf_cid              TEXT NOT NULL,
+  leaf_mh_key           TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
   contract              VARCHAR(66),
   contract_hash         VARCHAR(66),
   post_state            VARCHAR(66),

db/migrations/00013_create_cid_indexes.sql

@@ -28,9 +28,9 @@ CREATE INDEX tx_src_index ON eth.transaction_cids USING btree (src);
 -- receipt indexes
 CREATE INDEX rct_tx_id_index ON eth.receipt_cids USING btree (tx_id);
 
-CREATE INDEX rct_cid_index ON eth.receipt_cids USING btree (cid);
+CREATE INDEX rct_leaf_cid_index ON eth.receipt_cids USING btree (leaf_cid);
 
-CREATE INDEX rct_mh_index ON eth.receipt_cids USING btree (mh_key);
+CREATE INDEX rct_leaf_mh_index ON eth.receipt_cids USING btree (leaf_mh_key);
 
 CREATE INDEX rct_contract_index ON eth.receipt_cids USING btree (contract);
 
@@ -91,8 +91,8 @@ DROP INDEX eth.state_header_id_index;
 -- receipt indexes
 DROP INDEX eth.rct_contract_hash_index;
 DROP INDEX eth.rct_contract_index;
-DROP INDEX eth.rct_mh_index;
-DROP INDEX eth.rct_cid_index;
+DROP INDEX eth.rct_leaf_mh_index;
+DROP INDEX eth.rct_leaf_cid_index;
 DROP INDEX eth.rct_tx_id_index;
 
 -- transaction indexes

db/migrations/00014_create_stored_functions.sql

@@ -1,37 +1,18 @@
 -- +goose Up
 -- +goose StatementBegin
--- returns if a storage node at the provided path was removed in the range > the provided height and <= the provided block hash
-CREATE OR REPLACE FUNCTION was_storage_removed(path BYTEA, height BIGINT, hash VARCHAR(66)) RETURNS BOOLEAN
-AS $$
-SELECT exists(SELECT 1
-              FROM eth.storage_cids
-                INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id)
-                INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
-              WHERE storage_path = path
-                AND block_number > height
-                AND block_number <= (SELECT block_number
-                                     FROM eth.header_cids
-                                     WHERE block_hash = hash)
-                AND storage_cids.node_type = 3
-              LIMIT 1);
-$$ LANGUAGE SQL;
--- +goose StatementEnd
-
--- +goose StatementBegin
--- returns if a state node at the provided path was removed in the range > the provided height and <= the provided block hash
-CREATE OR REPLACE FUNCTION was_state_removed(path BYTEA, height BIGINT, hash VARCHAR(66)) RETURNS BOOLEAN
-AS $$
-SELECT exists(SELECT 1
+-- returns if a state leaf node was removed within the provided block number
+CREATE OR REPLACE FUNCTION was_state_leaf_removed(key character varying, hash character varying)
+    RETURNS boolean AS $$
+    SELECT state_cids.node_type = 3
     FROM eth.state_cids
              INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
-              WHERE state_path = path
-                AND block_number > height
+    WHERE state_leaf_key = key
       AND block_number <= (SELECT block_number
                            FROM eth.header_cids
                            WHERE block_hash = hash)
-                AND state_cids.node_type = 3
-              LIMIT 1);
-$$ LANGUAGE SQL;
+    ORDER BY block_number DESC LIMIT 1;
+$$
+language sql;
 -- +goose StatementEnd
 
 -- +goose StatementBegin
@@ -150,8 +131,7 @@ LANGUAGE 'plpgsql';
 -- +goose StatementEnd
 
 -- +goose Down
-DROP FUNCTION was_storage_removed;
-DROP FUNCTION was_state_removed;
+DROP FUNCTION was_state_leaf_removed;
 DROP FUNCTION canonical_header_id;
 DROP FUNCTION canonical_header_from_array;
 DROP FUNCTION has_child;

db/migrations/00016_create_eth_log_cids_table.sql

@@ -4,7 +4,7 @@ CREATE TABLE eth.log_cids (
     leaf_cid            TEXT NOT NULL,
     leaf_mh_key         TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
     receipt_id          INTEGER NOT NULL REFERENCES eth.receipt_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-    address             VARCHAR(66),
+    address             VARCHAR(66) NOT NULL,
     log_data            BYTEA,
     index               INTEGER NOT NULL,
     topic0              VARCHAR(66),
@@ -19,6 +19,7 @@ CREATE INDEX log_mh_index ON eth.log_cids USING btree (leaf_mh_key);
 CREATE INDEX log_cid_index ON  eth.log_cids USING btree (leaf_cid);
 
 CREATE INDEX log_rct_id_index ON eth.log_cids USING btree (receipt_id);
+
 --
 -- Name: log_topic0_index; Type: INDEX; Schema: eth; Owner: -
 --

docker-compose.test.yml

@@ -0,0 +1,19 @@
+version: '3.2'
+
+services:
+  statediff-migrations:
+    restart: on-failure
+    depends_on:
+      - test-db
+    image: vulcanize/statediff-migrations:latest
+
+  test-db:
+    restart: always
+    image: postgres:10.12-alpine
+    command: ["postgres", "-c", "log_statement=all"]
+    environment:
+      POSTGRES_USER: "vdbm"
+      POSTGRES_DB: "vulcanize_testing"
+      POSTGRES_PASSWORD: "password"
+    ports:
+      - "127.0.0.1:8066:5432"

docker-compose.yml

@@ -0,0 +1,16 @@
+version: '3.2'
+
+services:
+  ipld-eth-db:
+    restart: always
+    image: vulcanize/ipld-eth-db
+    build: .
+    environment:
+      POSTGRES_USER: "vdbm"
+      POSTGRES_DB: "vulcanize_public"
+      POSTGRES_PASSWORD: "password"
+      hostname: db
+    volumes:
+      - ./schema.sql:/docker-entrypoint-initdb.d/init.sql
+    ports:
+      - "127.0.0.1:8077:5432"

migrations/00015_create_access_list_table.sql

@@ -1,15 +0,0 @@
--- +goose Up
-CREATE TABLE eth.access_list_element (
-   id                    SERIAL PRIMARY KEY,
-   tx_id                 INTEGER NOT NULL REFERENCES eth.transaction_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-   index                 INTEGER NOT NULL,
-   address               VARCHAR(66),
-   storage_keys          VARCHAR(66)[],
-   UNIQUE (tx_id, index)
-);
-
-CREATE INDEX accesss_list_element_address_index ON eth.access_list_element USING btree (address);
-
--- +goose Down
-DROP INDEX eth.accesss_list_element_address_index;
-DROP TABLE eth.access_list_element;

schema.sql

@@ -2,8 +2,8 @@
 -- PostgreSQL database dump
 --
 
--- Dumped from database version 14beta3
--- Dumped by pg_dump version 14beta3
+-- Dumped from database version 10.12
+-- Dumped by pg_dump version 13.4
 
 SET statement_timeout = 0;
 SET lock_timeout = 0;
@@ -25,8 +25,6 @@ CREATE SCHEMA eth;
 
 SET default_tablespace = '';
 
-SET default_table_access_method = heap;
-
 --
 -- Name: header_cids; Type: TABLE; Schema: eth; Owner: -
 --
@@ -224,43 +222,20 @@ $$;
 
 
 --
--- Name: was_state_removed(bytea, bigint, character varying); Type: FUNCTION; Schema: public; Owner: -
+-- Name: was_state_leaf_removed(character varying, character varying); Type: FUNCTION; Schema: public; Owner: -
 --
 
-CREATE FUNCTION public.was_state_removed(path bytea, height bigint, hash character varying) RETURNS boolean
+CREATE FUNCTION public.was_state_leaf_removed(key character varying, hash character varying) RETURNS boolean
     LANGUAGE sql
     AS $$
-SELECT exists(SELECT 1
+    SELECT state_cids.node_type = 3
     FROM eth.state_cids
              INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
-              WHERE state_path = path
-                AND block_number > height
+    WHERE state_leaf_key = key
       AND block_number <= (SELECT block_number
                            FROM eth.header_cids
                            WHERE block_hash = hash)
-                AND state_cids.node_type = 3
-              LIMIT 1);
-$$;
-
-
---
--- Name: was_storage_removed(bytea, bigint, character varying); Type: FUNCTION; Schema: public; Owner: -
---
-
-CREATE FUNCTION public.was_storage_removed(path bytea, height bigint, hash character varying) RETURNS boolean
-    LANGUAGE sql
-    AS $$
-SELECT exists(SELECT 1
-              FROM eth.storage_cids
-                INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id)
-                INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
-              WHERE storage_path = path
-                AND block_number > height
-                AND block_number <= (SELECT block_number
-                                     FROM eth.header_cids
-                                     WHERE block_hash = hash)
-                AND storage_cids.node_type = 3
-              LIMIT 1);
+    ORDER BY block_number DESC LIMIT 1;
 $$;
 
 
@@ -326,7 +301,7 @@ CREATE TABLE eth.log_cids (
     leaf_cid text NOT NULL,
     leaf_mh_key text NOT NULL,
     receipt_id integer NOT NULL,
-    address character varying(66),
+    address character varying(66) NOT NULL,
     log_data bytea,
     index integer NOT NULL,
     topic0 character varying(66),
@@ -363,8 +338,8 @@ ALTER SEQUENCE eth.log_cids_id_seq OWNED BY eth.log_cids.id;
 CREATE TABLE eth.receipt_cids (
     id integer NOT NULL,
     tx_id integer NOT NULL,
-    cid text NOT NULL,
-    mh_key text NOT NULL,
+    leaf_cid text NOT NULL,
+    leaf_mh_key text NOT NULL,
     contract character varying(66),
     contract_hash character varying(66),
     post_state character varying(66),
@@ -1011,13 +986,6 @@ CREATE INDEX log_topic2_index ON eth.log_cids USING btree (topic2);
 CREATE INDEX log_topic3_index ON eth.log_cids USING btree (topic3);
 
 
---
--- Name: rct_cid_index; Type: INDEX; Schema: eth; Owner: -
---
-
-CREATE INDEX rct_cid_index ON eth.receipt_cids USING btree (cid);
-
-
 --
 -- Name: rct_contract_hash_index; Type: INDEX; Schema: eth; Owner: -
 --
@@ -1033,10 +1001,17 @@ CREATE INDEX rct_contract_index ON eth.receipt_cids USING btree (contract);
 
 
 --
--- Name: rct_mh_index; Type: INDEX; Schema: eth; Owner: -
+-- Name: rct_leaf_cid_index; Type: INDEX; Schema: eth; Owner: -
 --
 
-CREATE INDEX rct_mh_index ON eth.receipt_cids USING btree (mh_key);
+CREATE INDEX rct_leaf_cid_index ON eth.receipt_cids USING btree (leaf_cid);
+
+
+--
+-- Name: rct_leaf_mh_index; Type: INDEX; Schema: eth; Owner: -
+--
+
+CREATE INDEX rct_leaf_mh_index ON eth.receipt_cids USING btree (leaf_mh_key);
 
 
 --
@@ -1197,49 +1172,49 @@ CREATE INDEX tx_src_index ON eth.transaction_cids USING btree (src);
 -- Name: header_cids header_cids_ai; Type: TRIGGER; Schema: eth; Owner: -
 --
 
-CREATE TRIGGER header_cids_ai AFTER INSERT ON eth.header_cids FOR EACH ROW EXECUTE FUNCTION eth.graphql_subscription('header_cids', 'id');
+CREATE TRIGGER header_cids_ai AFTER INSERT ON eth.header_cids FOR EACH ROW EXECUTE PROCEDURE eth.graphql_subscription('header_cids', 'id');
 
 
 --
 -- Name: receipt_cids receipt_cids_ai; Type: TRIGGER; Schema: eth; Owner: -
 --
 
-CREATE TRIGGER receipt_cids_ai AFTER INSERT ON eth.receipt_cids FOR EACH ROW EXECUTE FUNCTION eth.graphql_subscription('receipt_cids', 'id');
+CREATE TRIGGER receipt_cids_ai AFTER INSERT ON eth.receipt_cids FOR EACH ROW EXECUTE PROCEDURE eth.graphql_subscription('receipt_cids', 'id');
 
 
 --
 -- Name: state_accounts state_accounts_ai; Type: TRIGGER; Schema: eth; Owner: -
 --
 
-CREATE TRIGGER state_accounts_ai AFTER INSERT ON eth.state_accounts FOR EACH ROW EXECUTE FUNCTION eth.graphql_subscription('state_accounts', 'id');
+CREATE TRIGGER state_accounts_ai AFTER INSERT ON eth.state_accounts FOR EACH ROW EXECUTE PROCEDURE eth.graphql_subscription('state_accounts', 'id');
 
 
 --
 -- Name: state_cids state_cids_ai; Type: TRIGGER; Schema: eth; Owner: -
 --
 
-CREATE TRIGGER state_cids_ai AFTER INSERT ON eth.state_cids FOR EACH ROW EXECUTE FUNCTION eth.graphql_subscription('state_cids', 'id');
+CREATE TRIGGER state_cids_ai AFTER INSERT ON eth.state_cids FOR EACH ROW EXECUTE PROCEDURE eth.graphql_subscription('state_cids', 'id');
 
 
 --
 -- Name: storage_cids storage_cids_ai; Type: TRIGGER; Schema: eth; Owner: -
 --
 
-CREATE TRIGGER storage_cids_ai AFTER INSERT ON eth.storage_cids FOR EACH ROW EXECUTE FUNCTION eth.graphql_subscription('storage_cids', 'id');
+CREATE TRIGGER storage_cids_ai AFTER INSERT ON eth.storage_cids FOR EACH ROW EXECUTE PROCEDURE eth.graphql_subscription('storage_cids', 'id');
 
 
 --
 -- Name: transaction_cids transaction_cids_ai; Type: TRIGGER; Schema: eth; Owner: -
 --
 
-CREATE TRIGGER transaction_cids_ai AFTER INSERT ON eth.transaction_cids FOR EACH ROW EXECUTE FUNCTION eth.graphql_subscription('transaction_cids', 'id');
+CREATE TRIGGER transaction_cids_ai AFTER INSERT ON eth.transaction_cids FOR EACH ROW EXECUTE PROCEDURE eth.graphql_subscription('transaction_cids', 'id');
 
 
 --
 -- Name: uncle_cids uncle_cids_ai; Type: TRIGGER; Schema: eth; Owner: -
 --
 
-CREATE TRIGGER uncle_cids_ai AFTER INSERT ON eth.uncle_cids FOR EACH ROW EXECUTE FUNCTION eth.graphql_subscription('uncle_cids', 'id');
+CREATE TRIGGER uncle_cids_ai AFTER INSERT ON eth.uncle_cids FOR EACH ROW EXECUTE PROCEDURE eth.graphql_subscription('uncle_cids', 'id');
 
 
 --
@@ -1283,11 +1258,11 @@ ALTER TABLE ONLY eth.log_cids
 
 
 --
--- Name: receipt_cids receipt_cids_mh_key_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: -
+-- Name: receipt_cids receipt_cids_leaf_mh_key_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: -
 --
 
 ALTER TABLE ONLY eth.receipt_cids
-    ADD CONSTRAINT receipt_cids_mh_key_fkey FOREIGN KEY (mh_key) REFERENCES public.blocks(key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
+    ADD CONSTRAINT receipt_cids_leaf_mh_key_fkey FOREIGN KEY (leaf_mh_key) REFERENCES public.blocks(key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
 
 
 --

scripts/check_diff.sh

@@ -0,0 +1,7 @@
+lines=$( git diff --no-index ./db/migration_schema.sql schema.sql | wc -l )
+if [ $lines -gt 0 ]; then
+    echo "There are differences in schema"
+    git diff --no-index ./db/migration_schema.sql schema.sql
+    exit 1
+fi
+echo "Schema is latest"

scripts/startup_script.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+# Runs the db migrations
+set +x
+
+# Construct the connection string for postgres
+VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable
+
+# Run the DB migrations
+echo "Connecting with: $VDB_PG_CONNECT"
+echo "Running database migrations"
+./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up
+
+# If the db migrations ran without err
+if [[ $? -eq 0 ]]; then
+    echo "Migration process ran successfully"
+else
+    echo "Could not run migrations. Are the database details correct?"
+    exit 1
+fi