Allow edited release tags to run the pipeline

.github/workflows/on-main.yaml

@@ -1,25 +0,0 @@
-name: Docker Compose Build
-
-on:
-  push:
-    branches:
-      - main
-
-jobs:
-  build:
-    name: Run docker build
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Get the version
-        id: vars
-        run: echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
-      - name: Run docker build
-        run: make docker-build
-      - name: Tag docker image
-        run: docker tag vulcanize/ipld-eth-db docker.pkg.github.com/vulcanize/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.sha}}
-      - name: Docker Login
-        run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
-      - name: Docker Push
-        run: docker push docker.pkg.github.com/vulcanize/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.sha}}
-

.github/workflows/on-pr.yaml

@@ -3,61 +3,61 @@ name: Docker Build
 on: [pull_request]
 
 jobs:
-  # concise_migration_diff:
+  concise_migration_diff:
-  #   name: Verify concise migration and generated schema
+    name: Verify concise migration and generated schema
-  #   runs-on: ubuntu-latest
+    runs-on: ubuntu-latest
-  #   steps:
+    steps:
-  #     - uses: actions/checkout@v2
+      - uses: actions/checkout@v2
-  #     - name: Run docker concise migration build
+      - name: Run docker concise migration build
-  #       run: make docker-concise-migration-build
+        run: make docker-concise-migration-build
-  #     - name: Run database
+      - name: Run database
-  #       run: docker-compose -f docker-compose.test.yml up -d test-db
+        run: docker-compose -f docker-compose.test.yml up -d test-db
-  #     - name: Test concise migration
+      - name: Test concise migration
-  #       run: |
+        run: |
-  #         sleep 10
+          sleep 10
-  #         docker run --rm --network host -e DATABASE_USER=vdbm -e DATABASE_PASSWORD=password \
+          docker run --rm --network host -e DATABASE_USER=vdbm -e DATABASE_PASSWORD=password \
-  #         -e DATABASE_HOSTNAME=127.0.0.1 -e DATABASE_PORT=8066 -e DATABASE_NAME=vulcanize_testing \
+          -e DATABASE_HOSTNAME=127.0.0.1 -e DATABASE_PORT=8066 -e DATABASE_NAME=vulcanize_testing \
-  #         vulcanize/concise-migration-build
+          vulcanize/concise-migration-build
-  #     - name: Verify schema is latest
+      - name: Verify schema is latest
-  #       run: |
+        run: |
-  #         PGPASSWORD="password" pg_dump -h localhost -p 8066 -U vdbm vulcanize_testing --no-owner --schema-only > ./db/migration_schema.sql
+          PGPASSWORD="password" pg_dump -h localhost -p 8066 -U vdbm vulcanize_testing --no-owner --schema-only > ./db/migration_schema.sql
-  #         ./scripts/check_diff.sh ./db/migration_schema.sql db/schema.sql
+          ./scripts/check_diff.sh ./db/migration_schema.sql db/schema.sql
 
-  # incremental_migration_diff:
+  incremental_migration_diff:
-  #   name: Compare conscise migration schema with incremental migration.
+    name: Compare conscise migration schema with incremental migration.
-  #   runs-on: ubuntu-latest
+    runs-on: ubuntu-latest
-  #   steps:
+    steps:
-  #     - uses: actions/checkout@v2
+      - uses: actions/checkout@v2
-  #     - name: Run database
+      - name: Run database
-  #       run: docker-compose -f docker-compose.test.yml up -d test-db statediff-migrations
+        run: docker-compose -f docker-compose.test.yml up -d test-db statediff-migrations
-  #     - name: Test incremental migration
+      - name: Test incremental migration
-  #       run: |
+        run: |
-  #         sleep 10
+          sleep 10
-  #         docker run --rm --network host -e DATABASE_USER=vdbm -e DATABASE_PASSWORD=password \
+          docker run --rm --network host -e DATABASE_USER=vdbm -e DATABASE_PASSWORD=password \
-  #         -e DATABASE_HOSTNAME=127.0.0.1 -e DATABASE_PORT=8066 -e DATABASE_NAME=vulcanize_testing \
+          -e DATABASE_HOSTNAME=127.0.0.1 -e DATABASE_PORT=8066 -e DATABASE_NAME=vulcanize_testing \
-  #         vulcanize/statediff-migrations:v0.9.0
+          vulcanize/statediff-migrations:v0.9.0
-  #     - name: Verify schema is latest
+      - name: Verify schema is latest
-  #       run: |
+        run: |
-  #         PGPASSWORD="password" pg_dump -h localhost -p 8066 -U vdbm vulcanize_testing --no-owner --schema-only > ./db/migration_schema.sql
+          PGPASSWORD="password" pg_dump -h localhost -p 8066 -U vdbm vulcanize_testing --no-owner --schema-only > ./db/migration_schema.sql
-  #         ./scripts/check_diff.sh db/schema.sql ./db/migration_schema.sql
+          ./scripts/check_diff.sh db/schema.sql ./db/migration_schema.sql
 
-  # migration:
+  migration:
-  #   name: Compare up and down migration
+    name: Compare up and down migration
-  #   env:
+    env:
-  #     GOPATH: /tmp/go
+      GOPATH: /tmp/go
-  #   strategy:
+    strategy:
-  #     matrix:
+      matrix:
-  #       go-version: [ 1.16.x ]
+        go-version: [ 1.16.x ]
-  #       os: [ ubuntu-latest ]
+        os: [ ubuntu-latest ]
-  #   runs-on: ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
-  #   steps:
+    steps:
-  #     - name: Create GOPATH
+      - name: Create GOPATH
-  #       run: mkdir -p /tmp/go
+        run: mkdir -p /tmp/go
-  #     - name: Install Go
+      - name: Install Go
-  #       uses: actions/setup-go@v2
+        uses: actions/setup-go@v2
-  #       with:
+        with:
-  #         go-version: ${{ matrix.go-version }}
+          go-version: ${{ matrix.go-version }}
-  #     - uses: actions/checkout@v2
+      - uses: actions/checkout@v2
-  #     - name: Test migration
+      - name: Test migration
-  #       run: |
+        run: |
-  #         timeout 5m make test-migrations
+          timeout 5m make test-migrations

.github/workflows/publish.yaml

@@ -1,11 +1,28 @@
 name: Publish Docker image
 on:
   release:
-    types: [published]
+    types: [published, edited]
 jobs:
+  build:
+    name: Run docker build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Get the version
+        id: vars
+        run: echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
+      - name: Run docker build
+        run: make docker-build
+      - name: Tag docker image
+        run: docker tag vulcanize/ipld-eth-db docker.pkg.github.com/vulcanize/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.sha}}
+      - name: Docker Login
+        run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
+      - name: Docker Push
+        run: docker push docker.pkg.github.com/vulcanize/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.sha}}
   push_to_registries:
     name: Push Docker image to Docker Hub
     runs-on: ubuntu-latest
+    needs: build
     steps:
       - name: Get the version
         id: vars
@@ -21,4 +38,4 @@ jobs:
       - name: Tag docker image
         run: docker tag docker.pkg.github.com/vulcanize/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.sha}} vulcanize/ipld-eth-db:${{steps.vars.outputs.tag}}
       - name: Docker Push to Docker Hub
-        run: docker push vulcanize/ipld-eth-db:${{steps.vars.outputs.tag}}
+        run: docker push vulcanize/ipld-eth-db:${{steps.vars.outputs.tag}}

Dockerfile

@@ -1,28 +1,3 @@
-FROM golang:1.16-alpine as builder
+FROM postgres:12-alpine
 
-RUN apk --update --no-cache add make git g++ linux-headers
+COPY ./schema.sql /docker-entrypoint-initdb.d/init.sql
-
-ADD . /go/src/github.com/vulcanize/ipld-eth-db
-
-# Build migration tool
-WORKDIR /go/src/github.com/pressly
-RUN git clone https://github.com/pressly/goose.git
-WORKDIR /go/src/github.com/pressly/goose/cmd/goose
-RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -tags='no_sqlite3' -o goose .
-
-# app container
-FROM alpine
-
-WORKDIR /app
-
-COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-db/scripts/startup_script.sh .
-
-# copy over files for multi-node setup
-COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-db/docker-compose.test.yml docker-multi-node/docker-compose.test.yml
-COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-db/scripts/init-access-node.sh docker-multi-node/scripts/init-access-node.sh
-COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-db/scripts/init-data-node.sh docker-multi-node/scripts/init-data-node.sh
-
-COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose goose
-COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-db/db/migrations migrations/vulcanizedb
-
-ENTRYPOINT ["/app/startup_script.sh"]

README.md

@@ -2,44 +2,4 @@
 Schemas and utils for IPLD ETH Postgres database
 
 ## Database UML
-![](vulcanize_db.png)
+![](vulcanize_db.png)
-
-## Run
-
-* Remove any existing containers / volumes:
-
-  ```bash
-  docker-compose down -v --remove-orphans
-  ```
-
-* Spin up an access node and three data nodes using [docker-compose.test.yml](./docker-compose.test.yml):
-
-  ```bash
-  docker-compose -f docker-compose.test.yml up
-  ```
-
-  Following final output should be seen on all the nodes:
-
-    ```
-    LOG:  TimescaleDB background worker launcher connected to shared catalogs
-    ```
-
-* Edit [startup_script.sh](./scripts/startup_script.sh) to change the number of migrations to be run:
-
-  ```bash
-  ./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up-to 24
-  ```
-
-* In another `ipld-eth-db` terminal window, build an image `migrations-test` using [Dockerfile](./db/Dockerfile):
-
-  ```bash
-  docker build -t migrations-test -f ./db/Dockerfile .
-  ```
-
-* After the access and data nodes have spun up, start a container using `migrations-test` image to run the db migrations:
-
-  ```bash
-  # Here, we are running the container using host network.
-  # So connect to access node on 127.0.0.1:8066
-  docker run --rm --network host -e DATABASE_USER=postgres -e DATABASE_PASSWORD=password -e DATABASE_HOSTNAME=127.0.0.1 -e DATABASE_PORT=8066 -e DATABASE_NAME=vulcanize_testing_v4 migrations-test
-  ```

db/migrations/00001_create_ipfs_blocks_table.sql

@@ -1,9 +1,7 @@
 -- +goose Up
 CREATE TABLE IF NOT EXISTS public.blocks (
-    block_number BIGINT NOT NULL,
+  key TEXT PRIMARY KEY,
-    key TEXT NOT NULL,
+  data BYTEA NOT NULL
-    data BYTEA NOT NULL,
-    PRIMARY KEY (key, block_number)
 );
 
 -- +goose Down

db/migrations/00002_create_nodes_table.sql

@@ -1,10 +1,10 @@
 -- +goose Up
-CREATE TABLE IF NOT EXISTS nodes (
+CREATE TABLE nodes (
-    genesis_block VARCHAR(66),
+  genesis_block VARCHAR(66),
-    network_id    VARCHAR,
+  network_id    VARCHAR,
-    node_id       VARCHAR(128) PRIMARY KEY,
+  node_id       VARCHAR(128) PRIMARY KEY,
-    client_name   VARCHAR,
+  client_name   VARCHAR,
-    chain_id      INTEGER DEFAULT 1
+  chain_id      INTEGER DEFAULT 1
 );
 
 -- +goose Down

db/migrations/00004_create_eth_header_cids_table.sql

@@ -1,11 +1,11 @@
 -- +goose Up
-CREATE TABLE IF NOT EXISTS eth.header_cids (
+CREATE TABLE eth.header_cids (
     block_number          BIGINT NOT NULL,
-    block_hash            VARCHAR(66) NOT NULL,
+    block_hash            VARCHAR(66) PRIMARY KEY,
     parent_hash           VARCHAR(66) NOT NULL,
     cid                   TEXT NOT NULL,
     td                    NUMERIC NOT NULL,
-    node_id               VARCHAR(128) NOT NULL,
+    node_id               VARCHAR(128) NOT NULL REFERENCES nodes (node_id) ON DELETE CASCADE,
     reward                NUMERIC NOT NULL,
     state_root            VARCHAR(66) NOT NULL,
     tx_root               VARCHAR(66) NOT NULL,
@@ -13,10 +13,9 @@ CREATE TABLE IF NOT EXISTS eth.header_cids (
     uncle_root            VARCHAR(66) NOT NULL,
     bloom                 BYTEA NOT NULL,
     timestamp             BIGINT NOT NULL,
-    mh_key                TEXT NOT NULL,
+    mh_key                TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
     times_validated       INTEGER NOT NULL DEFAULT 1,
-    coinbase              VARCHAR(66) NOT NULL,
+    coinbase              VARCHAR(66) NOT NULL
-    PRIMARY KEY (block_hash, block_number)
 );
 
 -- +goose Down

db/migrations/00005_create_eth_uncle_cids_table.sql

@@ -1,13 +1,11 @@
 -- +goose Up
-CREATE TABLE IF NOT EXISTS eth.uncle_cids (
+CREATE TABLE eth.uncle_cids (
-    block_number          BIGINT NOT NULL,
+    block_hash            VARCHAR(66) PRIMARY KEY,
-    block_hash            VARCHAR(66) NOT NULL,
+    header_id             VARCHAR(66) NOT NULL REFERENCES eth.header_cids (block_hash) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-    header_id             VARCHAR(66) NOT NULL,
     parent_hash           VARCHAR(66) NOT NULL,
     cid                   TEXT NOT NULL,
     reward                NUMERIC NOT NULL,
-    mh_key                TEXT NOT NULL,
+    mh_key                TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED
-    PRIMARY KEY (block_hash, block_number)
 );
 
 -- +goose Down

db/migrations/00006_create_eth_transaction_cids_table.sql

@@ -1,17 +1,15 @@
 -- +goose Up
-CREATE TABLE IF NOT EXISTS eth.transaction_cids (
+CREATE TABLE eth.transaction_cids (
-    block_number          BIGINT NOT NULL,
+    header_id             VARCHAR(66) NOT NULL REFERENCES eth.header_cids (block_hash) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-    header_id             VARCHAR(66) NOT NULL,
+    tx_hash               VARCHAR(66) PRIMARY KEY,
-    tx_hash               VARCHAR(66) NOT NULL,
     cid                   TEXT NOT NULL,
     dst                   VARCHAR(66) NOT NULL,
     src                   VARCHAR(66) NOT NULL,
     index                 INTEGER NOT NULL,
-    mh_key                TEXT NOT NULL,
+    mh_key                TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
     tx_data               BYTEA,
     tx_type               INTEGER,
-    value                 NUMERIC,
+    value                 NUMERIC
-    PRIMARY KEY (tx_hash, block_number)
 );
 
 -- +goose Down

db/migrations/00007_create_eth_receipt_cids_table.sql

@@ -1,15 +1,13 @@
 -- +goose Up
-CREATE TABLE IF NOT EXISTS eth.receipt_cids (
+CREATE TABLE eth.receipt_cids (
-    block_number          BIGINT NOT NULL,
+    tx_id                 VARCHAR(66) PRIMARY KEY REFERENCES eth.transaction_cids (tx_hash) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-    tx_id                 VARCHAR(66) NOT NULL,
     leaf_cid              TEXT NOT NULL,
     contract              VARCHAR(66),
     contract_hash         VARCHAR(66),
-    leaf_mh_key           TEXT NOT NULL,
+    leaf_mh_key           TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
     post_state            VARCHAR(66),
     post_status           INTEGER,
-    log_root              VARCHAR(66),
+    log_root              VARCHAR(66)
-    PRIMARY KEY (tx_id, block_number)
 );
 
 -- +goose Down

db/migrations/00008_create_eth_state_cids_table.sql

@@ -1,14 +1,13 @@
 -- +goose Up
-CREATE TABLE IF NOT EXISTS eth.state_cids (
+CREATE TABLE eth.state_cids (
-    block_number          BIGINT NOT NULL,
+    header_id             VARCHAR(66) NOT NULL REFERENCES eth.header_cids (block_hash) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-    header_id             VARCHAR(66) NOT NULL,
     state_leaf_key        VARCHAR(66),
     cid                   TEXT NOT NULL,
     state_path            BYTEA NOT NULL,
     node_type             INTEGER NOT NULL,
     diff                  BOOLEAN NOT NULL DEFAULT FALSE,
-    mh_key                TEXT NOT NULL,
+    mh_key                TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-    PRIMARY KEY (state_path, header_id, block_number)
+    PRIMARY KEY (header_id, state_path)
 );
 
 -- +goose Down

db/migrations/00009_create_eth_storage_cids_table.sql

@@ -1,6 +1,5 @@
 -- +goose Up
-CREATE TABLE IF NOT EXISTS eth.storage_cids (
+CREATE TABLE eth.storage_cids (
-    block_number          BIGINT NOT NULL,
     header_id             VARCHAR(66) NOT NULL,
     state_path            BYTEA NOT NULL,
     storage_leaf_key      VARCHAR(66),
@@ -8,8 +7,9 @@ CREATE TABLE IF NOT EXISTS eth.storage_cids (
     storage_path          BYTEA NOT NULL,
     node_type             INTEGER NOT NULL,
     diff                  BOOLEAN NOT NULL DEFAULT FALSE,
-    mh_key                TEXT NOT NULL,
+    mh_key                TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-    PRIMARY KEY (storage_path, state_path, header_id, block_number)
+    FOREIGN KEY (header_id, state_path) REFERENCES eth.state_cids (header_id, state_path) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
+    PRIMARY KEY (header_id, state_path, storage_path)
 );
 
 -- +goose Down

db/migrations/00010_create_eth_state_accounts_table.sql

@@ -1,13 +1,13 @@
 -- +goose Up
-CREATE TABLE IF NOT EXISTS eth.state_accounts (
+CREATE TABLE eth.state_accounts (
-    block_number          BIGINT NOT NULL,
     header_id             VARCHAR(66) NOT NULL,
     state_path            BYTEA NOT NULL,
     balance               NUMERIC NOT NULL,
     nonce                 BIGINT NOT NULL,
     code_hash             BYTEA NOT NULL,
     storage_root          VARCHAR(66) NOT NULL,
-    PRIMARY KEY (state_path, header_id, block_number)
+    FOREIGN KEY (header_id, state_path) REFERENCES eth.state_cids (header_id, state_path) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
+    PRIMARY KEY (header_id, state_path)
 );
 
 -- +goose Down

db/migrations/00011_create_eth_access_list_elements_table.sql

@@ -1,11 +1,10 @@
 -- +goose Up
-CREATE TABLE IF NOT EXISTS eth.access_list_elements (
+CREATE TABLE eth.access_list_elements (
-    block_number          BIGINT NOT NULL,
+    tx_id                 VARCHAR(66) NOT NULL REFERENCES eth.transaction_cids (tx_hash) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-    tx_id                 VARCHAR(66) NOT NULL,
     index                 INTEGER NOT NULL,
     address               VARCHAR(66),
     storage_keys          VARCHAR(66)[],
-    PRIMARY KEY (tx_id, index, block_number)
+    PRIMARY KEY (tx_id, index)
 );
 
 -- +goose Down

db/migrations/00012_create_eth_log_cids_table.sql

@@ -1,9 +1,8 @@
 -- +goose Up
-CREATE TABLE IF NOT EXISTS eth.log_cids (
+CREATE TABLE eth.log_cids (
-    block_number        BIGINT NOT NULL,
     leaf_cid            TEXT NOT NULL,
-    leaf_mh_key         TEXT NOT NULL,
+    leaf_mh_key         TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-    rct_id              VARCHAR(66) NOT NULL,
+    rct_id              VARCHAR(66) NOT NULL REFERENCES eth.receipt_cids (tx_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
     address             VARCHAR(66) NOT NULL,
     index               INTEGER NOT NULL,
     topic0              VARCHAR(66),
@@ -11,7 +10,7 @@ CREATE TABLE IF NOT EXISTS eth.log_cids (
     topic2              VARCHAR(66),
     topic3              VARCHAR(66),
     log_data            BYTEA,
-    PRIMARY KEY (rct_id, index, block_number)
+    PRIMARY KEY (rct_id, index)
 );
 
 -- +goose Down

db/migrations/00014_create_cid_indexes.sql

@@ -1,61 +1,52 @@
 -- +goose Up
 -- header indexes
-CREATE INDEX header_block_number_index ON eth.header_cids USING brin (block_number);
+CREATE INDEX block_number_index ON eth.header_cids USING brin (block_number);
-CREATE UNIQUE INDEX header_cid_index ON eth.header_cids USING btree (cid, block_number);
+CREATE UNIQUE INDEX header_cid_index ON eth.header_cids USING btree (cid);
-CREATE UNIQUE INDEX header_mh_block_number_index ON eth.header_cids USING btree (mh_key, block_number);
+CREATE UNIQUE INDEX header_mh_index ON eth.header_cids USING btree (mh_key);
 CREATE INDEX state_root_index ON eth.header_cids USING btree (state_root);
 CREATE INDEX timestamp_index ON eth.header_cids USING brin (timestamp);
 
 -- uncle indexes
-CREATE INDEX uncle_block_number_index ON eth.uncle_cids USING brin (block_number);
-CREATE UNIQUE INDEX uncle_mh_block_number_index ON eth.uncle_cids USING btree (mh_key, block_number);
 CREATE INDEX uncle_header_id_index ON eth.uncle_cids USING btree (header_id);
 
 -- transaction indexes
-CREATE INDEX tx_block_number_index ON eth.transaction_cids USING brin (block_number);
 CREATE INDEX tx_header_id_index ON eth.transaction_cids USING btree (header_id);
-CREATE UNIQUE INDEX tx_cid_index ON eth.transaction_cids USING btree (cid, block_number);
+CREATE UNIQUE INDEX tx_cid_index ON eth.transaction_cids USING btree (cid);
-CREATE UNIQUE INDEX tx_mh_block_number_index ON eth.transaction_cids USING btree (mh_key, block_number);
+CREATE UNIQUE INDEX tx_mh_index ON eth.transaction_cids USING btree (mh_key);
 CREATE INDEX tx_dst_index ON eth.transaction_cids USING btree (dst);
 CREATE INDEX tx_src_index ON eth.transaction_cids USING btree (src);
 
 -- receipt indexes
-CREATE INDEX rct_block_number_index ON eth.receipt_cids USING brin (block_number);
 CREATE INDEX rct_leaf_cid_index ON eth.receipt_cids USING btree (leaf_cid);
-CREATE INDEX rct_leaf_mh_block_number_index ON eth.receipt_cids USING btree (leaf_mh_key, block_number);
+CREATE INDEX rct_leaf_mh_index ON eth.receipt_cids USING btree (leaf_mh_key);
 CREATE INDEX rct_contract_index ON eth.receipt_cids USING btree (contract);
 CREATE INDEX rct_contract_hash_index ON eth.receipt_cids USING btree (contract_hash);
 
 -- state node indexes
-CREATE INDEX state_block_number_index ON eth.state_cids USING brin (block_number);
 CREATE INDEX state_leaf_key_index ON eth.state_cids USING btree (state_leaf_key);
 CREATE INDEX state_cid_index ON eth.state_cids USING btree (cid);
-CREATE INDEX state_mh_block_number_index ON eth.state_cids USING btree (mh_key, block_number);
+CREATE INDEX state_mh_index ON eth.state_cids USING btree (mh_key);
-CREATE INDEX state_header_id_index ON eth.state_cids USING btree (header_id);
+CREATE INDEX state_path_index ON eth.state_cids USING btree (state_path);
 CREATE INDEX state_node_type_index ON eth.state_cids USING btree (node_type);
 
 -- storage node indexes
-CREATE INDEX storage_block_number_index ON eth.storage_cids USING brin (block_number);
 CREATE INDEX storage_state_path_index ON eth.storage_cids USING btree (state_path);
 CREATE INDEX storage_leaf_key_index ON eth.storage_cids USING btree (storage_leaf_key);
 CREATE INDEX storage_cid_index ON eth.storage_cids USING btree (cid);
-CREATE INDEX storage_mh_block_number_index ON eth.storage_cids USING btree (mh_key, block_number);
+CREATE INDEX storage_mh_index ON eth.storage_cids USING btree (mh_key);
-CREATE INDEX storage_header_id_index ON eth.storage_cids USING btree (header_id);
+CREATE INDEX storage_path_index ON eth.storage_cids USING btree (storage_path);
 CREATE INDEX storage_node_type_index ON eth.storage_cids USING btree (node_type);
 
 -- state accounts indexes
-CREATE INDEX account_block_number_index ON eth.state_accounts USING brin (block_number);
+CREATE INDEX account_state_path_index ON eth.state_accounts USING btree (state_path);
-CREATE INDEX account_header_id_index ON eth.state_accounts USING btree (header_id);
+CREATE INDEX storage_root_index ON eth.state_accounts USING btree (storage_root);
-CREATE INDEX account_storage_root_index ON eth.state_accounts USING btree (storage_root);
 
 -- access list indexes
-CREATE INDEX access_list_block_number_index ON eth.access_list_elements USING brin (block_number);
 CREATE INDEX access_list_element_address_index ON eth.access_list_elements USING btree (address);
 CREATE INDEX access_list_storage_keys_index ON eth.access_list_elements USING gin (storage_keys);
 
 -- log indexes
-CREATE INDEX log_block_number_index ON eth.log_cids USING brin (block_number);
+CREATE INDEX log_mh_index ON eth.log_cids USING btree (leaf_mh_key);
-CREATE INDEX log_leaf_mh_block_number_index ON eth.log_cids USING btree (leaf_mh_key, block_number);
 CREATE INDEX log_cid_index ON  eth.log_cids USING btree (leaf_cid);
 CREATE INDEX log_address_index ON eth.log_cids USING btree (address);
 CREATE INDEX log_topic0_index ON eth.log_cids USING btree (topic0);
@@ -71,59 +62,50 @@ DROP INDEX eth.log_topic1_index;
 DROP INDEX eth.log_topic0_index;
 DROP INDEX eth.log_address_index;
 DROP INDEX eth.log_cid_index;
-DROP INDEX eth.log_leaf_mh_block_number_index;
+DROP INDEX eth.log_mh_index;
-DROP INDEX eth.log_block_number_index;
 
 -- access list indexes
 DROP INDEX eth.access_list_storage_keys_index;
 DROP INDEX eth.access_list_element_address_index;
-DROP INDEX eth.access_list_block_number_index;
 
 -- state account indexes
-DROP INDEX eth.account_storage_root_index;
+DROP INDEX eth.storage_root_index;
-DROP index eth.account_header_id_index;
+DROP index eth.account_state_path_index;
-DROP INDEX eth.account_block_number_index;
 
 -- storage node indexes
 DROP INDEX eth.storage_node_type_index;
-DROP INDEX eth.storage_header_id_index;
+DROP INDEX eth.storage_path_index;
-DROP INDEX eth.storage_mh_block_number_index;
+DROP INDEX eth.storage_mh_index;
 DROP INDEX eth.storage_cid_index;
 DROP INDEX eth.storage_leaf_key_index;
 DROP INDEX eth.storage_state_path_index;
-DROP INDEX eth.storage_block_number_index;
 
 -- state node indexes
 DROP INDEX eth.state_node_type_index;
-DROP INDEX eth.state_header_id_index;
+DROP INDEX eth.state_path_index;
-DROP INDEX eth.state_mh_block_number_index;
+DROP INDEX eth.state_mh_index;
 DROP INDEX eth.state_cid_index;
 DROP INDEX eth.state_leaf_key_index;
-DROP INDEX eth.state_block_number_index;
 
 -- receipt indexes
 DROP INDEX eth.rct_contract_hash_index;
 DROP INDEX eth.rct_contract_index;
-DROP INDEX eth.rct_leaf_mh_block_number_index;
+DROP INDEX eth.rct_leaf_mh_index;
 DROP INDEX eth.rct_leaf_cid_index;
-DROP INDEX eth.rct_block_number_index;
 
 -- transaction indexes
 DROP INDEX eth.tx_src_index;
 DROP INDEX eth.tx_dst_index;
-DROP INDEX eth.tx_mh_block_number_index;
+DROP INDEX eth.tx_mh_index;
 DROP INDEX eth.tx_cid_index;
 DROP INDEX eth.tx_header_id_index;
-DROP INDEX eth.tx_block_number_index;
 
 -- uncle indexes
-DROP INDEX eth.uncle_block_number_index;
-DROP INDEX eth.uncle_mh_block_number_index;
 DROP INDEX eth.uncle_header_id_index;
 
 -- header indexes
 DROP INDEX eth.timestamp_index;
 DROP INDEX eth.state_root_index;
-DROP INDEX eth.header_mh_block_number_index;
+DROP INDEX eth.header_mh_index;
 DROP INDEX eth.header_cid_index;
-DROP INDEX eth.header_block_number_index;
+DROP INDEX eth.block_number_index;

db/migrations/00015_create_stored_functions.sql

@@ -0,0 +1,138 @@
+-- +goose Up
+-- +goose StatementBegin
+-- returns if a state leaf node was removed within the provided block number
+CREATE OR REPLACE FUNCTION was_state_leaf_removed(key character varying, hash character varying)
+    RETURNS boolean AS $$
+    SELECT state_cids.node_type = 3
+    FROM eth.state_cids
+             INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.block_hash)
+    WHERE state_leaf_key = key
+      AND block_number <= (SELECT block_number
+                           FROM eth.header_cids
+                           WHERE block_hash = hash)
+    ORDER BY block_number DESC LIMIT 1;
+$$
+language sql;
+-- +goose StatementEnd
+
+-- +goose StatementBegin
+CREATE TYPE child_result AS (
+    has_child BOOLEAN,
+    children eth.header_cids[]
+);
+
+CREATE OR REPLACE FUNCTION has_child(hash VARCHAR(66), height BIGINT) RETURNS child_result AS
+$BODY$
+DECLARE
+  child_height INT;
+  temp_child eth.header_cids;
+  new_child_result child_result;
+BEGIN
+  child_height = height + 1;
+  -- short circuit if there are no children
+  SELECT exists(SELECT 1
+              FROM eth.header_cids
+              WHERE parent_hash = hash
+                AND block_number = child_height
+              LIMIT 1)
+  INTO new_child_result.has_child;
+  -- collect all the children for this header
+  IF new_child_result.has_child THEN
+    FOR temp_child IN
+    SELECT * FROM eth.header_cids WHERE parent_hash = hash AND block_number = child_height
+    LOOP
+      new_child_result.children = array_append(new_child_result.children, temp_child);
+    END LOOP;
+  END IF;
+  RETURN new_child_result;
+END
+$BODY$
+LANGUAGE 'plpgsql';
+-- +goose StatementEnd
+
+-- +goose StatementBegin
+CREATE OR REPLACE FUNCTION canonical_header_from_array(headers eth.header_cids[]) RETURNS eth.header_cids AS
+$BODY$
+DECLARE
+  canonical_header eth.header_cids;
+  canonical_child eth.header_cids;
+  header eth.header_cids;
+  current_child_result child_result;
+  child_headers eth.header_cids[];
+  current_header_with_child eth.header_cids;
+  has_children_count INT DEFAULT 0;
+BEGIN
+  -- for each header in the provided set
+  FOREACH header IN ARRAY headers
+  LOOP
+    -- check if it has any children
+    current_child_result = has_child(header.block_hash, header.block_number);
+    IF current_child_result.has_child THEN
+      -- if it does, take note
+      has_children_count = has_children_count + 1;
+      current_header_with_child = header;
+      -- and add the children to the growing set of child headers
+      child_headers = array_cat(child_headers, current_child_result.children);
+    END IF;
+  END LOOP;
+  -- if none of the headers had children, none is more canonical than the other
+  IF has_children_count = 0 THEN
+    -- return the first one selected
+    SELECT * INTO canonical_header FROM unnest(headers) LIMIT 1;
+  -- if only one header had children, it can be considered the heaviest/canonical header of the set
+  ELSIF has_children_count = 1 THEN
+    -- return the only header with a child
+    canonical_header = current_header_with_child;
+  -- if there are multiple headers with children
+  ELSE
+    -- find the canonical header from the child set
+    canonical_child = canonical_header_from_array(child_headers);
+    -- the header that is parent to this header, is the canonical header at this level
+    SELECT * INTO canonical_header FROM unnest(headers)
+    WHERE block_hash = canonical_child.parent_hash;
+  END IF;
+  RETURN canonical_header;
+END
+$BODY$
+LANGUAGE 'plpgsql';
+-- +goose StatementEnd
+
+-- +goose StatementBegin
+CREATE OR REPLACE FUNCTION canonical_header_hash(height BIGINT) RETURNS character varying AS
+$BODY$
+DECLARE
+  canonical_header eth.header_cids;
+  headers eth.header_cids[];
+  header_count INT;
+  temp_header eth.header_cids;
+BEGIN
+  -- collect all headers at this height
+  FOR temp_header IN
+  SELECT * FROM eth.header_cids WHERE block_number = height
+  LOOP
+    headers = array_append(headers, temp_header);
+  END LOOP;
+  -- count the number of headers collected
+  header_count = array_length(headers, 1);
+  -- if we have less than 1 header, return NULL
+  IF header_count IS NULL OR header_count < 1 THEN
+    RETURN NULL;
+  -- if we have one header, return its hash
+  ELSIF header_count = 1 THEN
+    RETURN headers[1].block_hash;
+  -- if we have multiple headers we need to determine which one is canonical
+  ELSE
+    canonical_header = canonical_header_from_array(headers);
+    RETURN canonical_header.block_hash;
+  END IF;
+END;
+$BODY$
+LANGUAGE 'plpgsql';
+-- +goose StatementEnd
+
+-- +goose Down
+DROP FUNCTION was_state_leaf_removed;
+DROP FUNCTION canonical_header_hash;
+DROP FUNCTION canonical_header_from_array;
+DROP FUNCTION has_child;
+DROP TYPE child_result;

db/migrations/00018_update_db_version.sql

@@ -0,0 +1,6 @@
+-- +goose Up
+INSERT INTO public.db_version (singleton, version) VALUES (true, 'v3.0.0')
+ON CONFLICT (singleton) DO UPDATE SET (version, tstamp) = ('v3.0.0', NOW());
+
+-- +goose Down
+DELETE FROM public.db_version WHERE version = 'v3.0.0';

db/migrations/00020_update_db_version.sql

@@ -1,6 +0,0 @@
--- +goose Up
-INSERT INTO public.db_version (singleton, version) VALUES (true, 'v4.0.0')
-ON CONFLICT (singleton) DO UPDATE SET (version, tstamp) = ('v4.0.0', NOW());
-
--- +goose Down
-DELETE FROM public.db_version WHERE version = 'v4.0.0';

db/migrations/00021_add_data_nodes.sql

@@ -1,16 +0,0 @@
--- +goose NO TRANSACTION
--- +goose Up
--- this is generated by a script
-SELECT add_data_node('dn3', host => 'data-node-3', port => 5432, database => 'vulcanize_testing_v4', password => 'password');
-SELECT add_data_node('dn2', host => 'data-node-2', port => 5432, database => 'vulcanize_testing_v4', password => 'password');
-SELECT add_data_node('dn1', host => 'data-node-1', port => 5432, database => 'vulcanize_testing_v4', password => 'password');
-
-CALL distributed_exec($$ CREATE SCHEMA eth $$);
-
--- +goose Down
--- this is generated by a script
-CALL distributed_exec($$ DROP SCHEMA eth $$);
-
-SELECT delete_data_node('dn1', force => true, if_attached => true);
-SELECT delete_data_node('dn2', force => true, if_attached => true);
-SELECT delete_data_node('dn3', force => true, if_attached => true);

db/migrations/00022_convert_to_distributed_hypertables.sql

@@ -1,130 +0,0 @@
--- +goose Up
--- creating distributed hypertables from duplicate tables for now as we are getting the following error (while running geth
--- unit tests) if regular tables are directly converted to distributed hypertables
--- error: "cannot PREPARE a transaction that has executed LISTEN, UNLISTEN, or NOTIFY"
-
--- create new regular tables
-CREATE TABLE eth.log_cids_i (LIKE eth.log_cids INCLUDING ALL);
-CREATE TABLE eth.access_list_elements_i (LIKE eth.access_list_elements INCLUDING ALL);
-CREATE TABLE eth.state_accounts_i (LIKE eth.state_accounts INCLUDING ALL);
-CREATE TABLE eth.storage_cids_i (LIKE eth.storage_cids INCLUDING ALL);
-CREATE TABLE eth.state_cids_i (LIKE eth.state_cids INCLUDING ALL);
-CREATE TABLE eth.receipt_cids_i (LIKE eth.receipt_cids INCLUDING ALL);
-CREATE TABLE eth.transaction_cids_i (LIKE eth.transaction_cids INCLUDING ALL);
-CREATE TABLE eth.uncle_cids_i (LIKE eth.uncle_cids INCLUDING ALL);
-CREATE TABLE eth.header_cids_i (LIKE eth.header_cids INCLUDING ALL);
-CREATE TABLE public.blocks_i (LIKE public.blocks INCLUDING ALL);
-
--- turn them into distributed hypertables
-SELECT create_distributed_hypertable('public.blocks_i', 'block_number', chunk_time_interval => 32768);
-SELECT create_distributed_hypertable('eth.header_cids_i', 'block_number', chunk_time_interval => 32768);
-SELECT create_distributed_hypertable('eth.uncle_cids_i', 'block_number', chunk_time_interval => 32768);
-SELECT create_distributed_hypertable('eth.transaction_cids_i', 'block_number', chunk_time_interval => 32768);
-SELECT create_distributed_hypertable('eth.receipt_cids_i', 'block_number', chunk_time_interval => 32768);
-SELECT create_distributed_hypertable('eth.state_cids_i', 'block_number', chunk_time_interval => 32768);
-SELECT create_distributed_hypertable('eth.storage_cids_i', 'block_number', chunk_time_interval => 32768);
-SELECT create_distributed_hypertable('eth.state_accounts_i', 'block_number', chunk_time_interval => 32768);
-SELECT create_distributed_hypertable('eth.access_list_elements_i', 'block_number', chunk_time_interval => 32768);
-SELECT create_distributed_hypertable('eth.log_cids_i', 'block_number', chunk_time_interval => 32768);
-
--- migrate data
-INSERT INTO eth.log_cids_i (SELECT * FROM eth.log_cids);
-INSERT INTO eth.access_list_elements_i (SELECT * FROM eth.access_list_elements);
-INSERT INTO eth.state_accounts_i (SELECT * FROM eth.state_accounts);
-INSERT INTO eth.storage_cids_i (SELECT * FROM eth.storage_cids);
-INSERT INTO eth.state_cids_i (SELECT * FROM eth.state_cids);
-INSERT INTO eth.receipt_cids_i (SELECT * FROM eth.receipt_cids);
-INSERT INTO eth.transaction_cids_i (SELECT * FROM eth.transaction_cids);
-INSERT INTO eth.uncle_cids_i (SELECT * FROM eth.uncle_cids);
-INSERT INTO eth.header_cids_i (SELECT * FROM eth.header_cids);
-INSERT INTO public.blocks_i (SELECT * FROM public.blocks);
-
--- drop tables
-DROP TABLE eth.log_cids;
-DROP TABLE eth.access_list_elements;
-DROP TABLE eth.state_accounts;
-DROP TABLE eth.storage_cids;
-DROP TABLE eth.state_cids;
-DROP TABLE eth.receipt_cids;
-DROP TABLE eth.transaction_cids;
-DROP TABLE eth.uncle_cids;
-DROP TABLE eth.header_cids;
-DROP TABLE public.blocks;
-
--- rename distributed hypertables
-ALTER TABLE eth.log_cids_i RENAME TO log_cids;
-ALTER TABLE eth.access_list_elements_i RENAME TO access_list_elements;
-ALTER TABLE eth.state_accounts_i RENAME TO state_accounts;
-ALTER TABLE eth.storage_cids_i RENAME TO storage_cids;
-ALTER TABLE eth.state_cids_i RENAME TO state_cids;
-ALTER TABLE eth.receipt_cids_i RENAME TO receipt_cids;
-ALTER TABLE eth.transaction_cids_i RENAME TO transaction_cids;
-ALTER TABLE eth.uncle_cids_i RENAME TO uncle_cids;
-ALTER TABLE eth.header_cids_i RENAME TO header_cids;
-ALTER TABLE public.blocks_i RENAME TO blocks;
-CALL distributed_exec('ALTER TABLE eth.log_cids_i RENAME TO log_cids');
-CALL distributed_exec('ALTER TABLE eth.access_list_elements_i RENAME TO access_list_elements');
-CALL distributed_exec('ALTER TABLE eth.state_accounts_i RENAME TO state_accounts');
-CALL distributed_exec('ALTER TABLE eth.storage_cids_i RENAME TO storage_cids');
-CALL distributed_exec('ALTER TABLE eth.state_cids_i RENAME TO state_cids');
-CALL distributed_exec('ALTER TABLE eth.receipt_cids_i RENAME TO receipt_cids');
-CALL distributed_exec('ALTER TABLE eth.transaction_cids_i RENAME TO transaction_cids');
-CALL distributed_exec('ALTER TABLE eth.uncle_cids_i RENAME TO uncle_cids');
-CALL distributed_exec('ALTER TABLE eth.header_cids_i RENAME TO header_cids');
-CALL distributed_exec('ALTER TABLE public.blocks_i RENAME TO blocks');
-
--- update version
-INSERT INTO public.db_version (singleton, version) VALUES (true, 'v4.0.00-dh')
-    ON CONFLICT (singleton) DO UPDATE SET (version, tstamp) = ('v4.0.0-dh', NOW());
-
--- +goose Down
-INSERT INTO public.db_version (singleton, version) VALUES (true, 'v4.0.0')
-    ON CONFLICT (singleton) DO UPDATE SET (version, tstamp) = ('v4.0.0', NOW());
--- reversing conversion to hypertables requires migrating all data from every chunk back to a single table
--- create new regular tables
-CREATE TABLE eth.log_cids_i (LIKE eth.log_cids INCLUDING ALL);
-CREATE TABLE eth.access_list_elements_i (LIKE eth.access_list_elements INCLUDING ALL);
-CREATE TABLE eth.state_accounts_i (LIKE eth.state_accounts INCLUDING ALL);
-CREATE TABLE eth.storage_cids_i (LIKE eth.storage_cids INCLUDING ALL);
-CREATE TABLE eth.state_cids_i (LIKE eth.state_cids INCLUDING ALL);
-CREATE TABLE eth.receipt_cids_i (LIKE eth.receipt_cids INCLUDING ALL);
-CREATE TABLE eth.transaction_cids_i (LIKE eth.transaction_cids INCLUDING ALL);
-CREATE TABLE eth.uncle_cids_i (LIKE eth.uncle_cids INCLUDING ALL);
-CREATE TABLE eth.header_cids_i (LIKE eth.header_cids INCLUDING ALL);
-CREATE TABLE public.blocks_i (LIKE public.blocks INCLUDING ALL);
-
--- migrate data
-INSERT INTO eth.log_cids_i (SELECT * FROM eth.log_cids);
-INSERT INTO eth.access_list_elements_i (SELECT * FROM eth.access_list_elements);
-INSERT INTO eth.state_accounts_i (SELECT * FROM eth.state_accounts);
-INSERT INTO eth.storage_cids_i (SELECT * FROM eth.storage_cids);
-INSERT INTO eth.state_cids_i (SELECT * FROM eth.state_cids);
-INSERT INTO eth.receipt_cids_i (SELECT * FROM eth.receipt_cids);
-INSERT INTO eth.transaction_cids_i (SELECT * FROM eth.transaction_cids);
-INSERT INTO eth.uncle_cids_i (SELECT * FROM eth.uncle_cids);
-INSERT INTO eth.header_cids_i (SELECT * FROM eth.header_cids);
-INSERT INTO public.blocks_i (SELECT * FROM public.blocks);
-
--- drop distributed hypertables
-DROP TABLE eth.log_cids;
-DROP TABLE eth.access_list_elements;
-DROP TABLE eth.state_accounts;
-DROP TABLE eth.storage_cids;
-DROP TABLE eth.state_cids;
-DROP TABLE eth.receipt_cids;
-DROP TABLE eth.transaction_cids;
-DROP TABLE eth.uncle_cids;
-DROP TABLE eth.header_cids;
-DROP TABLE public.blocks;
-
--- rename hypertable tables
-ALTER TABLE eth.log_cids_i RENAME TO log_cids;
-ALTER TABLE eth.access_list_elements_i RENAME TO access_list_elements;
-ALTER TABLE eth.state_accounts_i RENAME TO state_accounts;
-ALTER TABLE eth.storage_cids_i RENAME TO storage_cids;
-ALTER TABLE eth.state_cids_i RENAME TO state_cids;
-ALTER TABLE eth.receipt_cids_i RENAME TO receipt_cids;
-ALTER TABLE eth.transaction_cids_i RENAME TO transaction_cids;
-ALTER TABLE eth.uncle_cids_i RENAME TO uncle_cids;
-ALTER TABLE eth.header_cids_i RENAME TO header_cids;
-ALTER TABLE public.blocks_i RENAME TO blocks;

db/migrations/00023_attach_data_nodes.sql

@@ -1,5 +0,0 @@
--- +goose Up
--- to be generated by a script
-
--- +goose Down
--- to be generated by a script

db/migrations/00024_create_stored_functions.sql

@@ -1,248 +0,0 @@
--- +goose Up
--- +goose StatementBegin
--- returns if a state leaf node was removed within the provided block number
-CREATE OR REPLACE FUNCTION was_state_leaf_removed(key character varying, hash character varying)
-    RETURNS boolean AS $$
-    SELECT state_cids.node_type = 3
-    FROM eth.state_cids
-             INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.block_hash)
-    WHERE state_leaf_key = key
-      AND state_cids.block_number <= (SELECT block_number
-                           FROM eth.header_cids
-                           WHERE block_hash = hash)
-    ORDER BY state_cids.block_number DESC LIMIT 1;
-$$
-language sql;
--- +goose StatementEnd
-
--- +goose StatementBegin
-CREATE TYPE child_result AS (
-    has_child BOOLEAN,
-    children eth.header_cids[]
-);
-
-CREATE OR REPLACE FUNCTION has_child(hash VARCHAR(66), height BIGINT) RETURNS child_result AS
-$BODY$
-DECLARE
-  child_height INT;
-  temp_child eth.header_cids;
-  new_child_result child_result;
-BEGIN
-  child_height = height + 1;
-  -- short circuit if there are no children
-  SELECT exists(SELECT 1
-              FROM eth.header_cids
-              WHERE parent_hash = hash
-                AND block_number = child_height
-              LIMIT 1)
-  INTO new_child_result.has_child;
-  -- collect all the children for this header
-  IF new_child_result.has_child THEN
-    FOR temp_child IN
-    SELECT * FROM eth.header_cids WHERE parent_hash = hash AND block_number = child_height
-    LOOP
-      new_child_result.children = array_append(new_child_result.children, temp_child);
-    END LOOP;
-  END IF;
-  RETURN new_child_result;
-END
-$BODY$
-LANGUAGE 'plpgsql';
--- +goose StatementEnd
-
--- +goose StatementBegin
-CREATE OR REPLACE FUNCTION canonical_header_from_array(headers eth.header_cids[]) RETURNS eth.header_cids AS
-$BODY$
-DECLARE
-  canonical_header eth.header_cids;
-  canonical_child eth.header_cids;
-  header eth.header_cids;
-  current_child_result child_result;
-  child_headers eth.header_cids[];
-  current_header_with_child eth.header_cids;
-  has_children_count INT DEFAULT 0;
-BEGIN
-  -- for each header in the provided set
-  FOREACH header IN ARRAY headers
-  LOOP
-    -- check if it has any children
-    current_child_result = has_child(header.block_hash, header.block_number);
-    IF current_child_result.has_child THEN
-      -- if it does, take note
-      has_children_count = has_children_count + 1;
-      current_header_with_child = header;
-      -- and add the children to the growing set of child headers
-      child_headers = array_cat(child_headers, current_child_result.children);
-    END IF;
-  END LOOP;
-  -- if none of the headers had children, none is more canonical than the other
-  IF has_children_count = 0 THEN
-    -- return the first one selected
-    SELECT * INTO canonical_header FROM unnest(headers) LIMIT 1;
-  -- if only one header had children, it can be considered the heaviest/canonical header of the set
-  ELSIF has_children_count = 1 THEN
-    -- return the only header with a child
-    canonical_header = current_header_with_child;
-  -- if there are multiple headers with children
-  ELSE
-    -- find the canonical header from the child set
-    canonical_child = canonical_header_from_array(child_headers);
-    -- the header that is parent to this header, is the canonical header at this level
-    SELECT * INTO canonical_header FROM unnest(headers)
-    WHERE block_hash = canonical_child.parent_hash;
-  END IF;
-  RETURN canonical_header;
-END
-$BODY$
-LANGUAGE 'plpgsql';
--- +goose StatementEnd
-
--- +goose StatementBegin
-CREATE OR REPLACE FUNCTION canonical_header_hash(height BIGINT) RETURNS character varying AS
-$BODY$
-DECLARE
-  canonical_header eth.header_cids;
-  headers eth.header_cids[];
-  header_count INT;
-  temp_header eth.header_cids;
-BEGIN
-  -- collect all headers at this height
-  FOR temp_header IN
-  SELECT * FROM eth.header_cids WHERE block_number = height
-  LOOP
-    headers = array_append(headers, temp_header);
-  END LOOP;
-  -- count the number of headers collected
-  header_count = array_length(headers, 1);
-  -- if we have less than 1 header, return NULL
-  IF header_count IS NULL OR header_count < 1 THEN
-    RETURN NULL;
-  -- if we have one header, return its hash
-  ELSIF header_count = 1 THEN
-    RETURN headers[1].block_hash;
-  -- if we have multiple headers we need to determine which one is canonical
-  ELSE
-    canonical_header = canonical_header_from_array(headers);
-    RETURN canonical_header.block_hash;
-  END IF;
-END
-$BODY$
-LANGUAGE 'plpgsql';
--- +goose StatementEnd
-
--- +goose StatementBegin
-CREATE TYPE state_node_result AS (
-    data                  BYTEA,
-    state_leaf_key        VARCHAR(66),
-    cid                   TEXT,
-    state_path            BYTEA,
-    node_type             INTEGER,
-    mh_key                TEXT
-);
--- +goose StatementEnd
-
--- +goose StatementBegin
-CREATE OR REPLACE FUNCTION state_snapshot(starting_height BIGINT, ending_height BIGINT) RETURNS void AS
-$BODY$
-DECLARE
-    canonical_hash VARCHAR(66);
-    results state_node_result[];
-BEGIN
-    -- get the canonical hash for the header at ending_height
-    canonical_hash = canonical_header_hash(ending_height);
-    IF canonical_hash IS NULL THEN
-        RAISE EXCEPTION 'cannot create state snapshot, no header can be found at height %', ending_height;
-    END IF;
-
-    -- select all of the state nodes for this snapshot: the latest state node record at every unique path
-    SELECT DISTINCT ON (state_path) blocks.data, state_cids.state_leaf_key, state_cids.cid, state_cids.state_path,
-        state_cids.node_type, state_cids.mh_key
-    INTO results
-    FROM eth.state_cids
-        INNER JOIN public.blocks
-            ON (state_cids.mh_key, state_cids.block_number) = (blocks.key, blocks.block_number)
-    WHERE state_cids.block_number BETWEEN starting_height AND ending_height
-    ORDER BY state_path, block_number DESC;
-
-    -- from the set returned above, insert public.block records at the ending_height block number
-    INSERT INTO public.blocks (block_number, key, data)
-    SELECT ending_height, r.mh_key, r.data
-    FROM results r;
-
-    -- from the set returned above, insert eth.state_cids records at the ending_height block number
-    -- anchoring all the records to the canonical header found at ending_height
-    INSERT INTO eth.state_cids (block_number, header_id, state_leaf_key, cid, state_path, node_type, diff, mh_key)
-    SELECT ending_height, canonical_hash, r.state_leaf_key, r.cid, r.state_path, r.node_type, false, r.mh_key
-    FROM results r
-    ON CONFLICT (state_path, header_id) DO NOTHING;
-END
-$BODY$
-LANGUAGE 'plpgsql';
--- +goose StatementEnd
-
--- +goose StatementBegin
-CREATE TYPE storage_node_result AS (
-    data                  BYTEA,
-    state_path            BYTEA,
-    storage_leaf_key      VARCHAR(66),
-    cid                   TEXT,
-    storage_path          BYTEA,
-    node_type             INTEGER,
-    mh_key                TEXT
-);
--- +goose StatementEnd
-
--- +goose StatementBegin
--- this should only be ran after a state_snapshot has been completed
--- this should probably be rolled together with state_snapshot into a single procedure...
-CREATE OR REPLACE FUNCTION storage_snapshot(starting_height BIGINT, ending_height BIGINT) RETURNS void AS
-$BODY$
-DECLARE
-    canonical_hash VARCHAR(66);
-    results storage_node_result[];
-BEGIN
-    -- get the canonical hash for the header at ending_height
-    SELECT canonical_header_hash(ending_height) INTO canonical_hash;
-    IF canonical_hash IS NULL THEN
-        RAISE EXCEPTION 'cannot create state snapshot, no header can be found at height %', ending_height;
-    END IF;
-
-    -- select all of the storage nodes for this snapshot: the latest storage node record at every unique state leaf key
-    SELECT DISTINCT ON (state_leaf_key, storage_path) block.data, storage_cids.state_path, storage_cids.storage_leaf_key,
-     storage_cids.cid, storage_cids.storage_path, storage_cids.node_type, storage_cids.mh_key
-    INTO results
-    FROM eth.storage_cids
-        INNER JOIN public.blocks
-        ON (storage_cids.mh_key, storage_cids.block_number) = (blocks.key, blocks.block_number)
-        INNER JOIN eth.state_cids
-        ON (storage_cids.state_path, storage_cids.header_id) = (state_cids.state_path, state_cids.header_id)
-    WHERE storage_cids.block_number BETWEEN starting_height AND ending_height
-    ORDER BY state_path, storage_path, block_number DESC;
-
-    -- from the set returned above, insert public.block records at the ending_height block number
-    INSERT INTO public.blocks (block_number, key, data)
-    SELECT ending_height, r.mh_key, r.data
-    FROM results r;
-
-    -- from the set returned above, insert eth.state_cids records at the ending_height block number
-    -- anchoring all the records to the canonical header found at ending_height
-    INSERT INTO eth.storage_cids (block_number, header_id, state_path, storage_leaf_key, cid, storage_path,
-                              node_type, diff, mh_key)
-    SELECT ending_height, canonical_hash, r.state_path, r.storage_leaf_key, r.cid, r.storage_path, r.node_type, false, r.mh_key
-    FROM results r
-    ON CONFLICT (storage_path, state_path, header_id) DO NOTHING;
-END
-$BODY$
-LANGUAGE 'plpgsql';
--- +goose StatementEnd
-
--- +goose Down
-DROP FUNCTION storage_snapshot;
-DROP TYPE storage_node_result;
-DROP FUNCTION state_snapshot;
-DROP TYPE state_node_result;
-DROP FUNCTION was_state_leaf_removed;
-DROP FUNCTION canonical_header_hash;
-DROP FUNCTION canonical_header_from_array;
-DROP FUNCTION has_child;
-DROP TYPE child_result;

db/migrations/00025_set_replication_factor.sql

@@ -1,27 +0,0 @@
--- +goose Up
--- to be generated by a script
-
-SELECT set_replication_factor('public.blocks', 3);
-SELECT set_replication_factor('eth.header_cids', 3);
-SELECT set_replication_factor('eth.uncle_cids', 3);
-SELECT set_replication_factor('eth.transaction_cids', 3);
-SELECT set_replication_factor('eth.receipt_cids', 3);
-SELECT set_replication_factor('eth.state_cids', 3);
-SELECT set_replication_factor('eth.storage_cids', 3);
-SELECT set_replication_factor('eth.state_accounts', 3);
-SELECT set_replication_factor('eth.access_list_elements', 3);
-SELECT set_replication_factor('eth.log_cids', 3);
-
--- +goose Down
--- to be generated by a script
-
-SELECT set_replication_factor('public.blocks', 1);
-SELECT set_replication_factor('eth.header_cids', 1);
-SELECT set_replication_factor('eth.uncle_cids', 1);
-SELECT set_replication_factor('eth.transaction_cids', 1);
-SELECT set_replication_factor('eth.receipt_cids', 1);
-SELECT set_replication_factor('eth.state_cids', 1);
-SELECT set_replication_factor('eth.storage_cids', 1);
-SELECT set_replication_factor('eth.state_accounts', 1);
-SELECT set_replication_factor('eth.access_list_elements', 1);
-SELECT set_replication_factor('eth.log_cids', 1);

db/post_batch_processing_migrations/00016_create_ipld_pk_constraint.sql

@@ -0,0 +1,7 @@
+-- +goose Up
+ALTER TABLE public.blocks
+ADD CONSTRAINT pk_public_blocks PRIMARY KEY (key);
+
+-- +goose Down
+ALTER TABLE public.blocks
+DROP CONSTRAINT pk_public_blocks;

db/post_batch_processing_migrations/00017_update_db_version.sql

@@ -0,0 +1,8 @@
+-- +goose Up
+INSERT INTO public.db_version (singleton, version) VALUES (true, 'v3.0.0')
+ON CONFLICT (singleton) DO UPDATE SET (version, tstamp) = ('v3.0.0', NOW());
+
+-- +goose Down
+DELETE FROM public.db_version WHERE version = 'v3.0.0';
+INSERT INTO public.db_version (singleton, version) VALUES (true, 'v0.3.2')
+ON CONFLICT (singleton) DO UPDATE SET (version, tstamp) = ('v0.3.2', NOW());

db/post_batch_processing_migrations/00018_create_log_indexes.sql

@@ -0,0 +1,15 @@
+-- +goose Up
+CREATE INDEX log_mh_index ON eth.log_cids USING btree (leaf_mh_key);
+CREATE INDEX block_number_index ON eth.header_cids USING brin (block_number);
+CREATE INDEX header_block_hash_index ON eth.header_cids USING btree (block_hash);
+CREATE INDEX tx_header_id_index ON eth.transaction_cids USING btree (header_id);
+CREATE INDEX tx_tx_hash_index ON eth.transaction_cids USING btree (tx_hash);
+CREATE INDEX log_cids_index ON eth.log_cids USING btree (rct_id, index);
+
+-- +goose Down
+DROP INDEX eth.log_cids_index;
+DROP INDEX eth.tx_tx_hash_index;
+DROP INDEX eth.tx_header_id_index;
+DROP INDEX eth.header_block_hash_index;
+DROP INDEX eth.block_number_index;
+DROP INDEX eth.log_mh_index;

db/post_batch_processing_migrations/00019_create_log_mhk_fk.sql

@@ -0,0 +1,9 @@
+-- +goose Up
+ALTER TABLE eth.log_cids
+ADD CONSTRAINT fk_log_leaf_mh_key
+    FOREIGN KEY (leaf_mh_key) REFERENCES public.blocks (key)
+    ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
+
+-- +goose Down
+ALTER TABLE eth.log_cids
+DROP CONSTRAINT fk_log_leaf_mh_key;

db/post_batch_processing_migrations/00020_disable_log_indexes_and_drop_mhk_fk.sql

@@ -0,0 +1,53 @@
+-- +goose Up
+UPDATE pg_index
+SET indisready=false
+WHERE indrelid = (
+    SELECT oid
+    FROM pg_class
+    WHERE relname='eth.log_cids'
+);
+UPDATE pg_index
+SET indisready=false
+WHERE indrelid = (
+    SELECT oid
+    FROM pg_class
+    WHERE relname='eth.transaction_cids'
+);
+UPDATE pg_index
+SET indisready=false
+WHERE indrelid = (
+    SELECT oid
+    FROM pg_class
+    WHERE relname='eth.header_cids'
+);
+
+ALTER TABLE eth.log_cids
+DROP CONSTRAINT fk_log_leaf_mh_key;
+
+-- +goose Down
+ALTER TABLE eth.log_cids
+ADD CONSTRAINT fk_log_leaf_mh_key
+    FOREIGN KEY (leaf_mh_key) REFERENCES public.blocks (key)
+    ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
+
+UPDATE pg_index
+SET indisready=true
+WHERE indrelid = (
+    SELECT oid
+    FROM pg_class
+    WHERE relname='eth.header_cids'
+);
+UPDATE pg_index
+SET indisready=true
+WHERE indrelid = (
+    SELECT oid
+    FROM pg_class
+    WHERE relname='eth.transaction_cids'
+);
+UPDATE pg_index
+SET indisready=true
+WHERE indrelid = (
+    SELECT oid
+    FROM pg_class
+    WHERE relname='eth.log_cids'
+);

db/post_batch_processing_migrations/00021_reenable_and_drop_log_indexes.sql

@@ -0,0 +1,53 @@
+-- +goose Up
+UPDATE pg_index
+SET indisready=true
+WHERE indrelid = (
+    SELECT oid
+    FROM pg_class
+    WHERE relname='eth.header_cids'
+);
+UPDATE pg_index
+SET indisready=true
+WHERE indrelid = (
+    SELECT oid
+    FROM pg_class
+    WHERE relname='eth.transaction_cids'
+);
+UPDATE pg_index
+SET indisready=true
+WHERE indrelid = (
+    SELECT oid
+    FROM pg_class
+    WHERE relname='eth.log_cids'
+);
+
+DROP INDEX eth.tx_header_id_index;
+DROP INDEX eth.block_number_index;
+DROP INDEX eth.log_mh_index;
+
+-- +goose Down
+CREATE INDEX log_mh_index ON eth.log_cids USING btree (leaf_mh_key);
+CREATE INDEX block_number_index ON eth.header_cids USING brin (block_number);
+CREATE INDEX tx_header_id_index ON eth.transaction_cids USING btree (header_id);
+
+UPDATE pg_index
+SET indisready=false
+WHERE indrelid = (
+    SELECT oid
+    FROM pg_class
+    WHERE relname='eth.log_cids'
+);
+UPDATE pg_index
+SET indisready=false
+WHERE indrelid = (
+    SELECT oid
+    FROM pg_class
+    WHERE relname='eth.transaction_cids'
+);
+UPDATE pg_index
+SET indisready=false
+WHERE indrelid = (
+    SELECT oid
+    FROM pg_class
+    WHERE relname='eth.header_cids'
+);

db/post_batch_processing_migrations/00022_drop_non_unique_indexes.sql

@@ -0,0 +1,9 @@
+-- +goose Up
+DROP INDEX eth.log_cids_index;
+DROP INDEX eth.tx_tx_hash_index;
+DROP INDEX eth.header_block_hash_index;
+
+-- +goose Down
+CREATE INDEX header_block_hash_index ON eth.header_cids USING btree (block_hash);
+CREATE INDEX tx_tx_hash_index ON eth.transaction_cids USING btree (tx_hash);
+CREATE INDEX log_cids_index ON eth.log_cids USING btree (rct_id, index);

db/post_batch_processing_migrations/00023_create_pk_constraints.sql

@@ -0,0 +1,61 @@
+-- +goose Up
+ALTER TABLE public.nodes
+ADD CONSTRAINT pk_public_nodes PRIMARY KEY (node_id);
+
+ALTER TABLE eth.header_cids
+ADD CONSTRAINT pk_eth_header_cids PRIMARY KEY (block_hash);
+
+ALTER TABLE eth.uncle_cids
+ADD CONSTRAINT pk_eth_uncle_cids PRIMARY KEY (block_hash);
+
+ALTER TABLE eth.transaction_cids
+ADD CONSTRAINT pk_eth_transaction_cids PRIMARY KEY (tx_hash);
+
+ALTER TABLE eth.receipt_cids
+ADD CONSTRAINT pk_eth_receipt_cids PRIMARY KEY (tx_id);
+
+ALTER TABLE eth.access_list_elements
+ADD CONSTRAINT pk_eth_access_list_elements PRIMARY KEY (tx_id, index);
+
+ALTER TABLE eth.log_cids
+ADD CONSTRAINT pk_eth_log_cids PRIMARY KEY (rct_id, index);
+
+ALTER TABLE eth.state_cids
+ADD CONSTRAINT pk_eth_state_cids PRIMARY KEY (header_id, state_path);
+
+ALTER TABLE eth.storage_cids
+ADD CONSTRAINT pk_eth_storage_cids PRIMARY KEY (header_id, state_path, storage_path);
+
+ALTER TABLE eth.state_accounts
+ADD CONSTRAINT pk_eth_state_accounts PRIMARY KEY (header_id, state_path);
+
+-- +goose Down
+ALTER TABLE eth.state_accounts
+DROP CONSTRAINT pk_eth_state_accounts;
+
+ALTER TABLE eth.storage_cids
+DROP CONSTRAINT pk_eth_storage_cids;
+
+ALTER TABLE eth.state_cids
+DROP CONSTRAINT pk_eth_state_cids;
+
+ALTER TABLE eth.log_cids
+DROP CONSTRAINT pk_eth_log_cids;
+
+ALTER TABLE eth.access_list_elements
+DROP CONSTRAINT pk_eth_access_list_elements;
+
+ALTER TABLE eth.receipt_cids
+DROP CONSTRAINT pk_eth_receipt_cids;
+
+ALTER TABLE eth.transaction_cids
+DROP CONSTRAINT pk_eth_transaction_cids;
+
+ALTER TABLE eth.uncle_cids
+DROP CONSTRAINT pk_eth_uncle_cids;
+
+ALTER TABLE eth.header_cids
+DROP CONSTRAINT pk_eth_header_cids;
+
+ALTER TABLE public.nodes
+DROP CONSTRAINT pk_public_nodes;

db/post_batch_processing_migrations/00024_create_eth_meta_schema.sql

@@ -0,0 +1,5 @@
+-- +goose Up
+CREATE SCHEMA eth_meta;
+
+-- +goose Down
+DROP SCHEMA eth_meta;

db/post_batch_processing_migrations/00025_create_watched_addresses_table.sql

@@ -0,0 +1,10 @@
+-- +goose Up
+CREATE TABLE eth_meta.watched_addresses (
+    address                 VARCHAR(66) PRIMARY KEY,
+    created_at              BIGINT NOT NULL,
+    watched_at              BIGINT NOT NULL,
+    last_filled_at          BIGINT NOT NULL DEFAULT 0
+);
+
+-- +goose Down
+DROP TABLE eth_meta.watched_addresses;

db/post_batch_processing_migrations/00026_create_known_gaps_table.sql

@@ -0,0 +1,10 @@
+-- +goose Up
+CREATE TABLE eth_meta.known_gaps (
+     starting_block_number bigint PRIMARY KEY,
+     ending_block_number bigint,
+     checked_out boolean,
+     processing_key bigint
+);
+
+-- +goose Down
+DROP TABLE eth_meta.known_gaps;

db/post_batch_processing_migrations/00027_make_tables_logged.sql

@@ -0,0 +1,25 @@
+-- +goose Up
+ALTER TABLE public.blocks SET LOGGED;
+ALTER TABLE public.nodes SET LOGGED;
+ALTER TABLE eth.header_cids SET LOGGED;
+ALTER TABLE eth.uncle_cids SET LOGGED;
+ALTER TABLE eth.transaction_cids SET LOGGED;
+ALTER TABLE eth.receipt_cids SET LOGGED;
+ALTER TABLE eth.state_cids SET LOGGED;
+ALTER TABLE eth.storage_cids SET LOGGED;
+ALTER TABLE eth.state_accounts SET LOGGED;
+ALTER TABLE eth.access_list_elements SET LOGGED;
+ALTER TABLE eth.log_cids SET LOGGED;
+
+-- +goose Down
+ALTER TABLE public.blocks SET UNLOGGED;
+ALTER TABLE public.nodes SET UNLOGGED;
+ALTER TABLE eth.header_cids SET UNLOGGED;
+ALTER TABLE eth.uncle_cids SET UNLOGGED;
+ALTER TABLE eth.transaction_cids SET UNLOGGED;
+ALTER TABLE eth.receipt_cids SET UNLOGGED;
+ALTER TABLE eth.state_cids SET UNLOGGED;
+ALTER TABLE eth.storage_cids SET UNLOGGED;
+ALTER TABLE eth.state_accounts SET UNLOGGED;
+ALTER TABLE eth.access_list_elements SET UNLOGGED;
+ALTER TABLE eth.log_cids SET UNLOGGED;

db/post_batch_processing_migrations/00028_create_fk_relations.sql

@@ -0,0 +1,129 @@
+-- +goose Up
+ALTER TABLE eth.header_cids
+ADD CONSTRAINT fk_header_mh_key
+    FOREIGN KEY (mh_key) REFERENCES public.blocks (key)
+    ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
+
+ALTER TABLE eth.header_cids
+ADD CONSTRAINT fk_header_node_id
+    FOREIGN KEY (node_id) REFERENCES public.nodes (node_id)
+    ON DELETE CASCADE;
+
+ALTER TABLE eth.uncle_cids
+ADD CONSTRAINT fk_uncle_mh_key
+    FOREIGN KEY (mh_key) REFERENCES public.blocks (key)
+    ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
+
+ALTER TABLE eth.uncle_cids
+ADD CONSTRAINT fk_uncle_header_id
+    FOREIGN KEY (header_id) REFERENCES eth.header_cids (block_hash)
+    ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
+
+ALTER TABLE eth.transaction_cids
+ADD CONSTRAINT fk_tx_mh_key
+    FOREIGN KEY (mh_key) REFERENCES public.blocks (key)
+    ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
+
+ALTER TABLE eth.transaction_cids
+ADD CONSTRAINT fk_tx_header_id
+    FOREIGN KEY (header_id) REFERENCES eth.header_cids (block_hash)
+    ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
+
+ALTER TABLE eth.receipt_cids
+ADD CONSTRAINT fk_rct_leaf_mh_key
+    FOREIGN KEY (leaf_mh_key) REFERENCES public.blocks (key)
+    ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
+
+ALTER TABLE eth.receipt_cids
+ADD CONSTRAINT fk_rct_tx_id
+    FOREIGN KEY (tx_id) REFERENCES eth.transaction_cids (tx_hash)
+    ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
+
+ALTER TABLE eth.state_cids
+ADD CONSTRAINT fk_state_mh_key
+    FOREIGN KEY (mh_key) REFERENCES public.blocks (key)
+    ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
+
+ALTER TABLE eth.state_cids
+ADD CONSTRAINT fk_state_header_id
+    FOREIGN KEY (header_id) REFERENCES eth.header_cids (block_hash)
+    ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
+
+ALTER TABLE eth.storage_cids
+ADD CONSTRAINT fk_storage_mh_key
+    FOREIGN KEY (mh_key) REFERENCES public.blocks (key)
+    ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
+
+ALTER TABLE eth.storage_cids
+ADD CONSTRAINT fk_storage_header_id_state_path
+    FOREIGN KEY (header_id, state_path) REFERENCES eth.state_cids (header_id, state_path)
+    ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
+
+ALTER TABLE eth.state_accounts
+ADD CONSTRAINT fk_account_header_id_state_path
+    FOREIGN KEY (header_id, state_path) REFERENCES eth.state_cids (header_id, state_path)
+    ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
+
+ALTER TABLE eth.access_list_elements
+ADD CONSTRAINT fk_access_list_tx_id
+    FOREIGN KEY (tx_id) REFERENCES eth.transaction_cids (tx_hash)
+    ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
+
+ALTER TABLE eth.log_cids
+ADD CONSTRAINT fk_log_leaf_mh_key
+    FOREIGN KEY (leaf_mh_key) REFERENCES public.blocks (key)
+    ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
+
+ALTER TABLE eth.log_cids
+ADD CONSTRAINT fk_log_rct_id
+    FOREIGN KEY (rct_id) REFERENCES eth.receipt_cids (tx_id)
+    ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
+
+-- +goose Down
+ALTER TABLE eth.header_cids
+DROP CONSTRAINT fk_header_mh_key;
+
+ALTER TABLE eth.header_cids
+DROP CONSTRAINT fk_header_node_id;
+
+ALTER TABLE eth.uncle_cids
+DROP CONSTRAINT fk_uncle_mh_key;
+
+ALTER TABLE eth.uncle_cids
+DROP CONSTRAINT fk_uncle_header_id;
+
+ALTER TABLE eth.transaction_cids
+DROP CONSTRAINT fk_tx_mh_key;
+
+ALTER TABLE eth.transaction_cids
+DROP CONSTRAINT fk_tx_header_id;
+
+ALTER TABLE eth.receipt_cids
+DROP CONSTRAINT fk_rct_leaf_mh_key;
+
+ALTER TABLE eth.receipt_cids
+DROP CONSTRAINT fk_rct_tx_id;
+
+ALTER TABLE eth.state_cids
+DROP CONSTRAINT fk_state_mh_key;
+
+ALTER TABLE eth.state_cids
+DROP CONSTRAINT fk_state_header_id;
+
+ALTER TABLE eth.storage_cids
+DROP CONSTRAINT fk_storage_mh_key;
+
+ALTER TABLE eth.storage_cids
+DROP CONSTRAINT fk_storage_header_id_state_path;
+
+ALTER TABLE eth.state_accounts
+DROP CONSTRAINT fk_account_header_id_state_path;
+
+ALTER TABLE eth.access_list_elements
+DROP CONSTRAINT fk_access_list_tx_id;
+
+ALTER TABLE eth.log_cids
+DROP CONSTRAINT fk_log_leaf_mh_key;
+
+ALTER TABLE eth.log_cids
+DROP CONSTRAINT fk_log_rct_id;

db/post_batch_processing_migrations/00029_create_postgraphile_comments.sql

@@ -0,0 +1,14 @@
+-- +goose Up
+COMMENT ON TABLE public.nodes IS E'@name NodeInfo';
+COMMENT ON TABLE eth.transaction_cids IS E'@name EthTransactionCids';
+COMMENT ON TABLE eth.header_cids IS E'@name EthHeaderCids';
+COMMENT ON COLUMN public.nodes.node_id IS E'@name ChainNodeID';
+COMMENT ON COLUMN eth.header_cids.node_id IS E'@name EthNodeID';
+
+-- +goose Down
+
+COMMENT ON TABLE public.nodes IS NULL;
+COMMENT ON TABLE eth.transaction_cids IS NULL;
+COMMENT ON TABLE eth.header_cids IS NULL;
+COMMENT ON COLUMN public.nodes.node_id IS NULL;
+COMMENT ON COLUMN eth.header_cids.node_id IS NULL;

db/post_batch_processing_migrations/00030_create_cid_indexes.sql

@@ -0,0 +1,111 @@
+-- +goose Up
+-- header indexes
+CREATE INDEX block_number_index ON eth.header_cids USING brin (block_number);
+CREATE UNIQUE INDEX header_cid_index ON eth.header_cids USING btree (cid);
+CREATE UNIQUE INDEX header_mh_index ON eth.header_cids USING btree (mh_key);
+CREATE INDEX state_root_index ON eth.header_cids USING btree (state_root);
+CREATE INDEX timestamp_index ON eth.header_cids USING brin (timestamp);
+
+-- uncle indexes
+CREATE INDEX uncle_header_id_index ON eth.uncle_cids USING btree (header_id);
+
+-- transaction indexes
+CREATE INDEX tx_header_id_index ON eth.transaction_cids USING btree (header_id);
+CREATE UNIQUE INDEX tx_cid_index ON eth.transaction_cids USING btree (cid);
+CREATE UNIQUE INDEX tx_mh_index ON eth.transaction_cids USING btree (mh_key);
+CREATE INDEX tx_dst_index ON eth.transaction_cids USING btree (dst);
+CREATE INDEX tx_src_index ON eth.transaction_cids USING btree (src);
+
+-- receipt indexes
+CREATE INDEX rct_leaf_cid_index ON eth.receipt_cids USING btree (leaf_cid);
+CREATE INDEX rct_leaf_mh_index ON eth.receipt_cids USING btree (leaf_mh_key);
+CREATE INDEX rct_contract_index ON eth.receipt_cids USING btree (contract);
+CREATE INDEX rct_contract_hash_index ON eth.receipt_cids USING btree (contract_hash);
+
+-- state node indexes
+CREATE INDEX state_leaf_key_index ON eth.state_cids USING btree (state_leaf_key);
+CREATE INDEX state_cid_index ON eth.state_cids USING btree (cid);
+CREATE INDEX state_mh_index ON eth.state_cids USING btree (mh_key);
+CREATE INDEX state_path_index ON eth.state_cids USING btree (state_path);
+CREATE INDEX state_node_type_index ON eth.state_cids USING btree (node_type);
+
+-- storage node indexes
+CREATE INDEX storage_state_path_index ON eth.storage_cids USING btree (state_path);
+CREATE INDEX storage_leaf_key_index ON eth.storage_cids USING btree (storage_leaf_key);
+CREATE INDEX storage_cid_index ON eth.storage_cids USING btree (cid);
+CREATE INDEX storage_mh_index ON eth.storage_cids USING btree (mh_key);
+CREATE INDEX storage_path_index ON eth.storage_cids USING btree (storage_path);
+CREATE INDEX storage_node_type_index ON eth.storage_cids USING btree (node_type);
+
+-- state accounts indexes
+CREATE INDEX account_state_path_index ON eth.state_accounts USING btree (state_path);
+CREATE INDEX storage_root_index ON eth.state_accounts USING btree (storage_root);
+
+-- access list indexes
+CREATE INDEX access_list_element_address_index ON eth.access_list_elements USING btree (address);
+CREATE INDEX access_list_storage_keys_index ON eth.access_list_elements USING gin (storage_keys);
+
+-- log indexes
+CREATE INDEX log_cid_index ON  eth.log_cids USING btree (leaf_cid);
+CREATE INDEX log_mh_index ON eth.log_cids USING btree (leaf_mh_key);
+CREATE INDEX log_address_index ON eth.log_cids USING btree (address);
+CREATE INDEX log_topic0_index ON eth.log_cids USING btree (topic0);
+CREATE INDEX log_topic1_index ON eth.log_cids USING btree (topic1);
+CREATE INDEX log_topic2_index ON eth.log_cids USING btree (topic2);
+CREATE INDEX log_topic3_index ON eth.log_cids USING btree (topic3);
+
+-- +goose Down
+-- log indexes
+DROP INDEX eth.log_topic3_index;
+DROP INDEX eth.log_topic2_index;
+DROP INDEX eth.log_topic1_index;
+DROP INDEX eth.log_topic0_index;
+DROP INDEX eth.log_address_index;
+DROP INDEX eth.log_mh_index;
+DROP INDEX eth.log_cid_index;
+
+-- access list indexes
+DROP INDEX eth.access_list_storage_keys_index;
+DROP INDEX eth.access_list_element_address_index;
+
+-- state account indexes
+DROP INDEX eth.storage_root_index;
+DROP index eth.account_state_path_index;
+
+-- storage node indexes
+DROP INDEX eth.storage_node_type_index;
+DROP INDEX eth.storage_path_index;
+DROP INDEX eth.storage_mh_index;
+DROP INDEX eth.storage_cid_index;
+DROP INDEX eth.storage_leaf_key_index;
+DROP INDEX eth.storage_state_path_index;
+
+-- state node indexes
+DROP INDEX eth.state_node_type_index;
+DROP INDEX eth.state_path_index;
+DROP INDEX eth.state_mh_index;
+DROP INDEX eth.state_cid_index;
+DROP INDEX eth.state_leaf_key_index;
+
+-- receipt indexes
+DROP INDEX eth.rct_contract_hash_index;
+DROP INDEX eth.rct_contract_index;
+DROP INDEX eth.rct_leaf_mh_index;
+DROP INDEX eth.rct_leaf_cid_index;
+
+-- transaction indexes
+DROP INDEX eth.tx_src_index;
+DROP INDEX eth.tx_dst_index;
+DROP INDEX eth.tx_mh_index;
+DROP INDEX eth.tx_cid_index;
+DROP INDEX eth.tx_header_id_index;
+
+-- uncle indexes
+DROP INDEX eth.uncle_header_id_index;
+
+-- header indexes
+DROP INDEX eth.timestamp_index;
+DROP INDEX eth.state_root_index;
+DROP INDEX eth.header_mh_index;
+DROP INDEX eth.header_cid_index;
+DROP INDEX eth.block_number_index;

db/post_batch_processing_migrations/00031_create_stored_functions.sql

@@ -0,0 +1,139 @@
+-- +goose Up
+-- +goose StatementBegin
+-- returns if a state leaf node was removed within the provided block number
+CREATE OR REPLACE FUNCTION was_state_leaf_removed(key character varying, hash character varying)
+    RETURNS boolean AS $$
+    SELECT state_cids.node_type = 3
+    FROM eth.state_cids
+             INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.block_hash)
+    WHERE state_leaf_key = key
+      AND block_number <= (SELECT block_number
+                           FROM eth.header_cids
+                           WHERE block_hash = hash)
+    ORDER BY block_number DESC LIMIT 1;
+$$
+language sql;
+-- +goose StatementEnd
+
+-- +goose StatementBegin
+CREATE TYPE child_result AS (
+    has_child BOOLEAN,
+    children eth.header_cids[]
+);
+
+
+CREATE OR REPLACE FUNCTION has_child(hash VARCHAR(66), height BIGINT) RETURNS child_result AS
+$BODY$
+DECLARE
+  child_height INT;
+  temp_child eth.header_cids;
+  new_child_result child_result;
+BEGIN
+  child_height = height + 1;
+  -- short circuit if there are no children
+  SELECT exists(SELECT 1
+              FROM eth.header_cids
+              WHERE parent_hash = hash
+                AND block_number = child_height
+              LIMIT 1)
+  INTO new_child_result.has_child;
+  -- collect all the children for this header
+  IF new_child_result.has_child THEN
+    FOR temp_child IN
+    SELECT * FROM eth.header_cids WHERE parent_hash = hash AND block_number = child_height
+    LOOP
+      new_child_result.children = array_append(new_child_result.children, temp_child);
+    END LOOP;
+  END IF;
+  RETURN new_child_result;
+END
+$BODY$
+LANGUAGE 'plpgsql';
+-- +goose StatementEnd
+
+-- +goose StatementBegin
+CREATE OR REPLACE FUNCTION canonical_header_from_array(headers eth.header_cids[]) RETURNS eth.header_cids AS
+$BODY$
+DECLARE
+  canonical_header eth.header_cids;
+  canonical_child eth.header_cids;
+  header eth.header_cids;
+  current_child_result child_result;
+  child_headers eth.header_cids[];
+  current_header_with_child eth.header_cids;
+  has_children_count INT DEFAULT 0;
+BEGIN
+  -- for each header in the provided set
+  FOREACH header IN ARRAY headers
+  LOOP
+    -- check if it has any children
+    current_child_result = has_child(header.block_hash, header.block_number);
+    IF current_child_result.has_child THEN
+      -- if it does, take note
+      has_children_count = has_children_count + 1;
+      current_header_with_child = header;
+      -- and add the children to the growing set of child headers
+      child_headers = array_cat(child_headers, current_child_result.children);
+    END IF;
+  END LOOP;
+  -- if none of the headers had children, none is more canonical than the other
+  IF has_children_count = 0 THEN
+    -- return the first one selected
+    SELECT * INTO canonical_header FROM unnest(headers) LIMIT 1;
+  -- if only one header had children, it can be considered the heaviest/canonical header of the set
+  ELSIF has_children_count = 1 THEN
+    -- return the only header with a child
+    canonical_header = current_header_with_child;
+  -- if there are multiple headers with children
+  ELSE
+    -- find the canonical header from the child set
+    canonical_child = canonical_header_from_array(child_headers);
+    -- the header that is parent to this header, is the canonical header at this level
+    SELECT * INTO canonical_header FROM unnest(headers)
+    WHERE block_hash = canonical_child.parent_hash;
+  END IF;
+  RETURN canonical_header;
+END
+$BODY$
+LANGUAGE 'plpgsql';
+-- +goose StatementEnd
+
+-- +goose StatementBegin
+CREATE OR REPLACE FUNCTION canonical_header_hash(height BIGINT) RETURNS character varying AS
+$BODY$
+DECLARE
+  canonical_header eth.header_cids;
+  headers eth.header_cids[];
+  header_count INT;
+  temp_header eth.header_cids;
+BEGIN
+  -- collect all headers at this height
+  FOR temp_header IN
+  SELECT * FROM eth.header_cids WHERE block_number = height
+  LOOP
+    headers = array_append(headers, temp_header);
+  END LOOP;
+  -- count the number of headers collected
+  header_count = array_length(headers, 1);
+  -- if we have less than 1 header, return NULL
+  IF header_count IS NULL OR header_count < 1 THEN
+    RETURN NULL;
+  -- if we have one header, return its hash
+  ELSIF header_count = 1 THEN
+    RETURN headers[1].block_hash;
+  -- if we have multiple headers we need to determine which one is canonical
+  ELSE
+    canonical_header = canonical_header_from_array(headers);
+    RETURN canonical_header.block_hash;
+  END IF;
+END;
+$BODY$
+LANGUAGE 'plpgsql';
+-- +goose StatementEnd
+
+-- +goose Down
+DROP FUNCTION was_state_leaf_removed;
+DROP FUNCTION canonical_header_hash;
+DROP FUNCTION canonical_header_from_array;
+DROP FUNCTION has_child;
+DROP TYPE child_result;

db/post_batch_processing_migrations/00032_create_postgraphile_triggers.sql

@@ -0,0 +1,116 @@
+-- +goose Up
+-- Name: graphql_subscription(); Type: FUNCTION; Schema: eth; Owner: -
+
+-- +goose StatementBegin
+CREATE FUNCTION eth.graphql_subscription() RETURNS TRIGGER AS $$
+DECLARE
+obj jsonb;
+BEGIN
+    IF (TG_TABLE_NAME = 'state_cids') OR (TG_TABLE_NAME = 'state_accounts') THEN
+             obj := json_build_array(
+                        TG_TABLE_NAME,
+                        NEW.header_id,
+                        NEW.state_path
+                    );
+    ELSIF (TG_TABLE_NAME = 'storage_cids') THEN
+         obj := json_build_array(
+                    TG_TABLE_NAME,
+                    NEW.header_id,
+                    NEW.state_path,
+                    NEW.storage_path
+                );
+    ELSIF (TG_TABLE_NAME = 'log_cids')w THEN
+         obj := json_build_array(
+                    TG_TABLE_NAME,
+                    NEW.rct_id,
+                    NEW.index
+                );
+    ELSIF (TG_TABLE_NAME = 'receipt_cids') THEN
+         obj := json_build_array(
+                    TG_TABLE_NAME,
+                    NEW.tx_id
+                );
+    ELSIF (TG_TABLE_NAME = 'transaction_cids') THEN
+         obj := json_build_array(
+                    TG_TABLE_NAME,
+                    NEW.tx_hash
+                );
+    ELSIF (TG_TABLE_NAME = 'access_list_elements') THEN
+         obj := json_build_array(
+                    TG_TABLE_NAME,
+                    NEW.tx_id,
+                    NEW.index
+                );
+    ELSIF (TG_TABLE_NAME = 'uncle_cids') OR (TG_TABLE_NAME = 'header_cids') THEN
+         obj := json_build_array(
+                    TG_TABLE_NAME,
+                    NEW.block_hash
+                );
+END IF;
+
+    perform pg_notify('postgraphile:' || TG_RELNAME , json_build_object(
+            '__node__', obj
+            )::text
+        );
+RETURN NEW;
+END;
+$$ language plpgsql;
+-- +goose StatementEnd
+
+CREATE TRIGGER trg_eth_header_cids
+    AFTER INSERT ON eth.header_cids
+    FOR EACH ROW
+    EXECUTE PROCEDURE eth.graphql_subscription();
+
+CREATE TRIGGER trg_eth_uncle_cids
+    AFTER INSERT ON eth.uncle_cids
+    FOR EACH ROW
+    EXECUTE PROCEDURE eth.graphql_subscription();
+
+CREATE TRIGGER trg_eth_transaction_cids
+    AFTER INSERT ON eth.transaction_cids
+    FOR EACH ROW
+    EXECUTE PROCEDURE eth.graphql_subscription();
+
+CREATE TRIGGER trg_eth_receipt_cids
+    AFTER INSERT ON eth.receipt_cids
+    FOR EACH ROW
+    EXECUTE PROCEDURE eth.graphql_subscription();
+
+CREATE TRIGGER trg_eth_state_cids
+    AFTER INSERT ON eth.state_cids
+    FOR EACH ROW
+    EXECUTE PROCEDURE eth.graphql_subscription();
+
+CREATE TRIGGER trg_eth_log_cids
+    AFTER INSERT ON eth.log_cids
+    FOR EACH ROW
+    EXECUTE PROCEDURE eth.graphql_subscription();
+
+CREATE TRIGGER trg_eth_storage_cids
+    AFTER INSERT ON eth.storage_cids
+    FOR EACH ROW
+    EXECUTE PROCEDURE eth.graphql_subscription();
+
+CREATE TRIGGER trg_eth_state_accounts
+    AFTER INSERT ON eth.state_accounts
+    FOR EACH ROW
+    EXECUTE PROCEDURE eth.graphql_subscription();
+
+CREATE TRIGGER trg_eth_access_list_elements
+    AFTER INSERT ON eth.access_list_elements
+    FOR EACH ROW
+    EXECUTE PROCEDURE eth.graphql_subscription();
+
+-- +goose Down
+DROP TRIGGER trg_eth_uncle_cids ON eth.uncle_cids;
+DROP TRIGGER trg_eth_transaction_cids ON eth.transaction_cids;
+DROP TRIGGER trg_eth_storage_cids ON eth.storage_cids;
+DROP TRIGGER trg_eth_state_cids ON eth.state_cids;
+DROP TRIGGER trg_eth_state_accounts ON eth.state_accounts;
+DROP TRIGGER trg_eth_receipt_cids ON eth.receipt_cids;
+DROP TRIGGER trg_eth_header_cids ON eth.header_cids;
+DROP TRIGGER trg_eth_log_cids ON eth.log_cids;
+DROP TRIGGER trg_eth_access_list_elements ON eth.access_list_elements;
+
+DROP FUNCTION eth.graphql_subscription();

db/pre_batch_processing_migrations/00001_create_ipfs_blocks_table.sql

@@ -0,0 +1,8 @@
+-- +goose Up
+CREATE TABLE IF NOT EXISTS public.blocks (
+  key TEXT NOT NULL,
+  data BYTEA NOT NULL
+);
+
+-- +goose Down
+DROP TABLE public.blocks;

db/pre_batch_processing_migrations/00002_create_nodes_table.sql

@@ -0,0 +1,11 @@
+-- +goose Up
+CREATE TABLE nodes (
+    genesis_block VARCHAR(66),
+    network_id    VARCHAR,
+    node_id       VARCHAR(128) NOT NULL,
+    client_name   VARCHAR,
+    chain_id      INTEGER DEFAULT 1
+);
+
+-- +goose Down
+DROP TABLE nodes;

db/pre_batch_processing_migrations/00003_create_eth_schema.sql

@@ -0,0 +1,5 @@
+-- +goose Up
+CREATE SCHEMA eth;
+
+-- +goose Down
+DROP SCHEMA eth;

db/pre_batch_processing_migrations/00004_create_eth_header_cids_table.sql

@@ -0,0 +1,22 @@
+-- +goose Up
+CREATE TABLE eth.header_cids (
+     block_number          BIGINT NOT NULL,
+     block_hash            VARCHAR(66) NOT NULL,
+     parent_hash           VARCHAR(66) NOT NULL,
+     cid                   TEXT NOT NULL,
+     td                    NUMERIC NOT NULL,
+     node_id               VARCHAR(128) NOT NULL,
+     reward                NUMERIC NOT NULL,
+     state_root            VARCHAR(66) NOT NULL,
+     tx_root               VARCHAR(66) NOT NULL,
+     receipt_root          VARCHAR(66) NOT NULL,
+     uncle_root            VARCHAR(66) NOT NULL,
+     bloom                 BYTEA NOT NULL,
+     timestamp             BIGINT NOT NULL,
+     mh_key                TEXT NOT NULL,
+     times_validated       INTEGER NOT NULL DEFAULT 1,
+     coinbase              VARCHAR(66) NOT NULL
+);
+
+-- +goose Down
+DROP TABLE eth.header_cids;

db/pre_batch_processing_migrations/00005_create_eth_uncle_cids_table.sql

@@ -0,0 +1,12 @@
+-- +goose Up
+CREATE TABLE eth.uncle_cids (
+    block_hash            VARCHAR(66) NOT NULL,
+    header_id             VARCHAR(66) NOT NULL,
+    parent_hash           VARCHAR(66) NOT NULL,
+    cid                   TEXT NOT NULL,
+    reward                NUMERIC NOT NULL,
+    mh_key                TEXT NOT NULL
+);
+
+-- +goose Down
+DROP TABLE eth.uncle_cids;

db/pre_batch_processing_migrations/00006_create_eth_transaction_cids_table.sql

@@ -0,0 +1,16 @@
+-- +goose Up
+CREATE TABLE eth.transaction_cids (
+    header_id             VARCHAR(66) NOT NULL,
+    tx_hash               VARCHAR(66) NOT NULL,
+    cid                   TEXT NOT NULL,
+    dst                   VARCHAR(66) NOT NULL,
+    src                   VARCHAR(66) NOT NULL,
+    index                 INTEGER NOT NULL,
+    mh_key                TEXT NOT NULL,
+    tx_data               BYTEA,
+    tx_type               INTEGER,
+    value                 NUMERIC
+);
+
+-- +goose Down
+DROP TABLE eth.transaction_cids;

db/pre_batch_processing_migrations/00007_create_eth_receipt_cids_table.sql

@@ -0,0 +1,14 @@
+-- +goose Up
+CREATE TABLE eth.receipt_cids (
+    tx_id                 VARCHAR(66) NOT NULL,
+    leaf_cid              TEXT NOT NULL,
+    contract              VARCHAR(66),
+    contract_hash         VARCHAR(66),
+    leaf_mh_key           TEXT NOT NULL,
+    post_state            VARCHAR(66),
+    post_status           INTEGER,
+    log_root              VARCHAR(66)
+);
+
+-- +goose Down
+DROP TABLE eth.receipt_cids;

db/pre_batch_processing_migrations/00008_create_eth_state_cids_table.sql

@@ -0,0 +1,13 @@
+-- +goose Up
+CREATE TABLE eth.state_cids (
+    header_id             VARCHAR(66) NOT NULL,
+    state_leaf_key        VARCHAR(66),
+    cid                   TEXT NOT NULL,
+    state_path            BYTEA NOT NULL,
+    node_type             INTEGER NOT NULL,
+    diff                  BOOLEAN NOT NULL DEFAULT FALSE,
+    mh_key                TEXT NOT NULL
+);
+
+-- +goose Down
+DROP TABLE eth.state_cids;

db/pre_batch_processing_migrations/00009_create_eth_storage_cids_table.sql

@@ -0,0 +1,14 @@
+-- +goose Up
+CREATE TABLE eth.storage_cids (
+    header_id             VARCHAR(66) NOT NULL,
+    state_path            BYTEA NOT NULL,
+    storage_leaf_key      VARCHAR(66),
+    cid                   TEXT NOT NULL,
+    storage_path          BYTEA NOT NULL,
+    node_type             INTEGER NOT NULL,
+    diff                  BOOLEAN NOT NULL DEFAULT FALSE,
+    mh_key                TEXT NOT NULL
+);
+
+-- +goose Down
+DROP TABLE eth.storage_cids;

db/pre_batch_processing_migrations/00010_create_eth_state_accounts_table.sql

@@ -0,0 +1,12 @@
+-- +goose Up
+CREATE TABLE eth.state_accounts (
+    header_id             VARCHAR(66) NOT NULL,
+    state_path            BYTEA NOT NULL,
+    balance               NUMERIC NOT NULL,
+    nonce                 BIGINT NOT NULL,
+    code_hash             BYTEA NOT NULL,
+    storage_root          VARCHAR(66) NOT NULL
+);
+
+-- +goose Down
+DROP TABLE eth.state_accounts;

db/pre_batch_processing_migrations/00011_create_eth_access_list_elements_table.sql

@@ -0,0 +1,10 @@
+-- +goose Up
+CREATE TABLE eth.access_list_elements (
+    tx_id                 VARCHAR(66) NOT NULL,
+    index                 INTEGER NOT NULL,
+    address               VARCHAR(66),
+    storage_keys          VARCHAR(66)[]
+);
+
+-- +goose Down
+DROP TABLE eth.access_list_elements;

db/pre_batch_processing_migrations/00012_create_eth_log_cids_table.sql

@@ -0,0 +1,17 @@
+-- +goose Up
+CREATE TABLE eth.log_cids (
+    leaf_cid            TEXT NOT NULL,
+    leaf_mh_key         TEXT NOT NULL,
+    rct_id              VARCHAR(66) NOT NULL,
+    address             VARCHAR(66) NOT NULL,
+    index               INTEGER NOT NULL,
+    topic0              VARCHAR(66),
+    topic1              VARCHAR(66),
+    topic2              VARCHAR(66),
+    topic3              VARCHAR(66),
+    log_data            BYTEA
+);
+
+-- +goose Down
+-- log indexes
+DROP TABLE eth.log_cids;

db/pre_batch_processing_migrations/00013_make_tables_unlogged.sql

@@ -0,0 +1,25 @@
+-- +goose Up
+ALTER TABLE public.blocks SET UNLOGGED;
+ALTER TABLE public.nodes SET UNLOGGED;
+ALTER TABLE eth.header_cids SET UNLOGGED;
+ALTER TABLE eth.uncle_cids SET UNLOGGED;
+ALTER TABLE eth.transaction_cids SET UNLOGGED;
+ALTER TABLE eth.receipt_cids SET UNLOGGED;
+ALTER TABLE eth.state_cids SET UNLOGGED;
+ALTER TABLE eth.storage_cids SET UNLOGGED;
+ALTER TABLE eth.state_accounts SET UNLOGGED;
+ALTER TABLE eth.access_list_elements SET UNLOGGED;
+ALTER TABLE eth.log_cids SET UNLOGGED;
+
+-- +goose Down
+ALTER TABLE public.blocks SET LOGGED;
+ALTER TABLE public.nodes SET LOGGED;
+ALTER TABLE eth.header_cids SET LOGGED;
+ALTER TABLE eth.uncle_cids SET LOGGED;
+ALTER TABLE eth.transaction_cids SET LOGGED;
+ALTER TABLE eth.receipt_cids SET LOGGED;
+ALTER TABLE eth.state_cids SET LOGGED;
+ALTER TABLE eth.storage_cids SET LOGGED;
+ALTER TABLE eth.state_accounts SET LOGGED;
+ALTER TABLE eth.access_list_elements SET LOGGED;
+ALTER TABLE eth.log_cids SET LOGGED;

db/pre_batch_processing_migrations/00014_create_eth_db_version_table.sql

@@ -0,0 +1,9 @@
+-- +goose Up
+CREATE TABLE IF NOT EXISTS public.db_version (
+    singleton BOOLEAN NOT NULL DEFAULT TRUE UNIQUE CHECK (singleton),
+    version TEXT NOT NULL,
+    tstamp TIMESTAMP WITHOUT TIME ZONE DEFAULT NOW()
+);
+
+-- +goose Down
+DROP TABLE public.db_version;

db/pre_batch_processing_migrations/00015_update_db_version.sql

@@ -0,0 +1,6 @@
+-- +goose Up
+INSERT INTO public.db_version (singleton, version) VALUES (true, 'v0.3.2')
+ON CONFLICT (singleton) DO UPDATE SET (version, tstamp) = ('v0.3.2', NOW());
+
+-- +goose Down
+DELETE FROM public.db_version WHERE version = 'v0.3.2';

docker-compose.test.yml

@@ -1,62 +1,19 @@
 version: '3.2'
 
 services:
-  access-node:
+  statediff-migrations:
-    image: timescale/timescaledb:latest-pg14
+    restart: on-failure
-    restart: always
-    container_name: access-node
     depends_on:
-      - data-node-1
+      - test-db
-      - data-node-2
+    image: vulcanize/statediff-migrations:v0.9.0
-      - data-node-3
-    command: ["postgres", "-c", "log_statement=all"]
-    environment:
-      POSTGRES_USER: "postgres"
-      POSTGRES_DB: "vulcanize_testing_v4"
-      POSTGRES_PASSWORD: "password"
-    ports:
-      - "127.0.0.1:8066:5432"
-    volumes:
-      - ./scripts/init-access-node.sh:/docker-entrypoint-initdb.d/init-access-node.sh
 
-  data-node-1:
+  test-db:
-    image: timescale/timescaledb:latest-pg14
+    restart: always
-    restart: unless-stopped
+    image: postgres:10.12-alpine
-    container_name: data-node-1
     command: ["postgres", "-c", "log_statement=all"]
     environment:
-      POSTGRES_USER: "postgres"
+      POSTGRES_USER: "vdbm"
-      POSTGRES_DB: "vulcanize_testing_v4"
+      POSTGRES_DB: "vulcanize_testing"
       POSTGRES_PASSWORD: "password"
     ports:
-      - "127.0.0.1:8067:5432"
+      - "127.0.0.1:8066:5432"
-    volumes:
-      - ./scripts/init-data-node.sh:/docker-entrypoint-initdb.d/init-data-node.sh
-
-  data-node-2:
-    image: timescale/timescaledb:latest-pg14
-    restart: unless-stopped
-    container_name: data-node-2
-    command: ["postgres", "-c", "log_statement=all"]
-    environment:
-      POSTGRES_USER: "postgres"
-      POSTGRES_DB: "vulcanize_testing_v4"
-      POSTGRES_PASSWORD: "password"
-    ports:
-      - "127.0.0.1:8068:5432"
-    volumes:
-      - ./scripts/init-data-node.sh:/docker-entrypoint-initdb.d/init-data-node.sh
-
-  data-node-3:
-    image: timescale/timescaledb:latest-pg14
-    restart: unless-stopped
-    container_name: data-node-3
-    command: ["postgres", "-c", "log_statement=all"]
-    environment:
-      POSTGRES_USER: "postgres"
-      POSTGRES_DB: "vulcanize_testing_v4"
-      POSTGRES_PASSWORD: "password"
-    ports:
-      - "127.0.0.1:8069:5432"
-    volumes:
-      - ./scripts/init-data-node.sh:/docker-entrypoint-initdb.d/init-data-node.sh

schema.sql

@@ -2,8 +2,8 @@
 -- PostgreSQL database dump
 --
 
--- Dumped from database version 14.2
+-- Dumped from database version 14beta3
--- Dumped by pg_dump version 14.2
+-- Dumped by pg_dump version 14beta3
 
 SET statement_timeout = 0;
 SET lock_timeout = 0;
@@ -16,20 +16,6 @@ SET xmloption = content;
 SET client_min_messages = warning;
 SET row_security = off;
 
---
--- Name: timescaledb; Type: EXTENSION; Schema: -; Owner: -
---
-
-CREATE EXTENSION IF NOT EXISTS timescaledb WITH SCHEMA public;
-
-
---
--- Name: EXTENSION timescaledb; Type: COMMENT; Schema: -; Owner: -
---
-
-COMMENT ON EXTENSION timescaledb IS 'Enables scalable inserts and complex queries for time-series data';
-
-
 --
 -- Name: eth; Type: SCHEMA; Schema: -; Owner: -
 --
@@ -96,35 +82,6 @@ CREATE TYPE public.child_result AS (
 );
 
 
---
--- Name: state_node_result; Type: TYPE; Schema: public; Owner: -
---
-
-CREATE TYPE public.state_node_result AS (
-	data bytea,
-	state_leaf_key character varying(66),
-	cid text,
-	state_path bytea,
-	node_type integer,
-	mh_key text
-);
-
-
---
--- Name: storage_node_result; Type: TYPE; Schema: public; Owner: -
---
-
-CREATE TYPE public.storage_node_result AS (
-	data bytea,
-	state_path bytea,
-	storage_leaf_key character varying(66),
-	cid text,
-	storage_path bytea,
-	node_type integer,
-	mh_key text
-);
-
-
 --
 -- Name: graphql_subscription(); Type: FUNCTION; Schema: eth; Owner: -
 --
@@ -267,7 +224,7 @@ BEGIN
     canonical_header = canonical_header_from_array(headers);
     RETURN canonical_header.block_hash;
   END IF;
-END
+END;
 $$;
 
 
@@ -304,87 +261,6 @@ END
 $$;
 
 
---
--- Name: state_snapshot(bigint, bigint); Type: FUNCTION; Schema: public; Owner: -
---
-
-CREATE FUNCTION public.state_snapshot(starting_height bigint, ending_height bigint) RETURNS void
-    LANGUAGE plpgsql
-    AS $$
-DECLARE
-    canonical_hash VARCHAR(66);
-    results state_node_result[];
-BEGIN
-    -- get the canonical hash for the header at ending_height
-    canonical_hash = canonical_header_hash(ending_height);
-    IF canonical_hash IS NULL THEN
-        RAISE EXCEPTION 'cannot create state snapshot, no header can be found at height %', ending_height;
-    END IF;
-    -- select all of the state nodes for this snapshot: the latest state node record at every unique path
-    SELECT DISTINCT ON (state_path) blocks.data, state_cids.state_leaf_key, state_cids.cid, state_cids.state_path,
-        state_cids.node_type, state_cids.mh_key
-    INTO results
-    FROM eth.state_cids
-        INNER JOIN public.blocks
-            ON (state_cids.mh_key, state_cids.block_number) = (blocks.key, blocks.block_number)
-    WHERE state_cids.block_number BETWEEN starting_height AND ending_height
-    ORDER BY state_path, block_number DESC;
-    -- from the set returned above, insert public.block records at the ending_height block number
-    INSERT INTO public.blocks (block_number, key, data)
-    SELECT ending_height, r.mh_key, r.data
-    FROM results r;
-    -- from the set returned above, insert eth.state_cids records at the ending_height block number
-    -- anchoring all the records to the canonical header found at ending_height
-    INSERT INTO eth.state_cids (block_number, header_id, state_leaf_key, cid, state_path, node_type, diff, mh_key)
-    SELECT ending_height, canonical_hash, r.state_leaf_key, r.cid, r.state_path, r.node_type, false, r.mh_key
-    FROM results r
-    ON CONFLICT (state_path, header_id) DO NOTHING;
-END
-$$;
-
-
---
--- Name: storage_snapshot(bigint, bigint); Type: FUNCTION; Schema: public; Owner: -
---
-
-CREATE FUNCTION public.storage_snapshot(starting_height bigint, ending_height bigint) RETURNS void
-    LANGUAGE plpgsql
-    AS $$
-DECLARE
-    canonical_hash VARCHAR(66);
-    results storage_node_result[];
-BEGIN
-    -- get the canonical hash for the header at ending_height
-    SELECT canonical_header_hash(ending_height) INTO canonical_hash;
-    IF canonical_hash IS NULL THEN
-        RAISE EXCEPTION 'cannot create state snapshot, no header can be found at height %', ending_height;
-    END IF;
-    -- select all of the storage nodes for this snapshot: the latest storage node record at every unique state leaf key
-    SELECT DISTINCT ON (state_leaf_key, storage_path) block.data, storage_cids.state_path, storage_cids.storage_leaf_key,
-     storage_cids.cid, storage_cids.storage_path, storage_cids.node_type, storage_cids.mh_key
-    INTO results
-    FROM eth.storage_cids
-        INNER JOIN public.blocks
-        ON (storage_cids.mh_key, storage_cids.block_number) = (blocks.key, blocks.block_number)
-        INNER JOIN eth.state_cids
-        ON (storage_cids.state_path, storage_cids.header_id) = (state_cids.state_path, state_cids.header_id)
-    WHERE storage_cids.block_number BETWEEN starting_height AND ending_height
-    ORDER BY state_path, storage_path, block_number DESC;
-    -- from the set returned above, insert public.block records at the ending_height block number
-    INSERT INTO public.blocks (block_number, key, data)
-    SELECT ending_height, r.mh_key, r.data
-    FROM results r;
-    -- from the set returned above, insert eth.state_cids records at the ending_height block number
-    -- anchoring all the records to the canonical header found at ending_height
-    INSERT INTO eth.storage_cids (block_number, header_id, state_path, storage_leaf_key, cid, storage_path,
-                              node_type, diff, mh_key)
-    SELECT ending_height, canonical_hash, r.state_path, r.storage_leaf_key, r.cid, r.storage_path, r.node_type, false, r.mh_key
-    FROM results r
-    ON CONFLICT (storage_path, state_path, header_id) DO NOTHING;
-END
-$$;
-
-
 --
 -- Name: was_state_leaf_removed(character varying, character varying); Type: FUNCTION; Schema: public; Owner: -
 --
@@ -396,10 +272,10 @@ CREATE FUNCTION public.was_state_leaf_removed(key character varying, hash charac
     FROM eth.state_cids
              INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.block_hash)
     WHERE state_leaf_key = key
-      AND state_cids.block_number <= (SELECT block_number
+      AND block_number <= (SELECT block_number
                            FROM eth.header_cids
                            WHERE block_hash = hash)
-    ORDER BY state_cids.block_number DESC LIMIT 1;
+    ORDER BY block_number DESC LIMIT 1;
 $$;
 
 
@@ -408,7 +284,6 @@ $$;
 --
 
 CREATE TABLE eth.access_list_elements (
-    block_number bigint NOT NULL,
     tx_id character varying(66) NOT NULL,
     index integer NOT NULL,
     address character varying(66),
@@ -421,7 +296,6 @@ CREATE TABLE eth.access_list_elements (
 --
 
 CREATE TABLE eth.log_cids (
-    block_number bigint NOT NULL,
     leaf_cid text NOT NULL,
     leaf_mh_key text NOT NULL,
     rct_id character varying(66) NOT NULL,
@@ -440,7 +314,6 @@ CREATE TABLE eth.log_cids (
 --
 
 CREATE TABLE eth.receipt_cids (
-    block_number bigint NOT NULL,
     tx_id character varying(66) NOT NULL,
     leaf_cid text NOT NULL,
     contract character varying(66),
@@ -457,7 +330,6 @@ CREATE TABLE eth.receipt_cids (
 --
 
 CREATE TABLE eth.state_accounts (
-    block_number bigint NOT NULL,
     header_id character varying(66) NOT NULL,
     state_path bytea NOT NULL,
     balance numeric NOT NULL,
@@ -472,7 +344,6 @@ CREATE TABLE eth.state_accounts (
 --
 
 CREATE TABLE eth.state_cids (
-    block_number bigint NOT NULL,
     header_id character varying(66) NOT NULL,
     state_leaf_key character varying(66),
     cid text NOT NULL,
@@ -488,7 +359,6 @@ CREATE TABLE eth.state_cids (
 --
 
 CREATE TABLE eth.storage_cids (
-    block_number bigint NOT NULL,
     header_id character varying(66) NOT NULL,
     state_path bytea NOT NULL,
     storage_leaf_key character varying(66),
@@ -505,7 +375,6 @@ CREATE TABLE eth.storage_cids (
 --
 
 CREATE TABLE eth.transaction_cids (
-    block_number bigint NOT NULL,
     header_id character varying(66) NOT NULL,
     tx_hash character varying(66) NOT NULL,
     cid text NOT NULL,
@@ -531,7 +400,6 @@ COMMENT ON TABLE eth.transaction_cids IS '@name EthTransactionCids';
 --
 
 CREATE TABLE eth.uncle_cids (
-    block_number bigint NOT NULL,
     block_hash character varying(66) NOT NULL,
     header_id character varying(66) NOT NULL,
     parent_hash character varying(66) NOT NULL,
@@ -570,7 +438,6 @@ CREATE TABLE eth_meta.watched_addresses (
 --
 
 CREATE TABLE public.blocks (
-    block_number bigint NOT NULL,
     key text NOT NULL,
     data bytea NOT NULL
 );
@@ -659,7 +526,7 @@ ALTER TABLE ONLY public.goose_db_version ALTER COLUMN id SET DEFAULT nextval('pu
 --
 
 ALTER TABLE ONLY eth.access_list_elements
-    ADD CONSTRAINT access_list_elements_pkey PRIMARY KEY (tx_id, index, block_number);
+    ADD CONSTRAINT access_list_elements_pkey PRIMARY KEY (tx_id, index);
 
 
 --
@@ -667,7 +534,7 @@ ALTER TABLE ONLY eth.access_list_elements
 --
 
 ALTER TABLE ONLY eth.header_cids
-    ADD CONSTRAINT header_cids_pkey PRIMARY KEY (block_hash, block_number);
+    ADD CONSTRAINT header_cids_pkey PRIMARY KEY (block_hash);
 
 
 --
@@ -675,7 +542,7 @@ ALTER TABLE ONLY eth.header_cids
 --
 
 ALTER TABLE ONLY eth.log_cids
-    ADD CONSTRAINT log_cids_pkey PRIMARY KEY (rct_id, index, block_number);
+    ADD CONSTRAINT log_cids_pkey PRIMARY KEY (rct_id, index);
 
 
 --
@@ -683,7 +550,7 @@ ALTER TABLE ONLY eth.log_cids
 --
 
 ALTER TABLE ONLY eth.receipt_cids
-    ADD CONSTRAINT receipt_cids_pkey PRIMARY KEY (tx_id, block_number);
+    ADD CONSTRAINT receipt_cids_pkey PRIMARY KEY (tx_id);
 
 
 --
@@ -691,7 +558,7 @@ ALTER TABLE ONLY eth.receipt_cids
 --
 
 ALTER TABLE ONLY eth.state_accounts
-    ADD CONSTRAINT state_accounts_pkey PRIMARY KEY (state_path, header_id, block_number);
+    ADD CONSTRAINT state_accounts_pkey PRIMARY KEY (header_id, state_path);
 
 
 --
@@ -699,7 +566,7 @@ ALTER TABLE ONLY eth.state_accounts
 --
 
 ALTER TABLE ONLY eth.state_cids
-    ADD CONSTRAINT state_cids_pkey PRIMARY KEY (state_path, header_id, block_number);
+    ADD CONSTRAINT state_cids_pkey PRIMARY KEY (header_id, state_path);
 
 
 --
@@ -707,7 +574,7 @@ ALTER TABLE ONLY eth.state_cids
 --
 
 ALTER TABLE ONLY eth.storage_cids
-    ADD CONSTRAINT storage_cids_pkey PRIMARY KEY (storage_path, state_path, header_id, block_number);
+    ADD CONSTRAINT storage_cids_pkey PRIMARY KEY (header_id, state_path, storage_path);
 
 
 --
@@ -715,7 +582,7 @@ ALTER TABLE ONLY eth.storage_cids
 --
 
 ALTER TABLE ONLY eth.transaction_cids
-    ADD CONSTRAINT transaction_cids_pkey PRIMARY KEY (tx_hash, block_number);
+    ADD CONSTRAINT transaction_cids_pkey PRIMARY KEY (tx_hash);
 
 
 --
@@ -723,7 +590,7 @@ ALTER TABLE ONLY eth.transaction_cids
 --
 
 ALTER TABLE ONLY eth.uncle_cids
-    ADD CONSTRAINT uncle_cids_pkey PRIMARY KEY (block_hash, block_number);
+    ADD CONSTRAINT uncle_cids_pkey PRIMARY KEY (block_hash);
 
 
 --
@@ -747,7 +614,7 @@ ALTER TABLE ONLY eth_meta.watched_addresses
 --
 
 ALTER TABLE ONLY public.blocks
-    ADD CONSTRAINT blocks_pkey PRIMARY KEY (key, block_number);
+    ADD CONSTRAINT blocks_pkey PRIMARY KEY (key);
 
 
 --
@@ -774,13 +641,6 @@ ALTER TABLE ONLY public.nodes
     ADD CONSTRAINT nodes_pkey PRIMARY KEY (node_id);
 
 
---
--- Name: access_list_block_number_index; Type: INDEX; Schema: eth; Owner: -
---
-
-CREATE INDEX access_list_block_number_index ON eth.access_list_elements USING brin (block_number);
-
-
 --
 -- Name: access_list_element_address_index; Type: INDEX; Schema: eth; Owner: -
 --
@@ -796,45 +656,31 @@ CREATE INDEX access_list_storage_keys_index ON eth.access_list_elements USING gi
 
 
 --
--- Name: account_block_number_index; Type: INDEX; Schema: eth; Owner: -
+-- Name: account_state_path_index; Type: INDEX; Schema: eth; Owner: -
 --
 
-CREATE INDEX account_block_number_index ON eth.state_accounts USING brin (block_number);
+CREATE INDEX account_state_path_index ON eth.state_accounts USING btree (state_path);
 
 
 --
--- Name: account_header_id_index; Type: INDEX; Schema: eth; Owner: -
+-- Name: block_number_index; Type: INDEX; Schema: eth; Owner: -
 --
 
-CREATE INDEX account_header_id_index ON eth.state_accounts USING btree (header_id);
+CREATE INDEX block_number_index ON eth.header_cids USING brin (block_number);
-
-
---
--- Name: account_storage_root_index; Type: INDEX; Schema: eth; Owner: -
---
-
-CREATE INDEX account_storage_root_index ON eth.state_accounts USING btree (storage_root);
-
-
---
--- Name: header_block_number_index; Type: INDEX; Schema: eth; Owner: -
---
-
-CREATE INDEX header_block_number_index ON eth.header_cids USING brin (block_number);
 
 
 --
 -- Name: header_cid_index; Type: INDEX; Schema: eth; Owner: -
 --
 
-CREATE UNIQUE INDEX header_cid_index ON eth.header_cids USING btree (cid, block_number);
+CREATE UNIQUE INDEX header_cid_index ON eth.header_cids USING btree (cid);
 
 
 --
--- Name: header_mh_block_number_index; Type: INDEX; Schema: eth; Owner: -
+-- Name: header_mh_index; Type: INDEX; Schema: eth; Owner: -
 --
 
-CREATE UNIQUE INDEX header_mh_block_number_index ON eth.header_cids USING btree (mh_key, block_number);
+CREATE UNIQUE INDEX header_mh_index ON eth.header_cids USING btree (mh_key);
 
 
 --
@@ -844,13 +690,6 @@ CREATE UNIQUE INDEX header_mh_block_number_index ON eth.header_cids USING btree
 CREATE INDEX log_address_index ON eth.log_cids USING btree (address);
 
 
---
--- Name: log_block_number_index; Type: INDEX; Schema: eth; Owner: -
---
-
-CREATE INDEX log_block_number_index ON eth.log_cids USING brin (block_number);
-
-
 --
 -- Name: log_cid_index; Type: INDEX; Schema: eth; Owner: -
 --
@@ -859,10 +698,10 @@ CREATE INDEX log_cid_index ON eth.log_cids USING btree (leaf_cid);
 
 
 --
--- Name: log_leaf_mh_block_number_index; Type: INDEX; Schema: eth; Owner: -
+-- Name: log_mh_index; Type: INDEX; Schema: eth; Owner: -
 --
 
-CREATE INDEX log_leaf_mh_block_number_index ON eth.log_cids USING btree (leaf_mh_key, block_number);
+CREATE INDEX log_mh_index ON eth.log_cids USING btree (leaf_mh_key);
 
 
 --
@@ -893,13 +732,6 @@ CREATE INDEX log_topic2_index ON eth.log_cids USING btree (topic2);
 CREATE INDEX log_topic3_index ON eth.log_cids USING btree (topic3);
 
 
---
--- Name: rct_block_number_index; Type: INDEX; Schema: eth; Owner: -
---
-
-CREATE INDEX rct_block_number_index ON eth.receipt_cids USING brin (block_number);
-
-
 --
 -- Name: rct_contract_hash_index; Type: INDEX; Schema: eth; Owner: -
 --
@@ -922,17 +754,10 @@ CREATE INDEX rct_leaf_cid_index ON eth.receipt_cids USING btree (leaf_cid);
 
 
 --
--- Name: rct_leaf_mh_block_number_index; Type: INDEX; Schema: eth; Owner: -
+-- Name: rct_leaf_mh_index; Type: INDEX; Schema: eth; Owner: -
 --
 
-CREATE INDEX rct_leaf_mh_block_number_index ON eth.receipt_cids USING btree (leaf_mh_key, block_number);
+CREATE INDEX rct_leaf_mh_index ON eth.receipt_cids USING btree (leaf_mh_key);
-
-
---
--- Name: state_block_number_index; Type: INDEX; Schema: eth; Owner: -
---
-
-CREATE INDEX state_block_number_index ON eth.state_cids USING brin (block_number);
 
 
 --
@@ -942,13 +767,6 @@ CREATE INDEX state_block_number_index ON eth.state_cids USING brin (block_number
 CREATE INDEX state_cid_index ON eth.state_cids USING btree (cid);
 
 
---
--- Name: state_header_id_index; Type: INDEX; Schema: eth; Owner: -
---
-
-CREATE INDEX state_header_id_index ON eth.state_cids USING btree (header_id);
-
-
 --
 -- Name: state_leaf_key_index; Type: INDEX; Schema: eth; Owner: -
 --
@@ -957,10 +775,10 @@ CREATE INDEX state_leaf_key_index ON eth.state_cids USING btree (state_leaf_key)
 
 
 --
--- Name: state_mh_block_number_index; Type: INDEX; Schema: eth; Owner: -
+-- Name: state_mh_index; Type: INDEX; Schema: eth; Owner: -
 --
 
-CREATE INDEX state_mh_block_number_index ON eth.state_cids USING btree (mh_key, block_number);
+CREATE INDEX state_mh_index ON eth.state_cids USING btree (mh_key);
 
 
 --
@@ -970,6 +788,13 @@ CREATE INDEX state_mh_block_number_index ON eth.state_cids USING btree (mh_key,
 CREATE INDEX state_node_type_index ON eth.state_cids USING btree (node_type);
 
 
+--
+-- Name: state_path_index; Type: INDEX; Schema: eth; Owner: -
+--
+
+CREATE INDEX state_path_index ON eth.state_cids USING btree (state_path);
+
+
 --
 -- Name: state_root_index; Type: INDEX; Schema: eth; Owner: -
 --
@@ -977,13 +802,6 @@ CREATE INDEX state_node_type_index ON eth.state_cids USING btree (node_type);
 CREATE INDEX state_root_index ON eth.header_cids USING btree (state_root);
 
 
---
--- Name: storage_block_number_index; Type: INDEX; Schema: eth; Owner: -
---
-
-CREATE INDEX storage_block_number_index ON eth.storage_cids USING brin (block_number);
-
-
 --
 -- Name: storage_cid_index; Type: INDEX; Schema: eth; Owner: -
 --
@@ -991,13 +809,6 @@ CREATE INDEX storage_block_number_index ON eth.storage_cids USING brin (block_nu
 CREATE INDEX storage_cid_index ON eth.storage_cids USING btree (cid);
 
 
---
--- Name: storage_header_id_index; Type: INDEX; Schema: eth; Owner: -
---
-
-CREATE INDEX storage_header_id_index ON eth.storage_cids USING btree (header_id);
-
-
 --
 -- Name: storage_leaf_key_index; Type: INDEX; Schema: eth; Owner: -
 --
@@ -1006,10 +817,10 @@ CREATE INDEX storage_leaf_key_index ON eth.storage_cids USING btree (storage_lea
 
 
 --
--- Name: storage_mh_block_number_index; Type: INDEX; Schema: eth; Owner: -
+-- Name: storage_mh_index; Type: INDEX; Schema: eth; Owner: -
 --
 
-CREATE INDEX storage_mh_block_number_index ON eth.storage_cids USING btree (mh_key, block_number);
+CREATE INDEX storage_mh_index ON eth.storage_cids USING btree (mh_key);
 
 
 --
@@ -1019,6 +830,20 @@ CREATE INDEX storage_mh_block_number_index ON eth.storage_cids USING btree (mh_k
 CREATE INDEX storage_node_type_index ON eth.storage_cids USING btree (node_type);
 
 
+--
+-- Name: storage_path_index; Type: INDEX; Schema: eth; Owner: -
+--
+
+CREATE INDEX storage_path_index ON eth.storage_cids USING btree (storage_path);
+
+
+--
+-- Name: storage_root_index; Type: INDEX; Schema: eth; Owner: -
+--
+
+CREATE INDEX storage_root_index ON eth.state_accounts USING btree (storage_root);
+
+
 --
 -- Name: storage_state_path_index; Type: INDEX; Schema: eth; Owner: -
 --
@@ -1033,18 +858,11 @@ CREATE INDEX storage_state_path_index ON eth.storage_cids USING btree (state_pat
 CREATE INDEX timestamp_index ON eth.header_cids USING brin ("timestamp");
 
 
---
--- Name: tx_block_number_index; Type: INDEX; Schema: eth; Owner: -
---
-
-CREATE INDEX tx_block_number_index ON eth.transaction_cids USING brin (block_number);
-
-
 --
 -- Name: tx_cid_index; Type: INDEX; Schema: eth; Owner: -
 --
 
-CREATE UNIQUE INDEX tx_cid_index ON eth.transaction_cids USING btree (cid, block_number);
+CREATE UNIQUE INDEX tx_cid_index ON eth.transaction_cids USING btree (cid);
 
 
 --
@@ -1062,10 +880,10 @@ CREATE INDEX tx_header_id_index ON eth.transaction_cids USING btree (header_id);
 
 
 --
--- Name: tx_mh_block_number_index; Type: INDEX; Schema: eth; Owner: -
+-- Name: tx_mh_index; Type: INDEX; Schema: eth; Owner: -
 --
 
-CREATE UNIQUE INDEX tx_mh_block_number_index ON eth.transaction_cids USING btree (mh_key, block_number);
+CREATE UNIQUE INDEX tx_mh_index ON eth.transaction_cids USING btree (mh_key);
 
 
 --
@@ -1075,13 +893,6 @@ CREATE UNIQUE INDEX tx_mh_block_number_index ON eth.transaction_cids USING btree
 CREATE INDEX tx_src_index ON eth.transaction_cids USING btree (src);
 
 
---
--- Name: uncle_block_number_index; Type: INDEX; Schema: eth; Owner: -
---
-
-CREATE INDEX uncle_block_number_index ON eth.uncle_cids USING brin (block_number);
-
-
 --
 -- Name: uncle_header_id_index; Type: INDEX; Schema: eth; Owner: -
 --
@@ -1089,20 +900,6 @@ CREATE INDEX uncle_block_number_index ON eth.uncle_cids USING brin (block_number
 CREATE INDEX uncle_header_id_index ON eth.uncle_cids USING btree (header_id);
 
 
---
--- Name: uncle_mh_block_number_index; Type: INDEX; Schema: eth; Owner: -
---
-
-CREATE UNIQUE INDEX uncle_mh_block_number_index ON eth.uncle_cids USING btree (mh_key, block_number);
-
-
---
--- Name: blocks_block_number_idx; Type: INDEX; Schema: public; Owner: -
---
-
-CREATE INDEX blocks_block_number_idx ON public.blocks USING btree (block_number DESC);
-
-
 --
 -- Name: access_list_elements trg_eth_access_list_elements; Type: TRIGGER; Schema: eth; Owner: -
 --
@@ -1166,6 +963,134 @@ CREATE TRIGGER trg_eth_transaction_cids AFTER INSERT ON eth.transaction_cids FOR
 CREATE TRIGGER trg_eth_uncle_cids AFTER INSERT ON eth.uncle_cids FOR EACH ROW EXECUTE FUNCTION eth.graphql_subscription();
 
 
+--
+-- Name: access_list_elements access_list_elements_tx_id_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: -
+--
+
+ALTER TABLE ONLY eth.access_list_elements
+    ADD CONSTRAINT access_list_elements_tx_id_fkey FOREIGN KEY (tx_id) REFERENCES eth.transaction_cids(tx_hash) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
+
+
+--
+-- Name: header_cids header_cids_mh_key_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: -
+--
+
+ALTER TABLE ONLY eth.header_cids
+    ADD CONSTRAINT header_cids_mh_key_fkey FOREIGN KEY (mh_key) REFERENCES public.blocks(key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
+
+
+--
+-- Name: header_cids header_cids_node_id_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: -
+--
+
+ALTER TABLE ONLY eth.header_cids
+    ADD CONSTRAINT header_cids_node_id_fkey FOREIGN KEY (node_id) REFERENCES public.nodes(node_id) ON DELETE CASCADE;
+
+
+--
+-- Name: log_cids log_cids_leaf_mh_key_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: -
+--
+
+ALTER TABLE ONLY eth.log_cids
+    ADD CONSTRAINT log_cids_leaf_mh_key_fkey FOREIGN KEY (leaf_mh_key) REFERENCES public.blocks(key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
+
+
+--
+-- Name: log_cids log_cids_rct_id_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: -
+--
+
+ALTER TABLE ONLY eth.log_cids
+    ADD CONSTRAINT log_cids_rct_id_fkey FOREIGN KEY (rct_id) REFERENCES eth.receipt_cids(tx_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
+
+
+--
+-- Name: receipt_cids receipt_cids_leaf_mh_key_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: -
+--
+
+ALTER TABLE ONLY eth.receipt_cids
+    ADD CONSTRAINT receipt_cids_leaf_mh_key_fkey FOREIGN KEY (leaf_mh_key) REFERENCES public.blocks(key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
+
+
+--
+-- Name: receipt_cids receipt_cids_tx_id_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: -
+--
+
+ALTER TABLE ONLY eth.receipt_cids
+    ADD CONSTRAINT receipt_cids_tx_id_fkey FOREIGN KEY (tx_id) REFERENCES eth.transaction_cids(tx_hash) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
+
+
+--
+-- Name: state_accounts state_accounts_header_id_state_path_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: -
+--
+
+ALTER TABLE ONLY eth.state_accounts
+    ADD CONSTRAINT state_accounts_header_id_state_path_fkey FOREIGN KEY (header_id, state_path) REFERENCES eth.state_cids(header_id, state_path) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
+
+
+--
+-- Name: state_cids state_cids_header_id_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: -
+--
+
+ALTER TABLE ONLY eth.state_cids
+    ADD CONSTRAINT state_cids_header_id_fkey FOREIGN KEY (header_id) REFERENCES eth.header_cids(block_hash) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
+
+
+--
+-- Name: state_cids state_cids_mh_key_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: -
+--
+
+ALTER TABLE ONLY eth.state_cids
+    ADD CONSTRAINT state_cids_mh_key_fkey FOREIGN KEY (mh_key) REFERENCES public.blocks(key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
+
+
+--
+-- Name: storage_cids storage_cids_header_id_state_path_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: -
+--
+
+ALTER TABLE ONLY eth.storage_cids
+    ADD CONSTRAINT storage_cids_header_id_state_path_fkey FOREIGN KEY (header_id, state_path) REFERENCES eth.state_cids(header_id, state_path) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
+
+
+--
+-- Name: storage_cids storage_cids_mh_key_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: -
+--
+
+ALTER TABLE ONLY eth.storage_cids
+    ADD CONSTRAINT storage_cids_mh_key_fkey FOREIGN KEY (mh_key) REFERENCES public.blocks(key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
+
+
+--
+-- Name: transaction_cids transaction_cids_header_id_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: -
+--
+
+ALTER TABLE ONLY eth.transaction_cids
+    ADD CONSTRAINT transaction_cids_header_id_fkey FOREIGN KEY (header_id) REFERENCES eth.header_cids(block_hash) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
+
+
+--
+-- Name: transaction_cids transaction_cids_mh_key_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: -
+--
+
+ALTER TABLE ONLY eth.transaction_cids
+    ADD CONSTRAINT transaction_cids_mh_key_fkey FOREIGN KEY (mh_key) REFERENCES public.blocks(key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
+
+
+--
+-- Name: uncle_cids uncle_cids_header_id_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: -
+--
+
+ALTER TABLE ONLY eth.uncle_cids
+    ADD CONSTRAINT uncle_cids_header_id_fkey FOREIGN KEY (header_id) REFERENCES eth.header_cids(block_hash) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
+
+
+--
+-- Name: uncle_cids uncle_cids_mh_key_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: -
+--
+
+ALTER TABLE ONLY eth.uncle_cids
+    ADD CONSTRAINT uncle_cids_mh_key_fkey FOREIGN KEY (mh_key) REFERENCES public.blocks(key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
+
+
 --
 -- PostgreSQL database dump complete
 --

scripts/add_nodes.sh

@@ -1,138 +0,0 @@
-#!/bin/bash
-# Guards
-if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ] || [ -z "$4" ] || [ -z "$5" ]
-  then
-    echo "Env variables not provided"
-    echo "Usage: ./add_nodes.sh <MIGRATION_FILE_PATH:string> <NODE_NAME:string> <NODE_HOST:string> <NODE_PORT:numeric/string> <NODE_DATABASE:string> <EXECUTE_SQL:bool>"
-    echo "Only <EXECUTE_SQL> is optional"
-    exit 1
-fi
-if [ -z "$6" ]
-  then
-    echo "EXECUTE_SQL not set, will not run statements against an access server"
-  else
-    echo "EXECUTE_SQL is set, will run stataments against an access server"
-    echo "Note: this mode is not recommended except in the case when the migration has already been applied with previous
-    nodes and we need/want to add more while still recording them in the existing migration (and adding their Down statements to said migration)"
-    echo "Expected environment variables:"
-    echo "DATABASE_HOSTNAME, DATABASE_NAME, DATABASE_PORT, DATABASE_USER"
-    echo "For now, DATABASE_PASSWORD will be prompted for on each psql call"
-fi
-
-# Remote DB node info
-export MIGRATION_FILE_PATH=$1
-export NODE_NAME=$2
-export NODE_HOST=$3
-export NODE_PORT=$4
-export NODE_DATABASE=$5
-printf "Enter the ${NODE_HOST} database password:\n"
-stty -echo
-read NODE_PASSWORD
-stty echo
-export NODE_PASSWORD
-
-if ! [ -z "$6" ]
-  then
-    # Access DB info
-    echo "heeeeey"
-    export DATABASE_HOSTNAME=localhost
-    export DATABASE_PORT=5432
-    export DATABASE_USER=vdbm
-    export DATABASE_NAME=vulcanize_db
-fi
-
-# Array of distributed hypertable names
-declare -a tables_names=("public.blocks" "eth.header_cid" "eth.uncle_cids" "eth.transaction_cids"
-                "eth.receipt_cids" "eth.state_cid" "eth.storage_cids" "eth.state_accounts"
-                "eth.access_list_elements" "eth.log_cids"
-                )
-# Array to append Up statements to for later (optional) execution
-declare -a up_stmts=()
-
-echo "Writing Up and Down statements to provided migration file at ${migration_file_path}"
-
-# Create add node statement
-up_add_pg_str="SELECT add_data_node('${NODE_NAME}', host => '${NODE_HOST}', port => ${NODE_PORT}, database => '${NODE_DATABASE}', password => '${NODE_PASSWORD}');"
-up_stmts+=(${up_add_pg_str})
-
-# Insert at the 3rd line of the file
-sed -i.bak '3 i\
-'"${up_add_pg_str}"'
-' "${MIGRATION_FILE_PATH}"
-
-# Check for error
-if [[ $? -eq 0 ]]; then
-    echo "Wrote Up add node statement ${up_add_pg_str}"
-else
-    echo "Could not write Up add node statement ${up_add_pg_str}. Is the migration file path correct?"
-    exit 1
-fi
-
-# Create attach node statements
-for table_name in "${tables_names[@]}"
-do
-  up_attach_pg_str="SELECT attach_data_node('${NODE_NAME}', '${table_name}', if_not_attached => true);"
-  up_stmts+=(${up_attach_pg_str})
-  # Insert at the 4th line of the file
-  sed -i.bak '4 i\
-'"${up_attach_pg_str}"'
-' "${MIGRATION_FILE_PATH}"
-  # Check for error
-  if [[ $? -eq 0 ]]; then
-      echo "Wrote Up attach node statement ${up_attach_pg_str}"
-  else
-      echo "Could not write Up attach node statement ${up_attach_pg_str}. Is the migration file path correct?"
-      exit 1
-  fi
-done
-
-## Create detach and remove node statement
-down_attach_pg_str="SELECT detach_data_node('${NODE_NAME}', force => true, if_attached = true);"
-down_add_pg_str="SELECT delete_data_node('${NODE_NAME}', force => true, if_attached => true);"
-
-# Append them at the last line in the file
-sed -i.bak '$ a\
-'"${down_attach_pg_str}"'
-' "${MIGRATION_FILE_PATH}"
-# Check for error
-if [[ $? -eq 0 ]]; then
-    echo "Wrote Down attach node statement ${down_attach_pg_str}"
-else
-    echo "Could not write Down attach node statement ${down_attach_pg_str}. Is the migration file path correct?"
-    exit 1
-fi
-# Append them at the last line in the file
-sed -i.bak '$ a\
-'"${down_add_pg_str}"'
-' "${MIGRATION_FILE_PATH}"
-# Check for error
-if [[ $? -eq 0 ]]; then
-    echo "Wrote Down add node statement ${down_add_pg_str}"
-else
-    echo "Could not write Down add node statement ${down_add_pg_str}. Is the migration file path correct?"
-    exit 1
-fi
-
-# Execute Up statements on the server if we are in that mode
-if [ -z "$6" ]
-  then
-    echo "Done!"
-    exit 0
-fi
-
-echo "Executing Up statements against provided server"
-
-for up_stmt in "${up_stmts[@]}"
-do
-  psql -c '\x' -c "${up_stmt}" -h $DATABASE_HOSTNAME -p $DATABASE_PORT -U $DATABASE_USER -d $DATABASE_NAME -W
-  # Check for error
-  if [[ $? -eq 0 ]]; then
-      echo "Executed Up statement ${up_stmt}}"
-  else
-      echo "Could not execute Up statement ${up_stmt}. Is the migration file path correct?"
-      exit 1
-  fi
-done
-
-echo "Done!"
-exit 0

scripts/init-access-node.sh

@@ -1,29 +0,0 @@
-#!/bin/sh
-set -e
-
-# https://docs.timescale.com/timescaledb/latest/how-to-guides/multinode-timescaledb/multinode-config/
-
-# To achieve good query performance you need to enable partition-wise aggregation on the access node. This pushes down aggregation queries to the data nodes.
-# https://www.postgresql.org/docs/12/runtime-config-query.html#enable_partitionwise_aggregate
-sed -ri "s!^#?(enable_partitionwise_aggregate)\s*=.*!\1 = on!" /var/lib/postgresql/data/postgresql.conf
-grep "enable_partitionwise_aggregate" /var/lib/postgresql/data/postgresql.conf
-
-# JIT should be set to off on the access node as JIT currently doesn't work well with distributed queries.
-# https://www.postgresql.org/docs/12/runtime-config-query.html#jit
-sed -ri "s!^#?(jit)\s*=.*!\1 = off!" /var/lib/postgresql/data/postgresql.conf
-grep "jit" /var/lib/postgresql/data/postgresql.conf
-
-# https://docs.timescale.com/timescaledb/latest/how-to-guides/multinode-timescaledb/multinode-auth/
-# https://docs.timescale.com/timescaledb/latest/how-to-guides/multinode-timescaledb/multinode-auth/#password-authentication
-
-# Set password_encryption = 'scram-sha-256' in postgresql.conf on the access node.
-sed -ri "s!^#?(password_encryption)\s*=.*!\1 = 'scram-sha-256'!" /var/lib/postgresql/data/postgresql.conf
-grep "password_encryption" /var/lib/postgresql/data/postgresql.conf
-
-# Append to data/passfile *:*:*:ROLE:ROLE_PASSWORD
-# This file stores the passwords for each role that the access node connects to on the data nodes.
-echo "*:*:*:postgres:password">>/var/lib/postgresql/data/passfile
-chmod 0600 /var/lib/postgresql/data/passfile
-
-# Add "host  all  all  ACCESS_NODE_IP  scram-sha-256" pg_hba.conf on the data nodes.
-# Skipped. Using default "host  all  all  all  scram-sha-256" for now.

scripts/init-data-node.sh

@@ -1,27 +0,0 @@
-#!/bin/sh
-set -e
-
-# https://docs.timescale.com/timescaledb/latest/how-to-guides/configuration/timescaledb-config/#timescaledb-last-tuned-string
-# https://docs.timescale.com/timescaledb/latest/how-to-guides/multi-node-setup/required-configuration/
-
-# It is necessary to change the parameter max_prepared_transactions to a non-zero value ('150' is recommended).
-# https://www.postgresql.org/docs/12/runtime-config-resource.html#max_prepared_transactions
-sed -ri "s!^#?(max_prepared_transactions)\s*=.*!\1 = 150!" /var/lib/postgresql/data/postgresql.conf
-grep "max_prepared_transactions" /var/lib/postgresql/data/postgresql.conf
-
-# Statement timeout should be disabled on the data nodes and managed through the access node configuration if desired.
-# https://www.postgresql.org/docs/12/runtime-config-client.html#statement_timeout
-sed -ri "s!^#?(statement_timeout)\s*=.*!\1 = 0!" /var/lib/postgresql/data/postgresql.conf
-grep "statement_timeout" /var/lib/postgresql/data/postgresql.conf
-
-# On the data nodes, set the wal_level to logical or higher to move or copy chunks between data nodes.
-# https://www.postgresql.org/docs/14/runtime-config-wal.html
-sed -ri "s!^#?(wal_level)\s*=.*!\1 = 'logical'!" /var/lib/postgresql/data/postgresql.conf
-grep "wal_level" /var/lib/postgresql/data/postgresql.conf
-
-# https://docs.timescale.com/timescaledb/latest/how-to-guides/multinode-timescaledb/multinode-auth/
-# https://docs.timescale.com/timescaledb/latest/how-to-guides/multinode-timescaledb/multinode-auth/#password-authentication
-
-# Set password_encryption = 'scram-sha-256' in postgresql.conf on the data node.
-sed -ri "s!^#?(password_encryption)\s*=.*!\1 = 'scram-sha-256'!" /var/lib/postgresql/data/postgresql.conf
-grep "password_encryption" /var/lib/postgresql/data/postgresql.conf

scripts/startup_script.sh

@@ -7,9 +7,8 @@ VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME
 
 # Run the DB migrations
 echo "Connecting with: $VDB_PG_CONNECT"
-sleep 15
 echo "Running database migrations"
-./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up-to 24
+./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up
 
 # If the db migrations ran without err
 if [[ $? -eq 0 ]]; then