incomplete tag

* Add block hash to primary keys in transactions, receipts and logs tables

* Add block hash in postgraphile triggers for transactions, receipts and logs tables

* Make indexes on transaction cid and mh_key non-unique

.github/workflows/on-pr.yaml

@@ -3,6 +3,13 @@ name: Docker Build
 on: [pull_request]
 
 jobs:
+  build:
+    name: Run docker build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Run docker build
+        run: make docker-build
   test:
     # Add a dummy job to avoid failing GitHub CI checks.
     # Other checks to be added later.

.github/workflows/publish.yaml

@@ -8,34 +8,42 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Get the version
-        id: vars
-        run: echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
-      - name: Run docker build
-        run: make docker-build
-      - name: Tag docker image
-        run: docker tag vulcanize/ipld-eth-db docker.pkg.github.com/vulcanize/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.sha}}
-      - name: Docker Login
-        run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
-      - name: Docker Push
-        run: docker push docker.pkg.github.com/vulcanize/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.sha}}
-  push_to_registries:
-    name: Push Docker image to Docker Hub
-    runs-on: ubuntu-latest
-    needs: build
-    steps:
       - name: Get the version
         id: vars
         run: |
           echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
           echo ::set-output name=tag::$(echo ${GITHUB_REF#refs/tags/})
-      - name: Docker Login to Github Registry
-        run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
-      - name: Docker Pull
-        run: docker pull docker.pkg.github.com/vulcanize/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.sha}}
-      - name: Docker Login to Docker Registry
-        run: echo ${{ secrets.VULCANIZEJENKINS_PAT }} | docker login -u vulcanizejenkins --password-stdin
+      - name: Run docker build
+        run: make docker-build
       - name: Tag docker image
-        run: docker tag docker.pkg.github.com/vulcanize/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.sha}} vulcanize/ipld-eth-db:${{steps.vars.outputs.tag}}
-      - name: Docker Push to Docker Hub
-        run: docker push vulcanize/ipld-eth-db:${{steps.vars.outputs.tag}}
+        run: docker tag cerc-io/ipld-eth-db git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.sha}}
+      - name: Tag docker image
+        run: docker tag git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.sha}} git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.tag}}
+      - name: Docker Login
+        run: echo ${{ secrets.GITEA_TOKEN }} | docker login https://git.vdb.to -u cerccicd --password-stdin
+      - name: Docker Push
+        run: docker push git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.sha}}
+      - name: Docker Push with TAG to git.vdb.to
+        run: docker push git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.tag}}
+  #push_to_registries:
+  #  name: Push Docker image to Docker Hub
+  #  runs-on: ubuntu-latest
+  #  needs: build
+  #  steps:
+  #    - name: Get the version
+  #     id: vars
+  #      run: |
+  #        echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
+  #        echo ::set-output name=tag::$(echo ${GITHUB_REF#refs/tags/})
+  #    - name: Docker Login to Github Registry
+  #      run: echo ${{ secrets.GITEA_TOKEN }} | docker login https://git.vdb.to -u cerccicd --password-stdin
+  #    - name: Docker Pull
+  #      run: docker pull git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.sha}}
+  #    - name: Docker Push to Docker Hub
+      #      run: docker push cerc-io/ipld-eth-db:${{steps.vars.outputs.tag}}
+  #    - name: Docker Login to Docker Registry
+  #      run: echo ${{ secrets.VULCANIZEJENKINS_PAT }} | docker login -u cerccicd --password-stdin
+  #    - name: Tag docker image
+  #      run: docker tag git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.sha}} cerc-io/ipld-eth-db:${{steps.vars.outputs.tag}}
+  #    - name: Docker Push to Docker Hub
+  #      run: docker push cerc-io/ipld-eth-db:${{steps.vars.outputs.tag}}

.gitignore

@@ -1 +1,2 @@
-.idea/
+.idea/
+.vscode

Dockerfile

@@ -1,12 +1,13 @@
-FROM golang:1.16-alpine as builder
+FROM golang:1.18-alpine as builder
 
 RUN apk --update --no-cache add make git g++ linux-headers
 
-ADD . /go/src/github.com/vulcanize/ipld-eth-db
+ADD . /go/src/github.com/cerc-io/ipld-eth-db
 
 # Build migration tool
 WORKDIR /go/src/github.com/pressly
-RUN git clone https://github.com/pressly/goose.git
+ARG GOOSE_VER="v3.6.1"
+RUN git clone --depth 1 --branch ${GOOSE_VER} https://github.com/pressly/goose.git
 WORKDIR /go/src/github.com/pressly/goose/cmd/goose
 RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -tags='no_sqlite3' -o goose .
 
@@ -15,12 +16,9 @@ FROM alpine
 
 WORKDIR /app
 
-COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-db/scripts/startup_script.sh .
-
-# copy over file for TimescaleDB setup
-COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-db/docker-compose.test.yml docker-tsdb/docker-compose.test.yml
+COPY --from=builder /go/src/github.com/cerc-io/ipld-eth-db/scripts/startup_script.sh .
 
 COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose goose
-COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-db/db/migrations migrations/vulcanizedb
+COPY --from=builder /go/src/github.com/cerc-io/ipld-eth-db/db/migrations migrations/vulcanizedb
 
 ENTRYPOINT ["/app/startup_script.sh"]

Makefile

@@ -115,12 +115,12 @@ import:
 ## Build docker image with schema
 .PHONY: docker-build
 docker-build:
-	docker-compose build
+	docker-compose -f docker-compose.test.yml build
 
-## Build docker image for migration
-.PHONY: docker-concise-migration-build
-docker-concise-migration-build:
-	docker build -t vulcanize/concise-migration-build -f ./db/Dockerfile .
+# ## Build docker image for migration
+# .PHONY: docker-concise-migration-build
+# docker-concise-migration-build:
+# 	docker build -t vulcanize/concise-migration-build -f ./db/Dockerfile .
 
 .PHONY: test-migrations
 test-migrations: $(GOOSE)

README.md

@@ -12,28 +12,24 @@ Schemas and utils for IPLD ETH Postgres database
   docker-compose down -v --remove-orphans
   ```
 
-* Spin up a TimescaleDB instance using [docker-compose.test.yml](./docker-compose.test.yml):
+* Spin up `ipld-eth-db` using an existing image:
 
-  ```bash
-  docker-compose -f docker-compose.test.yml up
-  ```
+  * Update image source used for running the migrations in [docker-compose.yml](./docker-compose.yml) (if required).
 
-  Following final output should be seen:
+  * Run:
 
     ```
-    LOG:  TimescaleDB background worker launcher connected to shared catalogs
+    docker-compose -f docker-compose.yml up
     ```
 
-* In another `ipld-eth-db` terminal window, build an image `migrations-test` using [Dockerfile](./db/Dockerfile):
+* Spin up `ipld-eth-db` using a locally built image:
 
-  ```bash
-  docker build -t migrations-test -f ./db/Dockerfile .
-  ```
+  * Update [Dockerfile](./Dockerfile) (if required).
 
-* Start a container using `migrations-test` image to run the db migrations:
+  * Update build context used for running the migrations in [docker-compose.test.yml](./docker-compose.test.yml) (if required).
 
-  ```bash
-  # Here, we are running the container using host network.
-  # So connect to TimescaleDB on 127.0.0.1:8066
-  docker run --rm --network host -e DATABASE_USER=vdbm -e DATABASE_PASSWORD=password -e DATABASE_HOSTNAME=127.0.0.1 -e DATABASE_PORT=8066 -e DATABASE_NAME=vulcanize_testing_v4 migrations-test
-  ```
+  * Run:
+
+    ```
+    docker-compose -f docker-compose.test.yml up --build
+    ```

db/Dockerfile

@@ -1,23 +0,0 @@
-FROM golang:1.16-alpine as builder
-
-RUN apk --update --no-cache add make git g++ linux-headers
-
-ADD . /go/src/github.com/vulcanize/ipld-eth-db
-
-# Build migration tool
-WORKDIR /go/src/github.com/pressly
-RUN git clone https://github.com/pressly/goose.git
-WORKDIR /go/src/github.com/pressly/goose/cmd/goose
-RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -tags='no_sqlite3' -o goose .
-
-# app container
-FROM alpine
-
-WORKDIR /app
-
-COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-db/scripts/startup_script.sh .
-
-COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose goose
-COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-db/db/migrations migrations/vulcanizedb
-
-ENTRYPOINT ["/app/startup_script.sh"]

db/migrations/00006_create_eth_transaction_cids_table.sql

@@ -11,7 +11,7 @@ CREATE TABLE IF NOT EXISTS eth.transaction_cids (
     tx_data               BYTEA,
     tx_type               INTEGER,
     value                 NUMERIC,
-    PRIMARY KEY (tx_hash, block_number)
+    PRIMARY KEY (tx_hash, header_id, block_number)
 );
 
 -- +goose Down

db/migrations/00007_create_eth_receipt_cids_table.sql

@@ -1,6 +1,7 @@
 -- +goose Up
 CREATE TABLE IF NOT EXISTS eth.receipt_cids (
     block_number          BIGINT NOT NULL,
+    header_id             VARCHAR(66) NOT NULL,
     tx_id                 VARCHAR(66) NOT NULL,
     leaf_cid              TEXT NOT NULL,
     contract              VARCHAR(66),
@@ -9,7 +10,7 @@ CREATE TABLE IF NOT EXISTS eth.receipt_cids (
     post_state            VARCHAR(66),
     post_status           INTEGER,
     log_root              VARCHAR(66),
-    PRIMARY KEY (tx_id, block_number)
+    PRIMARY KEY (tx_id, header_id, block_number)
 );
 
 -- +goose Down

db/migrations/00012_create_eth_log_cids_table.sql

@@ -1,6 +1,7 @@
 -- +goose Up
 CREATE TABLE IF NOT EXISTS eth.log_cids (
     block_number        BIGINT NOT NULL,
+    header_id           VARCHAR(66) NOT NULL,
     leaf_cid            TEXT NOT NULL,
     leaf_mh_key         TEXT NOT NULL,
     rct_id              VARCHAR(66) NOT NULL,
@@ -11,7 +12,7 @@ CREATE TABLE IF NOT EXISTS eth.log_cids (
     topic2              VARCHAR(66),
     topic3              VARCHAR(66),
     log_data            BYTEA,
-    PRIMARY KEY (rct_id, index, block_number)
+    PRIMARY KEY (rct_id, index, header_id, block_number)
 );
 
 -- +goose Down

db/migrations/00014_create_cid_indexes.sql

@@ -14,13 +14,14 @@ CREATE INDEX uncle_header_id_index ON eth.uncle_cids USING btree (header_id);
 -- transaction indexes
 CREATE INDEX tx_block_number_index ON eth.transaction_cids USING brin (block_number);
 CREATE INDEX tx_header_id_index ON eth.transaction_cids USING btree (header_id);
-CREATE UNIQUE INDEX tx_cid_index ON eth.transaction_cids USING btree (cid, block_number);
-CREATE UNIQUE INDEX tx_mh_block_number_index ON eth.transaction_cids USING btree (mh_key, block_number);
+CREATE INDEX tx_cid_index ON eth.transaction_cids USING btree (cid, block_number);
+CREATE INDEX tx_mh_block_number_index ON eth.transaction_cids USING btree (mh_key, block_number);
 CREATE INDEX tx_dst_index ON eth.transaction_cids USING btree (dst);
 CREATE INDEX tx_src_index ON eth.transaction_cids USING btree (src);
 
 -- receipt indexes
 CREATE INDEX rct_block_number_index ON eth.receipt_cids USING brin (block_number);
+CREATE INDEX rct_header_id_index ON eth.receipt_cids USING btree (header_id);
 CREATE INDEX rct_leaf_cid_index ON eth.receipt_cids USING btree (leaf_cid);
 CREATE INDEX rct_leaf_mh_block_number_index ON eth.receipt_cids USING btree (leaf_mh_key, block_number);
 CREATE INDEX rct_contract_index ON eth.receipt_cids USING btree (contract);
@@ -55,6 +56,7 @@ CREATE INDEX access_list_storage_keys_index ON eth.access_list_elements USING gi
 
 -- log indexes
 CREATE INDEX log_block_number_index ON eth.log_cids USING brin (block_number);
+CREATE INDEX log_header_id_index ON eth.log_cids USING btree (header_id);
 CREATE INDEX log_leaf_mh_block_number_index ON eth.log_cids USING btree (leaf_mh_key, block_number);
 CREATE INDEX log_cid_index ON  eth.log_cids USING btree (leaf_cid);
 CREATE INDEX log_address_index ON eth.log_cids USING btree (address);
@@ -72,6 +74,7 @@ DROP INDEX eth.log_topic0_index;
 DROP INDEX eth.log_address_index;
 DROP INDEX eth.log_cid_index;
 DROP INDEX eth.log_leaf_mh_block_number_index;
+DROP INDEX eth.log_header_id_index;
 DROP INDEX eth.log_block_number_index;
 
 -- access list indexes
@@ -106,6 +109,7 @@ DROP INDEX eth.rct_contract_hash_index;
 DROP INDEX eth.rct_contract_index;
 DROP INDEX eth.rct_leaf_mh_block_number_index;
 DROP INDEX eth.rct_leaf_cid_index;
+DROP INDEX eth.rct_header_id_index;
 DROP INDEX eth.rct_block_number_index;
 
 -- transaction indexes

db/migrations/00015_create_postgraphile_triggers.sql

@@ -22,17 +22,20 @@ BEGIN
     ELSIF (TG_TABLE_NAME = 'log_cids') THEN
          obj := json_build_array(
                     TG_TABLE_NAME,
+                    NEW.header_id,
                     NEW.rct_id,
                     NEW.index
                 );
     ELSIF (TG_TABLE_NAME = 'receipt_cids') THEN
          obj := json_build_array(
                     TG_TABLE_NAME,
+                    NEW.header_id,
                     NEW.tx_id
                 );
     ELSIF (TG_TABLE_NAME = 'transaction_cids') THEN
          obj := json_build_array(
                     TG_TABLE_NAME,
+                    NEW.header_id,
                     NEW.tx_hash
                 );
     ELSIF (TG_TABLE_NAME = 'access_list_elements') THEN

db/migrations/00022_create_stored_functions.sql

@@ -155,26 +155,26 @@ BEGIN
     END IF;
 
     -- select all of the state nodes for this snapshot: the latest state node record at every unique path
-    SELECT DISTINCT ON (state_path) blocks.data, state_cids.state_leaf_key, state_cids.cid, state_cids.state_path,
-        state_cids.node_type, state_cids.mh_key
-    INTO results
+    SELECT ARRAY (SELECT DISTINCT ON (state_path) ROW (blocks.data, state_cids.state_leaf_key, state_cids.cid, state_cids.state_path,
+        state_cids.node_type, state_cids.mh_key)
     FROM eth.state_cids
         INNER JOIN public.blocks
             ON (state_cids.mh_key, state_cids.block_number) = (blocks.key, blocks.block_number)
     WHERE state_cids.block_number BETWEEN starting_height AND ending_height
-    ORDER BY state_path, block_number DESC;
+    ORDER BY state_path, state_cids.block_number DESC)
+    INTO results;
 
     -- from the set returned above, insert public.block records at the ending_height block number
     INSERT INTO public.blocks (block_number, key, data)
     SELECT ending_height, r.mh_key, r.data
-    FROM results r;
+    FROM unnest(results) r;
 
     -- from the set returned above, insert eth.state_cids records at the ending_height block number
     -- anchoring all the records to the canonical header found at ending_height
     INSERT INTO eth.state_cids (block_number, header_id, state_leaf_key, cid, state_path, node_type, diff, mh_key)
     SELECT ending_height, canonical_hash, r.state_leaf_key, r.cid, r.state_path, r.node_type, false, r.mh_key
-    FROM results r
-    ON CONFLICT (state_path, header_id) DO NOTHING;
+    FROM unnest(results) r
+    ON CONFLICT (state_path, header_id, block_number) DO NOTHING;
 END
 $BODY$
 LANGUAGE 'plpgsql';
@@ -208,29 +208,29 @@ BEGIN
     END IF;
 
     -- select all of the storage nodes for this snapshot: the latest storage node record at every unique state leaf key
-    SELECT DISTINCT ON (state_leaf_key, storage_path) block.data, storage_cids.state_path, storage_cids.storage_leaf_key,
-     storage_cids.cid, storage_cids.storage_path, storage_cids.node_type, storage_cids.mh_key
-    INTO results
+    SELECT ARRAY (SELECT DISTINCT ON (state_leaf_key, storage_path) ROW (blocks.data, storage_cids.state_path, storage_cids.storage_leaf_key,
+     storage_cids.cid, storage_cids.storage_path, storage_cids.node_type, storage_cids.mh_key)
     FROM eth.storage_cids
         INNER JOIN public.blocks
         ON (storage_cids.mh_key, storage_cids.block_number) = (blocks.key, blocks.block_number)
         INNER JOIN eth.state_cids
         ON (storage_cids.state_path, storage_cids.header_id) = (state_cids.state_path, state_cids.header_id)
     WHERE storage_cids.block_number BETWEEN starting_height AND ending_height
-    ORDER BY state_path, storage_path, block_number DESC;
+    ORDER BY state_leaf_key, storage_path, storage_cids.state_path, storage_cids.block_number DESC)
+    INTO results;
 
     -- from the set returned above, insert public.block records at the ending_height block number
     INSERT INTO public.blocks (block_number, key, data)
     SELECT ending_height, r.mh_key, r.data
-    FROM results r;
+    FROM unnest(results) r;
 
     -- from the set returned above, insert eth.state_cids records at the ending_height block number
     -- anchoring all the records to the canonical header found at ending_height
     INSERT INTO eth.storage_cids (block_number, header_id, state_path, storage_leaf_key, cid, storage_path,
                               node_type, diff, mh_key)
     SELECT ending_height, canonical_hash, r.state_path, r.storage_leaf_key, r.cid, r.storage_path, r.node_type, false, r.mh_key
-    FROM results r
-    ON CONFLICT (storage_path, state_path, header_id) DO NOTHING;
+    FROM unnest(results) r
+    ON CONFLICT (storage_path, state_path, header_id, block_number) DO NOTHING;
 END
 $BODY$
 LANGUAGE 'plpgsql';

docker-compose.test.yml

@@ -1,14 +1,27 @@
 version: '3.2'
 
 services:
-  access-node:
+  migrations:
+    restart: on-failure
+    depends_on:
+      - ipld-eth-db
+    image: cerc-io/ipld-eth-db
+    # Build image using local context
+    build: .
+    environment:
+      DATABASE_USER: "vdbm"
+      DATABASE_NAME: "vulcanize_testing"
+      DATABASE_PASSWORD: "password"
+      DATABASE_HOSTNAME: "ipld-eth-db"
+      DATABASE_PORT: 5432
+
+  ipld-eth-db:
     image: timescale/timescaledb:latest-pg14
     restart: always
-    container_name: access-node
     command: ["postgres", "-c", "log_statement=all"]
     environment:
       POSTGRES_USER: "vdbm"
-      POSTGRES_DB: "vulcanize_testing_v4"
+      POSTGRES_DB: "vulcanize_testing"
       POSTGRES_PASSWORD: "password"
     ports:
-      - "127.0.0.1:8066:5432"
+      - "127.0.0.1:8077:5432"

docker-compose.yml

@@ -1,14 +1,26 @@
 version: '3.2'
 
 services:
+  migrations:
+    restart: on-failure
+    depends_on:
+      - ipld-eth-db
+    # Use an existing image
+    image: cerc-io/ipld-eth-db:v4.2.1-alpha-unstable
+    environment:
+      DATABASE_USER: "vdbm"
+      DATABASE_NAME: "vulcanize_testing"
+      DATABASE_PASSWORD: "password"
+      DATABASE_HOSTNAME: "ipld-eth-db"
+      DATABASE_PORT: 5432
+
   ipld-eth-db:
+    image: timescale/timescaledb:latest-pg14
     restart: always
-    image: vulcanize/ipld-eth-db
-    build: .
+    command: ["postgres", "-c", "log_statement=all"]
     environment:
       POSTGRES_USER: "vdbm"
       POSTGRES_DB: "vulcanize_testing"
       POSTGRES_PASSWORD: "password"
-      hostname: db
     ports:
-      - "127.0.0.1:8077:5432"
+      - "127.0.0.1:8077:5432"

scripts/startup_script.sh

@@ -13,6 +13,7 @@ echo "Running database migrations"
 # If the db migrations ran without err
 if [[ $? -eq 0 ]]; then
     echo "Migration process ran successfully"
+    tail -f /dev/null
 else
     echo "Could not run migrations. Are the database details correct?"
     exit 1