Use a specific tag while building migration tool (#101)

* Add block hash to primary keys in transactions, receipts and logs tables

* Add block hash in postgraphile triggers for transactions, receipts and logs tables

* Make indexes on transaction cid and mh_key non-unique

.gitignore

@@ -1 +1,2 @@
-.idea/
+.idea/
+.vscode

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.16-alpine as builder
+FROM golang:1.18-alpine as builder
 
 RUN apk --update --no-cache add make git g++ linux-headers
 
@@ -6,7 +6,8 @@ ADD . /go/src/github.com/vulcanize/ipld-eth-db
 
 # Build migration tool
 WORKDIR /go/src/github.com/pressly
-RUN git clone https://github.com/pressly/goose.git
+ARG GOOSE_VER="v3.6.1"
+RUN git clone --depth 1 --branch ${GOOSE_VER} https://github.com/pressly/goose.git
 WORKDIR /go/src/github.com/pressly/goose/cmd/goose
 RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -tags='no_sqlite3' -o goose .
 

Makefile

@@ -117,10 +117,10 @@ import:
 docker-build:
 	docker-compose -f docker-compose.test.yml build
 
-## Build docker image for migration
-.PHONY: docker-concise-migration-build
-docker-concise-migration-build:
-	docker build -t vulcanize/concise-migration-build -f ./db/Dockerfile .
+# ## Build docker image for migration
+# .PHONY: docker-concise-migration-build
+# docker-concise-migration-build:
+# 	docker build -t vulcanize/concise-migration-build -f ./db/Dockerfile .
 
 .PHONY: test-migrations
 test-migrations: $(GOOSE)

README.md

@@ -31,5 +31,5 @@ Schemas and utils for IPLD ETH Postgres database
   * Run:
 
     ```
-    docker-compose -f docker-compose.test.yml up
+    docker-compose -f docker-compose.test.yml up --build
     ```

db/Dockerfile

@@ -1,23 +0,0 @@
-FROM golang:1.16-alpine as builder
-
-RUN apk --update --no-cache add make git g++ linux-headers
-
-ADD . /go/src/github.com/vulcanize/ipld-eth-db
-
-# Build migration tool
-WORKDIR /go/src/github.com/pressly
-RUN git clone https://github.com/pressly/goose.git
-WORKDIR /go/src/github.com/pressly/goose/cmd/goose
-RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -tags='no_sqlite3' -o goose .
-
-# app container
-FROM alpine
-
-WORKDIR /app
-
-COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-db/scripts/startup_script.sh .
-
-COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose goose
-COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-db/db/migrations migrations/vulcanizedb
-
-ENTRYPOINT ["/app/startup_script.sh"]

db/migrations/00006_create_eth_transaction_cids_table.sql

@@ -11,7 +11,7 @@ CREATE TABLE IF NOT EXISTS eth.transaction_cids (
     tx_data               BYTEA,
     tx_type               INTEGER,
     value                 NUMERIC,
-    PRIMARY KEY (tx_hash, block_number)
+    PRIMARY KEY (tx_hash, header_id, block_number)
 );
 
 -- +goose Down

db/migrations/00007_create_eth_receipt_cids_table.sql

@@ -1,6 +1,7 @@
 -- +goose Up
 CREATE TABLE IF NOT EXISTS eth.receipt_cids (
     block_number          BIGINT NOT NULL,
+    header_id             VARCHAR(66) NOT NULL,
     tx_id                 VARCHAR(66) NOT NULL,
     leaf_cid              TEXT NOT NULL,
     contract              VARCHAR(66),
@@ -9,7 +10,7 @@ CREATE TABLE IF NOT EXISTS eth.receipt_cids (
     post_state            VARCHAR(66),
     post_status           INTEGER,
     log_root              VARCHAR(66),
-    PRIMARY KEY (tx_id, block_number)
+    PRIMARY KEY (tx_id, header_id, block_number)
 );
 
 -- +goose Down

db/migrations/00012_create_eth_log_cids_table.sql

@@ -1,6 +1,7 @@
 -- +goose Up
 CREATE TABLE IF NOT EXISTS eth.log_cids (
     block_number        BIGINT NOT NULL,
+    header_id           VARCHAR(66) NOT NULL,
     leaf_cid            TEXT NOT NULL,
     leaf_mh_key         TEXT NOT NULL,
     rct_id              VARCHAR(66) NOT NULL,
@@ -11,7 +12,7 @@ CREATE TABLE IF NOT EXISTS eth.log_cids (
     topic2              VARCHAR(66),
     topic3              VARCHAR(66),
     log_data            BYTEA,
-    PRIMARY KEY (rct_id, index, block_number)
+    PRIMARY KEY (rct_id, index, header_id, block_number)
 );
 
 -- +goose Down

db/migrations/00014_create_cid_indexes.sql

@@ -14,13 +14,14 @@ CREATE INDEX uncle_header_id_index ON eth.uncle_cids USING btree (header_id);
 -- transaction indexes
 CREATE INDEX tx_block_number_index ON eth.transaction_cids USING brin (block_number);
 CREATE INDEX tx_header_id_index ON eth.transaction_cids USING btree (header_id);
-CREATE UNIQUE INDEX tx_cid_index ON eth.transaction_cids USING btree (cid, block_number);
-CREATE UNIQUE INDEX tx_mh_block_number_index ON eth.transaction_cids USING btree (mh_key, block_number);
+CREATE INDEX tx_cid_index ON eth.transaction_cids USING btree (cid, block_number);
+CREATE INDEX tx_mh_block_number_index ON eth.transaction_cids USING btree (mh_key, block_number);
 CREATE INDEX tx_dst_index ON eth.transaction_cids USING btree (dst);
 CREATE INDEX tx_src_index ON eth.transaction_cids USING btree (src);
 
 -- receipt indexes
 CREATE INDEX rct_block_number_index ON eth.receipt_cids USING brin (block_number);
+CREATE INDEX rct_header_id_index ON eth.receipt_cids USING btree (header_id);
 CREATE INDEX rct_leaf_cid_index ON eth.receipt_cids USING btree (leaf_cid);
 CREATE INDEX rct_leaf_mh_block_number_index ON eth.receipt_cids USING btree (leaf_mh_key, block_number);
 CREATE INDEX rct_contract_index ON eth.receipt_cids USING btree (contract);
@@ -55,6 +56,7 @@ CREATE INDEX access_list_storage_keys_index ON eth.access_list_elements USING gi
 
 -- log indexes
 CREATE INDEX log_block_number_index ON eth.log_cids USING brin (block_number);
+CREATE INDEX log_header_id_index ON eth.log_cids USING btree (header_id);
 CREATE INDEX log_leaf_mh_block_number_index ON eth.log_cids USING btree (leaf_mh_key, block_number);
 CREATE INDEX log_cid_index ON  eth.log_cids USING btree (leaf_cid);
 CREATE INDEX log_address_index ON eth.log_cids USING btree (address);
@@ -72,6 +74,7 @@ DROP INDEX eth.log_topic0_index;
 DROP INDEX eth.log_address_index;
 DROP INDEX eth.log_cid_index;
 DROP INDEX eth.log_leaf_mh_block_number_index;
+DROP INDEX eth.log_header_id_index;
 DROP INDEX eth.log_block_number_index;
 
 -- access list indexes
@@ -106,6 +109,7 @@ DROP INDEX eth.rct_contract_hash_index;
 DROP INDEX eth.rct_contract_index;
 DROP INDEX eth.rct_leaf_mh_block_number_index;
 DROP INDEX eth.rct_leaf_cid_index;
+DROP INDEX eth.rct_header_id_index;
 DROP INDEX eth.rct_block_number_index;
 
 -- transaction indexes

db/migrations/00015_create_postgraphile_triggers.sql

@@ -22,17 +22,20 @@ BEGIN
     ELSIF (TG_TABLE_NAME = 'log_cids') THEN
          obj := json_build_array(
                     TG_TABLE_NAME,
+                    NEW.header_id,
                     NEW.rct_id,
                     NEW.index
                 );
     ELSIF (TG_TABLE_NAME = 'receipt_cids') THEN
          obj := json_build_array(
                     TG_TABLE_NAME,
+                    NEW.header_id,
                     NEW.tx_id
                 );
     ELSIF (TG_TABLE_NAME = 'transaction_cids') THEN
          obj := json_build_array(
                     TG_TABLE_NAME,
+                    NEW.header_id,
                     NEW.tx_hash
                 );
     ELSIF (TG_TABLE_NAME = 'access_list_elements') THEN

docker-compose.yml

@@ -6,7 +6,7 @@ services:
     depends_on:
       - ipld-eth-db
     # Use an existing image
-    image: vulcanize/ipld-eth-db:v4.1.1-alpha
+    image: vulcanize/ipld-eth-db:v4.2.1-alpha
     environment:
       DATABASE_USER: "vdbm"
       DATABASE_NAME: "vulcanize_testing"