Allow edited release tags to run the pipeline

update the pre- and post- batch sets with new meta schema and tables

.github/workflows/on-main.yaml

@@ -1,25 +0,0 @@
-name: Docker Compose Build
-
-on:
-  push:
-    branches:
-      - main
-
-jobs:
-  build:
-    name: Run docker build
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Get the version
-        id: vars
-        run: echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
-      - name: Run docker build
-        run: make docker-build
-      - name: Tag docker image
-        run: docker tag vulcanize/ipld-eth-db docker.pkg.github.com/vulcanize/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.sha}}
-      - name: Docker Login
-        run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
-      - name: Docker Push
-        run: docker push docker.pkg.github.com/vulcanize/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.sha}}
-

.github/workflows/publish.yaml

@@ -1,11 +1,28 @@
 name: Publish Docker image
 on:
   release:
-    types: [published]
+    types: [published, edited]
 jobs:
+  build:
+    name: Run docker build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Get the version
+        id: vars
+        run: echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
+      - name: Run docker build
+        run: make docker-build
+      - name: Tag docker image
+        run: docker tag vulcanize/ipld-eth-db docker.pkg.github.com/vulcanize/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.sha}}
+      - name: Docker Login
+        run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
+      - name: Docker Push
+        run: docker push docker.pkg.github.com/vulcanize/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.sha}}
   push_to_registries:
     name: Push Docker image to Docker Hub
     runs-on: ubuntu-latest
+    needs: build
     steps:
       - name: Get the version
         id: vars
@@ -21,4 +38,4 @@ jobs:
       - name: Tag docker image
         run: docker tag docker.pkg.github.com/vulcanize/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.sha}} vulcanize/ipld-eth-db:${{steps.vars.outputs.tag}}
       - name: Docker Push to Docker Hub
-        run: docker push vulcanize/ipld-eth-db:${{steps.vars.outputs.tag}}
+        run: docker push vulcanize/ipld-eth-db:${{steps.vars.outputs.tag}}

Makefile

@@ -8,7 +8,7 @@ BIN = $(GOPATH)/bin
 ## Migration tool
 GOOSE = $(BIN)/goose
 $(BIN)/goose:
-	GO111MODULE=off go get -u github.com/pressly/goose/cmd/goose
+	go get -u github.com/pressly/goose/cmd/goose
 
 .PHONY: installtools
 installtools: | $(GOOSE)

db/migrations/00014_create_cid_indexes.sql

@@ -1,8 +1,8 @@
 -- +goose Up
 -- header indexes
 CREATE INDEX block_number_index ON eth.header_cids USING brin (block_number);
-CREATE INDEX header_cid_index ON eth.header_cids USING btree (cid);
-CREATE INDEX header_mh_index ON eth.header_cids USING btree (mh_key);
+CREATE UNIQUE INDEX header_cid_index ON eth.header_cids USING btree (cid);
+CREATE UNIQUE INDEX header_mh_index ON eth.header_cids USING btree (mh_key);
 CREATE INDEX state_root_index ON eth.header_cids USING btree (state_root);
 CREATE INDEX timestamp_index ON eth.header_cids USING brin (timestamp);
 
@@ -11,8 +11,8 @@ CREATE INDEX uncle_header_id_index ON eth.uncle_cids USING btree (header_id);
 
 -- transaction indexes
 CREATE INDEX tx_header_id_index ON eth.transaction_cids USING btree (header_id);
-CREATE INDEX tx_cid_index ON eth.transaction_cids USING btree (cid);
-CREATE INDEX tx_mh_index ON eth.transaction_cids USING btree (mh_key);
+CREATE UNIQUE INDEX tx_cid_index ON eth.transaction_cids USING btree (cid);
+CREATE UNIQUE INDEX tx_mh_index ON eth.transaction_cids USING btree (mh_key);
 CREATE INDEX tx_dst_index ON eth.transaction_cids USING btree (dst);
 CREATE INDEX tx_src_index ON eth.transaction_cids USING btree (src);
 

db/migrations/00019_create_eth_meta_schema.sql

@@ -0,0 +1,5 @@
+-- +goose Up
+CREATE SCHEMA eth_meta;
+
+-- +goose Down
+DROP SCHEMA eth_meta;

db/migrations/00020_create_watched_addresses_table.sql

@@ -0,0 +1,10 @@
+-- +goose Up
+CREATE TABLE eth_meta.watched_addresses (
+    address                 VARCHAR(66) PRIMARY KEY,
+    created_at              BIGINT NOT NULL,
+    watched_at              BIGINT NOT NULL,
+    last_filled_at          BIGINT NOT NULL DEFAULT 0
+);
+
+-- +goose Down
+DROP TABLE eth_meta.watched_addresses;

db/migrations/00021_create_known_gaps_table.sql

@@ -0,0 +1,10 @@
+-- +goose Up
+CREATE TABLE eth_meta.known_gaps (
+    starting_block_number bigint PRIMARY KEY,
+    ending_block_number bigint,
+    checked_out boolean,
+    processing_key bigint
+);
+
+-- +goose Down
+DROP TABLE eth_meta.known_gaps;

db/post_batch_processing_migrations/00023_create_pk_constraints.sql

@@ -30,32 +30,32 @@ ALTER TABLE eth.state_accounts
 ADD CONSTRAINT pk_eth_state_accounts PRIMARY KEY (header_id, state_path);
 
 -- +goose Down
-ALTER TABLE public.nodes
-DROP CONSTRAINT pk_public_nodes;
-
-ALTER TABLE eth.header_cids
-DROP CONSTRAINT pk_eth_header_cids;
-
-ALTER TABLE eth.uncle_cids
-DROP CONSTRAINT pk_eth_uncle_cids;
-
-ALTER TABLE eth.transaction_cids
-DROP CONSTRAINT pk_eth_transaction_cids;
-
-ALTER TABLE eth.receipt_cids
-DROP CONSTRAINT pk_eth_receipt_cids;
-
-ALTER TABLE eth.access_list_elements
-DROP CONSTRAINT pk_eth_access_list_elements;
-
-ALTER TABLE eth.log_cids
-DROP CONSTRAINT pk_eth_log_cids;
-
-ALTER TABLE eth.state_cids
-DROP CONSTRAINT pk_eth_state_cids;
+ALTER TABLE eth.state_accounts
+DROP CONSTRAINT pk_eth_state_accounts;
 
 ALTER TABLE eth.storage_cids
 DROP CONSTRAINT pk_eth_storage_cids;
 
-ALTER TABLE eth.state_accounts
-DROP CONSTRAINT pk_eth_state_accounts;
+ALTER TABLE eth.state_cids
+DROP CONSTRAINT pk_eth_state_cids;
+
+ALTER TABLE eth.log_cids
+DROP CONSTRAINT pk_eth_log_cids;
+
+ALTER TABLE eth.access_list_elements
+DROP CONSTRAINT pk_eth_access_list_elements;
+
+ALTER TABLE eth.receipt_cids
+DROP CONSTRAINT pk_eth_receipt_cids;
+
+ALTER TABLE eth.transaction_cids
+DROP CONSTRAINT pk_eth_transaction_cids;
+
+ALTER TABLE eth.uncle_cids
+DROP CONSTRAINT pk_eth_uncle_cids;
+
+ALTER TABLE eth.header_cids
+DROP CONSTRAINT pk_eth_header_cids;
+
+ALTER TABLE public.nodes
+DROP CONSTRAINT pk_public_nodes;

db/post_batch_processing_migrations/00024_create_eth_meta_schema.sql

@@ -0,0 +1,5 @@
+-- +goose Up
+CREATE SCHEMA eth_meta;
+
+-- +goose Down
+DROP SCHEMA eth_meta;

db/post_batch_processing_migrations/00025_create_watched_addresses_table.sql

@@ -0,0 +1,10 @@
+-- +goose Up
+CREATE TABLE eth_meta.watched_addresses (
+    address                 VARCHAR(66) PRIMARY KEY,
+    created_at              BIGINT NOT NULL,
+    watched_at              BIGINT NOT NULL,
+    last_filled_at          BIGINT NOT NULL DEFAULT 0
+);
+
+-- +goose Down
+DROP TABLE eth_meta.watched_addresses;

db/post_batch_processing_migrations/00026_create_known_gaps_table.sql

@@ -0,0 +1,10 @@
+-- +goose Up
+CREATE TABLE eth_meta.known_gaps (
+     starting_block_number bigint PRIMARY KEY,
+     ending_block_number bigint,
+     checked_out boolean,
+     processing_key bigint
+);
+
+-- +goose Down
+DROP TABLE eth_meta.known_gaps;

db/post_batch_processing_migrations/00030_create_cid_indexes.sql

@@ -1,8 +1,8 @@
 -- +goose Up
 -- header indexes
 CREATE INDEX block_number_index ON eth.header_cids USING brin (block_number);
-CREATE INDEX header_cid_index ON eth.header_cids USING btree (cid);
-CREATE INDEX header_mh_index ON eth.header_cids USING btree (mh_key);
+CREATE UNIQUE INDEX header_cid_index ON eth.header_cids USING btree (cid);
+CREATE UNIQUE INDEX header_mh_index ON eth.header_cids USING btree (mh_key);
 CREATE INDEX state_root_index ON eth.header_cids USING btree (state_root);
 CREATE INDEX timestamp_index ON eth.header_cids USING brin (timestamp);
 
@@ -11,8 +11,8 @@ CREATE INDEX uncle_header_id_index ON eth.uncle_cids USING btree (header_id);
 
 -- transaction indexes
 CREATE INDEX tx_header_id_index ON eth.transaction_cids USING btree (header_id);
-CREATE INDEX tx_cid_index ON eth.transaction_cids USING btree (cid);
-CREATE INDEX tx_mh_index ON eth.transaction_cids USING btree (mh_key);
+CREATE UNIQUE INDEX tx_cid_index ON eth.transaction_cids USING btree (cid);
+CREATE UNIQUE INDEX tx_mh_index ON eth.transaction_cids USING btree (mh_key);
 CREATE INDEX tx_dst_index ON eth.transaction_cids USING btree (dst);
 CREATE INDEX tx_src_index ON eth.transaction_cids USING btree (src);
 

schema.sql

@@ -23,6 +23,13 @@ SET row_security = off;
 CREATE SCHEMA eth;
 
 
+--
+-- Name: eth_meta; Type: SCHEMA; Schema: -; Owner: -
+--
+
+CREATE SCHEMA eth_meta;
+
+
 SET default_tablespace = '';
 
 SET default_table_access_method = heap;
@@ -402,6 +409,30 @@ CREATE TABLE eth.uncle_cids (
 );
 
 
+--
+-- Name: known_gaps; Type: TABLE; Schema: eth_meta; Owner: -
+--
+
+CREATE TABLE eth_meta.known_gaps (
+    starting_block_number bigint NOT NULL,
+    ending_block_number bigint,
+    checked_out boolean,
+    processing_key bigint
+);
+
+
+--
+-- Name: watched_addresses; Type: TABLE; Schema: eth_meta; Owner: -
+--
+
+CREATE TABLE eth_meta.watched_addresses (
+    address character varying(66) NOT NULL,
+    created_at bigint NOT NULL,
+    watched_at bigint NOT NULL,
+    last_filled_at bigint DEFAULT 0 NOT NULL
+);
+
+
 --
 -- Name: blocks; Type: TABLE; Schema: public; Owner: -
 --
@@ -562,6 +593,22 @@ ALTER TABLE ONLY eth.uncle_cids
     ADD CONSTRAINT uncle_cids_pkey PRIMARY KEY (block_hash);
 
 
+--
+-- Name: known_gaps known_gaps_pkey; Type: CONSTRAINT; Schema: eth_meta; Owner: -
+--
+
+ALTER TABLE ONLY eth_meta.known_gaps
+    ADD CONSTRAINT known_gaps_pkey PRIMARY KEY (starting_block_number);
+
+
+--
+-- Name: watched_addresses watched_addresses_pkey; Type: CONSTRAINT; Schema: eth_meta; Owner: -
+--
+
+ALTER TABLE ONLY eth_meta.watched_addresses
+    ADD CONSTRAINT watched_addresses_pkey PRIMARY KEY (address);
+
+
 --
 -- Name: blocks blocks_pkey; Type: CONSTRAINT; Schema: public; Owner: -
 --
@@ -626,14 +673,14 @@ CREATE INDEX block_number_index ON eth.header_cids USING brin (block_number);
 -- Name: header_cid_index; Type: INDEX; Schema: eth; Owner: -
 --
 
-CREATE INDEX header_cid_index ON eth.header_cids USING btree (cid);
+CREATE UNIQUE INDEX header_cid_index ON eth.header_cids USING btree (cid);
 
 
 --
 -- Name: header_mh_index; Type: INDEX; Schema: eth; Owner: -
 --
 
-CREATE INDEX header_mh_index ON eth.header_cids USING btree (mh_key);
+CREATE UNIQUE INDEX header_mh_index ON eth.header_cids USING btree (mh_key);
 
 
 --
@@ -815,7 +862,7 @@ CREATE INDEX timestamp_index ON eth.header_cids USING brin ("timestamp");
 -- Name: tx_cid_index; Type: INDEX; Schema: eth; Owner: -
 --
 
-CREATE INDEX tx_cid_index ON eth.transaction_cids USING btree (cid);
+CREATE UNIQUE INDEX tx_cid_index ON eth.transaction_cids USING btree (cid);
 
 
 --
@@ -836,7 +883,7 @@ CREATE INDEX tx_header_id_index ON eth.transaction_cids USING btree (header_id);
 -- Name: tx_mh_index; Type: INDEX; Schema: eth; Owner: -
 --
 
-CREATE INDEX tx_mh_index ON eth.transaction_cids USING btree (mh_key);
+CREATE UNIQUE INDEX tx_mh_index ON eth.transaction_cids USING btree (mh_key);
 
 
 --