rename blocks table to eth_blocks so that we don't clash with the ipfs blocks table; dockerfile and startup_script for the seed node

db/migrations/00001_create_blocks_table.sql

@@ -1,5 +1,5 @@
 -- +goose Up
-CREATE TABLE public.blocks (
+CREATE TABLE public.eth_blocks (
   id            SERIAL PRIMARY KEY,
   difficulty    BIGINT,
   extra_data    VARCHAR,
@@ -20,4 +20,4 @@ CREATE TABLE public.blocks (
 
 
 -- +goose Down
-DROP TABLE public.blocks;
+DROP TABLE public.eth_blocks;

db/migrations/00002_create_full_sync_transactions_table.sql

@@ -1,7 +1,7 @@
 -- +goose Up
 CREATE TABLE full_sync_transactions (
   id          SERIAL PRIMARY KEY,
-  block_id    INTEGER NOT NULL REFERENCES blocks(id) ON DELETE CASCADE,
+  block_id    INTEGER NOT NULL REFERENCES eth_blocks(id) ON DELETE CASCADE,
   gas_limit    NUMERIC,
   gas_price    NUMERIC,
   hash        VARCHAR(66),

db/migrations/00003_add_block_index_to_blocks.sql

@@ -1,5 +1,5 @@
 -- +goose Up
-CREATE INDEX number_index ON blocks (number);
+CREATE INDEX number_index ON eth_blocks (number);
 
 
 -- +goose Down

db/migrations/00006_add_node_fk_to_blocks.sql

@@ -1,5 +1,5 @@
 -- +goose Up
-ALTER TABLE blocks
+ALTER TABLE eth_blocks
   ADD COLUMN node_id INTEGER NOT NULL,
   ADD CONSTRAINT node_fk
 FOREIGN KEY (node_id)
@@ -7,5 +7,5 @@ REFERENCES nodes (id)
 ON DELETE CASCADE;
 
 -- +goose Down
-ALTER TABLE blocks
+ALTER TABLE eth_blocks
   DROP COLUMN node_id;

db/migrations/00008_add_node_block_number_unique_constraint_to_blocks.sql

@@ -1,7 +1,7 @@
 -- +goose Up
-ALTER TABLE blocks
+ALTER TABLE eth_blocks
   ADD CONSTRAINT node_id_block_number_uc UNIQUE (number, node_id);
 
 -- +goose Down
-ALTER TABLE blocks
+ALTER TABLE eth_blocks
   DROP CONSTRAINT node_id_block_number_uc;

db/migrations/00010_add_node_id_index_to_blocks.sql

@@ -1,5 +1,5 @@
 -- +goose Up
-CREATE INDEX node_id_index ON blocks (node_id);
+CREATE INDEX node_id_index ON eth_blocks (node_id);
 
 -- +goose Down
 DROP INDEX node_id_index;

db/migrations/00020_rename_node_table.sql

@@ -7,14 +7,14 @@ ALTER TABLE public.eth_nodes DROP CONSTRAINT node_uc;
 ALTER TABLE public.eth_nodes
   ADD CONSTRAINT eth_node_uc UNIQUE (genesis_block, network_id, eth_node_id);
 
-ALTER TABLE public.blocks RENAME COLUMN node_id TO eth_node_id;
+ALTER TABLE public.eth_blocks RENAME COLUMN node_id TO eth_node_id;
 
-ALTER TABLE public.blocks DROP CONSTRAINT node_id_block_number_uc;
+ALTER TABLE public.eth_blocks DROP CONSTRAINT node_id_block_number_uc;
-ALTER TABLE public.blocks
+ALTER TABLE public.eth_blocks
   ADD CONSTRAINT eth_node_id_block_number_uc UNIQUE (number, eth_node_id);
 
-ALTER TABLE public.blocks DROP CONSTRAINT node_fk;
+ALTER TABLE public.eth_blocks DROP CONSTRAINT node_fk;
-ALTER TABLE public.blocks
+ALTER TABLE public.eth_blocks
   ADD CONSTRAINT node_fk
 FOREIGN KEY (eth_node_id) REFERENCES eth_nodes (id) ON DELETE CASCADE;
 
@@ -31,13 +31,13 @@ ALTER TABLE public.nodes
 ALTER TABLE public.nodes
   ADD CONSTRAINT node_uc UNIQUE (genesis_block, network_id, node_id);
 
-ALTER TABLE public.blocks RENAME COLUMN eth_node_id TO node_id;
+ALTER TABLE public.eth_blocks RENAME COLUMN eth_node_id TO node_id;
 
-ALTER TABLE public.blocks DROP CONSTRAINT eth_node_id_block_number_uc;
+ALTER TABLE public.eth_blocks DROP CONSTRAINT eth_node_id_block_number_uc;
-ALTER TABLE public.blocks
+ALTER TABLE public.eth_blocks
   ADD CONSTRAINT node_id_block_number_uc UNIQUE (number, node_id);
 
-ALTER TABLE public.blocks DROP CONSTRAINT node_fk;
+ALTER TABLE public.eth_blocks DROP CONSTRAINT node_fk;
-ALTER TABLE public.blocks
+ALTER TABLE public.eth_blocks
   ADD CONSTRAINT node_fk
 FOREIGN KEY (node_id) REFERENCES nodes (id) ON DELETE CASCADE;

db/migrations/00021_associate_receipts_with_blocks.sql

@@ -11,9 +11,9 @@ ALTER TABLE full_sync_receipts
   ALTER COLUMN block_id SET NOT NULL;
 
 ALTER TABLE full_sync_receipts
-  ADD CONSTRAINT blocks_fk
+  ADD CONSTRAINT eth_blocks_fk
 FOREIGN KEY (block_id)
-REFERENCES blocks (id)
+REFERENCES eth_blocks (id)
 ON DELETE CASCADE;
 
 ALTER TABLE full_sync_receipts

db/migrations/00022_add_eth_node_fingerprint_to_blocks.sql

@@ -1,16 +1,16 @@
 -- +goose Up
-ALTER TABLE blocks
+ALTER TABLE eth_blocks
   ADD COLUMN eth_node_fingerprint VARCHAR(128);
 
-UPDATE blocks
+UPDATE eth_blocks
   SET eth_node_fingerprint = (
-    SELECT eth_node_id FROM eth_nodes WHERE eth_nodes.id = blocks.eth_node_id
+    SELECT eth_node_id FROM eth_nodes WHERE eth_nodes.id = eth_blocks.eth_node_id
   );
 
-ALTER TABLE blocks
+ALTER TABLE eth_blocks
   ALTER COLUMN eth_node_fingerprint SET NOT NULL;
 
 
 -- +goose Down
-ALTER TABLE blocks
+ALTER TABLE eth_blocks
   DROP COLUMN eth_node_fingerprint;

db/migrations/00028_create_uncles_table.sql

@@ -2,7 +2,7 @@
 CREATE TABLE public.uncles (
   id                    SERIAL PRIMARY KEY,
   hash                  VARCHAR(66) NOT NULL,
-  block_id              INTEGER NOT NULL REFERENCES blocks (id) ON DELETE CASCADE,
+  block_id              INTEGER NOT NULL REFERENCES eth_blocks (id) ON DELETE CASCADE,
   reward                NUMERIC NOT NULL,
   miner                 VARCHAR(42) NOT NULL,
   raw                   JSONB,

db/schema.sql

@@ -1175,4 +1175,3 @@ ALTER TABLE ONLY public.uncles
 --
 -- PostgreSQL database dump complete
 --
-

dockerfiles/seed_node/Dockerfile

@@ -0,0 +1,61 @@
+FROM golang:alpine as builder
+
+RUN apk --update --no-cache add make git g++
+# DEBUG
+RUN apk add busybox-extras
+
+# Build statically linked vDB binary (wonky path because of Dep)
+WORKDIR /go/src/github.com/vulcanize/vulcanizedb
+ADD . .
+RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' .
+
+# Build statically linked IPFS binary (statically linked and wonky path because we need to use custom fork)
+WORKDIR /go/src/github.com/ipfs/go-ipfs
+ADD . .
+RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' .
+
+# Build statically linked Geth binary (statically linked and wonky path because we need to use custom fork)
+WORKDIR /go/src/github.com/ethereum/go-ethereum
+ADD . .
+RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' .
+
+# Build migration tool
+RUN go get -u -d github.com/pressly/goose/cmd/goose
+WORKDIR /go/src/github.com/pressly/goose/cmd/goose
+RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -tags='no_mysql no_sqlite' -o goose
+
+WORKDIR /go/src/github.com/vulcanize/vulcanizedb
+
+# app container
+FROM alpine
+WORKDIR /app
+
+ARG USER
+ARG config_file=environments/example.toml
+ARG vdb_command=syncPublishScreenAndServe
+ARG vdb_pg_connect="postgres://$USER@/vulcanize_public?sslmode=disable"
+ARG vdb_dbname="vulcanize_public"
+
+# setup environment
+ENV VDB_COMMAND="$vdb_command"
+ENV VDB_PG_CONNECT="$vdb_pg_connect"
+
+RUN adduser -Du 5000 $USER
+USER $USER
+
+# chown first so dir is writable
+# note: using $USER is merged, but not in the stable release yet
+COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/vulcanizedb/$config_file config.toml
+COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/vulcanizedb/dockerfiles/seed_node/startup_script.sh .
+
+# keep binaries immutable
+COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/vulcanizedb vulcanizedb
+COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose goose
+COPY --from=builder /go/src/github.com/vulcanize/vulcanizedb/db/migrations migrations/vulcanizedb
+COPY --from=builder /go/src/github.com/ipfs/go-ipfs ipfs
+COPY --from=builder /go/src/github.com/ethereum/go-ethereum geth
+
+# DEBUG
+COPY --from=builder /usr/bin/telnet /bin/telnet
+
+CMD ["./startup_script.sh"]

dockerfiles/seed_node/startup_script.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# Runs the migrations and starts the syncPublishScreenAndServe service
+
+# Exit if the variable tests fail
+set -e
+
+# Check the database variables are set
+test $DATABASE_NAME
+test $DATABASE_HOSTNAME
+test $DATABASE_PORT
+test $DATABASE_USER
+test $DATABASE_PASSWORD
+
+# Export our database variables so that the IPFS Postgres plugin can use them
+export IPFS_PGHOST=$DATABASE_HOSTNAME
+export IPFS_PGUSER=$DATABASE_USER
+export IPFS_PGDATABASE=$DATABASE_NAME
+export IPFS_PGPORT=$DATABASE_PORT
+export IPFS_PGPASSWORD=$DATABASE_PASSWORD
+
+# Construct the connection string for postgres
+CONNECT_STRING=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable
+echo "Connecting with: $CONNECT_STRING"
+
+set +e
+
+# Run the DB migrations
+./goose postgres "$CONNECT_STRING" up
+if [ $? -eq 0 ]; then
+  # Fire up the services
+  ipfs ipfs init --profile=postgresds
+  geth --statediff --statediff.streamblock --ws --syncmode=full
+  ./vulcanizedb syncPublishScreenAndServe --config environments/seedNodeStaging.toml &
+else
+  echo "Could not run migrations. Are the database details correct?"
+fi
+wait

pkg/contract_watcher/full/retriever/block_retriever.go

@@ -64,7 +64,7 @@ func (r *blockRetriever) retrieveFirstBlockFromReceipts(contractAddr string) (in
 	}
 	err := r.db.Get(
 		&firstBlock,
-		`SELECT number FROM blocks
+		`SELECT number FROM eth_blocks
 		       WHERE id = (SELECT block_id FROM full_sync_receipts
                            WHERE contract_address_id = $1
 		                   ORDER BY block_id ASC
@@ -92,7 +92,7 @@ func (r *blockRetriever) RetrieveMostRecentBlock() (int64, error) {
 	var lastBlock int64
 	err := r.db.Get(
 		&lastBlock,
-		"SELECT number FROM blocks ORDER BY number DESC LIMIT 1",
+		"SELECT number FROM eth_blocks ORDER BY number DESC LIMIT 1",
 	)
 
 	return lastBlock, err

pkg/contract_watcher/shared/helpers/test_helpers/database.go

@@ -265,7 +265,7 @@ func TearDown(db *postgres.DB) {
 	_, err = tx.Exec(`DELETE FROM addresses`)
 	Expect(err).NotTo(HaveOccurred())
 
-	_, err = tx.Exec(`DELETE FROM blocks`)
+	_, err = tx.Exec(`DELETE FROM eth_blocks`)
 	Expect(err).NotTo(HaveOccurred())
 
 	_, err = tx.Exec(`DELETE FROM headers`)

pkg/datastore/postgres/repositories/block_repository.go

@@ -19,8 +19,10 @@ package repositories
 import (
 	"database/sql"
 	"errors"
+
 	"github.com/jmoiron/sqlx"
 	"github.com/sirupsen/logrus"
+
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
@@ -43,7 +45,7 @@ func NewBlockRepository(database *postgres.DB) *BlockRepository {
 func (blockRepository BlockRepository) SetBlocksStatus(chainHead int64) error {
 	cutoff := chainHead - blocksFromHeadBeforeFinal
 	_, err := blockRepository.database.Exec(`
-                  UPDATE blocks SET is_final = TRUE
+                  UPDATE eth_blocks SET is_final = TRUE
                   WHERE is_final = FALSE AND number < $1`,
 		cutoff)
 
@@ -74,7 +76,7 @@ func (blockRepository BlockRepository) MissingBlockNumbers(startingBlockNumber i
           FROM (
               SELECT generate_series($1::INT, $2::INT) AS all_block_numbers) series
           WHERE all_block_numbers NOT IN (
-		  	  SELECT number FROM blocks WHERE eth_node_fingerprint = $3
+			  SELECT number FROM eth_blocks WHERE eth_node_fingerprint = $3
 		  ) `,
 		startingBlockNumber,
 		highestBlockNumber, nodeID)
@@ -102,7 +104,7 @@ func (blockRepository BlockRepository) GetBlock(blockNumber int64) (core.Block,
                        extra_data,
                        reward,
                        uncles_reward
-               FROM blocks
+               FROM eth_blocks
                WHERE eth_node_id = $1 AND number = $2`, blockRepository.database.NodeID, blockNumber)
 	savedBlock, err := blockRepository.loadBlock(blockRows)
 	if err != nil {
@@ -124,7 +126,7 @@ func (blockRepository BlockRepository) insertBlock(block core.Block) (int64, err
 		return 0, postgres.ErrBeginTransactionFailed(beginErr)
 	}
 	insertBlockErr := tx.QueryRow(
-		`INSERT INTO blocks
+		`INSERT INTO eth_blocks
                 (eth_node_id, number, gas_limit, gas_used, time, difficulty, hash, nonce, parent_hash, size, uncle_hash, is_final, miner, extra_data, reward, uncles_reward, eth_node_fingerprint)
                 VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)
                 RETURNING id `,
@@ -260,7 +262,7 @@ func (blockRepository BlockRepository) getBlockHash(block core.Block) (string, b
 	// TODO: handle possible error
 	blockRepository.database.Get(&retrievedBlockHash,
 		`SELECT hash
-               FROM blocks
+               FROM eth_blocks
                WHERE number = $1 AND eth_node_id = $2`,
 		block.Number, blockRepository.database.NodeID)
 	return retrievedBlockHash, blockExists(retrievedBlockHash)
@@ -287,7 +289,7 @@ func blockExists(retrievedBlockHash string) bool {
 
 func (blockRepository BlockRepository) removeBlock(blockNumber int64) error {
 	_, err := blockRepository.database.Exec(
-		`DELETE FROM blocks WHERE number=$1 AND eth_node_id=$2`,
+		`DELETE FROM eth_blocks WHERE number=$1 AND eth_node_id=$2`,
 		blockNumber, blockRepository.database.NodeID)
 	if err != nil {
 		return postgres.ErrDBDeleteFailed(err)

pkg/ipfs/retreiver.go

@@ -67,6 +67,8 @@ func (ecr *EthCIDRetriever) RetrieveCIDs(streamFilters config.Subscription) ([]C
 	}
 	log.Debug("backfill starting block:", streamFilters.StartingBlock)
 	log.Debug("backfill ending block:", endingBlock)
+	// THIS IS SUPER EXPENSIVE HAVING TO CYCLE THROUGH EACH BLOCK, NEED BETTER WAY TO FETCH CIDS
+	// WHILE STILL MAINTAINING RELATION INFO ABOUT WHAT BLOCK THE CIDS BELONG TO
 	for i := streamFilters.StartingBlock; i <= endingBlock; i++ {
 		cw := CidWrapper{}
 		if !streamFilters.HeaderFilter.Off {

test_config/test_config.go

@@ -88,6 +88,7 @@ func NewTestDB(node core.Node) *postgres.DB {
 func CleanTestDB(db *postgres.DB) {
 	db.MustExec("DELETE FROM addresses")
 	db.MustExec("DELETE FROM blocks")
+	db.MustExec("DELETE FROM eth_blocks")
 	db.MustExec("DELETE FROM checked_headers")
 	// can't delete from eth_nodes since this function is called after the required eth_node is persisted
 	db.MustExec("DELETE FROM full_sync_logs")