Upgrade to v5 schema

Now uses:
* ipld direct_by_leaf StateDB for basic queries
* trie_by_cid StateDB for trie slice and proof queries

Also:
* vulcanize => cerc refactor
* Backend method to close dbs
* state tests are in multiple packages, to allow separate ginkgo suites
* removes gap-filler module
* integration tests and github workflows
* run stack-orchestrator for testnet
* fix various issues with tests, hardhat server, dockerfile
* fix cmd flags / env vars
* fix flaky tests and clean up code
* remove unused code, scripts
* remove outdated docs
* update version

pkg/shared/functions.go

@@ -17,11 +17,8 @@
 package shared
 
 import (
-	"github.com/cerc-io/ipld-eth-server/v4/pkg/log"
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/log"
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ipfs/go-cid"
-	blockstore "github.com/ipfs/go-ipfs-blockstore"
-	dshelp "github.com/ipfs/go-ipfs-ds-help"
 	"github.com/jmoiron/sqlx"
 )
 
@@ -47,33 +44,3 @@ func Rollback(tx *sqlx.Tx) {
 		log.Error(err)
 	}
 }
-
-// FetchIPLDByMhKeyAndBlockNumber is used to retrieve an ipld from Postgres blockstore with the provided tx, mhkey string and blockNumber
-func FetchIPLDByMhKeyAndBlockNumber(tx *sqlx.Tx, mhKey string, blockNumber uint64) ([]byte, error) {
-	pgStr := `SELECT data FROM ipld.blocks WHERE key = $1 AND block_number = $2`
-	var block []byte
-	return block, tx.Get(&block, pgStr, mhKey, blockNumber)
-}
-
-// FetchIPLD is used to retrieve an IPLD from Postgres mhkey and blockNumber
-func FetchIPLD(db *sqlx.DB, mhKey string, blockNumber uint64) ([]byte, error) {
-	pgStr := `SELECT data FROM ipld.blocks WHERE key = $1 AND block_number = $2`
-	var block []byte
-	return block, db.Get(&block, pgStr, mhKey, blockNumber)
-}
-
-// MultihashKeyFromCID converts a cid into a blockstore-prefixed multihash db key string
-func MultihashKeyFromCID(c cid.Cid) string {
-	dbKey := dshelp.MultihashToDsKey(c.Hash())
-	return blockstore.BlockPrefix.String() + dbKey.String()
-}
-
-// MultihashKeyFromCIDString converts a cid string into a blockstore-prefixed multihash db key string
-func MultihashKeyFromCIDString(c string) (string, error) {
-	dc, err := cid.Decode(c)
-	if err != nil {
-		return "", err
-	}
-	dbKey := dshelp.MultihashToDsKey(dc.Hash())
-	return blockstore.BlockPrefix.String() + dbKey.String(), nil
-}

pkg/shared/test_helpers.go

@@ -19,8 +19,6 @@ package shared
 import (
 	"bytes"
 	"context"
-	"os"
-	"strconv"
 
 	. "github.com/onsi/gomega"
 
@@ -46,7 +44,8 @@ func IPLDsContainBytes(iplds []models.IPLDModel, b []byte) bool {
 
 // SetupDB is use to setup a db for watcher tests
 func SetupDB() *sqlx.DB {
-	config := getTestDBConfig()
+	config, err := postgres.TestConfig.WithEnv()
+	Expect(err).NotTo(HaveOccurred())
 
 	db, err := NewDB(config.DbConnectionString(), config)
 	Expect(err).NotTo(HaveOccurred())
@@ -60,6 +59,8 @@ func TearDownDB(db *sqlx.DB) {
 	Expect(err).NotTo(HaveOccurred())
 	_, err = tx.Exec(`DELETE FROM nodes`)
 	Expect(err).NotTo(HaveOccurred())
+	_, err = tx.Exec(`DELETE FROM ipld.blocks`)
+	Expect(err).NotTo(HaveOccurred())
 	_, err = tx.Exec(`DELETE FROM eth.header_cids`)
 	Expect(err).NotTo(HaveOccurred())
 	_, err = tx.Exec(`DELETE FROM eth.uncle_cids`)
@@ -72,12 +73,6 @@ func TearDownDB(db *sqlx.DB) {
 	Expect(err).NotTo(HaveOccurred())
 	_, err = tx.Exec(`DELETE FROM eth.storage_cids`)
 	Expect(err).NotTo(HaveOccurred())
-	_, err = tx.Exec(`DELETE FROM eth.state_accounts`)
-	Expect(err).NotTo(HaveOccurred())
-	_, err = tx.Exec(`DELETE FROM eth.access_list_elements`)
-	Expect(err).NotTo(HaveOccurred())
-	_, err = tx.Exec(`DELETE FROM blocks`)
-	Expect(err).NotTo(HaveOccurred())
 	_, err = tx.Exec(`DELETE FROM eth.log_cids`)
 	Expect(err).NotTo(HaveOccurred())
 	_, err = tx.Exec(`DELETE FROM eth_meta.watched_addresses`)
@@ -96,20 +91,10 @@ func SetupTestStateDiffIndexer(ctx context.Context, chainConfig *params.ChainCon
 		ChainID:      params.TestChainConfig.ChainID.Uint64(),
 	}
 
-	_, stateDiffIndexer, err := indexer.NewStateDiffIndexer(ctx, chainConfig, testInfo, getTestDBConfig())
+	dbconfig, err := postgres.TestConfig.WithEnv()
+	Expect(err).NotTo(HaveOccurred())
+	_, stateDiffIndexer, err := indexer.NewStateDiffIndexer(ctx, chainConfig, testInfo, dbconfig)
 	Expect(err).NotTo(HaveOccurred())
 
 	return stateDiffIndexer
 }
-
-func getTestDBConfig() postgres.Config {
-	port, _ := strconv.Atoi(os.Getenv("DATABASE_PORT"))
-	return postgres.Config{
-		Hostname:     os.Getenv("DATABASE_HOSTNAME"),
-		DatabaseName: os.Getenv("DATABASE_NAME"),
-		Username:     os.Getenv("DATABASE_USER"),
-		Password:     os.Getenv("DATABASE_PASSWORD"),
-		Port:         port,
-		Driver:       postgres.SQLX,
-	}
-}