From e015d4333d07deb6a4f540a6397c86bdff285c50 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Wed, 22 Mar 2023 12:53:33 +0800 Subject: [PATCH 1/4] update docs --- postgres/doc.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/postgres/doc.md b/postgres/doc.md index 5a4fc75..a1bfba7 100644 --- a/postgres/doc.md +++ b/postgres/doc.md @@ -18,7 +18,7 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/trie" "github.com/jmoiron/sqlx" - "github.com/vulcanize/ipfs-ethdb/v5/postgres" + "github.com/vulcanize/ipfs-ethdb/v5/postgres/v1" ) func main() { @@ -31,7 +31,11 @@ func main() { trieNodeIterator := t.NodeIterator([]byte{}) // do stuff with trie node iterator - database := pgipfsethdb.NewDatabase(db) + database := pgipfsethdb.NewDatabase(db, pgipfsethdb.CacheConfig{ + Name: "db", + Size: 3000000, // 3MB + ExpiryDuration: time.Hour, + }) stateDatabase := state.NewDatabase(database) stateDB, _ := state.New(common.Hash{}, stateDatabase, nil) stateDBNodeIterator := state.NewNodeIterator(stateDB) From 4695984e1db747ea0442e3f0375417c2e2be6cf9 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Thu, 30 Mar 2023 23:38:23 +0800 Subject: [PATCH 2/4] vulc => cerc --- README.md | 6 +++--- postgres/doc.md | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index c08a2c1..3b38abc 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ ## ipfs-ethdb -[![Go Report Card](https://goreportcard.com/badge/github.com/vulcanize/ipfs-ethdb)](https://goreportcard.com/report/github.com/vulcanize/ipfs-ethdb) +[![Go Report Card](https://goreportcard.com/badge/github.com/cerc/ipfs-ethdb)](https://goreportcard.com/report/github.com/cerc-io/ipfs-ethdb) > go-ethereum ethdb interfaces for Ethereum state data stored in IPFS @@ -11,7 +11,7 @@ interfacing with a state database. These interfaces are used to build higher-lev which are used to perform the bulk of state related needs. Ethereum data can be stored on IPFS, standard codecs for Etheruem data are defined in the [go-cid](https://github.com/ipfs/go-cid) library. -Using our [statediffing geth client](https://github.com/vulcanize/go-ethereum/releases/tag/v1.9.11-statediff-0.0.2) it is feasible to extract every single +Using our [statediffing geth client](https://github.com/cerc-io/go-ethereum/releases/tag/v1.9.11-statediff-0.0.2) it is feasible to extract every single state and storage node and publish it to IPFS. Geth stores state data in leveldb as key-value pairs between the keccak256 hash of the rlp-encoded object and the rlp-encoded object. @@ -50,7 +50,7 @@ import ( "github.com/ipfs/go-ipfs/core" "github.com/ipfs/go-ipfs/repo/fsrepo" "github.com/jmoiron/sqlx" - "github.com/vulcanize/ipfs-ethdb/v5" + "github.com/cerc-io/ipfs-ethdb/v5" ) func main() { diff --git a/postgres/doc.md b/postgres/doc.md index a1bfba7..b778f95 100644 --- a/postgres/doc.md +++ b/postgres/doc.md @@ -1,6 +1,6 @@ ## ipfs-ethdb -IPFS has been [extended](https://github.com/vulcanize/go-ipfs/releases/tag/v0.4.22-alpha) to [use Postgres](https://github.com/vulcanize/go-ipfs-config/releases/tag/v0.0.8-alpha) as a backing [datastore](https://github.com/ipfs/go-ds-sql/tree/master/postgres). +IPFS has been [extended](https://github.com/cerc-io/go-ipfs/releases/tag/v0.4.22-alpha) to [use Postgres](https://github.com/cerc-io/go-ipfs-config/releases/tag/v0.0.8-alpha) as a backing [datastore](https://github.com/ipfs/go-ds-sql/tree/master/postgres). Interfacing directly with the IPFS-backing Postgres database has some advantages over using the blockservice interface. Namely, batching of IPFS writes with other Postgres writes and avoiding lock contention on the ipfs repository (lockfile located at the `IPFS_PATH`). The downside is that we forgo the block-exchange capabilities of the blockservice, and are only able to fetch data contained in the local datastore. @@ -18,7 +18,7 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/trie" "github.com/jmoiron/sqlx" - "github.com/vulcanize/ipfs-ethdb/v5/postgres/v1" + "github.com/cerc-io/ipfs-ethdb/v5/postgres/v1" ) func main() { From 4a4ae2f2ea3818cbd4ae462e9b2c5d95896d31da Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Wed, 22 Mar 2023 12:53:42 +0800 Subject: [PATCH 3/4] use statediff indexer config; other nits --- postgres/shared/util.go | 26 ++++++++++++++++---------- postgres/v0/database.go | 2 +- postgres/v1/database.go | 6 ++++++ util.go | 2 +- 4 files changed, 24 insertions(+), 12 deletions(-) diff --git a/postgres/shared/util.go b/postgres/shared/util.go index 93ad211..8c29a74 100644 --- a/postgres/shared/util.go +++ b/postgres/shared/util.go @@ -16,22 +16,28 @@ package shared -import "github.com/jmoiron/sqlx" +import ( + "github.com/jmoiron/sqlx" -/* - Hostname: "localhost", - Port: 8077, - DatabaseName: "cerc_testing", - Username: "vdbm", - Password: "password", -*/ + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" +) + +var ( + testDBConfig, _ = postgres.Config{ + Hostname: "localhost", + DatabaseName: "cerc_testing", + Username: "vdbm", + Password: "password", + Port: 8077, + Driver: "SQLX", + }.WithEnv() +) // TestDB connect to the testing database // it assumes the database has the IPFS ipld.blocks table present // DO NOT use a production db for the test db, as it will remove all contents of the ipld.blocks table func TestDB() (*sqlx.DB, error) { - connectStr := "postgresql://vdbm:password@localhost:8077/cerc_testing?sslmode=disable" - return sqlx.Connect("postgres", connectStr) + return sqlx.Connect("postgres", testDBConfig.DbConnectionString()) } // ResetTestDB drops all rows in the test db ipld.blocks table diff --git a/postgres/v0/database.go b/postgres/v0/database.go index 41a0400..4872df4 100644 --- a/postgres/v0/database.go +++ b/postgres/v0/database.go @@ -119,7 +119,7 @@ func (d *Database) dbGet(key string) ([]byte, error) { var data []byte err := d.db.Get(&data, getPgStr, key) if err == sql.ErrNoRows { - log.Warn("Database miss for key", key) + log.Warn("Database miss for key ", key) } return data, err diff --git a/postgres/v1/database.go b/postgres/v1/database.go index ef514f7..87ea9e9 100644 --- a/postgres/v1/database.go +++ b/postgres/v1/database.go @@ -40,6 +40,12 @@ var ( putPgStr = "INSERT INTO ipld.blocks (key, data, block_number) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING" deletePgStr = "DELETE FROM ipld.blocks WHERE key = $1" dbSizePgStr = "SELECT pg_database_size(current_database())" + + DefaultCacheConfig = CacheConfig{ + Name: "db", + Size: 3000000, // 3MB + ExpiryDuration: time.Hour, + } ) var _ ethdb.Database = &Database{} diff --git a/util.go b/util.go index 5735952..544fa74 100644 --- a/util.go +++ b/util.go @@ -29,7 +29,7 @@ func Keccak256ToCid(h []byte, codec uint64) (cid.Cid, error) { if err != nil { return cid.Cid{}, err } - return cid.NewCidV1(codec, multihash.Multihash(buf)), nil + return cid.NewCidV1(codec, buf), nil } // NewBlock takes a keccak256 hash key and the rlp []byte value it was derived from and creates an ipfs block object From a1e3f601b10d0b974ce51fc7ceb6838b73ae1fab Mon Sep 17 00:00:00 2001 From: i-norden Date: Sun, 9 Apr 2023 12:11:46 -0500 Subject: [PATCH 4/4] revert using vdb fork of geth --- postgres/shared/util.go | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/postgres/shared/util.go b/postgres/shared/util.go index 8c29a74..d4b0bcb 100644 --- a/postgres/shared/util.go +++ b/postgres/shared/util.go @@ -18,26 +18,21 @@ package shared import ( "github.com/jmoiron/sqlx" - - "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" -) - -var ( - testDBConfig, _ = postgres.Config{ - Hostname: "localhost", - DatabaseName: "cerc_testing", - Username: "vdbm", - Password: "password", - Port: 8077, - Driver: "SQLX", - }.WithEnv() ) +/* + Hostname: "localhost", + Port: 8077, + DatabaseName: "cerc_testing", + Username: "vdbm", + Password: "password", +*/ // TestDB connect to the testing database // it assumes the database has the IPFS ipld.blocks table present // DO NOT use a production db for the test db, as it will remove all contents of the ipld.blocks table func TestDB() (*sqlx.DB, error) { - return sqlx.Connect("postgres", testDBConfig.DbConnectionString()) + connectStr := "postgresql://vdbm:password@localhost:8077/cerc_testing?sslmode=disable" + return sqlx.Connect("postgres", connectStr) } // ResetTestDB drops all rows in the test db ipld.blocks table