From 4d8d410b5b3b07e53efdcdcb14f78d98a65df1c1 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Fri, 22 Jan 2021 16:39:15 -0600 Subject: [PATCH] update github actions; linting --- .github/workflows/on-master.yaml | 1 + cmd/geth/config.go | 2 +- core/blockchain.go | 2 +- crypto/signify/signify_fuzz.go | 1 - internal/ethapi/api.go | 2 +- params/version.go | 6 +-- statediff/indexer/indexer.go | 25 ++++++------ statediff/indexer/indexer_test.go | 37 +++++++++--------- statediff/indexer/ipfs/ipld/trie_node.go | 48 +++++++++++++++--------- statediff/indexer/metrics.go | 8 ++-- statediff/indexer/shared/functions.go | 10 ++--- statediff/indexer/shared/types.go | 8 ++-- statediff/mainnet_tests/builder_test.go | 2 +- trie/encoding.go | 10 ++--- 14 files changed, 87 insertions(+), 75 deletions(-) diff --git a/.github/workflows/on-master.yaml b/.github/workflows/on-master.yaml index e53dc88b5..e37ac6a7e 100644 --- a/.github/workflows/on-master.yaml +++ b/.github/workflows/on-master.yaml @@ -3,6 +3,7 @@ name: Docker Build and publish to Github on: push: branches: + - v1.9.25-statediff - v1.9.24-statediff - v1.9.23-statediff - v1.9.11-statediff diff --git a/cmd/geth/config.go b/cmd/geth/config.go index dfb0d4aa7..3d5f63ca9 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -24,9 +24,9 @@ import ( "reflect" "unicode" - "gopkg.in/urfave/cli.v1" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/statediff" + "gopkg.in/urfave/cli.v1" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/eth" diff --git a/core/blockchain.go b/core/blockchain.go index 08f8927d8..e4e3b7a1d 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -132,7 +132,7 @@ type CacheConfig struct { Preimages bool // Whether to store preimage of trie key to the disk SnapshotWait bool // Wait for snapshot construction on startup. TODO(karalabe): This is a dirty hack for testing, nuke it - StateDiffing bool // Whether or not the statediffing service is running + StateDiffing bool // Whether or not the statediffing service is running } // defaultCacheConfig are the default caching values if none are specified by the diff --git a/crypto/signify/signify_fuzz.go b/crypto/signify/signify_fuzz.go index d1bcf356a..cc084c655 100644 --- a/crypto/signify/signify_fuzz.go +++ b/crypto/signify/signify_fuzz.go @@ -25,7 +25,6 @@ import ( "log" "os" "os/exec" - "runtime" fuzz "github.com/google/gofuzz" "github.com/jedisct1/go-minisign" diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 4aa91bef2..34499afd4 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -935,7 +935,7 @@ func (s *PublicBlockChainAPI) Call(ctx context.Context, args CallArgs, blockNrOr timeout := 5 * time.Second d, ok := ctx.Deadline() if ok { - timeout = d.Sub(time.Now()) + timeout = time.Until(d) } result, err := DoCall(ctx, s.b, args, blockNrOrHash, accounts, vm.Config{}, timeout, s.b.RPCGasCap()) if err != nil { diff --git a/params/version.go b/params/version.go index 3c7f6e7cb..cf520de2d 100644 --- a/params/version.go +++ b/params/version.go @@ -21,9 +21,9 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 9 // Minor version component of the current release - VersionPatch = 25 // Patch version component of the current release + VersionMajor = 1 // Major version component of the current release + VersionMinor = 9 // Minor version component of the current release + VersionPatch = 25 // Patch version component of the current release VersionMeta = "statediff-0.0.13" // Version metadata to append to the version string ) diff --git a/statediff/indexer/indexer.go b/statediff/indexer/indexer.go index 2a6bac3f2..c9e0c6369 100644 --- a/statediff/indexer/indexer.go +++ b/statediff/indexer/indexer.go @@ -31,16 +31,15 @@ import ( "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" - - node "github.com/ipfs/go-ipld-format" - "github.com/jmoiron/sqlx" - "github.com/multiformats/go-multihash" - "github.com/ethereum/go-ethereum/statediff/indexer/ipfs/ipld" "github.com/ethereum/go-ethereum/statediff/indexer/models" "github.com/ethereum/go-ethereum/statediff/indexer/postgres" "github.com/ethereum/go-ethereum/statediff/indexer/shared" sdtypes "github.com/ethereum/go-ethereum/statediff/types" + + node "github.com/ipfs/go-ipld-format" + "github.com/jmoiron/sqlx" + "github.com/multiformats/go-multihash" ) var ( @@ -135,21 +134,21 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip shared.Rollback(tx) panic(p) } else { - tDiff := time.Now().Sub(t) + tDiff := time.Since(t) indexerMetrics.tStateStoreCodeProcessing.Update(tDiff) traceMsg += fmt.Sprintf("state, storage, and code storage processing time: %s\r\n", tDiff.String()) t = time.Now() err = tx.Commit() - tDiff = time.Now().Sub(t) + tDiff = time.Since(t) indexerMetrics.tPostgresCommit.Update(tDiff) traceMsg += fmt.Sprintf("postgres transaction commit duration: %s\r\n", tDiff.String()) } - traceMsg += fmt.Sprintf(" TOTAL PROCESSING DURATION: %s\r\n", time.Now().Sub(start).String()) + traceMsg += fmt.Sprintf(" TOTAL PROCESSING DURATION: %s\r\n", time.Since(start).String()) log.Debug(traceMsg) return err }, } - tDiff := time.Now().Sub(t) + tDiff := time.Since(t) indexerMetrics.tFreePostgres.Update(tDiff) traceMsg += fmt.Sprintf("time spent waiting for free postgres tx: %s:\r\n", tDiff.String()) @@ -160,7 +159,7 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip if err != nil { return nil, err } - tDiff = time.Now().Sub(t) + tDiff = time.Since(t) indexerMetrics.tHeaderProcessing.Update(tDiff) traceMsg += fmt.Sprintf("header processing time: %s\r\n", tDiff.String()) t = time.Now() @@ -168,7 +167,7 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip if err := sdi.processUncles(tx, headerID, height, uncleNodes); err != nil { return nil, err } - tDiff = time.Now().Sub(t) + tDiff = time.Since(t) indexerMetrics.tUncleProcessing.Update(tDiff) traceMsg += fmt.Sprintf("uncle processing time: %s\r\n", tDiff.String()) t = time.Now() @@ -185,7 +184,7 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip }); err != nil { return nil, err } - tDiff = time.Now().Sub(t) + tDiff = time.Since(t) indexerMetrics.tTxAndRecProcessing.Update(tDiff) traceMsg += fmt.Sprintf("tx and receipt processing time: %s\r\n", tDiff.String()) t = time.Now() @@ -241,7 +240,7 @@ func (sdi *StateDiffIndexer) processUncles(tx *sqlx.Tx, headerID int64, blockNum return nil } -// processArgs bundles arugments to processReceiptsAndTxs +// processArgs bundles arguments to processReceiptsAndTxs type processArgs struct { headerID int64 blockNumber *big.Int diff --git a/statediff/indexer/indexer_test.go b/statediff/indexer/indexer_test.go index 1a7c76763..d2ce5984e 100644 --- a/statediff/indexer/indexer_test.go +++ b/statediff/indexer/indexer_test.go @@ -21,21 +21,21 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/params" - "github.com/ipfs/go-cid" - "github.com/ipfs/go-ipfs-blockstore" - "github.com/ipfs/go-ipfs-ds-help" - - ind "github.com/ethereum/go-ethereum/statediff/indexer" + "github.com/ethereum/go-ethereum/statediff/indexer" "github.com/ethereum/go-ethereum/statediff/indexer/mocks" - eth "github.com/ethereum/go-ethereum/statediff/indexer/models" + "github.com/ethereum/go-ethereum/statediff/indexer/models" "github.com/ethereum/go-ethereum/statediff/indexer/postgres" "github.com/ethereum/go-ethereum/statediff/indexer/shared" + + "github.com/ipfs/go-cid" + blockstore "github.com/ipfs/go-ipfs-blockstore" + dshelp "github.com/ipfs/go-ipfs-ds-help" ) var ( db *postgres.DB err error - indexer *ind.StateDiffIndexer + ind *indexer.StateDiffIndexer ipfsPgGet = `SELECT data FROM public.blocks WHERE key = $1` ) @@ -51,9 +51,9 @@ func setup(t *testing.T) { if err != nil { t.Fatal(err) } - indexer = ind.NewStateDiffIndexer(params.MainnetChainConfig, db) - var tx *ind.BlockTx - tx, err = indexer.PushBlock( + ind = indexer.NewStateDiffIndexer(params.MainnetChainConfig, db) + var tx *indexer.BlockTx + tx, err = ind.PushBlock( mocks.MockBlock, mocks.MockReceipts, mocks.MockBlock.Difficulty()) @@ -62,7 +62,7 @@ func setup(t *testing.T) { } defer tx.Close() for _, node := range mocks.StateDiffs { - err = indexer.PushStateNode(tx, node) + err = ind.PushStateNode(tx, node) if err != nil { t.Fatal(err) } @@ -72,7 +72,7 @@ func setup(t *testing.T) { } func tearDown(t *testing.T) { - ind.TearDownDB(t, db) + indexer.TearDownDB(t, db) } func TestPublishAndIndexer(t *testing.T) { @@ -195,7 +195,7 @@ func TestPublishAndIndexer(t *testing.T) { setup(t) defer tearDown(t) // check that state nodes were properly indexed and published - stateNodes := make([]eth.StateNodeModel, 0) + stateNodes := make([]models.StateNodeModel, 0) pgStr := `SELECT state_cids.id, state_cids.cid, state_cids.state_leaf_key, state_cids.node_type, state_cids.state_path, state_cids.header_id FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) WHERE header_cids.block_number = $1` @@ -217,7 +217,7 @@ func TestPublishAndIndexer(t *testing.T) { t.Fatal(err) } pgStr = `SELECT * from eth.state_accounts WHERE state_id = $1` - var account eth.StateAccountModel + var account models.StateAccountModel err = db.Get(&account, pgStr, stateNode.ID) if err != nil { t.Fatal(err) @@ -227,7 +227,7 @@ func TestPublishAndIndexer(t *testing.T) { shared.ExpectEqual(t, stateNode.StateKey, common.BytesToHash(mocks.ContractLeafKey).Hex()) shared.ExpectEqual(t, stateNode.Path, []byte{'\x06'}) shared.ExpectEqual(t, data, mocks.ContractLeafNode) - shared.ExpectEqual(t, account, eth.StateAccountModel{ + shared.ExpectEqual(t, account, models.StateAccountModel{ ID: account.ID, StateID: stateNode.ID, Balance: "0", @@ -241,7 +241,7 @@ func TestPublishAndIndexer(t *testing.T) { shared.ExpectEqual(t, stateNode.StateKey, common.BytesToHash(mocks.AccountLeafKey).Hex()) shared.ExpectEqual(t, stateNode.Path, []byte{'\x0c'}) shared.ExpectEqual(t, data, mocks.AccountLeafNode) - shared.ExpectEqual(t, account, eth.StateAccountModel{ + shared.ExpectEqual(t, account, models.StateAccountModel{ ID: account.ID, StateID: stateNode.ID, Balance: "1000", @@ -251,14 +251,13 @@ func TestPublishAndIndexer(t *testing.T) { }) } } - pgStr = `SELECT * from eth.state_accounts WHERE state_id = $1` }) t.Run("Publish and index storage IPLDs in a single tx", func(t *testing.T) { setup(t) defer tearDown(t) // check that storage nodes were properly indexed - storageNodes := make([]eth.StorageNodeWithStateKeyModel, 0) + storageNodes := make([]models.StorageNodeWithStateKeyModel, 0) pgStr := `SELECT storage_cids.cid, state_cids.state_leaf_key, storage_cids.storage_leaf_key, storage_cids.node_type, storage_cids.storage_path FROM eth.storage_cids, eth.state_cids, eth.header_cids WHERE storage_cids.state_id = state_cids.id @@ -269,7 +268,7 @@ func TestPublishAndIndexer(t *testing.T) { t.Fatal(err) } shared.ExpectEqual(t, len(storageNodes), 1) - shared.ExpectEqual(t, storageNodes[0], eth.StorageNodeWithStateKeyModel{ + shared.ExpectEqual(t, storageNodes[0], models.StorageNodeWithStateKeyModel{ CID: mocks.StorageCID.String(), NodeType: 2, StorageKey: common.BytesToHash(mocks.StorageLeafKey).Hex(), diff --git a/statediff/indexer/ipfs/ipld/trie_node.go b/statediff/indexer/ipfs/ipld/trie_node.go index 788f76db8..afa421bcc 100644 --- a/statediff/indexer/ipfs/ipld/trie_node.go +++ b/statediff/indexer/ipfs/ipld/trie_node.go @@ -25,6 +25,12 @@ import ( node "github.com/ipfs/go-ipld-format" ) +const ( + extension = "extension" + leaf = "leaf" + branch = "branch" +) + // TrieNode is the general abstraction for //ethereum IPLD trie nodes. type TrieNode struct { @@ -68,17 +74,23 @@ func decodeTrieNode(c cid.Cid, b []byte, return nil, err } - if nodeKind == "extension" { + if nodeKind == extension { elements, err = parseTrieNodeExtension(decoded, codec) + if err != nil { + return nil, err + } } - if nodeKind == "leaf" { + if nodeKind == leaf { elements, err = leafDecoder(decoded) + if err != nil { + return nil, err + } } - if nodeKind != "extension" && nodeKind != "leaf" { + if nodeKind != extension && nodeKind != leaf { return nil, fmt.Errorf("unexpected nodeKind returned from decoder") } case 17: - nodeKind = "branch" + nodeKind = branch elements, err = parseTrieNodeBranch(i, codec) if err != nil { return nil, err @@ -102,22 +114,22 @@ func decodeCompactKey(i []interface{}) (string, []interface{}, error) { switch first[0] / 16 { case '\x00': - return "extension", []interface{}{ + return extension, []interface{}{ nibbleToByte(first)[2:], last, }, nil case '\x01': - return "extension", []interface{}{ + return extension, []interface{}{ nibbleToByte(first)[1:], last, }, nil case '\x02': - return "leaf", []interface{}{ + return leaf, []interface{}{ nibbleToByte(first)[2:], last, }, nil case '\x03': - return "leaf", []interface{}{ + return leaf, []interface{}{ nibbleToByte(first)[1:], last, }, nil @@ -167,11 +179,11 @@ func parseTrieNodeBranch(i []interface{}, codec uint64) ([]interface{}, error) { // and returning the object found as well as the remaining path to traverse func (t *TrieNode) Resolve(p []string) (interface{}, []string, error) { switch t.nodeKind { - case "extension": + case extension: return t.resolveTrieNodeExtension(p) - case "leaf": + case leaf: return t.resolveTrieNodeLeaf(p) - case "branch": + case branch: return t.resolveTrieNodeBranch(p) default: return nil, nil, fmt.Errorf("nodeKind case not implemented") @@ -188,13 +200,13 @@ func (t *TrieNode) Tree(p string, depth int) []string { var out []string switch t.nodeKind { - case "extension": + case extension: var val string for _, e := range t.elements[0].([]byte) { val += fmt.Sprintf("%x", e) } return []string{val} - case "branch": + case branch: for i, elem := range t.elements { if _, ok := elem.(*cid.Cid); ok { out = append(out, fmt.Sprintf("%x", i)) @@ -261,9 +273,9 @@ func (t *TrieNode) MarshalJSON() ([]byte, error) { var out map[string]interface{} switch t.nodeKind { - case "extension": + case extension: fallthrough - case "leaf": + case leaf: var hexPrefix string for _, e := range t.elements[0].([]byte) { hexPrefix += fmt.Sprintf("%x", e) @@ -285,9 +297,9 @@ func (t *TrieNode) MarshalJSON() ([]byte, error) { hexPrefix: t.elements[1], } - case "branch": + case branch: out = map[string]interface{}{ - "type": "branch", + "type": branch, "0": t.elements[0], "1": t.elements[1], "2": t.elements[2], @@ -432,7 +444,7 @@ func getHexIndex(s string) int { return -1 } - c := byte(s[0]) + c := s[0] switch { case '0' <= c && c <= '9': return int(c - '0') diff --git a/statediff/indexer/metrics.go b/statediff/indexer/metrics.go index fc0727eda..e407d4d40 100644 --- a/statediff/indexer/metrics.go +++ b/statediff/indexer/metrics.go @@ -117,8 +117,8 @@ func (met *dbMetricsHandles) Update(stats sql.DBStats) { met.open.Update(int64(stats.OpenConnections)) met.inUse.Update(int64(stats.InUse)) met.idle.Update(int64(stats.Idle)) - met.waitedFor.Inc(int64(stats.WaitCount)) - met.blockedMilliseconds.Inc(int64(stats.WaitDuration.Milliseconds())) - met.closedMaxIdle.Inc(int64(stats.MaxIdleClosed)) - met.closedMaxLifetime.Inc(int64(stats.MaxLifetimeClosed)) + met.waitedFor.Inc(stats.WaitCount) + met.blockedMilliseconds.Inc(stats.WaitDuration.Milliseconds()) + met.closedMaxIdle.Inc(stats.MaxIdleClosed) + met.closedMaxLifetime.Inc(stats.MaxLifetimeClosed) } diff --git a/statediff/indexer/shared/functions.go b/statediff/indexer/shared/functions.go index 257100289..92d5e6f2f 100644 --- a/statediff/indexer/shared/functions.go +++ b/statediff/indexer/shared/functions.go @@ -22,14 +22,14 @@ import ( "github.com/ethereum/go-ethereum/statediff/indexer/ipfs/ipld" "github.com/ipfs/go-cid" - "github.com/ipfs/go-ipfs-blockstore" - "github.com/ipfs/go-ipfs-ds-help" - node "github.com/ipfs/go-ipld-format" + blockstore "github.com/ipfs/go-ipfs-blockstore" + dshelp "github.com/ipfs/go-ipfs-ds-help" + format "github.com/ipfs/go-ipld-format" "github.com/jmoiron/sqlx" "github.com/multiformats/go-multihash" ) -// HandleZeroAddrPointer will return an emtpy string for a nil address pointer +// HandleZeroAddrPointer will return an empty string for a nil address pointer func HandleZeroAddrPointer(to *common.Address) string { if to == nil { return "" @@ -53,7 +53,7 @@ func Rollback(tx *sqlx.Tx) { } // PublishIPLD is used to insert an IPLD into Postgres blockstore with the provided tx -func PublishIPLD(tx *sqlx.Tx, i node.Node) error { +func PublishIPLD(tx *sqlx.Tx, i format.Node) error { dbKey := dshelp.MultihashToDsKey(i.Cid().Hash()) prefixedKey := blockstore.BlockPrefix.String() + dbKey.String() raw := i.RawData() diff --git a/statediff/indexer/shared/types.go b/statediff/indexer/shared/types.go index a9792522b..544d4e07e 100644 --- a/statediff/indexer/shared/types.go +++ b/statediff/indexer/shared/types.go @@ -16,9 +16,11 @@ package shared -import "github.com/ethereum/go-ethereum/common" -import "github.com/ethereum/go-ethereum/statediff/types" -import "github.com/ethereum/go-ethereum/statediff/indexer/models" +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/statediff/indexer/models" + "github.com/ethereum/go-ethereum/statediff/types" +) // Trie struct used to flag node as leaf or not type TrieNode struct { diff --git a/statediff/mainnet_tests/builder_test.go b/statediff/mainnet_tests/builder_test.go index 6412331ff..f261c7fc8 100644 --- a/statediff/mainnet_tests/builder_test.go +++ b/statediff/mainnet_tests/builder_test.go @@ -463,7 +463,7 @@ func loadBlockFromRLPFile(filename string) (*types.Block, []byte, error) { } func TestBuilderOnMainnetBlocks(t *testing.T) { - chain, _ := core.NewBlockChain(db, nil, params.MainnetChainConfig, ethash.NewFaker(), vm.Config{}, nil) + chain, _ := core.NewBlockChain(db, nil, params.MainnetChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil) _, err := chain.InsertChain([]*types.Block{block1, block2, block3}) if err != nil { t.Error(err) diff --git a/trie/encoding.go b/trie/encoding.go index 3a5f632e2..ace45700c 100644 --- a/trie/encoding.go +++ b/trie/encoding.go @@ -60,8 +60,8 @@ func hexToCompact(hex []byte) []byte { // needed for the representation func hexToCompactInPlace(hex []byte) int { var ( - hexLen= len(hex) // length of the hex input - firstByte= byte(0) + hexLen = len(hex) // length of the hex input + firstByte = byte(0) ) // Check if we have a terminator there if hexLen > 0 && hex[hexLen-1] == 16 { @@ -69,9 +69,9 @@ func hexToCompactInPlace(hex []byte) int { hexLen-- // last part was the terminator, ignore that } var ( - binLen= hexLen/2 + 1 - ni= 0 // index in hex - bi= 1 // index in bin (compact) + binLen = hexLen/2 + 1 + ni = 0 // index in hex + bi = 1 // index in bin (compact) ) if hexLen&1 == 1 { firstByte |= 1 << 4 // odd flag