diff --git a/core/blockchain.go b/core/blockchain.go index cf1a65cd5..0c56a13c9 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1337,7 +1337,7 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. break } if bc.cacheConfig.ProcessingStateDiffs { - if !bc.allowedRootToBeDereferenced(root.(common.Hash)) { + if !bc.rootAllowedToBeDereferenced(root.(common.Hash)) { bc.triegc.Push(root, number) break } else { @@ -1406,7 +1406,7 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. // since we need the state tries of the current block and its parent in-memory // in order to process statediffs, we should avoid dereferencing roots until // its statediff and its child have been processed -func (bc *BlockChain) allowedRootToBeDereferenced(root common.Hash) bool { +func (bc *BlockChain) rootAllowedToBeDereferenced(root common.Hash) bool { diffProcessedForSelfAndChildCount := 2 count := bc.stateDiffsProcessed[root] return count >= diffProcessedForSelfAndChildCount diff --git a/eth/config.go b/eth/config.go index 8ed014e24..e67e060cc 100644 --- a/eth/config.go +++ b/eth/config.go @@ -151,8 +151,6 @@ type Config struct { // RPCGasCap is the global gas cap for eth-call variants. RPCGasCap *big.Int `toml:",omitempty"` - StateDiff bool - // Checkpoint is a hardcoded checkpoint which can be nil. Checkpoint *params.TrustedCheckpoint `toml:",omitempty"` diff --git a/statediff/builder.go b/statediff/builder.go index 765152e87..ad3acce52 100644 --- a/statediff/builder.go +++ b/statediff/builder.go @@ -25,7 +25,6 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/crypto" @@ -318,15 +317,3 @@ func (sdb *builder) buildStorageDiffsFromTrie(it trie.NodeIterator) ([]StorageDi return storageDiffs, nil } - -func (sdb *builder) addressByPath(path []byte) (*common.Address, error) { - log.Debug("Looking up address from path", "path", hexutil.Encode(append([]byte("secure-key-"), path...))) - addrBytes, err := sdb.chainDB.Get(append([]byte("secure-key-"), hexToKeyBytes(path)...)) - if err != nil { - log.Error("Error looking up address via path", "path", hexutil.Encode(append([]byte("secure-key-"), path...)), "error", err) - return nil, err - } - addr := common.BytesToAddress(addrBytes) - log.Debug("Address found", "Address", addr) - return &addr, nil -} diff --git a/statediff/doc.go b/statediff/doc.go index 0e6d5f3e1..35c48c02d 100644 --- a/statediff/doc.go +++ b/statediff/doc.go @@ -15,11 +15,11 @@ // along with the go-ethereum library. If not, see . /* -This work is adapted from work by Charles Crain at https://github.com/jpmorganchase/quorum/blob/9b7fd9af8082795eeeb6863d9746f12b82dd5078/statediff/statediff.go - Package statediff provides an auxiliary service that processes state diff objects from incoming chain events, relaying the objects to any rpc subscriptions. +This work is adapted from work by Charles Crain at https://github.com/jpmorganchase/quorum/blob/9b7fd9af8082795eeeb6863d9746f12b82dd5078/statediff/statediff.go + The service is spun up using the below CLI flags --statediff: boolean flag, turns on the service --statediff.streamblock: boolean flag, configures the service to associate and stream out the rest of the block data with the state diffs. @@ -27,7 +27,16 @@ The service is spun up using the below CLI flags --statediff.pathsandproofs: boolean flag, tells service to generate paths and proofs for the diffed storage and state trie leaf nodes. --statediff.watchedaddresses: string slice flag, used to limit the state diffing process to the given addresses. Usage: --statediff.watchedaddresses=addr1 --statediff.watchedaddresses=addr2 --statediff.watchedaddresses=addr3 -If you wish to use the websocket endpoint to subscribe to the statediff service, be sure to open up the Websocket RPC server with the `--ws` flag. +If you wish to use the websocket endpoint to subscribe to the statediff service, be sure to open up the Websocket RPC server with the `--ws` flag. The IPC-RPC server is turned on by default. + +The statediffing services works only with `--syncmode="full", but -importantly- does not require garbage collection to be turned off (does not require an archival node). + +e.g. + +$ ./geth --statediff --statediff.streamblock --ws --syncmode "full" + +This starts up the geth node in full sync mode, starts up the statediffing service, and opens up the websocket endpoint to subscribe to the service. +Because the "streamblock" flag has been turned on, the service will strean out block data (headers, transactions, and receipts) along with the diffed state and storage leafs. Rpc subscriptions to the service can be created using the rpc.Client.Subscribe() method, with the "statediff" namespace, a statediff.Payload channel, and the name of the statediff api's rpc method- "stream". @@ -41,7 +50,7 @@ for { select { case stateDiffPayload := <- stateDiffPayloadChan: processPayload(stateDiffPayload) - case err := <= rpcSub.Err(): + case err := <- rpcSub.Err(): log.Error(err) } } diff --git a/statediff/helpers.go b/statediff/helpers.go index 89852b55c..52ae1c392 100644 --- a/statediff/helpers.go +++ b/statediff/helpers.go @@ -37,7 +37,7 @@ func sortKeys(data AccountsMap) []string { return keys } -// BytesToNiblePath +// bytesToNiblePath converts the byte representation of a path to its string representation func bytesToNiblePath(path []byte) string { if hasTerm(path) { path = path[:len(path)-1] @@ -53,6 +53,8 @@ func bytesToNiblePath(path []byte) string { return nibblePath } +// findIntersection finds the set of strings from both arrays that are equivalent (same key as same index) +// this is used to find which keys have been both "deleted" and "created" i.e. they were "updated". func findIntersection(a, b []string) []string { lenA := len(a) lenB := len(b) @@ -63,13 +65,13 @@ func findIntersection(a, b []string) []string { } for { switch strings.Compare(a[iOfA], b[iOfB]) { - // a[iOfA] < b[iOfB] + // -1 when a[iOfA] < b[iOfB] case -1: iOfA++ if iOfA >= lenA { return updates } - // a[iOfA] == b[iOfB] + // 0 when a[iOfA] == b[iOfB] case 0: updates = append(updates, a[iOfA]) iOfA++ @@ -77,7 +79,7 @@ func findIntersection(a, b []string) []string { if iOfA >= lenA || iOfB >= lenB { return updates } - // a[iOfA] > b[iOfB] + // 1 when a[iOfA] > b[iOfB] case 1: iOfB++ if iOfB >= lenB { @@ -88,30 +90,11 @@ func findIntersection(a, b []string) []string { } +// pathToStr converts the NodeIterator path to a string representation func pathToStr(it trie.NodeIterator) string { return bytesToNiblePath(it.Path()) } -// Duplicated from trie/encoding.go -func hexToKeyBytes(hex []byte) []byte { - if hasTerm(hex) { - hex = hex[:len(hex)-1] - } - if len(hex)&1 != 0 { - panic("can't convert hex key of odd length") - } - key := make([]byte, (len(hex)+1)/2) - decodeNibbles(hex, key) - - return key -} - -func decodeNibbles(nibbles []byte, bytes []byte) { - for bi, ni := 0, 0; ni < len(nibbles); bi, ni = bi+1, ni+2 { - bytes[bi] = nibbles[ni]<<4 | nibbles[ni+1] - } -} - // hasTerm returns whether a hex key has the terminator flag. func hasTerm(s []byte) bool { return len(s) > 0 && s[len(s)-1] == 16 diff --git a/statediff/service.go b/statediff/service.go index be3dc6cf9..658f4c8a8 100644 --- a/statediff/service.go +++ b/statediff/service.go @@ -176,7 +176,7 @@ func (sds *Service) processStateDiff(currentBlock, parentBlock *types.Block) err payload.ReceiptsRlp = receiptBuff.Bytes() } - // If we have any websocket subscriptions listening in, send the data to them + // If we have any rpc subscriptions listening in, send the data to them sds.send(payload) return nil } diff --git a/statediff/service_test.go b/statediff/service_test.go index 1508542b7..6119f6ecb 100644 --- a/statediff/service_test.go +++ b/statediff/service_test.go @@ -131,27 +131,27 @@ func testErrorInChainEventLoop(t *testing.T) { for i, payload := range payloads { if !bytes.Equal(payload.ReceiptsRlp, expectedReceiptsRlp[i]) { t.Error("Test failure:", t.Name()) - t.Logf("Actual reeipt rlp for payload %d does not equal expected.\nactual: %+v\nexpected: %+v", i, payload.ReceiptsRlp, expectedReceiptsRlp[i]) + t.Logf("Actual receipt rlp for payload %d does not equal expected.\nactual: %+v\nexpected: %+v", i, payload.ReceiptsRlp, expectedReceiptsRlp[i]) } } if !reflect.DeepEqual(builder.BlockHash, testBlock2.Hash()) { t.Error("Test failure:", t.Name()) - t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", builder.BlockHash, testBlock2.Hash()) + t.Logf("Actual blockhash does not equal expected.\nactual:%+v\nexpected: %+v", builder.BlockHash, testBlock2.Hash()) } if !bytes.Equal(builder.OldStateRoot.Bytes(), parentBlock2.Root().Bytes()) { t.Error("Test failure:", t.Name()) - t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", builder.OldStateRoot, parentBlock2.Root()) + t.Logf("Actual root does not equal expected.\nactual:%+v\nexpected: %+v", builder.OldStateRoot, parentBlock2.Root()) } if !bytes.Equal(builder.NewStateRoot.Bytes(), testBlock2.Root().Bytes()) { t.Error("Test failure:", t.Name()) - t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", builder.NewStateRoot, testBlock2.Root()) + t.Logf("Actual root does not equal expected.\nactual:%+v\nexpected: %+v", builder.NewStateRoot, testBlock2.Root()) } //look up the parent block from its hash expectedHashes := []common.Hash{testBlock1.ParentHash(), testBlock2.ParentHash()} if !reflect.DeepEqual(blockChain.ParentHashesLookedUp, expectedHashes) { t.Error("Test failure:", t.Name()) - t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", blockChain.ParentHashesLookedUp, expectedHashes) + t.Logf("Actual parent hash does not equal expected.\nactual:%+v\nexpected: %+v", blockChain.ParentHashesLookedUp, expectedHashes) } }