review fixes/changes

core/blockchain.go

@@ -1337,7 +1337,7 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.
 					break
 				}
 				if bc.cacheConfig.ProcessingStateDiffs {
-					if !bc.allowedRootToBeDereferenced(root.(common.Hash)) {
+					if !bc.rootAllowedToBeDereferenced(root.(common.Hash)) {
 						bc.triegc.Push(root, number)
 						break
 					} else {
@@ -1406,7 +1406,7 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.
 // since we need the state tries of the current block and its parent in-memory
 // in order to process statediffs, we should avoid dereferencing roots until
 // its statediff and its child have been processed
-func (bc *BlockChain) allowedRootToBeDereferenced(root common.Hash) bool {
+func (bc *BlockChain) rootAllowedToBeDereferenced(root common.Hash) bool {
 	diffProcessedForSelfAndChildCount := 2
 	count := bc.stateDiffsProcessed[root]
 	return count >= diffProcessedForSelfAndChildCount

eth/config.go

@@ -151,8 +151,6 @@ type Config struct {
 	// RPCGasCap is the global gas cap for eth-call variants.
 	RPCGasCap *big.Int `toml:",omitempty"`
 
-	StateDiff bool
-
 	// Checkpoint is a hardcoded checkpoint which can be nil.
 	Checkpoint *params.TrustedCheckpoint `toml:",omitempty"`
 

statediff/builder.go

@@ -25,7 +25,6 @@ import (
 	"math/big"
 
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/crypto"
@@ -318,15 +317,3 @@ func (sdb *builder) buildStorageDiffsFromTrie(it trie.NodeIterator) ([]StorageDi
 
 	return storageDiffs, nil
 }
-
-func (sdb *builder) addressByPath(path []byte) (*common.Address, error) {
-	log.Debug("Looking up address from path", "path", hexutil.Encode(append([]byte("secure-key-"), path...)))
-	addrBytes, err := sdb.chainDB.Get(append([]byte("secure-key-"), hexToKeyBytes(path)...))
-	if err != nil {
-		log.Error("Error looking up address via path", "path", hexutil.Encode(append([]byte("secure-key-"), path...)), "error", err)
-		return nil, err
-	}
-	addr := common.BytesToAddress(addrBytes)
-	log.Debug("Address found", "Address", addr)
-	return &addr, nil
-}

statediff/doc.go

@@ -15,11 +15,11 @@
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
 /*
-This work is adapted from work by Charles Crain at https://github.com/jpmorganchase/quorum/blob/9b7fd9af8082795eeeb6863d9746f12b82dd5078/statediff/statediff.go
-
 Package statediff provides an auxiliary service that processes state diff objects from incoming chain events,
 relaying the objects to any rpc subscriptions.
 
+This work is adapted from work by Charles Crain at https://github.com/jpmorganchase/quorum/blob/9b7fd9af8082795eeeb6863d9746f12b82dd5078/statediff/statediff.go
+
 The service is spun up using the below CLI flags
 --statediff: boolean flag, turns on the service
 --statediff.streamblock: boolean flag, configures the service to associate and stream out the rest of the block data with the state diffs.
@@ -27,7 +27,16 @@ The service is spun up using the below CLI flags
 --statediff.pathsandproofs: boolean flag, tells service to generate paths and proofs for the diffed storage and state trie leaf nodes.
 --statediff.watchedaddresses: string slice flag, used to limit the state diffing process to the given addresses. Usage: --statediff.watchedaddresses=addr1 --statediff.watchedaddresses=addr2 --statediff.watchedaddresses=addr3
 
-If you wish to use the websocket endpoint to subscribe to the statediff service, be sure to open up the Websocket RPC server with the `--ws` flag.
+If you wish to use the websocket endpoint to subscribe to the statediff service, be sure to open up the Websocket RPC server with the `--ws` flag. The IPC-RPC server is turned on by default.
+
+The statediffing services works only with `--syncmode="full", but -importantly- does not require garbage collection to be turned off (does not require an archival node).
+
+e.g.
+
+$ ./geth --statediff --statediff.streamblock --ws --syncmode "full"
+
+This starts up the geth node in full sync mode, starts up the statediffing service, and opens up the websocket endpoint to subscribe to the service.
+Because the "streamblock" flag has been turned on, the service will strean out block data (headers, transactions, and receipts) along with the diffed state and storage leafs.
 
 Rpc subscriptions to the service can be created using the rpc.Client.Subscribe() method,
 with the "statediff" namespace, a statediff.Payload channel, and the name of the statediff api's rpc method- "stream".
@@ -41,7 +50,7 @@ for {
 	select {
 	case stateDiffPayload := <- stateDiffPayloadChan:
 		processPayload(stateDiffPayload)
-	case err := <= rpcSub.Err():
+	case err := <- rpcSub.Err():
 		log.Error(err)
 	}
 }

statediff/helpers.go

@@ -37,7 +37,7 @@ func sortKeys(data AccountsMap) []string {
 	return keys
 }
 
-// BytesToNiblePath
+// bytesToNiblePath converts the byte representation of a path to its string representation
 func bytesToNiblePath(path []byte) string {
 	if hasTerm(path) {
 		path = path[:len(path)-1]
@@ -53,6 +53,8 @@ func bytesToNiblePath(path []byte) string {
 	return nibblePath
 }
 
+// findIntersection finds the set of strings from both arrays that are equivalent (same key as same index)
+// this is used to find which keys have been both "deleted" and "created" i.e. they were "updated".
 func findIntersection(a, b []string) []string {
 	lenA := len(a)
 	lenB := len(b)
@@ -63,13 +65,13 @@ func findIntersection(a, b []string) []string {
 	}
 	for {
 		switch strings.Compare(a[iOfA], b[iOfB]) {
-		// a[iOfA] < b[iOfB]
+		// -1 when a[iOfA] < b[iOfB]
 		case -1:
 			iOfA++
 			if iOfA >= lenA {
 				return updates
 			}
-			// a[iOfA] == b[iOfB]
+			// 0 when a[iOfA] == b[iOfB]
 		case 0:
 			updates = append(updates, a[iOfA])
 			iOfA++
@@ -77,7 +79,7 @@ func findIntersection(a, b []string) []string {
 			if iOfA >= lenA || iOfB >= lenB {
 				return updates
 			}
-			// a[iOfA] > b[iOfB]
+			// 1 when a[iOfA] > b[iOfB]
 		case 1:
 			iOfB++
 			if iOfB >= lenB {
@@ -88,30 +90,11 @@ func findIntersection(a, b []string) []string {
 
 }
 
+// pathToStr converts the NodeIterator path to a string representation
 func pathToStr(it trie.NodeIterator) string {
 	return bytesToNiblePath(it.Path())
 }
 
-// Duplicated from trie/encoding.go
-func hexToKeyBytes(hex []byte) []byte {
-	if hasTerm(hex) {
-		hex = hex[:len(hex)-1]
-	}
-	if len(hex)&1 != 0 {
-		panic("can't convert hex key of odd length")
-	}
-	key := make([]byte, (len(hex)+1)/2)
-	decodeNibbles(hex, key)
-
-	return key
-}
-
-func decodeNibbles(nibbles []byte, bytes []byte) {
-	for bi, ni := 0, 0; ni < len(nibbles); bi, ni = bi+1, ni+2 {
-		bytes[bi] = nibbles[ni]<<4 | nibbles[ni+1]
-	}
-}
-
 // hasTerm returns whether a hex key has the terminator flag.
 func hasTerm(s []byte) bool {
 	return len(s) > 0 && s[len(s)-1] == 16

statediff/service.go

@@ -176,7 +176,7 @@ func (sds *Service) processStateDiff(currentBlock, parentBlock *types.Block) err
 		payload.ReceiptsRlp = receiptBuff.Bytes()
 	}
 
-	// If we have any websocket subscriptions listening in, send the data to them
+	// If we have any rpc subscriptions listening in, send the data to them
 	sds.send(payload)
 	return nil
 }

statediff/service_test.go

@@ -131,27 +131,27 @@ func testErrorInChainEventLoop(t *testing.T) {
 	for i, payload := range payloads {
 		if !bytes.Equal(payload.ReceiptsRlp, expectedReceiptsRlp[i]) {
 			t.Error("Test failure:", t.Name())
-			t.Logf("Actual reeipt rlp for payload %d does not equal expected.\nactual: %+v\nexpected: %+v", i, payload.ReceiptsRlp, expectedReceiptsRlp[i])
+			t.Logf("Actual receipt rlp for payload %d does not equal expected.\nactual: %+v\nexpected: %+v", i, payload.ReceiptsRlp, expectedReceiptsRlp[i])
 		}
 	}
 
 	if !reflect.DeepEqual(builder.BlockHash, testBlock2.Hash()) {
 		t.Error("Test failure:", t.Name())
-		t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", builder.BlockHash, testBlock2.Hash())
+		t.Logf("Actual blockhash does not equal expected.\nactual:%+v\nexpected: %+v", builder.BlockHash, testBlock2.Hash())
 	}
 	if !bytes.Equal(builder.OldStateRoot.Bytes(), parentBlock2.Root().Bytes()) {
 		t.Error("Test failure:", t.Name())
-		t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", builder.OldStateRoot, parentBlock2.Root())
+		t.Logf("Actual root does not equal expected.\nactual:%+v\nexpected: %+v", builder.OldStateRoot, parentBlock2.Root())
 	}
 	if !bytes.Equal(builder.NewStateRoot.Bytes(), testBlock2.Root().Bytes()) {
 		t.Error("Test failure:", t.Name())
-		t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", builder.NewStateRoot, testBlock2.Root())
+		t.Logf("Actual root does not equal expected.\nactual:%+v\nexpected: %+v", builder.NewStateRoot, testBlock2.Root())
 	}
 	//look up the parent block from its hash
 	expectedHashes := []common.Hash{testBlock1.ParentHash(), testBlock2.ParentHash()}
 	if !reflect.DeepEqual(blockChain.ParentHashesLookedUp, expectedHashes) {
 		t.Error("Test failure:", t.Name())
-		t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", blockChain.ParentHashesLookedUp, expectedHashes)
+		t.Logf("Actual parent hash does not equal expected.\nactual:%+v\nexpected: %+v", blockChain.ParentHashesLookedUp, expectedHashes)
 	}
 }