update pkgs to v5

pkg/api.go

@@ -45,11 +45,6 @@ func (api *PublicStateDiffAPI) StateDiffAt(ctx context.Context, blockNumber uint
 	return api.sds.StateDiffAt(blockNumber, params)
 }
 
-// StateTrieAt returns a state trie payload at the specific blockheight
-func (api *PublicStateDiffAPI) StateTrieAt(ctx context.Context, blockNumber uint64, params sd.Params) (*sd.Payload, error) {
-	return api.sds.StateTrieAt(blockNumber, params)
-}
-
 // WriteStateDiffAt writes a state diff object directly to DB at the specific blockheight
 func (api *PublicStateDiffAPI) WriteStateDiffAt(ctx context.Context, blockNumber uint64, params sd.Params) error {
 	return api.sds.WriteStateDiffAt(blockNumber, params)

pkg/builder.go

@@ -55,11 +55,11 @@ func NewBuilder(stateCache state.Database, workers uint) (sd.Builder, error) {
 
 // BuildStateDiffObject builds a statediff object from two blocks and the provided parameters
 func (sdb *builder) BuildStateDiffObject(args sd.Args, params sd.Params) (sdtypes.StateObject, error) {
-	var stateNodes []sdtypes.StateNode
+	var stateNodes []sdtypes.StateLeafNode
-	var codeAndCodeHashes []sdtypes.CodeAndCodeHash
+	var codeAndCodeHashes []sdtypes.IPLD
 	err := sdb.WriteStateDiffObject(
 		args,
-		params, sd.StateNodeAppender(&stateNodes), sd.CodeMappingAppender(&codeAndCodeHashes))
+		params, sd.StateNodeAppender(&stateNodes), sd.IPLDMappingAppender(&codeAndCodeHashes))
 	if err != nil {
 		return sdtypes.StateObject{}, err
 	}
@@ -67,12 +67,12 @@ func (sdb *builder) BuildStateDiffObject(args sd.Args, params sd.Params) (sdtype
 		BlockHash:   args.BlockHash,
 		BlockNumber: args.BlockNumber,
 		Nodes:       stateNodes,
-		CodeAndCodeHashes: codeAndCodeHashes,
+		IPLDs:       codeAndCodeHashes,
 	}, nil
 }
 
 // WriteStateDiffObject writes a statediff object to output callback
-func (sdb *builder) WriteStateDiffObject(args sd.Args, params sd.Params, output sdtypes.StateNodeSink, codeOutput sdtypes.CodeSink) error {
+func (sdb *builder) WriteStateDiffObject(args sd.Args, params sd.Params, output sdtypes.StateNodeSink, codeOutput sdtypes.IPLDSink) error {
 	// Load tries for old and new states
 	oldTrie, err := sdb.StateCache.OpenTrie(args.OldStateRoot)
 	if err != nil {
@@ -101,12 +101,12 @@ func (sdb *builder) WriteStateDiffObject(args sd.Args, params sd.Params, output
 	}
 
 	// Dispatch workers to process trie data; sync and collect results here via channels
-	nodeChan := make(chan sdtypes.StateNode)
+	nodeChan := make(chan sdtypes.StateLeafNode)
-	codeChan := make(chan sdtypes.CodeAndCodeHash)
+	codeChan := make(chan sdtypes.IPLD)
 
 	go func() {
-		nodeSender := func(node sdtypes.StateNode) error { nodeChan <- node; return nil }
+		nodeSender := func(node sdtypes.StateLeafNode) error { nodeChan <- node; return nil }
-		codeSender := func(code sdtypes.CodeAndCodeHash) error { codeChan <- code; return nil }
+		ipldSender := func(code sdtypes.IPLD) error { codeChan <- code; return nil }
 		var wg sync.WaitGroup
 
 		for w := uint(0); w < sdb.numWorkers; w++ {
@@ -115,12 +115,7 @@ func (sdb *builder) WriteStateDiffObject(args sd.Args, params sd.Params, output
 				defer wg.Done()
 				var err error
 				logger := log.New("hash", args.BlockHash.Hex(), "number", args.BlockNumber)
-				if !params.IntermediateStateNodes {
+				err = sdb.BuildStateDiffWithIntermediateStateNodes(iterPairs[worker], params, nodeSender, ipldSender, logger)
-					err = sdb.BuildStateDiffWithoutIntermediateStateNodes(iterPairs[worker], params, nodeSender, codeSender, logger)
-				} else {
-					err = sdb.BuildStateDiffWithIntermediateStateNodes(iterPairs[worker], params, nodeSender, codeSender, logger)
-				}
-
 				if err != nil {
 					logrus.Errorf("buildStateDiff error for worker %d, params %+v", worker, params)
 				}

pkg/helpers.go

@@ -1,72 +0,0 @@
-// Copyright 2019 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-// Contains a batch of utility type declarations used by the tests. As the node
-// operates on unique types, a lot of them are needed to check various features.
-
-package statediff
-
-import (
-	"sort"
-	"strings"
-)
-
-func sortKeys(data AccountMap) []string {
-	keys := make([]string, 0, len(data))
-	for key := range data {
-		keys = append(keys, key)
-	}
-	sort.Strings(keys)
-
-	return keys
-}
-
-// findIntersection finds the set of strings from both arrays that are equivalent
-// a and b must first be sorted
-// this is used to find which keys have been both "deleted" and "created" i.e. they were updated
-func findIntersection(a, b []string) []string {
-	lenA := len(a)
-	lenB := len(b)
-	iOfA, iOfB := 0, 0
-	updates := make([]string, 0)
-	if iOfA >= lenA || iOfB >= lenB {
-		return updates
-	}
-	for {
-		switch strings.Compare(a[iOfA], b[iOfB]) {
-		// -1 when a[iOfA] < b[iOfB]
-		case -1:
-			iOfA++
-			if iOfA >= lenA {
-				return updates
-			}
-			// 0 when a[iOfA] == b[iOfB]
-		case 0:
-			updates = append(updates, a[iOfA])
-			iOfA++
-			iOfB++
-			if iOfA >= lenA || iOfB >= lenB {
-				return updates
-			}
-			// 1 when a[iOfA] > b[iOfB]
-		case 1:
-			iOfB++
-			if iOfB >= lenB {
-				return updates
-			}
-		}
-	}
-}

pkg/rpc/check.go

@@ -1,37 +0,0 @@
-// VulcanizeDB
-// Copyright © 2020 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package rpc
-
-import "github.com/ethereum/go-ethereum/rpc"
-
-// checkModuleAvailability check that all names given in modules are actually
-// available API services.
-func checkModuleAvailability(modules []string, apis []rpc.API) (bad, available []string) {
-	availableSet := make(map[string]struct{})
-	for _, api := range apis {
-		if _, ok := availableSet[api.Namespace]; !ok {
-			availableSet[api.Namespace] = struct{}{}
-			available = append(available, api.Namespace)
-		}
-	}
-	for _, name := range modules {
-		if _, ok := availableSet[name]; !ok {
-			bad = append(bad, name)
-		}
-	}
-	return bad, available
-}

pkg/service.go

@@ -53,8 +53,6 @@ type StateDiffService interface {
 	StateDiffAt(blockNumber uint64, params sd.Params) (*sd.Payload, error)
 	// StateDiffFor method to get state diff object at specific block
 	StateDiffFor(blockHash common.Hash, params sd.Params) (*sd.Payload, error)
-	// StateTrieAt method to get state trie object at specific block
-	StateTrieAt(blockNumber uint64, params sd.Params) (*sd.Payload, error)
 	// WriteStateDiffAt method to write state diff object directly to DB
 	WriteStateDiffAt(blockNumber uint64, params sd.Params) error
 	// WriteStateDiffFor method to get state trie object at specific block
@@ -344,34 +342,6 @@ func (sds *Service) newPayload(stateObject []byte, block *types.Block, params sd
 	return payload, nil
 }
 
-// StateTrieAt returns a state trie object payload at the specified blockheight
-// This operation cannot be performed back past the point of db pruning; it requires an archival node for historical data
-func (sds *Service) StateTrieAt(blockNumber uint64, params sd.Params) (*sd.Payload, error) {
-	currentBlock, err := sds.lvlDBReader.GetBlockByNumber(blockNumber)
-	if err != nil {
-		return nil, err
-	}
-	logrus.Infof("sending state trie at block %d", blockNumber)
-
-	// compute leaf paths of watched addresses in the params
-	params.ComputeWatchedAddressesLeafPaths()
-
-	return sds.processStateTrie(currentBlock, params)
-}
-
-func (sds *Service) processStateTrie(block *types.Block, params sd.Params) (*sd.Payload, error) {
-	stateNodes, err := sds.Builder.BuildStateTrieObject(block)
-	if err != nil {
-		return nil, err
-	}
-	stateTrieRlp, err := rlp.EncodeToBytes(&stateNodes)
-	if err != nil {
-		return nil, err
-	}
-	logrus.Infof("state trie object at block %d is %d bytes in length", block.Number().Uint64(), len(stateTrieRlp))
-	return sds.newPayload(stateTrieRlp, block, params)
-}
-
 // Start is used to begin the service
 func (sds *Service) Start() error {
 	logrus.Info("starting statediff service")
@@ -461,11 +431,11 @@ func (sds *Service) writeStateDiff(block *types.Block, parentRoot common.Hash, p
 		return err
 	}
 	// defer handling of commit/rollback for any return case
-	output := func(node sdtypes.StateNode) error {
+	output := func(node sdtypes.StateLeafNode) error {
 		return sds.indexer.PushStateNode(tx, node, block.Hash().String())
 	}
-	codeOutput := func(c sdtypes.CodeAndCodeHash) error {
+	codeOutput := func(c sdtypes.IPLD) error {
-		return sds.indexer.PushCodeAndCodeHash(tx, c)
+		return sds.indexer.PushIPLD(tx, c)
 	}
 	prom.SetTimeMetric(prom.T_BLOCK_PROCESSING, time.Now().Sub(t))
 	t = time.Now()

pkg/types.go

@@ -20,23 +20,9 @@
 package statediff
 
 import (
-	"github.com/ethereum/go-ethereum/core/types"
 	sd "github.com/ethereum/go-ethereum/statediff"
-	sdTypes "github.com/ethereum/go-ethereum/statediff/types"
 )
 
-// AccountMap is a mapping of hex encoded path => account wrapper
-type AccountMap map[string]accountWrapper
-
-// accountWrapper is used to temporary associate the unpacked node with its raw values
-type accountWrapper struct {
-	Account   *types.StateAccount
-	NodeType  sdTypes.NodeType
-	Path      []byte
-	NodeValue []byte
-	LeafKey   []byte
-}
-
 // RangeRequest holds range quest work params
 type RangeRequest struct {
 	Start, Stop uint64