From 41ac23ab8afbd263d2c30813ea5b5d8053fc27ad Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Thu, 7 Feb 2019 15:33:42 -0600 Subject: [PATCH] Use a node's LeafKey as an identifier instead of the address It was proving difficult to find look the address up from a given path with a full node (sometimes the value wouldn't exist in the disk db). So, instead, for now we are using the node's LeafKey with is a Keccak256 hash of the address, so if we know the address we can figure out which LeafKey it matches up to. --- statediff/builder/builder.go | 41 +++++++++++------------- statediff/builder/builder_test.go | 46 +++++++++++++++------------ statediff/builder/helpers.go | 3 +- statediff/builder/struct.go | 11 ++++--- statediff/publisher/csv.go | 4 +-- statediff/publisher/publisher_test.go | 8 ++--- statediff/testhelpers/test_data.go | 20 ++++++++---- 7 files changed, 71 insertions(+), 62 deletions(-) diff --git a/statediff/builder/builder.go b/statediff/builder/builder.go index 716fd5693..dbb7d3d8b 100644 --- a/statediff/builder/builder.go +++ b/statediff/builder/builder.go @@ -41,6 +41,8 @@ type builder struct { blockChain *core.BlockChain } +type AccountsMap map[common.Hash]*state.Account + func NewBuilder(db ethdb.Database, blockChain *core.BlockChain) *builder { return &builder{ chainDB: db, @@ -112,33 +114,28 @@ func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, block }, nil } -func (sdb *builder) collectDiffNodes(a, b trie.NodeIterator) (map[common.Address]*state.Account, error) { - var diffAccounts = make(map[common.Address]*state.Account) +func (sdb *builder) collectDiffNodes(a, b trie.NodeIterator) (AccountsMap, error) { + var diffAccounts = make(AccountsMap) it, _ := trie.NewDifferenceIterator(a, b) for { log.Debug("Current Path and Hash", "path", pathToStr(it), "hashold", it.Hash()) if it.Leaf() { - // lookup address - path := make([]byte, len(it.Path())-1) - copy(path, it.Path()) - addr, err := sdb.addressByPath(path) - if err != nil { - log.Error("Error looking up address via path", "path", path, "error", err) - return nil, err - } + leafKey := make([]byte, len(it.LeafKey())) + copy(leafKey, it.LeafKey()) + leafKeyHash := common.BytesToHash(leafKey) // lookup account state var account state.Account if err := rlp.DecodeBytes(it.LeafBlob(), &account); err != nil { - log.Error("Error looking up account via address", "address", addr, "error", err) + log.Error("Error looking up account via address", "address", leafKeyHash, "error", err) return nil, err } // record account to diffs (creation if we are looking at new - old; deletion if old - new) - log.Debug("Account lookup successful", "address", addr, "account", account) - diffAccounts[*addr] = &account + log.Debug("Account lookup successful", "address", leafKeyHash, "account", account) + diffAccounts[leafKeyHash] = &account } cont := it.Next(true) if !cont { @@ -149,8 +146,8 @@ func (sdb *builder) collectDiffNodes(a, b trie.NodeIterator) (map[common.Address return diffAccounts, nil } -func (sdb *builder) buildDiffEventual(accounts map[common.Address]*state.Account) (map[common.Address]AccountDiff, error) { - accountDiffs := make(map[common.Address]AccountDiff) +func (sdb *builder) buildDiffEventual(accounts AccountsMap) (AccountDiffsMap, error) { + accountDiffs := make(AccountDiffsMap) for addr, val := range accounts { sr := val.Root storageDiffs, err := sdb.buildStorageDiffsEventual(sr) @@ -176,11 +173,11 @@ func (sdb *builder) buildDiffEventual(accounts map[common.Address]*state.Account return accountDiffs, nil } -func (sdb *builder) buildDiffIncremental(creations map[common.Address]*state.Account, deletions map[common.Address]*state.Account, updatedKeys []string) (map[common.Address]AccountDiff, error) { - updatedAccounts := make(map[common.Address]AccountDiff) +func (sdb *builder) buildDiffIncremental(creations AccountsMap, deletions AccountsMap, updatedKeys []string) (AccountDiffsMap, error) { + updatedAccounts := make(AccountDiffsMap) for _, val := range updatedKeys { - createdAcc := creations[common.HexToAddress(val)] - deletedAcc := deletions[common.HexToAddress(val)] + createdAcc := creations[common.HexToHash(val)] + deletedAcc := deletions[common.HexToHash(val)] oldSR := deletedAcc.Root newSR := createdAcc.Root if storageDiffs, err := sdb.buildStorageDiffsIncremental(oldSR, newSR); err != nil { @@ -194,15 +191,15 @@ func (sdb *builder) buildDiffIncremental(creations map[common.Address]*state.Acc nHexRoot := createdAcc.Root.Hex() contractRoot := DiffString{Value: &nHexRoot} - updatedAccounts[common.HexToAddress(val)] = AccountDiff{ + updatedAccounts[common.HexToHash(val)] = AccountDiff{ Nonce: nonce, Balance: balance, CodeHash: codeHash, ContractRoot: contractRoot, Storage: storageDiffs, } - delete(creations, common.HexToAddress(val)) - delete(deletions, common.HexToAddress(val)) + delete(creations, common.HexToHash(val)) + delete(deletions, common.HexToHash(val)) } } return updatedAccounts, nil diff --git a/statediff/builder/builder_test.go b/statediff/builder/builder_test.go index 62bb94de5..72189c05d 100644 --- a/statediff/builder/builder_test.go +++ b/statediff/builder/builder_test.go @@ -15,6 +15,7 @@ import ( "github.com/ethereum/go-ethereum/params" b "github.com/ethereum/go-ethereum/statediff/builder" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/statediff/testhelpers" ) var ( @@ -22,26 +23,32 @@ var ( testBankKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") testBankAddress = crypto.PubkeyToAddress(testBankKey.PublicKey) //0x71562b71999873DB5b286dF957af199Ec94617F7 + bankLeafKey = testhelpers.AddressToLeafKey(testBankAddress) testBankFunds = big.NewInt(100000000) genesis = core.GenesisBlockForTesting(testdb, testBankAddress, testBankFunds) account1Key, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") account2Key, _ = crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee") account1Addr = crypto.PubkeyToAddress(account1Key.PublicKey) //0x703c4b2bD70c169f5717101CaeE543299Fc946C7 + account1LeafKey = testhelpers.AddressToLeafKey(account1Addr) account2Addr = crypto.PubkeyToAddress(account2Key.PublicKey) //0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e + account2LeafKey = testhelpers.AddressToLeafKey(account2Addr) contractCode = common.Hex2Bytes("608060405234801561001057600080fd5b50602060405190810160405280600160ff16815250600090600161003592919061003b565b506100a5565b826064810192821561006f579160200282015b8281111561006e578251829060ff1690559160200191906001019061004e565b5b50905061007c9190610080565b5090565b6100a291905b8082111561009e576000816000905550600101610086565b5090565b90565b610124806100b46000396000f3fe6080604052348015600f57600080fd5b5060043610604f576000357c01000000000000000000000000000000000000000000000000000000009004806360cd2685146054578063c16431b9146093575b600080fd5b607d60048036036020811015606857600080fd5b810190808035906020019092919050505060c8565b6040518082815260200191505060405180910390f35b60c66004803603604081101560a757600080fd5b81019080803590602001909291908035906020019092919050505060e0565b005b6000808260648110151560d757fe5b01549050919050565b8060008360648110151560ef57fe5b0181905550505056fea165627a7a7230582064e918c3140a117bf3aa65865a9b9e83fae21ad1720506e7933b2a9f54bb40260029") contractAddr common.Address - emptyAccountDiffEventualMap = make(map[common.Address]b.AccountDiff) - emptyAccountDiffIncrementalMap = make(map[common.Address]b.AccountDiff) + contractLeafKey common.Hash + emptyAccountDiffEventualMap = make(b.AccountDiffsMap) + emptyAccountDiffIncrementalMap = make(b.AccountDiffsMap) block0Hash, block1Hash, block2Hash, block3Hash common.Hash block0, block1, block2, block3 *types.Block builder b.Builder miningReward = int64(2000000000000000000) burnAddress = common.HexToAddress("0x0") + burnLeafKey = testhelpers.AddressToLeafKey(burnAddress) ) func TestBuilder(t *testing.T) { _, blockMap, chain := makeChain(3, genesis) + contractLeafKey = testhelpers.AddressToLeafKey(contractAddr) defer chain.Stop() block0Hash = common.HexToHash("0xd1721cfd0b29c36fd7a68f25c128e86413fb666a6e1d68e89b875bd299262661") block1Hash = common.HexToHash("0xbbe88de60ba33a3f18c0caa37d827bfb70252e19e40a07cd34041696c35ecb1a") @@ -115,15 +122,15 @@ func TestBuilder(t *testing.T) { &b.StateDiff{ BlockNumber: block1.Number().Int64(), BlockHash: block1.Hash(), - CreatedAccounts: map[common.Address]b.AccountDiff{ - account1Addr: { + CreatedAccounts: b.AccountDiffsMap{ + account1LeafKey: { Nonce: b.DiffUint64{Value: &nonce0}, Balance: b.DiffBigInt{Value: big.NewInt(balanceChange10000)}, CodeHash: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", ContractRoot: b.DiffString{Value: &originalContractRoot}, Storage: map[string]b.DiffStorage{}, }, - burnAddress: { + burnLeafKey: { Nonce: b.DiffUint64{Value: &nonce0}, Balance: b.DiffBigInt{Value: big.NewInt(miningReward)}, CodeHash: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", @@ -132,8 +139,8 @@ func TestBuilder(t *testing.T) { }, }, DeletedAccounts: emptyAccountDiffEventualMap, - UpdatedAccounts: map[common.Address]b.AccountDiff{ - testBankAddress: { + UpdatedAccounts: b.AccountDiffsMap{ + bankLeafKey: { Nonce: b.DiffUint64{Value: &nonce1}, Balance: b.DiffBigInt{Value: big.NewInt(testBankFunds.Int64() - balanceChange10000)}, CodeHash: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", @@ -156,15 +163,15 @@ func TestBuilder(t *testing.T) { &b.StateDiff{ BlockNumber: block2.Number().Int64(), BlockHash: block2.Hash(), - CreatedAccounts: map[common.Address]b.AccountDiff{ - account2Addr: { + CreatedAccounts: b.AccountDiffsMap{ + account2LeafKey: { Nonce: b.DiffUint64{Value: &nonce0}, Balance: b.DiffBigInt{Value: big.NewInt(balanceChange1000)}, CodeHash: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", ContractRoot: b.DiffString{Value: &originalContractRoot}, Storage: map[string]b.DiffStorage{}, }, - contractAddr: { + contractLeafKey: { Nonce: b.DiffUint64{Value: &nonce1}, Balance: b.DiffBigInt{Value: big.NewInt(0)}, CodeHash: "0x753f98a8d4328b15636e46f66f2cb4bc860100aa17967cc145fcd17d1d4710ea", @@ -177,22 +184,22 @@ func TestBuilder(t *testing.T) { }, }, DeletedAccounts: emptyAccountDiffEventualMap, - UpdatedAccounts: map[common.Address]b.AccountDiff{ - testBankAddress: { + UpdatedAccounts: b.AccountDiffsMap{ + bankLeafKey: { Nonce: b.DiffUint64{Value: &nonce2}, Balance: b.DiffBigInt{Value: big.NewInt(block1BankBalance - balanceChange1000)}, CodeHash: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", ContractRoot: b.DiffString{Value: &originalContractRoot}, Storage: map[string]b.DiffStorage{}, }, - account1Addr: { + account1LeafKey: { Nonce: b.DiffUint64{Value: &nonce2}, Balance: b.DiffBigInt{Value: big.NewInt(block1Account1Balance - balanceChange1000 + balanceChange1000)}, CodeHash: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", ContractRoot: b.DiffString{Value: &originalContractRoot}, Storage: map[string]b.DiffStorage{}, }, - burnAddress: { + burnLeafKey: { Nonce: b.DiffUint64{Value: &nonce0}, Balance: b.DiffBigInt{Value: big.NewInt(miningReward + miningReward)}, CodeHash: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", @@ -215,17 +222,17 @@ func TestBuilder(t *testing.T) { &b.StateDiff{ BlockNumber: block3.Number().Int64(), BlockHash: block3.Hash(), - CreatedAccounts: map[common.Address]b.AccountDiff{}, + CreatedAccounts: b.AccountDiffsMap{}, DeletedAccounts: emptyAccountDiffEventualMap, - UpdatedAccounts: map[common.Address]b.AccountDiff{ - account2Addr: { + UpdatedAccounts: b.AccountDiffsMap{ + account2LeafKey: { Nonce: b.DiffUint64{Value: &nonce0}, Balance: b.DiffBigInt{Value: big.NewInt(block2Account2Balance + miningReward)}, CodeHash: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", ContractRoot: b.DiffString{Value: &originalContractRoot}, Storage: map[string]b.DiffStorage{}, }, - contractAddr: { + contractLeafKey: { Nonce: b.DiffUint64{Value: &nonce1}, Balance: b.DiffBigInt{Value: big.NewInt(0)}, CodeHash: "0x753f98a8d4328b15636e46f66f2cb4bc860100aa17967cc145fcd17d1d4710ea", @@ -236,7 +243,7 @@ func TestBuilder(t *testing.T) { Value: &updatedStorageValue}, }, }, - testBankAddress: { + bankLeafKey: { Nonce: b.DiffUint64{Value: &nonce3}, Balance: b.DiffBigInt{Value: big.NewInt(99989000)}, CodeHash: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", @@ -254,7 +261,6 @@ func TestBuilder(t *testing.T) { if err != nil { t.Error(err) } - fields := []string{"BlockNumber", "BlockHash", "DeletedAccounts", "UpdatedAccounts", "CreatedAccounts"} for _, field := range fields { diff --git a/statediff/builder/helpers.go b/statediff/builder/helpers.go index 891525cc2..339454776 100644 --- a/statediff/builder/helpers.go +++ b/statediff/builder/helpers.go @@ -24,11 +24,10 @@ import ( "strings" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/trie" ) -func sortKeys(data map[common.Address]*state.Account) []string { +func sortKeys(data AccountsMap) []string { var keys []string for key := range data { keys = append(keys, key.Hex()) diff --git a/statediff/builder/struct.go b/statediff/builder/struct.go index 295b09f10..416d8e816 100644 --- a/statediff/builder/struct.go +++ b/statediff/builder/struct.go @@ -26,12 +26,13 @@ import ( "github.com/ethereum/go-ethereum/common" ) +type AccountDiffsMap map[common.Hash]AccountDiff type StateDiff struct { - BlockNumber int64 `json:"blockNumber" gencodec:"required"` - BlockHash common.Hash `json:"blockHash" gencodec:"required"` - CreatedAccounts map[common.Address]AccountDiff `json:"createdAccounts" gencodec:"required"` - DeletedAccounts map[common.Address]AccountDiff `json:"deletedAccounts" gencodec:"required"` - UpdatedAccounts map[common.Address]AccountDiff `json:"updatedAccounts" gencodec:"required"` + BlockNumber int64 `json:"blockNumber" gencodec:"required"` + BlockHash common.Hash `json:"blockHash" gencodec:"required"` + CreatedAccounts AccountDiffsMap `json:"createdAccounts" gencodec:"required"` + DeletedAccounts AccountDiffsMap `json:"deletedAccounts" gencodec:"required"` + UpdatedAccounts AccountDiffsMap `json:"updatedAccounts" gencodec:"required"` encoded []byte err error diff --git a/statediff/publisher/csv.go b/statediff/publisher/csv.go index 54526951a..13971a5c8 100644 --- a/statediff/publisher/csv.go +++ b/statediff/publisher/csv.go @@ -15,7 +15,7 @@ var ( Headers = []string{ "blockNumber", "blockHash", "accountAction", "codeHash", "nonceValue", "balanceValue", "contractRoot", "storageDiffPaths", - "accountAddress", "storageKey", "storageValue", + "accountLeafKey", "storageKey", "storageValue", } timeStampFormat = "20060102150405.00000" @@ -81,7 +81,7 @@ func accumulateAccountRows(sd builder.StateDiff) [][]string { return accountRows } -func formatAccountData(accountAddr common.Address, accountDiff builder.AccountDiff, sd builder.StateDiff, accountAction string) [][]string { +func formatAccountData(accountAddr common.Hash, accountDiff builder.AccountDiff, sd builder.StateDiff, accountAction string) [][]string { blockNumberString := strconv.FormatInt(sd.BlockNumber, 10) blockHash := sd.BlockHash.String() codeHash := accountDiff.CodeHash diff --git a/statediff/publisher/publisher_test.go b/statediff/publisher/publisher_test.go index 8a26fb4e2..76aaf961e 100644 --- a/statediff/publisher/publisher_test.go +++ b/statediff/publisher/publisher_test.go @@ -35,7 +35,7 @@ var expectedCreatedAccountRow = []string{ strconv.FormatInt(testhelpers.NewBalanceValue, 10), testhelpers.ContractRoot, testhelpers.StoragePath, - testhelpers.ContractAddress, + testhelpers.ContractLeafKey.Hex(), "0000000000000000000000000000000000000000000000000000000000000001", testhelpers.StorageValue, } @@ -49,7 +49,7 @@ var expectedCreatedAccountWithoutStorageUpdateRow = []string{ strconv.FormatInt(testhelpers.NewBalanceValue, 10), testhelpers.ContractRoot, "", - testhelpers.AnotherContractAddress, + testhelpers.AnotherContractLeafKey.Hex(), "", "", } @@ -63,7 +63,7 @@ var expectedUpdatedAccountRow = []string{ strconv.FormatInt(testhelpers.NewBalanceValue, 10), testhelpers.ContractRoot, testhelpers.StoragePath, - testhelpers.ContractAddress, + testhelpers.ContractLeafKey.Hex(), "0000000000000000000000000000000000000000000000000000000000000001", testhelpers.StorageValue, } @@ -77,7 +77,7 @@ var expectedDeletedAccountRow = []string{ strconv.FormatInt(testhelpers.NewBalanceValue, 10), testhelpers.ContractRoot, testhelpers.StoragePath, - testhelpers.ContractAddress, + testhelpers.ContractLeafKey.Hex(), "0000000000000000000000000000000000000000000000000000000000000001", testhelpers.StorageValue, } diff --git a/statediff/testhelpers/test_data.go b/statediff/testhelpers/test_data.go index a24f32c15..b3ca15bc2 100644 --- a/statediff/testhelpers/test_data.go +++ b/statediff/testhelpers/test_data.go @@ -6,8 +6,14 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/statediff/builder" + "github.com/ethereum/go-ethereum/crypto" ) + +func AddressToLeafKey(address common.Address) common.Hash { + return common.BytesToHash(crypto.Keccak256(address[:])) +} + var ( BlockNumber = rand.Int63() BlockHash = "0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73" @@ -24,18 +30,18 @@ var ( }} emptyStorage = map[string]builder.DiffStorage{} address = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592") + ContractLeafKey = AddressToLeafKey(address) anotherAddress = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476593") - ContractAddress = address.String() - AnotherContractAddress = anotherAddress.String() - CreatedAccountDiffs = map[common.Address]builder.AccountDiff{ - address: { + AnotherContractLeafKey = AddressToLeafKey(anotherAddress) + CreatedAccountDiffs = builder.AccountDiffsMap{ + ContractLeafKey: { Nonce: builder.DiffUint64{Value: &NewNonceValue}, Balance: builder.DiffBigInt{Value: big.NewInt(NewBalanceValue)}, ContractRoot: builder.DiffString{Value: &ContractRoot}, CodeHash: CodeHash, Storage: storage, }, - anotherAddress: { + AnotherContractLeafKey: { Nonce: builder.DiffUint64{Value: &NewNonceValue}, Balance: builder.DiffBigInt{Value: big.NewInt(NewBalanceValue)}, CodeHash: CodeHash, @@ -44,7 +50,7 @@ var ( }, } - UpdatedAccountDiffs = map[common.Address]builder.AccountDiff{address: { + UpdatedAccountDiffs = builder.AccountDiffsMap{ContractLeafKey: { Nonce: builder.DiffUint64{Value: &NewNonceValue}, Balance: builder.DiffBigInt{Value: big.NewInt(NewBalanceValue)}, CodeHash: CodeHash, @@ -52,7 +58,7 @@ var ( Storage: storage, }} - DeletedAccountDiffs = map[common.Address]builder.AccountDiff{address: { + DeletedAccountDiffs = builder.AccountDiffsMap{ContractLeafKey: { Nonce: builder.DiffUint64{Value: &NewNonceValue}, Balance: builder.DiffBigInt{Value: big.NewInt(NewBalanceValue)}, ContractRoot: builder.DiffString{Value: &ContractRoot},