cerc-io/ipld-eth-server
12
0
Fork 1
Code Issues 10 Pull Requests 2 Actions Packages Projects Releases 37 Wiki Activity

integrate backfill into storage watcher; documentation for storage backfill

Browse Source
This commit is contained in:
Ian Norden
2019-12-02 11:06:28 -06:00
parent b454b61777
commit 37f4a2d603
18 changed files with 526 additions and 297 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libraries/shared/storage/backfill.go
+27 -65
View File
@@ -17,12 +17,9 @@
package storage
import (
"bytes"
"errors"
"fmt"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/statediff"
"github.com/sirupsen/logrus"
@@ -31,52 +28,39 @@ import (
"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
)
// IBackFiller is the backfilling interface
type IBackFiller interface {
BackFill(bfa BackFillerArgs) (map[common.Hash][]utils.StorageDiff, error)
// BackFiller is the backfilling interface
type BackFiller interface {
BackFill(startingBlock, endingBlock uint64) ([]utils.StorageDiff, error)
}
// BackFiller is the backfilling struct
type BackFiller struct {
sdf fetcher.IStateDiffFetcher
// backFiller is the backfilling struct
type backFiller struct {
fetcher fetcher.StateDiffFetcher
}
// BackFillerArgs are used to pass configuration params to the backfiller
type BackFillerArgs struct {
// mapping of hashed addresses to a list of the storage key hashes we want to collect at that address
WantedStorage map[common.Hash][]common.Hash
StartingBlock uint64
EndingBlock uint64
}
// NewStorageBackFiller returns a IBackFiller
func NewStorageBackFiller(bc fetcher.BatchClient) IBackFiller {
return &BackFiller{
sdf: fetcher.NewStateDiffFetcher(bc),
// NewStorageBackFiller returns a BackFiller
func NewStorageBackFiller(fetcher fetcher.StateDiffFetcher) BackFiller {
return &backFiller{
fetcher: fetcher,
}
}
// BackFill uses the provided config to fetch and return the state diff at the specified blocknumber
// StateDiffAt(ctx context.Context, blockNumber uint64) (*Payload, error)
func (bf *BackFiller) BackFill(bfa BackFillerArgs) (map[common.Hash][]utils.StorageDiff, error) {
results := make(map[common.Hash][]utils.StorageDiff, len(bfa.WantedStorage))
if bfa.EndingBlock < bfa.StartingBlock {
func (bf *backFiller) BackFill(startingBlock, endingBlock uint64) ([]utils.StorageDiff, error) {
results := make([]utils.StorageDiff, 0)
if endingBlock < startingBlock {
return nil, errors.New("backfill: ending block number needs to be greater than starting block number")
}
blockHeights := make([]uint64, 0, bfa.EndingBlock-bfa.StartingBlock+1)
for i := bfa.StartingBlock; i <= bfa.EndingBlock; i++ {
blockHeights := make([]uint64, 0, endingBlock-startingBlock+1)
for i := startingBlock; i <= endingBlock; i++ {
blockHeights = append(blockHeights, i)
}
payloads, err := bf.sdf.FetchStateDiffsAt(blockHeights)
payloads, err := bf.fetcher.FetchStateDiffsAt(blockHeights)
if err != nil {
return nil, err
}
for _, payload := range payloads {
block := new(types.Block)
blockDecodeErr := rlp.DecodeBytes(payload.BlockRlp, block)
if blockDecodeErr != nil {
return nil, blockDecodeErr
}
stateDiff := new(statediff.StateDiff)
stateDiffDecodeErr := rlp.DecodeBytes(payload.StateDiffRlp, stateDiff)
if stateDiffDecodeErr != nil {
@@ -84,42 +68,20 @@ func (bf *BackFiller) BackFill(bfa BackFillerArgs) (map[common.Hash][]utils.Stor
}
accounts := utils.GetAccountsFromDiff(*stateDiff)
for _, account := range accounts {
if wantedHashedAddress(bfa.WantedStorage, common.BytesToHash(account.Key)) {
logrus.Trace(fmt.Sprintf("iterating through %d Storage values on account", len(account.Storage)))
for _, storage := range account.Storage {
if wantedHashedStorageKey(bfa.WantedStorage[common.BytesToHash(account.Key)], storage.Key) {
diff, formatErr := utils.FromGethStateDiff(account, stateDiff, storage)
logrus.Trace("adding storage diff to out channel",
"keccak of address: ", diff.HashedAddress.Hex(),
"block height: ", diff.BlockHeight,
"storage key: ", diff.StorageKey.Hex(),
"storage value: ", diff.StorageValue.Hex())
if formatErr != nil {
return nil, formatErr
}
results[diff.HashedAddress] = append(results[diff.HashedAddress], diff)
}
logrus.Trace(fmt.Sprintf("iterating through %d Storage values on account", len(account.Storage)))
for _, storage := range account.Storage {
diff, formatErr := utils.FromGethStateDiff(account, stateDiff, storage)
if formatErr != nil {
return nil, formatErr
}
logrus.Trace("adding storage diff to results",
"keccak of address: ", diff.HashedAddress.Hex(),
"block height: ", diff.BlockHeight,
"storage key: ", diff.StorageKey.Hex(),
"storage value: ", diff.StorageValue.Hex())
results = append(results, diff)
}
}
}
return results, nil
}
func wantedHashedAddress(wantedStorage map[common.Hash][]common.Hash, hashedKey common.Hash) bool {
for addrHash := range wantedStorage {
if bytes.Equal(addrHash.Bytes(), hashedKey.Bytes()) {
return true
}
}
return false
}
func wantedHashedStorageKey(wantedKeys []common.Hash, keyBytes []byte) bool {
for _, key := range wantedKeys {
if bytes.Equal(key.Bytes(), keyBytes) {
return true
}
}
return false
}
libraries/shared/storage/backfill_test.go
+13 -93
View File
@@ -18,94 +18,43 @@ package storage_test
import (
"bytes"
"encoding/json"
"errors"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/statediff"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
"github.com/vulcanize/vulcanizedb/libraries/shared/storage"
"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
"github.com/vulcanize/vulcanizedb/libraries/shared/test_data"
"github.com/vulcanize/vulcanizedb/pkg/geth/client"
)
type mockClient struct {
MappedStateDiffAt map[uint64][]byte
}
// SetReturnDiffAt method to set what statediffs the mock client returns
func (mc *mockClient) SetReturnDiffAt(height uint64, diffPayload statediff.Payload) error {
if mc.MappedStateDiffAt == nil {
mc.MappedStateDiffAt = make(map[uint64][]byte)
}
by, err := json.Marshal(diffPayload)
if err != nil {
return err
}
mc.MappedStateDiffAt[height] = by
return nil
}
// BatchCall mockClient method to simulate batch call to geth
func (mc *mockClient) BatchCall(batch []client.BatchElem) error {
if mc.MappedStateDiffAt == nil {
return errors.New("mockclient needs to be initialized with statediff payloads and errors")
}
for _, batchElem := range batch {
if len(batchElem.Args) != 1 {
return errors.New("expected batch elem to contain single argument")
}
blockHeight, ok := batchElem.Args[0].(uint64)
if !ok {
return errors.New("expected batch elem argument to be a uint64")
}
err := json.Unmarshal(mc.MappedStateDiffAt[blockHeight], batchElem.Result)
if err != nil {
return err
}
}
return nil
}
var _ = Describe("BackFiller", func() {
Describe("BackFill", func() {
var (
mc *mockClient
backFiller storage.IBackFiller
fetcher *mocks.StateDiffFetcher
backFiller storage.BackFiller
)
BeforeEach(func() {
mc = new(mockClient)
setDiffAtErr1 := mc.SetReturnDiffAt(test_data.BlockNumber.Uint64(), test_data.MockStatediffPayload)
Expect(setDiffAtErr1).ToNot(HaveOccurred())
setDiffAtErr2 := mc.SetReturnDiffAt(test_data.BlockNumber2.Uint64(), test_data.MockStatediffPayload2)
Expect(setDiffAtErr2).ToNot(HaveOccurred())
backFiller = storage.NewStorageBackFiller(mc)
fetcher = new(mocks.StateDiffFetcher)
fetcher.SetPayloadsToReturn(map[uint64]*statediff.Payload{
test_data.BlockNumber.Uint64(): &test_data.MockStatediffPayload,
test_data.BlockNumber2.Uint64(): &test_data.MockStatediffPayload2,
})
backFiller = storage.NewStorageBackFiller(fetcher)
})
It("Batch calls statediff_stateDiffAt", func() {
backFillArgs := storage.BackFillerArgs{
WantedStorage: map[common.Hash][]common.Hash{
test_data.ContractLeafKey: {common.BytesToHash(test_data.StorageKey)},
test_data.AnotherContractLeafKey: {common.BytesToHash(test_data.StorageKey)},
},
StartingBlock: test_data.BlockNumber.Uint64(),
EndingBlock: test_data.BlockNumber2.Uint64(),
}
backFillStorage, backFillErr := backFiller.BackFill(backFillArgs)
backFillStorage, backFillErr := backFiller.BackFill(test_data.BlockNumber.Uint64(), test_data.BlockNumber2.Uint64())
Expect(backFillErr).ToNot(HaveOccurred())
Expect(len(backFillStorage)).To(Equal(2))
Expect(len(backFillStorage[test_data.ContractLeafKey])).To(Equal(1))
Expect(len(backFillStorage[test_data.AnotherContractLeafKey])).To(Equal(3))
Expect(backFillStorage[test_data.ContractLeafKey][0]).To(Equal(test_data.CreatedExpectedStorageDiff))
Expect(len(backFillStorage)).To(Equal(4))
// Can only rlp encode the slice of diffs as part of a struct
// Rlp encoding allows us to compare content of the slices when the order in the slice may vary
expectedDiffStruct := struct {
diffs []utils.StorageDiff
}{
[]utils.StorageDiff{
test_data.CreatedExpectedStorageDiff,
test_data.UpdatedExpectedStorageDiff,
test_data.UpdatedExpectedStorageDiff2,
test_data.DeletedExpectedStorageDiff,
@@ -116,40 +65,11 @@ var _ = Describe("BackFiller", func() {
receivedDiffStruct := struct {
diffs []utils.StorageDiff
}{
backFillStorage[test_data.AnotherContractLeafKey],
backFillStorage,
}
receivedDiffBytes, rlpErr2 := rlp.EncodeToBytes(receivedDiffStruct)
Expect(rlpErr2).ToNot(HaveOccurred())
Expect(bytes.Equal(expectedDiffBytes, receivedDiffBytes)).To(BeTrue())
})
It("Only returns storage for provided addresses (address hashes)", func() {
backFillArgs := storage.BackFillerArgs{
WantedStorage: map[common.Hash][]common.Hash{
test_data.ContractLeafKey: {common.BytesToHash(test_data.StorageKey)},
},
StartingBlock: test_data.BlockNumber.Uint64(),
EndingBlock: test_data.BlockNumber2.Uint64(),
}
backFillStorage, backFillErr := backFiller.BackFill(backFillArgs)
Expect(backFillErr).ToNot(HaveOccurred())
Expect(len(backFillStorage)).To(Equal(1))
Expect(len(backFillStorage[test_data.ContractLeafKey])).To(Equal(1))
Expect(len(backFillStorage[test_data.AnotherContractLeafKey])).To(Equal(0))
Expect(backFillStorage[test_data.ContractLeafKey][0]).To(Equal(test_data.CreatedExpectedStorageDiff))
})
It("Only returns storage for provided storage keys", func() {
backFillArgs := storage.BackFillerArgs{
WantedStorage: map[common.Hash][]common.Hash{
test_data.ContractLeafKey: nil,
},
StartingBlock: test_data.BlockNumber.Uint64(),
EndingBlock: test_data.BlockNumber2.Uint64(),
}
backFillStorage, backFillErr := backFiller.BackFill(backFillArgs)
Expect(backFillErr).ToNot(HaveOccurred())
Expect(len(backFillStorage)).To(Equal(0))
})
})
})