cerc-io/ipld-eth-server
12
0
Fork 1
Code Issues 10 Pull Requests 2 Actions Packages Projects Releases 37 Wiki Activity

pair with new statediffing geth version; travis tests will fail til release is up

Browse Source
This commit is contained in:
Ian Norden
2020-05-21 15:34:43 -05:00
parent 5fb1cc0696
commit 0101c4791a
26 changed files with 169 additions and 180 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libraries/shared/fetcher/geth_rpc_storage_fetcher.go
+5 -5
View File
@@ -45,7 +45,7 @@ func NewGethRPCStorageFetcher(streamer streamer.Streamer) GethRPCStorageFetcher
func (fetcher GethRPCStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageDiffInput, errs chan<- error) {
ethStatediffPayloadChan := fetcher.StatediffPayloadChan
clientSubscription, clientSubErr := fetcher.streamer.Stream(ethStatediffPayloadChan)
clientSubscription, clientSubErr := fetcher.streamer.Stream(ethStatediffPayloadChan, statediff.Params{})
if clientSubErr != nil {
errs <- clientSubErr
panic(fmt.Sprintf("Error creating a geth client subscription: %v", clientSubErr))
@@ -55,8 +55,8 @@ func (fetcher GethRPCStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageD
for {
diff := <-ethStatediffPayloadChan
logrus.Trace("received a statediff")
stateDiff := new(statediff.StateDiff)
decodeErr := rlp.DecodeBytes(diff.StateDiffRlp, stateDiff)
stateDiff := new(statediff.StateObject)
decodeErr := rlp.DecodeBytes(diff.StateObjectRlp, stateDiff)
if decodeErr != nil {
logrus.Warn("Error decoding state diff into RLP: ", decodeErr)
errs <- decodeErr
@@ -65,8 +65,8 @@ func (fetcher GethRPCStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageD
accounts := utils.GetAccountsFromDiff(*stateDiff)
logrus.Trace(fmt.Sprintf("iterating through %d accounts on stateDiff for block %d", len(accounts), stateDiff.BlockNumber))
for _, account := range accounts {
logrus.Trace(fmt.Sprintf("iterating through %d Storage values on account with key %s", len(account.Storage), common.BytesToHash(account.LeafKey).Hex()))
for _, storage := range account.Storage {
logrus.Trace(fmt.Sprintf("iterating through %d Storage values on account with key %s", len(account.StorageNodes), common.BytesToHash(account.LeafKey).Hex()))
for _, storage := range account.StorageNodes {
diff, formatErr := utils.FromGethStateDiff(account, stateDiff, storage)
if formatErr != nil {
logrus.Error("failed to format utils.StorageDiff from storage with key: ", common.BytesToHash(storage.LeafKey), "from account with key: ", common.BytesToHash(account.LeafKey))
libraries/shared/fetcher/geth_rpc_storage_fetcher_test.go
+9 -8
View File
@@ -33,13 +33,14 @@ import (
type MockStoragediffStreamer struct {
subscribeError error
PassedPayloadChan chan statediff.Payload
PassedParams statediff.Params
streamPayloads []statediff.Payload
}
func (streamer *MockStoragediffStreamer) Stream(statediffPayloadChan chan statediff.Payload) (*rpc.ClientSubscription, error) {
func (streamer *MockStoragediffStreamer) Stream(statediffPayloadChan chan statediff.Payload, params statediff.Params) (*rpc.ClientSubscription, error) {
clientSubscription := rpc.ClientSubscription{}
streamer.PassedPayloadChan = statediffPayloadChan
streamer.PassedParams = params
go func() {
for _, payload := range streamer.streamPayloads {
streamer.PassedPayloadChan <- payload
@@ -148,19 +149,19 @@ var _ = Describe("Geth RPC Storage Fetcher", func() {
It("adds errors to error channel if formatting the diff as a StateDiff object fails", func(done Done) {
accountDiffs := test_data.CreatedAccountDiffs
accountDiffs[0].Storage = []statediff.StorageDiff{test_data.StorageWithBadValue}
accountDiffs[0].StorageNodes = []statediff.StorageNode{test_data.StorageWithBadValue}
stateDiff := statediff.StateDiff{
BlockNumber: test_data.BlockNumber,
BlockHash: common.HexToHash(test_data.BlockHash),
CreatedAccounts: accountDiffs,
stateDiff := statediff.StateObject{
BlockNumber: test_data.BlockNumber,
BlockHash: common.HexToHash(test_data.BlockHash),
Nodes: accountDiffs,
}
stateDiffRlp, err := rlp.EncodeToBytes(stateDiff)
Expect(err).NotTo(HaveOccurred())
badStatediffPayload := statediff.Payload{
StateDiffRlp: stateDiffRlp,
StateObjectRlp: stateDiffRlp,
}
streamer.SetPayloads([]statediff.Payload{badStatediffPayload})
libraries/shared/mocks/batch_client.go
+6 -6
View File
@@ -51,12 +51,12 @@ func (mc *BackFillerClient) BatchCall(batch []client.BatchElem) error {
return errors.New("mockclient needs to be initialized with statediff payloads and errors")
}
for _, batchElem := range batch {
if len(batchElem.Args) != 1 {
return errors.New("expected batch elem to contain single argument")
if len(batchElem.Args) < 1 {
return errors.New("expected batch elem to contain an argument(s)")
}
blockHeight, ok := batchElem.Args[0].(uint64)
if !ok {
return errors.New("expected batch elem argument to be a uint64")
return errors.New("expected first batch elem argument to be a uint64")
}
err := json.Unmarshal(mc.MappedStateDiffAt[blockHeight], batchElem.Result)
if err != nil {
@@ -72,12 +72,12 @@ func (mc *BackFillerClient) BatchCallContext(ctx context.Context, batch []rpc.Ba
return errors.New("mockclient needs to be initialized with statediff payloads and errors")
}
for _, batchElem := range batch {
if len(batchElem.Args) != 1 {
return errors.New("expected batch elem to contain single argument")
if len(batchElem.Args) < 1 {
return errors.New("expected batch elem to contain an argument(s)")
}
blockHeight, ok := batchElem.Args[0].(uint64)
if !ok {
return errors.New("expected batch elem argument to be a uint64")
return errors.New("expected batch elem first argument to be a uint64")
}
err := json.Unmarshal(mc.MappedStateDiffAt[blockHeight], batchElem.Result)
if err != nil {
libraries/shared/storage/backfiller.go
+4 -4
View File
@@ -120,16 +120,16 @@ func (bf *backFiller) backFillRange(blockHeights []uint64, diffChan chan utils.S
errChan <- fetchErr
}
for _, payload := range payloads {
stateDiff := new(statediff.StateDiff)
stateDiffDecodeErr := rlp.DecodeBytes(payload.StateDiffRlp, stateDiff)
stateDiff := new(statediff.StateObject)
stateDiffDecodeErr := rlp.DecodeBytes(payload.StateObjectRlp, stateDiff)
if stateDiffDecodeErr != nil {
errChan <- stateDiffDecodeErr
continue
}
accounts := utils.GetAccountsFromDiff(*stateDiff)
for _, account := range accounts {
logrus.Trace(fmt.Sprintf("iterating through %d Storage values on account with key %s", len(account.Storage), common.BytesToHash(account.LeafKey).Hex()))
for _, storage := range account.Storage {
logrus.Trace(fmt.Sprintf("iterating through %d Storage values on account with key %s", len(account.StorageNodes), common.BytesToHash(account.LeafKey).Hex()))
for _, storage := range account.StorageNodes {
diff, formatErr := utils.FromGethStateDiff(account, stateDiff, storage)
if formatErr != nil {
logrus.Error("failed to format utils.StorageDiff from storage with key: ", common.BytesToHash(storage.LeafKey), "from account with key: ", common.BytesToHash(account.LeafKey))
libraries/shared/storage/utils/diff.go
+3 -4
View File
@@ -57,7 +57,7 @@ func FromParityCsvRow(csvRow []string) (StorageDiffInput, error) {
}, nil
}
func FromGethStateDiff(account statediff.AccountDiff, stateDiff *statediff.StateDiff, storage statediff.StorageDiff) (StorageDiffInput, error) {
func FromGethStateDiff(account statediff.StateNode, stateDiff *statediff.StateObject, storage statediff.StorageNode) (StorageDiffInput, error) {
var decodedValue []byte
err := rlp.DecodeBytes(storage.NodeValue, &decodedValue)
if err != nil {
@@ -84,7 +84,6 @@ func HexToKeccak256Hash(hex string) common.Hash {
return crypto.Keccak256Hash(common.FromHex(hex))
}
func GetAccountsFromDiff(stateDiff statediff.StateDiff) []statediff.AccountDiff {
accounts := append(stateDiff.CreatedAccounts, stateDiff.UpdatedAccounts...)
return append(accounts, stateDiff.DeletedAccounts...)
func GetAccountsFromDiff(stateDiff statediff.StateObject) []statediff.StateNode {
return stateDiff.Nodes
}
libraries/shared/storage/utils/diff_test.go
+4 -4
View File
@@ -67,8 +67,8 @@ var _ = Describe("Storage row parsing", func() {
Describe("FromGethStateDiff", func() {
var (
accountDiff = statediff.AccountDiff{LeafKey: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 0}}
stateDiff = &statediff.StateDiff{
accountDiff = statediff.StateNode{LeafKey: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 0}}
stateDiff = &statediff.StateObject{
BlockNumber: big.NewInt(rand.Int63()),
BlockHash: fakes.FakeHash,
}
@@ -79,7 +79,7 @@ var _ = Describe("Storage row parsing", func() {
storageValueRlp, encodeErr := rlp.EncodeToBytes(storageValueBytes)
Expect(encodeErr).NotTo(HaveOccurred())
storageDiff := statediff.StorageDiff{
storageDiff := statediff.StorageNode{
LeafKey: []byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1},
NodeValue: storageValueRlp,
NodeType: statediff.Leaf,
@@ -104,7 +104,7 @@ var _ = Describe("Storage row parsing", func() {
storageValueRlp, encodeErr := rlp.EncodeToBytes(storageValueBytes)
Expect(encodeErr).NotTo(HaveOccurred())
storageDiff := statediff.StorageDiff{
storageDiff := statediff.StorageNode{
LeafKey: []byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1},
NodeValue: storageValueRlp,
NodeType: statediff.Leaf,
libraries/shared/streamer/statediff_streamer.go
+3 -3
View File
@@ -26,7 +26,7 @@ import (
// Streamer is the interface for streaming a statediff subscription
type Streamer interface {
Stream(payloadChan chan statediff.Payload) (*rpc.ClientSubscription, error)
Stream(payloadChan chan statediff.Payload, params statediff.Params) (*rpc.ClientSubscription, error)
}
// StateDiffStreamer is the underlying struct for the StateDiffStreamer interface
@@ -42,7 +42,7 @@ func NewStateDiffStreamer(client core.RPCClient) Streamer {
}
// Stream is the main loop for subscribing to data from the Geth state diff process
func (sds *StateDiffStreamer) Stream(payloadChan chan statediff.Payload) (*rpc.ClientSubscription, error) {
func (sds *StateDiffStreamer) Stream(payloadChan chan statediff.Payload, params statediff.Params) (*rpc.ClientSubscription, error) {
logrus.Info("streaming diffs from geth")
return sds.Client.Subscribe("statediff", payloadChan, "stream")
return sds.Client.Subscribe("statediff", payloadChan, "stream", params)
}
libraries/shared/streamer/statediff_streamer_test.go
+7 -2
View File
@@ -28,9 +28,14 @@ var _ = Describe("StateDiff Streamer", func() {
client := &fakes.MockRPCClient{}
streamer := streamer.NewStateDiffStreamer(client)
payloadChan := make(chan statediff.Payload)
_, err := streamer.Stream(payloadChan)
params := statediff.Params{
IncludeBlock: true,
IncludeTD: true,
IncludeReceipts: true,
}
_, err := streamer.Stream(payloadChan, params)
Expect(err).NotTo(HaveOccurred())
client.AssertSubscribeCalledWith("statediff", payloadChan, []interface{}{"stream"})
client.AssertSubscribeCalledWith("statediff", payloadChan, []interface{}{"stream", params})
})
})
libraries/shared/test_data/statediff.go
+31 -35
View File
@@ -40,7 +40,7 @@ var (
StorageKey = common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001").Bytes()
SmallStorageValue = common.Hex2Bytes("03")
SmallStorageValueRlp, _ = rlp.EncodeToBytes(SmallStorageValue)
storageWithSmallValue = []statediff.StorageDiff{{
storageWithSmallValue = []statediff.StorageNode{{
LeafKey: StorageKey,
NodeValue: SmallStorageValueRlp,
NodeType: statediff.Leaf,
@@ -48,13 +48,13 @@ var (
}}
LargeStorageValue = common.Hex2Bytes("00191b53778c567b14b50ba0000")
LargeStorageValueRlp, _ = rlp.EncodeToBytes(LargeStorageValue)
storageWithLargeValue = []statediff.StorageDiff{{
storageWithLargeValue = []statediff.StorageNode{{
LeafKey: StorageKey,
NodeValue: LargeStorageValueRlp,
Path: StoragePath,
NodeType: statediff.Leaf,
}}
StorageWithBadValue = statediff.StorageDiff{
StorageWithBadValue = statediff.StorageNode{
LeafKey: StorageKey,
NodeValue: []byte{0, 1, 2},
NodeType: statediff.Leaf,
@@ -74,44 +74,40 @@ var (
CodeHash: CodeHash,
}
valueBytes, _ = rlp.EncodeToBytes(testAccount)
CreatedAccountDiffs = []statediff.AccountDiff{
CreatedAccountDiffs = []statediff.StateNode{
{
LeafKey: ContractLeafKey.Bytes(),
NodeValue: valueBytes,
Storage: storageWithSmallValue,
LeafKey: ContractLeafKey.Bytes(),
NodeValue: valueBytes,
StorageNodes: storageWithSmallValue,
},
}
UpdatedAccountDiffs = []statediff.AccountDiff{{
LeafKey: AnotherContractLeafKey.Bytes(),
NodeValue: valueBytes,
Storage: storageWithLargeValue,
UpdatedAccountDiffs = []statediff.StateNode{{
LeafKey: AnotherContractLeafKey.Bytes(),
NodeValue: valueBytes,
StorageNodes: storageWithLargeValue,
}}
UpdatedAccountDiffs2 = []statediff.AccountDiff{{
LeafKey: AnotherContractLeafKey.Bytes(),
NodeValue: valueBytes,
Storage: storageWithSmallValue,
UpdatedAccountDiffs2 = []statediff.StateNode{{
LeafKey: AnotherContractLeafKey.Bytes(),
NodeValue: valueBytes,
StorageNodes: storageWithSmallValue,
}}
DeletedAccountDiffs = []statediff.AccountDiff{{
LeafKey: AnotherContractLeafKey.Bytes(),
NodeValue: valueBytes,
Storage: storageWithSmallValue,
DeletedAccountDiffs = []statediff.StateNode{{
LeafKey: AnotherContractLeafKey.Bytes(),
NodeValue: valueBytes,
StorageNodes: storageWithSmallValue,
}}
MockStateDiff = statediff.StateDiff{
BlockNumber: BlockNumber,
BlockHash: common.HexToHash(BlockHash),
CreatedAccounts: CreatedAccountDiffs,
DeletedAccounts: DeletedAccountDiffs,
UpdatedAccounts: UpdatedAccountDiffs,
MockStateDiff = statediff.StateObject{
BlockNumber: BlockNumber,
BlockHash: common.HexToHash(BlockHash),
Nodes: append(append(CreatedAccountDiffs, UpdatedAccountDiffs...), DeletedAccountDiffs...),
}
MockStateDiff2 = statediff.StateDiff{
BlockNumber: BlockNumber2,
BlockHash: common.HexToHash(BlockHash2),
CreatedAccounts: nil,
DeletedAccounts: nil,
UpdatedAccounts: UpdatedAccountDiffs2,
MockStateDiff2 = statediff.StateObject{
BlockNumber: BlockNumber2,
BlockHash: common.HexToHash(BlockHash2),
Nodes: UpdatedAccountDiffs2,
}
MockStateDiffBytes, _ = rlp.EncodeToBytes(MockStateDiff)
MockStateDiff2Bytes, _ = rlp.EncodeToBytes(MockStateDiff2)
@@ -144,12 +140,12 @@ var (
MockBlockRlp2, _ = rlp.EncodeToBytes(MockBlock2)
MockStatediffPayload = statediff.Payload{
BlockRlp: MockBlockRlp,
StateDiffRlp: MockStateDiffBytes,
BlockRlp: MockBlockRlp,
StateObjectRlp: MockStateDiffBytes,
}
MockStatediffPayload2 = statediff.Payload{
BlockRlp: MockBlockRlp2,
StateDiffRlp: MockStateDiff2Bytes,
BlockRlp: MockBlockRlp2,
StateObjectRlp: MockStateDiff2Bytes,
}
CreatedExpectedStorageDiff = utils.StorageDiffInput{