[WIP] Add a service to fill indexing gap for watched addresses #135

Closed
prathamesh0 wants to merge 16 commits from pm-watched-addresses into master
4 changed files with 550 additions and 958 deletions
Showing only changes of commit 4e976c62f1 - Show all commits

View File

@ -30,7 +30,6 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/statediff"
sdtypes "github.com/ethereum/go-ethereum/statediff/types"
"github.com/mailgun/groupcache/v2"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
@ -305,7 +304,6 @@ func startStateTrieValidator(config *s.Config, server s.Server) {
type WatchedAddress struct {
Address string `db:"address"`
Kind int `db:"kind"`
CreatedAt uint64 `db:"created_at"`
WatchedAt uint64 `db:"watched_at"`
LastFilledAt uint64 `db:"last_filled_at"`
@ -343,15 +341,7 @@ func startWatchedAddressGapFiller(config *s.Config) {
fillAddresses := []interface{}{}
for _, fillWatchedAddress := range fillWatchedAddresses {
if blockNumber >= fillWatchedAddress.startBlock && blockNumber <= fillWatchedAddress.endBlock {
switch fillWatchedAddress.Kind {
case sdtypes.WatchedAddress.Int():
params.WatchedAddresses = append(params.WatchedAddresses, common.HexToAddress(fillWatchedAddress.Address))
case sdtypes.WatchedStorageSlot.Int():
params.WatchedStorageSlots = append(params.WatchedStorageSlots, common.HexToHash(fillWatchedAddress.Address))
default:
log.Fatalf("Unexpected kind %d:", fillWatchedAddress.Kind)
}
params.WatchedAddresses = append(params.WatchedAddresses, common.HexToAddress(fillWatchedAddress.Address))
fillAddresses = append(fillAddresses, fillWatchedAddress.Address)
}
}

View File

@ -36,6 +36,10 @@ type StorageKey struct {
Key string `json:"key"`
}
type CountIncremented struct {
BlockNumber int64 `json:"blockNumber"`
}
const srvUrl = "http://localhost:3000"
func DeployContract() (*ContractDeployed, error) {
@ -105,22 +109,21 @@ func DeploySLVContract() (*ContractDeployed, error) {
return &contract, nil
}
func IncrementCountA(addr string) error {
_, err := http.Get(fmt.Sprintf("%s/v1/incrementCountA?addr=%s", srvUrl, addr))
func IncrementCount(addr string, count string) (*CountIncremented, error) {
res, err := http.Get(fmt.Sprintf("%s/v1/incrementCount%s?addr=%s", srvUrl, count, addr))
if err != nil {
return err
return nil, err
}
return nil
}
var blockNumber CountIncremented
func IncrementCountB(addr string) error {
_, err := http.Get(fmt.Sprintf("%s/v1/incrementCountB?addr=%s", srvUrl, addr))
decoder := json.NewDecoder(res.Body)
err = decoder.Decode(&blockNumber)
if err != nil {
return err
return nil, err
}
return nil
return &blockNumber, nil
}
func GetStorageSlotKey(contract string, label string) (*StorageKey, error) {
@ -146,13 +149,7 @@ func ClearWatchedAddresses(gethRPCClient *rpc.Client) error {
args := []sdtypes.WatchAddressArg{}
// Clear watched addresses
gethErr := gethRPCClient.Call(nil, gethMethod, statediff.ClearAddresses, args)
if gethErr != nil {
return gethErr
}
// Clear watched storage slots
gethErr = gethRPCClient.Call(nil, gethMethod, statediff.ClearStorageSlots, args)
gethErr := gethRPCClient.Call(nil, gethMethod, statediff.Clear, args)
if gethErr != nil {
return gethErr
}

File diff suppressed because it is too large Load Diff

View File

@ -8,7 +8,6 @@ import (
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethclient"
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/statediff"
@ -18,12 +17,6 @@ import (
integration "github.com/vulcanize/ipld-eth-server/test"
)
var (
ctx = context.Background()
ipldClient *ethclient.Client
)
var _ = Describe("Watched address gap filling service integration test", func() {
dbWrite, err := strconv.ParseBool(os.Getenv("DB_WRITE"))
Expect(err).To(BeNil())
@ -39,42 +32,33 @@ var _ = Describe("Watched address gap filling service integration test", func()
Expect(err).ToNot(HaveOccurred())
ipldEthHttpPath := "http://127.0.0.1:8081"
ipldClient, err = ethclient.Dial(ipldEthHttpPath)
ipldClient, err := ethclient.Dial(ipldEthHttpPath)
Expect(err).ToNot(HaveOccurred())
ipldRPCClient, err := rpc.Dial(ipldEthHttpPath)
Expect(err).ToNot(HaveOccurred())
var (
ctx = context.Background()
contractErr error
txErr error
GLD1 *integration.ContractDeployed
GLD2 *integration.ContractDeployed
SLV1 *integration.ContractDeployed
SLV2 *integration.ContractDeployed
SLV3 *integration.ContractDeployed
countAIndex string
countBIndex string
countAStorageHash common.Hash
countBStorageHash common.Hash
totalSupplyIndex = "0x2"
totalSuppyStorageHash = crypto.Keccak256Hash(common.HexToHash(totalSupplyIndex).Bytes())
countAIndex string
countBIndex string
oldCountA1 = big.NewInt(0)
oldCountB1 = big.NewInt(0)
oldCountA2 = big.NewInt(0)
oldCountB2 = big.NewInt(0)
oldCountA3 = big.NewInt(0)
oldCountB3 = big.NewInt(0)
updatedCountA1 = big.NewInt(1)
updatedCountB1 = big.NewInt(1)
updatedCountA2 = big.NewInt(1)
updatedCountB2 = big.NewInt(1)
updatedCountA3 = big.NewInt(1)
updatedCountB3 = big.NewInt(1)
SLV2CountBIncrementedAt *integration.CountIncremented
)
BeforeEach(func() {
@ -84,7 +68,7 @@ var _ = Describe("Watched address gap filling service integration test", func()
})
It("test init", func() {
// Clear out watched addresses | storage slots
// Clear out watched addresses
err := integration.ClearWatchedAddresses(gethRPCClient)
Expect(err).ToNot(HaveOccurred())
@ -93,7 +77,7 @@ var _ = Describe("Watched address gap filling service integration test", func()
Expect(contractErr).ToNot(HaveOccurred())
// Watch GLD1 contract
operation := statediff.AddAddresses
operation := statediff.Add
args := []sdtypes.WatchAddressArg{
{
Address: GLD1.Address,
@ -103,49 +87,33 @@ var _ = Describe("Watched address gap filling service integration test", func()
ipldErr := ipldRPCClient.Call(nil, ipldMethod, operation, args)
Expect(ipldErr).ToNot(HaveOccurred())
// Deploy a GLD contract
GLD2, contractErr = integration.DeployContract()
Expect(contractErr).ToNot(HaveOccurred())
// Deploy three SLV contracts and update storage slots
// Deploy two SLV contracts and update storage slots
SLV1, contractErr = integration.DeploySLVContract()
Expect(contractErr).ToNot(HaveOccurred())
txErr = integration.IncrementCountA(SLV1.Address)
Expect(txErr).ToNot(HaveOccurred())
txErr = integration.IncrementCountB(SLV1.Address)
_, txErr = integration.IncrementCount(SLV1.Address, "A")
Expect(txErr).ToNot(HaveOccurred())
SLV2, contractErr = integration.DeploySLVContract()
Expect(contractErr).ToNot(HaveOccurred())
txErr = integration.IncrementCountA(SLV2.Address)
_, txErr = integration.IncrementCount(SLV2.Address, "A")
Expect(txErr).ToNot(HaveOccurred())
txErr = integration.IncrementCountB(SLV2.Address)
Expect(txErr).ToNot(HaveOccurred())
SLV3, contractErr = integration.DeploySLVContract()
Expect(contractErr).ToNot(HaveOccurred())
txErr = integration.IncrementCountA(SLV3.Address)
Expect(txErr).ToNot(HaveOccurred())
txErr = integration.IncrementCountB(SLV3.Address)
SLV2CountBIncrementedAt, txErr = integration.IncrementCount(SLV2.Address, "B")
Expect(txErr).ToNot(HaveOccurred())
// Get storage slot keys
storageSlotAKey, err := integration.GetStorageSlotKey("SLVToken", "countA")
Expect(err).ToNot(HaveOccurred())
countAIndex = storageSlotAKey.Key
countAStorageHash = crypto.Keccak256Hash(common.HexToHash(countAIndex).Bytes())
storageSlotBKey, err := integration.GetStorageSlotKey("SLVToken", "countB")
Expect(err).ToNot(HaveOccurred())
countBIndex = storageSlotBKey.Key
countBStorageHash = crypto.Keccak256Hash(common.HexToHash(countBIndex).Bytes())
})
defer It("test cleanup", func() {
// Clear out watched addresses | storage slots
// Clear out watched addresses
err := integration.ClearWatchedAddresses(gethRPCClient)
Expect(err).ToNot(HaveOccurred())
})
@ -153,19 +121,12 @@ var _ = Describe("Watched address gap filling service integration test", func()
Context("previously unwatched contract watched", func() {
It("indexes state only for watched contract", func() {
// WatchedAddresses = [GLD1]
// WatchedStorageSlots = []
// SLV1, countA
countA1Storage, err := ipldClient.StorageAt(ctx, common.HexToAddress(SLV1.Address), common.HexToHash(countAIndex), nil)
Expect(err).ToNot(HaveOccurred())
countA1 := new(big.Int).SetBytes(countA1Storage)
Expect(countA1.String()).To(Equal(oldCountA1.String()))
// SLV1, countB
countB1Storage, err := ipldClient.StorageAt(ctx, common.HexToAddress(SLV1.Address), common.HexToHash(countBIndex), nil)
Expect(err).ToNot(HaveOccurred())
countB1 := new(big.Int).SetBytes(countB1Storage)
Expect(countB1.String()).To(Equal(oldCountB1.String()))
// SLV2, countA
countA2Storage, err := ipldClient.StorageAt(ctx, common.HexToAddress(SLV2.Address), common.HexToHash(countAIndex), nil)
Expect(err).ToNot(HaveOccurred())
@ -177,18 +138,6 @@ var _ = Describe("Watched address gap filling service integration test", func()
Expect(err).ToNot(HaveOccurred())
countB2 := new(big.Int).SetBytes(countB2Storage)
Expect(countB2.String()).To(Equal(oldCountB2.String()))
// SLV3, countA
countA3Storage, err := ipldClient.StorageAt(ctx, common.HexToAddress(SLV3.Address), common.HexToHash(countAIndex), nil)
Expect(err).ToNot(HaveOccurred())
countA3 := new(big.Int).SetBytes(countA3Storage)
Expect(countA3.String()).To(Equal(oldCountA3.String()))
// SLV3, countB
countB3Storage, err := ipldClient.StorageAt(ctx, common.HexToAddress(SLV3.Address), common.HexToHash(countBIndex), nil)
Expect(err).ToNot(HaveOccurred())
countB3 := new(big.Int).SetBytes(countB3Storage)
Expect(countB3.String()).To(Equal(oldCountB3.String()))
})
It("indexes past state on watching a contract", func() {
@ -199,26 +148,19 @@ var _ = Describe("Watched address gap filling service integration test", func()
CreatedAt: uint64(SLV1.BlockNumber),
},
}
ipldErr := ipldRPCClient.Call(nil, ipldMethod, statediff.AddAddresses, args)
ipldErr := ipldRPCClient.Call(nil, ipldMethod, statediff.Add, args)
Expect(ipldErr).ToNot(HaveOccurred())
// Sleep for service interval + few extra seconds
time.Sleep(time.Duration(serviceInterval+2) * time.Second)
// WatchedAddresses = [GLD1, SLV1]
// WatchedStorageSlots = []
// SLV1, countA
countA1Storage, err := ipldClient.StorageAt(ctx, common.HexToAddress(SLV1.Address), common.HexToHash(countAIndex), nil)
Expect(err).ToNot(HaveOccurred())
countA1 := new(big.Int).SetBytes(countA1Storage)
Expect(countA1.String()).To(Equal(updatedCountA1.String()))
// SLV1, countB
countB1Storage, err := ipldClient.StorageAt(ctx, common.HexToAddress(SLV1.Address), common.HexToHash(countBIndex), nil)
Expect(err).ToNot(HaveOccurred())
countB1 := new(big.Int).SetBytes(countB1Storage)
Expect(countB1.String()).To(Equal(updatedCountB1.String()))
// SLV2, countA
countA2Storage, err := ipldClient.StorageAt(ctx, common.HexToAddress(SLV2.Address), common.HexToHash(countAIndex), nil)
Expect(err).ToNot(HaveOccurred())
@ -230,142 +172,57 @@ var _ = Describe("Watched address gap filling service integration test", func()
Expect(err).ToNot(HaveOccurred())
countB2 := new(big.Int).SetBytes(countB2Storage)
Expect(countB2.String()).To(Equal(oldCountB2.String()))
// SLV3, countA
countA3Storage, err := ipldClient.StorageAt(ctx, common.HexToAddress(SLV3.Address), common.HexToHash(countAIndex), nil)
Expect(err).ToNot(HaveOccurred())
countA3 := new(big.Int).SetBytes(countA3Storage)
Expect(countA3.String()).To(Equal(oldCountA3.String()))
// SLV3, countB
countB3Storage, err := ipldClient.StorageAt(ctx, common.HexToAddress(SLV3.Address), common.HexToHash(countBIndex), nil)
Expect(err).ToNot(HaveOccurred())
countB3 := new(big.Int).SetBytes(countB3Storage)
Expect(countB3.String()).To(Equal(oldCountB3.String()))
})
})
Context("one storage slot being watched", func() {
It("indexer past state only for watched storage slots of watched contracts", func() {
// Watch countA
Context("previously unwatched contract watched (different 'created_at')", func() {
It("indexes past state from 'created_at' onwards on watching a contract", func() {
// Watch SLV2 (created_at -> countB incremented) contract
args := []sdtypes.WatchAddressArg{
{
Address: countAStorageHash.Hex(),
CreatedAt: uint64(SLV1.BlockNumber),
Address: SLV2.Address,
CreatedAt: uint64(SLV2CountBIncrementedAt.BlockNumber),
},
}
ipldErr := ipldRPCClient.Call(nil, ipldMethod, statediff.AddStorageSlots, args)
ipldErr := ipldRPCClient.Call(nil, ipldMethod, statediff.Add, args)
Expect(ipldErr).ToNot(HaveOccurred())
// Watch SLV2 contract
args = []sdtypes.WatchAddressArg{
// Sleep for service interval + few extra seconds
time.Sleep(time.Duration(serviceInterval+2) * time.Second)
// WatchedAddresses = [GLD1, SLV1, SLV2]
// SLV2, countA
countA2Storage, err := ipldClient.StorageAt(ctx, common.HexToAddress(SLV2.Address), common.HexToHash(countAIndex), nil)
Expect(err).ToNot(HaveOccurred())
countA2 := new(big.Int).SetBytes(countA2Storage)
Expect(countA2.String()).To(Equal(oldCountA2.String()))
// SLV2, countB
countB2Storage, err := ipldClient.StorageAt(ctx, common.HexToAddress(SLV2.Address), common.HexToHash(countBIndex), nil)
Expect(err).ToNot(HaveOccurred())
countB2 := new(big.Int).SetBytes(countB2Storage)
Expect(countB2.String()).To(Equal(updatedCountB2.String()))
})
It("indexes missing past state on watching a contract from an earlier 'created_at'", func() {
// Clear out watched addresses
err := integration.ClearWatchedAddresses(gethRPCClient)
Expect(err).ToNot(HaveOccurred())
// Watch SLV2 (created_at -> deployment) contract
args := []sdtypes.WatchAddressArg{
{
Address: SLV2.Address,
CreatedAt: uint64(SLV2.BlockNumber),
},
}
ipldErr = ipldRPCClient.Call(nil, ipldMethod, statediff.AddAddresses, args)
ipldErr := ipldRPCClient.Call(nil, ipldMethod, statediff.Add, args)
Expect(ipldErr).ToNot(HaveOccurred())
// Sleep for service interval + few extra seconds
time.Sleep(time.Duration(serviceInterval+2) * time.Second)
// WatchedAddresses = [GLD1, SLV1, SLV2]
// WatchedStorageSlots = [countA]
// SLV2, countA
countA2Storage, err := ipldClient.StorageAt(ctx, common.HexToAddress(SLV2.Address), common.HexToHash(countAIndex), nil)
Expect(err).ToNot(HaveOccurred())
countA2 := new(big.Int).SetBytes(countA2Storage)
Expect(countA2.String()).To(Equal(updatedCountA2.String()))
// SLV2, countB
countB2Storage, err := ipldClient.StorageAt(ctx, common.HexToAddress(SLV2.Address), common.HexToHash(countBIndex), nil)
Expect(err).ToNot(HaveOccurred())
countB2 := new(big.Int).SetBytes(countB2Storage)
Expect(countB2.String()).To(Equal(oldCountB2.String()))
// SLV3, countA
countA3Storage, err := ipldClient.StorageAt(ctx, common.HexToAddress(SLV3.Address), common.HexToHash(countAIndex), nil)
Expect(err).ToNot(HaveOccurred())
countA3 := new(big.Int).SetBytes(countA3Storage)
Expect(countA3.String()).To(Equal(oldCountA3.String()))
// SLV3, countB
countB3Storage, err := ipldClient.StorageAt(ctx, common.HexToAddress(SLV3.Address), common.HexToHash(countBIndex), nil)
Expect(err).ToNot(HaveOccurred())
countB3 := new(big.Int).SetBytes(countB3Storage)
Expect(countB3.String()).To(Equal(oldCountB3.String()))
})
})
Context("previously unwatched storage slot watched", func() {
It("indexes past state only for watched storage slots updated after created at", func() {
// Watch countB with created_at = SLV3.BlockNumber
args := []sdtypes.WatchAddressArg{
{
Address: countBStorageHash.Hex(),
CreatedAt: uint64(SLV3.BlockNumber),
},
}
ipldErr := ipldRPCClient.Call(nil, ipldMethod, statediff.AddStorageSlots, args)
Expect(ipldErr).ToNot(HaveOccurred())
// Sleep for service interval + few extra seconds
time.Sleep(time.Duration(serviceInterval+2) * time.Second)
// WatchedAddresses = [GLD1, SLV1, SLV2]
// WatchedStorageSlots = [countA, countB] (countB -> created_at = SLV3.BlockNumber)
// SLV2, countA
countA2Storage, err := ipldClient.StorageAt(ctx, common.HexToAddress(SLV2.Address), common.HexToHash(countAIndex), nil)
Expect(err).ToNot(HaveOccurred())
countA2 := new(big.Int).SetBytes(countA2Storage)
Expect(countA2.String()).To(Equal(updatedCountA2.String()))
// SLV2, countB
countB2Storage, err := ipldClient.StorageAt(ctx, common.HexToAddress(SLV2.Address), common.HexToHash(countBIndex), nil)
Expect(err).ToNot(HaveOccurred())
countB2 := new(big.Int).SetBytes(countB2Storage)
Expect(countB2.String()).To(Equal(oldCountB2.String()))
// SLV3, countA
countA3Storage, err := ipldClient.StorageAt(ctx, common.HexToAddress(SLV3.Address), common.HexToHash(countAIndex), nil)
Expect(err).ToNot(HaveOccurred())
countA3 := new(big.Int).SetBytes(countA3Storage)
Expect(countA3.String()).To(Equal(oldCountA3.String()))
// SLV3, countB
countB3Storage, err := ipldClient.StorageAt(ctx, common.HexToAddress(SLV3.Address), common.HexToHash(countBIndex), nil)
Expect(err).ToNot(HaveOccurred())
countB3 := new(big.Int).SetBytes(countB3Storage)
Expect(countB3.String()).To(Equal(oldCountB3.String()))
})
It("indexes past state for watched storage slots of watched contracts", func() {
// Unwatch countB
args := []sdtypes.WatchAddressArg{
{
Address: countBStorageHash.Hex(),
CreatedAt: uint64(SLV3.BlockNumber),
},
}
ipldErr := ipldRPCClient.Call(nil, ipldMethod, statediff.RemoveStorageSlots, args)
Expect(ipldErr).ToNot(HaveOccurred())
// Watch countB with created_at = SLV1.BlockNumber
args = []sdtypes.WatchAddressArg{
{
Address: countBStorageHash.Hex(),
CreatedAt: uint64(SLV1.BlockNumber),
},
}
ipldErr = ipldRPCClient.Call(nil, ipldMethod, statediff.AddStorageSlots, args)
Expect(ipldErr).ToNot(HaveOccurred())
// Sleep for service interval + few extra seconds
time.Sleep(time.Duration(serviceInterval+2) * time.Second)
// WatchedAddresses = [GLD1, SLV1, SLV2]
// WatchedStorageSlots = [countA, countB] (countB -> created_at = SLV1.BlockNumber)
// WatchedAddresses = [SLV2]
// SLV2, countA
countA2Storage, err := ipldClient.StorageAt(ctx, common.HexToAddress(SLV2.Address), common.HexToHash(countAIndex), nil)
Expect(err).ToNot(HaveOccurred())
@ -377,66 +234,6 @@ var _ = Describe("Watched address gap filling service integration test", func()
Expect(err).ToNot(HaveOccurred())
countB2 := new(big.Int).SetBytes(countB2Storage)
Expect(countB2.String()).To(Equal(updatedCountB2.String()))
// SLV3, countA
countA3Storage, err := ipldClient.StorageAt(ctx, common.HexToAddress(SLV3.Address), common.HexToHash(countAIndex), nil)
Expect(err).ToNot(HaveOccurred())
countA3 := new(big.Int).SetBytes(countA3Storage)
Expect(countA3.String()).To(Equal(oldCountA3.String()))
// SLV3, countB
countB3Storage, err := ipldClient.StorageAt(ctx, common.HexToAddress(SLV3.Address), common.HexToHash(countBIndex), nil)
Expect(err).ToNot(HaveOccurred())
countB3 := new(big.Int).SetBytes(countB3Storage)
Expect(countB3.String()).To(Equal(oldCountB3.String()))
})
})
Context("contract watched along with other contract and its storage slots", func() {
It("indexes past state for watched storage slots of watched contracts", func() {
// Watch totalSupply
args := []sdtypes.WatchAddressArg{
{
Address: totalSuppyStorageHash.Hex(),
CreatedAt: uint64(GLD1.BlockNumber),
},
}
ipldErr := ipldRPCClient.Call(nil, ipldMethod, statediff.AddStorageSlots, args)
Expect(ipldErr).ToNot(HaveOccurred())
// Watch GLD2 and SLV3 contracts
args = []sdtypes.WatchAddressArg{
{
Address: GLD2.Address,
CreatedAt: uint64(GLD2.BlockNumber),
},
{
Address: SLV3.Address,
CreatedAt: uint64(SLV3.BlockNumber),
},
}
ipldErr = ipldRPCClient.Call(nil, ipldMethod, statediff.AddAddresses, args)
Expect(ipldErr).ToNot(HaveOccurred())
// Sleep for service interval + few extra seconds
time.Sleep(time.Duration(serviceInterval+2) * time.Second)
// WatchedAddresses = [GLD1, SLV1, SLV2, GLD2, SLV3]
// WatchedStorageSlots = [countA, countB, totalSupply]
// SLV3, countA
countA3Storage, err := ipldClient.StorageAt(ctx, common.HexToAddress(SLV3.Address), common.HexToHash(countAIndex), nil)
Expect(err).ToNot(HaveOccurred())
countA3 := new(big.Int).SetBytes(countA3Storage)
Expect(countA3.String()).To(Equal(updatedCountA3.String()))
// SLV3, countB
countB3Storage, err := ipldClient.StorageAt(ctx, common.HexToAddress(SLV3.Address), common.HexToHash(countBIndex), nil)
Expect(err).ToNot(HaveOccurred())
countB3 := new(big.Int).SetBytes(countB3Storage)
Expect(countB3.String()).To(Equal(updatedCountB3.String()))
})
})
// TODO:
// Add test: watched storage slot not having any previously watched related contracts.
})