cerc-io/ipld-eth-server
12
0
Fork 1
Code Issues 10 Pull Requests 2 Actions Packages Projects Releases 37 Wiki Activity

(VDB-950) Write raw diffs before transforming

Browse Source 
- Raw field we can reference by FK for related data
- Enables replay for unwatched or mistransformed diffs
This commit is contained in:
Rob Mulholand
2019-12-03 14:51:17 -06:00
committed by Ian Norden
parent 1178940047
commit 56ce8bdb41
45 changed files with 543 additions and 1815 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libraries/shared/fetcher/csv_tail_storage_fetcher.go
+1 -1
View File
@@ -32,7 +32,7 @@ func NewCsvTailStorageFetcher(tailer fs.Tailer) CsvTailStorageFetcher {
return CsvTailStorageFetcher{tailer: tailer}
}
func (storageFetcher CsvTailStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageDiff, errs chan<- error) {
func (storageFetcher CsvTailStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageDiffInput, errs chan<- error) {
t, tailErr := storageFetcher.tailer.Tail()
if tailErr != nil {
errs <- tailErr
libraries/shared/fetcher/csv_tail_storage_fetcher_test.go
+2 -2
View File
@@ -34,13 +34,13 @@ var _ = Describe("Csv Tail Storage Fetcher", func() {
var (
errorsChannel chan error
mockTailer *fakes.MockTailer
diffsChannel chan utils.StorageDiff
diffsChannel chan utils.StorageDiffInput
storageFetcher fetcher.CsvTailStorageFetcher
)
BeforeEach(func() {
errorsChannel = make(chan error)
diffsChannel = make(chan utils.StorageDiff)
diffsChannel = make(chan utils.StorageDiffInput)
mockTailer = fakes.NewMockTailer()
storageFetcher = fetcher.NewCsvTailStorageFetcher(mockTailer)
})
libraries/shared/fetcher/geth_rpc_storage_fetcher.go
+1 -1
View File
@@ -43,7 +43,7 @@ func NewGethRPCStorageFetcher(streamer streamer.Streamer) GethRPCStorageFetcher
}
}
func (fetcher GethRPCStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageDiff, errs chan<- error) {
func (fetcher GethRPCStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageDiffInput, errs chan<- error) {
ethStatediffPayloadChan := fetcher.StatediffPayloadChan
clientSubscription, clientSubErr := fetcher.streamer.Stream(ethStatediffPayloadChan)
if clientSubErr != nil {
libraries/shared/fetcher/geth_rpc_storage_fetcher_test.go
+5 -5
View File
@@ -60,13 +60,13 @@ func (streamer *MockStoragediffStreamer) SetPayloads(payloads []statediff.Payloa
var _ = Describe("Geth RPC Storage Fetcher", func() {
var streamer MockStoragediffStreamer
var statediffFetcher fetcher.GethRPCStorageFetcher
var storagediffChan chan utils.StorageDiff
var storagediffChan chan utils.StorageDiffInput
var errorChan chan error
BeforeEach(func() {
streamer = MockStoragediffStreamer{}
statediffFetcher = fetcher.NewGethRPCStorageFetcher(&streamer)
storagediffChan = make(chan utils.StorageDiff)
storagediffChan = make(chan utils.StorageDiffInput)
errorChan = make(chan error)
})
@@ -113,21 +113,21 @@ var _ = Describe("Geth RPC Storage Fetcher", func() {
height := test_data.BlockNumber
intHeight := int(height.Int64())
createdExpectedStorageDiff := utils.StorageDiff{
createdExpectedStorageDiff := utils.StorageDiffInput{
HashedAddress: common.BytesToHash(test_data.ContractLeafKey[:]),
BlockHash: common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"),
BlockHeight: intHeight,
StorageKey: common.BytesToHash(test_data.StorageKey),
StorageValue: common.BytesToHash(test_data.SmallStorageValue),
}
updatedExpectedStorageDiff := utils.StorageDiff{
updatedExpectedStorageDiff := utils.StorageDiffInput{
HashedAddress: common.BytesToHash(test_data.AnotherContractLeafKey[:]),
BlockHash: common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"),
BlockHeight: intHeight,
StorageKey: common.BytesToHash(test_data.StorageKey),
StorageValue: common.BytesToHash(test_data.LargeStorageValue),
}
deletedExpectedStorageDiff := utils.StorageDiff{
deletedExpectedStorageDiff := utils.StorageDiffInput{
HashedAddress: common.BytesToHash(test_data.AnotherContractLeafKey[:]),
BlockHash: common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"),
BlockHeight: intHeight,
libraries/shared/fetcher/storage_fetcher_interface.go
+1 -1
View File
@@ -17,5 +17,5 @@ package fetcher
import "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
type IStorageFetcher interface {
FetchStorageDiffs(out chan<- utils.StorageDiff, errs chan<- error)
FetchStorageDiffs(out chan<- utils.StorageDiffInput, errs chan<- error)
}