(VDB-950) Write raw diffs before transforming

- Raw field we can reference by FK for related data
- Enables replay for unwatched or mistransformed diffs

libraries/shared/mocks/backfiller.go

@@ -24,23 +24,23 @@ import (
 
 // BackFiller mock for tests
 type BackFiller struct {
-	StorageDiffsToReturn []utils.StorageDiff
+	StorageDiffsToReturn []utils.StorageDiffInput
 	BackFillErrs         []error
 	PassedEndingBlock    uint64
 }
 
 // SetStorageDiffsToReturn for tests
-func (backFiller *BackFiller) SetStorageDiffsToReturn(diffs []utils.StorageDiff) {
+func (backFiller *BackFiller) SetStorageDiffsToReturn(diffs []utils.StorageDiffInput) {
 	backFiller.StorageDiffsToReturn = diffs
 }
 
 // BackFill mock method
-func (backFiller *BackFiller) BackFill(startingBlock, endingBlock uint64, backFill chan utils.StorageDiff, errChan chan error, done chan bool) error {
+func (backFiller *BackFiller) BackFill(startingBlock, endingBlock uint64, backFill chan utils.StorageDiffInput, errChan chan error, done chan bool) error {
 	if endingBlock < startingBlock {
 		return errors.New("backfill: ending block number needs to be greater than starting block number")
 	}
 	backFiller.PassedEndingBlock = endingBlock
-	go func(backFill chan utils.StorageDiff, errChan chan error, done chan bool) {
+	go func(backFill chan utils.StorageDiffInput, errChan chan error, done chan bool) {
 		errLen := len(backFiller.BackFillErrs)
 		for i, diff := range backFiller.StorageDiffsToReturn {
 			if i < errLen {

libraries/shared/mocks/storage_fetcher.go

@@ -22,7 +22,7 @@ import (
 
 // StorageFetcher is a mock fetcher for use in tests with backfilling
 type StorageFetcher struct {
-	DiffsToReturn []utils.StorageDiff
+	DiffsToReturn []utils.StorageDiffInput
 	ErrsToReturn  []error
 }
 
@@ -32,7 +32,7 @@ func NewStorageFetcher() *StorageFetcher {
 }
 
 // FetchStorageDiffs mock method
-func (fetcher *StorageFetcher) FetchStorageDiffs(out chan<- utils.StorageDiff, errs chan<- error) {
+func (fetcher *StorageFetcher) FetchStorageDiffs(out chan<- utils.StorageDiffInput, errs chan<- error) {
 	for _, err := range fetcher.ErrsToReturn {
 		errs <- err
 	}

libraries/shared/mocks/storage_queue.go

@@ -24,37 +24,36 @@ import (
 type MockStorageQueue struct {
 	AddCalled       bool
 	AddError        error
-	AddPassedDiffs  map[int]utils.StorageDiff
+	AddPassedDiffs  []utils.PersistedStorageDiff
 	DeleteErr       error
-	DeletePassedIds []int
+	DeletePassedIds []int64
 	GetAllErr       error
-	DiffsToReturn   map[int]utils.StorageDiff
+	DiffsToReturn   []utils.PersistedStorageDiff
 	GetAllCalled    bool
 }
 
 // Add mock method
-func (queue *MockStorageQueue) Add(diff utils.StorageDiff) error {
+func (queue *MockStorageQueue) Add(diff utils.PersistedStorageDiff) error {
 	queue.AddCalled = true
-	if queue.AddPassedDiffs == nil {
-		queue.AddPassedDiffs = make(map[int]utils.StorageDiff)
-	}
-	queue.AddPassedDiffs[diff.ID] = diff
+	queue.AddPassedDiffs = append(queue.AddPassedDiffs, diff)
 	return queue.AddError
 }
 
 // Delete mock method
-func (queue *MockStorageQueue) Delete(id int) error {
+func (queue *MockStorageQueue) Delete(id int64) error {
 	queue.DeletePassedIds = append(queue.DeletePassedIds, id)
-	delete(queue.DiffsToReturn, id)
+	var diffs []utils.PersistedStorageDiff
+	for _, diff := range queue.DiffsToReturn {
+		if diff.ID != id {
+			diffs = append(diffs, diff)
+		}
+	}
+	queue.DiffsToReturn = diffs
 	return queue.DeleteErr
 }
 
 // GetAll mock method
-func (queue *MockStorageQueue) GetAll() ([]utils.StorageDiff, error) {
+func (queue *MockStorageQueue) GetAll() ([]utils.PersistedStorageDiff, error) {
 	queue.GetAllCalled = true
-	diffs := make([]utils.StorageDiff, 0)
-	for _, diff := range queue.DiffsToReturn {
-		diffs = append(diffs, diff)
-	}
-	return diffs, queue.GetAllErr
+	return queue.DiffsToReturn, queue.GetAllErr
 }

libraries/shared/mocks/storage_repository.go

@@ -22,16 +22,14 @@ import (
 )
 
 type MockStorageRepository struct {
-	CreateErr         error
-	PassedBlockNumber int
-	PassedBlockHash   string
-	PassedMetadata    utils.StorageValueMetadata
-	PassedValue       interface{}
+	CreateErr      error
+	PassedDiffID   int64
+	PassedMetadata utils.StorageValueMetadata
+	PassedValue    interface{}
 }
 
-func (repository *MockStorageRepository) Create(blockNumber int, blockHash string, metadata utils.StorageValueMetadata, value interface{}) error {
-	repository.PassedBlockNumber = blockNumber
-	repository.PassedBlockHash = blockHash
+func (repository *MockStorageRepository) Create(diffID int64, metadata utils.StorageValueMetadata, value interface{}) error {
+	repository.PassedDiffID = diffID
 	repository.PassedMetadata = metadata
 	repository.PassedValue = value
 	return repository.CreateErr

libraries/shared/mocks/storage_transformer.go

@@ -28,15 +28,12 @@ import (
 type MockStorageTransformer struct {
 	KeccakOfAddress common.Hash
 	ExecuteErr      error
-	PassedDiffs     map[int]utils.StorageDiff
+	PassedDiffs     []utils.PersistedStorageDiff
 }
 
 // Execute mock method
-func (transformer *MockStorageTransformer) Execute(diff utils.StorageDiff) error {
-	if transformer.PassedDiffs == nil {
-		transformer.PassedDiffs = make(map[int]utils.StorageDiff)
-	}
-	transformer.PassedDiffs[diff.ID] = diff
+func (transformer *MockStorageTransformer) Execute(diff utils.PersistedStorageDiff) error {
+	transformer.PassedDiffs = append(transformer.PassedDiffs, diff)
 	return transformer.ExecuteErr
 }