(VDB-950) Write raw diffs before transforming

- Raw field we can reference by FK for related data
- Enables replay for unwatched or mistransformed diffs

pkg/datastore/postgres/repositories/storage_diff_repository.go

@@ -0,0 +1,47 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package repositories
+
+import (
+	"database/sql"
+
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+var ErrDuplicateDiff = sql.ErrNoRows
+
+type StorageDiffRepository struct {
+	db *postgres.DB
+}
+
+func NewStorageDiffRepository(db *postgres.DB) StorageDiffRepository {
+	return StorageDiffRepository{db: db}
+}
+
+func (repository StorageDiffRepository) CreateStorageDiff(input utils.StorageDiffInput) (int64, error) {
+	var storageDiffID int64
+	row := repository.db.QueryRowx(`INSERT INTO public.storage_diff
+		(hashed_address, block_height, block_hash, storage_key, storage_value) VALUES ($1, $2, $3, $4, $5)
+		ON CONFLICT DO NOTHING RETURNING id`, input.HashedAddress.Bytes(), input.BlockHeight, input.BlockHash.Bytes(),
+		input.StorageKey.Bytes(), input.StorageValue.Bytes())
+	err := row.Scan(&storageDiffID)
+	if err != nil && err == sql.ErrNoRows {
+		return 0, ErrDuplicateDiff
+	}
+	return storageDiffID, err
+}

pkg/datastore/postgres/repositories/storage_diff_repository_test.go

@@ -0,0 +1,83 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package repositories_test
+
+import (
+	"database/sql"
+	"math/rand"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/test_data"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
+	"github.com/vulcanize/vulcanizedb/test_config"
+)
+
+var _ = Describe("Storage diffs repository", func() {
+	var (
+		db              *postgres.DB
+		repo            repositories.StorageDiffRepository
+		fakeStorageDiff utils.StorageDiffInput
+	)
+
+	BeforeEach(func() {
+		db = test_config.NewTestDB(test_config.NewTestNode())
+		test_config.CleanTestDB(db)
+		repo = repositories.NewStorageDiffRepository(db)
+		fakeStorageDiff = utils.StorageDiffInput{
+			HashedAddress: test_data.FakeHash(),
+			BlockHash:     test_data.FakeHash(),
+			BlockHeight:   rand.Int(),
+			StorageKey:    test_data.FakeHash(),
+			StorageValue:  test_data.FakeHash(),
+		}
+	})
+
+	Describe("CreateStorageDiff", func() {
+		It("adds a storage diff to the db, returning id", func() {
+			id, createErr := repo.CreateStorageDiff(fakeStorageDiff)
+
+			Expect(createErr).NotTo(HaveOccurred())
+			Expect(id).NotTo(BeZero())
+			var persisted utils.PersistedStorageDiff
+			getErr := db.Get(&persisted, `SELECT * FROM public.storage_diff`)
+			Expect(getErr).NotTo(HaveOccurred())
+			Expect(persisted.ID).To(Equal(id))
+			Expect(persisted.HashedAddress).To(Equal(fakeStorageDiff.HashedAddress))
+			Expect(persisted.BlockHash).To(Equal(fakeStorageDiff.BlockHash))
+			Expect(persisted.BlockHeight).To(Equal(fakeStorageDiff.BlockHeight))
+			Expect(persisted.StorageKey).To(Equal(fakeStorageDiff.StorageKey))
+			Expect(persisted.StorageValue).To(Equal(fakeStorageDiff.StorageValue))
+		})
+
+		It("does not duplicate storage diffs", func() {
+			_, createErr := repo.CreateStorageDiff(fakeStorageDiff)
+			Expect(createErr).NotTo(HaveOccurred())
+
+			_, createTwoErr := repo.CreateStorageDiff(fakeStorageDiff)
+			Expect(createTwoErr).To(HaveOccurred())
+			Expect(createTwoErr).To(MatchError(sql.ErrNoRows))
+
+			var count int
+			getErr := db.Get(&count, `SELECT count(*) FROM public.storage_diff`)
+			Expect(getErr).NotTo(HaveOccurred())
+			Expect(count).To(Equal(1))
+		})
+	})
+})

pkg/datastore/repository.go

@@ -19,6 +19,7 @@ package datastore
 import (
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/jmoiron/sqlx"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/filters"
 )
@@ -83,6 +84,10 @@ type HeaderSyncReceiptRepository interface {
 	CreateFullSyncReceiptInTx(blockID int64, receipt core.Receipt, tx *sqlx.Tx) (int64, error)
 }
 
+type StorageDiffRepository interface {
+	CreateStorageDiff(input utils.StorageDiffInput) (int64, error)
+}
+
 type WatchedEventRepository interface {
 	GetWatchedEvents(name string) ([]*core.WatchedEvent, error)
 }

pkg/fakes/mock_storage_diff_repository.go

@@ -0,0 +1,32 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package fakes
+
+import (
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+)
+
+type MockStorageDiffRepository struct {
+	CreatePassedInputs []utils.StorageDiffInput
+	CreateReturnID     int64
+	CreateReturnError  error
+}
+
+func (repository *MockStorageDiffRepository) CreateStorageDiff(input utils.StorageDiffInput) (int64, error) {
+	repository.CreatePassedInputs = append(repository.CreatePassedInputs, input)
+	return repository.CreateReturnID, repository.CreateReturnError
+}