(VDB-929) Minimize storage key lookup bespoke code

- Extract shared namespace for looking up and hashing keys - Require storage transformers only to implement a loader that associates known keys with metadata - Move key loader/lookup utils to utils directory to avoid multiple "storage" packages in imports
2019-10-28 14:29:09 -05:00
parent b7675316b4
commit b8fec5e4e3
12 changed files with 412 additions and 98 deletions
@@ -31,15 +31,15 @@ The storage transformer depends on contract-specific implementations of code cap

 ```golang
 func (transformer Transformer) Execute(row shared.StorageDiffRow) error {
-	metadata, lookupErr := transformer.Mappings.Lookup(row.StorageKey)
+	metadata, lookupErr := transformer.StorageKeysLookup.Lookup(diff.StorageKey)
 	if lookupErr != nil {
 		return lookupErr
 	}
-	value, decodeErr := shared.Decode(row, metadata)
+	value, decodeErr := utils.Decode(diff, metadata)
 	if decodeErr != nil {
 		return decodeErr
 	}
-	return transformer.Repository.Create(row.BlockHeight, row.BlockHash.Hex(), metadata, value)
+	return transformer.Repository.Create(diff.BlockHeight, diff.BlockHash.Hex(), metadata, value)
 }
 ```

@@ -47,20 +47,36 @@ func (transformer Transformer) Execute(row shared.StorageDiffRow) error {

 In order to watch an additional smart contract, a developer must create three things:

-1. Mappings - specify how to identify keys in the contract's storage trie.
+1. StorageKeysLoader - identify keys in the contract's storage trie, providing metadata to describe how associated values should be decoded.
 1. Repository - specify how to persist a parsed version of the storage value matching the recognized storage key.
 1. Instance - create an instance of the storage transformer that uses your mappings and repository.

-### Mappings
+### StorageKeysLoader
+
+A `StorageKeysLoader` is used by the `StorageKeysLookup` object on a storage transformer.

 ```golang
-type Mappings interface {
-	Lookup(key common.Hash) (shared.StorageValueMetadata, error)
+type KeysLoader interface {
+	LoadMappings() (map[common.Hash]utils.StorageValueMetadata, error)
 	SetDB(db *postgres.DB)
 }
 ```

-A contract-specific implementation of the mappings interface enables the storage transformer to fetch metadata associated with a storage key.
+When a key is not found, the lookup object refreshes its known keys by calling the loader.
+
+```golang
+func (lookup *keysLookup) refreshMappings() error {
+	var err error
+	lookup.mappings, err = lookup.loader.LoadMappings()
+	if err != nil {
+		return err
+	}
+	lookup.mappings = utils.AddHashedKeys(lookup.mappings)
+	return nil
+}
+```
+
+A contract-specific implementation of the loader enables the storage transformer to fetch metadata associated with a storage key.

 Storage metadata contains: the name of the variable matching the storage key, a raw version of any keys associated with the variable (if the variable is a mapping), and the variable's type.

@@ -72,7 +88,7 @@ type StorageValueMetadata struct {
 }
 ```

-Keys are only relevant if the variable is a mapping. For example, in the following Solidity code:
+The `Keys` field on the metadata is only relevant if the variable is a mapping. For example, in the following Solidity code:

 ```solidity
 pragma solidity ^0.4.0;
@@ -85,7 +101,7 @@ contract Contract {

 The metadata for variable `x` would not have any associated keys, but the metadata for a storage key associated with `y` would include the address used to specify that key's index in the mapping.

-The `SetDB` function is required for the mappings to connect to the database.
+The `SetDB` function is required for the storage key loader to connect to the database.
 A database connection may be desired when keys in a mapping variable need to be read from log events (e.g. to lookup what addresses may exist in `y`, above).

 ### Repository
@@ -0,0 +1,28 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package storage
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+type KeysLoader interface {
+	LoadMappings() (map[common.Hash]utils.StorageValueMetadata, error)
+	SetDB(db *postgres.DB)
+}
@@ -0,0 +1,66 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package storage
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+type KeysLookup interface {
+	Lookup(key common.Hash) (utils.StorageValueMetadata, error)
+	SetDB(db *postgres.DB)
+}
+
+type keysLookup struct {
+	loader   KeysLoader
+	mappings map[common.Hash]utils.StorageValueMetadata
+}
+
+func NewKeysLookup(loader KeysLoader) KeysLookup {
+	return &keysLookup{loader: loader, mappings: make(map[common.Hash]utils.StorageValueMetadata)}
+}
+
+func (lookup *keysLookup) Lookup(key common.Hash) (utils.StorageValueMetadata, error) {
+	metadata, ok := lookup.mappings[key]
+	if !ok {
+		refreshErr := lookup.refreshMappings()
+		if refreshErr != nil {
+			return metadata, refreshErr
+		}
+		metadata, ok = lookup.mappings[key]
+		if !ok {
+			return metadata, utils.ErrStorageKeyNotFound{Key: key.Hex()}
+		}
+	}
+	return metadata, nil
+}
+
+func (lookup *keysLookup) refreshMappings() error {
+	var err error
+	lookup.mappings, err = lookup.loader.LoadMappings()
+	if err != nil {
+		return err
+	}
+	lookup.mappings = utils.AddHashedKeys(lookup.mappings)
+	return nil
+}
+
+func (lookup *keysLookup) SetDB(db *postgres.DB) {
+	lookup.loader.SetDB(db)
+}
@@ -0,0 +1,113 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package storage_test
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/factories/storage"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"github.com/vulcanize/vulcanizedb/test_config"
+)
+
+var _ = Describe("Storage keys lookup", func() {
+	var (
+		fakeMetadata = utils.GetStorageValueMetadata("name", map[utils.Key]string{}, utils.Uint256)
+		lookup       storage.KeysLookup
+		loader       *mocks.MockStorageKeysLoader
+	)
+
+	BeforeEach(func() {
+		loader = &mocks.MockStorageKeysLoader{}
+		lookup = storage.NewKeysLookup(loader)
+	})
+
+	Describe("Lookup", func() {
+		Describe("when key not found", func() {
+			It("refreshes keys", func() {
+				loader.StorageKeyMappings = map[common.Hash]utils.StorageValueMetadata{fakes.FakeHash: fakeMetadata}
+				_, err := lookup.Lookup(fakes.FakeHash)
+
+				Expect(err).NotTo(HaveOccurred())
+				Expect(loader.LoadMappingsCallCount).To(Equal(1))
+			})
+
+			It("returns error if refreshing keys fails", func() {
+				loader.LoadMappingsError = fakes.FakeError
+
+				_, err := lookup.Lookup(fakes.FakeHash)
+
+				Expect(err).To(HaveOccurred())
+				Expect(err).To(MatchError(fakes.FakeError))
+			})
+		})
+
+		Describe("when key found", func() {
+			BeforeEach(func() {
+				loader.StorageKeyMappings = map[common.Hash]utils.StorageValueMetadata{fakes.FakeHash: fakeMetadata}
+				_, err := lookup.Lookup(fakes.FakeHash)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(loader.LoadMappingsCallCount).To(Equal(1))
+			})
+
+			It("does not refresh keys", func() {
+				_, err := lookup.Lookup(fakes.FakeHash)
+
+				Expect(err).NotTo(HaveOccurred())
+				Expect(loader.LoadMappingsCallCount).To(Equal(1))
+			})
+		})
+
+		It("returns metadata for loaded static key", func() {
+			loader.StorageKeyMappings = map[common.Hash]utils.StorageValueMetadata{fakes.FakeHash: fakeMetadata}
+
+			metadata, err := lookup.Lookup(fakes.FakeHash)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(metadata).To(Equal(fakeMetadata))
+		})
+
+		It("returns metadata for hashed version of key (accommodates keys emitted from Geth)", func() {
+			loader.StorageKeyMappings = map[common.Hash]utils.StorageValueMetadata{fakes.FakeHash: fakeMetadata}
+
+			hashedKey := common.BytesToHash(crypto.Keccak256(fakes.FakeHash.Bytes()))
+			metadata, err := lookup.Lookup(hashedKey)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(metadata).To(Equal(fakeMetadata))
+		})
+
+		It("returns key not found error if key not found", func() {
+			_, err := lookup.Lookup(fakes.FakeHash)
+
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(utils.ErrStorageKeyNotFound{Key: fakes.FakeHash.Hex()}))
+		})
+	})
+
+	Describe("SetDB", func() {
+		It("sets the db on the loader", func() {
+			lookup.SetDB(test_config.NewTestDB(test_config.NewTestNode()))
+
+			Expect(loader.SetDBCalled).To(BeTrue())
+		})
+	})
+})
@@ -18,20 +18,19 @@ package storage

 import (
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/storage"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 )

 type Transformer struct {
-	HashedAddress common.Hash
-	Mappings      storage.Mappings
-	Repository    Repository
+	HashedAddress     common.Hash
+	StorageKeysLookup KeysLookup
+	Repository        Repository
 }

 func (transformer Transformer) NewTransformer(db *postgres.DB) transformer.StorageTransformer {
-	transformer.Mappings.SetDB(db)
+	transformer.StorageKeysLookup.SetDB(db)
 	transformer.Repository.SetDB(db)
 	return transformer
 }
@@ -41,7 +40,7 @@ func (transformer Transformer) KeccakContractAddress() common.Hash {
 }

 func (transformer Transformer) Execute(diff utils.StorageDiff) error {
-	metadata, lookupErr := transformer.Mappings.Lookup(diff.StorageKey)
+	metadata, lookupErr := transformer.StorageKeysLookup.Lookup(diff.StorageKey)
 	if lookupErr != nil {
 		return lookupErr
 	}
@@ -28,18 +28,18 @@ import (

 var _ = Describe("Storage transformer", func() {
 	var (
-		mappings   *mocks.MockMappings
-		repository *mocks.MockStorageRepository
-		t          storage.Transformer
+		storageKeysLookup *mocks.MockStorageKeysLookup
+		repository        *mocks.MockStorageRepository
+		t                 storage.Transformer
 	)

 	BeforeEach(func() {
-		mappings = &mocks.MockMappings{}
+		storageKeysLookup = &mocks.MockStorageKeysLookup{}
 		repository = &mocks.MockStorageRepository{}
 		t = storage.Transformer{
-			HashedAddress: common.Hash{},
-			Mappings:      mappings,
-			Repository:    repository,
+			HashedAddress:     common.Hash{},
+			StorageKeysLookup: storageKeysLookup,
+			Repository:        repository,
 		}
 	})

@@ -53,11 +53,11 @@ var _ = Describe("Storage transformer", func() {
 	It("looks up metadata for storage key", func() {
 		t.Execute(utils.StorageDiff{})

-		Expect(mappings.LookupCalled).To(BeTrue())
+		Expect(storageKeysLookup.LookupCalled).To(BeTrue())
 	})

 	It("returns error if lookup fails", func() {
-		mappings.LookupErr = fakes.FakeError
+		storageKeysLookup.LookupErr = fakes.FakeError

 		err := t.Execute(utils.StorageDiff{})

@@ -67,7 +67,7 @@ var _ = Describe("Storage transformer", func() {

 	It("creates storage row with decoded data", func() {
 		fakeMetadata := utils.StorageValueMetadata{Type: utils.Address}
-		mappings.Metadata = fakeMetadata
+		storageKeysLookup.Metadata = fakeMetadata
 		rawValue := common.HexToAddress("0x12345")
 		fakeBlockNumber := 123
 		fakeBlockHash := "0x67890"
@@ -91,7 +91,7 @@ var _ = Describe("Storage transformer", func() {
 	It("returns error if creating row fails", func() {
 		rawValue := common.HexToAddress("0x12345")
 		fakeMetadata := utils.StorageValueMetadata{Type: utils.Address}
-		mappings.Metadata = fakeMetadata
+		storageKeysLookup.Metadata = fakeMetadata
 		repository.CreateErr = fakes.FakeError

 		err := t.Execute(utils.StorageDiff{StorageValue: rawValue.Hash()})
@@ -118,7 +118,7 @@ var _ = Describe("Storage transformer", func() {
 		}

 		It("passes the decoded data items to the repository", func() {
-			mappings.Metadata = fakeMetadata
+			storageKeysLookup.Metadata = fakeMetadata
 			fakeRow := utils.StorageDiff{
 				HashedAddress: common.Hash{},
 				BlockHash:     common.HexToHash(fakeBlockHash),
@@ -140,7 +140,7 @@ var _ = Describe("Storage transformer", func() {
 		})

 		It("returns error if creating a row fails", func() {
-			mappings.Metadata = fakeMetadata
+			storageKeysLookup.Metadata = fakeMetadata
 			repository.CreateErr = fakes.FakeError

 			err := t.Execute(utils.StorageDiff{StorageValue: rawValue.Hash()})