diff --git a/cmd/parseStorageDiffs.go b/cmd/parseStorageDiffs.go index f826d3cb..d1aee398 100644 --- a/cmd/parseStorageDiffs.go +++ b/cmd/parseStorageDiffs.go @@ -58,6 +58,7 @@ func parseStorageDiffs() { // TODO: configure transformers watcher := shared.NewStorageWatcher(tailer, db) watcher.AddTransformers([]storage.TransformerInitializer{ + transformers.GetCatStorageTransformer().NewTransformer, transformers.GetPitStorageTransformer().NewTransformer, transformers.GetVatStorageTransformer().NewTransformer, transformers.GetVowStorageTransformer().NewTransformer, diff --git a/pkg/transformers/storage_diffs/maker/cat/mappings.go b/pkg/transformers/storage_diffs/maker/cat/mappings.go new file mode 100644 index 00000000..69d7bbf0 --- /dev/null +++ b/pkg/transformers/storage_diffs/maker/cat/mappings.go @@ -0,0 +1,199 @@ +package cat + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/sirupsen/logrus" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs" + "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/maker" + "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared" + "strconv" +) + +const ( + NFlip = "nflip" + Live = "live" + Vat = "vat" + Pit = "pit" + Vow = "vow" + + IlkFlip = "flip" + IlkChop = "chop" + IlkLump = "lump" + + FlipIlk = "ilk" + FlipUrn = "urn" + FlipInk = "ink" + FlipTab = "tab" +) + +var ( + // wards takes up index 0 + IlksMappingIndex = storage_diffs.IndexOne // bytes32 => flip address; chop (ray), lump (wad) uint256 + FlipsMappingIndex = storage_diffs.IndexTwo // uint256 => ilk, urn bytes32; ink, tab uint256 (both wad) + + NFlipKey = common.HexToHash(storage_diffs.IndexThree) + NFlipMetadata = shared.GetStorageValueMetadata(NFlip, nil, shared.Uint256) + + LiveKey = common.HexToHash(storage_diffs.IndexFour) + LiveMetadata = shared.GetStorageValueMetadata(Live, nil, shared.Uint256) + + VatKey = common.HexToHash(storage_diffs.IndexFive) + VatMetadata = shared.GetStorageValueMetadata(Vat, nil, shared.Address) + + PitKey = common.HexToHash(storage_diffs.IndexSix) + PitMetadata = shared.GetStorageValueMetadata(Pit, nil, shared.Address) + + VowKey = common.HexToHash(storage_diffs.IndexSeven) + VowMetadata = shared.GetStorageValueMetadata(Vow, nil, shared.Address) +) + +type CatMappings struct { + StorageRepository maker.IMakerStorageRepository + mappings map[common.Hash]shared.StorageValueMetadata +} + +func (mappings CatMappings) Lookup(key common.Hash) (shared.StorageValueMetadata, error) { + metadata, ok := mappings.mappings[key] + if !ok { + err := mappings.loadMappings() + if err != nil { + return metadata, err + } + metadata, ok = mappings.mappings[key] + if !ok { + return metadata, shared.ErrStorageKeyNotFound{Key: key.Hex()} + } + } + return metadata, nil +} + +func (mappings *CatMappings) SetDB(db *postgres.DB) { + mappings.StorageRepository.SetDB(db) +} + +func (mappings *CatMappings) loadMappings() error { + mappings.mappings = loadStaticMappings() + ilkErr := mappings.loadIlkKeys() + if ilkErr != nil { + return ilkErr + } + + flipsErr := mappings.loadFlipsKeys() + if flipsErr != nil { + return flipsErr + } + + return nil +} + +func loadStaticMappings() map[common.Hash]shared.StorageValueMetadata { + mappings := make(map[common.Hash]shared.StorageValueMetadata) + mappings[NFlipKey] = NFlipMetadata + mappings[LiveKey] = LiveMetadata + mappings[VatKey] = VatMetadata + mappings[PitKey] = PitMetadata + mappings[VowKey] = VowMetadata + return mappings +} + +// Ilks +func (mappings *CatMappings) loadIlkKeys() error { + ilks, err := mappings.StorageRepository.GetIlks() + if err != nil { + return err + } + for _, ilk := range ilks { + mappings.mappings[getIlkFlipKey(ilk)] = getIlkFlipMetadata(ilk) + mappings.mappings[getIlkChopKey(ilk)] = getIlkChopMetadata(ilk) + mappings.mappings[getIlkLumpKey(ilk)] = getIlkLumpMetadata(ilk) + } + return nil +} + +func getIlkFlipKey(ilk string) common.Hash { + return storage_diffs.GetMapping(IlksMappingIndex, ilk) +} + +func getIlkFlipMetadata(ilk string) shared.StorageValueMetadata { + keys := map[shared.Key]string{shared.Ilk: ilk} + return shared.GetStorageValueMetadata(IlkFlip, keys, shared.Address) +} + +func getIlkChopKey(ilk string) common.Hash { + return storage_diffs.GetIncrementedKey(getIlkFlipKey(ilk), 1) +} + +func getIlkChopMetadata(ilk string) shared.StorageValueMetadata { + keys := map[shared.Key]string{shared.Ilk: ilk} + return shared.GetStorageValueMetadata(IlkChop, keys, shared.Uint256) +} + +func getIlkLumpKey(ilk string) common.Hash { + return storage_diffs.GetIncrementedKey(getIlkFlipKey(ilk), 2) +} + +func getIlkLumpMetadata(ilk string) shared.StorageValueMetadata { + keys := map[shared.Key]string{shared.Ilk: ilk} + return shared.GetStorageValueMetadata(IlkLump, keys, shared.Uint256) +} + +// Flip ID increments each time it happens, so we just need the biggest flip ID from the DB +// and we can interpolate the sequence [0..max]. This makes sure we track all earlier flips, +// even if we've missed events +func (mappings CatMappings) loadFlipsKeys() error { + maxFlip, err := mappings.StorageRepository.GetMaxFlip() + if err != nil { + logrus.Error("loadFlipsKeys: error getting max flip: ", err) + return err + } else if maxFlip == nil { // No flips occurred yet + return nil + } + + last := maxFlip.Int64() + var flipStr string + for flip := 0; int64(flip) <= last; flip++ { + flipStr = strconv.Itoa(flip) + mappings.mappings[getFlipIlkKey(flipStr)] = getFlipIlkMetadata(flipStr) + mappings.mappings[getFlipUrnKey(flipStr)] = getFlipUrnMetadata(flipStr) + mappings.mappings[getFlipInkKey(flipStr)] = getFlipInkMetadata(flipStr) + mappings.mappings[getFlipTabKey(flipStr)] = getFlipTabMetadata(flipStr) + } + return nil +} + +func getFlipIlkKey(nflip string) common.Hash { + return storage_diffs.GetMapping(FlipsMappingIndex, nflip) +} + +func getFlipIlkMetadata(nflip string) shared.StorageValueMetadata { + keys := map[shared.Key]string{shared.Flip: nflip} + return shared.GetStorageValueMetadata(FlipIlk, keys, shared.Bytes32) +} + +func getFlipUrnKey(nflip string) common.Hash { + return storage_diffs.GetIncrementedKey(getFlipIlkKey(nflip), 1) +} + +func getFlipUrnMetadata(nflip string) shared.StorageValueMetadata { + keys := map[shared.Key]string{shared.Flip: nflip} + return shared.GetStorageValueMetadata(FlipUrn, keys, shared.Bytes32) +} + +func getFlipInkKey(nflip string) common.Hash { + return storage_diffs.GetIncrementedKey(getFlipIlkKey(nflip), 2) +} + +func getFlipInkMetadata(nflip string) shared.StorageValueMetadata { + keys := map[shared.Key]string{shared.Flip: nflip} + return shared.GetStorageValueMetadata(FlipInk, keys, shared.Uint256) +} + +func getFlipTabKey(nflip string) common.Hash { + return storage_diffs.GetIncrementedKey(getFlipIlkKey(nflip), 3) +} + +func getFlipTabMetadata(nflip string) shared.StorageValueMetadata { + keys := map[shared.Key]string{shared.Flip: nflip} + return shared.GetStorageValueMetadata(FlipTab, keys, shared.Uint256) +} diff --git a/pkg/transformers/storage_diffs/maker/cat/repository.go b/pkg/transformers/storage_diffs/maker/cat/repository.go new file mode 100644 index 00000000..ce59c22e --- /dev/null +++ b/pkg/transformers/storage_diffs/maker/cat/repository.go @@ -0,0 +1,176 @@ +package cat + +import ( + "fmt" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared" +) + +type CatStorageRepository struct { + db *postgres.DB +} + +func (repository *CatStorageRepository) Create(blockNumber int, blockHash string, metadata shared.StorageValueMetadata, value interface{}) error { + switch metadata.Name { + case NFlip: + return repository.insertNFlip(blockNumber, blockHash, value.(string)) + case Live: + return repository.insertLive(blockNumber, blockHash, value.(string)) + case Vat: + return repository.insertVat(blockNumber, blockHash, value.(string)) + case Pit: + return repository.insertPit(blockNumber, blockHash, value.(string)) + case Vow: + return repository.insertVow(blockNumber, blockHash, value.(string)) + case IlkFlip: + return repository.insertIlkFlip(blockNumber, blockHash, metadata, value.(string)) + case IlkChop: + return repository.insertIlkChop(blockNumber, blockHash, metadata, value.(string)) + case IlkLump: + return repository.insertIlkLump(blockNumber, blockHash, metadata, value.(string)) + case FlipIlk: + return repository.insertFlipIlk(blockNumber, blockHash, metadata, value.(string)) + case FlipUrn: + return repository.insertFlipUrn(blockNumber, blockHash, metadata, value.(string)) + case FlipInk: + return repository.insertFlipInk(blockNumber, blockHash, metadata, value.(string)) + case FlipTab: + return repository.insertFlipTab(blockNumber, blockHash, metadata, value.(string)) + default: + panic(fmt.Sprintf("unrecognized vat contract storage name: %s", metadata.Name)) + } +} + +func (repository *CatStorageRepository) SetDB(db *postgres.DB) { + repository.db = db +} + +func (repository *CatStorageRepository) insertNFlip(blockNumber int, blockHash string, nflip string) error { + _, writeErr := repository.db.Exec( + `INSERT INTO maker.cat_nflip (block_number, block_hash, nflip) VALUES ($1, $2, $3)`, + blockNumber, blockHash, nflip) + return writeErr +} + +func (repository *CatStorageRepository) insertLive(blockNumber int, blockHash string, live string) error { + _, writeErr := repository.db.Exec( + `INSERT INTO maker.cat_live (block_number, block_hash, live) VALUES ($1, $2, $3 )`, + blockNumber, blockHash, live) + return writeErr +} + +func (repository *CatStorageRepository) insertVat(blockNumber int, blockHash string, vat string) error { + _, writeErr := repository.db.Exec( + `INSERT INTO maker.cat_vat (block_number, block_hash, vat) VALUES ($1, $2, $3 )`, + blockNumber, blockHash, vat) + return writeErr +} + +func (repository *CatStorageRepository) insertPit(blockNumber int, blockHash string, pit string) error { + _, writeErr := repository.db.Exec( + `INSERT INTO maker.cat_pit (block_number, block_hash, pit) VALUES ($1, $2, $3 )`, + blockNumber, blockHash, pit) + return writeErr +} + +func (repository *CatStorageRepository) insertVow(blockNumber int, blockHash string, vow string) error { + _, writeErr := repository.db.Exec( + `INSERT INTO maker.cat_vow (block_number, block_hash, vow) VALUES ($1, $2, $3 )`, + blockNumber, blockHash, vow) + return writeErr +} + +// Ilks mapping: bytes32 => flip address; chop (ray), lump (wad) uint256 +func (repository *CatStorageRepository) insertIlkFlip(blockNumber int, blockHash string, metadata shared.StorageValueMetadata, flip string) error { + ilk, err := getIlk(metadata.Keys) + if err != nil { + return err + } + _, writeErr := repository.db.Exec( + `INSERT INTO maker.cat_ilk_flip (block_number, block_hash, ilk, flip) VALUES ($1, $2, $3, $4)`, + blockNumber, blockHash, ilk, flip) + return writeErr +} + +func (repository *CatStorageRepository) insertIlkChop(blockNumber int, blockHash string, metadata shared.StorageValueMetadata, chop string) error { + ilk, err := getIlk(metadata.Keys) + if err != nil { + return err + } + _, writeErr := repository.db.Exec( + `INSERT INTO maker.cat_ilk_chop (block_number, block_hash, ilk, chop) VALUES ($1, $2, $3, $4)`, + blockNumber, blockHash, ilk, chop) + return writeErr +} + +func (repository *CatStorageRepository) insertIlkLump(blockNumber int, blockHash string, metadata shared.StorageValueMetadata, lump string) error { + ilk, err := getIlk(metadata.Keys) + if err != nil { + return err + } + _, writeErr := repository.db.Exec( + `INSERT INTO maker.cat_ilk_lump (block_number, block_hash, ilk, lump) VALUES ($1, $2, $3, $4)`, + blockNumber, blockHash, ilk, lump) + return writeErr +} + +// Flips mapping: uint256 => ilk, urn bytes32; ink, tab uint256 (both wad) +func (repository *CatStorageRepository) insertFlipIlk(blockNumber int, blockHash string, metadata shared.StorageValueMetadata, ilk string) error { + nflip, err := getFlip(metadata.Keys) + if err != nil { + return err + } + _, writeErr := repository.db.Exec( + `INSERT INTO maker.cat_flip_ilk (block_number, block_hash, nflip, ilk) VALUES ($1, $2, $3, $4)`, + blockNumber, blockHash, nflip, ilk) + return writeErr +} + +func (repository *CatStorageRepository) insertFlipUrn(blockNumber int, blockHash string, metadata shared.StorageValueMetadata, urn string) error { + nflip, err := getFlip(metadata.Keys) + if err != nil { + return err + } + _, writeErr := repository.db.Exec( + `INSERT INTO maker.cat_flip_urn (block_number, block_hash, nflip, urn) VALUES ($1, $2, $3, $4)`, + blockNumber, blockHash, nflip, urn) + return writeErr +} + +func (repository *CatStorageRepository) insertFlipInk(blockNumber int, blockHash string, metadata shared.StorageValueMetadata, ink string) error { + nflip, err := getFlip(metadata.Keys) + if err != nil { + return err + } + _, writeErr := repository.db.Exec( + `INSERT INTO maker.cat_flip_ink (block_number, block_hash, nflip, ink) VALUES ($1, $2, $3, $4)`, + blockNumber, blockHash, nflip, ink) + return writeErr +} + +func (repository *CatStorageRepository) insertFlipTab(blockNumber int, blockHash string, metadata shared.StorageValueMetadata, tab string) error { + nflip, err := getFlip(metadata.Keys) + if err != nil { + return err + } + _, writeErr := repository.db.Exec( + `INSERT INTO maker.cat_flip_tab (block_number, block_hash, nflip, tab) VALUES ($1, $2, $3, $4)`, + blockNumber, blockHash, nflip, tab) + return writeErr +} + +func getIlk(keys map[shared.Key]string) (string, error) { + ilk, ok := keys[shared.Ilk] + if !ok { + return "", shared.ErrMetadataMalformed{MissingData: shared.Ilk} + } + return ilk, nil +} + +func getFlip(keys map[shared.Key]string) (string, error) { + nflip, ok := keys[shared.Flip] + if !ok { + return "", shared.ErrMetadataMalformed{MissingData: shared.Flip} + } + return nflip, nil +} diff --git a/pkg/transformers/storage_diffs/maker/maker_storage_repository.go b/pkg/transformers/storage_diffs/maker/maker_storage_repository.go index 44ff4998..a9813005 100644 --- a/pkg/transformers/storage_diffs/maker/maker_storage_repository.go +++ b/pkg/transformers/storage_diffs/maker/maker_storage_repository.go @@ -16,7 +16,11 @@ package maker -import "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" +import ( + "database/sql" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "math/big" +) type Urn struct { Ilk string @@ -25,6 +29,7 @@ type Urn struct { type IMakerStorageRepository interface { GetDaiKeys() ([]string, error) + GetMaxFlip() (*big.Int, error) GetGemKeys() ([]Urn, error) GetIlks() ([]string, error) GetSinKeys() ([]string, error) @@ -48,6 +53,16 @@ func (repository *MakerStorageRepository) GetDaiKeys() ([]string, error) { return daiKeys, err } +func (repository *MakerStorageRepository) GetMaxFlip() (*big.Int, error) { + var maxFlip big.Int + err := repository.db.Get(&maxFlip, `SELECT MAX(nflip) FROM maker.bite`) + if err == sql.ErrNoRows { + // No flips have occurred; this is different from flip 0 having occurred + return nil, nil + } + return &maxFlip, err +} + func (repository *MakerStorageRepository) GetGemKeys() ([]Urn, error) { var gems []Urn err := repository.db.Select(&gems, ` diff --git a/pkg/transformers/storage_diffs/maker/test_helpers/maker_storage_repository.go b/pkg/transformers/storage_diffs/maker/test_helpers/maker_storage_repository.go index 161cac53..3ce7c08c 100644 --- a/pkg/transformers/storage_diffs/maker/test_helpers/maker_storage_repository.go +++ b/pkg/transformers/storage_diffs/maker/test_helpers/maker_storage_repository.go @@ -3,6 +3,7 @@ package test_helpers import ( "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/maker" + "math/big" ) type MockMakerStorageRepository struct { @@ -14,11 +15,14 @@ type MockMakerStorageRepository struct { GetGemKeysError error GetIlksCalled bool GetIlksError error + GetMaxFlipCalled bool + GetMaxFlipError error GetSinKeysCalled bool GetSinKeysError error GetUrnsCalled bool GetUrnsError error Ilks []string + MaxFlip *big.Int SinKeys []string Urns []maker.Urn } @@ -38,6 +42,11 @@ func (repository *MockMakerStorageRepository) GetIlks() ([]string, error) { return repository.Ilks, repository.GetIlksError } +func (repository *MockMakerStorageRepository) GetMaxFlip() (*big.Int, error) { + repository.GetMaxFlipCalled = true + return repository.MaxFlip, repository.GetMaxFlipError +} + func (repository *MockMakerStorageRepository) GetSinKeys() ([]string, error) { repository.GetSinKeysCalled = true return repository.SinKeys, repository.GetSinKeysError diff --git a/pkg/transformers/storage_diffs/shared/decoder.go b/pkg/transformers/storage_diffs/shared/decoder.go index 753b2416..7591f8c0 100644 --- a/pkg/transformers/storage_diffs/shared/decoder.go +++ b/pkg/transformers/storage_diffs/shared/decoder.go @@ -27,6 +27,8 @@ func Decode(row StorageDiffRow, metadata StorageValueMetadata) (interface{}, err return decodeUint256(row.StorageValue.Bytes()), nil case Address: return decodeAddress(row.StorageValue.Bytes()), nil + case Bytes32: + return row.StorageValue.Hex(), nil default: return nil, ErrTypeNotFound{} } diff --git a/pkg/transformers/storage_diffs/shared/value.go b/pkg/transformers/storage_diffs/shared/value.go index 4bc59d7f..e5543bce 100644 --- a/pkg/transformers/storage_diffs/shared/value.go +++ b/pkg/transformers/storage_diffs/shared/value.go @@ -27,8 +27,9 @@ const ( type Key string const ( - Ilk Key = "ilk" - Guy Key = "guy" + Ilk Key = "ilk" + Guy Key = "guy" + Flip Key = "flip" ) type StorageValueMetadata struct { diff --git a/pkg/transformers/storage_transformers.go b/pkg/transformers/storage_transformers.go index a0d80d1d..48e2771b 100644 --- a/pkg/transformers/storage_transformers.go +++ b/pkg/transformers/storage_transformers.go @@ -5,11 +5,20 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/transformers/factories/storage" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/maker" + "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/maker/cat" "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/maker/pit" "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/maker/vat" "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/maker/vow" ) +func GetCatStorageTransformer() storage.Transformer { + return storage.Transformer{ + Address: common.HexToAddress(constants.CatContractAddress()), + Mappings: &cat.CatMappings{StorageRepository: &maker.MakerStorageRepository{}}, + Repository: &cat.CatStorageRepository{}, + } +} + func GetPitStorageTransformer() storage.Transformer { return storage.Transformer{ Address: common.HexToAddress(constants.PitContractAddress()),