Add Cat storage transformer

cmd/parseStorageDiffs.go

@@ -58,6 +58,7 @@ func parseStorageDiffs() {
 	// TODO: configure transformers
 	watcher := shared.NewStorageWatcher(tailer, db)
 	watcher.AddTransformers([]storage.TransformerInitializer{
+		transformers.GetCatStorageTransformer().NewTransformer,
 		transformers.GetPitStorageTransformer().NewTransformer,
 		transformers.GetVatStorageTransformer().NewTransformer,
 		transformers.GetVowStorageTransformer().NewTransformer,

pkg/transformers/storage_diffs/maker/cat/mappings.go

@@ -0,0 +1,199 @@
+package cat
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/sirupsen/logrus"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/maker"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared"
+	"strconv"
+)
+
+const (
+	NFlip = "nflip"
+	Live  = "live"
+	Vat   = "vat"
+	Pit   = "pit"
+	Vow   = "vow"
+
+	IlkFlip = "flip"
+	IlkChop = "chop"
+	IlkLump = "lump"
+
+	FlipIlk = "ilk"
+	FlipUrn = "urn"
+	FlipInk = "ink"
+	FlipTab = "tab"
+)
+
+var (
+	// wards takes up index 0
+	IlksMappingIndex  = storage_diffs.IndexOne // bytes32 => flip address; chop (ray), lump (wad) uint256
+	FlipsMappingIndex = storage_diffs.IndexTwo // uint256 => ilk, urn bytes32; ink, tab uint256 (both wad)
+
+	NFlipKey      = common.HexToHash(storage_diffs.IndexThree)
+	NFlipMetadata = shared.GetStorageValueMetadata(NFlip, nil, shared.Uint256)
+
+	LiveKey      = common.HexToHash(storage_diffs.IndexFour)
+	LiveMetadata = shared.GetStorageValueMetadata(Live, nil, shared.Uint256)
+
+	VatKey      = common.HexToHash(storage_diffs.IndexFive)
+	VatMetadata = shared.GetStorageValueMetadata(Vat, nil, shared.Address)
+
+	PitKey      = common.HexToHash(storage_diffs.IndexSix)
+	PitMetadata = shared.GetStorageValueMetadata(Pit, nil, shared.Address)
+
+	VowKey      = common.HexToHash(storage_diffs.IndexSeven)
+	VowMetadata = shared.GetStorageValueMetadata(Vow, nil, shared.Address)
+)
+
+type CatMappings struct {
+	StorageRepository maker.IMakerStorageRepository
+	mappings          map[common.Hash]shared.StorageValueMetadata
+}
+
+func (mappings CatMappings) Lookup(key common.Hash) (shared.StorageValueMetadata, error) {
+	metadata, ok := mappings.mappings[key]
+	if !ok {
+		err := mappings.loadMappings()
+		if err != nil {
+			return metadata, err
+		}
+		metadata, ok = mappings.mappings[key]
+		if !ok {
+			return metadata, shared.ErrStorageKeyNotFound{Key: key.Hex()}
+		}
+	}
+	return metadata, nil
+}
+
+func (mappings *CatMappings) SetDB(db *postgres.DB) {
+	mappings.StorageRepository.SetDB(db)
+}
+
+func (mappings *CatMappings) loadMappings() error {
+	mappings.mappings = loadStaticMappings()
+	ilkErr := mappings.loadIlkKeys()
+	if ilkErr != nil {
+		return ilkErr
+	}
+
+	flipsErr := mappings.loadFlipsKeys()
+	if flipsErr != nil {
+		return flipsErr
+	}
+
+	return nil
+}
+
+func loadStaticMappings() map[common.Hash]shared.StorageValueMetadata {
+	mappings := make(map[common.Hash]shared.StorageValueMetadata)
+	mappings[NFlipKey] = NFlipMetadata
+	mappings[LiveKey] = LiveMetadata
+	mappings[VatKey] = VatMetadata
+	mappings[PitKey] = PitMetadata
+	mappings[VowKey] = VowMetadata
+	return mappings
+}
+
+// Ilks
+func (mappings *CatMappings) loadIlkKeys() error {
+	ilks, err := mappings.StorageRepository.GetIlks()
+	if err != nil {
+		return err
+	}
+	for _, ilk := range ilks {
+		mappings.mappings[getIlkFlipKey(ilk)] = getIlkFlipMetadata(ilk)
+		mappings.mappings[getIlkChopKey(ilk)] = getIlkChopMetadata(ilk)
+		mappings.mappings[getIlkLumpKey(ilk)] = getIlkLumpMetadata(ilk)
+	}
+	return nil
+}
+
+func getIlkFlipKey(ilk string) common.Hash {
+	return storage_diffs.GetMapping(IlksMappingIndex, ilk)
+}
+
+func getIlkFlipMetadata(ilk string) shared.StorageValueMetadata {
+	keys := map[shared.Key]string{shared.Ilk: ilk}
+	return shared.GetStorageValueMetadata(IlkFlip, keys, shared.Address)
+}
+
+func getIlkChopKey(ilk string) common.Hash {
+	return storage_diffs.GetIncrementedKey(getIlkFlipKey(ilk), 1)
+}
+
+func getIlkChopMetadata(ilk string) shared.StorageValueMetadata {
+	keys := map[shared.Key]string{shared.Ilk: ilk}
+	return shared.GetStorageValueMetadata(IlkChop, keys, shared.Uint256)
+}
+
+func getIlkLumpKey(ilk string) common.Hash {
+	return storage_diffs.GetIncrementedKey(getIlkFlipKey(ilk), 2)
+}
+
+func getIlkLumpMetadata(ilk string) shared.StorageValueMetadata {
+	keys := map[shared.Key]string{shared.Ilk: ilk}
+	return shared.GetStorageValueMetadata(IlkLump, keys, shared.Uint256)
+}
+
+// Flip ID increments each time it happens, so we just need the biggest flip ID from the DB
+// and we can interpolate the sequence [0..max]. This makes sure we track all earlier flips,
+// even if we've missed events
+func (mappings CatMappings) loadFlipsKeys() error {
+	maxFlip, err := mappings.StorageRepository.GetMaxFlip()
+	if err != nil {
+		logrus.Error("loadFlipsKeys: error getting max flip: ", err)
+		return err
+	} else if maxFlip == nil { // No flips occurred yet
+		return nil
+	}
+
+	last := maxFlip.Int64()
+	var flipStr string
+	for flip := 0; int64(flip) <= last; flip++ {
+		flipStr = strconv.Itoa(flip)
+		mappings.mappings[getFlipIlkKey(flipStr)] = getFlipIlkMetadata(flipStr)
+		mappings.mappings[getFlipUrnKey(flipStr)] = getFlipUrnMetadata(flipStr)
+		mappings.mappings[getFlipInkKey(flipStr)] = getFlipInkMetadata(flipStr)
+		mappings.mappings[getFlipTabKey(flipStr)] = getFlipTabMetadata(flipStr)
+	}
+	return nil
+}
+
+func getFlipIlkKey(nflip string) common.Hash {
+	return storage_diffs.GetMapping(FlipsMappingIndex, nflip)
+}
+
+func getFlipIlkMetadata(nflip string) shared.StorageValueMetadata {
+	keys := map[shared.Key]string{shared.Flip: nflip}
+	return shared.GetStorageValueMetadata(FlipIlk, keys, shared.Bytes32)
+}
+
+func getFlipUrnKey(nflip string) common.Hash {
+	return storage_diffs.GetIncrementedKey(getFlipIlkKey(nflip), 1)
+}
+
+func getFlipUrnMetadata(nflip string) shared.StorageValueMetadata {
+	keys := map[shared.Key]string{shared.Flip: nflip}
+	return shared.GetStorageValueMetadata(FlipUrn, keys, shared.Bytes32)
+}
+
+func getFlipInkKey(nflip string) common.Hash {
+	return storage_diffs.GetIncrementedKey(getFlipIlkKey(nflip), 2)
+}
+
+func getFlipInkMetadata(nflip string) shared.StorageValueMetadata {
+	keys := map[shared.Key]string{shared.Flip: nflip}
+	return shared.GetStorageValueMetadata(FlipInk, keys, shared.Uint256)
+}
+
+func getFlipTabKey(nflip string) common.Hash {
+	return storage_diffs.GetIncrementedKey(getFlipIlkKey(nflip), 3)
+}
+
+func getFlipTabMetadata(nflip string) shared.StorageValueMetadata {
+	keys := map[shared.Key]string{shared.Flip: nflip}
+	return shared.GetStorageValueMetadata(FlipTab, keys, shared.Uint256)
+}

pkg/transformers/storage_diffs/maker/cat/repository.go

@@ -0,0 +1,176 @@
+package cat
+
+import (
+	"fmt"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared"
+)
+
+type CatStorageRepository struct {
+	db *postgres.DB
+}
+
+func (repository *CatStorageRepository) Create(blockNumber int, blockHash string, metadata shared.StorageValueMetadata, value interface{}) error {
+	switch metadata.Name {
+	case NFlip:
+		return repository.insertNFlip(blockNumber, blockHash, value.(string))
+	case Live:
+		return repository.insertLive(blockNumber, blockHash, value.(string))
+	case Vat:
+		return repository.insertVat(blockNumber, blockHash, value.(string))
+	case Pit:
+		return repository.insertPit(blockNumber, blockHash, value.(string))
+	case Vow:
+		return repository.insertVow(blockNumber, blockHash, value.(string))
+	case IlkFlip:
+		return repository.insertIlkFlip(blockNumber, blockHash, metadata, value.(string))
+	case IlkChop:
+		return repository.insertIlkChop(blockNumber, blockHash, metadata, value.(string))
+	case IlkLump:
+		return repository.insertIlkLump(blockNumber, blockHash, metadata, value.(string))
+	case FlipIlk:
+		return repository.insertFlipIlk(blockNumber, blockHash, metadata, value.(string))
+	case FlipUrn:
+		return repository.insertFlipUrn(blockNumber, blockHash, metadata, value.(string))
+	case FlipInk:
+		return repository.insertFlipInk(blockNumber, blockHash, metadata, value.(string))
+	case FlipTab:
+		return repository.insertFlipTab(blockNumber, blockHash, metadata, value.(string))
+	default:
+		panic(fmt.Sprintf("unrecognized vat contract storage name: %s", metadata.Name))
+	}
+}
+
+func (repository *CatStorageRepository) SetDB(db *postgres.DB) {
+	repository.db = db
+}
+
+func (repository *CatStorageRepository) insertNFlip(blockNumber int, blockHash string, nflip string) error {
+	_, writeErr := repository.db.Exec(
+		`INSERT INTO maker.cat_nflip (block_number, block_hash, nflip) VALUES ($1, $2, $3)`,
+		blockNumber, blockHash, nflip)
+	return writeErr
+}
+
+func (repository *CatStorageRepository) insertLive(blockNumber int, blockHash string, live string) error {
+	_, writeErr := repository.db.Exec(
+		`INSERT INTO maker.cat_live (block_number, block_hash, live) VALUES ($1, $2, $3 )`,
+		blockNumber, blockHash, live)
+	return writeErr
+}
+
+func (repository *CatStorageRepository) insertVat(blockNumber int, blockHash string, vat string) error {
+	_, writeErr := repository.db.Exec(
+		`INSERT INTO maker.cat_vat (block_number, block_hash, vat) VALUES ($1, $2, $3 )`,
+		blockNumber, blockHash, vat)
+	return writeErr
+}
+
+func (repository *CatStorageRepository) insertPit(blockNumber int, blockHash string, pit string) error {
+	_, writeErr := repository.db.Exec(
+		`INSERT INTO maker.cat_pit (block_number, block_hash, pit) VALUES ($1, $2, $3 )`,
+		blockNumber, blockHash, pit)
+	return writeErr
+}
+
+func (repository *CatStorageRepository) insertVow(blockNumber int, blockHash string, vow string) error {
+	_, writeErr := repository.db.Exec(
+		`INSERT INTO maker.cat_vow (block_number, block_hash, vow) VALUES ($1, $2, $3 )`,
+		blockNumber, blockHash, vow)
+	return writeErr
+}
+
+// Ilks mapping: bytes32 => flip address; chop (ray), lump (wad) uint256
+func (repository *CatStorageRepository) insertIlkFlip(blockNumber int, blockHash string, metadata shared.StorageValueMetadata, flip string) error {
+	ilk, err := getIlk(metadata.Keys)
+	if err != nil {
+		return err
+	}
+	_, writeErr := repository.db.Exec(
+		`INSERT INTO maker.cat_ilk_flip (block_number, block_hash, ilk, flip) VALUES ($1, $2, $3, $4)`,
+		blockNumber, blockHash, ilk, flip)
+	return writeErr
+}
+
+func (repository *CatStorageRepository) insertIlkChop(blockNumber int, blockHash string, metadata shared.StorageValueMetadata, chop string) error {
+	ilk, err := getIlk(metadata.Keys)
+	if err != nil {
+		return err
+	}
+	_, writeErr := repository.db.Exec(
+		`INSERT INTO maker.cat_ilk_chop (block_number, block_hash, ilk, chop) VALUES ($1, $2, $3, $4)`,
+		blockNumber, blockHash, ilk, chop)
+	return writeErr
+}
+
+func (repository *CatStorageRepository) insertIlkLump(blockNumber int, blockHash string, metadata shared.StorageValueMetadata, lump string) error {
+	ilk, err := getIlk(metadata.Keys)
+	if err != nil {
+		return err
+	}
+	_, writeErr := repository.db.Exec(
+		`INSERT INTO maker.cat_ilk_lump (block_number, block_hash, ilk, lump) VALUES ($1, $2, $3, $4)`,
+		blockNumber, blockHash, ilk, lump)
+	return writeErr
+}
+
+// Flips mapping: uint256 => ilk, urn bytes32; ink, tab uint256 (both wad)
+func (repository *CatStorageRepository) insertFlipIlk(blockNumber int, blockHash string, metadata shared.StorageValueMetadata, ilk string) error {
+	nflip, err := getFlip(metadata.Keys)
+	if err != nil {
+		return err
+	}
+	_, writeErr := repository.db.Exec(
+		`INSERT INTO maker.cat_flip_ilk (block_number, block_hash, nflip, ilk) VALUES ($1, $2, $3, $4)`,
+		blockNumber, blockHash, nflip, ilk)
+	return writeErr
+}
+
+func (repository *CatStorageRepository) insertFlipUrn(blockNumber int, blockHash string, metadata shared.StorageValueMetadata, urn string) error {
+	nflip, err := getFlip(metadata.Keys)
+	if err != nil {
+		return err
+	}
+	_, writeErr := repository.db.Exec(
+		`INSERT INTO maker.cat_flip_urn (block_number, block_hash, nflip, urn) VALUES ($1, $2, $3, $4)`,
+		blockNumber, blockHash, nflip, urn)
+	return writeErr
+}
+
+func (repository *CatStorageRepository) insertFlipInk(blockNumber int, blockHash string, metadata shared.StorageValueMetadata, ink string) error {
+	nflip, err := getFlip(metadata.Keys)
+	if err != nil {
+		return err
+	}
+	_, writeErr := repository.db.Exec(
+		`INSERT INTO maker.cat_flip_ink (block_number, block_hash, nflip, ink) VALUES ($1, $2, $3, $4)`,
+		blockNumber, blockHash, nflip, ink)
+	return writeErr
+}
+
+func (repository *CatStorageRepository) insertFlipTab(blockNumber int, blockHash string, metadata shared.StorageValueMetadata, tab string) error {
+	nflip, err := getFlip(metadata.Keys)
+	if err != nil {
+		return err
+	}
+	_, writeErr := repository.db.Exec(
+		`INSERT INTO maker.cat_flip_tab (block_number, block_hash, nflip, tab) VALUES ($1, $2, $3, $4)`,
+		blockNumber, blockHash, nflip, tab)
+	return writeErr
+}
+
+func getIlk(keys map[shared.Key]string) (string, error) {
+	ilk, ok := keys[shared.Ilk]
+	if !ok {
+		return "", shared.ErrMetadataMalformed{MissingData: shared.Ilk}
+	}
+	return ilk, nil
+}
+
+func getFlip(keys map[shared.Key]string) (string, error) {
+	nflip, ok := keys[shared.Flip]
+	if !ok {
+		return "", shared.ErrMetadataMalformed{MissingData: shared.Flip}
+	}
+	return nflip, nil
+}

pkg/transformers/storage_diffs/maker/maker_storage_repository.go

@@ -16,7 +16,11 @@
 
 package maker
 
-import "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+import (
+	"database/sql"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"math/big"
+)
 
 type Urn struct {
 	Ilk string
@@ -25,6 +29,7 @@ type Urn struct {
 
 type IMakerStorageRepository interface {
 	GetDaiKeys() ([]string, error)
+	GetMaxFlip() (*big.Int, error)
 	GetGemKeys() ([]Urn, error)
 	GetIlks() ([]string, error)
 	GetSinKeys() ([]string, error)
@@ -48,6 +53,16 @@ func (repository *MakerStorageRepository) GetDaiKeys() ([]string, error) {
 	return daiKeys, err
 }
 
+func (repository *MakerStorageRepository) GetMaxFlip() (*big.Int, error) {
+	var maxFlip big.Int
+	err := repository.db.Get(&maxFlip, `SELECT MAX(nflip) FROM maker.bite`)
+	if err == sql.ErrNoRows {
+		// No flips have occurred; this is different from flip 0 having occurred
+		return nil, nil
+	}
+	return &maxFlip, err
+}
+
 func (repository *MakerStorageRepository) GetGemKeys() ([]Urn, error) {
 	var gems []Urn
 	err := repository.db.Select(&gems, `

pkg/transformers/storage_diffs/maker/test_helpers/maker_storage_repository.go

@@ -3,6 +3,7 @@ package test_helpers
 import (
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/maker"
+	"math/big"
 )
 
 type MockMakerStorageRepository struct {
@@ -14,11 +15,14 @@ type MockMakerStorageRepository struct {
 	GetGemKeysError  error
 	GetIlksCalled    bool
 	GetIlksError     error
+	GetMaxFlipCalled bool
+	GetMaxFlipError  error
 	GetSinKeysCalled bool
 	GetSinKeysError  error
 	GetUrnsCalled    bool
 	GetUrnsError     error
 	Ilks             []string
+	MaxFlip          *big.Int
 	SinKeys          []string
 	Urns             []maker.Urn
 }
@@ -38,6 +42,11 @@ func (repository *MockMakerStorageRepository) GetIlks() ([]string, error) {
 	return repository.Ilks, repository.GetIlksError
 }
 
+func (repository *MockMakerStorageRepository) GetMaxFlip() (*big.Int, error) {
+	repository.GetMaxFlipCalled = true
+	return repository.MaxFlip, repository.GetMaxFlipError
+}
+
 func (repository *MockMakerStorageRepository) GetSinKeys() ([]string, error) {
 	repository.GetSinKeysCalled = true
 	return repository.SinKeys, repository.GetSinKeysError

pkg/transformers/storage_diffs/shared/decoder.go

@@ -27,6 +27,8 @@ func Decode(row StorageDiffRow, metadata StorageValueMetadata) (interface{}, err
 		return decodeUint256(row.StorageValue.Bytes()), nil
 	case Address:
 		return decodeAddress(row.StorageValue.Bytes()), nil
+	case Bytes32:
+		return row.StorageValue.Hex(), nil
 	default:
 		return nil, ErrTypeNotFound{}
 	}

pkg/transformers/storage_diffs/shared/value.go

@@ -27,8 +27,9 @@ const (
 type Key string
 
 const (
-	Ilk Key = "ilk"
+	Ilk  Key = "ilk"
-	Guy Key = "guy"
+	Guy  Key = "guy"
+	Flip Key = "flip"
 )
 
 type StorageValueMetadata struct {

pkg/transformers/storage_transformers.go

@@ -5,11 +5,20 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/factories/storage"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/maker"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/maker/cat"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/maker/pit"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/maker/vat"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/maker/vow"
 )
 
+func GetCatStorageTransformer() storage.Transformer {
+	return storage.Transformer{
+		Address:    common.HexToAddress(constants.CatContractAddress()),
+		Mappings:   &cat.CatMappings{StorageRepository: &maker.MakerStorageRepository{}},
+		Repository: &cat.CatStorageRepository{},
+	}
+}
+
 func GetPitStorageTransformer() storage.Transformer {
 	return storage.Transformer{
 		Address:    common.HexToAddress(constants.PitContractAddress()),