Backfill Pit.file log events

2018-09-04 16:07:35 -05:00
parent 53635ec362
commit db4b891398
32 changed files with 707 additions and 25 deletions
@@ -0,0 +1,13 @@
+package pit_file
+
+import (
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
+)
+
+var PitFileConfig = shared.TransformerConfig{
+	ContractAddress:     "0xff3f2400f1600f3f493a9a92704a29b96795af1a", // temporary address from Ganache deploy
+	ContractAbi:         shared.PitABI,
+	Topics:              []string{shared.PitFileSignature},
+	StartingBlockNumber: 0,
+	EndingBlockNumber:   100,
+}
@@ -0,0 +1,32 @@
+package pit_file
+
+import (
+	"bytes"
+	"encoding/json"
+	"github.com/ethereum/go-ethereum/core/types"
+	"math/big"
+)
+
+type Converter interface {
+	ToModel(contractAddress string, contractAbi string, ethLog types.Log) (PitFileModel, error)
+}
+
+type PitFileConverter struct {
+}
+
+func (PitFileConverter) ToModel(contractAddress string, contractAbi string, ethLog types.Log) (entity PitFileModel, err error) {
+	ilk := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
+	what := string(bytes.Trim(ethLog.Topics[3].Bytes(), "\x00"))
+	itemByteLength := 32
+	riskBytes := ethLog.Data[len(ethLog.Data)-itemByteLength:]
+	risk := big.NewInt(0).SetBytes(riskBytes).String()
+
+	raw, err := json.Marshal(ethLog)
+	return PitFileModel{
+		Ilk:              ilk,
+		What:             what,
+		Risk:             risk,
+		TransactionIndex: ethLog.TxIndex,
+		Raw:              raw,
+	}, err
+}
@@ -0,0 +1,20 @@
+package pit_file_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
+)
+
+var _ = Describe("Pit file converter", func() {
+	It("converts a log to an entity", func() {
+		converter := pit_file.PitFileConverter{}
+
+		model, err := converter.ToModel(test_data.PitAddress, shared.PitABI, test_data.EthPitFileLog)
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(model).To(Equal(test_data.PitFileModel))
+	})
+})
@@ -0,0 +1,9 @@
+package pit_file
+
+type PitFileModel struct {
+	Ilk              string
+	What             string
+	Risk             string
+	TransactionIndex uint   `db:"tx_idx"`
+	Raw              []byte `db:"raw_log"`
+}
@@ -0,0 +1,13 @@
+package pit_file_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestPitFile(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "PitFile Suite")
+}
@@ -0,0 +1,48 @@
+package pit_file
+
+import (
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+type Repository interface {
+	Create(headerID int64, transactionIndex uint, model PitFileModel) error
+	MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
+}
+
+type PitFileRepository struct {
+	db *postgres.DB
+}
+
+func NewPitFileRepository(db *postgres.DB) PitFileRepository {
+	return PitFileRepository{
+		db: db,
+	}
+}
+
+func (repository PitFileRepository) Create(headerID int64, transactionIndex uint, model PitFileModel) error {
+	_, err := repository.db.Exec(
+		`INSERT into maker.pit_file (header_id, ilk, what, risk, tx_idx, raw_log)
+        VALUES($1, $2, $3, $4::NUMERIC, $5, $6)`,
+		headerID, model.Ilk, model.What, model.Risk, model.TransactionIndex, model.Raw,
+	)
+	return err
+}
+
+func (repository PitFileRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
+	var result []core.Header
+	err := repository.db.Select(
+		&result,
+		`SELECT headers.id, headers.block_number FROM headers
+               LEFT JOIN maker.pit_file on headers.id = header_id
+               WHERE header_id ISNULL
+               AND headers.block_number >= $1
+               AND headers.block_number <= $2
+               AND headers.eth_node_fingerprint = $3`,
+		startingBlockNumber,
+		endingBlockNumber,
+		repository.db.Node.ID,
+	)
+
+	return result, err
+}
@@ -0,0 +1,130 @@
+package pit_file_test
+
+import (
+	"database/sql"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
+	"github.com/vulcanize/vulcanizedb/test_config"
+)
+
+var _ = Describe("Pit file repository", func() {
+	Describe("Create", func() {
+		It("adds a pit file event", func() {
+			db := test_config.NewTestDB(core.Node{})
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
+			Expect(err).NotTo(HaveOccurred())
+			pitFileRepository := pit_file.NewPitFileRepository(db)
+
+			err = pitFileRepository.Create(headerID, 123, test_data.PitFileModel)
+
+			Expect(err).NotTo(HaveOccurred())
+			var dbPitFile pit_file.PitFileModel
+			err = db.Get(&dbPitFile, `SELECT ilk, what, risk, tx_idx, raw_log FROM maker.pit_file WHERE header_id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(dbPitFile.Ilk).To(Equal(test_data.PitFileModel.Ilk))
+			Expect(dbPitFile.What).To(Equal(test_data.PitFileModel.What))
+			Expect(dbPitFile.Risk).To(Equal(test_data.PitFileModel.Risk))
+			Expect(dbPitFile.TransactionIndex).To(Equal(test_data.PitFileModel.TransactionIndex))
+			// The raw bytes scanned out of the DB are compressed (missing whitespace), such that they don't strictly equal the input bytes
+			Expect(dbPitFile.Raw).To(MatchJSON(test_data.PitFileModel.Raw))
+		})
+
+		It("does not duplicate pit file events", func() {
+			db := test_config.NewTestDB(core.Node{})
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
+			Expect(err).NotTo(HaveOccurred())
+			pitFileRepository := pit_file.NewPitFileRepository(db)
+			err = pitFileRepository.Create(headerID, 123, test_data.PitFileModel)
+			Expect(err).NotTo(HaveOccurred())
+
+			err = pitFileRepository.Create(headerID, 123, test_data.PitFileModel)
+
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
+		})
+
+		It("removes pit file if corresponding header is deleted", func() {
+			db := test_config.NewTestDB(core.Node{})
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
+			Expect(err).NotTo(HaveOccurred())
+			pitFileRepository := pit_file.NewPitFileRepository(db)
+			err = pitFileRepository.Create(headerID, 123, test_data.PitFileModel)
+			Expect(err).NotTo(HaveOccurred())
+
+			_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
+
+			Expect(err).NotTo(HaveOccurred())
+			var dbPitFile pit_file.PitFileModel
+			err = db.Get(&dbPitFile, `SELECT ilk, what, risk, tx_idx, raw_log FROM maker.pit_file WHERE header_id = $1`, headerID)
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(sql.ErrNoRows))
+		})
+	})
+
+	Describe("MissingHeaders", func() {
+		It("returns headers with no associated pit file event", func() {
+			db := test_config.NewTestDB(core.Node{})
+			test_config.CleanTestDB(db)
+			headerRepository := repositories.NewHeaderRepository(db)
+			startingBlockNumber := int64(1)
+			pitFileBlockNumber := int64(2)
+			endingBlockNumber := int64(3)
+			blockNumbers := []int64{startingBlockNumber, pitFileBlockNumber, endingBlockNumber, endingBlockNumber + 1}
+			var headerIDs []int64
+			for _, n := range blockNumbers {
+				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				headerIDs = append(headerIDs, headerID)
+				Expect(err).NotTo(HaveOccurred())
+			}
+			pitFileRepository := pit_file.NewPitFileRepository(db)
+			err := pitFileRepository.Create(headerIDs[1], 123, test_data.PitFileModel)
+			Expect(err).NotTo(HaveOccurred())
+
+			headers, err := pitFileRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(headers)).To(Equal(2))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
+		})
+
+		It("only returns headers associated with the current node", func() {
+			db := test_config.NewTestDB(core.Node{})
+			test_config.CleanTestDB(db)
+			blockNumbers := []int64{1, 2, 3}
+			headerRepository := repositories.NewHeaderRepository(db)
+			dbTwo := test_config.NewTestDB(core.Node{ID: "second"})
+			headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
+			var headerIDs []int64
+			for _, n := range blockNumbers {
+				headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				Expect(err).NotTo(HaveOccurred())
+				headerIDs = append(headerIDs, headerID)
+				_, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n})
+				Expect(err).NotTo(HaveOccurred())
+			}
+			pitFileRepository := pit_file.NewPitFileRepository(db)
+			pitFileRepositoryTwo := pit_file.NewPitFileRepository(dbTwo)
+			err := pitFileRepository.Create(headerIDs[0], 0, test_data.PitFileModel)
+			Expect(err).NotTo(HaveOccurred())
+
+			nodeOneMissingHeaders, err := pitFileRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(nodeOneMissingHeaders)).To(Equal(len(blockNumbers) - 1))
+
+			nodeTwoMissingHeaders, err := pitFileRepositoryTwo.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(nodeTwoMissingHeaders)).To(Equal(len(blockNumbers)))
+		})
+	})
+})
@@ -0,0 +1,56 @@
+package pit_file
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
+)
+
+type PitFileTransformer struct {
+	Config     shared.TransformerConfig
+	Converter  Converter
+	Fetcher    shared.LogFetcher
+	Repository Repository
+}
+
+type PitFileTransformerInitializer struct {
+	Config shared.TransformerConfig
+}
+
+func (initializer PitFileTransformerInitializer) NewPitFileTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer {
+	converter := PitFileConverter{}
+	fetcher := shared.NewFetcher(blockChain)
+	repository := NewPitFileRepository(db)
+	return PitFileTransformer{
+		Config:     initializer.Config,
+		Converter:  converter,
+		Fetcher:    fetcher,
+		Repository: repository,
+	}
+}
+
+func (transformer PitFileTransformer) Execute() error {
+	missingHeaders, err := transformer.Repository.MissingHeaders(transformer.Config.StartingBlockNumber, transformer.Config.EndingBlockNumber)
+	if err != nil {
+		return err
+	}
+	for _, header := range missingHeaders {
+		topics := [][]common.Hash{{common.HexToHash(shared.PitFileSignature)}}
+		matchingLogs, err := transformer.Fetcher.FetchLogs(PitFileConfig.ContractAddress, topics, header.BlockNumber)
+		if err != nil {
+			return err
+		}
+		for _, log := range matchingLogs {
+			model, err := transformer.Converter.ToModel(PitFileConfig.ContractAddress, shared.PitABI, log)
+			if err != nil {
+				return err
+			}
+			err = transformer.Repository.Create(header.Id, log.TxIndex, model)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
@@ -0,0 +1,162 @@
+package pit_file_test
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks"
+	pit_file_mocks "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks/pit_file"
+)
+
+var _ = Describe("Pit file transformer", func() {
+	It("gets missing headers for block numbers specified in config", func() {
+		repository := &pit_file_mocks.MockPitFileRepository{}
+		transformer := pit_file.PitFileTransformer{
+			Config:     pit_file.PitFileConfig,
+			Fetcher:    &mocks.MockLogFetcher{},
+			Converter:  &pit_file_mocks.MockPitFileConverter{},
+			Repository: repository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(repository.PassedStartingBlockNumber).To(Equal(pit_file.PitFileConfig.StartingBlockNumber))
+		Expect(repository.PassedEndingBlockNumber).To(Equal(pit_file.PitFileConfig.EndingBlockNumber))
+	})
+
+	It("returns error if repository returns error for missing headers", func() {
+		repository := &pit_file_mocks.MockPitFileRepository{}
+		repository.SetMissingHeadersErr(fakes.FakeError)
+		transformer := pit_file.PitFileTransformer{
+			Fetcher:    &mocks.MockLogFetcher{},
+			Converter:  &pit_file_mocks.MockPitFileConverter{},
+			Repository: repository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
+	It("fetches logs for missing headers", func() {
+		fetcher := &mocks.MockLogFetcher{}
+		repository := &pit_file_mocks.MockPitFileRepository{}
+		repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}, {BlockNumber: 2}})
+		transformer := pit_file.PitFileTransformer{
+			Fetcher:    fetcher,
+			Converter:  &pit_file_mocks.MockPitFileConverter{},
+			Repository: repository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fetcher.FetchedBlocks).To(Equal([]int64{1, 2}))
+		Expect(fetcher.FetchedContractAddress).To(Equal(pit_file.PitFileConfig.ContractAddress))
+		Expect(fetcher.FetchedTopics).To(Equal([][]common.Hash{{common.HexToHash(shared.PitFileSignature)}}))
+	})
+
+	It("returns error if fetcher returns error", func() {
+		fetcher := &mocks.MockLogFetcher{}
+		fetcher.SetFetcherError(fakes.FakeError)
+		repository := &pit_file_mocks.MockPitFileRepository{}
+		repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}})
+		transformer := pit_file.PitFileTransformer{
+			Fetcher:    fetcher,
+			Converter:  &pit_file_mocks.MockPitFileConverter{},
+			Repository: repository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
+	It("converts matching logs", func() {
+		converter := &pit_file_mocks.MockPitFileConverter{}
+		fetcher := &mocks.MockLogFetcher{}
+		fetcher.SetFetchedLogs([]types.Log{test_data.EthPitFileLog})
+		repository := &pit_file_mocks.MockPitFileRepository{}
+		repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}})
+		transformer := pit_file.PitFileTransformer{
+			Fetcher:    fetcher,
+			Converter:  converter,
+			Repository: repository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(converter.PassedContractAddress).To(Equal(pit_file.PitFileConfig.ContractAddress))
+		Expect(converter.PassedContractABI).To(Equal(pit_file.PitFileConfig.ContractAbi))
+		Expect(converter.PassedLog).To(Equal(test_data.EthPitFileLog))
+	})
+
+	It("returns error if converter returns error", func() {
+		converter := &pit_file_mocks.MockPitFileConverter{}
+		converter.SetConverterError(fakes.FakeError)
+		fetcher := &mocks.MockLogFetcher{}
+		fetcher.SetFetchedLogs([]types.Log{test_data.EthPitFileLog})
+		repository := &pit_file_mocks.MockPitFileRepository{}
+		repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}})
+		transformer := pit_file.PitFileTransformer{
+			Fetcher:    fetcher,
+			Converter:  converter,
+			Repository: repository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
+	It("persists pit file model", func() {
+		converter := &pit_file_mocks.MockPitFileConverter{}
+		fetcher := &mocks.MockLogFetcher{}
+		fetcher.SetFetchedLogs([]types.Log{test_data.EthPitFileLog})
+		repository := &pit_file_mocks.MockPitFileRepository{}
+		fakeHeader := core.Header{BlockNumber: 1, Id: 2}
+		repository.SetMissingHeaders([]core.Header{fakeHeader})
+		transformer := pit_file.PitFileTransformer{
+			Fetcher:    fetcher,
+			Converter:  converter,
+			Repository: repository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id))
+		Expect(repository.PassedTransactionIndex).To(Equal(test_data.EthPitFileLog.TxIndex))
+		Expect(repository.PassedModel).To(Equal(test_data.PitFileModel))
+	})
+
+	It("returns error if repository returns error for create", func() {
+		converter := &pit_file_mocks.MockPitFileConverter{}
+		fetcher := &mocks.MockLogFetcher{}
+		fetcher.SetFetchedLogs([]types.Log{test_data.EthPitFileLog})
+		repository := &pit_file_mocks.MockPitFileRepository{}
+		repository.SetMissingHeaders([]core.Header{{BlockNumber: 1, Id: 2}})
+		repository.SetCreateError(fakes.FakeError)
+		transformer := pit_file.PitFileTransformer{
+			Fetcher:    fetcher,
+			Converter:  converter,
+			Repository: repository,
+		}
+
+		err := transformer.Execute()
+
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+})