Deal transformer (#19)

2018-09-13 09:06:26 -05:00
parent d69063636e
commit 939ead0c82
23 changed files with 791 additions and 14 deletions
@@ -0,0 +1,25 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package deal
+
+import "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
+
+var Config = shared.TransformerConfig{
+	ContractAddress:     shared.FlipperContractAddress,
+	ContractAbi:         shared.FlipperABI,
+	Topics:              []string{shared.DealSignature},
+	StartingBlockNumber: 0,
+	EndingBlockNumber:   100,
+}
@@ -0,0 +1,58 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package deal
+
+import (
+	"encoding/json"
+	"errors"
+
+	"github.com/ethereum/go-ethereum/core/types"
+)
+
+type Converter interface {
+	ToModel(ethLog types.Log) (DealModel, error)
+}
+
+type DealConverter struct{}
+
+func NewDealConverter() DealConverter {
+	return DealConverter{}
+}
+
+func (DealConverter) ToModel(ethLog types.Log) (DealModel, error) {
+	err := validateLog(ethLog)
+	if err != nil {
+		return DealModel{}, err
+	}
+
+	bidId := ethLog.Topics[2].Big()
+	raw, err := json.Marshal(ethLog)
+	if err != nil {
+		return DealModel{}, err
+	}
+
+	return DealModel{
+		BidId:            bidId.String(),
+		TransactionIndex: ethLog.TxIndex,
+		Raw:              raw,
+	}, nil
+}
+
+func validateLog(ethLog types.Log) error {
+	if len(ethLog.Topics) < 3 {
+		return errors.New("deal log does not contain expected topics")
+	}
+	return nil
+}
@@ -0,0 +1,45 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package deal_test
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/deal"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
+)
+
+var _ = Describe("Flip Deal Converter", func() {
+	It("converts a log to a model", func() {
+		converter := deal.DealConverter{}
+
+		model, err := converter.ToModel(test_data.DealLogNote)
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(model).To(Equal(test_data.DealModel))
+	})
+
+	It("returns an error if the expected amount of topics aren't in the log", func() {
+		converter := deal.DealConverter{}
+		invalidLog := test_data.DealLogNote
+		invalidLog.Topics = []common.Hash{}
+		model, err := converter.ToModel(invalidLog)
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError("deal log does not contain expected topics"))
+		Expect(model).To(Equal(deal.DealModel{}))
+	})
+})
@@ -0,0 +1,19 @@
+package deal_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"io/ioutil"
+	"log"
+)
+
+func TestFlipDeal(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Deal Suite")
+}
+
+var _ = BeforeSuite(func() {
+	log.SetOutput(ioutil.Discard)
+})
@@ -0,0 +1,21 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package deal
+
+type DealModel struct {
+	BidId            string `db:"bid_id"`
+	TransactionIndex uint   `db:"tx_idx"`
+	Raw              []byte `db:"raw_log"`
+}
@@ -0,0 +1,56 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package deal
+
+import (
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+type Repository interface {
+	Create(headerId int64, model DealModel) error
+	MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
+}
+type DealRepository struct {
+	db *postgres.DB
+}
+
+func NewDealRepository(database *postgres.DB) DealRepository {
+	return DealRepository{db: database}
+}
+func (r DealRepository) Create(headerId int64, model DealModel) error {
+	_, err := r.db.Exec(
+		`INSERT into maker.deal (header_id, bid_id, tx_idx, raw_log)
+         VALUES($1, $2, $3, $4)`,
+		headerId, model.BidId, model.TransactionIndex, model.Raw,
+	)
+	return err
+}
+func (r DealRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
+	var missingHeaders []core.Header
+	err := r.db.Select(
+		&missingHeaders,
+		`SELECT headers.id, headers.block_number FROM headers
+               LEFT JOIN maker.deal on headers.id = header_id
+               WHERE header_id ISNULL
+               AND headers.block_number >= $1
+               AND headers.block_number <= $2
+               AND headers.eth_node_fingerprint = $3`,
+		startingBlockNumber,
+		endingBlockNumber,
+		r.db.Node.ID,
+	)
+	return missingHeaders, err
+}
@@ -0,0 +1,138 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package deal_test
+
+import (
+	"math/rand"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/deal"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
+	"github.com/vulcanize/vulcanizedb/test_config"
+)
+
+var _ = Describe("Deal Repository", func() {
+	var node core.Node
+	var db *postgres.DB
+	var dealRepository deal.DealRepository
+	var headerRepository repositories.HeaderRepository
+	var headerId int64
+	var err error
+
+	BeforeEach(func() {
+		node = test_config.NewTestNode()
+		db = test_config.NewTestDB(node)
+		test_config.CleanTestDB(db)
+		dealRepository = deal.NewDealRepository(db)
+		headerRepository = repositories.NewHeaderRepository(db)
+	})
+	Describe("Create", func() {
+		BeforeEach(func() {
+			headerId, err = headerRepository.CreateOrUpdateHeader(core.Header{})
+			Expect(err).NotTo(HaveOccurred())
+			err := dealRepository.Create(headerId, test_data.DealModel)
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It("persists a deal record", func() {
+			var count int
+			db.QueryRow(`SELECT count(*) FROM maker.deal`).Scan(&count)
+			Expect(count).To(Equal(1))
+			var dbResult deal.DealModel
+			err = db.Get(&dbResult, `SELECT bid_id, tx_idx, raw_log FROM maker.deal WHERE header_id = $1`, headerId)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(dbResult.BidId).To(Equal(test_data.DealModel.BidId))
+			Expect(dbResult.TransactionIndex).To(Equal(test_data.DealModel.TransactionIndex))
+			Expect(dbResult.Raw).To(MatchJSON(test_data.DealModel.Raw))
+		})
+
+		It("returns an error if inserting a deal record fails", func() {
+			err = dealRepository.Create(headerId, test_data.DealModel)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
+		})
+
+		It("deletes the deal record if its corresponding header record is deleted", func() {
+			var count int
+			err = db.QueryRow(`SELECT count(*) from maker.deal`).Scan(&count)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(count).To(Equal(1))
+			_, err = db.Exec(`DELETE FROM headers where id = $1`, headerId)
+			Expect(err).NotTo(HaveOccurred())
+			err = db.QueryRow(`SELECT count(*) from maker.deal`).Scan(&count)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(count).To(Equal(0))
+		})
+	})
+
+	Describe("MissingHeaders", func() {
+		var dealBlockNumber int64
+		var startingBlockNumber int64
+		var endingBlockNumber int64
+		var blockNumbers []int64
+
+		BeforeEach(func() {
+			dealBlockNumber = rand.Int63()
+			startingBlockNumber = dealBlockNumber - 1
+			endingBlockNumber = dealBlockNumber + 1
+			outOfRangeBlockNumber := dealBlockNumber + 2
+			blockNumbers = []int64{startingBlockNumber, dealBlockNumber, endingBlockNumber, outOfRangeBlockNumber}
+			var headerIds []int64
+			for _, number := range blockNumbers {
+				headerId, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: number})
+				Expect(err).NotTo(HaveOccurred())
+				headerIds = append(headerIds, headerId)
+			}
+			dealRepository.Create(headerIds[1], test_data.DealModel)
+		})
+
+		It("returns header records that don't have a corresponding deals", func() {
+			missingHeaders, err := dealRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(missingHeaders)).To(Equal(2))
+			Expect(missingHeaders[0].BlockNumber).To(Equal(startingBlockNumber))
+			Expect(missingHeaders[1].BlockNumber).To(Equal(endingBlockNumber))
+		})
+
+		It("only returns missing headers for the given node", func() {
+			node2 := core.Node{}
+			db2 := test_config.NewTestDB(node2)
+			dealRepository2 := deal.NewDealRepository(db2)
+			headerRepository2 := repositories.NewHeaderRepository(db2)
+			var node2HeaderIds []int64
+			for _, number := range blockNumbers {
+				id, err := headerRepository2.CreateOrUpdateHeader(core.Header{BlockNumber: number})
+				node2HeaderIds = append(node2HeaderIds, id)
+				Expect(err).NotTo(HaveOccurred())
+			}
+			missingHeadersNode1, err := dealRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(missingHeadersNode1)).To(Equal(2))
+			Expect(missingHeadersNode1[0].BlockNumber).To(Equal(startingBlockNumber))
+			Expect(missingHeadersNode1[1].BlockNumber).To(Equal(endingBlockNumber))
+			missingHeadersNode2, err := dealRepository2.MissingHeaders(startingBlockNumber, endingBlockNumber)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(missingHeadersNode2)).To(Equal(3))
+			Expect(missingHeadersNode2[0].BlockNumber).To(Equal(startingBlockNumber))
+			Expect(missingHeadersNode2[1].BlockNumber).To(Equal(dealBlockNumber))
+			Expect(missingHeadersNode2[2].BlockNumber).To(Equal(endingBlockNumber))
+		})
+	})
+})
@@ -0,0 +1,77 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package deal
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
+	"log"
+)
+
+type DealTransformer struct {
+	Config     shared.TransformerConfig
+	Converter  Converter
+	Fetcher    shared.LogFetcher
+	Repository Repository
+}
+
+type DealTransformerInitializer struct {
+	Config shared.TransformerConfig
+}
+
+func (i DealTransformerInitializer) NewDealTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer {
+	converter := NewDealConverter()
+	fetcher := shared.NewFetcher(blockChain)
+	repository := NewDealRepository(db)
+	return DealTransformer{
+		Config:     i.Config,
+		Converter:  converter,
+		Fetcher:    fetcher,
+		Repository: repository,
+	}
+}
+
+func (t DealTransformer) Execute() error {
+	config := t.Config
+	topics := [][]common.Hash{{common.HexToHash(shared.DealSignature)}}
+
+	headers, err := t.Repository.MissingHeaders(config.StartingBlockNumber, config.EndingBlockNumber)
+	if err != nil {
+		return err
+	}
+
+	for _, header := range headers {
+		ethLogs, err := t.Fetcher.FetchLogs(config.ContractAddress, topics, header.BlockNumber)
+		if err != nil {
+			log.Println("Error fetching deal logs:", err)
+			return err
+		}
+		for _, ethLog := range ethLogs {
+			model, err := t.Converter.ToModel(ethLog)
+			if err != nil {
+				log.Println("Error converting deal log", err)
+				return err
+			}
+			err = t.Repository.Create(header.Id, model)
+			if err != nil {
+				log.Println("Error persisting deal record", err)
+				return err
+			}
+		}
+	}
+	return err
+}
@@ -0,0 +1,120 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package deal_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/deal"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks"
+	deal_mocks "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks/deal"
+	"math/rand"
+)
+
+var _ = Describe("DealTransformer", func() {
+	var config = deal.Config
+	var dealRepository deal_mocks.MockDealRepository
+	var fetcher mocks.MockLogFetcher
+	var converter deal_mocks.MockDealConverter
+	var transformer deal.DealTransformer
+
+	BeforeEach(func() {
+		dealRepository = deal_mocks.MockDealRepository{}
+		fetcher = mocks.MockLogFetcher{}
+		converter = deal_mocks.MockDealConverter{}
+		transformer = deal.DealTransformer{
+			Repository: &dealRepository,
+			Config:     config,
+			Fetcher:    &fetcher,
+			Converter:  &converter,
+		}
+	})
+
+	It("gets missing headers", func() {
+		err := transformer.Execute()
+		Expect(err).NotTo(HaveOccurred())
+		Expect(dealRepository.PassedStartingBlockNumber).To(Equal(config.StartingBlockNumber))
+		Expect(dealRepository.PassedEndingBlockNumber).To(Equal(config.EndingBlockNumber))
+	})
+
+	It("returns an error if fetching the missing headers fails", func() {
+		dealRepository.SetMissingHeadersErr(fakes.FakeError)
+		err := transformer.Execute()
+		Expect(err).To(HaveOccurred())
+	})
+
+	It("fetches logs for each missing header", func() {
+		header1 := core.Header{BlockNumber: rand.Int63()}
+		header2 := core.Header{BlockNumber: rand.Int63()}
+		dealRepository.SetMissingHeaders([]core.Header{header1, header2})
+		err := transformer.Execute()
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fetcher.FetchedContractAddress).To(Equal(config.ContractAddress))
+		expectedTopics := [][]common.Hash{{common.HexToHash(shared.DealSignature)}}
+		Expect(fetcher.FetchedTopics).To(Equal(expectedTopics))
+		Expect(fetcher.FetchedBlocks).To(Equal([]int64{header1.BlockNumber, header2.BlockNumber}))
+	})
+
+	It("returns an error if fetching logs fails", func() {
+		dealRepository.SetMissingHeaders([]core.Header{{}})
+		fetcher.SetFetcherError(fakes.FakeError)
+		err := transformer.Execute()
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
+	It("converts each eth log to a Model", func() {
+		dealRepository.SetMissingHeaders([]core.Header{{}})
+		fetcher.SetFetchedLogs([]types.Log{test_data.DealLogNote})
+		err := transformer.Execute()
+		Expect(err).NotTo(HaveOccurred())
+		Expect(converter.LogsToConvert).To(Equal([]types.Log{test_data.DealLogNote}))
+	})
+
+	It("returns an error if converting the eth log fails", func() {
+		dealRepository.SetMissingHeaders([]core.Header{{}})
+		fetcher.SetFetchedLogs([]types.Log{test_data.DealLogNote})
+		converter.SetConverterError(fakes.FakeError)
+		err := transformer.Execute()
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError(fakes.FakeError))
+	})
+
+	It("persists each model as a Deal record", func() {
+		header1 := core.Header{Id: rand.Int63()}
+		header2 := core.Header{Id: rand.Int63()}
+		dealRepository.SetMissingHeaders([]core.Header{header1, header2})
+		fetcher.SetFetchedLogs([]types.Log{test_data.DealLogNote})
+		err := transformer.Execute()
+		Expect(err).NotTo(HaveOccurred())
+		Expect(dealRepository.PassedDealModels).To(Equal([]deal.DealModel{test_data.DealModel, test_data.DealModel}))
+		Expect(dealRepository.PassedHeaderIDs).To(Equal([]int64{header1.Id, header2.Id}))
+	})
+
+	It("returns an error if persisting deal record fails", func() {
+		dealRepository.SetMissingHeaders([]core.Header{{}})
+		dealRepository.SetCreateError(fakes.FakeError)
+		fetcher.SetFetchedLogs([]types.Log{test_data.DealLogNote})
+		err := transformer.Execute()
+		Expect(err).To(HaveOccurred())
+	})
+})