From 2e72a7e940c38dc90dab98b252dc6a41d082df6b Mon Sep 17 00:00:00 2001 From: Taka Goto Date: Wed, 17 Oct 2018 10:53:06 -0500 Subject: [PATCH] vat flux transformer --- cmd/continuousLogSync.go | 1 + pkg/transformers/shared/constants.go | 5 +- .../shared/event_signature_generator_test.go | 7 + .../test_data/mocks/vat_flux/converter.go | 22 ++ .../test_data/mocks/vat_flux/repository.go | 65 +++++ pkg/transformers/test_data/vat_flux.go | 52 ++++ pkg/transformers/transformers.go | 3 + pkg/transformers/vat_flux/config.go | 11 + pkg/transformers/vat_flux/converter.go | 75 ++++++ pkg/transformers/vat_flux/converter_test.go | 54 +++++ pkg/transformers/vat_flux/model.go | 24 ++ pkg/transformers/vat_flux/repository.go | 84 +++++++ pkg/transformers/vat_flux/repository_test.go | 226 ++++++++++++++++++ pkg/transformers/vat_flux/transformer.go | 63 +++++ pkg/transformers/vat_flux/transformer_test.go | 178 ++++++++++++++ .../vat_flux/vat_flux_suite_test.go | 19 ++ pkg/transformers/vat_grab/tranformer.go | 2 +- test_config/test_config.go | 1 + 18 files changed, 890 insertions(+), 2 deletions(-) create mode 100644 pkg/transformers/test_data/mocks/vat_flux/converter.go create mode 100644 pkg/transformers/test_data/mocks/vat_flux/repository.go create mode 100644 pkg/transformers/test_data/vat_flux.go create mode 100644 pkg/transformers/vat_flux/config.go create mode 100644 pkg/transformers/vat_flux/converter.go create mode 100644 pkg/transformers/vat_flux/converter_test.go create mode 100644 pkg/transformers/vat_flux/model.go create mode 100644 pkg/transformers/vat_flux/repository.go create mode 100644 pkg/transformers/vat_flux/repository_test.go create mode 100644 pkg/transformers/vat_flux/transformer.go create mode 100644 pkg/transformers/vat_flux/transformer_test.go create mode 100644 pkg/transformers/vat_flux/vat_flux_suite_test.go diff --git a/cmd/continuousLogSync.go b/cmd/continuousLogSync.go index fefd665d..c3138173 100644 --- a/cmd/continuousLogSync.go +++ b/cmd/continuousLogSync.go @@ -111,6 +111,7 @@ func buildTransformerInitializerMap() map[string]shared2.TransformerInitializer transformerInitializerMap["vatFold"] = transformers.VatFoldTransformerInitializer transformerInitializerMap["vatToll"] = transformers.VatTollTransformerInitializer transformerInitializerMap["vatTune"] = transformers.VatTuneTransformerInitializer + transformerInitializerMap["vatFlux"] = transformers.VatFluxTransformerInitializer return transformerInitializerMap } diff --git a/pkg/transformers/shared/constants.go b/pkg/transformers/shared/constants.go index 40234ff6..df43dc7c 100644 --- a/pkg/transformers/shared/constants.go +++ b/pkg/transformers/shared/constants.go @@ -34,7 +34,8 @@ var ( PipContractAddress = "0x9FfFE440258B79c5d6604001674A4722FfC0f7Bc" PitContractAddress = "0xe7cf3198787c9a4daac73371a38f29aaeeced87e" RepContractAddress = "0xf88bbdc1e2718f8857f30a180076ec38d53cf296" - VatContractAddress = "0xcd726790550afcd77e9a7a47e86a3f9010af126b" + //VatContractAddress = "0xcd726790550afcd77e9a7a47e86a3f9010af126b" + VatContractAddress = "0x239e6f0ab02713f1f8aa90ebeded9fc66dc96cd6" //TODO: get pit and drip file method signatures directly from the ABI biteMethod = GetSolidityMethodSignature(CatABI, "Bite") @@ -62,6 +63,7 @@ var ( vatFoldMethod = GetSolidityMethodSignature(VatABI, "fold") vatTollMethod = GetSolidityMethodSignature(VatABI, "toll") vatTuneMethod = GetSolidityMethodSignature(VatABI, "tune") + vatFluxMethod = GetSolidityMethodSignature(VatABI, "flux") BiteSignature = GetEventSignature(biteMethod) DealSignature = GetLogNoteSignature(dealMethod) @@ -88,4 +90,5 @@ var ( VatFoldSignature = GetLogNoteSignature(vatFoldMethod) VatTollSignature = GetLogNoteSignature(vatTollMethod) VatTuneSignature = GetLogNoteSignature(vatTuneMethod) + VatFluxSignature = GetLogNoteSignature(vatFluxMethod) ) diff --git a/pkg/transformers/shared/event_signature_generator_test.go b/pkg/transformers/shared/event_signature_generator_test.go index 7c96cf99..44be477d 100644 --- a/pkg/transformers/shared/event_signature_generator_test.go +++ b/pkg/transformers/shared/event_signature_generator_test.go @@ -133,6 +133,13 @@ var _ = Describe("Event signature generator", func() { Expect(expected).To(Equal(actual)) }) + It("gets the vat flux method signature", func() { + expected := "flux(bytes32,bytes32,bytes32,int256)" + actual := shared.GetSolidityMethodSignature(shared.VatABI, "flux") + + Expect(expected).To(Equal(actual)) + }) + It("gets the flip deal method signature", func() { expected := "deal(uint256)" actual := shared.GetSolidityMethodSignature(shared.FlipperABI, "deal") diff --git a/pkg/transformers/test_data/mocks/vat_flux/converter.go b/pkg/transformers/test_data/mocks/vat_flux/converter.go new file mode 100644 index 00000000..c8cfd15a --- /dev/null +++ b/pkg/transformers/test_data/mocks/vat_flux/converter.go @@ -0,0 +1,22 @@ +package vat_flux + +import ( + "github.com/ethereum/go-ethereum/core/types" + + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_flux" +) + +type MockVatFlux struct { + err error + PassedLogs []types.Log +} + +func (converter *MockVatFlux) ToModels(ethLogs []types.Log) ([]vat_flux.VatFluxModel, error) { + converter.PassedLogs = ethLogs + return []vat_flux.VatFluxModel{test_data.VatFluxModel}, converter.err +} + +func (converter *MockVatFlux) SetConverterError(e error) { + converter.err = e +} diff --git a/pkg/transformers/test_data/mocks/vat_flux/repository.go b/pkg/transformers/test_data/mocks/vat_flux/repository.go new file mode 100644 index 00000000..332ba278 --- /dev/null +++ b/pkg/transformers/test_data/mocks/vat_flux/repository.go @@ -0,0 +1,65 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vat_flux + +import ( + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_flux" +) + +type MockVatFluxRepository struct { + createErr error + markHeaderCheckedErr error + MarkHeaderCheckedPassedHeaderID int64 + missingHeaders []core.Header + missingHeadersErr error + PassedStartingBlockNumber int64 + PassedEndingBlockNumber int64 + PassedHeaderID int64 + PassedModels []vat_flux.VatFluxModel +} + +func (repository *MockVatFluxRepository) MarkCheckedHeader(headerId int64) error { + repository.MarkHeaderCheckedPassedHeaderID = headerId + return repository.markHeaderCheckedErr +} + +func (repository *MockVatFluxRepository) Create(headerID int64, models []vat_flux.VatFluxModel) error { + repository.PassedHeaderID = headerID + repository.PassedModels = models + return repository.createErr +} + +func (repository *MockVatFluxRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + repository.PassedStartingBlockNumber = startingBlockNumber + repository.PassedEndingBlockNumber = endingBlockNumber + return repository.missingHeaders, repository.missingHeadersErr +} + +func (repository *MockVatFluxRepository) SetMarkHeaderCheckedErr(e error) { + repository.markHeaderCheckedErr = e +} + +func (repository *MockVatFluxRepository) SetMissingHeadersErr(e error) { + repository.missingHeadersErr = e +} + +func (repository *MockVatFluxRepository) SetMissingHeaders(headers []core.Header) { + repository.missingHeaders = headers +} + +func (repository *MockVatFluxRepository) SetCreateError(e error) { + repository.createErr = e +} diff --git a/pkg/transformers/test_data/vat_flux.go b/pkg/transformers/test_data/vat_flux.go new file mode 100644 index 00000000..76dc4f49 --- /dev/null +++ b/pkg/transformers/test_data/vat_flux.go @@ -0,0 +1,52 @@ +/* + * Copyright 2018 Vulcanize + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package test_data + +import ( + "encoding/json" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_flux" +) + +var VatFluxLog = types.Log{ + Address: common.HexToAddress("0x239e6f0ab02713f1f8aa90ebeded9fc66dc96cd6"), + Topics: []common.Hash{ + common.HexToHash("0xa6e4182100000000000000000000000000000000000000000000000000000000"), + common.HexToHash("0x0000000000000000000000007340e006f4135ba6970d43bf43d88dcad4e7a8ca"), + common.HexToHash("0x0000000000000000000000007fa9ef6609ca7921112231f8f195138ebba29770"), + common.HexToHash("0x00000000000000000000000093086347c52a8878af71bb818509d484c6a2e1bf"), + }, + Data: hexutil.MustDecode("0x00000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000084a6e418217340e006f4135ba6970d43bf43d88dcad4e7a8ca00000000000000000000000007fa9ef6609ca7921112231f8f195138ebba297700000000000000000000000093086347c52a8878af71bb818509d484c6a2e1bf000000000000000000000000000000000000000000000000000000000000000000000000000000000000007b"), + BlockNumber: 23, + TxHash: common.HexToHash("0xf98681bab9b8c75bd8aa4a7d0a8142ff527c5ea8fa54f3c2835d4533838b2e6f"), + TxIndex: 0, + BlockHash: common.HexToHash("0xc3fe212ad4f81ade1265af6de2b4bb50d962b1a4db06aabc982e7f9cb0972c2d"), + Index: 0, + Removed: false, +} + +var rawFluxLog, _ = json.Marshal(VatFluxLog) +var VatFluxModel = vat_flux.VatFluxModel{ + Ilk: "0x7340e006f4135BA6970D43bf43d88DCAD4e7a8CA", + Src: "0x07Fa9eF6609cA7921112231F8f195138ebbA2977", + Dst: "0x93086347c52a8878af71bB818509d484c6a2e1bF", + Rad: "123", + TransactionIndex: 0, + Raw: rawFluxLog, +} diff --git a/pkg/transformers/transformers.go b/pkg/transformers/transformers.go index e13e6c59..a7e5bb8c 100644 --- a/pkg/transformers/transformers.go +++ b/pkg/transformers/transformers.go @@ -37,6 +37,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/tend" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_flux" "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_fold" "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_grab" "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_heal" @@ -75,6 +76,7 @@ var ( VatFoldTransformerInitializer = vat_fold.VatFoldTransformerInitializer{Config: vat_fold.VatFoldConfig}.NewVatFoldTransformer VatTollTransformerInitializer = vat_toll.VatTollTransformerInitializer{Config: vat_toll.VatTollConfig}.NewVatTollTransformer VatTuneTransformerInitializer = vat_tune.VatTuneTransformerInitializer{Config: vat_tune.VatTuneConfig}.NewVatTuneTransformer + VatFluxTransformerInitializer = vat_flux.VatFluxTransformerInitializer{Config: vat_flux.VatFluxConfig}.NewVatFluxTransformer ) func TransformerInitializers() []shared.TransformerInitializer { @@ -104,5 +106,6 @@ func TransformerInitializers() []shared.TransformerInitializer { VatFoldTransformerInitializer, VatTollTransformerInitializer, VatTuneTransformerInitializer, + VatFluxTransformerInitializer, } } diff --git a/pkg/transformers/vat_flux/config.go b/pkg/transformers/vat_flux/config.go new file mode 100644 index 00000000..c7f3449f --- /dev/null +++ b/pkg/transformers/vat_flux/config.go @@ -0,0 +1,11 @@ +package vat_flux + +import "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + +var VatFluxConfig = shared.TransformerConfig{ + ContractAddresses: []string{shared.VatContractAddress}, + ContractAbi: shared.VatABI, + Topics: []string{shared.VatFluxSignature}, + StartingBlockNumber: 0, + EndingBlockNumber: 10000000, +} diff --git a/pkg/transformers/vat_flux/converter.go b/pkg/transformers/vat_flux/converter.go new file mode 100644 index 00000000..47dac29a --- /dev/null +++ b/pkg/transformers/vat_flux/converter.go @@ -0,0 +1,75 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vat_flux + +import ( + "encoding/json" + "errors" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "math/big" +) + +type Converter interface { + ToModels(ethLogs []types.Log) ([]VatFluxModel, error) +} + +type VatFluxConverter struct{} + +func (VatFluxConverter) ToModels(ethLogs []types.Log) ([]VatFluxModel, error) { + var models []VatFluxModel + for _, ethLog := range ethLogs { + err := verifyLog(ethLog) + if err != nil { + return nil, err + } + + ilk := common.BytesToAddress(ethLog.Topics[1].Bytes()[:]) + src := common.BytesToAddress(ethLog.Topics[2].Bytes()[:]) + dst := common.BytesToAddress(ethLog.Topics[3].Bytes()[:]) + radBytes := shared.GetDataBytesAtIndex(-1, ethLog.Data) + rad := big.NewInt(0).SetBytes(radBytes).String() + + if err != nil { + return nil, err + } + + rawLogJson, err := json.Marshal(ethLog) + if err != nil { + return nil, err + } + + model := VatFluxModel{ + Ilk: ilk.String(), + Src: src.String(), + Dst: dst.String(), + Rad: rad, + TransactionIndex: ethLog.TxIndex, + Raw: rawLogJson, + } + + models = append(models, model) + } + + return models, nil +} + +func verifyLog(log types.Log) error { + if len(log.Topics) < 4 { + return errors.New("log missing topics") + } + return nil +} diff --git a/pkg/transformers/vat_flux/converter_test.go b/pkg/transformers/vat_flux/converter_test.go new file mode 100644 index 00000000..d84ac555 --- /dev/null +++ b/pkg/transformers/vat_flux/converter_test.go @@ -0,0 +1,54 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vat_flux_test + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_flux" +) + +var _ = Describe("VatFlux converter", func() { + It("Converts logs to models", func() { + converter := vat_flux.VatFluxConverter{} + models, err := converter.ToModels([]types.Log{test_data.VatFluxLog}) + + Expect(err).NotTo(HaveOccurred()) + Expect(models[0].Ilk).To(Equal(test_data.VatFluxModel.Ilk)) + Expect(models[0].Src).To(Equal(test_data.VatFluxModel.Src)) + Expect(models[0].Dst).To(Equal(test_data.VatFluxModel.Dst)) + Expect(models[0].Rad).To(Equal(test_data.VatFluxModel.Rad)) + Expect(models[0].TransactionIndex).To(Equal(test_data.VatFluxModel.TransactionIndex)) + Expect(models[0].Raw).To(Equal(test_data.VatFluxModel.Raw)) + }) + + It("Returns an error there are missing topics", func() { + converter := vat_flux.VatFluxConverter{} + badLog := types.Log{ + Topics: []common.Hash{ + common.HexToHash("0x"), + common.HexToHash("0x"), + common.HexToHash("0x"), + }, + } + _, err := converter.ToModels([]types.Log{badLog}) + + Expect(err).To(HaveOccurred()) + }) +}) diff --git a/pkg/transformers/vat_flux/model.go b/pkg/transformers/vat_flux/model.go new file mode 100644 index 00000000..a63310af --- /dev/null +++ b/pkg/transformers/vat_flux/model.go @@ -0,0 +1,24 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vat_flux + +type VatFluxModel struct { + Ilk string + Src string + Dst string + Rad string + TransactionIndex uint `db:"tx_idx"` + Raw []byte `db:"raw_log"` +} diff --git a/pkg/transformers/vat_flux/repository.go b/pkg/transformers/vat_flux/repository.go new file mode 100644 index 00000000..412629b6 --- /dev/null +++ b/pkg/transformers/vat_flux/repository.go @@ -0,0 +1,84 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vat_flux + +import ( + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" +) + +type Repository interface { + Create(headerId int64, models []VatFluxModel) error + MissingHeaders(startingBlock, endingBlock int64) ([]core.Header, error) + MarkCheckedHeader(headerId int64) error +} + +type VatFluxRepository struct { + DB *postgres.DB +} + +func NewVatFluxRepository(db *postgres.DB) VatFluxRepository { + return VatFluxRepository{DB: db} +} + +func (repository VatFluxRepository) Create(headerId int64, models []VatFluxModel) error { + tx, err := repository.DB.Begin() + if err != nil { + return err + } + + for _, model := range models { + _, err := tx.Exec(`INSERT INTO maker.vat_flux (header_id, ilk, dst, src, rad, tx_idx, raw_log) + VALUES($1, $2, $3, $4, $5, $6, $7)`, + headerId, model.Ilk, model.Dst, model.Src, model.Rad, model.TransactionIndex, model.Raw) + if err != nil { + tx.Rollback() + return err + } + } + + _, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, vat_flux_checked) + VALUES($1, $2) + ON CONFLICT (header_id) DO + UPDATE SET vat_flux_checked = $2`, headerId, true) + if err != nil { + tx.Rollback() + return err + } + return tx.Commit() +} + +func (repository VatFluxRepository) MissingHeaders(startingBlock, endingBlock int64) ([]core.Header, error) { + var headers []core.Header + err := repository.DB.Select(&headers, + `SELECT headers.id, block_number from headers + LEFT JOIN checked_headers on headers.id = header_id + WHERE (header_id ISNULL OR vat_flux_checked IS FALSE) + AND headers.block_number >= $1 + AND headers.block_number <= $2 + AND headers.eth_node_fingerprint = $3`, + startingBlock, endingBlock, repository.DB.Node.ID) + + return headers, err +} + +func (repository VatFluxRepository) MarkCheckedHeader(headerId int64) error { + _, err := repository.DB.Exec(`INSERT INTO public.checked_headers (header_id, vat_flux_checked) + VALUES($1, $2) + ON CONFLICT (header_id) DO + UPDATE SET vat_flux_checked = $2`, headerId, true) + + return err +} diff --git a/pkg/transformers/vat_flux/repository_test.go b/pkg/transformers/vat_flux/repository_test.go new file mode 100644 index 00000000..b472e84b --- /dev/null +++ b/pkg/transformers/vat_flux/repository_test.go @@ -0,0 +1,226 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vat_flux_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_flux" + "github.com/vulcanize/vulcanizedb/test_config" +) + +var _ = Describe("VatFlux Repository", func() { + var db *postgres.DB + var repository vat_flux.VatFluxRepository + var headerRepository repositories.HeaderRepository + var headerId int64 + var err error + + BeforeEach(func() { + node := test_config.NewTestNode() + db = test_config.NewTestDB(node) + test_config.CleanTestDB(db) + repository = vat_flux.VatFluxRepository{DB: db} + headerRepository = repositories.NewHeaderRepository(db) + headerId, err = headerRepository.CreateOrUpdateHeader(core.Header{}) + Expect(err).NotTo(HaveOccurred()) + }) + + type VatFluxDBResult struct { + vat_flux.VatFluxModel + Id int + HeaderId int64 `db:"header_id"` + } + + type CheckedHeaderResult struct { + VatFluxChecked bool `db:"vat_flux_checked"` + } + + Describe("Create", func() { + It("persists vat flux records", func() { + anotherVatFlux := test_data.VatFluxModel + anotherVatFlux.TransactionIndex = test_data.VatFluxModel.TransactionIndex + 1 + err = repository.Create(headerId, []vat_flux.VatFluxModel{test_data.VatFluxModel, anotherVatFlux}) + + var dbResult []VatFluxDBResult + err = db.Select(&dbResult, `SELECT * from maker.vat_flux where header_id = $1`, headerId) + Expect(err).NotTo(HaveOccurred()) + Expect(len(dbResult)).To(Equal(2)) + Expect(dbResult[0].Ilk).To(Equal(test_data.VatFluxModel.Ilk)) + Expect(dbResult[0].Dst).To(Equal(test_data.VatFluxModel.Dst)) + Expect(dbResult[0].Src).To(Equal(test_data.VatFluxModel.Src)) + Expect(dbResult[0].Rad).To(Equal(test_data.VatFluxModel.Rad)) + Expect(dbResult[0].TransactionIndex).To(Equal(test_data.VatFluxModel.TransactionIndex)) + Expect(dbResult[1].TransactionIndex).To(Equal(test_data.VatFluxModel.TransactionIndex + 1)) + Expect(dbResult[0].Raw).To(MatchJSON(test_data.VatFluxModel.Raw)) + Expect(dbResult[0].HeaderId).To(Equal(headerId)) + }) + + It("returns an error if the insertion fails", func() { + err = repository.Create(headerId, []vat_flux.VatFluxModel{test_data.VatFluxModel}) + Expect(err).NotTo(HaveOccurred()) + err = repository.Create(headerId, []vat_flux.VatFluxModel{test_data.VatFluxModel}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint")) + }) + + It("marks the header as checked for vat flux logs", func() { + err = repository.Create(headerId, []vat_flux.VatFluxModel{test_data.VatFluxModel}) + Expect(err).NotTo(HaveOccurred()) + + var headerChecked bool + err = db.Get(&headerChecked, `SELECT vat_flux_checked FROM public.checked_headers WHERE header_id = $1`, headerId) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + + It("removes vat flux if corresponding header is deleted", func() { + err = repository.Create(headerId, []vat_flux.VatFluxModel{test_data.VatFluxModel}) + Expect(err).NotTo(HaveOccurred()) + + _, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerId) + + Expect(err).NotTo(HaveOccurred()) + var count int + err = db.QueryRow(`SELECT count(*) from maker.vat_flux`).Scan(&count) + Expect(err).NotTo(HaveOccurred()) + Expect(count).To(Equal(0)) + }) + + It("wraps create in a transaction", func() { + err = repository.Create(headerId, []vat_flux.VatFluxModel{test_data.VatFluxModel, test_data.VatFluxModel}) + Expect(err).To(HaveOccurred()) + var count int + err = repository.DB.QueryRowx(`SELECT count(*) FROM maker.vat_flux`).Scan(&count) + Expect(count).To(Equal(0)) + }) + }) + + Describe("MissingHeaders", func() { + It("returns headers that haven't been checked", func() { + startingBlock := GinkgoRandomSeed() + vatFluxBlock := startingBlock + 1 + endingBlock := startingBlock + 2 + outsideRangeBlock := startingBlock + 3 + + var headerIds []int64 + blockNumbers := []int64{startingBlock, vatFluxBlock, endingBlock, outsideRangeBlock} + for _, n := range blockNumbers { + headerId, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + Expect(err).NotTo(HaveOccurred()) + headerIds = append(headerIds, headerId) + } + + err = repository.MarkCheckedHeader(headerIds[0]) + Expect(err).NotTo(HaveOccurred()) + + headers, err := repository.MissingHeaders(startingBlock, endingBlock) + Expect(err).NotTo(HaveOccurred()) + Expect(headers[0].Id).To(Or(Equal(headerIds[1]), Equal(headerIds[2]))) + Expect(headers[1].Id).To(Or(Equal(headerIds[1]), Equal(headerIds[2]))) + Expect(len(headers)).To(Equal(2)) + }) + + It("returns header ids when checked_headers.vat_flux is false", func() { + startingBlock := GinkgoRandomSeed() + vatFluxBlock := startingBlock + 1 + endingBlock := startingBlock + 2 + outsideRangeBlock := startingBlock + 3 + + var headerIds []int64 + blockNumbers := []int64{startingBlock, vatFluxBlock, endingBlock, outsideRangeBlock} + for _, n := range blockNumbers { + headerId, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + Expect(err).NotTo(HaveOccurred()) + headerIds = append(headerIds, headerId) + } + + err = repository.MarkCheckedHeader(headerIds[0]) + _, err = repository.DB.Exec(`INSERT INTO checked_headers (header_id) VALUES ($1)`, headerIds[1]) + Expect(err).NotTo(HaveOccurred()) + + headers, err := repository.MissingHeaders(startingBlock, endingBlock) + Expect(err).NotTo(HaveOccurred()) + Expect(headers[0].Id).To(Or(Equal(headerIds[1]), Equal(headerIds[2]))) + Expect(headers[1].Id).To(Or(Equal(headerIds[1]), Equal(headerIds[2]))) + Expect(len(headers)).To(Equal(2)) + }) + + It("only returns header ids for the current node", func() { + startingBlock := GinkgoRandomSeed() + vatFluxBlock := startingBlock + 1 + endingBlock := startingBlock + 2 + outsideRangeBlock := startingBlock + 3 + db2 := test_config.NewTestDB(core.Node{ID: "second node"}) + headerRepository2 := repositories.NewHeaderRepository(db2) + repository2 := vat_flux.NewVatFluxRepository(db2) + + var headerIds []int64 + blockNumbers := []int64{startingBlock, vatFluxBlock, endingBlock, outsideRangeBlock} + for _, n := range blockNumbers { + headerId, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + Expect(err).NotTo(HaveOccurred()) + headerIds = append(headerIds, headerId) + + _, err = headerRepository2.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + Expect(err).NotTo(HaveOccurred()) + } + + err = repository.MarkCheckedHeader(headerIds[0]) + Expect(err).NotTo(HaveOccurred()) + + nodeOneMissingHeaders, err := repository.MissingHeaders(startingBlock, endingBlock) + Expect(err).NotTo(HaveOccurred()) + Expect(len(nodeOneMissingHeaders)).To(Equal(2)) + + nodeTwoMissingHeaders, err := repository2.MissingHeaders(startingBlock, endingBlock) + Expect(err).NotTo(HaveOccurred()) + Expect(len(nodeTwoMissingHeaders)).To(Equal(3)) + }) + }) + + Describe("MarkCheckedHeader", func() { + It("creates a new checked_header record", func() { + err := repository.MarkCheckedHeader(headerId) + Expect(err).NotTo(HaveOccurred()) + + var checkedHeaderResult = CheckedHeaderResult{} + err = db.Get(&checkedHeaderResult, `SELECT vat_flux_checked FROM checked_headers WHERE header_id = $1`, headerId) + Expect(err).NotTo(HaveOccurred()) + Expect(checkedHeaderResult.VatFluxChecked).To(BeTrue()) + }) + + It("updates an existing checked header", func() { + _, err := repository.DB.Exec(`INSERT INTO checked_headers (header_id) VALUES($1)`, headerId) + Expect(err).NotTo(HaveOccurred()) + + var checkedHeaderResult CheckedHeaderResult + err = db.Get(&checkedHeaderResult, `SELECT vat_flux_checked FROM checked_headers WHERE header_id = $1`, headerId) + Expect(err).NotTo(HaveOccurred()) + Expect(checkedHeaderResult.VatFluxChecked).To(BeFalse()) + + err = repository.MarkCheckedHeader(headerId) + Expect(err).NotTo(HaveOccurred()) + + err = db.Get(&checkedHeaderResult, `SELECT vat_flux_checked FROM checked_headers WHERE header_id = $1`, headerId) + Expect(err).NotTo(HaveOccurred()) + Expect(checkedHeaderResult.VatFluxChecked).To(BeTrue()) + }) + }) +}) diff --git a/pkg/transformers/vat_flux/transformer.go b/pkg/transformers/vat_flux/transformer.go new file mode 100644 index 00000000..0d0d8bf8 --- /dev/null +++ b/pkg/transformers/vat_flux/transformer.go @@ -0,0 +1,63 @@ +package vat_flux + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "log" +) + +type VatFluxTransformerInitializer struct { + Config shared.TransformerConfig +} + +func (initializer VatFluxTransformerInitializer) NewVatFluxTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer { + converter := VatFluxConverter{} + fetcher := shared.NewFetcher(blockChain) + repository := NewVatFluxRepository(db) + return VatFluxTransformer{ + Config: initializer.Config, + Converter: converter, + Fetcher: fetcher, + Repository: repository, + } +} + +type VatFluxTransformer struct { + Config shared.TransformerConfig + Converter Converter + Fetcher shared.LogFetcher + Repository Repository +} + +func (transformer VatFluxTransformer) Execute() error { + missingHeaders, err := transformer.Repository.MissingHeaders(transformer.Config.StartingBlockNumber, transformer.Config.EndingBlockNumber) + if err != nil { + return err + } + log.Printf("Fetching vat flux event logs for %d headers \n", len(missingHeaders)) + for _, header := range missingHeaders { + topics := [][]common.Hash{{common.HexToHash(shared.VatFluxSignature)}} + matchingLogs, err := transformer.Fetcher.FetchLogs(VatFluxConfig.ContractAddresses, topics, header.BlockNumber) + if err != nil { + return err + } + if len(matchingLogs) < 1 { + err = transformer.Repository.MarkCheckedHeader(header.Id) + if err != nil { + return err + } + continue + } + models, err := transformer.Converter.ToModels(matchingLogs) + if err != nil { + return err + } + err = transformer.Repository.Create(header.Id, models) + if err != nil { + return err + } + } + return nil +} diff --git a/pkg/transformers/vat_flux/transformer_test.go b/pkg/transformers/vat_flux/transformer_test.go new file mode 100644 index 00000000..18f68e27 --- /dev/null +++ b/pkg/transformers/vat_flux/transformer_test.go @@ -0,0 +1,178 @@ +package vat_flux_test + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/fakes" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks" + vat_flux_mocks "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks/vat_flux" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_flux" +) + +var _ = Describe("Vat flux transformer", func() { + It("gets missing headers for block numbers specified in config", func() { + repository := &vat_flux_mocks.MockVatFluxRepository{} + transformer := vat_flux.VatFluxTransformer{ + Config: vat_flux.VatFluxConfig, + Fetcher: &mocks.MockLogFetcher{}, + Converter: &vat_flux_mocks.MockVatFlux{}, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + Expect(repository.PassedStartingBlockNumber).To(Equal(vat_flux.VatFluxConfig.StartingBlockNumber)) + Expect(repository.PassedEndingBlockNumber).To(Equal(vat_flux.VatFluxConfig.EndingBlockNumber)) + }) + + It("returns error if repository returns error for missing headers", func() { + repository := &vat_flux_mocks.MockVatFluxRepository{} + repository.SetMissingHeadersErr(fakes.FakeError) + transformer := vat_flux.VatFluxTransformer{ + Fetcher: &mocks.MockLogFetcher{}, + Converter: &vat_flux_mocks.MockVatFlux{}, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + + It("fetches logs for missing headers", func() { + fetcher := &mocks.MockLogFetcher{} + repository := &vat_flux_mocks.MockVatFluxRepository{} + repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}, {BlockNumber: 2}}) + transformer := vat_flux.VatFluxTransformer{ + Fetcher: fetcher, + Converter: &vat_flux_mocks.MockVatFlux{}, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + Expect(fetcher.FetchedBlocks).To(Equal([]int64{1, 2})) + Expect(fetcher.FetchedContractAddresses).To(Equal([][]string{vat_flux.VatFluxConfig.ContractAddresses, vat_flux.VatFluxConfig.ContractAddresses})) + Expect(fetcher.FetchedTopics).To(Equal([][]common.Hash{{common.HexToHash(shared.VatFluxSignature)}})) + }) + + It("returns error if fetcher returns error", func() { + fetcher := &mocks.MockLogFetcher{} + fetcher.SetFetcherError(fakes.FakeError) + repository := &vat_flux_mocks.MockVatFluxRepository{} + repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) + transformer := vat_flux.VatFluxTransformer{ + Fetcher: fetcher, + Converter: &vat_flux_mocks.MockVatFlux{}, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + + It("returns error if marking header checked returns err", func() { + mockConverter := &vat_flux_mocks.MockVatFlux{} + mockRepository := &vat_flux_mocks.MockVatFluxRepository{} + mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}}) + mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError) + mockFetcher := &mocks.MockLogFetcher{} + transformer := vat_flux.VatFluxTransformer{ + Converter: mockConverter, + Fetcher: mockFetcher, + Repository: mockRepository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + + It("converts matching logs", func() { + converter := &vat_flux_mocks.MockVatFlux{} + fetcher := &mocks.MockLogFetcher{} + fetcher.SetFetchedLogs([]types.Log{test_data.VatFluxLog}) + repository := &vat_flux_mocks.MockVatFluxRepository{} + repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) + transformer := vat_flux.VatFluxTransformer{ + Fetcher: fetcher, + Converter: converter, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + Expect(converter.PassedLogs).To(Equal([]types.Log{test_data.VatFluxLog})) + }) + + It("returns error if converter returns error", func() { + converter := &vat_flux_mocks.MockVatFlux{} + converter.SetConverterError(fakes.FakeError) + fetcher := &mocks.MockLogFetcher{} + fetcher.SetFetchedLogs([]types.Log{test_data.VatFluxLog}) + repository := &vat_flux_mocks.MockVatFluxRepository{} + repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) + transformer := vat_flux.VatFluxTransformer{ + Fetcher: fetcher, + Converter: converter, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + + It("persists vat flux model", func() { + converter := &vat_flux_mocks.MockVatFlux{} + fetcher := &mocks.MockLogFetcher{} + fetcher.SetFetchedLogs([]types.Log{test_data.VatFluxLog}) + repository := &vat_flux_mocks.MockVatFluxRepository{} + fakeHeader := core.Header{BlockNumber: 1, Id: 2} + repository.SetMissingHeaders([]core.Header{fakeHeader}) + transformer := vat_flux.VatFluxTransformer{ + Fetcher: fetcher, + Converter: converter, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id)) + Expect(repository.PassedModels).To(Equal([]vat_flux.VatFluxModel{test_data.VatFluxModel})) + }) + + It("returns error if repository returns error for create", func() { + converter := &vat_flux_mocks.MockVatFlux{} + fetcher := &mocks.MockLogFetcher{} + fetcher.SetFetchedLogs([]types.Log{test_data.VatFluxLog}) + repository := &vat_flux_mocks.MockVatFluxRepository{} + repository.SetMissingHeaders([]core.Header{{BlockNumber: 1, Id: 2}}) + repository.SetCreateError(fakes.FakeError) + transformer := vat_flux.VatFluxTransformer{ + Fetcher: fetcher, + Converter: converter, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) +}) diff --git a/pkg/transformers/vat_flux/vat_flux_suite_test.go b/pkg/transformers/vat_flux/vat_flux_suite_test.go new file mode 100644 index 00000000..9a099bf2 --- /dev/null +++ b/pkg/transformers/vat_flux/vat_flux_suite_test.go @@ -0,0 +1,19 @@ +package vat_flux_test + +import ( + "io/ioutil" + "log" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestVatFlux(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "VatFlux Suite") +} + +var _ = BeforeSuite(func() { + log.SetOutput(ioutil.Discard) +}) diff --git a/pkg/transformers/vat_grab/tranformer.go b/pkg/transformers/vat_grab/tranformer.go index 47dc2820..6dd40aaf 100644 --- a/pkg/transformers/vat_grab/tranformer.go +++ b/pkg/transformers/vat_grab/tranformer.go @@ -36,7 +36,7 @@ func (transformer VatGrabTransformer) Execute() error { if err != nil { return err } - log.Printf("Fetching vat init event logs for %d headers \n", len(missingHeaders)) + log.Printf("Fetching vat grab event logs for %d headers \n", len(missingHeaders)) for _, header := range missingHeaders { topics := [][]common.Hash{{common.HexToHash(shared.VatGrabSignature)}} matchingLogs, err := transformer.Fetcher.FetchLogs(VatGrabConfig.ContractAddresses, topics, header.BlockNumber) diff --git a/test_config/test_config.go b/test_config/test_config.go index a0c03d9e..8d0403bd 100644 --- a/test_config/test_config.go +++ b/test_config/test_config.go @@ -103,6 +103,7 @@ func CleanTestDB(db *postgres.DB) { db.MustExec("DELETE FROM maker.vat_fold") db.MustExec("DELETE FROM maker.vat_toll") db.MustExec("DELETE FROM maker.vat_tune") + db.MustExec("DELETE FROM maker.vat_flux") db.MustExec("DELETE FROM receipts") db.MustExec("DELETE FROM transactions") db.MustExec("DELETE FROM watched_contracts")