From 2a518ad5181c154eabd1e44c9eb63ff817aa071f Mon Sep 17 00:00:00 2001 From: Rob Mulholand Date: Wed, 14 Nov 2018 16:33:29 -0600 Subject: [PATCH] (VDB-254) Convert numerical values for Pit.file ilk - convert data to ray when what == "spot" and wad when what == "line" --- .../integration_tests/pit_file_ilk.go | 50 +++++++++++++++++-- pkg/transformers/pit_file/ilk/converter.go | 21 ++++++-- .../pit_file/ilk/converter_test.go | 46 +++++++++++++++-- .../pit_file/ilk/repository_test.go | 18 +++---- pkg/transformers/test_data/pit_file.go | 45 +++++++++++++---- 5 files changed, 150 insertions(+), 30 deletions(-) diff --git a/pkg/transformers/integration_tests/pit_file_ilk.go b/pkg/transformers/integration_tests/pit_file_ilk.go index 7a255141..cdff6ee7 100644 --- a/pkg/transformers/integration_tests/pit_file_ilk.go +++ b/pkg/transformers/integration_tests/pit_file_ilk.go @@ -17,15 +17,15 @@ package integration_tests import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" - "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/ilk" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/test_config" ) var _ = Describe("PitFileIlk LogNoteTransformer", func() { - It("fetches and transforms a PitFileIlk event from Kovan chain", func() { + It("fetches and transforms a Pit.file ilk 'spot' event from Kovan", func() { blockNumber := int64(9103223) config := ilk.IlkFileConfig config.StartingBlockNumber = blockNumber @@ -59,6 +59,48 @@ var _ = Describe("PitFileIlk LogNoteTransformer", func() { Expect(len(dbResult)).To(Equal(1)) Expect(dbResult[0].Ilk).To(Equal("ETH")) Expect(dbResult[0].What).To(Equal("spot")) - Expect(dbResult[0].Data).To(Equal("139840000000000000000000000000")) + Expect(dbResult[0].Data).To(Equal("139.840000000000003410605131648")) + }) + + It("fetches and transforms a Pit.file ilk 'line' event from Kovan", func() { + blockNumber := int64(8762253) + config := ilk.IlkFileConfig + config.StartingBlockNumber = blockNumber + config.EndingBlockNumber = blockNumber + + rpcClient, ethClient, err := getClients(ipc) + Expect(err).NotTo(HaveOccurred()) + blockChain, err := getBlockChain(rpcClient, ethClient) + Expect(err).NotTo(HaveOccurred()) + + db := test_config.NewTestDB(blockChain.Node()) + test_config.CleanTestDB(db) + + err = persistHeader(db, blockNumber, blockChain) + Expect(err).NotTo(HaveOccurred()) + + initializer := factories.LogNoteTransformer{ + Config: config, + Fetcher: &shared.Fetcher{}, + Converter: &ilk.PitFileIlkConverter{}, + Repository: &ilk.PitFileIlkRepository{}, + } + transformer := initializer.NewLogNoteTransformer(db, blockChain) + err = transformer.Execute() + Expect(err).NotTo(HaveOccurred()) + + var dbResult []ilk.PitFileIlkModel + err = db.Select(&dbResult, `SELECT ilk, what, data from maker.pit_file_ilk`) + Expect(err).NotTo(HaveOccurred()) + + Expect(len(dbResult)).To(Equal(2)) + var pitFileIlkLineModel ilk.PitFileIlkModel + for _, result := range dbResult { + if result.What == "line" { + pitFileIlkLineModel = result + } + } + Expect(pitFileIlkLineModel.Ilk).To(Equal("REP")) + Expect(pitFileIlkLineModel.Data).To(Equal("2000000.000000000000000000")) }) }) diff --git a/pkg/transformers/pit_file/ilk/converter.go b/pkg/transformers/pit_file/ilk/converter.go index a99f2a14..6001786b 100644 --- a/pkg/transformers/pit_file/ilk/converter.go +++ b/pkg/transformers/pit_file/ilk/converter.go @@ -22,6 +22,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" ) @@ -36,8 +37,11 @@ func (PitFileIlkConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) } ilk := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00")) what := string(bytes.Trim(ethLog.Topics[3].Bytes(), "\x00")) - riskBytes := ethLog.Data[len(ethLog.Data)-constants.DataItemLength:] - risk := big.NewInt(0).SetBytes(riskBytes).String() + dataBytes := ethLog.Data[len(ethLog.Data)-constants.DataItemLength:] + data, err := getData(dataBytes, what) + if err != nil { + return nil, err + } raw, err := json.Marshal(ethLog) if err != nil { @@ -46,7 +50,7 @@ func (PitFileIlkConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) model := PitFileIlkModel{ Ilk: ilk, What: what, - Data: risk, + Data: data, LogIndex: ethLog.Index, TransactionIndex: ethLog.TxIndex, Raw: raw, @@ -56,6 +60,17 @@ func (PitFileIlkConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) return models, nil } +func getData(dataBytes []byte, what string) (string, error) { + n := big.NewInt(0).SetBytes(dataBytes).String() + if what == "spot" { + return shared.ConvertToRay(n), nil + } else if what == "line" { + return shared.ConvertToWad(n), nil + } else { + return "", errors.New("unexpected payload for 'what'") + } +} + func verifyLog(log types.Log) error { if len(log.Topics) < 4 { return errors.New("log missing topics") diff --git a/pkg/transformers/pit_file/ilk/converter_test.go b/pkg/transformers/pit_file/ilk/converter_test.go index 8ddfa4e4..bdaf4cc6 100644 --- a/pkg/transformers/pit_file/ilk/converter_test.go +++ b/pkg/transformers/pit_file/ilk/converter_test.go @@ -47,13 +47,49 @@ var _ = Describe("Pit file ilk converter", func() { Expect(err).To(HaveOccurred()) }) - It("converts a log to an model", func() { + It("returns error if 'what' field is unknown", func() { + log := types.Log{ + Address: test_data.EthPitFileIlkLineLog.Address, + Topics: []common.Hash{ + test_data.EthPitFileIlkLineLog.Topics[0], + test_data.EthPitFileIlkLineLog.Topics[1], + test_data.EthPitFileIlkLineLog.Topics[2], + common.HexToHash("0x1111111100000000000000000000000000000000000000000000000000000000"), + }, + Data: test_data.EthPitFileIlkLineLog.Data, + BlockNumber: test_data.EthPitFileIlkLineLog.BlockNumber, + TxHash: test_data.EthPitFileIlkLineLog.TxHash, + TxIndex: test_data.EthPitFileIlkLineLog.TxIndex, + BlockHash: test_data.EthPitFileIlkLineLog.BlockHash, + Index: test_data.EthPitFileIlkLineLog.Index, + } converter := ilk.PitFileIlkConverter{} - models, err := converter.ToModels([]types.Log{test_data.EthPitFileIlkLog}) + _, err := converter.ToModels([]types.Log{log}) - Expect(err).NotTo(HaveOccurred()) - Expect(len(models)).To(Equal(1)) - Expect(models[0].(ilk.PitFileIlkModel)).To(Equal(test_data.PitFileIlkModel)) + Expect(err).To(HaveOccurred()) }) + + Describe("when log is valid", func() { + It("converts to model with data converted to ray when what is 'spot'", func() { + converter := ilk.PitFileIlkConverter{} + + models, err := converter.ToModels([]types.Log{test_data.EthPitFileIlkSpotLog}) + + Expect(err).NotTo(HaveOccurred()) + Expect(len(models)).To(Equal(1)) + Expect(models[0].(ilk.PitFileIlkModel)).To(Equal(test_data.PitFileIlkSpotModel)) + }) + + It("converts to model with data converted to wad when what is 'line'", func() { + converter := ilk.PitFileIlkConverter{} + + models, err := converter.ToModels([]types.Log{test_data.EthPitFileIlkLineLog}) + + Expect(err).NotTo(HaveOccurred()) + Expect(len(models)).To(Equal(1)) + Expect(models[0].(ilk.PitFileIlkModel)).To(Equal(test_data.PitFileIlkLineModel)) + }) + }) + }) diff --git a/pkg/transformers/pit_file/ilk/repository_test.go b/pkg/transformers/pit_file/ilk/repository_test.go index b99f6ad5..22214f1b 100644 --- a/pkg/transformers/pit_file/ilk/repository_test.go +++ b/pkg/transformers/pit_file/ilk/repository_test.go @@ -44,12 +44,12 @@ var _ = Describe("Pit file ilk repository", func() { }) Describe("Create", func() { - modelWithDifferentLogIdx := test_data.PitFileIlkModel + modelWithDifferentLogIdx := test_data.PitFileIlkSpotModel modelWithDifferentLogIdx.LogIndex = modelWithDifferentLogIdx.LogIndex + 1 inputs := shared_behaviors.CreateBehaviorInputs{ CheckedHeaderColumnName: constants.PitFileIlkChecked, LogEventTableName: "maker.pit_file_ilk", - TestModel: test_data.PitFileIlkModel, + TestModel: test_data.PitFileIlkSpotModel, ModelWithDifferentLogIdx: modelWithDifferentLogIdx, Repository: &pitFileIlkRepository, } @@ -59,18 +59,18 @@ var _ = Describe("Pit file ilk repository", func() { It("adds a pit file ilk event", func() { headerID, err := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) Expect(err).NotTo(HaveOccurred()) - err = pitFileIlkRepository.Create(headerID, []interface{}{test_data.PitFileIlkModel}) + err = pitFileIlkRepository.Create(headerID, []interface{}{test_data.PitFileIlkSpotModel}) Expect(err).NotTo(HaveOccurred()) var dbPitFile ilk.PitFileIlkModel err = db.Get(&dbPitFile, `SELECT ilk, what, data, log_idx, tx_idx, raw_log FROM maker.pit_file_ilk WHERE header_id = $1`, headerID) Expect(err).NotTo(HaveOccurred()) - Expect(dbPitFile.Ilk).To(Equal(test_data.PitFileIlkModel.Ilk)) - Expect(dbPitFile.What).To(Equal(test_data.PitFileIlkModel.What)) - Expect(dbPitFile.Data).To(Equal(test_data.PitFileIlkModel.Data)) - Expect(dbPitFile.LogIndex).To(Equal(test_data.PitFileIlkModel.LogIndex)) - Expect(dbPitFile.TransactionIndex).To(Equal(test_data.PitFileIlkModel.TransactionIndex)) - Expect(dbPitFile.Raw).To(MatchJSON(test_data.PitFileIlkModel.Raw)) + Expect(dbPitFile.Ilk).To(Equal(test_data.PitFileIlkSpotModel.Ilk)) + Expect(dbPitFile.What).To(Equal(test_data.PitFileIlkSpotModel.What)) + Expect(dbPitFile.Data).To(Equal(test_data.PitFileIlkSpotModel.Data)) + Expect(dbPitFile.LogIndex).To(Equal(test_data.PitFileIlkSpotModel.LogIndex)) + Expect(dbPitFile.TransactionIndex).To(Equal(test_data.PitFileIlkSpotModel.TransactionIndex)) + Expect(dbPitFile.Raw).To(MatchJSON(test_data.PitFileIlkSpotModel.Raw)) }) }) diff --git a/pkg/transformers/test_data/pit_file.go b/pkg/transformers/test_data/pit_file.go index 2d40d99e..e1d53526 100644 --- a/pkg/transformers/test_data/pit_file.go +++ b/pkg/transformers/test_data/pit_file.go @@ -55,15 +55,42 @@ var PitFileDebtCeilingModel = debt_ceiling.PitFileDebtCeilingModel{ Raw: rawPitFileDebtCeilingLog, } -var EthPitFileIlkLog = types.Log{ +var EthPitFileIlkLineLog = types.Log{ Address: common.HexToAddress(constants.PitContractAddress), Topics: []common.Hash{ common.HexToHash("0x1a0b287e00000000000000000000000000000000000000000000000000000000"), - common.HexToHash("0x0000000000000000000000000f243e26db94b5426032e6dfa6007802dea2a614"), + common.HexToHash("0x00000000000000000000000064d922894153be9eef7b7218dc565d1d0ce2a092"), + common.HexToHash("0x66616b6520696c6b000000000000000000000000000000000000000000000000"), + common.HexToHash("0x6c696e6500000000000000000000000000000000000000000000000000000000"), + }, + Data: hexutil.MustDecode("0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000641a0b287e66616b6520696c6b0000000000000000000000000000000000000000000000006c696e6500000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e8d4a51000"), + BlockNumber: 12, + TxHash: common.HexToHash("0x2e27c962a697d4f7ec5d3206d0c058bd510f7593a711f082e55f3b62d44d8dd9"), + TxIndex: 112, + BlockHash: fakes.FakeHash, + Index: 15, + Removed: false, +} + +var rawPitFileIlkLineLog, _ = json.Marshal(EthPitFileIlkLineLog) +var PitFileIlkLineModel = ilk2.PitFileIlkModel{ + Ilk: "fake ilk", + What: "line", + Data: "0.000001000000000000", + LogIndex: EthPitFileIlkLineLog.Index, + TransactionIndex: EthPitFileIlkLineLog.TxIndex, + Raw: rawPitFileIlkLineLog, +} + +var EthPitFileIlkSpotLog = types.Log{ + Address: common.HexToAddress(constants.PitContractAddress), + Topics: []common.Hash{ + common.HexToHash("0x1a0b287e00000000000000000000000000000000000000000000000000000000"), + common.HexToHash("0x00000000000000000000000064d922894153be9eef7b7218dc565d1d0ce2a092"), common.HexToHash("0x66616b6520696c6b000000000000000000000000000000000000000000000000"), common.HexToHash("0x73706f7400000000000000000000000000000000000000000000000000000000"), }, - Data: hexutil.MustDecode("0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000641a0b287e66616b6520696c6b00000000000000000000000000000000000000000000000073706f7400000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007b"), + Data: hexutil.MustDecode("0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000641a0b287e66616b6520696c6b00000000000000000000000000000000000000000000000073706f7400000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e8d4a51000"), BlockNumber: 11, TxHash: common.HexToHash("0x1ba8125f60fa045c85b35df3983bee37db8627fbc32e3442a5cf17c85bb83f09"), TxIndex: 111, @@ -72,12 +99,12 @@ var EthPitFileIlkLog = types.Log{ Removed: false, } -var rawPitFileIlkLog, _ = json.Marshal(EthPitFileIlkLog) -var PitFileIlkModel = ilk2.PitFileIlkModel{ +var rawPitFileIlkSpotLog, _ = json.Marshal(EthPitFileIlkSpotLog) +var PitFileIlkSpotModel = ilk2.PitFileIlkModel{ Ilk: "fake ilk", What: "spot", - Data: big.NewInt(123).String(), - LogIndex: EthPitFileIlkLog.Index, - TransactionIndex: EthPitFileIlkLog.TxIndex, - Raw: rawPitFileIlkLog, + Data: "0.000000000000001000000000000", + LogIndex: EthPitFileIlkSpotLog.Index, + TransactionIndex: EthPitFileIlkSpotLog.TxIndex, + Raw: rawPitFileIlkSpotLog, }