From c1e10f09fbce2f1a4e217cf8b4afe3e2735043b0 Mon Sep 17 00:00:00 2001 From: Elizabeth Date: Fri, 9 Nov 2018 14:53:20 -0600 Subject: [PATCH] Vdb 67 convert numbers to ray and wad (#105) * Add method to convert values to ray or wad units * Convert data to ray or wad for cat_file_chop_lump * Use shared convert functions in price feed conversion * Pull common ray/wad values into vars * Fix after rebase with staging --- .../cat_file/chop_lump/converter.go | 19 +++++++- .../cat_file/chop_lump/converter_test.go | 25 ++++++++--- .../cat_file/chop_lump/repository_test.go | 38 +++++++++++----- .../integration_tests/cat_file.go | 4 +- .../integration_tests/price_feeds.go | 6 +-- pkg/transformers/price_feeds/converter.go | 3 +- pkg/transformers/shared/utilities.go | 31 +++++++++++++ pkg/transformers/shared/utilities_test.go | 16 +++++++ pkg/transformers/test_data/cat_file.go | 44 ++++++++++++++----- pkg/transformers/test_data/price_feed.go | 2 +- 10 files changed, 153 insertions(+), 35 deletions(-) diff --git a/pkg/transformers/cat_file/chop_lump/converter.go b/pkg/transformers/cat_file/chop_lump/converter.go index cb999727..bb60d5c0 100644 --- a/pkg/transformers/cat_file/chop_lump/converter.go +++ b/pkg/transformers/cat_file/chop_lump/converter.go @@ -19,10 +19,16 @@ import ( "encoding/json" "errors" "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "math/big" ) +var ( + chop = "chop" + lump = "lump" +) + type CatFileChopLumpConverter struct{} func (CatFileChopLumpConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) { @@ -44,7 +50,7 @@ func (CatFileChopLumpConverter) ToModels(ethLogs []types.Log) ([]interface{}, er result := CatFileChopLumpModel{ Ilk: ilk, What: what, - Data: data, + Data: convertData(what, data), TransactionIndex: ethLog.TxIndex, LogIndex: ethLog.Index, Raw: raw, @@ -54,6 +60,17 @@ func (CatFileChopLumpConverter) ToModels(ethLogs []types.Log) ([]interface{}, er return results, nil } +func convertData(what, data string) string { + var convertedData string + if what == chop { + convertedData = shared.ConvertToRay(data) + } else if what == lump { + convertedData = shared.ConvertToWad(data) + } + + return convertedData +} + func verifyLog(log types.Log) error { if len(log.Topics) < 4 { return errors.New("log missing topics") diff --git a/pkg/transformers/cat_file/chop_lump/converter_test.go b/pkg/transformers/cat_file/chop_lump/converter_test.go index 00bdd97b..81858d14 100644 --- a/pkg/transformers/cat_file/chop_lump/converter_test.go +++ b/pkg/transformers/cat_file/chop_lump/converter_test.go @@ -31,6 +31,24 @@ var _ = Describe("Cat file chop lump converter", func() { converter = chop_lump.CatFileChopLumpConverter{} }) + Context("chop events", func() { + It("converts a chop log to a model", func() { + models, err := converter.ToModels([]types.Log{test_data.EthCatFileChopLog}) + + Expect(err).NotTo(HaveOccurred()) + Expect(models).To(Equal([]interface{}{test_data.CatFileChopModel})) + }) + }) + + Context("lump events", func() { + It("converts a lump log to a model", func() { + models, err := converter.ToModels([]types.Log{test_data.EthCatFileLumpLog}) + + Expect(err).NotTo(HaveOccurred()) + Expect(models).To(Equal([]interface{}{test_data.CatFileLumpModel})) + }) + }) + It("returns err if log is missing topics", func() { badLog := types.Log{ Data: []byte{1, 1, 1, 1, 1}, @@ -48,11 +66,4 @@ var _ = Describe("Cat file chop lump converter", func() { _, err := converter.ToModels([]types.Log{badLog}) Expect(err).To(HaveOccurred()) }) - - It("converts a log to an model", func() { - models, err := converter.ToModels([]types.Log{test_data.EthCatFileChopLumpLog}) - - Expect(err).NotTo(HaveOccurred()) - Expect(models).To(Equal([]interface{}{test_data.CatFileChopLumpModel})) - }) }) diff --git a/pkg/transformers/cat_file/chop_lump/repository_test.go b/pkg/transformers/cat_file/chop_lump/repository_test.go index a80079f3..ec10826a 100644 --- a/pkg/transformers/cat_file/chop_lump/repository_test.go +++ b/pkg/transformers/cat_file/chop_lump/repository_test.go @@ -41,34 +41,52 @@ var _ = Describe("Cat file chop lump repository", func() { }) Describe("Create", func() { - modelWithDifferentLogIdx := test_data.CatFileChopLumpModel + modelWithDifferentLogIdx := test_data.CatFileChopModel modelWithDifferentLogIdx.LogIndex++ inputs := shared_behaviors.CreateBehaviorInputs{ CheckedHeaderColumnName: constants.CatFileChopLumpChecked, LogEventTableName: "maker.cat_file_chop_lump", - TestModel: test_data.CatFileChopLumpModel, + TestModel: test_data.CatFileChopModel, ModelWithDifferentLogIdx: modelWithDifferentLogIdx, Repository: &catFileRepository, } shared_behaviors.SharedRepositoryCreateBehaviors(&inputs) - It("adds a cat file chop lump event", func() { + It("adds a cat file chop event", func() { headerRepository := repositories.NewHeaderRepository(db) headerID, err := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) Expect(err).NotTo(HaveOccurred()) - err = catFileRepository.Create(headerID, []interface{}{test_data.CatFileChopLumpModel}) + err = catFileRepository.Create(headerID, []interface{}{test_data.CatFileChopModel}) Expect(err).NotTo(HaveOccurred()) var dbResult chop_lump.CatFileChopLumpModel err = db.Get(&dbResult, `SELECT ilk, what, data, tx_idx, log_idx, raw_log FROM maker.cat_file_chop_lump WHERE header_id = $1`, headerID) Expect(err).NotTo(HaveOccurred()) - Expect(dbResult.Ilk).To(Equal(test_data.CatFileChopLumpModel.Ilk)) - Expect(dbResult.What).To(Equal(test_data.CatFileChopLumpModel.What)) - Expect(dbResult.Data).To(Equal(test_data.CatFileChopLumpModel.Data)) - Expect(dbResult.TransactionIndex).To(Equal(test_data.CatFileChopLumpModel.TransactionIndex)) - Expect(dbResult.LogIndex).To(Equal(test_data.CatFileChopLumpModel.LogIndex)) - Expect(dbResult.Raw).To(MatchJSON(test_data.CatFileChopLumpModel.Raw)) + Expect(dbResult.Ilk).To(Equal(test_data.CatFileChopModel.Ilk)) + Expect(dbResult.What).To(Equal(test_data.CatFileChopModel.What)) + Expect(dbResult.Data).To(Equal(test_data.CatFileChopModel.Data)) + Expect(dbResult.TransactionIndex).To(Equal(test_data.CatFileChopModel.TransactionIndex)) + Expect(dbResult.LogIndex).To(Equal(test_data.CatFileChopModel.LogIndex)) + Expect(dbResult.Raw).To(MatchJSON(test_data.CatFileChopModel.Raw)) + }) + + It("adds a cat file lump event", func() { + headerRepository := repositories.NewHeaderRepository(db) + headerID, err := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) + Expect(err).NotTo(HaveOccurred()) + err = catFileRepository.Create(headerID, []interface{}{test_data.CatFileLumpModel}) + + Expect(err).NotTo(HaveOccurred()) + var dbResult chop_lump.CatFileChopLumpModel + err = db.Get(&dbResult, `SELECT ilk, what, data, tx_idx, log_idx, raw_log FROM maker.cat_file_chop_lump WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(dbResult.Ilk).To(Equal(test_data.CatFileLumpModel.Ilk)) + Expect(dbResult.What).To(Equal(test_data.CatFileLumpModel.What)) + Expect(dbResult.Data).To(Equal(test_data.CatFileLumpModel.Data)) + Expect(dbResult.TransactionIndex).To(Equal(test_data.CatFileLumpModel.TransactionIndex)) + Expect(dbResult.LogIndex).To(Equal(test_data.CatFileLumpModel.LogIndex)) + Expect(dbResult.Raw).To(MatchJSON(test_data.CatFileLumpModel.Raw)) }) }) diff --git a/pkg/transformers/integration_tests/cat_file.go b/pkg/transformers/integration_tests/cat_file.go index 26b03e80..35d24015 100644 --- a/pkg/transformers/integration_tests/cat_file.go +++ b/pkg/transformers/integration_tests/cat_file.go @@ -80,12 +80,12 @@ var _ = Describe("Cat File transformer", func() { Expect(dbResult[0].Ilk).To(Equal("REP")) Expect(dbResult[0].What).To(Equal("lump")) - Expect(dbResult[0].Data).To(Equal("10000000000000000000000")) + Expect(dbResult[0].Data).To(Equal("10000.000000000000000000")) Expect(dbResult[0].LogIndex).To(Equal(uint(3))) Expect(dbResult[1].Ilk).To(Equal("REP")) Expect(dbResult[1].What).To(Equal("chop")) - Expect(dbResult[1].Data).To(Equal("1000000000000000000000000000")) + Expect(dbResult[1].Data).To(Equal("1.000000000000000000000000000")) Expect(dbResult[1].LogIndex).To(Equal(uint(4))) }) diff --git a/pkg/transformers/integration_tests/price_feeds.go b/pkg/transformers/integration_tests/price_feeds.go index 75f5e90c..da5e9a5b 100644 --- a/pkg/transformers/integration_tests/price_feeds.go +++ b/pkg/transformers/integration_tests/price_feeds.go @@ -65,7 +65,7 @@ var _ = Describe("Price feeds transformer", func() { var model price_feeds.PriceFeedModel err = db.Get(&model, `SELECT block_number, medianizer_address, usd_value, tx_idx, raw_log FROM maker.price_feeds WHERE block_number = $1`, config.StartingBlockNumber) Expect(err).NotTo(HaveOccurred()) - Expect(model.UsdValue).To(Equal("207.314891143")) + Expect(model.UsdValue).To(Equal("207.314891143000011198")) Expect(model.MedianizerAddress).To(Equal(config.ContractAddresses[0])) }) @@ -92,7 +92,7 @@ var _ = Describe("Price feeds transformer", func() { var model price_feeds.PriceFeedModel err = db.Get(&model, `SELECT block_number, medianizer_address, usd_value, tx_idx, raw_log FROM maker.price_feeds WHERE block_number = $1`, config.StartingBlockNumber) Expect(err).NotTo(HaveOccurred()) - Expect(model.UsdValue).To(Equal("391.803979212")) + Expect(model.UsdValue).To(Equal("391.803979212000001553")) Expect(model.MedianizerAddress).To(Equal(config.ContractAddresses[0])) }) @@ -119,7 +119,7 @@ var _ = Describe("Price feeds transformer", func() { var model price_feeds.PriceFeedModel err = db.Get(&model, `SELECT block_number, medianizer_address, usd_value, tx_idx, raw_log FROM maker.price_feeds WHERE block_number = $1`, config.StartingBlockNumber) Expect(err).NotTo(HaveOccurred()) - Expect(model.UsdValue).To(Equal("12.8169284827")) + Expect(model.UsdValue).To(Equal("12.816928482699999847")) Expect(model.MedianizerAddress).To(Equal(config.ContractAddresses[0])) }) }) diff --git a/pkg/transformers/price_feeds/converter.go b/pkg/transformers/price_feeds/converter.go index 373d8eca..cd14573a 100644 --- a/pkg/transformers/price_feeds/converter.go +++ b/pkg/transformers/price_feeds/converter.go @@ -18,6 +18,7 @@ import ( "encoding/json" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" ) type PriceFeedConverter struct{} @@ -32,7 +33,7 @@ func (converter PriceFeedConverter) ToModels(ethLogs []types.Log) ([]interface{} model := PriceFeedModel{ BlockNumber: log.BlockNumber, MedianizerAddress: log.Address.String(), - UsdValue: Convert("wad", hexutil.Encode(log.Data), 15), + UsdValue: shared.ConvertToWad(hexutil.Encode(log.Data)), LogIndex: log.Index, TransactionIndex: log.TxIndex, Raw: raw, diff --git a/pkg/transformers/shared/utilities.go b/pkg/transformers/shared/utilities.go index ede35a28..073f8783 100644 --- a/pkg/transformers/shared/utilities.go +++ b/pkg/transformers/shared/utilities.go @@ -19,6 +19,15 @@ import ( "math/big" ) +var ( + rayBase = big.NewFloat(1e27) + wadBase = big.NewFloat(1e18) + rayPrecision = 27 + wadPrecision = 18 + ray = "ray" + wad = "wad" +) + func BigIntToInt64(value *big.Int) int64 { if value == nil { return int64(0) @@ -47,3 +56,25 @@ func GetDataBytesAtIndex(n int, logData []byte) []byte { } return []byte{} } + +func ConvertToRay(value string) string { + return convert(ray, value, rayPrecision) +} + +func ConvertToWad(value string) string { + return convert(wad, value, wadPrecision) +} + +func convert(conversion string, value string, precision int) string { + result := big.NewFloat(0.0) + bigFloat := big.NewFloat(0.0) + bigFloat.SetString(value) + + switch conversion { + case ray: + result.Quo(bigFloat, rayBase) + case wad: + result.Quo(bigFloat, wadBase) + } + return result.Text('f', precision) +} diff --git a/pkg/transformers/shared/utilities_test.go b/pkg/transformers/shared/utilities_test.go index 03e5ddc0..905bc1be 100644 --- a/pkg/transformers/shared/utilities_test.go +++ b/pkg/transformers/shared/utilities_test.go @@ -45,5 +45,21 @@ var _ = Describe("Shared utilities", func() { Expect(expected[:]).To(Equal(actual)) }) + + It("converts values to rays", func() { + rayOne := shared.ConvertToRay("123456789012345678901234567890") + Expect(rayOne).To(Equal("123.456789012345680589533003513")) + + rayTwo := shared.ConvertToRay("1234567890123456790123567890") + Expect(rayTwo).To(Equal("1.234567890123456912476740399")) + }) + + It("converts values to wads", func() { + wadOne := shared.ConvertToWad("12345678901234567890123") + Expect(wadOne).To(Equal("12345.678901234567092615")) + + wadTwo := shared.ConvertToWad("1234567890123456789") + Expect(wadTwo).To(Equal("1.234567890123456690")) + }) }) }) diff --git a/pkg/transformers/test_data/cat_file.go b/pkg/transformers/test_data/cat_file.go index 2884cdcb..4e12c4d2 100644 --- a/pkg/transformers/test_data/cat_file.go +++ b/pkg/transformers/test_data/cat_file.go @@ -23,10 +23,9 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/transformers/cat_file/flip" "github.com/vulcanize/vulcanizedb/pkg/transformers/cat_file/pit_vow" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" - "math/big" ) -var EthCatFileChopLumpLog = types.Log{ +var EthCatFileChopLog = types.Log{ Address: common.HexToAddress(constants.CatContractAddress), Topics: []common.Hash{ common.HexToHash("0x1a0b287e00000000000000000000000000000000000000000000000000000000"), @@ -34,7 +33,7 @@ var EthCatFileChopLumpLog = types.Log{ common.HexToHash("0x66616b6520696c6b000000000000000000000000000000000000000000000000"), common.HexToHash("0x63686f7000000000000000000000000000000000000000000000000000000000"), }, - Data: hexutil.MustDecode("0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000641a0b287e66616b6520696c6b00000000000000000000000000000000000000000000000063686f700000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000075bcd15"), + Data: hexutil.MustDecode("0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000641a0b287e66616b6520696c6b00000000000000000000000000000000000000000000000063686f700000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000018EE90FF6C373E0EE4E3F0AD2"), BlockNumber: 110, TxHash: common.HexToHash("0xe32dfe6afd7ea28475569756fc30f0eea6ad4cfd32f67436ff1d1c805e4382df"), TxIndex: 13, @@ -42,15 +41,40 @@ var EthCatFileChopLumpLog = types.Log{ Index: 1, Removed: false, } - -var rawCatFileChopLumpLog, _ = json.Marshal(EthCatFileChopLumpLog) -var CatFileChopLumpModel = chop_lump.CatFileChopLumpModel{ +var rawCatFileChopLog, _ = json.Marshal(EthCatFileChopLog) +var CatFileChopModel = chop_lump.CatFileChopLumpModel{ Ilk: "fake ilk", What: "chop", - Data: big.NewInt(123456789).String(), - TransactionIndex: EthCatFileChopLumpLog.TxIndex, - LogIndex: EthCatFileChopLumpLog.Index, - Raw: rawCatFileChopLumpLog, + Data: "123.456789012345680589533003513", + TransactionIndex: EthCatFileChopLog.TxIndex, + LogIndex: EthCatFileChopLog.Index, + Raw: rawCatFileChopLog, +} + +var EthCatFileLumpLog = types.Log{ + Address: common.HexToAddress(constants.CatContractAddress), + Topics: []common.Hash{ + common.HexToHash("0x1a0b287e00000000000000000000000000000000000000000000000000000000"), + common.HexToHash("0x00000000000000000000000064d922894153be9eef7b7218dc565d1d0ce2a092"), + common.HexToHash("0x66616b6520696c6b000000000000000000000000000000000000000000000000"), + common.HexToHash("0x6c756d7000000000000000000000000000000000000000000000000000000000"), + }, + Data: hexutil.MustDecode("0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000641a0b287e66616b6520696c6b00000000000000000000000000000000000000000000000063686f700000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000029D42B64E76714244CB"), + BlockNumber: 110, + TxHash: common.HexToHash("0xe32dfe6afd7ea28475569756fc30f0eea6ad4cfd32f67436ff1d1c805e4382df"), + TxIndex: 15, + BlockHash: common.HexToHash("0x2764998a4e048d4c4ba45ea40fd5efaa8e2d4f1dd2b15425a6c6a3dea7f1064a"), + Index: 3, + Removed: false, +} +var rawCatFileLumpLog, _ = json.Marshal(EthCatFileLumpLog) +var CatFileLumpModel = chop_lump.CatFileChopLumpModel{ + Ilk: "fake ilk", + What: "lump", + Data: "12345.678901234567092615", + TransactionIndex: EthCatFileLumpLog.TxIndex, + LogIndex: EthCatFileLumpLog.Index, + Raw: rawCatFileLumpLog, } var EthCatFileFlipLog = types.Log{ diff --git a/pkg/transformers/test_data/price_feed.go b/pkg/transformers/test_data/price_feed.go index a0c1df94..e4ce5a5e 100644 --- a/pkg/transformers/test_data/price_feed.go +++ b/pkg/transformers/test_data/price_feed.go @@ -45,7 +45,7 @@ var rawPriceFeedLog, _ = json.Marshal(EthPriceFeedLog) var PriceFeedModel = price_feeds.PriceFeedModel{ BlockNumber: blockNumber, MedianizerAddress: EthPriceFeedLog.Address.String(), - UsdValue: "378.6599388897", + UsdValue: "378.659938889700015352", LogIndex: EthPriceFeedLog.Index, TransactionIndex: EthPriceFeedLog.TxIndex, Raw: rawPriceFeedLog,