Vdb 67 convert numbers to ray and wad (#105)

* Add method to convert values to ray or wad units

* Convert data to ray or wad for cat_file_chop_lump

* Use shared convert functions in price feed conversion

* Pull common ray/wad values into vars

* Fix after rebase with staging

pkg/transformers/cat_file/chop_lump/converter.go

@@ -19,10 +19,16 @@ import (
 	"encoding/json"
 	"errors"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
 	"math/big"
 )
 
+var (
+	chop = "chop"
+	lump = "lump"
+)
+
 type CatFileChopLumpConverter struct{}
 
 func (CatFileChopLumpConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) {
@@ -44,7 +50,7 @@ func (CatFileChopLumpConverter) ToModels(ethLogs []types.Log) ([]interface{}, er
 		result := CatFileChopLumpModel{
 			Ilk:              ilk,
 			What:             what,
-			Data:             data,
+			Data:             convertData(what, data),
 			TransactionIndex: ethLog.TxIndex,
 			LogIndex:         ethLog.Index,
 			Raw:              raw,
@@ -54,6 +60,17 @@ func (CatFileChopLumpConverter) ToModels(ethLogs []types.Log) ([]interface{}, er
 	return results, nil
 }
 
+func convertData(what, data string) string {
+	var convertedData string
+	if what == chop {
+		convertedData = shared.ConvertToRay(data)
+	} else if what == lump {
+		convertedData = shared.ConvertToWad(data)
+	}
+
+	return convertedData
+}
+
 func verifyLog(log types.Log) error {
 	if len(log.Topics) < 4 {
 		return errors.New("log missing topics")

pkg/transformers/cat_file/chop_lump/converter_test.go

@@ -31,6 +31,24 @@ var _ = Describe("Cat file chop lump converter", func() {
 		converter = chop_lump.CatFileChopLumpConverter{}
 	})
 
+	Context("chop events", func() {
+		It("converts a chop log to a model", func() {
+			models, err := converter.ToModels([]types.Log{test_data.EthCatFileChopLog})
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(models).To(Equal([]interface{}{test_data.CatFileChopModel}))
+		})
+	})
+
+	Context("lump events", func() {
+		It("converts a lump log to a model", func() {
+			models, err := converter.ToModels([]types.Log{test_data.EthCatFileLumpLog})
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(models).To(Equal([]interface{}{test_data.CatFileLumpModel}))
+		})
+	})
+
 	It("returns err if log is missing topics", func() {
 		badLog := types.Log{
 			Data: []byte{1, 1, 1, 1, 1},
@@ -48,11 +66,4 @@ var _ = Describe("Cat file chop lump converter", func() {
 		_, err := converter.ToModels([]types.Log{badLog})
 		Expect(err).To(HaveOccurred())
 	})
-
-	It("converts a log to an model", func() {
-		models, err := converter.ToModels([]types.Log{test_data.EthCatFileChopLumpLog})
-
-		Expect(err).NotTo(HaveOccurred())
-		Expect(models).To(Equal([]interface{}{test_data.CatFileChopLumpModel}))
-	})
 })

pkg/transformers/cat_file/chop_lump/repository_test.go

@@ -41,34 +41,52 @@ var _ = Describe("Cat file chop lump repository", func() {
 	})
 
 	Describe("Create", func() {
-		modelWithDifferentLogIdx := test_data.CatFileChopLumpModel
+		modelWithDifferentLogIdx := test_data.CatFileChopModel
 		modelWithDifferentLogIdx.LogIndex++
 		inputs := shared_behaviors.CreateBehaviorInputs{
 			CheckedHeaderColumnName:  constants.CatFileChopLumpChecked,
 			LogEventTableName:        "maker.cat_file_chop_lump",
-			TestModel:                test_data.CatFileChopLumpModel,
+			TestModel:                test_data.CatFileChopModel,
 			ModelWithDifferentLogIdx: modelWithDifferentLogIdx,
 			Repository:               &catFileRepository,
 		}
 
 		shared_behaviors.SharedRepositoryCreateBehaviors(&inputs)
 
-		It("adds a cat file chop lump event", func() {
+		It("adds a cat file chop event", func() {
 			headerRepository := repositories.NewHeaderRepository(db)
 			headerID, err := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader)
 			Expect(err).NotTo(HaveOccurred())
-			err = catFileRepository.Create(headerID, []interface{}{test_data.CatFileChopLumpModel})
+			err = catFileRepository.Create(headerID, []interface{}{test_data.CatFileChopModel})
 
 			Expect(err).NotTo(HaveOccurred())
 			var dbResult chop_lump.CatFileChopLumpModel
 			err = db.Get(&dbResult, `SELECT ilk, what, data, tx_idx, log_idx, raw_log FROM maker.cat_file_chop_lump WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
-			Expect(dbResult.Ilk).To(Equal(test_data.CatFileChopLumpModel.Ilk))
+			Expect(dbResult.Ilk).To(Equal(test_data.CatFileChopModel.Ilk))
-			Expect(dbResult.What).To(Equal(test_data.CatFileChopLumpModel.What))
+			Expect(dbResult.What).To(Equal(test_data.CatFileChopModel.What))
-			Expect(dbResult.Data).To(Equal(test_data.CatFileChopLumpModel.Data))
+			Expect(dbResult.Data).To(Equal(test_data.CatFileChopModel.Data))
-			Expect(dbResult.TransactionIndex).To(Equal(test_data.CatFileChopLumpModel.TransactionIndex))
+			Expect(dbResult.TransactionIndex).To(Equal(test_data.CatFileChopModel.TransactionIndex))
-			Expect(dbResult.LogIndex).To(Equal(test_data.CatFileChopLumpModel.LogIndex))
+			Expect(dbResult.LogIndex).To(Equal(test_data.CatFileChopModel.LogIndex))
-			Expect(dbResult.Raw).To(MatchJSON(test_data.CatFileChopLumpModel.Raw))
+			Expect(dbResult.Raw).To(MatchJSON(test_data.CatFileChopModel.Raw))
+		})
+
+		It("adds a cat file lump event", func() {
+			headerRepository := repositories.NewHeaderRepository(db)
+			headerID, err := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader)
+			Expect(err).NotTo(HaveOccurred())
+			err = catFileRepository.Create(headerID, []interface{}{test_data.CatFileLumpModel})
+
+			Expect(err).NotTo(HaveOccurred())
+			var dbResult chop_lump.CatFileChopLumpModel
+			err = db.Get(&dbResult, `SELECT ilk, what, data, tx_idx, log_idx, raw_log FROM maker.cat_file_chop_lump WHERE header_id = $1`, headerID)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(dbResult.Ilk).To(Equal(test_data.CatFileLumpModel.Ilk))
+			Expect(dbResult.What).To(Equal(test_data.CatFileLumpModel.What))
+			Expect(dbResult.Data).To(Equal(test_data.CatFileLumpModel.Data))
+			Expect(dbResult.TransactionIndex).To(Equal(test_data.CatFileLumpModel.TransactionIndex))
+			Expect(dbResult.LogIndex).To(Equal(test_data.CatFileLumpModel.LogIndex))
+			Expect(dbResult.Raw).To(MatchJSON(test_data.CatFileLumpModel.Raw))
 		})
 	})
 

pkg/transformers/integration_tests/cat_file.go

@@ -80,12 +80,12 @@ var _ = Describe("Cat File transformer", func() {
 
 		Expect(dbResult[0].Ilk).To(Equal("REP"))
 		Expect(dbResult[0].What).To(Equal("lump"))
-		Expect(dbResult[0].Data).To(Equal("10000000000000000000000"))
+		Expect(dbResult[0].Data).To(Equal("10000.000000000000000000"))
 		Expect(dbResult[0].LogIndex).To(Equal(uint(3)))
 
 		Expect(dbResult[1].Ilk).To(Equal("REP"))
 		Expect(dbResult[1].What).To(Equal("chop"))
-		Expect(dbResult[1].Data).To(Equal("1000000000000000000000000000"))
+		Expect(dbResult[1].Data).To(Equal("1.000000000000000000000000000"))
 		Expect(dbResult[1].LogIndex).To(Equal(uint(4)))
 	})
 

pkg/transformers/integration_tests/price_feeds.go

@@ -65,7 +65,7 @@ var _ = Describe("Price feeds transformer", func() {
 		var model price_feeds.PriceFeedModel
 		err = db.Get(&model, `SELECT block_number, medianizer_address, usd_value, tx_idx, raw_log FROM maker.price_feeds WHERE block_number = $1`, config.StartingBlockNumber)
 		Expect(err).NotTo(HaveOccurred())
-		Expect(model.UsdValue).To(Equal("207.314891143"))
+		Expect(model.UsdValue).To(Equal("207.314891143000011198"))
 		Expect(model.MedianizerAddress).To(Equal(config.ContractAddresses[0]))
 	})
 
@@ -92,7 +92,7 @@ var _ = Describe("Price feeds transformer", func() {
 		var model price_feeds.PriceFeedModel
 		err = db.Get(&model, `SELECT block_number, medianizer_address, usd_value, tx_idx, raw_log FROM maker.price_feeds WHERE block_number = $1`, config.StartingBlockNumber)
 		Expect(err).NotTo(HaveOccurred())
-		Expect(model.UsdValue).To(Equal("391.803979212"))
+		Expect(model.UsdValue).To(Equal("391.803979212000001553"))
 		Expect(model.MedianizerAddress).To(Equal(config.ContractAddresses[0]))
 	})
 
@@ -119,7 +119,7 @@ var _ = Describe("Price feeds transformer", func() {
 		var model price_feeds.PriceFeedModel
 		err = db.Get(&model, `SELECT block_number, medianizer_address, usd_value, tx_idx, raw_log FROM maker.price_feeds WHERE block_number = $1`, config.StartingBlockNumber)
 		Expect(err).NotTo(HaveOccurred())
-		Expect(model.UsdValue).To(Equal("12.8169284827"))
+		Expect(model.UsdValue).To(Equal("12.816928482699999847"))
 		Expect(model.MedianizerAddress).To(Equal(config.ContractAddresses[0]))
 	})
 })

pkg/transformers/price_feeds/converter.go

@@ -18,6 +18,7 @@ import (
 	"encoding/json"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
 )
 
 type PriceFeedConverter struct{}
@@ -32,7 +33,7 @@ func (converter PriceFeedConverter) ToModels(ethLogs []types.Log) ([]interface{}
 		model := PriceFeedModel{
 			BlockNumber:       log.BlockNumber,
 			MedianizerAddress: log.Address.String(),
-			UsdValue:          Convert("wad", hexutil.Encode(log.Data), 15),
+			UsdValue:          shared.ConvertToWad(hexutil.Encode(log.Data)),
 			LogIndex:          log.Index,
 			TransactionIndex:  log.TxIndex,
 			Raw:               raw,

pkg/transformers/shared/utilities.go

@@ -19,6 +19,15 @@ import (
 	"math/big"
 )
 
+var (
+	rayBase      = big.NewFloat(1e27)
+	wadBase      = big.NewFloat(1e18)
+	rayPrecision = 27
+	wadPrecision = 18
+	ray          = "ray"
+	wad          = "wad"
+)
+
 func BigIntToInt64(value *big.Int) int64 {
 	if value == nil {
 		return int64(0)
@@ -47,3 +56,25 @@ func GetDataBytesAtIndex(n int, logData []byte) []byte {
 	}
 	return []byte{}
 }
+
+func ConvertToRay(value string) string {
+	return convert(ray, value, rayPrecision)
+}
+
+func ConvertToWad(value string) string {
+	return convert(wad, value, wadPrecision)
+}
+
+func convert(conversion string, value string, precision int) string {
+	result := big.NewFloat(0.0)
+	bigFloat := big.NewFloat(0.0)
+	bigFloat.SetString(value)
+
+	switch conversion {
+	case ray:
+		result.Quo(bigFloat, rayBase)
+	case wad:
+		result.Quo(bigFloat, wadBase)
+	}
+	return result.Text('f', precision)
+}

pkg/transformers/shared/utilities_test.go

@@ -45,5 +45,21 @@ var _ = Describe("Shared utilities", func() {
 
 			Expect(expected[:]).To(Equal(actual))
 		})
+
+		It("converts values to rays", func() {
+			rayOne := shared.ConvertToRay("123456789012345678901234567890")
+			Expect(rayOne).To(Equal("123.456789012345680589533003513"))
+
+			rayTwo := shared.ConvertToRay("1234567890123456790123567890")
+			Expect(rayTwo).To(Equal("1.234567890123456912476740399"))
+		})
+
+		It("converts values to wads", func() {
+			wadOne := shared.ConvertToWad("12345678901234567890123")
+			Expect(wadOne).To(Equal("12345.678901234567092615"))
+
+			wadTwo := shared.ConvertToWad("1234567890123456789")
+			Expect(wadTwo).To(Equal("1.234567890123456690"))
+		})
 	})
 })

pkg/transformers/test_data/cat_file.go

@@ -23,10 +23,9 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/cat_file/flip"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/cat_file/pit_vow"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
-	"math/big"
 )
 
-var EthCatFileChopLumpLog = types.Log{
+var EthCatFileChopLog = types.Log{
 	Address: common.HexToAddress(constants.CatContractAddress),
 	Topics: []common.Hash{
 		common.HexToHash("0x1a0b287e00000000000000000000000000000000000000000000000000000000"),
@@ -34,7 +33,7 @@ var EthCatFileChopLumpLog = types.Log{
 		common.HexToHash("0x66616b6520696c6b000000000000000000000000000000000000000000000000"),
 		common.HexToHash("0x63686f7000000000000000000000000000000000000000000000000000000000"),
 	},
-	Data:        hexutil.MustDecode("0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000641a0b287e66616b6520696c6b00000000000000000000000000000000000000000000000063686f700000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000075bcd15"),
+	Data:        hexutil.MustDecode("0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000641a0b287e66616b6520696c6b00000000000000000000000000000000000000000000000063686f700000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000018EE90FF6C373E0EE4E3F0AD2"),
 	BlockNumber: 110,
 	TxHash:      common.HexToHash("0xe32dfe6afd7ea28475569756fc30f0eea6ad4cfd32f67436ff1d1c805e4382df"),
 	TxIndex:     13,
@@ -42,15 +41,40 @@ var EthCatFileChopLumpLog = types.Log{
 	Index:       1,
 	Removed:     false,
 }
-
+var rawCatFileChopLog, _ = json.Marshal(EthCatFileChopLog)
-var rawCatFileChopLumpLog, _ = json.Marshal(EthCatFileChopLumpLog)
+var CatFileChopModel = chop_lump.CatFileChopLumpModel{
-var CatFileChopLumpModel = chop_lump.CatFileChopLumpModel{
 	Ilk:              "fake ilk",
 	What:             "chop",
-	Data:             big.NewInt(123456789).String(),
+	Data:             "123.456789012345680589533003513",
-	TransactionIndex: EthCatFileChopLumpLog.TxIndex,
+	TransactionIndex: EthCatFileChopLog.TxIndex,
-	LogIndex:         EthCatFileChopLumpLog.Index,
+	LogIndex:         EthCatFileChopLog.Index,
-	Raw:              rawCatFileChopLumpLog,
+	Raw:              rawCatFileChopLog,
+}
+
+var EthCatFileLumpLog = types.Log{
+	Address: common.HexToAddress(constants.CatContractAddress),
+	Topics: []common.Hash{
+		common.HexToHash("0x1a0b287e00000000000000000000000000000000000000000000000000000000"),
+		common.HexToHash("0x00000000000000000000000064d922894153be9eef7b7218dc565d1d0ce2a092"),
+		common.HexToHash("0x66616b6520696c6b000000000000000000000000000000000000000000000000"),
+		common.HexToHash("0x6c756d7000000000000000000000000000000000000000000000000000000000"),
+	},
+	Data:        hexutil.MustDecode("0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000641a0b287e66616b6520696c6b00000000000000000000000000000000000000000000000063686f700000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000029D42B64E76714244CB"),
+	BlockNumber: 110,
+	TxHash:      common.HexToHash("0xe32dfe6afd7ea28475569756fc30f0eea6ad4cfd32f67436ff1d1c805e4382df"),
+	TxIndex:     15,
+	BlockHash:   common.HexToHash("0x2764998a4e048d4c4ba45ea40fd5efaa8e2d4f1dd2b15425a6c6a3dea7f1064a"),
+	Index:       3,
+	Removed:     false,
+}
+var rawCatFileLumpLog, _ = json.Marshal(EthCatFileLumpLog)
+var CatFileLumpModel = chop_lump.CatFileChopLumpModel{
+	Ilk:              "fake ilk",
+	What:             "lump",
+	Data:             "12345.678901234567092615",
+	TransactionIndex: EthCatFileLumpLog.TxIndex,
+	LogIndex:         EthCatFileLumpLog.Index,
+	Raw:              rawCatFileLumpLog,
 }
 
 var EthCatFileFlipLog = types.Log{

pkg/transformers/test_data/price_feed.go

@@ -45,7 +45,7 @@ var rawPriceFeedLog, _ = json.Marshal(EthPriceFeedLog)
 var PriceFeedModel = price_feeds.PriceFeedModel{
 	BlockNumber:       blockNumber,
 	MedianizerAddress: EthPriceFeedLog.Address.String(),
-	UsdValue:          "378.6599388897",
+	UsdValue:          "378.659938889700015352",
 	LogIndex:          EthPriceFeedLog.Index,
 	TransactionIndex:  EthPriceFeedLog.TxIndex,
 	Raw:               rawPriceFeedLog,