From 40408e83003e14972c976de6f5a8efcec0293d3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edvard=20H=C3=BCbinette?= Date: Thu, 15 Nov 2018 12:57:29 +0100 Subject: [PATCH] VDB-120 include tic in dent and tend (#118) * Add constants for TTL and tic for fakeHeader * Add tic to tend transformer * Update tests for tend * Fix string conversion bug in fakes * Fix tend integration tests after staging rebase * Add tic to dent transformer * Update dent tests * Change integration tests to use hardcoded block timestamp --- pkg/fakes/data.go | 13 ++++++--- pkg/transformers/dent/converter.go | 4 --- pkg/transformers/dent/model.go | 1 - pkg/transformers/dent/repository.go | 7 ++++- pkg/transformers/dent/repository_test.go | 12 +++++--- pkg/transformers/integration_tests/dent.go | 8 ++++++ pkg/transformers/integration_tests/tend.go | 32 +++++++++++++++++----- pkg/transformers/shared/constants/data.go | 3 ++ pkg/transformers/shared/utilities.go | 13 +++++++++ pkg/transformers/tend/converter.go | 4 --- pkg/transformers/tend/model.go | 1 - pkg/transformers/tend/repository.go | 9 ++++-- pkg/transformers/tend/repository_test.go | 9 ++++-- pkg/transformers/test_data/dent.go | 2 -- pkg/transformers/test_data/tend.go | 3 +- 15 files changed, 86 insertions(+), 35 deletions(-) diff --git a/pkg/fakes/data.go b/pkg/fakes/data.go index b949440e..d8236efc 100644 --- a/pkg/fakes/data.go +++ b/pkg/fakes/data.go @@ -6,18 +6,21 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" + "strconv" ) var ( - FakeError = errors.New("failed") - FakeHash = common.BytesToHash([]byte{1, 2, 3, 4, 5}) + FakeError = errors.New("failed") + FakeHash = common.BytesToHash([]byte{1, 2, 3, 4, 5}) + fakeTimestamp = int64(111111111) ) var rawFakeHeader, _ = json.Marshal(types.Header{}) var FakeHeader = core.Header{ Hash: FakeHash.String(), Raw: rawFakeHeader, - Timestamp: "111111111", + Timestamp: strconv.FormatInt(fakeTimestamp, 10), } func GetFakeHeader(blockNumber int64) core.Header { @@ -25,6 +28,8 @@ func GetFakeHeader(blockNumber int64) core.Header { Hash: FakeHash.String(), BlockNumber: blockNumber, Raw: rawFakeHeader, - Timestamp: "111111111", + Timestamp: strconv.FormatInt(fakeTimestamp, 10), } } + +var FakeHeaderTic = fakeTimestamp + constants.TTL diff --git a/pkg/transformers/dent/converter.go b/pkg/transformers/dent/converter.go index 29f47bdd..75ac93b3 100644 --- a/pkg/transformers/dent/converter.go +++ b/pkg/transformers/dent/converter.go @@ -40,9 +40,6 @@ func (c DentConverter) ToModels(ethLogs []types.Log) (result []interface{}, err lot := log.Topics[3].Big().String() bidValue := getBidValue(log) guy := common.HexToAddress(log.Topics[1].Hex()).String() - tic := "0" - //TODO: it is likely that the tic value will need to be added to an emitted event, - //so this will need to be updated at that point logIndex := log.Index transactionIndex := log.TxIndex @@ -57,7 +54,6 @@ func (c DentConverter) ToModels(ethLogs []types.Log) (result []interface{}, err Lot: lot, Bid: bidValue, Guy: guy, - Tic: tic, LogIndex: logIndex, TransactionIndex: transactionIndex, Raw: raw, diff --git a/pkg/transformers/dent/model.go b/pkg/transformers/dent/model.go index c6b4b37a..7f5bdfc1 100644 --- a/pkg/transformers/dent/model.go +++ b/pkg/transformers/dent/model.go @@ -19,7 +19,6 @@ type DentModel struct { Lot string Bid string Guy string - Tic string LogIndex uint `db:"log_idx"` TransactionIndex uint `db:"tx_idx"` Raw []byte `db:"raw_log"` diff --git a/pkg/transformers/dent/repository.go b/pkg/transformers/dent/repository.go index e86e6aac..80be4fdd 100644 --- a/pkg/transformers/dent/repository.go +++ b/pkg/transformers/dent/repository.go @@ -32,6 +32,11 @@ func (repository DentRepository) Create(headerID int64, models []interface{}) er return err } + tic, err := shared.GetTicInTx(headerID, tx) + if err != nil { + return err + } + for _, model := range models { dent, ok := model.(DentModel) if !ok { @@ -48,7 +53,7 @@ func (repository DentRepository) Create(headerID int64, models []interface{}) er _, err = tx.Exec( `INSERT into maker.dent (header_id, bid_id, lot, bid, guy, tic, log_idx, tx_idx, raw_log) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9)`, - headerID, dent.BidId, dent.Lot, dent.Bid, dent.Guy, dent.Tic, dent.LogIndex, dent.TransactionIndex, dent.Raw, + headerID, dent.BidId, dent.Lot, dent.Bid, dent.Guy, tic, dent.LogIndex, dent.TransactionIndex, dent.Raw, ) if err != nil { tx.Rollback() diff --git a/pkg/transformers/dent/repository_test.go b/pkg/transformers/dent/repository_test.go index b5d6d67d..bad19213 100644 --- a/pkg/transformers/dent/repository_test.go +++ b/pkg/transformers/dent/repository_test.go @@ -57,9 +57,9 @@ var _ = Describe("Dent Repository", func() { shared_behaviors.SharedRepositoryCreateBehaviors(&inputs) It("persists a dent record", func() { - headerId, err := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) + headerID, err := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) Expect(err).NotTo(HaveOccurred()) - err = dentRepository.Create(headerId, []interface{}{test_data.DentModel}) + err = dentRepository.Create(headerID, []interface{}{test_data.DentModel}) Expect(err).NotTo(HaveOccurred()) var count int @@ -67,16 +67,20 @@ var _ = Describe("Dent Repository", func() { Expect(count).To(Equal(1)) var dbResult dent.DentModel - err = db.Get(&dbResult, `SELECT bid_id, lot, bid, guy, tic, log_idx, tx_idx, raw_log FROM maker.dent WHERE header_id = $1`, headerId) + err = db.Get(&dbResult, `SELECT bid_id, lot, bid, guy, log_idx, tx_idx, raw_log FROM maker.dent WHERE header_id = $1`, headerID) Expect(err).NotTo(HaveOccurred()) Expect(dbResult.BidId).To(Equal(test_data.DentModel.BidId)) Expect(dbResult.Lot).To(Equal(test_data.DentModel.Lot)) Expect(dbResult.Bid).To(Equal(test_data.DentModel.Bid)) Expect(dbResult.Guy).To(Equal(test_data.DentModel.Guy)) - Expect(dbResult.Tic).To(Equal(test_data.DentModel.Tic)) Expect(dbResult.LogIndex).To(Equal(test_data.DentModel.LogIndex)) Expect(dbResult.TransactionIndex).To(Equal(test_data.DentModel.TransactionIndex)) Expect(dbResult.Raw).To(MatchJSON(test_data.DentModel.Raw)) + + var dbTic int64 + err = db.Get(&dbTic, `SELECT tic FROM maker.dent WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(dbTic).To(Equal(fakes.FakeHeaderTic)) }) }) diff --git a/pkg/transformers/integration_tests/dent.go b/pkg/transformers/integration_tests/dent.go index 716b257e..47e6c7ac 100644 --- a/pkg/transformers/integration_tests/dent.go +++ b/pkg/transformers/integration_tests/dent.go @@ -3,6 +3,7 @@ package integration_tests import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" @@ -55,6 +56,13 @@ var _ = Describe("Dent transformer", func() { Expect(dbResult[0].BidId).To(Equal("2")) Expect(dbResult[0].Guy).To(Equal("0x0000d8b4147eDa80Fec7122AE16DA2479Cbd7ffB")) Expect(dbResult[0].Lot).To(Equal("1000000000000000000000000000")) + + var dbTic int64 + err = db.Get(&dbTic, `SELECT tic FROM maker.dent`) + Expect(err).NotTo(HaveOccurred()) + + actualTic := 1538637780 + constants.TTL + Expect(dbTic).To(Equal(actualTic)) }) It("persists a flip dent log event", func() { diff --git a/pkg/transformers/integration_tests/tend.go b/pkg/transformers/integration_tests/tend.go index 4e146446..5fe2084e 100644 --- a/pkg/transformers/integration_tests/tend.go +++ b/pkg/transformers/integration_tests/tend.go @@ -17,11 +17,11 @@ package integration_tests import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/tend" "github.com/vulcanize/vulcanizedb/test_config" ) @@ -61,7 +61,7 @@ var _ = Describe("Tend LogNoteTransformer", func() { Expect(err).NotTo(HaveOccurred()) var dbResult []tend.TendModel - err = db.Select(&dbResult, `SELECT bid, bid_id, guy, lot, tic from maker.tend`) + err = db.Select(&dbResult, `SELECT bid, bid_id, guy, lot FROM maker.tend`) Expect(err).NotTo(HaveOccurred()) Expect(len(dbResult)).To(Equal(1)) @@ -69,7 +69,13 @@ var _ = Describe("Tend LogNoteTransformer", func() { Expect(dbResult[0].BidId).To(Equal("3")) Expect(dbResult[0].Guy).To(Equal("0x0000d8b4147eDa80Fec7122AE16DA2479Cbd7ffB")) Expect(dbResult[0].Lot).To(Equal("1000000000000000000")) - Expect(dbResult[0].Tic).To(Equal("0")) + + var dbTic int64 + err = db.Get(&dbTic, `SELECT tic FROM maker.tend`) + Expect(err).NotTo(HaveOccurred()) + + actualTic := 1538490276 + constants.TTL + Expect(dbTic).To(Equal(actualTic)) }) It("fetches and transforms a subsequent Flip Tend event from Kovan chain for the same auction", func() { @@ -92,7 +98,7 @@ var _ = Describe("Tend LogNoteTransformer", func() { Expect(err).NotTo(HaveOccurred()) var dbResult []tend.TendModel - err = db.Select(&dbResult, `SELECT bid, bid_id, guy, lot, tic from maker.tend`) + err = db.Select(&dbResult, `SELECT bid, bid_id, guy, lot from maker.tend`) Expect(err).NotTo(HaveOccurred()) Expect(len(dbResult)).To(Equal(1)) @@ -100,7 +106,13 @@ var _ = Describe("Tend LogNoteTransformer", func() { Expect(dbResult[0].BidId).To(Equal("3")) Expect(dbResult[0].Guy).To(Equal("0x0000d8b4147eDa80Fec7122AE16DA2479Cbd7ffB")) Expect(dbResult[0].Lot).To(Equal("1000000000000000000")) - Expect(dbResult[0].Tic).To(Equal("0")) + + var dbTic int64 + err = db.Get(&dbTic, `SELECT tic FROM maker.tend`) + Expect(err).NotTo(HaveOccurred()) + + actualTic := 1538491224 + constants.TTL + Expect(dbTic).To(Equal(actualTic)) }) It("fetches and transforms a Flap Tend event from the Kovan chain", func() { @@ -123,7 +135,7 @@ var _ = Describe("Tend LogNoteTransformer", func() { Expect(err).NotTo(HaveOccurred()) var dbResult []tend.TendModel - err = db.Select(&dbResult, `SELECT bid, bid_id, guy, lot, tic from maker.tend`) + err = db.Select(&dbResult, `SELECT bid, bid_id, guy, lot from maker.tend`) Expect(err).NotTo(HaveOccurred()) Expect(len(dbResult)).To(Equal(1)) @@ -131,6 +143,12 @@ var _ = Describe("Tend LogNoteTransformer", func() { Expect(dbResult[0].BidId).To(Equal("1")) Expect(dbResult[0].Guy).To(Equal("0x0000d8b4147eDa80Fec7122AE16DA2479Cbd7ffB")) Expect(dbResult[0].Lot).To(Equal("1000000000000000000")) - Expect(dbResult[0].Tic).To(Equal("0")) + + var dbTic int64 + err = db.Get(&dbTic, `SELECT tic FROM maker.tend`) + Expect(err).NotTo(HaveOccurred()) + + actualTic := 1538992860 + constants.TTL + Expect(dbTic).To(Equal(actualTic)) }) }) diff --git a/pkg/transformers/shared/constants/data.go b/pkg/transformers/shared/constants/data.go index 135160e7..001952ba 100644 --- a/pkg/transformers/shared/constants/data.go +++ b/pkg/transformers/shared/constants/data.go @@ -1,3 +1,6 @@ package constants var DataItemLength = 32 + +// TODO Grab this from DB, since it can change through governance +var TTL = int64(10800) // 60 * 60 * 3 == 10800 seconds == 3 hours diff --git a/pkg/transformers/shared/utilities.go b/pkg/transformers/shared/utilities.go index 073f8783..a97aef7b 100644 --- a/pkg/transformers/shared/utilities.go +++ b/pkg/transformers/shared/utilities.go @@ -15,6 +15,7 @@ package shared import ( + "database/sql" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "math/big" ) @@ -78,3 +79,15 @@ func convert(conversion string, value string, precision int) string { } return result.Text('f', precision) } + +// Grabs the block timestamp for an headerID, and adds the TTL constant +func GetTicInTx(headerID int64, tx *sql.Tx) (int64, error) { + var blockTimestamp int64 + err := tx.QueryRow(`SELECT block_timestamp FROM public.headers WHERE id = $1;`, headerID).Scan(&blockTimestamp) + if err != nil { + return 0, err + } + + tic := blockTimestamp + constants.TTL + return tic, nil +} diff --git a/pkg/transformers/tend/converter.go b/pkg/transformers/tend/converter.go index 632d718b..1b49a6ca 100644 --- a/pkg/transformers/tend/converter.go +++ b/pkg/transformers/tend/converter.go @@ -40,9 +40,6 @@ func (TendConverter) ToModels(ethLogs []types.Log) (results []interface{}, err e lastItem := ethLog.Data[lastDataItemStartIndex:] last := big.NewInt(0).SetBytes(lastItem) bidValue := last.String() - tic := "0" - //TODO: it is likely that the tic value will need to be added to an emitted event, - //so this will need to be updated at that point transactionIndex := ethLog.TxIndex logIndex := ethLog.Index @@ -56,7 +53,6 @@ func (TendConverter) ToModels(ethLogs []types.Log) (results []interface{}, err e Lot: lot, Bid: bidValue, Guy: guy, - Tic: tic, LogIndex: logIndex, TransactionIndex: transactionIndex, Raw: rawLog, diff --git a/pkg/transformers/tend/model.go b/pkg/transformers/tend/model.go index 8dedcadc..58c1d4ba 100644 --- a/pkg/transformers/tend/model.go +++ b/pkg/transformers/tend/model.go @@ -19,7 +19,6 @@ type TendModel struct { Lot string Bid string Guy string - Tic string LogIndex uint `db:"log_idx"` TransactionIndex uint `db:"tx_idx"` Raw []byte `db:"raw_log"` diff --git a/pkg/transformers/tend/repository.go b/pkg/transformers/tend/repository.go index 91437658..c42ed67b 100644 --- a/pkg/transformers/tend/repository.go +++ b/pkg/transformers/tend/repository.go @@ -32,6 +32,11 @@ func (repository TendRepository) Create(headerID int64, models []interface{}) er return err } + tic, err := shared.GetTicInTx(headerID, tx) + if err != nil { + return err + } + for _, model := range models { tend, ok := model.(TendModel) if !ok { @@ -47,8 +52,8 @@ func (repository TendRepository) Create(headerID int64, models []interface{}) er _, err = tx.Exec( `INSERT into maker.tend (header_id, bid_id, lot, bid, guy, tic, log_idx, tx_idx, raw_log) - VALUES($1, $2, $3::NUMERIC, $4::NUMERIC, $5, $6::NUMERIC, $7, $8, $9)`, - headerID, tend.BidId, tend.Lot, tend.Bid, tend.Guy, tend.Tic, tend.LogIndex, tend.TransactionIndex, tend.Raw, + VALUES($1, $2, $3::NUMERIC, $4::NUMERIC, $5, $6, $7, $8, $9)`, + headerID, tend.BidId, tend.Lot, tend.Bid, tend.Guy, tic, tend.LogIndex, tend.TransactionIndex, tend.Raw, ) if err != nil { diff --git a/pkg/transformers/tend/repository_test.go b/pkg/transformers/tend/repository_test.go index 79465cee..dae2d4ba 100644 --- a/pkg/transformers/tend/repository_test.go +++ b/pkg/transformers/tend/repository_test.go @@ -68,19 +68,22 @@ var _ = Describe("TendRepository", func() { Expect(count).To(Equal(1)) dbResult := tend.TendModel{} - err = db.Get(&dbResult, `SELECT bid_id, lot, bid, guy, tic, log_idx, tx_idx, raw_log FROM maker.tend WHERE header_id = $1`, headerID) + err = db.Get(&dbResult, `SELECT bid_id, lot, bid, guy, log_idx, tx_idx, raw_log FROM maker.tend WHERE header_id = $1`, headerID) Expect(err).NotTo(HaveOccurred()) Expect(dbResult.BidId).To(Equal(test_data.TendModel.BidId)) Expect(dbResult.Lot).To(Equal(test_data.TendModel.Lot)) Expect(dbResult.Bid).To(Equal(test_data.TendModel.Bid)) Expect(dbResult.Guy).To(Equal(test_data.TendModel.Guy)) - Expect(dbResult.Tic).To(Equal(test_data.TendModel.Tic)) Expect(dbResult.LogIndex).To(Equal(test_data.TendModel.LogIndex)) Expect(dbResult.TransactionIndex).To(Equal(test_data.TendModel.TransactionIndex)) Expect(dbResult.Raw).To(MatchJSON(test_data.TendModel.Raw)) - }) + var dbTic int64 + err = db.Get(&dbTic, `SELECT tic FROM maker.tend WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(dbTic).To(Equal(fakes.FakeHeaderTic)) + }) }) Describe("MarkHeaderChecked", func() { diff --git a/pkg/transformers/test_data/dent.go b/pkg/transformers/test_data/dent.go index 0acd5972..94e2a18e 100644 --- a/pkg/transformers/test_data/dent.go +++ b/pkg/transformers/test_data/dent.go @@ -31,7 +31,6 @@ var ( dentBidId = int64(1) dentLot = "11000000000000000000" dentBid = "3000000000000000000" - DentTic = "0" dentGuy = "0x64d922894153BE9EEf7b7218dc565d1D0Ce2a092" dentRawJson, _ = json.Marshal(DentLog) ) @@ -58,7 +57,6 @@ var DentModel = dent.DentModel{ Lot: dentLot, Bid: dentBid, Guy: dentGuy, - Tic: DentTic, LogIndex: DentLog.Index, TransactionIndex: DentLog.TxIndex, Raw: dentRawJson, diff --git a/pkg/transformers/test_data/tend.go b/pkg/transformers/test_data/tend.go index 3ea850f5..d8e2bc62 100644 --- a/pkg/transformers/test_data/tend.go +++ b/pkg/transformers/test_data/tend.go @@ -34,7 +34,7 @@ var ( tendGuy = "0x7d7bEe5fCfD8028cf7b00876C5b1421c800561A6" tendData = "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000644b43ed12000000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000049b9ca9a6943400000000000000000000000000000000000000000000000000000de0b6b3a7640000" tendTransactionHash = "0x7909c8793ded2b8348f5db623044fbc26bb7ab78ad5792897abdf68ddc1df63d" - TendTic = "0" + tendBlockHash = "0xa8ea87147c0a68daeb6b1d9f8c0937ba975a650809cab80d19c969e8d0df452c" ) var TendLogNote = types.Log{ @@ -60,7 +60,6 @@ var TendModel = tend.TendModel{ Lot: tendLot, Bid: tendBid, Guy: tendGuy, - Tic: TendTic, LogIndex: TendLogNote.Index, TransactionIndex: TendLogNote.TxIndex, Raw: rawTendLog,