cerc-io/ipld-eth-server
18
0
Fork 1
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases 37 Wiki Activity

Merge branch 'staging' into VDB-125-aggregate-log-fetching

Browse Source
This commit is contained in:
Edvard 2018-12-11 15:03:21 +01:00
commit ff47e2e06e
14 changed files with 168 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
environments/staging.toml
View File

@ -6,7 +6,7 @@ password = "vulcanize"
port = 5432
[client]
ipcPath = "http://147.75.64.249:8545"
ipcPath = "http://147.75.199.135:8545"
[datadog]
name = "maker_vdb_staging"

3
pkg/history/populate_blocks.go
View File

@ -1,7 +1,7 @@
package history
import (
"log"
log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore"
@ -10,7 +10,6 @@ import (
func PopulateMissingBlocks(blockchain core.BlockChain, blockRepository datastore.BlockRepository, startingBlockNumber int64) int {
lastBlock := blockchain.LastBlock().Int64()
blockRange := blockRepository.MissingBlockNumbers(startingBlockNumber, lastBlock, blockchain.Node().ID)
log.SetPrefix("")
log.Printf("Backfilling %d blocks\n\n", len(blockRange))
RetrieveAndUpdateBlocks(blockchain, blockRepository, blockRange)
return len(blockRange)

3
pkg/transformers/drip_file/ilk/converter.go
View File

@ -22,6 +22,7 @@ import (
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
)
@ -38,7 +39,7 @@ func (DripFileIlkConverter) ToModels(ethLogs []types.Log) ([]interface{}, error)
ilk := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
vow := string(bytes.Trim(ethLog.Topics[3].Bytes(), "\x00"))
taxBytes := ethLog.Data[len(ethLog.Data)-constants.DataItemLength:]
tax := big.NewInt(0).SetBytes(taxBytes).String()
tax := shared.ConvertToRay(big.NewInt(0).SetBytes(taxBytes).String())
raw, err := json.Marshal(ethLog)
if err != nil {
return nil, err

2
pkg/transformers/integration_tests/pit_file_debt_ceiling.go
View File

@ -74,6 +74,6 @@ var _ = Describe("PitFileDebtCeiling LogNoteTransformer", func() {
Expect(len(dbResult)).To(Equal(1))
Expect(dbResult[0].What).To(Equal("Line"))
Expect(dbResult[0].Data).To(Equal("10000000000000000000000000"))
Expect(dbResult[0].Data).To(Equal("10000000.000000000000000000"))
})
})

50
pkg/transformers/integration_tests/pit_file_ilk.go
View File

@ -18,15 +18,15 @@ import (
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/transformers/factories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/factories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/ilk"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/test_config"
)
var _ = Describe("PitFileIlk LogNoteTransformer", func() {
It("fetches and transforms a PitFileIlk event from Kovan chain", func() {
It("fetches and transforms a Pit.file ilk 'spot' event from Kovan", func() {
blockNumber := int64(9103223)
config := ilk.IlkFileConfig
config.StartingBlockNumber = blockNumber
@ -67,6 +67,48 @@ var _ = Describe("PitFileIlk LogNoteTransformer", func() {
Expect(len(dbResult)).To(Equal(1))
Expect(dbResult[0].Ilk).To(Equal("ETH"))
Expect(dbResult[0].What).To(Equal("spot"))
Expect(dbResult[0].Data).To(Equal("139840000000000000000000000000"))
Expect(dbResult[0].Data).To(Equal("139.840000000000003410605131648"))
})
It("fetches and transforms a Pit.file ilk 'line' event from Kovan", func() {
blockNumber := int64(8762253)
config := ilk.IlkFileConfig
config.StartingBlockNumber = blockNumber
config.EndingBlockNumber = blockNumber
rpcClient, ethClient, err := getClients(ipc)
Expect(err).NotTo(HaveOccurred())
blockChain, err := getBlockChain(rpcClient, ethClient)
Expect(err).NotTo(HaveOccurred())
db := test_config.NewTestDB(blockChain.Node())
test_config.CleanTestDB(db)
err = persistHeader(db, blockNumber, blockChain)
Expect(err).NotTo(HaveOccurred())
initializer := factories.LogNoteTransformer{
Config: config,
Fetcher: &shared.Fetcher{},
Converter: &ilk.PitFileIlkConverter{},
Repository: &ilk.PitFileIlkRepository{},
}
transformer := initializer.NewLogNoteTransformer(db, blockChain)
err = transformer.Execute()
Expect(err).NotTo(HaveOccurred())
var dbResult []ilk.PitFileIlkModel
err = db.Select(&dbResult, `SELECT ilk, what, data from maker.pit_file_ilk`)
Expect(err).NotTo(HaveOccurred())
Expect(len(dbResult)).To(Equal(2))
var pitFileIlkLineModel ilk.PitFileIlkModel
for _, result := range dbResult {
if result.What == "line" {
pitFileIlkLineModel = result
}
}
Expect(pitFileIlkLineModel.Ilk).To(Equal("REP"))
Expect(pitFileIlkLineModel.Data).To(Equal("2000000.000000000000000000"))
})
})

4
pkg/transformers/integration_tests/vat_fold.go
View File

@ -43,7 +43,7 @@ var _ = Describe("VatFold Transformer", func() {
})
It("transforms VatFold log events", func() {
blockNumber := int64(9051149)
blockNumber := int64(9367233)
config := vat_fold.VatFoldConfig
config.StartingBlockNumber = blockNumber
config.EndingBlockNumber = blockNumber
@ -75,6 +75,6 @@ var _ = Describe("VatFold Transformer", func() {
dbResult := dbResults[0]
Expect(dbResult.Ilk).To(Equal("REP"))
Expect(dbResult.Urn).To(Equal(common.HexToAddress("0x0000000000000000000000003728e9777b2a0a611ee0f89e00e01044ce4736d1").String()))
Expect(dbResult.Rate).To(Equal("0"))
Expect(dbResult.Rate).To(Equal("0.000000000000000000000000000"))
})
})

9
pkg/transformers/pit_file/debt_ceiling/converter.go
View File

@ -15,13 +15,12 @@
package debt_ceiling
import (
"encoding/json"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
"math/big"
"bytes"
"encoding/json"
"errors"
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
)
type PitFileDebtCeilingConverter struct{}
@ -34,7 +33,7 @@ func (PitFileDebtCeilingConverter) ToModels(ethLogs []types.Log) ([]interface{},
return nil, err
}
what := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
data := big.NewInt(0).SetBytes(ethLog.Topics[3].Bytes()).String()
data := shared.ConvertToWad(ethLog.Topics[3].Big().String())
raw, err := json.Marshal(ethLog)
if err != nil {

21
pkg/transformers/pit_file/ilk/converter.go
View File

@ -22,6 +22,7 @@ import (
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
)
@ -36,8 +37,11 @@ func (PitFileIlkConverter) ToModels(ethLogs []types.Log) ([]interface{}, error)
}
ilk := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
what := string(bytes.Trim(ethLog.Topics[3].Bytes(), "\x00"))
riskBytes := ethLog.Data[len(ethLog.Data)-constants.DataItemLength:]
risk := big.NewInt(0).SetBytes(riskBytes).String()
dataBytes := ethLog.Data[len(ethLog.Data)-constants.DataItemLength:]
data, err := getData(dataBytes, what)
if err != nil {
return nil, err
}
raw, err := json.Marshal(ethLog)
if err != nil {
@ -46,7 +50,7 @@ func (PitFileIlkConverter) ToModels(ethLogs []types.Log) ([]interface{}, error)
model := PitFileIlkModel{
Ilk: ilk,
What: what,
Data: risk,
Data: data,
LogIndex: ethLog.Index,
TransactionIndex: ethLog.TxIndex,
Raw: raw,
@ -56,6 +60,17 @@ func (PitFileIlkConverter) ToModels(ethLogs []types.Log) ([]interface{}, error)
return models, nil
}
func getData(dataBytes []byte, what string) (string, error) {
n := big.NewInt(0).SetBytes(dataBytes).String()
if what == "spot" {
return shared.ConvertToRay(n), nil
} else if what == "line" {
return shared.ConvertToWad(n), nil
} else {
return "", errors.New("unexpected payload for 'what'")
}
}
func verifyLog(log types.Log) error {
if len(log.Topics) < 4 {
return errors.New("log missing topics")

46
pkg/transformers/pit_file/ilk/converter_test.go
View File

@ -47,13 +47,49 @@ var _ = Describe("Pit file ilk converter", func() {
Expect(err).To(HaveOccurred())
})
It("converts a log to an model", func() {
It("returns error if 'what' field is unknown", func() {
log := types.Log{
Address: test_data.EthPitFileIlkLineLog.Address,
Topics: []common.Hash{
test_data.EthPitFileIlkLineLog.Topics[0],
test_data.EthPitFileIlkLineLog.Topics[1],
test_data.EthPitFileIlkLineLog.Topics[2],
common.HexToHash("0x1111111100000000000000000000000000000000000000000000000000000000"),
},
Data: test_data.EthPitFileIlkLineLog.Data,
BlockNumber: test_data.EthPitFileIlkLineLog.BlockNumber,
TxHash: test_data.EthPitFileIlkLineLog.TxHash,
TxIndex: test_data.EthPitFileIlkLineLog.TxIndex,
BlockHash: test_data.EthPitFileIlkLineLog.BlockHash,
Index: test_data.EthPitFileIlkLineLog.Index,
}
converter := ilk.PitFileIlkConverter{}
models, err := converter.ToModels([]types.Log{test_data.EthPitFileIlkLog})
_, err := converter.ToModels([]types.Log{log})
Expect(err).NotTo(HaveOccurred())
Expect(len(models)).To(Equal(1))
Expect(models[0].(ilk.PitFileIlkModel)).To(Equal(test_data.PitFileIlkModel))
Expect(err).To(HaveOccurred())
})
Describe("when log is valid", func() {
It("converts to model with data converted to ray when what is 'spot'", func() {
converter := ilk.PitFileIlkConverter{}
models, err := converter.ToModels([]types.Log{test_data.EthPitFileIlkSpotLog})
Expect(err).NotTo(HaveOccurred())
Expect(len(models)).To(Equal(1))
Expect(models[0].(ilk.PitFileIlkModel)).To(Equal(test_data.PitFileIlkSpotModel))
})
It("converts to model with data converted to wad when what is 'line'", func() {
converter := ilk.PitFileIlkConverter{}
models, err := converter.ToModels([]types.Log{test_data.EthPitFileIlkLineLog})
Expect(err).NotTo(HaveOccurred())
Expect(len(models)).To(Equal(1))
Expect(models[0].(ilk.PitFileIlkModel)).To(Equal(test_data.PitFileIlkLineModel))
})
})
})

18
pkg/transformers/pit_file/ilk/repository_test.go
View File

@ -44,12 +44,12 @@ var _ = Describe("Pit file ilk repository", func() {
})
Describe("Create", func() {
modelWithDifferentLogIdx := test_data.PitFileIlkModel
modelWithDifferentLogIdx := test_data.PitFileIlkSpotModel
modelWithDifferentLogIdx.LogIndex = modelWithDifferentLogIdx.LogIndex + 1
inputs := shared_behaviors.CreateBehaviorInputs{
CheckedHeaderColumnName: constants.PitFileIlkChecked,
LogEventTableName: "maker.pit_file_ilk",
TestModel: test_data.PitFileIlkModel,
TestModel: test_data.PitFileIlkSpotModel,
ModelWithDifferentLogIdx: modelWithDifferentLogIdx,
Repository: &pitFileIlkRepository,
}
@ -59,18 +59,18 @@ var _ = Describe("Pit file ilk repository", func() {
It("adds a pit file ilk event", func() {
headerID, err := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader)
Expect(err).NotTo(HaveOccurred())
err = pitFileIlkRepository.Create(headerID, []interface{}{test_data.PitFileIlkModel})
err = pitFileIlkRepository.Create(headerID, []interface{}{test_data.PitFileIlkSpotModel})
Expect(err).NotTo(HaveOccurred())
var dbPitFile ilk.PitFileIlkModel
err = db.Get(&dbPitFile, `SELECT ilk, what, data, log_idx, tx_idx, raw_log FROM maker.pit_file_ilk WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(dbPitFile.Ilk).To(Equal(test_data.PitFileIlkModel.Ilk))
Expect(dbPitFile.What).To(Equal(test_data.PitFileIlkModel.What))
Expect(dbPitFile.Data).To(Equal(test_data.PitFileIlkModel.Data))
Expect(dbPitFile.LogIndex).To(Equal(test_data.PitFileIlkModel.LogIndex))
Expect(dbPitFile.TransactionIndex).To(Equal(test_data.PitFileIlkModel.TransactionIndex))
Expect(dbPitFile.Raw).To(MatchJSON(test_data.PitFileIlkModel.Raw))
Expect(dbPitFile.Ilk).To(Equal(test_data.PitFileIlkSpotModel.Ilk))
Expect(dbPitFile.What).To(Equal(test_data.PitFileIlkSpotModel.What))
Expect(dbPitFile.Data).To(Equal(test_data.PitFileIlkSpotModel.Data))
Expect(dbPitFile.LogIndex).To(Equal(test_data.PitFileIlkSpotModel.LogIndex))
Expect(dbPitFile.TransactionIndex).To(Equal(test_data.PitFileIlkSpotModel.TransactionIndex))
Expect(dbPitFile.Raw).To(MatchJSON(test_data.PitFileIlkSpotModel.Raw))
})
})

4
pkg/transformers/test_data/drip_file.go
View File

@ -35,7 +35,7 @@ var EthDripFileIlkLog = types.Log{
common.HexToHash("0x66616b6520696c6b000000000000000000000000000000000000000000000000"),
common.HexToHash("0x66616b6520766f77000000000000000000000000000000000000000000000000"),
},
Data: hexutil.MustDecode("0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000641a0b287e66616b6520696c6b00000000000000000000000000000000000000000000000066616b6520766f77000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007b"),
Data: hexutil.MustDecode("0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000641a0b287e66616b6520696c6b00000000000000000000000000000000000000000000000066616b6520766f77000000000000000000000000000000000000000000000000000000000000000000000000000000000000009B3F7188CE95D16E5AE0000000"),
BlockNumber: 35,
TxHash: common.HexToHash("0xa1c31b7e6389470902237161263558615e60b40f2e63060b2f4aeafe92d57e5f"),
TxIndex: 12,
@ -48,7 +48,7 @@ var rawDripFileIlkLog, _ = json.Marshal(EthDripFileIlkLog)
var DripFileIlkModel = ilk2.DripFileIlkModel{
Ilk: "fake ilk",
Vow: "fake vow",
Tax: big.NewInt(123).String(),
Tax: "12300.000000000000000000000000000",
LogIndex: EthDripFileIlkLog.Index,
TransactionIndex: EthDripFileIlkLog.TxIndex,
Raw: rawDripFileIlkLog,

48
pkg/transformers/test_data/pit_file.go
View File

@ -25,6 +25,7 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/debt_ceiling"
ilk2 "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/ilk"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
)
@ -48,21 +49,48 @@ var EthPitFileDebtCeilingLog = types.Log{
var rawPitFileDebtCeilingLog, _ = json.Marshal(EthPitFileDebtCeilingLog)
var PitFileDebtCeilingModel = debt_ceiling.PitFileDebtCeilingModel{
What: "Line",
Data: big.NewInt(123456).String(),
Data: shared.ConvertToWad(big.NewInt(123456).String()),
LogIndex: EthPitFileDebtCeilingLog.Index,
TransactionIndex: EthPitFileDebtCeilingLog.TxIndex,
Raw: rawPitFileDebtCeilingLog,
}
var EthPitFileIlkLog = types.Log{
var EthPitFileIlkLineLog = types.Log{
Address: common.HexToAddress(constants.PitContractAddress),
Topics: []common.Hash{
common.HexToHash("0x1a0b287e00000000000000000000000000000000000000000000000000000000"),
common.HexToHash("0x0000000000000000000000000f243e26db94b5426032e6dfa6007802dea2a614"),
common.HexToHash("0x00000000000000000000000064d922894153be9eef7b7218dc565d1d0ce2a092"),
common.HexToHash("0x66616b6520696c6b000000000000000000000000000000000000000000000000"),
common.HexToHash("0x6c696e6500000000000000000000000000000000000000000000000000000000"),
},
Data: hexutil.MustDecode("0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000641a0b287e66616b6520696c6b0000000000000000000000000000000000000000000000006c696e6500000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e8d4a51000"),
BlockNumber: 12,
TxHash: common.HexToHash("0x2e27c962a697d4f7ec5d3206d0c058bd510f7593a711f082e55f3b62d44d8dd9"),
TxIndex: 112,
BlockHash: fakes.FakeHash,
Index: 15,
Removed: false,
}
var rawPitFileIlkLineLog, _ = json.Marshal(EthPitFileIlkLineLog)
var PitFileIlkLineModel = ilk2.PitFileIlkModel{
Ilk: "fake ilk",
What: "line",
Data: "0.000001000000000000",
LogIndex: EthPitFileIlkLineLog.Index,
TransactionIndex: EthPitFileIlkLineLog.TxIndex,
Raw: rawPitFileIlkLineLog,
}
var EthPitFileIlkSpotLog = types.Log{
Address: common.HexToAddress(constants.PitContractAddress),
Topics: []common.Hash{
common.HexToHash("0x1a0b287e00000000000000000000000000000000000000000000000000000000"),
common.HexToHash("0x00000000000000000000000064d922894153be9eef7b7218dc565d1d0ce2a092"),
common.HexToHash("0x66616b6520696c6b000000000000000000000000000000000000000000000000"),
common.HexToHash("0x73706f7400000000000000000000000000000000000000000000000000000000"),
},
Data: hexutil.MustDecode("0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000641a0b287e66616b6520696c6b00000000000000000000000000000000000000000000000073706f7400000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007b"),
Data: hexutil.MustDecode("0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000641a0b287e66616b6520696c6b00000000000000000000000000000000000000000000000073706f7400000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e8d4a51000"),
BlockNumber: 11,
TxHash: common.HexToHash("0x1ba8125f60fa045c85b35df3983bee37db8627fbc32e3442a5cf17c85bb83f09"),
TxIndex: 111,
@ -71,12 +99,12 @@ var EthPitFileIlkLog = types.Log{
Removed: false,
}
var rawPitFileIlkLog, _ = json.Marshal(EthPitFileIlkLog)
var PitFileIlkModel = ilk2.PitFileIlkModel{
var rawPitFileIlkSpotLog, _ = json.Marshal(EthPitFileIlkSpotLog)
var PitFileIlkSpotModel = ilk2.PitFileIlkModel{
Ilk: "fake ilk",
What: "spot",
Data: big.NewInt(123).String(),
LogIndex: EthPitFileIlkLog.Index,
TransactionIndex: EthPitFileIlkLog.TxIndex,
Raw: rawPitFileIlkLog,
Data: "0.000000000000001000000000000",
LogIndex: EthPitFileIlkSpotLog.Index,
TransactionIndex: EthPitFileIlkSpotLog.TxIndex,
Raw: rawPitFileIlkSpotLog,
}

2
pkg/transformers/test_data/vat_fold.go
View File

@ -47,7 +47,7 @@ var rawVatFoldLog, _ = json.Marshal(EthVatFoldLog)
var VatFoldModel = vat_fold.VatFoldModel{
Ilk: "REP",
Urn: "0x3728e9777B2a0a611ee0F89e00E01044ce4736d1",
Rate: "2",
Rate: "0.000000000000000000000000002",
LogIndex: EthVatFoldLog.Index,
TransactionIndex: EthVatFoldLog.TxIndex,
Raw: rawVatFoldLog,

3
pkg/transformers/vat_fold/converter.go
View File

@ -20,6 +20,7 @@ import (
"errors"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
)
@ -35,7 +36,7 @@ func (VatFoldConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) {
ilk := string(bytes.Trim(ethLog.Topics[1].Bytes(), "\x00"))
urn := common.BytesToAddress(ethLog.Topics[2].Bytes()).String()
rate := ethLog.Topics[3].Big().String()
rate := shared.ConvertToRay(ethLog.Topics[3].Big().String())
raw, err := json.Marshal(ethLog)
if err != nil {