Convert and persist LogNote events as dent records
This commit is contained in:
Elizabeth
2018-09-11 14:01:45 -05:00
committed by GitHub
parent d4a4748246
commit 9abe3ffa68
29 changed files with 1116 additions and 49 deletions
+25
View File
@@ -0,0 +1,25 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dent
import "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
var DentConfig = shared.TransformerConfig{
ContractAddress: shared.FlipperContractAddress,
ContractAbi: shared.FlipperABI,
Topics: []string{shared.DentFunctionSignature},
StartingBlockNumber: 0,
EndingBlockNumber: 100,
}
+87
View File
@@ -0,0 +1,87 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dent
import (
"encoding/json"
"errors"
"math/big"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
)
type Converter interface {
Convert(contractAddress string, contractAbi string, ethLog types.Log) (DentModel, error)
}
type DentConverter struct{}
func NewDentConverter() DentConverter {
return DentConverter{}
}
func (c DentConverter) Convert(contractAddress, contractAbi string, ethLog types.Log) (DentModel, error) {
err := validateLog(ethLog)
if err != nil {
return DentModel{}, err
}
bidId := ethLog.Topics[2].Big()
lot := ethLog.Topics[3].Big().String()
bidValue := getBidValue(ethLog)
guy := common.HexToAddress(ethLog.Topics[1].Hex()).String()
tic := "0"
//TODO: it is likely that the tic value will need to be added to an emitted event,
//so this will need to be updated at that point
transactionIndex := ethLog.TxIndex
raw, err := json.Marshal(ethLog)
if err != nil {
return DentModel{}, err
}
return DentModel{
BidId: bidId.String(),
Lot: lot,
Bid: bidValue,
Guy: guy,
Tic: tic,
TransactionIndex: transactionIndex,
Raw: raw,
}, nil
}
func validateLog(ethLog types.Log) error {
if len(ethLog.Data) <= 0 {
return errors.New("dent log data is empty")
}
if len(ethLog.Topics) < 4 {
return errors.New("dent log does not contain expected topics")
}
return nil
}
func getBidValue(ethLog types.Log) string {
itemByteLength := 32
lastDataItemStartIndex := len(ethLog.Data) - itemByteLength
lastItem := ethLog.Data[lastDataItemStartIndex:]
lastValue := big.NewInt(0).SetBytes(lastItem)
return lastValue.String()
}
+58
View File
@@ -0,0 +1,58 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dent_test
import (
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/transformers/dent"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
)
var _ = Describe("Dent Converter", func() {
var converter dent.DentConverter
BeforeEach(func() {
converter = dent.NewDentConverter()
})
It("converts an eth log to a db model", func() {
model, err := converter.Convert(shared.FlipperContractAddress, shared.FlipperABI, test_data.DentLog)
Expect(err).NotTo(HaveOccurred())
Expect(model).To(Equal(test_data.DentModel))
})
It("returns an error if the expected amount of topics aren't in the log", func() {
invalidLog := test_data.DentLog
invalidLog.Topics = []common.Hash{}
model, err := converter.Convert(shared.FlipperContractAddress, shared.FlipperABI, invalidLog)
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError("dent log does not contain expected topics"))
Expect(model).To(Equal(dent.DentModel{}))
})
It("returns an error if the log data is empty", func() {
emptyDataLog := test_data.DentLog
emptyDataLog.Data = []byte{}
model, err := converter.Convert(shared.FlipperContractAddress, shared.FlipperABI, emptyDataLog)
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError("dent log data is empty"))
Expect(model).To(Equal(dent.DentModel{}))
})
})
+19
View File
@@ -0,0 +1,19 @@
package dent_test
import (
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"io/ioutil"
"log"
)
func TestDent(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Dent Suite")
}
var _ = BeforeSuite(func() {
log.SetOutput(ioutil.Discard)
})
+25
View File
@@ -0,0 +1,25 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dent
type DentModel struct {
BidId string `db:"bid_id"`
Lot string
Bid string
Guy string
Tic string
TransactionIndex uint `db:"tx_idx"`
Raw []byte `db:"raw_log"`
}
+62
View File
@@ -0,0 +1,62 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dent
import (
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
type Repository interface {
Create(headerId int64, model DentModel) error
MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
}
type DentRepository struct {
db *postgres.DB
}
func NewDentRepository(database *postgres.DB) DentRepository {
return DentRepository{db: database}
}
func (r DentRepository) Create(headerId int64, model DentModel) error {
_, err := r.db.Exec(
`INSERT into maker.dent (header_id, bid_id, lot, bid, guy, tic, tx_idx, raw_log)
VALUES($1, $2, $3, $4, $5, $6, $7, $8)`,
headerId, model.BidId, model.Lot, model.Bid, model.Guy, model.Tic, model.TransactionIndex, model.Raw,
)
return err
}
func (r DentRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
var missingHeaders []core.Header
err := r.db.Select(
&missingHeaders,
`SELECT headers.id, headers.block_number FROM headers
LEFT JOIN maker.dent on headers.id = header_id
WHERE header_id ISNULL
AND headers.block_number >= $1
AND headers.block_number <= $2
AND headers.eth_node_fingerprint = $3`,
startingBlockNumber,
endingBlockNumber,
r.db.Node.ID,
)
return missingHeaders, err
}
+153
View File
@@ -0,0 +1,153 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dent_test
import (
"math/rand"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/dent"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
"github.com/vulcanize/vulcanizedb/test_config"
)
var _ = Describe("Dent Repository", func() {
var node core.Node
var db *postgres.DB
var dentRepository dent.DentRepository
var headerRepository repositories.HeaderRepository
var headerId int64
var err error
BeforeEach(func() {
node = test_config.NewTestNode()
db = test_config.NewTestDB(node)
test_config.CleanTestDB(db)
dentRepository = dent.NewDentRepository(db)
headerRepository = repositories.NewHeaderRepository(db)
})
Describe("Create", func() {
BeforeEach(func() {
headerId, err = headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
err := dentRepository.Create(headerId, test_data.DentModel)
Expect(err).NotTo(HaveOccurred())
})
It("persists a dent record", func() {
var count int
db.QueryRow(`SELECT count(*) FROM maker.dent`).Scan(&count)
Expect(count).To(Equal(1))
var dbResult dent.DentModel
err = db.Get(&dbResult, `SELECT bid_id, lot, bid, guy, tic, tx_idx, raw_log FROM maker.dent WHERE header_id = $1`, headerId)
Expect(err).NotTo(HaveOccurred())
Expect(dbResult.BidId).To(Equal(test_data.DentModel.BidId))
Expect(dbResult.Lot).To(Equal(test_data.DentModel.Lot))
Expect(dbResult.Bid).To(Equal(test_data.DentModel.Bid))
Expect(dbResult.Guy).To(Equal(test_data.DentModel.Guy))
Expect(dbResult.Tic).To(Equal(test_data.DentModel.Tic))
Expect(dbResult.TransactionIndex).To(Equal(test_data.DentModel.TransactionIndex))
Expect(dbResult.Raw).To(MatchJSON(test_data.DentModel.Raw))
})
It("returns an error if inserting a dent record fails", func() {
err = dentRepository.Create(headerId, test_data.DentModel)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
})
It("deletes the tend record if its corresponding header record is deleted", func() {
var count int
err = db.QueryRow(`SELECT count(*) from maker.dent`).Scan(&count)
Expect(err).NotTo(HaveOccurred())
Expect(count).To(Equal(1))
_, err = db.Exec(`DELETE FROM headers where id = $1`, headerId)
Expect(err).NotTo(HaveOccurred())
err = db.QueryRow(`SELECT count(*) from maker.dent`).Scan(&count)
Expect(err).NotTo(HaveOccurred())
Expect(count).To(Equal(0))
})
})
Describe("MissingHeaders", func() {
var dentBlockNumber int64
var startingBlockNumber int64
var endingBlockNumber int64
var blockNumbers []int64
BeforeEach(func() {
dentBlockNumber = rand.Int63()
startingBlockNumber = dentBlockNumber - 1
endingBlockNumber = dentBlockNumber + 1
outOfRangeBlockNumber := dentBlockNumber + 2
blockNumbers = []int64{startingBlockNumber, dentBlockNumber, endingBlockNumber, outOfRangeBlockNumber}
var headerIds []int64
for _, number := range blockNumbers {
headerId, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: number})
Expect(err).NotTo(HaveOccurred())
headerIds = append(headerIds, headerId)
}
dentRepository.Create(headerIds[1], test_data.DentModel)
})
It("returns header records that don't have a corresponding dents", func() {
missingHeaders, err := dentRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
Expect(err).NotTo(HaveOccurred())
Expect(len(missingHeaders)).To(Equal(2))
Expect(missingHeaders[0].BlockNumber).To(Equal(startingBlockNumber))
Expect(missingHeaders[1].BlockNumber).To(Equal(endingBlockNumber))
})
It("only returns missing headers for the given node", func() {
node2 := core.Node{}
db2 := test_config.NewTestDB(node2)
dentRepository2 := dent.NewDentRepository(db2)
headerRepository2 := repositories.NewHeaderRepository(db2)
var node2HeaderIds []int64
for _, number := range blockNumbers {
id, err := headerRepository2.CreateOrUpdateHeader(core.Header{BlockNumber: number})
node2HeaderIds = append(node2HeaderIds, id)
Expect(err).NotTo(HaveOccurred())
}
missingHeadersNode1, err := dentRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
Expect(err).NotTo(HaveOccurred())
Expect(len(missingHeadersNode1)).To(Equal(2))
Expect(missingHeadersNode1[0].BlockNumber).To(Equal(startingBlockNumber))
Expect(missingHeadersNode1[1].BlockNumber).To(Equal(endingBlockNumber))
missingHeadersNode2, err := dentRepository2.MissingHeaders(startingBlockNumber, endingBlockNumber)
Expect(err).NotTo(HaveOccurred())
Expect(len(missingHeadersNode2)).To(Equal(3))
Expect(missingHeadersNode2[0].BlockNumber).To(Equal(startingBlockNumber))
Expect(missingHeadersNode2[1].BlockNumber).To(Equal(dentBlockNumber))
Expect(missingHeadersNode2[2].BlockNumber).To(Equal(endingBlockNumber))
})
})
})
+80
View File
@@ -0,0 +1,80 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dent
import (
"log"
"github.com/ethereum/go-ethereum/common"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
)
type DentTransformer struct {
Config shared.TransformerConfig
Converter Converter
Fetcher shared.LogFetcher
Repository Repository
}
type DentTransformerInitializer struct {
Config shared.TransformerConfig
}
func (i DentTransformerInitializer) NewDentTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer {
converter := NewDentConverter()
fetcher := shared.NewFetcher(blockChain)
repository := NewDentRepository(db)
return DentTransformer{
Config: i.Config,
Converter: converter,
Fetcher: fetcher,
Repository: repository,
}
}
func (t DentTransformer) Execute() error {
config := t.Config
topics := [][]common.Hash{{common.HexToHash(shared.DentFunctionSignature)}}
headers, err := t.Repository.MissingHeaders(config.StartingBlockNumber, config.EndingBlockNumber)
for _, header := range headers {
ethLogs, err := t.Fetcher.FetchLogs(config.ContractAddress, topics, header.BlockNumber)
if err != nil {
log.Println("Error fetching dent logs:", err)
return err
}
for _, ethLog := range ethLogs {
model, err := t.Converter.Convert(config.ContractAddress, config.ContractAbi, ethLog)
if err != nil {
log.Println("Error converting dent log", err)
return err
}
err = t.Repository.Create(header.Id, model)
if err != nil {
log.Println("Error persisting dent record", err)
return err
}
}
}
return err
}
+131
View File
@@ -0,0 +1,131 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dent_test
import (
"math/rand"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/transformers/dent"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks"
dent_mocks "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks/dent"
)
var _ = Describe("DentTransformer", func() {
var config = dent.DentConfig
var dentRepository dent_mocks.MockDentRepository
var fetcher mocks.MockLogFetcher
var converter dent_mocks.MockDentConverter
var transformer dent.DentTransformer
BeforeEach(func() {
dentRepository = dent_mocks.MockDentRepository{}
fetcher = mocks.MockLogFetcher{}
converter = dent_mocks.MockDentConverter{}
transformer = dent.DentTransformer{
Repository: &dentRepository,
Config: config,
Fetcher: &fetcher,
Converter: &converter,
}
})
It("gets missing headers", func() {
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(dentRepository.PassedStartingBlockNumber).To(Equal(config.StartingBlockNumber))
Expect(dentRepository.PassedEndingBlockNumber).To(Equal(config.EndingBlockNumber))
})
It("returns an error if fetching the missing headers fails", func() {
dentRepository.SetMissingHeadersError(fakes.FakeError)
err := transformer.Execute()
Expect(err).To(HaveOccurred())
})
It("fetches logs for each missing header", func() {
header1 := core.Header{BlockNumber: rand.Int63()}
header2 := core.Header{BlockNumber: rand.Int63()}
dentRepository.SetMissingHeaders([]core.Header{header1, header2})
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(fetcher.FetchedContractAddress).To(Equal(config.ContractAddress))
expectedTopics := [][]common.Hash{{common.HexToHash(shared.DentFunctionSignature)}}
Expect(fetcher.FetchedTopics).To(Equal(expectedTopics))
Expect(fetcher.FetchedBlocks).To(Equal([]int64{header1.BlockNumber, header2.BlockNumber}))
})
It("returns an error if fetching logs fails", func() {
dentRepository.SetMissingHeaders([]core.Header{{}})
fetcher.SetFetcherError(fakes.FakeError)
err := transformer.Execute()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("converts each eth log to a Model", func() {
dentRepository.SetMissingHeaders([]core.Header{{}})
fetcher.SetFetchedLogs([]types.Log{test_data.DentLog})
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(converter.PassedContractAddress).To(Equal(config.ContractAddress))
Expect(converter.PassedContractAbi).To(Equal(config.ContractAbi))
Expect(converter.LogsToConvert).To(Equal([]types.Log{test_data.DentLog}))
})
It("returns an error if converting the eth log fails", func() {
dentRepository.SetMissingHeaders([]core.Header{{}})
fetcher.SetFetchedLogs([]types.Log{test_data.DentLog})
converter.SetConverterError(fakes.FakeError)
err := transformer.Execute()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("persists each model as a Dent record", func() {
header1 := core.Header{Id: rand.Int63()}
header2 := core.Header{Id: rand.Int63()}
dentRepository.SetMissingHeaders([]core.Header{header1, header2})
fetcher.SetFetchedLogs([]types.Log{test_data.DentLog})
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(dentRepository.PassedDentModels).To(Equal([]dent.DentModel{test_data.DentModel, test_data.DentModel}))
Expect(dentRepository.PassedHeaderIds).To(Equal([]int64{header1.Id, header2.Id}))
})
It("returns an error if persisting dent record fails", func() {
dentRepository.SetMissingHeaders([]core.Header{{}})
dentRepository.SetCreateError(fakes.FakeError)
fetcher.SetFetchedLogs([]types.Log{test_data.DentLog})
err := transformer.Execute()
Expect(err).To(HaveOccurred())
})
})