Add Drip drip transformer

This commit is contained in:
Rob Mulholand
2018-09-13 16:35:10 -05:00
parent 560305f601
commit f5595bee78
17 changed files with 830 additions and 2 deletions
+25
View File
@@ -0,0 +1,25 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package drip_drip
import "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
var DripDripConfig = shared.TransformerConfig{
ContractAddress: shared.DripContractAddress,
ContractAbi: shared.DripABI,
Topics: []string{shared.DripDripSignature},
StartingBlockNumber: 0,
EndingBlockNumber: 100,
}
+49
View File
@@ -0,0 +1,49 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package drip_drip
import (
"bytes"
"encoding/json"
"errors"
"github.com/ethereum/go-ethereum/core/types"
)
type Converter interface {
ToModel(ethLog types.Log) (DripDripModel, error)
}
type DripDripConverter struct{}
func (DripDripConverter) ToModel(ethLog types.Log) (DripDripModel, error) {
err := verifyLog(ethLog)
if err != nil {
return DripDripModel{}, err
}
ilk := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
raw, err := json.Marshal(ethLog)
return DripDripModel{
Ilk: ilk,
TransactionIndex: ethLog.TxIndex,
Raw: raw,
}, err
}
func verifyLog(log types.Log) error {
if len(log.Topics) < 3 {
return errors.New("log missing topics")
}
return nil
}
@@ -0,0 +1,44 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package drip_drip_test
import (
"github.com/ethereum/go-ethereum/core/types"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/transformers/drip_drip"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
)
var _ = Describe("Drip drip converter", func() {
It("returns err if log is missing topics", func() {
converter := drip_drip.DripDripConverter{}
badLog := types.Log{}
_, err := converter.ToModel(badLog)
Expect(err).To(HaveOccurred())
})
It("converts a log to an model", func() {
converter := drip_drip.DripDripConverter{}
model, err := converter.ToModel(test_data.EthDripDripLog)
Expect(err).NotTo(HaveOccurred())
Expect(model).To(Equal(test_data.DripDripModel))
})
})
@@ -0,0 +1,27 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package drip_drip_test
import (
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
func TestDripDrip(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "DripDrip Suite")
}
+21
View File
@@ -0,0 +1,21 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package drip_drip
type DripDripModel struct {
Ilk string
TransactionIndex uint `db:"tx_idx"`
Raw []byte `db:"raw_log"`
}
+59
View File
@@ -0,0 +1,59 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package drip_drip
import (
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
type Repository interface {
Create(headerID int64, model DripDripModel) error
MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
}
type DripDripRepository struct {
db *postgres.DB
}
func NewDripDripRepository(db *postgres.DB) DripDripRepository {
return DripDripRepository{db: db}
}
func (repository DripDripRepository) Create(headerID int64, model DripDripModel) error {
_, err := repository.db.Exec(
`INSERT into maker.drip_drip (header_id, ilk, tx_idx, raw_log)
VALUES($1, $2, $3, $4)`,
headerID, model.Ilk, model.TransactionIndex, model.Raw,
)
return err
}
func (repository DripDripRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
var result []core.Header
err := repository.db.Select(
&result,
`SELECT headers.id, headers.block_number FROM headers
LEFT JOIN maker.drip_drip on headers.id = header_id
WHERE header_id ISNULL
AND headers.block_number >= $1
AND headers.block_number <= $2
AND headers.eth_node_fingerprint = $3`,
startingBlockNumber,
endingBlockNumber,
repository.db.Node.ID,
)
return result, err
}
@@ -0,0 +1,143 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package drip_drip_test
import (
"database/sql"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/drip_drip"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
"github.com/vulcanize/vulcanizedb/test_config"
)
var _ = Describe("Drip drip repository", func() {
Describe("Create", func() {
It("adds a drip drip event", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
dripDripRepository := drip_drip.NewDripDripRepository(db)
err = dripDripRepository.Create(headerID, test_data.DripDripModel)
Expect(err).NotTo(HaveOccurred())
var dbPitFile drip_drip.DripDripModel
err = db.Get(&dbPitFile, `SELECT ilk, tx_idx, raw_log FROM maker.drip_drip WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(dbPitFile.Ilk).To(Equal(test_data.DripDripModel.Ilk))
Expect(dbPitFile.TransactionIndex).To(Equal(test_data.DripDripModel.TransactionIndex))
Expect(dbPitFile.Raw).To(MatchJSON(test_data.DripDripModel.Raw))
})
It("does not duplicate drip drip events", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
dripDripRepository := drip_drip.NewDripDripRepository(db)
err = dripDripRepository.Create(headerID, test_data.DripDripModel)
Expect(err).NotTo(HaveOccurred())
err = dripDripRepository.Create(headerID, test_data.DripDripModel)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
})
It("removes drip drip if corresponding header is deleted", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
dripDripRepository := drip_drip.NewDripDripRepository(db)
err = dripDripRepository.Create(headerID, test_data.DripDripModel)
Expect(err).NotTo(HaveOccurred())
_, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
var dbPitFile drip_drip.DripDripModel
err = db.Get(&dbPitFile, `SELECT ilk, tx_idx, raw_log FROM maker.drip_drip WHERE header_id = $1`, headerID)
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(sql.ErrNoRows))
})
})
Describe("MissingHeaders", func() {
It("returns headers with no associated drip drip event", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
headerRepository := repositories.NewHeaderRepository(db)
startingBlockNumber := int64(1)
dripDripBlockNumber := int64(2)
endingBlockNumber := int64(3)
blockNumbers := []int64{startingBlockNumber, dripDripBlockNumber, endingBlockNumber, endingBlockNumber + 1}
var headerIDs []int64
for _, n := range blockNumbers {
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
headerIDs = append(headerIDs, headerID)
Expect(err).NotTo(HaveOccurred())
}
dripDripRepository := drip_drip.NewDripDripRepository(db)
err := dripDripRepository.Create(headerIDs[1], test_data.DripDripModel)
Expect(err).NotTo(HaveOccurred())
headers, err := dripDripRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
Expect(err).NotTo(HaveOccurred())
Expect(len(headers)).To(Equal(2))
Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber)))
})
It("only returns headers associated with the current node", func() {
db := test_config.NewTestDB(core.Node{})
test_config.CleanTestDB(db)
blockNumbers := []int64{1, 2, 3}
headerRepository := repositories.NewHeaderRepository(db)
dbTwo := test_config.NewTestDB(core.Node{ID: "second"})
headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
var headerIDs []int64
for _, n := range blockNumbers {
headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n})
Expect(err).NotTo(HaveOccurred())
headerIDs = append(headerIDs, headerID)
_, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n})
Expect(err).NotTo(HaveOccurred())
}
dripDripRepository := drip_drip.NewDripDripRepository(db)
dripDripRepositoryTwo := drip_drip.NewDripDripRepository(dbTwo)
err := dripDripRepository.Create(headerIDs[0], test_data.DripDripModel)
Expect(err).NotTo(HaveOccurred())
nodeOneMissingHeaders, err := dripDripRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])
Expect(err).NotTo(HaveOccurred())
Expect(len(nodeOneMissingHeaders)).To(Equal(len(blockNumbers) - 1))
nodeTwoMissingHeaders, err := dripDripRepositoryTwo.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1])
Expect(err).NotTo(HaveOccurred())
Expect(len(nodeTwoMissingHeaders)).To(Equal(len(blockNumbers)))
})
})
})
+70
View File
@@ -0,0 +1,70 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package drip_drip
import (
"github.com/ethereum/go-ethereum/common"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
)
type DripDripTransformerInitializer struct {
Config shared.TransformerConfig
}
func (initializer DripDripTransformerInitializer) NewDripDripTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer {
converter := DripDripConverter{}
fetcher := shared.NewFetcher(blockChain)
repository := NewDripDripRepository(db)
return DripDripTransformer{
Config: initializer.Config,
Converter: converter,
Fetcher: fetcher,
Repository: repository,
}
}
type DripDripTransformer struct {
Config shared.TransformerConfig
Converter Converter
Fetcher shared.LogFetcher
Repository Repository
}
func (transformer DripDripTransformer) Execute() error {
missingHeaders, err := transformer.Repository.MissingHeaders(transformer.Config.StartingBlockNumber, transformer.Config.EndingBlockNumber)
if err != nil {
return err
}
for _, header := range missingHeaders {
topics := [][]common.Hash{{common.HexToHash(shared.DripDripSignature)}}
matchingLogs, err := transformer.Fetcher.FetchLogs(transformer.Config.ContractAddress, topics, header.BlockNumber)
if err != nil {
return err
}
for _, log := range matchingLogs {
model, err := transformer.Converter.ToModel(log)
if err != nil {
return err
}
err = transformer.Repository.Create(header.Id, model)
if err != nil {
return err
}
}
}
return nil
}
@@ -0,0 +1,175 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package drip_drip_test
import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/transformers/drip_drip"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks"
drip_drip_mocks "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks/drip_drip"
)
var _ = Describe("Drip drip transformer", func() {
It("gets missing headers for block numbers specified in config", func() {
repository := &drip_drip_mocks.MockDripDripRepository{}
transformer := drip_drip.DripDripTransformer{
Config: drip_drip.DripDripConfig,
Fetcher: &mocks.MockLogFetcher{},
Converter: &drip_drip_mocks.MockDripDripConverter{},
Repository: repository,
}
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(repository.PassedStartingBlockNumber).To(Equal(drip_drip.DripDripConfig.StartingBlockNumber))
Expect(repository.PassedEndingBlockNumber).To(Equal(drip_drip.DripDripConfig.EndingBlockNumber))
})
It("returns error if repository returns error for missing headers", func() {
repository := &drip_drip_mocks.MockDripDripRepository{}
repository.SetMissingHeadersErr(fakes.FakeError)
transformer := drip_drip.DripDripTransformer{
Fetcher: &mocks.MockLogFetcher{},
Converter: &drip_drip_mocks.MockDripDripConverter{},
Repository: repository,
}
err := transformer.Execute()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("fetches logs for missing headers", func() {
fetcher := &mocks.MockLogFetcher{}
repository := &drip_drip_mocks.MockDripDripRepository{}
repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}, {BlockNumber: 2}})
transformer := drip_drip.DripDripTransformer{
Config: drip_drip.DripDripConfig,
Fetcher: fetcher,
Converter: &drip_drip_mocks.MockDripDripConverter{},
Repository: repository,
}
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(fetcher.FetchedBlocks).To(Equal([]int64{1, 2}))
Expect(fetcher.FetchedContractAddress).To(Equal(drip_drip.DripDripConfig.ContractAddress))
Expect(fetcher.FetchedTopics).To(Equal([][]common.Hash{{common.HexToHash(shared.DripDripSignature)}}))
})
It("returns error if fetcher returns error", func() {
fetcher := &mocks.MockLogFetcher{}
fetcher.SetFetcherError(fakes.FakeError)
repository := &drip_drip_mocks.MockDripDripRepository{}
repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}})
transformer := drip_drip.DripDripTransformer{
Fetcher: fetcher,
Converter: &drip_drip_mocks.MockDripDripConverter{},
Repository: repository,
}
err := transformer.Execute()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("converts matching logs", func() {
converter := &drip_drip_mocks.MockDripDripConverter{}
fetcher := &mocks.MockLogFetcher{}
fetcher.SetFetchedLogs([]types.Log{test_data.EthDripDripLog})
repository := &drip_drip_mocks.MockDripDripRepository{}
repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}})
transformer := drip_drip.DripDripTransformer{
Fetcher: fetcher,
Converter: converter,
Repository: repository,
}
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(converter.PassedLog).To(Equal(test_data.EthDripDripLog))
})
It("returns error if converter returns error", func() {
converter := &drip_drip_mocks.MockDripDripConverter{}
converter.SetConverterError(fakes.FakeError)
fetcher := &mocks.MockLogFetcher{}
fetcher.SetFetchedLogs([]types.Log{test_data.EthDripDripLog})
repository := &drip_drip_mocks.MockDripDripRepository{}
repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}})
transformer := drip_drip.DripDripTransformer{
Fetcher: fetcher,
Converter: converter,
Repository: repository,
}
err := transformer.Execute()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("persists drip drip model", func() {
converter := &drip_drip_mocks.MockDripDripConverter{}
fetcher := &mocks.MockLogFetcher{}
fetcher.SetFetchedLogs([]types.Log{test_data.EthDripDripLog})
repository := &drip_drip_mocks.MockDripDripRepository{}
fakeHeader := core.Header{BlockNumber: 1, Id: 2}
repository.SetMissingHeaders([]core.Header{fakeHeader})
transformer := drip_drip.DripDripTransformer{
Fetcher: fetcher,
Converter: converter,
Repository: repository,
}
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id))
Expect(repository.PassedModel).To(Equal(test_data.DripDripModel))
})
It("returns error if repository returns error for create", func() {
converter := &drip_drip_mocks.MockDripDripConverter{}
fetcher := &mocks.MockLogFetcher{}
fetcher.SetFetchedLogs([]types.Log{test_data.EthDripDripLog})
repository := &drip_drip_mocks.MockDripDripRepository{}
repository.SetMissingHeaders([]core.Header{{BlockNumber: 1, Id: 2}})
repository.SetCreateError(fakes.FakeError)
transformer := drip_drip.DripDripTransformer{
Fetcher: fetcher,
Converter: converter,
Repository: repository,
}
err := transformer.Execute()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
})