Convert and persist LogNote events as dent records
This commit is contained in:
Elizabeth 2018-09-11 14:01:45 -05:00 committed by GitHub
parent d4a4748246
commit 9abe3ffa68
29 changed files with 1116 additions and 49 deletions

View File

@ -0,0 +1 @@
DROP TABLE maker.dent;

View File

@ -0,0 +1,11 @@
CREATE TABLE maker.dent (
db_id SERIAL PRIMARY KEY,
header_id INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE,
bid_id NUMERIC NOT NULL UNIQUE,
lot NUMERIC,
bid NUMERIC,
guy BYTEA,
tic NUMERIC,
tx_idx INTEGER NOT NUll,
raw_log JSONB
);

View File

@ -79,6 +79,43 @@ CREATE SEQUENCE maker.bite_id_seq
ALTER SEQUENCE maker.bite_id_seq OWNED BY maker.bite.id;
--
-- Name: dent; Type: TABLE; Schema: maker; Owner: -
--
CREATE TABLE maker.dent (
db_id integer NOT NULL,
header_id integer NOT NULL,
bid_id numeric NOT NULL,
lot numeric,
bid numeric,
guy bytea,
tic numeric,
tx_idx integer NOT NULL,
raw_log jsonb
);
--
-- Name: dent_db_id_seq; Type: SEQUENCE; Schema: maker; Owner: -
--
CREATE SEQUENCE maker.dent_db_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: dent_db_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: -
--
ALTER SEQUENCE maker.dent_db_id_seq OWNED BY maker.dent.db_id;
--
-- Name: flip_kick; Type: TABLE; Schema: maker; Owner: -
--
@ -709,6 +746,13 @@ CREATE VIEW public.watched_event_logs AS
ALTER TABLE ONLY maker.bite ALTER COLUMN id SET DEFAULT nextval('maker.bite_id_seq'::regclass);
--
-- Name: dent db_id; Type: DEFAULT; Schema: maker; Owner: -
--
ALTER TABLE ONLY maker.dent ALTER COLUMN db_id SET DEFAULT nextval('maker.dent_db_id_seq'::regclass);
--
-- Name: flip_kick db_id; Type: DEFAULT; Schema: maker; Owner: -
--
@ -837,6 +881,22 @@ ALTER TABLE ONLY maker.bite
ADD CONSTRAINT bite_pkey PRIMARY KEY (id);
--
-- Name: dent dent_bid_id_key; Type: CONSTRAINT; Schema: maker; Owner: -
--
ALTER TABLE ONLY maker.dent
ADD CONSTRAINT dent_bid_id_key UNIQUE (bid_id);
--
-- Name: dent dent_pkey; Type: CONSTRAINT; Schema: maker; Owner: -
--
ALTER TABLE ONLY maker.dent
ADD CONSTRAINT dent_pkey PRIMARY KEY (db_id);
--
-- Name: flip_kick flip_kick_id_key; Type: CONSTRAINT; Schema: maker; Owner: -
--
@ -1088,6 +1148,14 @@ ALTER TABLE ONLY maker.bite
ADD CONSTRAINT bite_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE;
--
-- Name: dent dent_header_id_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: -
--
ALTER TABLE ONLY maker.dent
ADD CONSTRAINT dent_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE;
--
-- Name: flip_kick flip_kick_header_id_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: -
--

View File

@ -22,7 +22,7 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/geth"
"github.com/vulcanize/vulcanizedb/pkg/transformers/utilities"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
)
type Converter interface {
@ -70,14 +70,14 @@ func (converter BiteConverter) ToModel(entity BiteEntity) (BiteModel, error) {
}
return BiteModel{
Id: utilities.ConvertNilToEmptyString(id),
Id: shared.ConvertNilToEmptyString(id),
Ilk: ilk,
Lad: lad,
Ink: utilities.ConvertNilToEmptyString(ink),
Art: utilities.ConvertNilToEmptyString(art),
IArt: utilities.ConvertNilToEmptyString(iArt),
Tab: utilities.ConvertNilToEmptyString(tab),
Flip: utilities.ConvertNilToEmptyString(flip),
Ink: shared.ConvertNilToEmptyString(ink),
Art: shared.ConvertNilToEmptyString(art),
IArt: shared.ConvertNilToEmptyString(iArt),
Tab: shared.ConvertNilToEmptyString(tab),
Flip: shared.ConvertNilToEmptyString(flip),
TransactionIndex: txIdx,
Raw: rawLogString,
}, nil

View File

@ -0,0 +1,25 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dent
import "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
var DentConfig = shared.TransformerConfig{
ContractAddress: shared.FlipperContractAddress,
ContractAbi: shared.FlipperABI,
Topics: []string{shared.DentFunctionSignature},
StartingBlockNumber: 0,
EndingBlockNumber: 100,
}

View File

@ -0,0 +1,87 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dent
import (
"encoding/json"
"errors"
"math/big"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
)
type Converter interface {
Convert(contractAddress string, contractAbi string, ethLog types.Log) (DentModel, error)
}
type DentConverter struct{}
func NewDentConverter() DentConverter {
return DentConverter{}
}
func (c DentConverter) Convert(contractAddress, contractAbi string, ethLog types.Log) (DentModel, error) {
err := validateLog(ethLog)
if err != nil {
return DentModel{}, err
}
bidId := ethLog.Topics[2].Big()
lot := ethLog.Topics[3].Big().String()
bidValue := getBidValue(ethLog)
guy := common.HexToAddress(ethLog.Topics[1].Hex()).String()
tic := "0"
//TODO: it is likely that the tic value will need to be added to an emitted event,
//so this will need to be updated at that point
transactionIndex := ethLog.TxIndex
raw, err := json.Marshal(ethLog)
if err != nil {
return DentModel{}, err
}
return DentModel{
BidId: bidId.String(),
Lot: lot,
Bid: bidValue,
Guy: guy,
Tic: tic,
TransactionIndex: transactionIndex,
Raw: raw,
}, nil
}
func validateLog(ethLog types.Log) error {
if len(ethLog.Data) <= 0 {
return errors.New("dent log data is empty")
}
if len(ethLog.Topics) < 4 {
return errors.New("dent log does not contain expected topics")
}
return nil
}
func getBidValue(ethLog types.Log) string {
itemByteLength := 32
lastDataItemStartIndex := len(ethLog.Data) - itemByteLength
lastItem := ethLog.Data[lastDataItemStartIndex:]
lastValue := big.NewInt(0).SetBytes(lastItem)
return lastValue.String()
}

View File

@ -0,0 +1,58 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dent_test
import (
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/transformers/dent"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
)
var _ = Describe("Dent Converter", func() {
var converter dent.DentConverter
BeforeEach(func() {
converter = dent.NewDentConverter()
})
It("converts an eth log to a db model", func() {
model, err := converter.Convert(shared.FlipperContractAddress, shared.FlipperABI, test_data.DentLog)
Expect(err).NotTo(HaveOccurred())
Expect(model).To(Equal(test_data.DentModel))
})
It("returns an error if the expected amount of topics aren't in the log", func() {
invalidLog := test_data.DentLog
invalidLog.Topics = []common.Hash{}
model, err := converter.Convert(shared.FlipperContractAddress, shared.FlipperABI, invalidLog)
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError("dent log does not contain expected topics"))
Expect(model).To(Equal(dent.DentModel{}))
})
It("returns an error if the log data is empty", func() {
emptyDataLog := test_data.DentLog
emptyDataLog.Data = []byte{}
model, err := converter.Convert(shared.FlipperContractAddress, shared.FlipperABI, emptyDataLog)
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError("dent log data is empty"))
Expect(model).To(Equal(dent.DentModel{}))
})
})

View File

@ -0,0 +1,19 @@
package dent_test
import (
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"io/ioutil"
"log"
)
func TestDent(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Dent Suite")
}
var _ = BeforeSuite(func() {
log.SetOutput(ioutil.Discard)
})

View File

@ -0,0 +1,25 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dent
type DentModel struct {
BidId string `db:"bid_id"`
Lot string
Bid string
Guy string
Tic string
TransactionIndex uint `db:"tx_idx"`
Raw []byte `db:"raw_log"`
}

View File

@ -0,0 +1,62 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dent
import (
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
type Repository interface {
Create(headerId int64, model DentModel) error
MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
}
type DentRepository struct {
db *postgres.DB
}
func NewDentRepository(database *postgres.DB) DentRepository {
return DentRepository{db: database}
}
func (r DentRepository) Create(headerId int64, model DentModel) error {
_, err := r.db.Exec(
`INSERT into maker.dent (header_id, bid_id, lot, bid, guy, tic, tx_idx, raw_log)
VALUES($1, $2, $3, $4, $5, $6, $7, $8)`,
headerId, model.BidId, model.Lot, model.Bid, model.Guy, model.Tic, model.TransactionIndex, model.Raw,
)
return err
}
func (r DentRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
var missingHeaders []core.Header
err := r.db.Select(
&missingHeaders,
`SELECT headers.id, headers.block_number FROM headers
LEFT JOIN maker.dent on headers.id = header_id
WHERE header_id ISNULL
AND headers.block_number >= $1
AND headers.block_number <= $2
AND headers.eth_node_fingerprint = $3`,
startingBlockNumber,
endingBlockNumber,
r.db.Node.ID,
)
return missingHeaders, err
}

View File

@ -0,0 +1,153 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dent_test
import (
"math/rand"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/dent"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
"github.com/vulcanize/vulcanizedb/test_config"
)
var _ = Describe("Dent Repository", func() {
var node core.Node
var db *postgres.DB
var dentRepository dent.DentRepository
var headerRepository repositories.HeaderRepository
var headerId int64
var err error
BeforeEach(func() {
node = test_config.NewTestNode()
db = test_config.NewTestDB(node)
test_config.CleanTestDB(db)
dentRepository = dent.NewDentRepository(db)
headerRepository = repositories.NewHeaderRepository(db)
})
Describe("Create", func() {
BeforeEach(func() {
headerId, err = headerRepository.CreateOrUpdateHeader(core.Header{})
Expect(err).NotTo(HaveOccurred())
err := dentRepository.Create(headerId, test_data.DentModel)
Expect(err).NotTo(HaveOccurred())
})
It("persists a dent record", func() {
var count int
db.QueryRow(`SELECT count(*) FROM maker.dent`).Scan(&count)
Expect(count).To(Equal(1))
var dbResult dent.DentModel
err = db.Get(&dbResult, `SELECT bid_id, lot, bid, guy, tic, tx_idx, raw_log FROM maker.dent WHERE header_id = $1`, headerId)
Expect(err).NotTo(HaveOccurred())
Expect(dbResult.BidId).To(Equal(test_data.DentModel.BidId))
Expect(dbResult.Lot).To(Equal(test_data.DentModel.Lot))
Expect(dbResult.Bid).To(Equal(test_data.DentModel.Bid))
Expect(dbResult.Guy).To(Equal(test_data.DentModel.Guy))
Expect(dbResult.Tic).To(Equal(test_data.DentModel.Tic))
Expect(dbResult.TransactionIndex).To(Equal(test_data.DentModel.TransactionIndex))
Expect(dbResult.Raw).To(MatchJSON(test_data.DentModel.Raw))
})
It("returns an error if inserting a dent record fails", func() {
err = dentRepository.Create(headerId, test_data.DentModel)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
})
It("deletes the tend record if its corresponding header record is deleted", func() {
var count int
err = db.QueryRow(`SELECT count(*) from maker.dent`).Scan(&count)
Expect(err).NotTo(HaveOccurred())
Expect(count).To(Equal(1))
_, err = db.Exec(`DELETE FROM headers where id = $1`, headerId)
Expect(err).NotTo(HaveOccurred())
err = db.QueryRow(`SELECT count(*) from maker.dent`).Scan(&count)
Expect(err).NotTo(HaveOccurred())
Expect(count).To(Equal(0))
})
})
Describe("MissingHeaders", func() {
var dentBlockNumber int64
var startingBlockNumber int64
var endingBlockNumber int64
var blockNumbers []int64
BeforeEach(func() {
dentBlockNumber = rand.Int63()
startingBlockNumber = dentBlockNumber - 1
endingBlockNumber = dentBlockNumber + 1
outOfRangeBlockNumber := dentBlockNumber + 2
blockNumbers = []int64{startingBlockNumber, dentBlockNumber, endingBlockNumber, outOfRangeBlockNumber}
var headerIds []int64
for _, number := range blockNumbers {
headerId, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: number})
Expect(err).NotTo(HaveOccurred())
headerIds = append(headerIds, headerId)
}
dentRepository.Create(headerIds[1], test_data.DentModel)
})
It("returns header records that don't have a corresponding dents", func() {
missingHeaders, err := dentRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
Expect(err).NotTo(HaveOccurred())
Expect(len(missingHeaders)).To(Equal(2))
Expect(missingHeaders[0].BlockNumber).To(Equal(startingBlockNumber))
Expect(missingHeaders[1].BlockNumber).To(Equal(endingBlockNumber))
})
It("only returns missing headers for the given node", func() {
node2 := core.Node{}
db2 := test_config.NewTestDB(node2)
dentRepository2 := dent.NewDentRepository(db2)
headerRepository2 := repositories.NewHeaderRepository(db2)
var node2HeaderIds []int64
for _, number := range blockNumbers {
id, err := headerRepository2.CreateOrUpdateHeader(core.Header{BlockNumber: number})
node2HeaderIds = append(node2HeaderIds, id)
Expect(err).NotTo(HaveOccurred())
}
missingHeadersNode1, err := dentRepository.MissingHeaders(startingBlockNumber, endingBlockNumber)
Expect(err).NotTo(HaveOccurred())
Expect(len(missingHeadersNode1)).To(Equal(2))
Expect(missingHeadersNode1[0].BlockNumber).To(Equal(startingBlockNumber))
Expect(missingHeadersNode1[1].BlockNumber).To(Equal(endingBlockNumber))
missingHeadersNode2, err := dentRepository2.MissingHeaders(startingBlockNumber, endingBlockNumber)
Expect(err).NotTo(HaveOccurred())
Expect(len(missingHeadersNode2)).To(Equal(3))
Expect(missingHeadersNode2[0].BlockNumber).To(Equal(startingBlockNumber))
Expect(missingHeadersNode2[1].BlockNumber).To(Equal(dentBlockNumber))
Expect(missingHeadersNode2[2].BlockNumber).To(Equal(endingBlockNumber))
})
})
})

View File

@ -0,0 +1,80 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dent
import (
"log"
"github.com/ethereum/go-ethereum/common"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
)
type DentTransformer struct {
Config shared.TransformerConfig
Converter Converter
Fetcher shared.LogFetcher
Repository Repository
}
type DentTransformerInitializer struct {
Config shared.TransformerConfig
}
func (i DentTransformerInitializer) NewDentTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer {
converter := NewDentConverter()
fetcher := shared.NewFetcher(blockChain)
repository := NewDentRepository(db)
return DentTransformer{
Config: i.Config,
Converter: converter,
Fetcher: fetcher,
Repository: repository,
}
}
func (t DentTransformer) Execute() error {
config := t.Config
topics := [][]common.Hash{{common.HexToHash(shared.DentFunctionSignature)}}
headers, err := t.Repository.MissingHeaders(config.StartingBlockNumber, config.EndingBlockNumber)
for _, header := range headers {
ethLogs, err := t.Fetcher.FetchLogs(config.ContractAddress, topics, header.BlockNumber)
if err != nil {
log.Println("Error fetching dent logs:", err)
return err
}
for _, ethLog := range ethLogs {
model, err := t.Converter.Convert(config.ContractAddress, config.ContractAbi, ethLog)
if err != nil {
log.Println("Error converting dent log", err)
return err
}
err = t.Repository.Create(header.Id, model)
if err != nil {
log.Println("Error persisting dent record", err)
return err
}
}
}
return err
}

View File

@ -0,0 +1,131 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dent_test
import (
"math/rand"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/transformers/dent"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks"
dent_mocks "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks/dent"
)
var _ = Describe("DentTransformer", func() {
var config = dent.DentConfig
var dentRepository dent_mocks.MockDentRepository
var fetcher mocks.MockLogFetcher
var converter dent_mocks.MockDentConverter
var transformer dent.DentTransformer
BeforeEach(func() {
dentRepository = dent_mocks.MockDentRepository{}
fetcher = mocks.MockLogFetcher{}
converter = dent_mocks.MockDentConverter{}
transformer = dent.DentTransformer{
Repository: &dentRepository,
Config: config,
Fetcher: &fetcher,
Converter: &converter,
}
})
It("gets missing headers", func() {
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(dentRepository.PassedStartingBlockNumber).To(Equal(config.StartingBlockNumber))
Expect(dentRepository.PassedEndingBlockNumber).To(Equal(config.EndingBlockNumber))
})
It("returns an error if fetching the missing headers fails", func() {
dentRepository.SetMissingHeadersError(fakes.FakeError)
err := transformer.Execute()
Expect(err).To(HaveOccurred())
})
It("fetches logs for each missing header", func() {
header1 := core.Header{BlockNumber: rand.Int63()}
header2 := core.Header{BlockNumber: rand.Int63()}
dentRepository.SetMissingHeaders([]core.Header{header1, header2})
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(fetcher.FetchedContractAddress).To(Equal(config.ContractAddress))
expectedTopics := [][]common.Hash{{common.HexToHash(shared.DentFunctionSignature)}}
Expect(fetcher.FetchedTopics).To(Equal(expectedTopics))
Expect(fetcher.FetchedBlocks).To(Equal([]int64{header1.BlockNumber, header2.BlockNumber}))
})
It("returns an error if fetching logs fails", func() {
dentRepository.SetMissingHeaders([]core.Header{{}})
fetcher.SetFetcherError(fakes.FakeError)
err := transformer.Execute()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("converts each eth log to a Model", func() {
dentRepository.SetMissingHeaders([]core.Header{{}})
fetcher.SetFetchedLogs([]types.Log{test_data.DentLog})
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(converter.PassedContractAddress).To(Equal(config.ContractAddress))
Expect(converter.PassedContractAbi).To(Equal(config.ContractAbi))
Expect(converter.LogsToConvert).To(Equal([]types.Log{test_data.DentLog}))
})
It("returns an error if converting the eth log fails", func() {
dentRepository.SetMissingHeaders([]core.Header{{}})
fetcher.SetFetchedLogs([]types.Log{test_data.DentLog})
converter.SetConverterError(fakes.FakeError)
err := transformer.Execute()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("persists each model as a Dent record", func() {
header1 := core.Header{Id: rand.Int63()}
header2 := core.Header{Id: rand.Int63()}
dentRepository.SetMissingHeaders([]core.Header{header1, header2})
fetcher.SetFetchedLogs([]types.Log{test_data.DentLog})
err := transformer.Execute()
Expect(err).NotTo(HaveOccurred())
Expect(dentRepository.PassedDentModels).To(Equal([]dent.DentModel{test_data.DentModel, test_data.DentModel}))
Expect(dentRepository.PassedHeaderIds).To(Equal([]int64{header1.Id, header2.Id}))
})
It("returns an error if persisting dent record fails", func() {
dentRepository.SetMissingHeaders([]core.Header{{}})
dentRepository.SetCreateError(fakes.FakeError)
fetcher.SetFetchedLogs([]types.Log{test_data.DentLog})
err := transformer.Execute()
Expect(err).To(HaveOccurred())
})
})

View File

@ -24,7 +24,7 @@ import (
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/geth"
"github.com/vulcanize/vulcanizedb/pkg/transformers/utilities"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
)
type Converter interface {
@ -58,13 +58,13 @@ func (FlipKickConverter) ToModel(flipKick FlipKickEntity) (FlipKickModel, error)
}
id := flipKick.Id.String()
lot := utilities.ConvertNilToEmptyString(flipKick.Lot.String())
bid := utilities.ConvertNilToEmptyString(flipKick.Bid.String())
lot := shared.ConvertNilToEmptyString(flipKick.Lot.String())
bid := shared.ConvertNilToEmptyString(flipKick.Bid.String())
gal := flipKick.Gal.String()
endValue := utilities.ConvertNilToZeroTimeValue(flipKick.End)
endValue := shared.ConvertNilToZeroTimeValue(flipKick.End)
end := time.Unix(endValue, 0)
urn := common.BytesToAddress(flipKick.Urn[:common.AddressLength]).String()
tab := utilities.ConvertNilToEmptyString(flipKick.Tab.String())
tab := shared.ConvertNilToEmptyString(flipKick.Tab.String())
rawLogJson, err := json.Marshal(flipKick.Raw)
if err != nil {
return FlipKickModel{}, err

View File

@ -0,0 +1,31 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package shared
import (
"github.com/ethereum/go-ethereum/common"
"math/big"
)
type Bid struct {
Bid *big.Int
Lot *big.Int
Guy common.Address
Tic *big.Int
End *big.Int
Lad [32]byte
Gal common.Address
Tab *big.Int
}

View File

@ -0,0 +1,45 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package shared
import (
"github.com/vulcanize/vulcanizedb/pkg/core"
)
type IBidFetcher interface {
FetchBid(contractAbi, contractAddress string, blockNumber int64, methodArgs interface{}) (Bid, error)
}
type BidFetcher struct {
blockChain core.BlockChain
}
func NewBidFetcher(blockchain core.BlockChain) IBidFetcher {
return BidFetcher{
blockChain: blockchain,
}
}
func (fetcher BidFetcher) FetchBid(contractAbi, contractAddress string, blockNumber int64, methodArgs interface{}) (Bid, error) {
method := "bids"
result := Bid{}
err := fetcher.blockChain.FetchContractData(contractAbi, contractAddress, method, methodArgs, &result, blockNumber)
if err != nil {
return result, err
}
return result, nil
}

View File

@ -0,0 +1,57 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package shared_test
import (
"math/big"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
)
var _ = Describe("Dent Bid BidFetcher", func() {
var blockChain *fakes.MockBlockChain
var fetcher shared.IBidFetcher
var blockNumber = int64(123)
var address = "0xfakeAddress"
var contractAbi = "contractAbi"
BeforeEach(func() {
blockChain = fakes.NewMockBlockChain()
fetcher = shared.NewBidFetcher(blockChain)
})
It("fetches a bid record for the given id", func() {
method := "bids"
bidId := big.NewInt(1)
result := &shared.Bid{}
_, err := fetcher.FetchBid(contractAbi, address, blockNumber, bidId)
Expect(err).NotTo(HaveOccurred())
blockChain.AssertFetchContractDataCalledWith(contractAbi, address, method, bidId, result, blockNumber)
})
It("returns an error if fetching the bid fails", func() {
blockChain.SetFetchContractDataErr(fakes.FakeError)
fetcher := shared.NewBidFetcher(blockChain)
_, err := fetcher.FetchBid(contractAbi, address, blockNumber, nil)
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
})

View File

@ -16,6 +16,7 @@ package shared
var (
biteMethod = "Bite(bytes32,bytes32,uint256,uint256,uint256,uint256,uint256)"
dentMethod = "dent(uint256,uint256,uint256)"
flipKickMethod = "Kick(uint256,uint256,uint256,address,uint48,bytes32,uint256)"
frobMethod = "Frob(bytes32,bytes32,uint256,uint256,int256,int256,uint256)"
pitFileDebtCeilingMethod = "file(bytes32,uint256)"
@ -31,6 +32,7 @@ var (
FlipperContractAddress = "0x6b59c42097e2fff7cad96cb08ceefd601081ad9c" //this is a temporary address deployed locally
BiteSignature = GetEventSignature(biteMethod)
DentFunctionSignature = GetLogNoteSignature(dentMethod)
FlipKickSignature = GetEventSignature(flipKickMethod)
FrobSignature = GetEventSignature(frobMethod)
LogValueSignature = GetEventSignature("")

View File

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package utilities
package shared
import "math/big"

View File

@ -16,37 +16,71 @@ package tend
import (
"encoding/json"
"errors"
"math/big"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
)
type Converter interface {
Convert(contractAddress string, contractAbi string, ethLog types.Log) (TendModel, error)
}
type TendConverter struct{}
type TendConverter struct {
BidFetcher shared.IBidFetcher
}
func NewTendConverter() TendConverter {
return TendConverter{}
}
func (c TendConverter) Convert(contractAddress string, contractAbi string, ethLog types.Log) (TendModel, error) {
entity := TendModel{}
entity.Guy = common.HexToAddress(ethLog.Topics[1].Hex()).String()
entity.BidId = ethLog.Topics[2].Big().String()
entity.Lot = ethLog.Topics[3].Big().String()
itemByteLength := 32
lastDataItemStartIndex := len(ethLog.Data) - itemByteLength
lastItem := ethLog.Data[lastDataItemStartIndex:]
last := big.NewInt(0).SetBytes(lastItem)
entity.Bid = last.String()
entity.Tic = "0" //TODO: how do we get the bid tic?
entity.TransactionIndex = ethLog.TxIndex
rawJson, err := json.Marshal(ethLog)
err := validateLog(ethLog)
if err != nil {
return TendModel{}, err
}
entity.Raw = string(rawJson)
return entity, err
bidId := ethLog.Topics[2].Big()
guy := common.HexToAddress(ethLog.Topics[1].Hex()).String()
lot := ethLog.Topics[3].Big().String()
lastDataItemStartIndex := len(ethLog.Data) - 32
lastItem := ethLog.Data[lastDataItemStartIndex:]
last := big.NewInt(0).SetBytes(lastItem)
bidValue := last.String()
tic := "0"
//TODO: it is likely that the tic value will need to be added to an emitted event,
//so this will need to be updated at that point
transactionIndex := ethLog.TxIndex
rawJson, err := json.Marshal(ethLog)
if err != nil {
return TendModel{}, err
}
raw := string(rawJson)
return TendModel{
BidId: bidId.String(),
Lot: lot,
Bid: bidValue,
Guy: guy,
Tic: tic,
TransactionIndex: transactionIndex,
Raw: raw,
}, nil
}
func validateLog(ethLog types.Log) error {
if len(ethLog.Data) <= 0 {
return errors.New("tend log note data is empty")
}
if len(ethLog.Topics) < 4 {
return errors.New("tend log does not contain expected topics")
}
return nil
}

View File

@ -18,6 +18,7 @@ import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/ethereum/go-ethereum/common"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/tend"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
@ -27,14 +28,34 @@ var _ = Describe("Tend TendConverter", func() {
var converter tend.TendConverter
BeforeEach(func() {
converter = tend.TendConverter{}
converter = tend.NewTendConverter()
})
Describe("Convert", func() {
It("converts an eth log to a db model", func() {
model, err := converter.Convert(shared.FlipperContractAddress, shared.FlipperABI, test_data.TendLogNote)
Expect(err).NotTo(HaveOccurred())
Expect(model).To(Equal(test_data.TendModel))
})
It("returns an error if the log data is empty", func() {
emptyDataLog := test_data.TendLogNote
emptyDataLog.Data = []byte{}
model, err := converter.Convert(shared.FlipperContractAddress, shared.FlipperABI, emptyDataLog)
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError("tend log note data is empty"))
Expect(model).To(Equal(tend.TendModel{}))
})
It("returns an error if the expected amount of topics aren't in the log", func() {
invalidLog := test_data.TendLogNote
invalidLog.Topics = []common.Hash{}
model, err := converter.Convert(shared.FlipperContractAddress, shared.FlipperABI, invalidLog)
Expect(err).To(MatchError("tend log does not contain expected topics"))
Expect(model).To(Equal(tend.TendModel{}))
})
})
})

View File

@ -36,16 +36,15 @@ type TendTransformerInitializer struct {
}
func (i TendTransformerInitializer) NewTendTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer {
converter := NewTendConverter()
fetcher := shared.NewFetcher(blockChain)
repository := NewTendRepository(db)
transformer := TendTransformer{
return TendTransformer{
Fetcher: fetcher,
Repository: repository,
Converter: TendConverter{},
Converter: converter,
Config: i.Config,
}
return transformer
}
func (t TendTransformer) Execute() error {

View File

@ -0,0 +1,64 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package test_data
import (
"encoding/json"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/transformers/dent"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"strconv"
)
var (
DentData = "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000645ff3a382000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000098a7d9b8314c000000000000000000000000000000000000000000000000000029a2241af62c0000"
DentTransactionHash = "0x5a210319fcd31eea5959fedb4a1b20881c21a21976e23ff19dff3b44cc1c71e8"
DentBlockHash = "0x105b771e04d7b8516f9291b1f006c46c09cfbff9efa8bc52498b171ff99d28b5"
dentBidId = int64(1)
dentLot = "11000000000000000000"
dentBid = "3000000000000000000"
DentTic = "0"
dentGuy = "0x64d922894153BE9EEf7b7218dc565d1D0Ce2a092"
dentRawJson, _ = json.Marshal(DentLog)
)
var DentLog = types.Log{
Address: common.StringToAddress(shared.FlipperContractAddress),
Topics: []common.Hash{
common.HexToHash("0x5ff3a38200000000000000000000000000000000000000000000000000000000"),
common.HexToHash("0x00000000000000000000000064d922894153be9eef7b7218dc565d1d0ce2a092"),
common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000001"),
common.HexToHash("0x00000000000000000000000000000000000000000000000098a7d9b8314c0000"),
},
Data: hexutil.MustDecode(DentData),
BlockNumber: 15,
TxHash: common.HexToHash(DentTransactionHash),
TxIndex: 5,
BlockHash: common.HexToHash(DentBlockHash),
Index: 2,
Removed: false,
}
var DentModel = dent.DentModel{
BidId: strconv.FormatInt(dentBidId, 10),
Lot: dentLot,
Bid: dentBid,
Guy: dentGuy,
Tic: DentTic,
TransactionIndex: DentLog.TxIndex,
Raw: dentRawJson,
}

View File

@ -0,0 +1,40 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dent
import (
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/transformers/dent"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
)
type MockDentConverter struct {
converterError error
PassedContractAddress string
PassedContractAbi string
LogsToConvert []types.Log
}
func (c *MockDentConverter) Convert(contractAddress string, contractAbi string, ethLog types.Log) (dent.DentModel, error) {
c.PassedContractAddress = contractAddress
c.PassedContractAbi = contractAbi
c.LogsToConvert = append(c.LogsToConvert, ethLog)
return test_data.DentModel, c.converterError
}
func (c *MockDentConverter) SetConverterError(err error) {
c.converterError = err
}

View File

@ -0,0 +1,56 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dent
import (
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/transformers/dent"
)
type MockDentRepository struct {
PassedStartingBlockNumber int64
PassedEndingBlockNumber int64
PassedDentModels []dent.DentModel
PassedHeaderIds []int64
missingHeaders []core.Header
missingHeadersError error
createError error
}
func (r *MockDentRepository) Create(headerId int64, model dent.DentModel) error {
r.PassedHeaderIds = append(r.PassedHeaderIds, headerId)
r.PassedDentModels = append(r.PassedDentModels, model)
return r.createError
}
func (r *MockDentRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
r.PassedStartingBlockNumber = startingBlockNumber
r.PassedEndingBlockNumber = endingBlockNumber
return r.missingHeaders, r.missingHeadersError
}
func (r *MockDentRepository) SetMissingHeadersError(err error) {
r.missingHeadersError = err
}
func (r *MockDentRepository) SetMissingHeaders(headers []core.Header) {
r.missingHeaders = headers
}
func (r *MockDentRepository) SetCreateError(err error) {
r.createError = err
}

View File

@ -16,7 +16,6 @@ package test_data
import (
"encoding/json"
"math/big"
"strconv"
"github.com/ethereum/go-ethereum/common"
@ -28,14 +27,14 @@ import (
)
var (
tendId = int64(10)
tendBidId = int64(10)
tendLot = "85000000000000000000"
tendBid = "1000000000000000000"
tendGuy = "0x7d7bEe5fCfD8028cf7b00876C5b1421c800561A6"
tic = new(big.Int).SetBytes([]byte{0})
tendData = "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000644b43ed12000000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000049b9ca9a6943400000000000000000000000000000000000000000000000000000de0b6b3a7640000"
tendTransactionHash = "0x7909c8793ded2b8348f5db623044fbc26bb7ab78ad5792897abdf68ddc1df63d"
tendBlockHash = "0xa8ea87147c0a68daeb6b1d9f8c0937ba975a650809cab80d19c969e8d0df452c"
TendTic = "0"
)
var TendLogNote = types.Log{
@ -57,11 +56,11 @@ var TendLogNote = types.Log{
var RawLogNoteJson, _ = json.Marshal(TendLogNote)
var TendModel = tend.TendModel{
BidId: strconv.FormatInt(tendId, 10),
BidId: strconv.FormatInt(tendBidId, 10),
Lot: tendLot,
Bid: tendBid,
Guy: tendGuy,
Tic: tic.String(),
Tic: TendTic,
TransactionIndex: TendLogNote.TxIndex,
Raw: string(RawLogNoteJson),
}

View File

@ -16,6 +16,7 @@ package transformers
import (
"github.com/vulcanize/vulcanizedb/pkg/transformers/bite"
"github.com/vulcanize/vulcanizedb/pkg/transformers/dent"
"github.com/vulcanize/vulcanizedb/pkg/transformers/flip_kick"
"github.com/vulcanize/vulcanizedb/pkg/transformers/frob"
"github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file"
@ -28,22 +29,20 @@ import (
)
func TransformerInitializers() []shared.TransformerInitializer {
flipKickConfig := flip_kick.FlipKickConfig
flipKickTransformerInitializer := flip_kick.FlipKickTransformerInitializer{Config: flipKickConfig}
frobConfig := frob.FrobConfig
frobTransformerInitializer := frob.FrobTransformerInitializer{Config: frobConfig}
biteTransformerInitializer := bite.BiteTransformerInitializer{Config: bite.BiteConfig}
dentTransformerInitializer := dent.DentTransformerInitializer{Config: dent.DentConfig}
flipKickTransformerInitializer := flip_kick.FlipKickTransformerInitializer{Config: flip_kick.FlipKickConfig}
frobTransformerInitializer := frob.FrobTransformerInitializer{Config: frob.FrobConfig}
pitFileConfig := pit_file.PitFileConfig
pitFileDebtCeilingTransformerInitializer := debt_ceiling.PitFileDebtCeilingTransformerInitializer{Config: pitFileConfig}
pitFileIlkTransformerInitializer := ilk.PitFileIlkTransformerInitializer{Config: pitFileConfig}
pitFileStabilityFeeTransformerInitializer := stability_fee.PitFileStabilityFeeTransformerInitializer{Config: pitFileConfig}
priceFeedConfig := price_feeds.PriceFeedConfig
priceFeedTransformerInitializer := price_feeds.PriceFeedTransformerInitializer{Config: priceFeedConfig}
tendConfig := tend.TendConfig
tendTransformerInitializer := tend.TendTransformerInitializer{Config: tendConfig}
biteTransformerInitializer := bite.BiteTransformerInitializer{Config: bite.BiteConfig}
priceFeedTransformerInitializer := price_feeds.PriceFeedTransformerInitializer{Config: price_feeds.PriceFeedConfig}
tendTransformerInitializer := tend.TendTransformerInitializer{Config: tend.TendConfig}
return []shared.TransformerInitializer{
biteTransformerInitializer.NewBiteTransformer,
dentTransformerInitializer.NewDentTransformer,
flipKickTransformerInitializer.NewFlipKickTransformer,
frobTransformerInitializer.NewFrobTransformer,
pitFileDebtCeilingTransformerInitializer.NewPitFileDebtCeilingTransformer,