cerc-io/ipld-eth-server
18
0
Fork 1
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases 37 Wiki Activity

remove pkg/transformers; extract shared files needed to

Browse Source 
libraries/shared; work on automated db migration management
This commit is contained in:
Ian Norden 2019-01-31 17:14:42 -06:00
parent 55fa9b8364
commit decc2a3caf
545 changed files with 431 additions and 15922 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.travis.yml
View File

@ -12,7 +12,8 @@ addons:
go_import_path: github.com/vulcanize/vulcanizedb
before_install:
# ginkgo golint dep goose
# ginkgo golint dep migrate
- echo -e "Host github.com\n\tHostName github.com\n\tUser git\n\tIdentityFile ~/.ssh/id_rsa\n" >> ~/.ssh/config
- make installtools
- bash ./scripts/install-postgres-10.sh
- npm install -g ganache-cli
@ -25,7 +26,7 @@ before_script:
- sudo -u postgres createdb vulcanize_private
- make version_migrations
- make migrate NAME=vulcanize_private
- bash ./pkg/transformers/start_test_chain.sh
- bash ./scripts/start_test_chain.sh
- cd postgraphile && yarn
script:
@ -38,5 +39,6 @@ notifications:
email: false
after_script:
- bash ./pkg/transformers/stop_test_chain.sh
- bash ./scripts/stop_test_chain.sh
- bash ./bin/deploy.sh

5
Makefile
View File

@ -48,16 +48,15 @@ lint:
.PHONY: test
test: | $(GINKGO) $(LINT)
go get -t ./...
go vet ./...
go fmt ./...
$(GINKGO) -r --skipPackage=integration_tests,integration
$(GINKGO) -r
.PHONY: integrationtest
integrationtest: | $(GINKGO) $(LINT)
go vet ./...
go fmt ./...
$(GINKGO) -r pkg/transformers/integration_tests/ integration_test/
$(GINKGO) -r integration_test/
.PHONY: dep
dep: | $(DEP)

23
cmd/composeAndExecute.go
View File

@ -23,6 +23,7 @@ import (
"time"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
"github.com/vulcanize/vulcanizedb/libraries/shared/watcher"
@ -47,8 +48,7 @@ var composeAndExecuteCmd = &cobra.Command{
ipcPath = "http://kovan0.vulcanize.io:8545"
[exporter]
filePath = "$GOPATH/src/github.com/vulcanize/vulcanizedb/plugins/"
fileName = "exporter"
name = "exporter"
[exporter.transformers]
transformer1 = "github.com/path/to/transformer1"
transformer2 = "github.com/path/to/transformer2"
@ -75,9 +75,20 @@ loaded into and executed over by a generic watcher`,
func composeAndExecute() {
// generate code to build the plugin according to the config file
autogenConfig = autogen.Config{
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/plugins",
FileName: viper.GetString("exporter.name"),
Save: viper.GetBool("exporter.save"),
Initializers: viper.GetStringMapString("exporter.transformers"),
Dependencies: viper.GetStringMapString("exporter.repositories"),
Migrations: viper.GetStringMapString("exporter.migrations"),
}
fmt.Println("generating plugin")
generator := autogen.NewGenerator(autogenConfig, databaseConfig)
err := generator.GenerateExporterPlugin()
if err != nil {
fmt.Println("generating plugin failed")
log.Fatal(err)
}
@ -86,14 +97,21 @@ func composeAndExecute() {
if err != nil {
log.Fatal(err)
}
if !autogenConfig.Save {
defer utils.ClearFiles(pluginPath)
}
fmt.Println("opening plugin")
plug, err := plugin.Open(pluginPath)
if err != nil {
fmt.Println("opening pluggin failed")
log.Fatal(err)
}
// Load the `Exporter` symbol from the plugin
fmt.Println("loading transformers from plugin")
symExporter, err := plug.Lookup("Exporter")
if err != nil {
fmt.Println("loading Exporter symbol failed")
log.Fatal(err)
}
@ -116,6 +134,7 @@ func composeAndExecute() {
w.AddTransformers(initializers)
// Execute over the TransformerInitializer set using the watcher
fmt.Println("executing transformers")
ticker := time.NewTicker(pollingInterval)
defer ticker.Stop()
for range ticker.C {

74
cmd/getSignatures.go
View File

@ -1,74 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"fmt"
"github.com/spf13/cobra"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
)
// getSignaturesCmd represents the getSignatures command
var getSignaturesCmd = &cobra.Command{
Use: "getSignatures",
Short: "A command to see transformer method and event signatures",
Long: `A convenience command to see method/event signatures for Maker transformers
vulcanizedb getSignatures`,
Run: func(cmd *cobra.Command, args []string) {
getSignatures()
},
}
func getSignatures() {
signatures := make(map[string]string)
signatures["BiteSignature"] = constants.GetBiteSignature()
signatures["CatFileChopLumpSignature"] = constants.GetCatFileChopLumpSignature()
signatures["CatFileFlipSignature"] = constants.GetCatFileFlipSignature()
signatures["CatFilePitVowSignature"] = constants.GetCatFilePitVowSignature()
signatures["DealSignature"] = constants.GetDealSignature()
signatures["DentFunctionSignature"] = constants.GetDentFunctionSignature()
signatures["DripDripSignature"] = constants.GetDripDripSignature()
signatures["DripFileIlkSignature"] = constants.GetDripFileIlkSignature()
signatures["DripFileRepoSignature"] = constants.GetDripFileRepoSignature()
signatures["DripFileVowSignature"] = constants.GetDripFileVowSignature()
signatures["FlapKickSignature"] = constants.GetFlapKickSignature()
signatures["FlipKickSignature"] = constants.GetFlipKickSignature()
signatures["FlopKickSignature"] = constants.GetFlopKickSignature()
signatures["FrobSignature"] = constants.GetFrobSignature()
signatures["LogValueSignature"] = constants.GetLogValueSignature()
signatures["PitFileDebtCeilingSignature"] = constants.GetPitFileDebtCeilingSignature()
signatures["PitFileIlkSignature"] = constants.GetPitFileIlkSignature()
signatures["TendFunctionSignature"] = constants.GetTendFunctionSignature()
signatures["VatFluxSignature"] = constants.GetVatFluxSignature()
signatures["VatFoldSignature"] = constants.GetVatFoldSignature()
signatures["VatGrabSignature"] = constants.GetVatGrabSignature()
signatures["VatHealSignature"] = constants.GetVatHealSignature()
signatures["VatInitSignature"] = constants.GetVatInitSignature()
signatures["VatMoveSignature"] = constants.GetVatMoveSignature()
signatures["VatSlipSignature"] = constants.GetVatSlipSignature()
signatures["VatTollSignature"] = constants.GetVatTollSignature()
signatures["VatTuneSignature"] = constants.GetVatTuneSignature()
signatures["VowFlogSignature"] = constants.GetVowFlogSignature()
for name, sig := range signatures {
fmt.Println(name, ": ", sig)
}
}
func init() {
rootCmd.AddCommand(getSignaturesCmd)
}

11
cmd/root.go
View File

@ -55,7 +55,7 @@ const (
var rootCmd = &cobra.Command{
Use: "vulcanizedb",
PersistentPreRun: configure,
PersistentPreRun: database,
}
func Execute() {
@ -65,7 +65,7 @@ func Execute() {
}
}
func configure(cmd *cobra.Command, args []string) {
func database(cmd *cobra.Command, args []string) {
ipc = viper.GetString("client.ipcpath")
levelDbPath = viper.GetString("client.leveldbpath")
storageDiffsPath = viper.GetString("filesystem.storageDiffsPath")
@ -76,13 +76,6 @@ func configure(cmd *cobra.Command, args []string) {
User: viper.GetString("database.user"),
Password: viper.GetString("database.password"),
}
autogenConfig = autogen.Config{
FilePath: viper.GetString("exporter.filePath"),
FileName: viper.GetString("exporter.fileName"),
Initializers: viper.GetStringMapString("exporter.transformers"),
Dependencies: viper.GetStringMapString("exporter.repositories"),
Migrations: viper.GetStringMapString("exporter.migrations"),
}
viper.Set("database.config", databaseConfig)
}

6
environments/compose.toml
View File

@ -12,8 +12,8 @@
name = "maker_vdb_staging"
[exporter]
filePath = "$GOPATH/src/github.com/vulcanize/vulcanizedb/plugins/"
fileName = "exporter"
name = "exporter"
save = false
[exporter.transformers]
bite = "github.com/vulcanize/mcd_transformers/transformers/bite"
cat_chop_lump = "github.com/vulcanize/mcd_transformers/transformers/cat_file/chop_lump"
@ -45,8 +45,6 @@
vow_flog = "github.com/vulcanize/mcd_transformers/transformers/vow_flog"
[exporter.repositories]
mcd_transformers = "github.com/vulcanize/mcd_transformers"
[exporter.migrations]
mcd_transformers = "db/migrations"
[contract]
[contract.address]

5
pkg/transformers/shared/log_chunker.go → libraries/shared/chunker/log_chunker.go
View File

@ -14,12 +14,13 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package shared
package chunker
import (
"strings"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"strings"
shared_t "github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
)

9
pkg/transformers/shared/log_chunker_test.go → libraries/shared/chunker/log_chunker_test.go
View File

@ -14,21 +14,22 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package shared_test
package chunker_test
import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
chunk "github.com/vulcanize/vulcanizedb/libraries/shared/chunker"
shared_t "github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
)
var _ = Describe("Log chunker", func() {
var (
configs []shared_t.TransformerConfig
chunker *shared.LogChunker
chunker *chunk.LogChunker
)
BeforeEach(func() {
@ -50,7 +51,7 @@ var _ = Describe("Log chunker", func() {
}
configs = []shared_t.TransformerConfig{configA, configB, configC}
chunker = shared.NewLogChunker()
chunker = chunk.NewLogChunker()
chunker.AddConfigs(configs)
})

15
pkg/transformers/price_feeds/constants.go → libraries/shared/constants/checked_headers.go
View File

@ -14,15 +14,12 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package price_feeds
package constants
import (
"errors"
"math/big"
)
type TransformerExecution bool
var (
ErrNoMatchingLog = errors.New("no matching log")
Ether = big.NewFloat(1e18)
Ray = big.NewFloat(1e27)
const (
HeaderRecheck TransformerExecution = true
HeaderMissing TransformerExecution = false
RecheckHeaderCap = "4"
)

9
pkg/transformers/factories/log_note_converter.go → libraries/shared/constants/data.go
View File

@ -14,10 +14,9 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package factories
package constants
import "github.com/ethereum/go-ethereum/core/types"
var DataItemLength = 32
type LogNoteConverter interface {
ToModels(ethLog []types.Log) ([]interface{}, error)
}
// TODO Grab this from DB, since it can change through governance
var TTL = int64(10800) // 60 * 60 * 3 == 10800 seconds == 3 hours

2
pkg/transformers/shared/log_fetcher.go → libraries/shared/fetcher/log_fetcher.go
View File

@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package shared
package fetcher
import (
"github.com/ethereum/go-ethereum"

8
pkg/transformers/shared/log_fetcher_test.go → libraries/shared/fetcher/log_fetcher_test.go
View File

@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package shared_test
package fetcher_test
import (
"github.com/ethereum/go-ethereum"
@ -22,16 +22,16 @@ import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
fetch "github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
)
var _ = Describe("Fetcher", func() {
Describe("FetchLogs", func() {
It("fetches logs based on the given query", func() {
blockChain := fakes.NewMockBlockChain()
fetcher := shared.NewFetcher(blockChain)
fetcher := fetch.NewFetcher(blockChain)
header := fakes.FakeHeader
addresses := []common.Address{
@ -59,7 +59,7 @@ var _ = Describe("Fetcher", func() {
It("returns an error if fetching the logs fails", func() {
blockChain := fakes.NewMockBlockChain()
blockChain.SetGetEthLogsWithCustomQueryErr(fakes.FakeError)
fetcher := shared.NewFetcher(blockChain)
fetcher := fetch.NewFetcher(blockChain)
_, err := fetcher.FetchLogs([]common.Address{}, []common.Hash{}, core.Header{})

16
pkg/transformers/test_data/mocks/mock_watcher_repository.go → libraries/shared/mocks/mock_watcher_repository.go
View File

@ -1,3 +1,19 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package mocks
import (

28
pkg/transformers/factories/repository.go → libraries/shared/mocks/storage_repository.go
View File

@ -14,17 +14,29 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package factories
package mocks
import (
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/libraries/shared/utils"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
type Repository interface {
Create(headerID int64, models []interface{}) error
MarkHeaderChecked(headerID int64) error
MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
RecheckHeaders(startingBlockNumber, endingBlockNUmber int64) ([]core.Header, error)
SetDB(db *postgres.DB)
type MockStorageRepository struct {
CreateErr error
PassedBlockNumber int
PassedBlockHash string
PassedMetadata utils.StorageValueMetadata
PassedValue interface{}
}
func (repository *MockStorageRepository) Create(blockNumber int, blockHash string, metadata utils.StorageValueMetadata, value interface{}) error {
repository.PassedBlockNumber = blockNumber
repository.PassedBlockHash = blockHash
repository.PassedMetadata = metadata
repository.PassedValue = value
return repository.CreateErr
}
func (*MockStorageRepository) SetDB(db *postgres.DB) {
panic("implement me")
}

40
pkg/transformers/price_feeds/model.go → libraries/shared/mocks/storage_transformer.go
View File

@ -14,35 +14,31 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package price_feeds
package mocks
import (
"math/big"
"github.com/ethereum/go-ethereum/common"
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
"github.com/vulcanize/vulcanizedb/libraries/shared/utils"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
type LogValueEntity struct {
Val common.Address
type MockStorageTransformer struct {
Address common.Address
ExecuteErr error
PassedRow utils.StorageDiffRow
}
type PriceFeedModel struct {
BlockNumber uint64 `db:"block_number"`
MedianizerAddress string `db:"medianizer_address"`
UsdValue string `db:"usd_value"`
LogIndex uint `db:"log_idx"`
TransactionIndex uint `db:"tx_idx"`
Raw []byte `db:"raw_log"`
func (transformer *MockStorageTransformer) Execute(row utils.StorageDiffRow) error {
transformer.PassedRow = row
return transformer.ExecuteErr
}
func Convert(conversion string, value string, prec int) string {
var bgflt = big.NewFloat(0.0)
bgflt.SetString(value)
switch conversion {
case "ray":
bgflt.Quo(bgflt, Ray)
case "wad":
bgflt.Quo(bgflt, Ether)
}
return bgflt.Text('g', prec)
func (transformer *MockStorageTransformer) ContractAddress() common.Address {
return transformer.Address
}
func (transformer *MockStorageTransformer) FakeTransformerInitializer(db *postgres.DB) transformer.StorageTransformer {
return transformer
}

20
pkg/transformers/test_data/mocks/transformer.go → libraries/shared/mocks/transformer.go
View File

@ -1,12 +1,28 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package mocks
import (
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
shared_t "github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
)
type MockTransformer struct {
@ -35,7 +51,7 @@ func (mh *MockTransformer) SetTransformerConfig(config shared_t.TransformerConfi
mh.config = config
}
func (mh *MockTransformer) FakeTransformerInitializer(db *postgres.DB) shared_t.Transformer {
func (mh *MockTransformer) FakeTransformerInitializer(db *postgres.DB) shared_t.EventTransformer {
return mh
}

21
pkg/transformers/shared/repository.go → libraries/shared/repository/repository.go
View File

@ -1,13 +1,30 @@
package shared
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package repository
import (
"bytes"
"database/sql"
"database/sql/driver"
"fmt"
"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
)
func GetOrCreateIlk(ilk string, db *postgres.DB) (int, error) {

10
pkg/transformers/shared/repository_utility_test.go → libraries/shared/repository/repository_utility_test.go
View File

@ -14,21 +14,23 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package shared_test
package repository_test
import (
"fmt"
"math/rand"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
shared "github.com/vulcanize/vulcanizedb/libraries/shared/repository"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
"github.com/vulcanize/vulcanizedb/test_config"
"math/rand"
)
var _ = Describe("Repository utilities", func() {

23
libraries/shared/transformer/transformer.go → libraries/shared/transformer/event_transformer.go
View File

@ -1,5 +1,18 @@
// Auto-gen this code for different transformer interfaces/configs
// based on config file to allow for more modularity
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package transformer
@ -8,16 +21,16 @@ import (
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
)
type Transformer interface {
type EventTransformer interface {
Execute(logs []types.Log, header core.Header, recheckHeaders constants.TransformerExecution) error
GetConfig() TransformerConfig
}
type TransformerInitializer func(db *postgres.DB) Transformer
type TransformerInitializer func(db *postgres.DB) EventTransformer
type TransformerConfig struct {
TransformerName string

14
pkg/transformers/storage_diffs/repository.go → libraries/shared/transformer/storage_transformer.go
View File

@ -14,14 +14,18 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package storage_diffs
package transformer
import (
"github.com/ethereum/go-ethereum/common"
"github.com/vulcanize/vulcanizedb/libraries/shared/utils"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared"
)
type Repository interface {
Create(blockNumber int, blockHash string, metadata shared.StorageValueMetadata, value interface{}) error
SetDB(db *postgres.DB)
type StorageTransformer interface {
Execute(row utils.StorageDiffRow) error
ContractAddress() common.Address
}
type StorageTransformerInitializer func(db *postgres.DB) StorageTransformer

35
libraries/shared/transformer/transformer_suite_test.go
View File

@ -1,35 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package transformer_test
import (
"io/ioutil"
"log"
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
func TestShared(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Shared Transformer Suite")
}
var _ = BeforeSuite(func() {
log.SetOutput(ioutil.Discard)
})

17
libraries/shared/transformer/transformer_test.go
View File

@ -1,17 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package transformer_test

5
pkg/transformers/storage_diffs/shared/decoder.go → libraries/shared/utils/decoder.go
View File

@ -14,12 +14,13 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package shared
package utils
import (
"fmt"
"github.com/ethereum/go-ethereum/common"
"math/big"
"github.com/ethereum/go-ethereum/common"
)
func Decode(row StorageDiffRow, metadata StorageValueMetadata) (interface{}, error) {

20
pkg/transformers/storage_diffs/shared/decoder_test.go → libraries/shared/utils/decoder_test.go
View File

@ -14,23 +14,25 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package shared_test
package utils_test
import (
"math/big"
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared"
"math/big"
"github.com/vulcanize/vulcanizedb/libraries/shared/utils"
)
var _ = Describe("Storage decoder", func() {
It("decodes uint256", func() {
fakeInt := common.HexToHash("0000000000000000000000000000000000000000000000000000000000000539")
row := shared.StorageDiffRow{StorageValue: fakeInt}
metadata := shared.StorageValueMetadata{Type: shared.Uint256}
row := utils.StorageDiffRow{StorageValue: fakeInt}
metadata := utils.StorageValueMetadata{Type: utils.Uint256}
result, err := shared.Decode(row, metadata)
result, err := utils.Decode(row, metadata)
Expect(err).NotTo(HaveOccurred())
Expect(result).To(Equal(big.NewInt(0).SetBytes(fakeInt.Bytes()).String()))
@ -38,10 +40,10 @@ var _ = Describe("Storage decoder", func() {
It("decodes address", func() {
fakeAddress := common.HexToAddress("0x12345")
row := shared.StorageDiffRow{StorageValue: fakeAddress.Hash()}
metadata := shared.StorageValueMetadata{Type: shared.Address}
row := utils.StorageDiffRow{StorageValue: fakeAddress.Hash()}
metadata := utils.StorageValueMetadata{Type: utils.Address}
result, err := shared.Decode(row, metadata)
result, err := utils.Decode(row, metadata)
Expect(err).NotTo(HaveOccurred())
Expect(result).To(Equal(fakeAddress.Hex()))

2
pkg/transformers/storage_diffs/shared/errors.go → libraries/shared/utils/errors.go
View File

@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package shared
package utils
import (
"fmt"

5
pkg/transformers/storage_diffs/shared/row.go → libraries/shared/utils/row.go
View File

@ -14,11 +14,12 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package shared
package utils
import (
"github.com/ethereum/go-ethereum/common"
"strconv"
"github.com/ethereum/go-ethereum/common"
)
const ExpectedRowLength = 5

13
pkg/transformers/storage_diffs/shared/row_test.go → libraries/shared/utils/row_test.go
View File

@ -14,13 +14,14 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package shared_test
package utils_test
import (
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared"
"github.com/vulcanize/vulcanizedb/libraries/shared/utils"
)
var _ = Describe("Storage row parsing", func() {
@ -32,7 +33,7 @@ var _ = Describe("Storage row parsing", func() {
storageValue := "0x654"
data := []string{contract, blockHash, blockHeight, storageKey, storageValue}
result, err := shared.FromStrings(data)
result, err := utils.FromStrings(data)
Expect(err).NotTo(HaveOccurred())
Expect(result.Contract).To(Equal(common.HexToAddress(contract)))
@ -43,14 +44,14 @@ var _ = Describe("Storage row parsing", func() {
})
It("returns an error if row is missing data", func() {
_, err := shared.FromStrings([]string{"0x123"})
_, err := utils.FromStrings([]string{"0x123"})
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(shared.ErrRowMalformed{Length: 1}))
Expect(err).To(MatchError(utils.ErrRowMalformed{Length: 1}))
})
It("returns error if block height malformed", func() {
_, err := shared.FromStrings([]string{"", "", "", "", ""})
_, err := utils.FromStrings([]string{"", "", "", "", ""})
Expect(err).To(HaveOccurred())
})

2
pkg/transformers/storage_diffs/shared/shared_suite_test.go → libraries/shared/utils/shared_suite_test.go
View File

@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package shared_test
package utils_test
import (
"github.com/sirupsen/logrus"

2
pkg/transformers/storage_diffs/shared/value.go → libraries/shared/utils/value.go
View File

@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package shared
package utils
type ValueType int

22
libraries/shared/watcher/event_watcher.go
View File

@ -22,26 +22,28 @@ import (
"github.com/ethereum/go-ethereum/common"
log "github.com/sirupsen/logrus"
chunk "github.com/vulcanize/vulcanizedb/libraries/shared/chunker"
"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
fetch "github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
repo "github.com/vulcanize/vulcanizedb/libraries/shared/repository"
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
)
type EventWatcher struct {
Transformers []transformer.Transformer
Transformers []transformer.EventTransformer
DB *postgres.DB
Fetcher shared.LogFetcher
Chunker shared.Chunker
Fetcher fetch.LogFetcher
Chunker chunk.Chunker
Addresses []common.Address
Topics []common.Hash
StartingBlock *int64
}
func NewEventWatcher(db *postgres.DB, bc core.BlockChain) EventWatcher {
chunker := shared.NewLogChunker()
fetcher := shared.NewFetcher(bc)
chunker := chunk.NewLogChunker()
fetcher := fetch.NewFetcher(bc)
return EventWatcher{
DB: db,
Fetcher: fetcher,
@ -83,13 +85,13 @@ func (watcher *EventWatcher) Execute(recheckHeaders constants.TransformerExecuti
return fmt.Errorf("No transformers added to watcher")
}
checkedColumnNames, err := shared.GetCheckedColumnNames(watcher.DB)
checkedColumnNames, err := repo.GetCheckedColumnNames(watcher.DB)
if err != nil {
return err
}
notCheckedSQL := shared.CreateNotCheckedSQL(checkedColumnNames, recheckHeaders)
notCheckedSQL := repo.CreateNotCheckedSQL(checkedColumnNames, recheckHeaders)
missingHeaders, err := shared.MissingHeaders(*watcher.StartingBlock, -1, watcher.DB, notCheckedSQL)
missingHeaders, err := repo.MissingHeaders(*watcher.StartingBlock, -1, watcher.DB, notCheckedSQL)
if err != nil {
log.Error("Fetching of missing headers failed in watcher!")
return err

4
libraries/shared/watcher/event_watcher_test.go
View File

@ -25,14 +25,14 @@ import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
"github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
"github.com/vulcanize/vulcanizedb/libraries/shared/watcher"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks"
"github.com/vulcanize/vulcanizedb/test_config"
)

25
libraries/shared/storage_watcher.go → libraries/shared/watcher/storage_watcher.go
View File

@ -14,17 +14,18 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package shared
package watcher
import (
"strings"
"reflect"
"github.com/ethereum/go-ethereum/common"
"github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/storage"
"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared"
"reflect"
"strings"
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
"github.com/vulcanize/vulcanizedb/libraries/shared/utils"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/fs"
)
@ -32,11 +33,11 @@ type StorageWatcher struct {
db *postgres.DB
tailer fs.Tailer
Queue IStorageQueue
Transformers map[common.Address]storage.Transformer
Transformers map[common.Address]transformer.StorageTransformer
}
func NewStorageWatcher(tailer fs.Tailer, db *postgres.DB) StorageWatcher {
transformers := make(map[common.Address]storage.Transformer)
transformers := make(map[common.Address]transformer.StorageTransformer)
queue := NewStorageQueue(db)
return StorageWatcher{
db: db,
@ -46,7 +47,7 @@ func NewStorageWatcher(tailer fs.Tailer, db *postgres.DB) StorageWatcher {
}
}
func (watcher StorageWatcher) AddTransformers(initializers []storage.TransformerInitializer) {
func (watcher StorageWatcher) AddTransformers(initializers []transformer.StorageTransformerInitializer) {
for _, initializer := range initializers {
transformer := initializer(watcher.db)
watcher.Transformers[transformer.ContractAddress()] = transformer
@ -59,13 +60,13 @@ func (watcher StorageWatcher) Execute() error {
return tailErr
}
for line := range t.Lines {
row, parseErr := shared.FromStrings(strings.Split(line.Text, ","))
row, parseErr := utils.FromStrings(strings.Split(line.Text, ","))
if parseErr != nil {
return parseErr
}
transformer, ok := watcher.Transformers[row.Contract]
if !ok {
logrus.Warn(shared.ErrContractNotFound{Contract: row.Contract.Hex()}.Error())
logrus.Warn(utils.ErrContractNotFound{Contract: row.Contract.Hex()}.Error())
continue
}
executeErr := transformer.Execute(row)
@ -85,5 +86,5 @@ func (watcher StorageWatcher) Execute() error {
}
func isKeyNotFound(executeErr error) bool {
return reflect.TypeOf(executeErr) == reflect.TypeOf(shared.ErrStorageKeyNotFound{})
return reflect.TypeOf(executeErr) == reflect.TypeOf(utils.ErrStorageKeyNotFound{})
}

72
libraries/shared/storage_watcher_test.go → libraries/shared/watcher/storage_watcher_test.go
View File

@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package shared_test
package watcher_test
import (
"errors"
@ -30,12 +30,12 @@ import (
. "github.com/onsi/gomega"
"github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/libraries/shared"
"github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
"github.com/vulcanize/vulcanizedb/libraries/shared/utils"
"github.com/vulcanize/vulcanizedb/libraries/shared/watcher"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/storage"
shared2 "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks"
"github.com/vulcanize/vulcanizedb/test_config"
)
@ -43,39 +43,39 @@ var _ = Describe("Storage Watcher", func() {
It("adds transformers", func() {
fakeAddress := common.HexToAddress("0x12345")
fakeTransformer := &mocks.MockStorageTransformer{Address: fakeAddress}
watcher := shared.NewStorageWatcher(&fakes.MockTailer{}, test_config.NewTestDB(core.Node{}))
w := watcher.NewStorageWatcher(&fakes.MockTailer{}, test_config.NewTestDB(core.Node{}))
watcher.AddTransformers([]storage.TransformerInitializer{fakeTransformer.FakeTransformerInitializer})
w.AddTransformers([]transformer.StorageTransformerInitializer{fakeTransformer.FakeTransformerInitializer})
Expect(watcher.Transformers[fakeAddress]).To(Equal(fakeTransformer))
Expect(w.Transformers[fakeAddress]).To(Equal(fakeTransformer))
})
It("reads the tail of the storage diffs file", func() {
mockTailer := fakes.NewMockTailer()
watcher := shared.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
w := watcher.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
assert(func(err error) {
Expect(err).To(BeNil())
Expect(mockTailer.TailCalled).To(BeTrue())
}, watcher, mockTailer, []*tail.Line{})
}, w, mockTailer, []*tail.Line{})
})
It("returns error if row parsing fails", func() {
mockTailer := fakes.NewMockTailer()
watcher := shared.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
w := watcher.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
line := &tail.Line{Text: "oops"}
assert(func(err error) {
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(shared2.ErrRowMalformed{Length: 1}))
}, watcher, mockTailer, []*tail.Line{line})
Expect(err).To(MatchError(utils.ErrRowMalformed{Length: 1}))
}, w, mockTailer, []*tail.Line{line})
})
It("logs error if no transformer can parse storage row", func() {
mockTailer := fakes.NewMockTailer()
address := common.HexToAddress("0x12345")
line := getFakeLine(address.Bytes())
watcher := shared.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
w := watcher.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
tempFile, err := ioutil.TempFile("", "log")
defer os.Remove(tempFile.Name())
Expect(err).NotTo(HaveOccurred())
@ -85,24 +85,24 @@ var _ = Describe("Storage Watcher", func() {
Expect(err).NotTo(HaveOccurred())
logContent, readErr := ioutil.ReadFile(tempFile.Name())
Expect(readErr).NotTo(HaveOccurred())
Expect(string(logContent)).To(ContainSubstring(shared2.ErrContractNotFound{Contract: address.Hex()}.Error()))
}, watcher, mockTailer, []*tail.Line{line})
Expect(string(logContent)).To(ContainSubstring(utils.ErrContractNotFound{Contract: address.Hex()}.Error()))
}, w, mockTailer, []*tail.Line{line})
})
It("executes transformer with storage row", func() {
address := []byte{1, 2, 3}
line := getFakeLine(address)
mockTailer := fakes.NewMockTailer()
watcher := shared.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
w := watcher.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
fakeTransformer := &mocks.MockStorageTransformer{Address: common.BytesToAddress(address)}
watcher.AddTransformers([]storage.TransformerInitializer{fakeTransformer.FakeTransformerInitializer})
w.AddTransformers([]transformer.StorageTransformerInitializer{fakeTransformer.FakeTransformerInitializer})
assert(func(err error) {
Expect(err).To(BeNil())
expectedRow, err := shared2.FromStrings(strings.Split(line.Text, ","))
expectedRow, err := utils.FromStrings(strings.Split(line.Text, ","))
Expect(err).NotTo(HaveOccurred())
Expect(fakeTransformer.PassedRow).To(Equal(expectedRow))
}, watcher, mockTailer, []*tail.Line{line})
}, w, mockTailer, []*tail.Line{line})
})
Describe("when executing transformer fails", func() {
@ -110,30 +110,30 @@ var _ = Describe("Storage Watcher", func() {
address := []byte{1, 2, 3}
line := getFakeLine(address)
mockTailer := fakes.NewMockTailer()
watcher := shared.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
w := watcher.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
mockQueue := &mocks.MockStorageQueue{}
watcher.Queue = mockQueue
keyNotFoundError := shared2.ErrStorageKeyNotFound{Key: "unknown_storage_key"}
w.Queue = mockQueue
keyNotFoundError := utils.ErrStorageKeyNotFound{Key: "unknown_storage_key"}
fakeTransformer := &mocks.MockStorageTransformer{Address: common.BytesToAddress(address), ExecuteErr: keyNotFoundError}
watcher.AddTransformers([]storage.TransformerInitializer{fakeTransformer.FakeTransformerInitializer})
w.AddTransformers([]transformer.StorageTransformerInitializer{fakeTransformer.FakeTransformerInitializer})
assert(func(err error) {
Expect(err).NotTo(HaveOccurred())
Expect(mockQueue.AddCalled).To(BeTrue())
}, watcher, mockTailer, []*tail.Line{line})
}, w, mockTailer, []*tail.Line{line})
})
It("logs error if queuing row fails", func() {
address := []byte{1, 2, 3}
line := getFakeLine(address)
mockTailer := fakes.NewMockTailer()
watcher := shared.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
w := watcher.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
mockQueue := &mocks.MockStorageQueue{}
mockQueue.AddError = fakes.FakeError
watcher.Queue = mockQueue
keyNotFoundError := shared2.ErrStorageKeyNotFound{Key: "unknown_storage_key"}
w.Queue = mockQueue
keyNotFoundError := utils.ErrStorageKeyNotFound{Key: "unknown_storage_key"}
fakeTransformer := &mocks.MockStorageTransformer{Address: common.BytesToAddress(address), ExecuteErr: keyNotFoundError}
watcher.AddTransformers([]storage.TransformerInitializer{fakeTransformer.FakeTransformerInitializer})
w.AddTransformers([]transformer.StorageTransformerInitializer{fakeTransformer.FakeTransformerInitializer})
tempFile, err := ioutil.TempFile("", "log")
defer os.Remove(tempFile.Name())
Expect(err).NotTo(HaveOccurred())
@ -145,17 +145,17 @@ var _ = Describe("Storage Watcher", func() {
logContent, readErr := ioutil.ReadFile(tempFile.Name())
Expect(readErr).NotTo(HaveOccurred())
Expect(string(logContent)).To(ContainSubstring(fakes.FakeError.Error()))
}, watcher, mockTailer, []*tail.Line{line})
}, w, mockTailer, []*tail.Line{line})
})
It("logs any other error", func() {
address := []byte{1, 2, 3}
line := getFakeLine(address)
mockTailer := fakes.NewMockTailer()
watcher := shared.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
w := watcher.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
executionError := errors.New("storage watcher failed attempting to execute transformer")
fakeTransformer := &mocks.MockStorageTransformer{Address: common.BytesToAddress(address), ExecuteErr: executionError}
watcher.AddTransformers([]storage.TransformerInitializer{fakeTransformer.FakeTransformerInitializer})
w.AddTransformers([]transformer.StorageTransformerInitializer{fakeTransformer.FakeTransformerInitializer})
tempFile, err := ioutil.TempFile("", "log")
defer os.Remove(tempFile.Name())
Expect(err).NotTo(HaveOccurred())
@ -166,12 +166,12 @@ var _ = Describe("Storage Watcher", func() {
logContent, readErr := ioutil.ReadFile(tempFile.Name())
Expect(readErr).NotTo(HaveOccurred())
Expect(string(logContent)).To(ContainSubstring(executionError.Error()))
}, watcher, mockTailer, []*tail.Line{line})
}, w, mockTailer, []*tail.Line{line})
})
})
})
func assert(assertion func(err error), watcher shared.StorageWatcher, mockTailer *fakes.MockTailer, lines []*tail.Line) {
func assert(assertion func(err error), watcher watcher.StorageWatcher, mockTailer *fakes.MockTailer, lines []*tail.Line) {
errs := make(chan error, 1)
done := make(chan bool, 1)
go execute(watcher, errs, done)
@ -190,8 +190,8 @@ func assert(assertion func(err error), watcher shared.StorageWatcher, mockTailer
}
}
func execute(watcher shared.StorageWatcher, errs chan error, done chan bool) {
err := watcher.Execute()
func execute(w watcher.StorageWatcher, errs chan error, done chan bool) {
err := w.Execute()
if err != nil {
errs <- err
} else {

5
pkg/autogen/config.go
View File

@ -26,11 +26,12 @@ import (
)
type Config struct {
Initializers map[string]string // Map of import aliases to transformer paths
Initializers map[string]string // Map of import aliases to transformer initializer paths
Dependencies map[string]string // Map of vendor dep names to their repositories
Migrations map[string]string // Map of migration names to their paths within the vendored deps
Migrations map[string]string // Map of vendor dep names to relative path from repository to db migrations
FilePath string
FileName string
Save bool
}
func (c *Config) GetPluginPaths() (string, string, error) {

187
pkg/autogen/generator.go
View File

@ -24,6 +24,7 @@ import (
"os/exec"
"path/filepath"
"strconv"
"strings"
. "github.com/dave/jennifer/jen"
@ -50,14 +51,14 @@ func NewGenerator(gc Config, dbc config.Database) *generator {
}
func (g *generator) GenerateExporterPlugin() error {
if g.GenConfig == nil {
return errors.New("generator needs a config file")
}
if g.GenConfig.FilePath == "" {
return errors.New("generator is missing file path")
}
if len(g.GenConfig.Initializers) < 1 {
return errors.New("generator needs to be configured with imports")
return errors.New("generator needs to be configured with TransformerInitializer import paths")
}
if len(g.GenConfig.Dependencies) < 1 {
return errors.New("generator needs to be configured with root repository path(s)")
}
if len(g.GenConfig.Migrations) < 1 {
fmt.Fprintf(os.Stderr, "warning: no db migration paths have been provided\r\n")
}
// Get plugin file paths
@ -66,14 +67,8 @@ func (g *generator) GenerateExporterPlugin() error {
return err
}
// Clear .go and .so files of the same name if they exist (overwrite)
err = utils.ClearFiles(goFile, soFile)
if err != nil {
return err
}
// Generate Exporter code
err = g.generateCode(goFile)
err = g.generateCode(goFile, soFile)
if err != nil {
return err
}
@ -83,24 +78,32 @@ func (g *generator) GenerateExporterPlugin() error {
if err != nil {
return err
}
defer g.cleanUp() // Clear these up when we are done building our plugin
// Clear tmp files and directories when we exit
defer g.cleanUp(goFile)
// Build the .go file into a .so plugin
err = exec.Command("go", "build", "-buildmode=plugin", "-o", soFile, goFile).Run()
if err != nil {
return err
return errors.New(fmt.Sprintf("unable to build .so file: %s", err.Error()))
}
// Run migrations only after successfully building .so file
return g.runMigrations()
}
// Generates the plugin code
func (g *generator) generateCode(goFile string) error {
func (g *generator) generateCode(goFile, soFile string) error {
// Clear .go and .so files of the same name if they exist
err := utils.ClearFiles(goFile, soFile)
if err != nil {
return err
}
// Begin code generation
f := NewFile("main")
f.HeaderComment("This exporter is generated to export the configured transformer initializers")
// Import TransformerInitializers
// Import TransformerInitializers specified in config
f.ImportAlias("github.com/vulcanize/vulcanizedb/libraries/shared/transformer", "interface")
for alias, imp := range g.GenConfig.Initializers {
f.ImportAlias(imp, alias)
@ -112,12 +115,10 @@ func (g *generator) generateCode(goFile string) error {
importedInitializers = append(importedInitializers, Qual(path, "TransformerInitializer"))
}
// Create Exporter variable with method to export a set of the configured TransformerInitializers
// Create Exporter variable with method to export the set of the imported TransformerInitializers
f.Type().Id("exporter").String()
f.Var().Id("Exporter").Id("exporter")
f.Func().Params(
Id("e").Id("exporter"),
).Id("Export").Params().Index().Qual(
f.Func().Params(Id("e").Id("exporter")).Id("Export").Params().Index().Qual(
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer",
"TransformerInitializer").Block(
Return(Index().Qual(
@ -128,6 +129,49 @@ func (g *generator) generateCode(goFile string) error {
return f.Save(goFile)
}
// Sets up temporary vendor libs and migration directories
func (g *generator) setupTempDirs() error {
// TODO: Less hacky way of handling plugin build deps
dirPath, err := utils.CleanPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/")
if err != nil {
return err
}
vendorPath := filepath.Join(dirPath, "vendor")
// Keep track of where we are writing transformer vendor libs, so that we can remove them afterwards
g.tmpVenDirs = make([]string, 0, len(g.GenConfig.Dependencies))
// Import transformer dependencies so that we build our plugin
for name, importPath := range g.GenConfig.Dependencies {
index := strings.Index(importPath, "/")
gitPath := importPath[:index] + ":" + importPath[index+1:]
importURL := "git@" + gitPath + ".git"
depPath := filepath.Join(vendorPath, importPath)
err = exec.Command("git", "clone", importURL, depPath).Run()
if err != nil {
return errors.New(fmt.Sprintf("unable to clone %s transformer dependency: %s", name, err.Error()))
}
err := os.RemoveAll(filepath.Join(depPath, "vendor/"))
if err != nil {
return err
}
g.tmpVenDirs = append(g.tmpVenDirs, depPath)
}
// Initialize temp directory for transformer migrations
g.tmpMigDir, err = utils.CleanPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/db/plugin_migrations")
if err != nil {
return err
}
err = os.RemoveAll(g.tmpMigDir)
if err != nil {
return errors.New(fmt.Sprintf("unable to remove file found at %s where tmp directory needs to be written", g.tmpMigDir))
}
return os.Mkdir(g.tmpMigDir, os.FileMode(0777))
}
func (g *generator) runMigrations() error {
// Get paths to db migrations
paths, err := g.GenConfig.GetMigrationsPaths()
@ -149,78 +193,8 @@ func (g *generator) runMigrations() error {
}
// Run the copied migrations
location := "file://" + g.tmpMigDir
pgStr := fmt.Sprintf("postgres://%s:%d/%s?sslmode=disable up", g.DBConfig.Hostname, g.DBConfig.Port, g.DBConfig.Name)
return exec.Command("migrate", "-source", location, pgStr).Run()
}
// Sets up temporary vendor libs and migration directories
func (g *generator) setupTempDirs() error {
// TODO: Less hacky way of handling plugin build deps
dirPath, err := utils.CleanPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/")
if err != nil {
return err
}
vendorPath := filepath.Join(dirPath, "vendor/")
/*
// Keep track of where we are writing transformer vendor libs, so that we can remove them afterwards
g.tmpVenDirs = make([]string, 0, len(g.GenConfig.Dependencies))
// Import transformer dependencies so that we build our plugin
for _, importPath := range g.GenConfig.Dependencies {
importURL := "https://" + importPath + ".git"
depPath := filepath.Join(vendorPath, importPath)
err = exec.Command("git", "clone", importURL, depPath).Run()
if err != nil {
return err
}
err := os.RemoveAll(filepath.Join(depPath, "vendor/"))
if err != nil {
return err
}
g.tmpVenDirs = append(g.tmpVenDirs, depPath)
}
*/
// Keep track of where we are writing transformer vendor libs, so that we can remove them afterwards
g.tmpVenDirs = make([]string, 0, len(g.GenConfig.Dependencies))
for _, importPath := range g.GenConfig.Dependencies {
depPath := filepath.Join(vendorPath, importPath)
g.tmpVenDirs = append(g.tmpVenDirs, depPath)
}
// Dep ensure to make sure vendor pkgs are in place for building the plugin
err = exec.Command("dep", "ensure").Run()
if err != nil {
return errors.New("failed to vendor transformer packages required to build plugin")
}
// Git checkout our head-state vendor libraries
// This is necessary because we currently need to manual edit our vendored
// go-ethereum abi library to allow for unpacking in empty interfaces and maps
// This can be removed once the PRs against geth merged
err = exec.Command("git", "checkout", dirPath).Run()
if err != nil {
return errors.New("failed to checkout vendored go-ethereum lib")
}
// Initialize temp directory for transformer migrations
g.tmpMigDir, err = utils.CleanPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/db/plugin_migrations")
if err != nil {
return err
}
stat, err := os.Stat(g.tmpMigDir)
if err == nil {
if !stat.IsDir() {
return errors.New(fmt.Sprintf("file %s found where directory is expected", stat.Name()))
}
} else if os.IsNotExist(err) {
os.Mkdir(g.tmpMigDir, os.FileMode(0777))
} else {
return err
}
return nil
pgStr := fmt.Sprintf("postgres://%s:%d/%s?sslmode=disable", g.DBConfig.Hostname, g.DBConfig.Port, g.DBConfig.Name)
return exec.Command("migrate", "-path", g.tmpMigDir, "-database", pgStr, "up").Run()
}
func (g *generator) createMigrationCopies(paths []string) error {
@ -230,21 +204,19 @@ func (g *generator) createMigrationCopies(paths []string) error {
return err
}
for _, file := range dir {
if file.IsDir() || len(file.Name()) < 15 { // (10 digit unix time stamp + x + .sql) is bare minimum
if file.IsDir() || len(file.Name()) < 15 || filepath.Ext(file.Name()) != ".sql" { // (10 digit unix time stamp + x + .sql) is bare minimum
continue
}
_, err := strconv.Atoi(file.Name()[:10])
if err != nil {
fmt.Fprintf(os.Stderr, "migration file name %s does not posses 10 digit timestamp prefix", file.Name())
fmt.Fprintf(os.Stderr, "migration file name %s does not posses 10 digit timestamp prefix\r\n", file.Name())
continue
}
if filepath.Ext(file.Name()) == "sql" {
src := filepath.Join(path, file.Name())
dst := filepath.Join(g.tmpMigDir, "1"+file.Name())
err = utils.CopyFile(src, dst)
if err != nil {
return err
}
src := filepath.Join(path, file.Name())
dst := filepath.Join(g.tmpMigDir, "1"+file.Name())
err = utils.CopyFile(src, dst)
if err != nil {
return err
}
}
}
@ -252,7 +224,14 @@ func (g *generator) createMigrationCopies(paths []string) error {
return nil
}
func (g *generator) cleanUp() error {
func (g *generator) cleanUp(goFile string) error {
if !g.GenConfig.Save {
err := utils.ClearFiles(goFile)
if err != nil {
return err
}
}
for _, venDir := range g.tmpVenDirs {
err := os.RemoveAll(venDir)
if err != nil {

208
pkg/autogen/generator_test.go
View File

@ -31,20 +31,10 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/bite"
"github.com/vulcanize/vulcanizedb/utils"
)
var localConfig = autogen.Config{
Initializers: map[string]string{
"bite": "github.com/vulcanize/vulcanizedb/pkg/autogen/test_helpers/bite",
"deal": "github.com/vulcanize/vulcanizedb/pkg/autogen/test_helpers/deal",
},
FileName: "localTestTransformerSet",
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/autogen/test_helpers/test/",
}
var externalConfig = autogen.Config{
var genConfig = autogen.Config{
Initializers: map[string]string{
"bite": "github.com/vulcanize/mcd_transformers/transformers/bite",
"deal": "github.com/vulcanize/mcd_transformers/transformers/deal",
@ -53,7 +43,8 @@ var externalConfig = autogen.Config{
"mcd_transformers": "github.com/vulcanize/mcd_transformers",
},
FileName: "externalTestTransformerSet",
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/autogen/test_helpers/test/",
FilePath: "$GOPATH/src/github.com/vulcanize/vulcanizedb/pkg/autogen/test_helpers/test",
Save: false,
}
type Exporter interface {
@ -71,147 +62,82 @@ var _ = Describe("Generator test", func() {
viper.SetConfigName("compose")
viper.AddConfigPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/environments/")
Describe("Using local config", func() {
BeforeEach(func() {
goPath, soPath, err = localConfig.GetPluginPaths()
Expect(err).ToNot(HaveOccurred())
g = autogen.NewGenerator(localConfig, config.Database{})
err = g.GenerateExporterPlugin()
Expect(err).ToNot(HaveOccurred())
})
AfterEach(func() {
err := utils.ClearFiles(goPath, soPath)
Expect(err).ToNot(HaveOccurred())
})
Describe("GenerateTransformerPlugin", func() {
It("It bundles the specified transformer initializers into a Exporter object and creates .so", func() {
plug, err := plugin.Open(soPath)
Expect(err).ToNot(HaveOccurred())
symExporter, err := plug.Lookup("Exporter")
Expect(err).ToNot(HaveOccurred())
exporter, ok := symExporter.(Exporter)
Expect(ok).To(Equal(true))
initializers := exporter.Export()
Expect(len(initializers)).To(Equal(2))
})
It("Loads our generated Exporter and uses it to import an arbitrary set of TransformerInitializers that we can execute over", func() {
db, bc = test_helpers.SetupDBandBC()
defer test_helpers.TearDown(db)
hr = repositories.NewHeaderRepository(db)
header1, err := bc.GetHeaderByNumber(9377319)
Expect(err).ToNot(HaveOccurred())
headerID, err = hr.CreateOrUpdateHeader(header1)
Expect(err).ToNot(HaveOccurred())
plug, err := plugin.Open(soPath)
Expect(err).ToNot(HaveOccurred())
symExporter, err := plug.Lookup("Exporter")
Expect(err).ToNot(HaveOccurred())
exporter, ok := symExporter.(Exporter)
Expect(ok).To(Equal(true))
initializers := exporter.Export()
w := watcher.NewWatcher(db, bc)
w.AddTransformers(initializers)
err = w.Execute()
Expect(err).ToNot(HaveOccurred())
type model struct {
bite.BiteModel
Id int64 `db:"id"`
HeaderId int64 `db:"header_id"`
}
returned := model{}
err = db.Get(&returned, `SELECT * FROM maker.bite WHERE header_id = $1`, headerID)
Expect(err).ToNot(HaveOccurred())
Expect(returned.Ilk).To(Equal("ETH"))
Expect(returned.Urn).To(Equal("0x0000d8b4147eDa80Fec7122AE16DA2479Cbd7ffB"))
Expect(returned.Ink).To(Equal("80000000000000000000"))
Expect(returned.Art).To(Equal("11000000000000000000000"))
Expect(returned.IArt).To(Equal("12496609999999999999992"))
Expect(returned.Tab).To(Equal("11000000000000000000000"))
Expect(returned.NFlip).To(Equal("7"))
Expect(returned.TransactionIndex).To(Equal(uint(1)))
Expect(returned.LogIndex).To(Equal(uint(4)))
})
})
BeforeEach(func() {
goPath, soPath, err = genConfig.GetPluginPaths()
Expect(err).ToNot(HaveOccurred())
g = autogen.NewGenerator(genConfig, config.Database{})
err = g.GenerateExporterPlugin()
Expect(err).ToNot(HaveOccurred())
})
Describe("Using external config", func() {
BeforeEach(func() {
goPath, soPath, err = externalConfig.GetPluginPaths()
AfterEach(func() {
err := utils.ClearFiles(goPath, soPath)
Expect(err).ToNot(HaveOccurred())
})
Describe("GenerateTransformerPlugin", func() {
It("It bundles the specified transformer initializers into a Exporter object and creates .so", func() {
plug, err := plugin.Open(soPath)
Expect(err).ToNot(HaveOccurred())
g = autogen.NewGenerator(externalConfig, config.Database{})
err = g.GenerateExporterPlugin()
symExporter, err := plug.Lookup("Exporter")
Expect(err).ToNot(HaveOccurred())
exporter, ok := symExporter.(Exporter)
Expect(ok).To(Equal(true))
initializers := exporter.Export()
Expect(len(initializers)).To(Equal(2))
})
AfterEach(func() {
err := utils.ClearFiles(goPath, soPath)
It("Loads our generated Exporter and uses it to import an arbitrary set of TransformerInitializers that we can execute over", func() {
db, bc = test_helpers.SetupDBandBC()
defer test_helpers.TearDown(db)
hr = repositories.NewHeaderRepository(db)
header1, err := bc.GetHeaderByNumber(9377319)
Expect(err).ToNot(HaveOccurred())
headerID, err = hr.CreateOrUpdateHeader(header1)
Expect(err).ToNot(HaveOccurred())
})
Describe("GenerateTransformerPlugin", func() {
It("It bundles the specified transformer initializers into a Exporter object and creates .so", func() {
plug, err := plugin.Open(soPath)
Expect(err).ToNot(HaveOccurred())
symExporter, err := plug.Lookup("Exporter")
Expect(err).ToNot(HaveOccurred())
exporter, ok := symExporter.(Exporter)
Expect(ok).To(Equal(true))
initializers := exporter.Export()
Expect(len(initializers)).To(Equal(2))
})
plug, err := plugin.Open(soPath)
Expect(err).ToNot(HaveOccurred())
symExporter, err := plug.Lookup("Exporter")
Expect(err).ToNot(HaveOccurred())
exporter, ok := symExporter.(Exporter)
Expect(ok).To(Equal(true))
initializers := exporter.Export()
It("Loads our generated Exporter and uses it to import an arbitrary set of TransformerInitializers that we can execute over", func() {
db, bc = test_helpers.SetupDBandBC()
defer test_helpers.TearDown(db)
w := watcher.NewWatcher(db, bc)
w.AddTransformers(initializers)
err = w.Execute()
Expect(err).ToNot(HaveOccurred())
hr = repositories.NewHeaderRepository(db)
header1, err := bc.GetHeaderByNumber(9377319)
Expect(err).ToNot(HaveOccurred())
headerID, err = hr.CreateOrUpdateHeader(header1)
Expect(err).ToNot(HaveOccurred())
type model struct {
Ilk string
Urn string
Ink string
Art string
IArt string
Tab string
NFlip string
LogIndex uint `db:"log_idx"`
TransactionIndex uint `db:"tx_idx"`
Raw []byte `db:"raw_log"`
Id int64 `db:"id"`
HeaderId int64 `db:"header_id"`
}
plug, err := plugin.Open(soPath)
Expect(err).ToNot(HaveOccurred())
symExporter, err := plug.Lookup("Exporter")
Expect(err).ToNot(HaveOccurred())
exporter, ok := symExporter.(Exporter)
Expect(ok).To(Equal(true))
initializers := exporter.Export()
returned := model{}
w := watcher.NewWatcher(db, bc)
w.AddTransformers(initializers)
err = w.Execute()
Expect(err).ToNot(HaveOccurred())
type model struct {
bite.BiteModel
Id int64 `db:"id"`
HeaderId int64 `db:"header_id"`
}
returned := model{}
err = db.Get(&returned, `SELECT * FROM maker.bite WHERE header_id = $1`, headerID)
Expect(err).ToNot(HaveOccurred())
Expect(returned.Ilk).To(Equal("ETH"))
Expect(returned.Urn).To(Equal("0x0000d8b4147eDa80Fec7122AE16DA2479Cbd7ffB"))
Expect(returned.Ink).To(Equal("80000000000000000000"))
Expect(returned.Art).To(Equal("11000000000000000000000"))
Expect(returned.IArt).To(Equal("12496609999999999999992"))
Expect(returned.Tab).To(Equal("11000000000000000000000"))
Expect(returned.NFlip).To(Equal("7"))
Expect(returned.TransactionIndex).To(Equal(uint(1)))
Expect(returned.LogIndex).To(Equal(uint(4)))
})
err = db.Get(&returned, `SELECT * FROM maker.bite WHERE header_id = $1`, headerID)
Expect(err).ToNot(HaveOccurred())
Expect(returned.Ilk).To(Equal("ETH"))
Expect(returned.Urn).To(Equal("0x0000d8b4147eDa80Fec7122AE16DA2479Cbd7ffB"))
Expect(returned.Ink).To(Equal("80000000000000000000"))
Expect(returned.Art).To(Equal("11000000000000000000000"))
Expect(returned.IArt).To(Equal("12496609999999999999992"))
Expect(returned.Tab).To(Equal("11000000000000000000000"))
Expect(returned.NFlip).To(Equal("7"))
Expect(returned.TransactionIndex).To(Equal(uint(1)))
Expect(returned.LogIndex).To(Equal(uint(4)))
})
})
})

8
pkg/autogen/test_helpers/bite/initializer.go
View File

@ -1,8 +0,0 @@
package bite
import (
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
"github.com/vulcanize/vulcanizedb/pkg/transformers"
)
var TransformerInitializer transformer.TransformerInitializer = transformers.GetBiteTransformer().NewTransformer

8
pkg/autogen/test_helpers/deal/initializer.go
View File

@ -1,8 +0,0 @@
package deal
import (
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
"github.com/vulcanize/vulcanizedb/pkg/transformers"
)
var TransformerInitializer transformer.TransformerInitializer = transformers.GetDealTransformer().NewLogNoteTransformer

2
pkg/autogen/test_helpers/test/README.md
View File

@ -1,3 +1,3 @@
### Test
This empty directory is for holding the output code generated, and then deleted, during the generator_tests
This empty directory is for holding the temporary .so and .go files generated during the generator_tests

2
pkg/fakes/data.go
View File

@ -24,8 +24,8 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
)
var (

129
pkg/transformers/DOCUMENTATION.md
View File

@ -1,129 +0,0 @@
# Transformers
## Architecture
Transformers fetch logs from Ethereum, convert/decode them into usable data, and then persist them in postgres.
A transformer consists of:
- A fetcher -> Fetches raw logs from the blockchain and encodes them as go datatypes
- A converter -> Converts this raw data into a human friendly representation suitable for consumption in the API
- A repository -> Abstracts the database
For Maker, vulcanize will be run in `lightSync` mode, so it will store all headers, and then fetchers pull relevant logs by making RPC calls.
## Event Types
For Maker there are two main types of log events that we're tracking:
1. Custom events that are defined in the contract solidity code.
1. `LogNote` events which utilize the [DSNote library](https://github.com/dapphub/ds-note).
1. `Note` events in the `Vat`
The transformer process for each of these different log types is the same, except for the converting process, as denoted below.
## Creating a Transformer
1. Pull an example event (from kovan / ganache etc.)
1. Add event & method sig, contract address, `checked_headers` column name, and label to relevant files in [`constants`](./shared/constants)
1. Write a test for the event sig in [`event_signature_generator_test.go`](./shared/event_signature_generator_test.go)
1. Create DB table (using [`create_migration`](../../scripts/create_migration))
1. Create columns in `checked_headers` in the _same_ migration
1. Add a line to clean the new table `CleanTestDB` (in [`test_config.go`](../../test_config/test_config.go))
1. Define `model.go`
1. Create test event in [`test_data`](./test_data)
1. Write converter + converter tests
1. Write repository + repository tests
1. Create converter + repository mocks
1. Create an config object [`shared.TransformerConfig`](./shared/transformer.go) in `config.go`
1. Wire up transformer in [`transformers.go`](./transformers.go), remembering to add it to `TransformerInitializers()`
1. Wire up transformer in [`continuousLogSync.go`](../../cmd/continuousLogSync.go)
1. Manually trigger an event and check that it gets persisted to postgres
1. Create an integration test for the shiny new transformer in [`integration_tests`](./integration_tests)
**Fetching Logs**
1. Generate an example raw log event, by either:
- Pulling the log directly from the Kovan deployment ([address.go](https://github.com/8thlight/maker-vulcanizedb/blob/master/pkg/transformers/shared/constants/address.go)).
- Deploying the contract to a local chain and emiting the event manually.
1. Fetch the logs from the chain based on the example event's topic zero:
- The topic zero is based on the keccak-256 hash of the log event's method signature. These are located in [`pkg/transformers/shared/constants/signature.go`](./shared/constants/signature.go).
- Fetching is done in batch from the [`watcher`](../../libraries/shared/watcher.go).
- The logs are then chunked up by the [`chunker`](./shared/log_chunker.go) before being delegated to each transformer.
**Coverting logs**
- **Converting most custom events** (such as FlopKick)
1. Convert the raw log into a Go struct.
- We've been using [go-ethereum's abigen tool](https://github.com/ethereum/go-ethereum/tree/master/cmd/abigen) to get the contract's ABI, and a Go struct that represents the event log. We will unpack the raw logs into this struct.
- To use abigen: `abigen --sol flip.sol --pkg flip --out {/path/to/output_file}`
- sol: this is the path to the solidity contract
- pkg: a package name for the generated Go code
- out: the file path for the generated Go code (optional)
- the output for `flop.sol` will include the FlopperAbi and the FlopperKick struct:
```go
type FlopperKick struct {
Id *big.Int
Lot *big.Int
Bid *big.Int
Gal common.Address
End *big.Int
Raw types.Log
}
```
- Using go-ethereum's `contract.UnpackLog` method we can unpack the raw log into the FlopperKick struct (which we're referring to as the `entity`).
- See the `ToEntity` method in [`pkg/transformers/flop_kick/converter.go`](./flop_kick/converter.go).
1. Convert the entity into a database model. See the `ToModel` method in `pkg/transformers/flop_kick/converter`.
- **Converting Price Feed custom events**
- Price Feed contracts use the [LogNote event](https://github.com/makerdao/medianizer/blob/master/src/medianizer.sol#L23)
- The LogNote event takes in the value of the price feed as it's sole argument, and does not index it. This means that this value can be taken directly from the log's data, and then properly converted using the `price_feeds.Convert` method (located in the model.go file).
- Since this conversion from raw log to model includes less fields than some others, we've chosen to convert it directly to the database model, skipping the `ToEntity` step.
- **Converting LogNote events** (such as tend)
- Since LogNote events are a generic structure, they depend on the method signature of the method that is calling them. For example, the `tend` method is called on the [flip.sol contract](https://github.com/makerdao/dss/blob/master/src/flip.sol#L117), and it's method signature looks like this: `tend(uint,uint,uint)`.
- The first four bytes of the Keccak-256 hashed method signature will be located in `topic[0]` on the log.
- The message sender will be in `topic[1]`.
- The first parameter passed to `tend` becomes `topic[2]`.
- The second parameter passed to `tend` will be `topic[3]`.
- Any additional parameters will be in the log's data field.
- More detail is located in the [DSNote repo](https://github.com/dapphub/ds-note).
**Get all MissingHeaders**
- Headers are inserted into VulcanizeDB as part of the `lightSync` command. Then for each transformer we check each header for matching logs.
- The MissingHeaders method queries the `checked_headers` table to see if the header has been checked for the given log type.
**Persist the log record to VulcanizeDB**
- Each event log has it's own table in the database, as well as it's own column in the `checked_headers` table.
- The `checked_headers` table allows us to keep track of which headers have been checked for a given log type.
- To create a new migration file: `./scripts/create_migration create_flop_kick`
- See `db/migrations/1536942529_create_flop_kick.up.sql`.
- The specific log event tables are all created in the `maker` schema.
- There is a one-many association between `headers` and the log
event tables. This is so that if a header is removed due to a reorg, the associated log event records are also removed.
- To run the migrations: `make migrate HOST=local_host PORT=5432 NAME=vulcanize_private`
- When a new log record is inserted into VulcanizeDB, we also need to make sure to insert a record into the `checked_headers` table for the given log type.
- We have been using the repository pattern (i.e. wrapping all SQL/ORM invocations in isolated namespaces per table) to interact with the database, see the `Create` method in `pkg/transformers/flop_kick/repository.go`.
**MarkHeaderChecked**
- There is a chance that a header does not have a log for the given transformer's log type, and in this instance we also want to record that the header has been "checked" so that we don't continue to query that header over and over.
- In the transformer we'll make sure to insert a row for the header indicating that it has been checked for the log type that the transformer is responsible for.
**Wire each component up in the transformer**
- We use a [`TransformerInitializer`](./shared/transformer.go) struct for each transformer so that we can inject ethRPC and postgresDB connections as well as configuration data (including the contract address, block range, etc.) into the transformer.
- See any of `pkg/transformers/flop_kick/transformer.go`
- All of the transformers are then initialized in `pkg/transformers/transformers.go` with their configuration.
- The transformers can be executed by using the `continuousLogSync` command, which can be configured to run specific transformers or all transformers.
## Useful Documents
[Ethereum Event ABI Specification](https://solidity.readthedocs.io/en/develop/abi-spec.html#events)

35
pkg/transformers/bite/bite_suite_test.go
View File

@ -1,35 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package bite_test
import (
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
log "github.com/sirupsen/logrus"
"io/ioutil"
)
func TestBite(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Bite Suite")
}
var _ = BeforeSuite(func() {
log.SetOutput(ioutil.Discard)
})

33
pkg/transformers/bite/config.go
View File

@ -1,33 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package bite
import (
shared_t "github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
)
func GetBiteConfig() shared_t.TransformerConfig {
return shared_t.TransformerConfig{
TransformerName: constants.BiteLabel,
ContractAddresses: []string{constants.CatContractAddress()},
ContractAbi: constants.CatABI(),
Topic: constants.GetBiteSignature(),
StartingBlockNumber: constants.CatDeploymentBlock(),
EndingBlockNumber: -1,
}
}

94
pkg/transformers/bite/converter.go
View File

@ -1,94 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package bite
import (
"encoding/json"
"fmt"
"github.com/ethereum/go-ethereum/accounts/abi/bind"
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/geth"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
)
type BiteConverter struct{}
func (BiteConverter) ToEntities(contractAbi string, ethLogs []types.Log) ([]interface{}, error) {
var entities []interface{}
for _, ethLog := range ethLogs {
entity := &BiteEntity{}
address := ethLog.Address
abi, err := geth.ParseAbi(contractAbi)
if err != nil {
return nil, err
}
contract := bind.NewBoundContract(address, abi, nil, nil, nil)
err = contract.UnpackLog(entity, "Bite", ethLog)
if err != nil {
return nil, err
}
entity.Raw = ethLog
entity.LogIndex = ethLog.Index
entity.TransactionIndex = ethLog.TxIndex
entities = append(entities, *entity)
}
return entities, nil
}
func (converter BiteConverter) ToModels(entities []interface{}) ([]interface{}, error) {
var models []interface{}
for _, entity := range entities {
biteEntity, ok := entity.(BiteEntity)
if !ok {
return nil, fmt.Errorf("entity of type %T, not %T", entity, BiteEntity{})
}
ilk := shared.GetHexWithoutPrefix(biteEntity.Ilk[:])
urn := shared.GetHexWithoutPrefix(biteEntity.Urn[:])
ink := biteEntity.Ink
art := biteEntity.Art
iArt := biteEntity.IArt
tab := biteEntity.Tab
flip := biteEntity.Flip
logIdx := biteEntity.LogIndex
txIdx := biteEntity.TransactionIndex
rawLog, err := json.Marshal(biteEntity.Raw)
if err != nil {
return nil, err
}
model := BiteModel{
Ilk: ilk,
Urn: urn,
Ink: shared.BigIntToString(ink),
Art: shared.BigIntToString(art),
IArt: shared.BigIntToString(iArt),
Tab: shared.BigIntToString(tab),
NFlip: shared.BigIntToString(flip),
LogIndex: logIdx,
TransactionIndex: txIdx,
Raw: rawLog,
}
models = append(models, model)
}
return models, nil
}

91
pkg/transformers/bite/converter_test.go
View File

@ -1,91 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package bite_test
import (
"encoding/json"
"github.com/ethereum/go-ethereum/core/types"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/transformers/bite"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
)
var _ = Describe("Bite Converter", func() {
var converter = bite.BiteConverter{}
Describe("ToEntity", func() {
It("converts an eth log to a bite entity", func() {
entities, err := converter.ToEntities(test_data.KovanCatABI, []types.Log{test_data.EthBiteLog})
Expect(err).NotTo(HaveOccurred())
Expect(len(entities)).To(Equal(1))
entity := entities[0]
Expect(entity).To(Equal(test_data.BiteEntity))
})
It("returns an error if converting log to entity fails", func() {
_, err := converter.ToEntities("error abi", []types.Log{test_data.EthBiteLog})
Expect(err).To(HaveOccurred())
})
})
Describe("ToModel", func() {
var emptyEntity = bite.BiteEntity{}
It("converts an Entity to a Model", func() {
models, err := converter.ToModels([]interface{}{test_data.BiteEntity})
Expect(err).NotTo(HaveOccurred())
Expect(len(models)).To(Equal(1))
model := models[0]
Expect(model).To(Equal(test_data.BiteModel))
})
It("returns an error if the entity type is wrong", func() {
_, err := converter.ToModels([]interface{}{test_data.WrongEntity{}})
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("entity of type test_data.WrongEntity, not bite.BiteEntity"))
})
It("handles nil values", func() {
emptyLog, err := json.Marshal(types.Log{})
Expect(err).NotTo(HaveOccurred())
expectedModel := bite.BiteModel{
Ilk: "0000000000000000000000000000000000000000000000000000000000000000",
Urn: "0000000000000000000000000000000000000000000000000000000000000000",
Ink: "",
Art: "",
IArt: "",
Tab: "",
NFlip: "",
TransactionIndex: 0,
Raw: emptyLog,
}
models, err := converter.ToModels([]interface{}{emptyEntity})
Expect(err).NotTo(HaveOccurred())
Expect(len(models)).To(Equal(1))
model := models[0]
Expect(model).To(Equal(expectedModel))
})
})
})

35
pkg/transformers/bite/entity.go
View File

@ -1,35 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package bite
import (
"github.com/ethereum/go-ethereum/core/types"
"math/big"
)
type BiteEntity struct {
Ilk [32]byte
Urn [32]byte
Ink *big.Int
Art *big.Int
Tab *big.Int
Flip *big.Int
IArt *big.Int
LogIndex uint
TransactionIndex uint
Raw types.Log
}

30
pkg/transformers/bite/model.go
View File

@ -1,30 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package bite
type BiteModel struct {
Ilk string
Urn string
Ink string
Art string
IArt string
Tab string
NFlip string
LogIndex uint `db:"log_idx"`
TransactionIndex uint `db:"tx_idx"`
Raw []byte `db:"raw_log"`
}

35
pkg/transformers/cat_file/chop_lump/chop_lump_suite_test.go
View File

@ -1,35 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package chop_lump_test
import (
log "github.com/sirupsen/logrus"
"io/ioutil"
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
func TestChopLump(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "ChopLump Suite")
}
var _ = BeforeSuite(func() {
log.SetOutput(ioutil.Discard)
})

33
pkg/transformers/cat_file/chop_lump/config.go
View File

@ -1,33 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package chop_lump
import (
shared_t "github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
)
func GetCatFileChopLumpConfig() shared_t.TransformerConfig {
return shared_t.TransformerConfig{
TransformerName: constants.CatFileChopLumpLabel,
ContractAddresses: []string{constants.CatContractAddress()},
ContractAbi: constants.CatABI(),
Topic: constants.GetCatFileChopLumpSignature(),
StartingBlockNumber: constants.CatDeploymentBlock(),
EndingBlockNumber: -1,
}
}

84
pkg/transformers/cat_file/chop_lump/converter.go
View File

@ -1,84 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package chop_lump
import (
"bytes"
"encoding/json"
"errors"
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
"math/big"
)
var (
chop = "chop"
lump = "lump"
)
type CatFileChopLumpConverter struct{}
func (CatFileChopLumpConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) {
var results []interface{}
for _, ethLog := range ethLogs {
err := verifyLog(ethLog)
if err != nil {
return nil, err
}
ilk := shared.GetHexWithoutPrefix(ethLog.Topics[2].Bytes())
what := string(bytes.Trim(ethLog.Topics[3].Bytes(), "\x00"))
dataBytes := ethLog.Data[len(ethLog.Data)-constants.DataItemLength:]
data := big.NewInt(0).SetBytes(dataBytes).String()
raw, err := json.Marshal(ethLog)
if err != nil {
return nil, err
}
result := CatFileChopLumpModel{
Ilk: ilk,
What: what,
Data: convertData(what, data),
TransactionIndex: ethLog.TxIndex,
LogIndex: ethLog.Index,
Raw: raw,
}
results = append(results, result)
}
return results, nil
}
func convertData(what, data string) string {
var convertedData string
if what == chop {
convertedData = shared.ConvertToRay(data)
} else if what == lump {
convertedData = shared.ConvertToWad(data)
}
return convertedData
}
func verifyLog(log types.Log) error {
if len(log.Topics) < 4 {
return errors.New("log missing topics")
}
if len(log.Data) < constants.DataItemLength {
return errors.New("log missing data")
}
return nil
}

71
pkg/transformers/cat_file/chop_lump/converter_test.go
View File

@ -1,71 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package chop_lump_test
import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/transformers/cat_file/chop_lump"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
)
var _ = Describe("Cat file chop lump converter", func() {
var converter chop_lump.CatFileChopLumpConverter
BeforeEach(func() {
converter = chop_lump.CatFileChopLumpConverter{}
})
Context("chop events", func() {
It("converts a chop log to a model", func() {
models, err := converter.ToModels([]types.Log{test_data.EthCatFileChopLog})
Expect(err).NotTo(HaveOccurred())
Expect(models).To(Equal([]interface{}{test_data.CatFileChopModel}))
})
})
Context("lump events", func() {
It("converts a lump log to a model", func() {
models, err := converter.ToModels([]types.Log{test_data.EthCatFileLumpLog})
Expect(err).NotTo(HaveOccurred())
Expect(models).To(Equal([]interface{}{test_data.CatFileLumpModel}))
})
})
It("returns err if log is missing topics", func() {
badLog := types.Log{
Data: []byte{1, 1, 1, 1, 1},
}
_, err := converter.ToModels([]types.Log{badLog})
Expect(err).To(HaveOccurred())
})
It("returns err if log is missing data", func() {
badLog := types.Log{
Topics: []common.Hash{{}, {}, {}, {}},
}
_, err := converter.ToModels([]types.Log{badLog})
Expect(err).To(HaveOccurred())
})
})

26
pkg/transformers/cat_file/chop_lump/model.go
View File

@ -1,26 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package chop_lump
type CatFileChopLumpModel struct {
Ilk string
What string
Data string
TransactionIndex uint `db:"tx_idx"`
LogIndex uint `db:"log_idx"`
Raw []byte `db:"raw_log"`
}

33
pkg/transformers/cat_file/flip/config.go
View File

@ -1,33 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package flip
import (
shared_t "github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
)
func GetCatFileFlipConfig() shared_t.TransformerConfig {
return shared_t.TransformerConfig{
TransformerName: constants.CatFileFlipLabel,
ContractAddresses: []string{constants.CatContractAddress()},
ContractAbi: constants.CatABI(),
Topic: constants.GetCatFileFlipSignature(),
StartingBlockNumber: constants.CatDeploymentBlock(),
EndingBlockNumber: -1,
}
}

68
pkg/transformers/cat_file/flip/converter.go
View File

@ -1,68 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package flip
import (
"bytes"
"encoding/json"
"errors"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
)
type CatFileFlipConverter struct{}
func (CatFileFlipConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) {
var results []interface{}
for _, ethLog := range ethLogs {
err := verifyLog(ethLog)
if err != nil {
return nil, err
}
ilk := shared.GetHexWithoutPrefix(ethLog.Topics[2].Bytes())
what := string(bytes.Trim(ethLog.Topics[3].Bytes(), "\x00"))
flipBytes := ethLog.Data[len(ethLog.Data)-constants.DataItemLength:]
flip := common.BytesToAddress(flipBytes).String()
raw, err := json.Marshal(ethLog)
if err != nil {
return nil, err
}
result := CatFileFlipModel{
Ilk: ilk,
What: what,
Flip: flip,
TransactionIndex: ethLog.TxIndex,
LogIndex: ethLog.Index,
Raw: raw,
}
results = append(results, result)
}
return results, nil
}
func verifyLog(log types.Log) error {
if len(log.Topics) < 4 {
return errors.New("log missing topics")
}
if len(log.Data) < constants.DataItemLength {
return errors.New("log missing data")
}
return nil
}

60
pkg/transformers/cat_file/flip/converter_test.go
View File

@ -1,60 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package flip_test
import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/transformers/cat_file/flip"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
)
var _ = Describe("Cat file flip converter", func() {
var converter flip.CatFileFlipConverter
BeforeEach(func() {
converter = flip.CatFileFlipConverter{}
})
It("returns err if log is missing topics", func() {
badLog := types.Log{
Data: []byte{1, 1, 1, 1, 1},
}
_, err := converter.ToModels([]types.Log{badLog})
Expect(err).To(HaveOccurred())
})
It("returns err if log is missing data", func() {
badLog := types.Log{
Topics: []common.Hash{{}, {}, {}, {}},
}
_, err := converter.ToModels([]types.Log{badLog})
Expect(err).To(HaveOccurred())
})
It("converts a log to an model", func() {
models, err := converter.ToModels([]types.Log{test_data.EthCatFileFlipLog})
Expect(err).NotTo(HaveOccurred())
Expect(models).To(Equal([]interface{}{test_data.CatFileFlipModel}))
})
})

35
pkg/transformers/cat_file/flip/flip_suite_test.go
View File

@ -1,35 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package flip_test
import (
log "github.com/sirupsen/logrus"
"io/ioutil"
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
func TestFlip(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Flip Suite")
}
var _ = BeforeSuite(func() {
log.SetOutput(ioutil.Discard)
})

26
pkg/transformers/cat_file/flip/model.go
View File

@ -1,26 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package flip
type CatFileFlipModel struct {
Ilk string
What string
Flip string
TransactionIndex uint `db:"tx_idx"`
LogIndex uint `db:"log_idx"`
Raw []byte `db:"raw_log"`
}

89
pkg/transformers/cat_file/flip/repository.go
View File

@ -1,89 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package flip
import (
"fmt"
log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
)
type CatFileFlipRepository struct {
db *postgres.DB
}
func (repository CatFileFlipRepository) Create(headerID int64, models []interface{}) error {
tx, dBaseErr := repository.db.Begin()
if dBaseErr != nil {
return dBaseErr
}
for _, model := range models {
flip, ok := model.(CatFileFlipModel)
if !ok {
rollbackErr := tx.Rollback()
if rollbackErr != nil {
log.Error("failed to rollback ", rollbackErr)
}
return fmt.Errorf("model of type %T, not %T", model, CatFileFlipModel{})
}
_, execErr := repository.db.Exec(
`INSERT into maker.cat_file_flip (header_id, ilk, what, flip, tx_idx, log_idx, raw_log)
VALUES($1, $2, $3, $4, $5, $6, $7)
ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET ilk = $2, what = $3, flip = $4, raw_log = $7;`,
headerID, flip.Ilk, flip.What, flip.Flip, flip.TransactionIndex, flip.LogIndex, flip.Raw,
)
if execErr != nil {
rollbackErr := tx.Rollback()
if rollbackErr != nil {
log.Error("failed to rollback ", rollbackErr)
}
return execErr
}
}
checkHeaderErr := shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.CatFileFlipChecked)
if checkHeaderErr != nil {
rollbackErr := tx.Rollback()
if rollbackErr != nil {
log.Error("failed to rollback ", rollbackErr)
}
return checkHeaderErr
}
return tx.Commit()
}
func (repository CatFileFlipRepository) MarkHeaderChecked(headerID int64) error {
return shared.MarkHeaderChecked(headerID, repository.db, constants.CatFileFlipChecked)
}
func (repository CatFileFlipRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.CatFileFlipChecked)
}
func (repository CatFileFlipRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.CatFileFlipChecked)
}
func (repository *CatFileFlipRepository) SetDB(db *postgres.DB) {
repository.db = db
}

88
pkg/transformers/cat_file/flip/repository_test.go
View File

@ -1,88 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package flip_test
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/datastore"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/transformers/cat_file/flip"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/shared_behaviors"
"github.com/vulcanize/vulcanizedb/test_config"
)
var _ = Describe("Cat file flip repository", func() {
var (
catFileFlipRepository flip.CatFileFlipRepository
db *postgres.DB
headerRepository datastore.HeaderRepository
)
BeforeEach(func() {
db = test_config.NewTestDB(test_config.NewTestNode())
test_config.CleanTestDB(db)
headerRepository = repositories.NewHeaderRepository(db)
catFileFlipRepository = flip.CatFileFlipRepository{}
catFileFlipRepository.SetDB(db)
})
Describe("Create", func() {
catFileFlipWithDifferentIdx := test_data.CatFileFlipModel
catFileFlipWithDifferentIdx.LogIndex++
inputs := shared_behaviors.CreateBehaviorInputs{
CheckedHeaderColumnName: constants.CatFileFlipChecked,
LogEventTableName: "maker.cat_file_flip",
TestModel: test_data.CatFileFlipModel,
ModelWithDifferentLogIdx: catFileFlipWithDifferentIdx,
Repository: &catFileFlipRepository,
}
shared_behaviors.SharedRepositoryCreateBehaviors(&inputs)
It("adds a cat file flip event", func() {
headerID, err := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader)
Expect(err).NotTo(HaveOccurred())
err = catFileFlipRepository.Create(headerID, []interface{}{test_data.CatFileFlipModel})
Expect(err).NotTo(HaveOccurred())
var dbResult flip.CatFileFlipModel
err = db.Get(&dbResult, `SELECT ilk, what, flip, tx_idx, log_idx, raw_log FROM maker.cat_file_flip WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(dbResult.Ilk).To(Equal(test_data.CatFileFlipModel.Ilk))
Expect(dbResult.What).To(Equal(test_data.CatFileFlipModel.What))
Expect(dbResult.Flip).To(Equal(test_data.CatFileFlipModel.Flip))
Expect(dbResult.TransactionIndex).To(Equal(test_data.CatFileFlipModel.TransactionIndex))
Expect(dbResult.LogIndex).To(Equal(test_data.CatFileFlipModel.LogIndex))
Expect(dbResult.Raw).To(MatchJSON(test_data.CatFileFlipModel.Raw))
})
})
Describe("MarkHeaderChecked", func() {
inputs := shared_behaviors.MarkedHeaderCheckedBehaviorInputs{
CheckedHeaderColumnName: constants.CatFileFlipChecked,
Repository: &catFileFlipRepository,
}
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
})
})

33
pkg/transformers/cat_file/pit_vow/config.go
View File

@ -1,33 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package pit_vow
import (
shared_t "github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
)
func GetCatFilePitVowConfig() shared_t.TransformerConfig {
return shared_t.TransformerConfig{
TransformerName: constants.CatFilePitVowLabel,
ContractAddresses: []string{constants.CatContractAddress()},
ContractAbi: constants.CatABI(),
Topic: constants.GetCatFilePitVowSignature(),
StartingBlockNumber: constants.CatDeploymentBlock(),
EndingBlockNumber: -1,
}
}

66
pkg/transformers/cat_file/pit_vow/converter.go
View File

@ -1,66 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package pit_vow
import (
"bytes"
"encoding/json"
"errors"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
)
type CatFilePitVowConverter struct{}
func (CatFilePitVowConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) {
var results []interface{}
for _, ethLog := range ethLogs {
err := verifyLog(ethLog)
if err != nil {
return nil, err
}
what := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
data := common.BytesToAddress(ethLog.Topics[3].Bytes()).String()
raw, err := json.Marshal(ethLog)
if err != nil {
return nil, err
}
result := CatFilePitVowModel{
What: what,
Data: data,
TransactionIndex: ethLog.TxIndex,
LogIndex: ethLog.Index,
Raw: raw,
}
results = append(results, result)
}
return results, nil
}
func verifyLog(log types.Log) error {
if len(log.Topics) < 4 {
return errors.New("log missing topics")
}
if len(log.Data) < constants.DataItemLength {
return errors.New("log missing data")
}
return nil
}

60
pkg/transformers/cat_file/pit_vow/converter_test.go
View File

@ -1,60 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package pit_vow_test
import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/transformers/cat_file/pit_vow"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
)
var _ = Describe("Cat file pit vow converter", func() {
var converter pit_vow.CatFilePitVowConverter
BeforeEach(func() {
converter = pit_vow.CatFilePitVowConverter{}
})
It("returns err if log is missing topics", func() {
badLog := types.Log{
Data: []byte{1, 1, 1, 1, 1},
}
_, err := converter.ToModels([]types.Log{badLog})
Expect(err).To(HaveOccurred())
})
It("returns err if log is missing data", func() {
badLog := types.Log{
Topics: []common.Hash{{}, {}, {}, {}},
}
_, err := converter.ToModels([]types.Log{badLog})
Expect(err).To(HaveOccurred())
})
It("converts a log to an model", func() {
models, err := converter.ToModels([]types.Log{test_data.EthCatFilePitVowLog})
Expect(err).NotTo(HaveOccurred())
Expect(models).To(Equal([]interface{}{test_data.CatFilePitVowModel}))
})
})

25
pkg/transformers/cat_file/pit_vow/model.go
View File

@ -1,25 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package pit_vow
type CatFilePitVowModel struct {
What string
Data string
TransactionIndex uint `db:"tx_idx"`
LogIndex uint `db:"log_idx"`
Raw []byte `db:"raw_log"`
}

35
pkg/transformers/cat_file/pit_vow/pit_vow_suite_test.go
View File

@ -1,35 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package pit_vow_test
import (
log "github.com/sirupsen/logrus"
"io/ioutil"
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
func TestPitVow(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "PitVow Suite")
}
var _ = BeforeSuite(func() {
log.SetOutput(ioutil.Discard)
})

89
pkg/transformers/cat_file/pit_vow/repository.go
View File

@ -1,89 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package pit_vow
import (
"fmt"
log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
)
type CatFilePitVowRepository struct {
db *postgres.DB
}
func (repository CatFilePitVowRepository) Create(headerID int64, models []interface{}) error {
tx, dBaseErr := repository.db.Begin()
if dBaseErr != nil {
return dBaseErr
}
for _, model := range models {
vow, ok := model.(CatFilePitVowModel)
if !ok {
rollbackErr := tx.Rollback()
if rollbackErr != nil {
log.Error("failed to rollback ", rollbackErr)
}
return fmt.Errorf("model of type %T, not %T", model, CatFilePitVowModel{})
}
_, execErr := repository.db.Exec(
`INSERT into maker.cat_file_pit_vow (header_id, what, data, tx_idx, log_idx, raw_log)
VALUES($1, $2, $3, $4, $5, $6)
ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET what = $2, data = $3, raw_log = $6;`,
headerID, vow.What, vow.Data, vow.TransactionIndex, vow.LogIndex, vow.Raw,
)
if execErr != nil {
rollbackErr := tx.Rollback()
if rollbackErr != nil {
log.Error("failed to rollback ", rollbackErr)
}
return execErr
}
}
checkHeaderErr := shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.CatFilePitVowChecked)
if checkHeaderErr != nil {
rollbackErr := tx.Rollback()
if rollbackErr != nil {
log.Error("failed to rollback ", rollbackErr)
}
return checkHeaderErr
}
return tx.Commit()
}
func (repository CatFilePitVowRepository) MarkHeaderChecked(headerID int64) error {
return shared.MarkHeaderChecked(headerID, repository.db, constants.CatFilePitVowChecked)
}
func (repository CatFilePitVowRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.CatFilePitVowChecked)
}
func (repository CatFilePitVowRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.CatFilePitVowChecked)
}
func (repository *CatFilePitVowRepository) SetDB(db *postgres.DB) {
repository.db = db
}

86
pkg/transformers/cat_file/pit_vow/repository_test.go
View File

@ -1,86 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package pit_vow_test
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/datastore"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/transformers/cat_file/pit_vow"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/shared_behaviors"
"github.com/vulcanize/vulcanizedb/test_config"
)
var _ = Describe("Cat file pit vow repository", func() {
var (
catFilePitVowRepository pit_vow.CatFilePitVowRepository
db *postgres.DB
headerRepository datastore.HeaderRepository
)
BeforeEach(func() {
db = test_config.NewTestDB(test_config.NewTestNode())
test_config.CleanTestDB(db)
headerRepository = repositories.NewHeaderRepository(db)
catFilePitVowRepository = pit_vow.CatFilePitVowRepository{}
catFilePitVowRepository.SetDB(db)
})
Describe("Create", func() {
modelWithDifferentLogIdx := test_data.CatFilePitVowModel
modelWithDifferentLogIdx.LogIndex++
inputs := shared_behaviors.CreateBehaviorInputs{
CheckedHeaderColumnName: constants.CatFilePitVowChecked,
LogEventTableName: "maker.cat_file_pit_vow",
TestModel: test_data.CatFilePitVowModel,
ModelWithDifferentLogIdx: modelWithDifferentLogIdx,
Repository: &catFilePitVowRepository,
}
shared_behaviors.SharedRepositoryCreateBehaviors(&inputs)
It("adds a cat file pit vow event", func() {
headerID, err := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader)
Expect(err).NotTo(HaveOccurred())
err = catFilePitVowRepository.Create(headerID, []interface{}{test_data.CatFilePitVowModel})
Expect(err).NotTo(HaveOccurred())
var dbResult pit_vow.CatFilePitVowModel
err = db.Get(&dbResult, `SELECT what, data, tx_idx, log_idx, raw_log FROM maker.cat_file_pit_vow WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(dbResult.What).To(Equal(test_data.CatFilePitVowModel.What))
Expect(dbResult.Data).To(Equal(test_data.CatFilePitVowModel.Data))
Expect(dbResult.TransactionIndex).To(Equal(test_data.CatFilePitVowModel.TransactionIndex))
Expect(dbResult.LogIndex).To(Equal(test_data.CatFilePitVowModel.LogIndex))
Expect(dbResult.Raw).To(MatchJSON(test_data.CatFilePitVowModel.Raw))
})
})
Describe("MarkHeaderChecked", func() {
inputs := shared_behaviors.MarkedHeaderCheckedBehaviorInputs{
CheckedHeaderColumnName: constants.CatFilePitVowChecked,
Repository: &catFilePitVowRepository,
}
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
})
})

35
pkg/transformers/deal/config.go
View File

@ -1,35 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package deal
import (
shared_t "github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
)
func GetDealConfig() shared_t.TransformerConfig {
return shared_t.TransformerConfig{
TransformerName: constants.DealLabel,
ContractAddresses: []string{constants.FlapperContractAddress(), constants.FlipperContractAddress(), constants.FlopperContractAddress()},
ContractAbi: constants.FlipperABI(),
Topic: constants.GetDealSignature(),
StartingBlockNumber: shared.MinInt64([]int64{
constants.FlapperDeploymentBlock(), constants.FlipperDeploymentBlock(), constants.FlopperDeploymentBlock()}),
EndingBlockNumber: -1,
}
}

59
pkg/transformers/deal/converter.go
View File

@ -1,59 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package deal
import (
"encoding/json"
"errors"
"github.com/ethereum/go-ethereum/core/types"
)
type DealConverter struct{}
func (DealConverter) ToModels(ethLogs []types.Log) (result []interface{}, err error) {
for _, log := range ethLogs {
err := validateLog(log)
if err != nil {
return nil, err
}
bidId := log.Topics[2].Big()
raw, err := json.Marshal(log)
if err != nil {
return nil, err
}
model := DealModel{
BidId: bidId.String(),
ContractAddress: log.Address.Hex(),
LogIndex: log.Index,
TransactionIndex: log.TxIndex,
Raw: raw,
}
result = append(result, model)
}
return result, nil
}
func validateLog(ethLog types.Log) error {
if len(ethLog.Topics) < 3 {
return errors.New("deal log does not contain expected topics")
}
return nil
}

50
pkg/transformers/deal/converter_test.go
View File

@ -1,50 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package deal_test
import (
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/transformers/deal"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
)
var _ = Describe("Flip Deal Converter", func() {
It("converts logs to models", func() {
converter := deal.DealConverter{}
models, err := converter.ToModels([]types.Log{test_data.DealLogNote})
Expect(err).NotTo(HaveOccurred())
Expect(len(models)).To(Equal(1))
Expect(models[0]).To(Equal(test_data.DealModel))
})
It("returns an error if the expected amount of topics aren't in the log", func() {
converter := deal.DealConverter{}
invalidLog := test_data.DealLogNote
invalidLog.Topics = []common.Hash{}
_, err := converter.ToModels([]types.Log{invalidLog})
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError("deal log does not contain expected topics"))
})
})

19
pkg/transformers/deal/deal_suite_test.go
View File

@ -1,19 +0,0 @@
package deal_test
import (
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
log "github.com/sirupsen/logrus"
"io/ioutil"
)
func TestFlipDeal(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Deal Suite")
}
var _ = BeforeSuite(func() {
log.SetOutput(ioutil.Discard)
})

25
pkg/transformers/deal/model.go
View File

@ -1,25 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package deal
type DealModel struct {
BidId string `db:"bid_id"`
ContractAddress string `db:"contract_address"`
LogIndex uint `db:"log_idx"`
TransactionIndex uint `db:"tx_idx"`
Raw []byte `db:"raw_log"`
}

90
pkg/transformers/deal/repository.go
View File

@ -1,90 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package deal
import (
"fmt"
log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
)
type DealRepository struct {
db *postgres.DB
}
func (repository DealRepository) Create(headerID int64, models []interface{}) error {
tx, dBaseErr := repository.db.Begin()
if dBaseErr != nil {
return dBaseErr
}
for _, model := range models {
dealModel, ok := model.(DealModel)
if !ok {
rollbackErr := tx.Rollback()
if rollbackErr != nil {
log.Error("failed to rollback ", rollbackErr)
}
return fmt.Errorf("model of type %T, not %T", model, DealModel{})
}
_, execErr := tx.Exec(
`INSERT into maker.deal (header_id, bid_id, contract_address, log_idx, tx_idx, raw_log)
VALUES($1, $2, $3, $4, $5, $6)
ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET bid_id = $2, contract_address = $3, raw_log = $6;`,
headerID, dealModel.BidId, dealModel.ContractAddress, dealModel.LogIndex, dealModel.TransactionIndex, dealModel.Raw,
)
if execErr != nil {
rollbackErr := tx.Rollback()
if rollbackErr != nil {
log.Error("failed to rollback ", rollbackErr)
}
return execErr
}
}
checkHeaderErr := shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.DealChecked)
if checkHeaderErr != nil {
rollbackErr := tx.Rollback()
if rollbackErr != nil {
log.Error("failed to rollback ", rollbackErr)
}
return checkHeaderErr
}
return tx.Commit()
}
func (repository DealRepository) MarkHeaderChecked(headerID int64) error {
return shared.MarkHeaderChecked(headerID, repository.db, constants.DealChecked)
}
func (repository DealRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DealChecked)
}
func (repository DealRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DealChecked)
}
func (repository *DealRepository) SetDB(db *postgres.DB) {
repository.db = db
}

89
pkg/transformers/deal/repository_test.go
View File

@ -1,89 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package deal_test
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/transformers/deal"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/shared_behaviors"
"github.com/vulcanize/vulcanizedb/test_config"
)
var _ = Describe("Deal Repository", func() {
var (
db *postgres.DB
dealRepository deal.DealRepository
headerRepository repositories.HeaderRepository
)
BeforeEach(func() {
db = test_config.NewTestDB(test_config.NewTestNode())
test_config.CleanTestDB(db)
dealRepository = deal.DealRepository{}
dealRepository.SetDB(db)
headerRepository = repositories.NewHeaderRepository(db)
})
Describe("Create", func() {
modelWithDifferentLogIdx := test_data.DealModel
modelWithDifferentLogIdx.LogIndex = modelWithDifferentLogIdx.LogIndex + 1
inputs := shared_behaviors.CreateBehaviorInputs{
CheckedHeaderColumnName: constants.DealChecked,
LogEventTableName: "maker.deal",
TestModel: test_data.DealModel,
ModelWithDifferentLogIdx: modelWithDifferentLogIdx,
Repository: &dealRepository,
}
shared_behaviors.SharedRepositoryCreateBehaviors(&inputs)
It("persists a deal record", func() {
headerId, err := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader)
Expect(err).NotTo(HaveOccurred())
err = dealRepository.Create(headerId, []interface{}{test_data.DealModel})
Expect(err).NotTo(HaveOccurred())
var count int
db.QueryRow(`SELECT count(*) FROM maker.deal`).Scan(&count)
Expect(count).To(Equal(1))
var dbResult deal.DealModel
err = db.Get(&dbResult, `SELECT bid_id, contract_address, log_idx, tx_idx, raw_log FROM maker.deal WHERE header_id = $1`, headerId)
Expect(err).NotTo(HaveOccurred())
Expect(dbResult.BidId).To(Equal(test_data.DealModel.BidId))
Expect(dbResult.ContractAddress).To(Equal(test_data.DealModel.ContractAddress))
Expect(dbResult.LogIndex).To(Equal(test_data.DealModel.LogIndex))
Expect(dbResult.TransactionIndex).To(Equal(test_data.DealModel.TransactionIndex))
Expect(dbResult.Raw).To(MatchJSON(test_data.DealModel.Raw))
})
})
Describe("MarkHeaderChecked", func() {
inputs := shared_behaviors.MarkedHeaderCheckedBehaviorInputs{
CheckedHeaderColumnName: constants.DealChecked,
Repository: &dealRepository,
}
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
})
})

34
pkg/transformers/dent/config.go
View File

@ -1,34 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package dent
import (
shared_t "github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
)
func GetDentConfig() shared_t.TransformerConfig {
return shared_t.TransformerConfig{
TransformerName: constants.DentLabel,
ContractAddresses: []string{constants.FlipperContractAddress(), constants.FlopperContractAddress()},
ContractAbi: constants.FlipperABI(),
Topic: constants.GetDentFunctionSignature(),
StartingBlockNumber: shared.MinInt64([]int64{constants.FlipperDeploymentBlock(), constants.FlopperDeploymentBlock()}),
EndingBlockNumber: -1,
}
}

88
pkg/transformers/dent/converter.go
View File

@ -1,88 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package dent
import (
"encoding/json"
"errors"
"math/big"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
)
type DentConverter struct{}
func NewDentConverter() DentConverter {
return DentConverter{}
}
func (c DentConverter) ToModels(ethLogs []types.Log) (result []interface{}, err error) {
for _, log := range ethLogs {
err := validateLog(log)
if err != nil {
return nil, err
}
bidId := log.Topics[2].Big()
lot := log.Topics[3].Big().String()
bidValue := getBidValue(log)
// TODO: verify guy is available on Topics[1] (looks like it may just be an int id)
guy := common.HexToAddress(log.Topics[1].Hex()).String()
logIndex := log.Index
transactionIndex := log.TxIndex
raw, err := json.Marshal(log)
if err != nil {
return nil, err
}
model := DentModel{
BidId: bidId.String(),
Lot: lot,
Bid: bidValue,
Guy: guy,
LogIndex: logIndex,
TransactionIndex: transactionIndex,
Raw: raw,
}
result = append(result, model)
}
return result, err
}
func validateLog(ethLog types.Log) error {
if len(ethLog.Data) <= 0 {
return errors.New("dent log data is empty")
}
if len(ethLog.Topics) < 4 {
return errors.New("dent log does not contain expected topics")
}
return nil
}
func getBidValue(ethLog types.Log) string {
itemByteLength := 32
lastDataItemStartIndex := len(ethLog.Data) - itemByteLength
lastItem := ethLog.Data[lastDataItemStartIndex:]
lastValue := big.NewInt(0).SetBytes(lastItem)
return lastValue.String()
}

59
pkg/transformers/dent/converter_test.go
View File

@ -1,59 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package dent_test
import (
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/transformers/dent"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
)
var _ = Describe("Dent Converter", func() {
var converter dent.DentConverter
BeforeEach(func() {
converter = dent.NewDentConverter()
})
It("converts an eth log to a db model", func() {
models, err := converter.ToModels([]types.Log{test_data.DentLog})
Expect(err).NotTo(HaveOccurred())
Expect(len(models)).To(Equal(1))
Expect(models[0].(dent.DentModel)).To(Equal(test_data.DentModel))
})
It("returns an error if the expected amount of topics aren't in the log", func() {
invalidLog := test_data.DentLog
invalidLog.Topics = []common.Hash{}
_, err := converter.ToModels([]types.Log{invalidLog})
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError("dent log does not contain expected topics"))
})
It("returns an error if the log data is empty", func() {
emptyDataLog := test_data.DentLog
emptyDataLog.Data = []byte{}
_, err := converter.ToModels([]types.Log{emptyDataLog})
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError("dent log data is empty"))
})
})

19
pkg/transformers/dent/dent_suite_test.go
View File

@ -1,19 +0,0 @@
package dent_test
import (
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
log "github.com/sirupsen/logrus"
"io/ioutil"
)
func TestDent(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Dent Suite")
}
var _ = BeforeSuite(func() {
log.SetOutput(ioutil.Discard)
})

27
pkg/transformers/dent/model.go
View File

@ -1,27 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package dent
type DentModel struct {
BidId string `db:"bid_id"`
Lot string
Bid string
Guy string
LogIndex uint `db:"log_idx"`
TransactionIndex uint `db:"tx_idx"`
Raw []byte `db:"raw_log"`
}

93
pkg/transformers/dent/repository.go
View File

@ -1,93 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package dent
import (
"fmt"
log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
)
type DentRepository struct {
db *postgres.DB
}
func (repository DentRepository) Create(headerID int64, models []interface{}) error {
tx, dBaseErr := repository.db.Begin()
if dBaseErr != nil {
return dBaseErr
}
tic, getTicErr := shared.GetTicInTx(headerID, tx)
if getTicErr != nil {
rollbackErr := tx.Rollback()
if rollbackErr != nil {
log.Error("failed to rollback ", rollbackErr)
}
return getTicErr
}
for _, model := range models {
dent, ok := model.(DentModel)
if !ok {
rollbackErr := tx.Rollback()
if rollbackErr != nil {
log.Error("failed to rollback ", rollbackErr)
}
return fmt.Errorf("model of type %T, not %T", model, DentModel{})
}
_, execErr := tx.Exec(
`INSERT into maker.dent (header_id, bid_id, lot, bid, guy, tic, log_idx, tx_idx, raw_log)
VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9)
ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET bid_Id = $2, lot = $3, bid = $4, guy = $5, tic = $6, raw_log = $9;`,
headerID, dent.BidId, dent.Lot, dent.Bid, dent.Guy, tic, dent.LogIndex, dent.TransactionIndex, dent.Raw,
)
if execErr != nil {
tx.Rollback()
return execErr
}
}
err := shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.DentChecked)
if err != nil {
tx.Rollback()
return err
}
return tx.Commit()
}
func (repository DentRepository) MarkHeaderChecked(headerId int64) error {
return shared.MarkHeaderChecked(headerId, repository.db, constants.DentChecked)
}
func (repository DentRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DentChecked)
}
func (repository DentRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DentChecked)
}
func (repository *DentRepository) SetDB(db *postgres.DB) {
repository.db = db
}

97
pkg/transformers/dent/repository_test.go
View File

@ -1,97 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package dent_test
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/transformers/dent"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/shared_behaviors"
"github.com/vulcanize/vulcanizedb/test_config"
)
var _ = Describe("Dent Repository", func() {
var (
db *postgres.DB
dentRepository dent.DentRepository
headerRepository repositories.HeaderRepository
)
BeforeEach(func() {
db = test_config.NewTestDB(test_config.NewTestNode())
test_config.CleanTestDB(db)
dentRepository = dent.DentRepository{}
dentRepository.SetDB(db)
headerRepository = repositories.NewHeaderRepository(db)
})
Describe("Create", func() {
modelWithDifferentLogIdx := test_data.DentModel
modelWithDifferentLogIdx.LogIndex++
inputs := shared_behaviors.CreateBehaviorInputs{
CheckedHeaderColumnName: constants.DentChecked,
LogEventTableName: "maker.dent",
TestModel: test_data.DentModel,
ModelWithDifferentLogIdx: modelWithDifferentLogIdx,
Repository: &dentRepository,
}
shared_behaviors.SharedRepositoryCreateBehaviors(&inputs)
It("persists a dent record", func() {
headerID, err := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader)
Expect(err).NotTo(HaveOccurred())
err = dentRepository.Create(headerID, []interface{}{test_data.DentModel})
Expect(err).NotTo(HaveOccurred())
var count int
db.QueryRow(`SELECT count(*) FROM maker.dent`).Scan(&count)
Expect(count).To(Equal(1))
var dbResult dent.DentModel
err = db.Get(&dbResult, `SELECT bid_id, lot, bid, guy, log_idx, tx_idx, raw_log FROM maker.dent WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(dbResult.BidId).To(Equal(test_data.DentModel.BidId))
Expect(dbResult.Lot).To(Equal(test_data.DentModel.Lot))
Expect(dbResult.Bid).To(Equal(test_data.DentModel.Bid))
Expect(dbResult.Guy).To(Equal(test_data.DentModel.Guy))
Expect(dbResult.LogIndex).To(Equal(test_data.DentModel.LogIndex))
Expect(dbResult.TransactionIndex).To(Equal(test_data.DentModel.TransactionIndex))
Expect(dbResult.Raw).To(MatchJSON(test_data.DentModel.Raw))
var dbTic int64
err = db.Get(&dbTic, `SELECT tic FROM maker.dent WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(dbTic).To(Equal(fakes.FakeHeaderTic))
})
})
Describe("MarkHeaderChecked", func() {
inputs := shared_behaviors.MarkedHeaderCheckedBehaviorInputs{
CheckedHeaderColumnName: constants.DentChecked,
Repository: &dentRepository,
}
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
})
})

32
pkg/transformers/drip_drip/config.go
View File

@ -1,32 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package drip_drip
import (
shared_t "github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
)
func GetDripDripConfig() shared_t.TransformerConfig {
return shared_t.TransformerConfig{
ContractAddresses: []string{constants.DripContractAddress()},
ContractAbi: constants.DripABI(),
Topic: constants.GetDripDripSignature(),
StartingBlockNumber: constants.DripDeploymentBlock(),
EndingBlockNumber: -1,
}
}

56
pkg/transformers/drip_drip/converter.go
View File

@ -1,56 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package drip_drip
import (
"encoding/json"
"errors"
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
)
type DripDripConverter struct{}
func (DripDripConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) {
var models []interface{}
for _, ethLog := range ethLogs {
err := verifyLog(ethLog)
if err != nil {
return nil, err
}
ilk := shared.GetHexWithoutPrefix(ethLog.Topics[2].Bytes())
raw, err := json.Marshal(ethLog)
if err != nil {
return nil, err
}
model := DripDripModel{
Ilk: ilk,
LogIndex: ethLog.Index,
TransactionIndex: ethLog.TxIndex,
Raw: raw,
}
models = append(models, model)
}
return models, nil
}
func verifyLog(log types.Log) error {
if len(log.Topics) < 3 {
return errors.New("log missing topics")
}
return nil
}

46
pkg/transformers/drip_drip/converter_test.go
View File

@ -1,46 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package drip_drip_test
import (
"github.com/ethereum/go-ethereum/core/types"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/transformers/drip_drip"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
)
var _ = Describe("Drip drip converter", func() {
It("returns err if log is missing topics", func() {
converter := drip_drip.DripDripConverter{}
badLog := types.Log{}
_, err := converter.ToModels([]types.Log{badLog})
Expect(err).To(HaveOccurred())
})
It("converts a log to an model", func() {
converter := drip_drip.DripDripConverter{}
model, err := converter.ToModels([]types.Log{test_data.EthDripDripLog})
Expect(err).NotTo(HaveOccurred())
Expect(model).To(Equal([]interface{}{test_data.DripDripModel}))
})
})

35
pkg/transformers/drip_drip/drip_drip_suite_test.go
View File

@ -1,35 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package drip_drip_test
import (
log "github.com/sirupsen/logrus"
"io/ioutil"
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
func TestDripDrip(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "DripDrip Suite")
}
var _ = BeforeSuite(func() {
log.SetOutput(ioutil.Discard)
})

24
pkg/transformers/drip_drip/model.go
View File

@ -1,24 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package drip_drip
type DripDripModel struct {
Ilk string
LogIndex uint `db:"log_idx"`
TransactionIndex uint `db:"tx_idx"`
Raw []byte `db:"raw_log"`
}

33
pkg/transformers/drip_file/ilk/config.go
View File

@ -1,33 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ilk
import (
shared_t "github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
)
func GetDripFileIlkConfig() shared_t.TransformerConfig {
return shared_t.TransformerConfig{
TransformerName: constants.DripFileIlkLabel,
ContractAddresses: []string{constants.DripContractAddress()},
ContractAbi: constants.DripABI(),
Topic: constants.GetDripFileIlkSignature(),
StartingBlockNumber: constants.DripDeploymentBlock(),
EndingBlockNumber: -1,
}
}

70
pkg/transformers/drip_file/ilk/converter.go
View File

@ -1,70 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ilk
import (
"encoding/json"
"errors"
"math/big"
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
)
type DripFileIlkConverter struct{}
func (DripFileIlkConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) {
var models []interface{}
for _, ethLog := range ethLogs {
err := verifyLog(ethLog)
if err != nil {
return nil, err
}
ilk := shared.GetHexWithoutPrefix(ethLog.Topics[2].Bytes())
vow := shared.GetHexWithoutPrefix(ethLog.Topics[3].Bytes())
taxBytes := ethLog.Data[len(ethLog.Data)-constants.DataItemLength:]
tax := shared.ConvertToRay(big.NewInt(0).SetBytes(taxBytes).String())
raw, err := json.Marshal(ethLog)
if err != nil {
return nil, err
}
model := DripFileIlkModel{
Ilk: ilk,
Vow: vow,
Tax: tax,
LogIndex: ethLog.Index,
TransactionIndex: ethLog.TxIndex,
Raw: raw,
}
models = append(models, model)
}
return models, nil
}
func verifyLog(log types.Log) error {
if len(log.Topics) < 4 {
return errors.New("log missing topics")
}
if len(log.Data) < constants.DataItemLength {
return errors.New("log missing data")
}
return nil
}

63
pkg/transformers/drip_file/ilk/converter_test.go
View File

@ -1,63 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ilk_test
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/ilk"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
)
var _ = Describe("Drip file ilk converter", func() {
It("returns err if log missing topics", func() {
converter := ilk.DripFileIlkConverter{}
badLog := types.Log{
Topics: []common.Hash{{}},
Data: []byte{1, 1, 1, 1, 1},
}
_, err := converter.ToModels([]types.Log{badLog})
Expect(err).To(HaveOccurred())
})
It("returns err if log missing data", func() {
converter := ilk.DripFileIlkConverter{}
badLog := types.Log{
Topics: []common.Hash{{}, {}, {}, {}},
}
_, err := converter.ToModels([]types.Log{badLog})
Expect(err).To(HaveOccurred())
})
It("converts a log to a model", func() {
converter := ilk.DripFileIlkConverter{}
models, err := converter.ToModels([]types.Log{test_data.EthDripFileIlkLog})
Expect(err).NotTo(HaveOccurred())
Expect(len(models)).To(Equal(1))
Expect(models[0].(ilk.DripFileIlkModel)).To(Equal(test_data.DripFileIlkModel))
})
})

35
pkg/transformers/drip_file/ilk/ilk_suite_test.go
View File

@ -1,35 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ilk_test
import (
log "github.com/sirupsen/logrus"
"io/ioutil"
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
func TestIlk(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Ilk Suite")
}
var _ = BeforeSuite(func() {
log.SetOutput(ioutil.Discard)
})

26
pkg/transformers/drip_file/ilk/model.go
View File

@ -1,26 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ilk
type DripFileIlkModel struct {
Ilk string
Vow string
Tax string
LogIndex uint `db:"log_idx"`
TransactionIndex uint `db:"tx_idx"`
Raw []byte `db:"raw_log"`
}

33
pkg/transformers/drip_file/repo/config.go
View File

@ -1,33 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package repo
import (
shared_t "github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
)
func GetDripFileRepoConfig() shared_t.TransformerConfig {
return shared_t.TransformerConfig{
TransformerName: constants.DripFileRepoLabel,
ContractAddresses: []string{constants.DripContractAddress()},
ContractAbi: constants.DripABI(),
Topic: constants.GetDripFileRepoSignature(),
StartingBlockNumber: constants.DripDeploymentBlock(),
EndingBlockNumber: -1,
}
}

61
pkg/transformers/drip_file/repo/converter.go
View File

@ -1,61 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package repo
import (
"bytes"
"encoding/json"
"errors"
"github.com/ethereum/go-ethereum/core/types"
"math/big"
)
type DripFileRepoConverter struct{}
func (DripFileRepoConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) {
var models []interface{}
for _, ethLog := range ethLogs {
err := verifyLog(ethLog)
if err != nil {
return nil, err
}
what := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
data := big.NewInt(0).SetBytes(ethLog.Topics[3].Bytes()).String()
raw, err := json.Marshal(ethLog)
if err != nil {
return nil, err
}
model := DripFileRepoModel{
What: what,
Data: data,
LogIndex: ethLog.Index,
TransactionIndex: ethLog.TxIndex,
Raw: raw,
}
models = append(models, model)
}
return models, nil
}
func verifyLog(log types.Log) error {
if len(log.Topics) < 4 {
return errors.New("log missing topics")
}
return nil
}

51
pkg/transformers/drip_file/repo/converter_test.go
View File

@ -1,51 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package repo_test
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/repo"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
)
var _ = Describe("Drip file repo converter", func() {
It("returns err if log missing topics", func() {
converter := repo.DripFileRepoConverter{}
badLog := types.Log{
Topics: []common.Hash{{}},
Data: []byte{1, 1, 1, 1, 1},
}
_, err := converter.ToModels([]types.Log{badLog})
Expect(err).To(HaveOccurred())
})
It("converts a log to a model", func() {
converter := repo.DripFileRepoConverter{}
models, err := converter.ToModels([]types.Log{test_data.EthDripFileRepoLog})
Expect(err).NotTo(HaveOccurred())
Expect(len(models)).To(Equal(1))
Expect(models[0].(repo.DripFileRepoModel)).To(Equal(test_data.DripFileRepoModel))
})
})

25
pkg/transformers/drip_file/repo/model.go
View File

@ -1,25 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package repo
type DripFileRepoModel struct {
What string
Data string
LogIndex uint `db:"log_idx"`
TransactionIndex uint `db:"tx_idx"`
Raw []byte `db:"raw_log"`
}

35
pkg/transformers/drip_file/repo/repo_suite_test.go
View File

@ -1,35 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package repo_test
import (
log "github.com/sirupsen/logrus"
"io/ioutil"
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
func TestRepo(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Repo Suite")
}
var _ = BeforeSuite(func() {
log.SetOutput(ioutil.Discard)
})

91
pkg/transformers/drip_file/repo/repository.go
View File

@ -1,91 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package repo
import (
"fmt"
log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
)
type DripFileRepoRepository struct {
db *postgres.DB
}
func (repository DripFileRepoRepository) Create(headerID int64, models []interface{}) error {
tx, dBaseErr := repository.db.Begin()
if dBaseErr != nil {
return dBaseErr
}
for _, model := range models {
repo, ok := model.(DripFileRepoModel)
if !ok {
rollbackErr := tx.Rollback()
if rollbackErr != nil {
log.Error("failed to rollback ", rollbackErr)
}
return fmt.Errorf("model of type %T, not %T", model, DripFileRepoModel{})
}
_, execErr := tx.Exec(
`INSERT into maker.drip_file_repo (header_id, what, data, log_idx, tx_idx, raw_log)
VALUES($1, $2, $3::NUMERIC, $4, $5, $6)`,
headerID, repo.What, repo.Data, repo.LogIndex, repo.TransactionIndex, repo.Raw,
)
if execErr != nil {
rollbackErr := tx.Rollback()
if rollbackErr != nil {
log.Error("failed to rollback ", rollbackErr)
}
return execErr
}
}
checkHeaderErr := shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.DripFileRepoChecked)
if checkHeaderErr != nil {
rollbackErr := tx.Rollback()
if rollbackErr != nil {
log.Error("failed to rollback ", rollbackErr)
}
return checkHeaderErr
}
return tx.Commit()
}
func (repository DripFileRepoRepository) MarkHeaderChecked(headerID int64) error {
return shared.MarkHeaderChecked(headerID, repository.db, constants.DripFileRepoChecked)
}
func (repository DripFileRepoRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DripFileRepoChecked)
}
func (repository DripFileRepoRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
return shared.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DripFileRepoChecked)
}
func (repository *DripFileRepoRepository) SetDB(db *postgres.DB) {
repository.db = db
}

87
pkg/transformers/drip_file/repo/repository_test.go
View File

@ -1,87 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package repo_test
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/datastore"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/repo"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/shared_behaviors"
"github.com/vulcanize/vulcanizedb/test_config"
)
var _ = Describe("Drip file repo repository", func() {
var (
db *postgres.DB
dripFileRepoRepository repo.DripFileRepoRepository
headerRepository datastore.HeaderRepository
)
BeforeEach(func() {
db = test_config.NewTestDB(test_config.NewTestNode())
test_config.CleanTestDB(db)
headerRepository = repositories.NewHeaderRepository(db)
dripFileRepoRepository = repo.DripFileRepoRepository{}
dripFileRepoRepository.SetDB(db)
})
Describe("Create", func() {
modelWithDifferentLogIdx := test_data.DripFileRepoModel
modelWithDifferentLogIdx.LogIndex++
inputs := shared_behaviors.CreateBehaviorInputs{
CheckedHeaderColumnName: constants.DripFileRepoChecked,
LogEventTableName: "maker.drip_file_repo",
TestModel: test_data.DripFileRepoModel,
ModelWithDifferentLogIdx: modelWithDifferentLogIdx,
Repository: &dripFileRepoRepository,
}
shared_behaviors.SharedRepositoryCreateBehaviors(&inputs)
It("adds a drip file repo event", func() {
headerID, err := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader)
Expect(err).NotTo(HaveOccurred())
err = dripFileRepoRepository.Create(headerID, []interface{}{test_data.DripFileRepoModel})
Expect(err).NotTo(HaveOccurred())
var dbDripFileRepo repo.DripFileRepoModel
err = db.Get(&dbDripFileRepo, `SELECT what, data, log_idx, tx_idx, raw_log FROM maker.drip_file_repo WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(dbDripFileRepo.What).To(Equal(test_data.DripFileRepoModel.What))
Expect(dbDripFileRepo.Data).To(Equal(test_data.DripFileRepoModel.Data))
Expect(dbDripFileRepo.LogIndex).To(Equal(test_data.DripFileRepoModel.LogIndex))
Expect(dbDripFileRepo.TransactionIndex).To(Equal(test_data.DripFileRepoModel.TransactionIndex))
Expect(dbDripFileRepo.Raw).To(MatchJSON(test_data.DripFileRepoModel.Raw))
})
})
Describe("MarkHeaderChecked", func() {
inputs := shared_behaviors.MarkedHeaderCheckedBehaviorInputs{
CheckedHeaderColumnName: constants.DripFileRepoChecked,
Repository: &dripFileRepoRepository,
}
shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs)
})
})

33
pkg/transformers/drip_file/vow/config.go
View File

@ -1,33 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package vow
import (
shared_t "github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
)
func GetDripFileVowConfig() shared_t.TransformerConfig {
return shared_t.TransformerConfig{
TransformerName: constants.DripFileVowLabel,
ContractAddresses: []string{constants.DripContractAddress()},
ContractAbi: constants.DripABI(),
Topic: constants.GetDripFileVowSignature(),
StartingBlockNumber: constants.DripDeploymentBlock(),
EndingBlockNumber: -1,
}
}

61
pkg/transformers/drip_file/vow/converter.go
View File

@ -1,61 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package vow
import (
"bytes"
"encoding/json"
"errors"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
)
type DripFileVowConverter struct{}
func (DripFileVowConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) {
var models []interface{}
for _, ethLog := range ethLogs {
err := verifyLog(ethLog)
if err != nil {
return nil, err
}
what := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00"))
data := common.BytesToAddress(ethLog.Topics[3].Bytes()).String()
raw, err := json.Marshal(ethLog)
if err != nil {
return nil, err
}
model := DripFileVowModel{
What: what,
Data: data,
LogIndex: ethLog.Index,
TransactionIndex: ethLog.TxIndex,
Raw: raw,
}
models = append(models, model)
}
return models, nil
}
func verifyLog(log types.Log) error {
if len(log.Topics) < 4 {
return errors.New("log missing topics")
}
return nil
}

51
pkg/transformers/drip_file/vow/converter_test.go
View File

@ -1,51 +0,0 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package vow_test
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/vow"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
)
var _ = Describe("Drip file repo converter", func() {
It("returns err if log missing topics", func() {
converter := vow.DripFileVowConverter{}
badLog := types.Log{
Topics: []common.Hash{{}},
Data: []byte{1, 1, 1, 1, 1},
}
_, err := converter.ToModels([]types.Log{badLog})
Expect(err).To(HaveOccurred())
})
It("converts a log to a model", func() {
converter := vow.DripFileVowConverter{}
models, err := converter.ToModels([]types.Log{test_data.EthDripFileVowLog})
Expect(err).NotTo(HaveOccurred())
Expect(len(models)).To(Equal(1))
Expect(models[0].(vow.DripFileVowModel)).To(Equal(test_data.DripFileVowModel))
})
})

Some files were not shown because too many files have changed in this diff Show More