Merge old private repo into vulcanize

This commit is contained in:
Matt Krump 2018-01-25 18:08:26 -06:00
commit aac2239097
55 changed files with 1718 additions and 579 deletions

2
.gitignore vendored
View File

@ -7,3 +7,5 @@ environments/*.toml
Vagrantfile
vagrant_bootstrap.sh
.vagrant
test_scripts/
vulcanizedb

View File

@ -8,11 +8,8 @@ addons:
postgresql: "9.6"
before_install:
# godo
- go get -u gopkg.in/godo.v2/cmd/godo
# dep
- go get -u github.com/golang/dep/cmd/dep
- dep ensure
# ginkgo
- go get -u github.com/onsi/ginkgo/ginkgo
# migrate
@ -30,7 +27,7 @@ install:
before_script:
- ./scripts/setup
- nohup ./scripts/start_private_blockchain </dev/null &
- godo migrate -- -e private
- make migrate HOST_NAME=localhost NAME=vulcanize_private PORT=5432
script:
- ginkgo -r

View File

@ -1,109 +0,0 @@
package main
import (
"log"
"fmt"
"github.com/vulcanize/vulcanizedb/cmd"
"github.com/vulcanize/vulcanizedb/pkg/config"
do "gopkg.in/godo.v2"
)
func parseEnvironment(context *do.Context) string {
environment := context.Args.MayString("", "environment", "env", "e")
if environment == "" {
log.Fatalln("--environment required")
}
return environment
}
func tasks(p *do.Project) {
p.Task("run", nil, func(context *do.Context) {
environment := parseEnvironment(context)
context.Start(`go run main.go --environment={{.environment}}`,
do.M{"environment": environment, "$in": "cmd/run"})
})
p.Task("vulcanizeDb", nil, func(context *do.Context) {
environment := parseEnvironment(context)
context.Start(`go run main.go --environment={{.environment}}`,
do.M{"environment": environment, "$in": "cmd/vulcanize_db"})
})
p.Task("populateBlocks", nil, func(context *do.Context) {
environment := parseEnvironment(context)
startingNumber := context.Args.MayInt(-1, "starting-number")
if startingNumber < 0 {
log.Fatalln("--starting-number required")
}
context.Start(`go run main.go --environment={{.environment}} --starting-number={{.startingNumber}}`,
do.M{"environment": environment, "startingNumber": startingNumber, "$in": "cmd/populate_blocks"})
})
p.Task("getLogs", nil, func(context *do.Context) {
environment := parseEnvironment(context)
contractHash := context.Args.MayString("", "contract-hash", "c")
if contractHash == "" {
log.Fatalln("--contract-hash required")
}
context.Start(`go run main.go --environment={{.environment}} --contract-hash={{.contractHash}}`,
do.M{
"environment": environment,
"contractHash": contractHash,
"$in": "cmd/get_logs",
})
})
p.Task("watchContract", nil, func(context *do.Context) {
environment := parseEnvironment(context)
contractHash := context.Args.MayString("", "contract-hash", "c")
abiFilepath := context.Args.MayString("", "abi-filepath", "a")
if contractHash == "" {
log.Fatalln("--contract-hash required")
}
context.Start(`go run main.go --environment={{.environment}} --contract-hash={{.contractHash}} --abi-filepath={{.abiFilepath}}`,
do.M{
"environment": environment,
"contractHash": contractHash,
"abiFilepath": abiFilepath,
"$in": "cmd/watch_contract",
})
})
p.Task("migrate", nil, func(context *do.Context) {
environment := parseEnvironment(context)
cfg := cmd.LoadConfig(environment)
connectString := config.DbConnectionString(cfg.Database)
migrate := fmt.Sprintf("migrate -database '%s' -path ./db/migrations up", connectString)
context.Bash(migrate)
})
p.Task("rollback", nil, func(context *do.Context) {
environment := parseEnvironment(context)
cfg := cmd.LoadConfig(environment)
connectString := config.DbConnectionString(cfg.Database)
migrate := fmt.Sprintf("migrate -database '%s' -path ./db/migrations down 1", connectString)
context.Bash(migrate)
})
p.Task("showContractSummary", nil, func(context *do.Context) {
environment := parseEnvironment(context)
contractHash := context.Args.MayString("", "contract-hash", "c")
blockNumber := context.Args.MayInt(-1, "block-number", "b")
if contractHash == "" {
log.Fatalln("--contract-hash required")
}
context.Start(`go run main.go --environment={{.environment}} --contract-hash={{.contractHash}} --block-number={{.blockNumber}}`,
do.M{"environment": environment,
"contractHash": contractHash,
"blockNumber": blockNumber,
"$in": "cmd/show_contract_summary"})
})
}
func main() {
do.Godo(tasks)
}

82
Gopkg.lock generated
View File

@ -31,6 +31,12 @@
revision = "4bb3c89d44e372e6a9ab85a8be0c9345265c763a"
version = "v1.7.3"
[[projects]]
name = "github.com/fsnotify/fsnotify"
packages = ["."]
revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9"
version = "v1.4.7"
[[projects]]
name = "github.com/go-stack/stack"
packages = ["."]
@ -49,6 +55,12 @@
packages = ["."]
revision = "553a641470496b2327abcac10b36396bd98e45c9"
[[projects]]
branch = "master"
name = "github.com/hashicorp/hcl"
packages = [".","hcl/ast","hcl/parser","hcl/scanner","hcl/strconv","hcl/token","json/parser","json/scanner","json/token"]
revision = "23c074d0eceb2b8a5bfdbb271ab780cde70f05a8"
[[projects]]
branch = "master"
name = "github.com/howeyc/gopass"
@ -61,6 +73,12 @@
packages = [".","dcps/internetgateway1","dcps/internetgateway2","httpu","scpd","soap","ssdp"]
revision = "dceda08e705b2acee36aab47d765ed801f64cfc7"
[[projects]]
name = "github.com/inconshreveable/mousetrap"
packages = ["."]
revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75"
version = "v1.0"
[[projects]]
name = "github.com/jackpal/go-nat-pmp"
packages = ["."]
@ -79,6 +97,12 @@
packages = [".","oid"]
revision = "83612a56d3dd153a94a629cd64925371c9adad78"
[[projects]]
name = "github.com/magiconair/properties"
packages = ["."]
revision = "d419a98cdbed11a922bf76f257b7c4be79b50e73"
version = "v1.7.4"
[[projects]]
name = "github.com/mattn/go-colorable"
packages = ["."]
@ -115,6 +139,18 @@
revision = "00c06406c2dd2e011f153a6502a21473676db33f"
version = "v1.0.0"
[[projects]]
branch = "master"
name = "github.com/mitchellh/go-homedir"
packages = ["."]
revision = "b8bc1bf767474819792c23f32d8286a45736f1c6"
[[projects]]
branch = "master"
name = "github.com/mitchellh/mapstructure"
packages = ["."]
revision = "b4575eea38cca1123ec2dc90c26529b5c5acfcff"
[[projects]]
name = "github.com/nozzle/throttler"
packages = ["."]
@ -133,6 +169,12 @@
revision = "c893efa28eb45626cdaa76c9f653b62488858837"
version = "v1.2.0"
[[projects]]
name = "github.com/pelletier/go-toml"
packages = ["."]
revision = "acdc4509485b587f5e675510c4f2c63e90ff68a8"
version = "v1.1.0"
[[projects]]
branch = "master"
name = "github.com/rcrowley/go-metrics"
@ -145,6 +187,42 @@
revision = "7af7a1e09ba336d2ea14b1ce73bf693c6837dbf6"
version = "v1.2"
[[projects]]
name = "github.com/spf13/afero"
packages = [".","mem"]
revision = "bb8f1927f2a9d3ab41c9340aa034f6b803f4359c"
version = "v1.0.2"
[[projects]]
name = "github.com/spf13/cast"
packages = ["."]
revision = "acbeb36b902d72a7a4c18e8f3241075e7ab763e4"
version = "v1.1.0"
[[projects]]
name = "github.com/spf13/cobra"
packages = ["."]
revision = "7b2c5ac9fc04fc5efafb60700713d4fa609b777b"
version = "v0.0.1"
[[projects]]
branch = "master"
name = "github.com/spf13/jwalterweatherman"
packages = ["."]
revision = "7c0cea34c8ece3fbeb2b27ab9b59511d360fb394"
[[projects]]
name = "github.com/spf13/pflag"
packages = ["."]
revision = "e57e3eeb33f795204c1ca35f56c44f83227c6e66"
version = "v1.0.0"
[[projects]]
name = "github.com/spf13/viper"
packages = ["."]
revision = "25b30aa063fc18e48662b86996252eabdcf2f0c7"
version = "v1.0.0"
[[projects]]
branch = "master"
name = "github.com/syndtr/goleveldb"
@ -172,7 +250,7 @@
[[projects]]
branch = "master"
name = "golang.org/x/text"
packages = ["encoding","encoding/charmap","encoding/htmlindex","encoding/internal","encoding/internal/identifier","encoding/japanese","encoding/korean","encoding/simplifiedchinese","encoding/traditionalchinese","encoding/unicode","internal/gen","internal/tag","internal/utf8internal","language","runes","transform","unicode/cldr"]
packages = ["encoding","encoding/charmap","encoding/htmlindex","encoding/internal","encoding/internal/identifier","encoding/japanese","encoding/korean","encoding/simplifiedchinese","encoding/traditionalchinese","encoding/unicode","internal/gen","internal/tag","internal/triegen","internal/ucd","internal/utf8internal","language","runes","transform","unicode/cldr","unicode/norm"]
revision = "be25de41fadfae372d6470bda81ca6beb55ef551"
[[projects]]
@ -208,6 +286,6 @@
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "2d7b9c5c88a94f3384b0cd754d35a3d7822a5858f439aaafe8c6477fb7c24f63"
inputs-digest = "46080c5bb453cb31ea731741d5defda225e2dc75bf7c11179d69422c82de37ca"
solver-name = "gps-cdcl"
solver-version = 1

View File

@ -32,3 +32,7 @@
[[constraint]]
branch = "master"
name = "github.com/lib/pq"
[[constraint]]
name = "github.com/spf13/cobra"
version = "0.0.1"

40
Makefile Normal file
View File

@ -0,0 +1,40 @@
HOST_NAME =
PORT =
NAME =
CONNECT_STRING=postgresql://$(HOST_NAME):$(PORT)/$(NAME)?sslmode=disable
$(MATTESMIGRATE):
go get -u -d github.com/mattes/migrate/cli github.com/lib/pq
go build -tags 'postgres' -o /usr/local/bin/migrate github.com/mattes/migrate/cli
$(DEP):
go get -u github.com/golang/dep/cmd/dep
$(GINKGO):
go get -u github.com/onsi/ginkgo/ginkgo
checkdbvars:
test -n "$(HOST_NAME)" # $$HOST_NAME
test -n "$(PORT)" # $$PORT
test -n "$(NAME)" # $$NAME
rollback: checkdbvars
migrate -database $(CONNECT_STRING) -path ./db/migrations down 1
pg_dump -O -s $(CONNECT_STRING) > db/schema.sql
migrate: $(MATTESMIGRATE) checkdbvars
migrate -database $(CONNECT_STRING) -path ./db/migrations up
pg_dump -O -s $(CONNECT_STRING) > db/schema.sql
import:
test -n "$(NAME)" # $$NAME
psql $(NAME) < db/schema.sql
dep: $(DEP)
dep ensure
build: dep
go build
test: $(GINKGO)
ginkgo -r

118
README.md
View File

@ -1,23 +1,50 @@
# Vulcanize DB
test change
[![Build Status](https://travis-ci.com/vulcanize/vulcanizedb.svg?token=GKv2Y33qsFnfYgejjvYx&branch=master)](https://travis-ci.com/vulcanize/vulcanizedb)
## Development Setup
[![Build Status](https://travis-ci.com/8thlight/vulcanizedb.svg?token=GKv2Y33qsFnfYgejjvYx&branch=master)](https://travis-ci.com/8thlight/vulcanizedb)
### Dependencies
- Go 1.9+
- https://github.com/golang/dep
- `go get -u github.com/golang/dep/cmd/dep`
- https://github.com/go-godo/godo
- `go get -u gopkg.in/godo.v2/cmd/godo`
- Postgres 10
- Go Ethereum
- https://ethereum.github.io/go-ethereum/downloads/
### Cloning the Repository
### Installation
go get github.com/vulcanize/vulcanizedb
### Setting up the Databases
1. Install Postgres
2. Create a superuser for yourself and make sure `psql --list` works without prompting for a password.
3. `createdb vulcanize_private`
4. `cd $GOPATH/src/github.com/vulcanize/vulcanizedb`
5. Import the schema:
`psql vulcanize_private < db/schema.sql`
or run the migrations:
`make migrate HOST_NAME=localhost NAME=vulcanize_public PORT=5432`
* See below for configuring additional environments
Adding a new migration: `./scripts/create_migration <migration-name>`
## Start syncing with postgres
1. Start geth
2. In a separate terminal start vulcanize_db
- `vulcanizedb sync --config <config.toml> --starting-block-number <block-numbe>`
* see `environments` for example config
## Watch specific contract events
1. Start geth
2. In a separate terminal start vulcanize_db
- `vulcanizedb sync --config <config.toml> --starting-block-number <block-numbe>`
3. Create event filter
- `vulcanizedb addFilter --config <config.toml> --filter-filepath <filter.json>`
* see `filters` for example filter
## Development Setup
### Cloning the Repository (Private repo only)
1. `git config --global url."git@github.com:".insteadOf "https://github.com/"`
- By default, `go get` does not work for private GitHub repos. This will fix that.
@ -25,32 +52,14 @@ test change
3. `cd $GOPATH/src/github.com/vulcanize/vulcanizedb`
4. `dep ensure`
### Setting up the Databases
### Creating/Using a test node
1. Install Postgres
2. Create a superuser for yourself and make sure `psql --list` works without prompting for a password.
3. `go get -u -d github.com/mattes/migrate/cli github.com/lib/pq`
4. `go build -tags 'postgres' -o /usr/local/bin/migrate github.com/mattes/migrate/cli`
5. `createdb vulcanize_private`
6. `cd $GOPATH/src/github.com/vulcanize/vulcanizedb`
7. `godo migrate -- --environment=<some-environment>`
* See below for configuring additional environments
Adding a new migration: `./scripts/create_migration <migration-name>`
### Creating/Using a Private Blockchain
Syncing the public blockchain takes many hours for the initial sync and will download 20+ GB of data.
Here are some instructions for creating a private blockchain that does not depend on having a network connection.
Syncing the against the public network takes many hours for the initial sync and will download 20+ GB of data.
Here are some instructions for creating a private test node that does not depend on having a network connection.
1. Run `./scripts/setup` to create a private blockchain with a new account.
* This will result in a warning.
2. Run `./scripts/start_private_blockchain`.
3. Run `godo run -- --environment=private` to start listener.
### Connecting to the Public Blockchain
`./scripts/start_blockchain`
### IPC File Paths
@ -59,52 +68,7 @@ The default location for Ethereum is:
- `$HOME/.ethereum` for Ubuntu
- `$GOPATH/src/gihub.com/vulcanize/vulcanizedb/test_data_dir/geth.ipc` for private blockchain.
**Note the location of the ipc file is outputted when you connect to a blockchain. It is needed to for configuration**
## Start Vulcanize DB
1. Start a blockchain.
2. In a separate terminal start vulcanize_db
- `godo vulcanizeDb -- --environment=<some-environment>`
## Running Listener
1. Start a blockchain.
2. In a separate terminal start listener (ipcDir location)
- `godo run -- --environment=<some-environment>`
## Retrieving Historical Data
1. Start a blockchain.
2. In a separate terminal start listener (ipcDir location)
- `godo populateBlocks -- --environment=<some-environment> --starting-number=<starting-block-number>`
## Retrieve Contract Attributes
1. Add contract ABI to contracts / environment directory:
```
vulcanizedb/
contracts/
public/
<contract-address>.json
private/
```
The name of the JSON file should correspond the contract's address.
2. Start watching the contract `godo watchContract -- --environment=<some-environment> --contract-hash=<contract-address>`
3. Request summary data `godo showContractSummary -- --environment=<some-environment> --contract-hash=<contract-address>`
## Retrieving Contract Logs
1. Get the logs for a specific contract
- `godo getLogs -- --environment=<some-environment> --contract-hash=<contract-address>`
### Configuring Additional Environments
You can create configuration files for additional environments.
* Among other things, it will require the IPC file path
* See `environments/private.toml` for an example
* You will need to do this if you want to run a node connecting to the public blockchain
**Note the location of the ipc file is printed to the console when you start geth. It is needed to for configuration**
## Running the Tests

75
cmd/addFilter.go Normal file
View File

@ -0,0 +1,75 @@
package cmd
import (
"encoding/json"
"io/ioutil"
"log"
"github.com/vulcanize/vulcanizedb/pkg/filters"
"github.com/vulcanize/vulcanizedb/pkg/geth"
"github.com/vulcanize/vulcanizedb/utils"
"github.com/spf13/cobra"
)
// addFilterCmd represents the addFilter command
var addFilterCmd = &cobra.Command{
Use: "addFilter",
Short: "Adds event filter to vulcanizedb",
Long: `An event filter is added to the vulcanize_db.
All events matching the filter conitions will be tracked
in vulcanizedb.
vulcanizedb addFilter --config config.toml --filter-filepath filter.json
The event filters are expected to match
the format described in the ethereum RPC wiki:
https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_newfilter
[{
"fromBlock": "0x1",
"toBlock": "0x2",
"address": "0x8888f1f195afa192cfee860698584c030f4c9db1",
"topics": ["0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b",
null,
"0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b",
"0x0000000000000000000000000aff3454fce5edbc8cca8697c15331677e6ebccc"]
}]
`,
Run: func(cmd *cobra.Command, args []string) {
addFilter()
},
}
var filterFilepath string
func init() {
rootCmd.AddCommand(addFilterCmd)
addFilterCmd.PersistentFlags().StringVar(&filterFilepath, "filter-filepath", "", "path/to/filter.json")
addFilterCmd.MarkFlagRequired("filter-filepath")
}
func addFilter() {
if filterFilepath == "" {
log.Fatal("filter-filepath required")
}
var logFilters filters.LogFilters
blockchain := geth.NewBlockchain(ipc)
repository := utils.LoadPostgres(databaseConfig, blockchain.Node())
absFilePath := utils.AbsFilePath(filterFilepath)
logFilterBytes, err := ioutil.ReadFile(absFilePath)
if err != nil {
log.Fatal(err)
}
err = json.Unmarshal(logFilterBytes, &logFilters)
if err != nil {
log.Fatal(err)
}
for _, filter := range logFilters {
err = repository.AddFilter(filter)
if err != nil {
log.Fatal(err)
}
}
}

View File

@ -1,71 +0,0 @@
package main
import (
"log"
"flag"
"math/big"
"time"
"github.com/vulcanize/vulcanizedb/cmd"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/geth"
)
func min(a, b int64) int64 {
if a < b {
return a
}
return b
}
const (
windowSize = 24
pollingInterval = 10 * time.Second
)
func main() {
environment := flag.String("environment", "", "Environment name")
contractHash := flag.String("contract-hash", "", "Contract hash to show summary")
ticker := time.NewTicker(pollingInterval)
defer ticker.Stop()
flag.Parse()
config := cmd.LoadConfig(*environment)
blockchain := geth.NewBlockchain(config.Client.IPCPath)
repository := cmd.LoadPostgres(config.Database, blockchain.Node())
lastBlockNumber := blockchain.LastBlock().Int64()
stepSize := int64(1000)
go func() {
for i := int64(0); i < lastBlockNumber; i = min(i+stepSize, lastBlockNumber) {
logs, err := blockchain.GetLogs(core.Contract{Hash: *contractHash}, big.NewInt(i), big.NewInt(i+stepSize))
log.Println("Backfilling Logs:", i)
if err != nil {
log.Println(err)
}
repository.CreateLogs(logs)
}
}()
done := make(chan struct{})
go func() { done <- struct{}{} }()
for range ticker.C {
select {
case <-done:
go func() {
z := &big.Int{}
z.Sub(blockchain.LastBlock(), big.NewInt(25))
log.Printf("Logs Window: %d - %d", z.Int64(), blockchain.LastBlock().Int64())
logs, _ := blockchain.GetLogs(core.Contract{Hash: *contractHash}, z, blockchain.LastBlock())
repository.CreateLogs(logs)
done <- struct{}{}
}()
default:
}
}
}

View File

@ -1,22 +0,0 @@
package main
import (
"flag"
"fmt"
"github.com/vulcanize/vulcanizedb/cmd"
"github.com/vulcanize/vulcanizedb/pkg/geth"
"github.com/vulcanize/vulcanizedb/pkg/history"
)
func main() {
environment := flag.String("environment", "", "Environment name")
startingBlockNumber := flag.Int("starting-number", -1, "First block to fill from")
flag.Parse()
config := cmd.LoadConfig(*environment)
blockchain := geth.NewBlockchain(config.Client.IPCPath)
repository := cmd.LoadPostgres(config.Database, blockchain.Node())
numberOfBlocksCreated := history.PopulateMissingBlocks(blockchain, repository, int64(*startingBlockNumber))
fmt.Printf("Populated %d blocks", numberOfBlocksCreated)
}

74
cmd/root.go Normal file
View File

@ -0,0 +1,74 @@
package cmd
import (
"fmt"
"os"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/mitchellh/go-homedir"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)
var cfgFile string
var databaseConfig config.Database
var ipc string
var rootCmd = &cobra.Command{
Use: "vulcanizedb",
PersistentPreRun: database,
}
func Execute() {
if err := rootCmd.Execute(); err != nil {
fmt.Println(err)
os.Exit(1)
}
}
func database(cmd *cobra.Command, args []string) {
ipc = viper.GetString("client.ipcpath")
databaseConfig = config.Database{
Name: viper.GetString("database.name"),
Hostname: viper.GetString("database.hostname"),
Port: viper.GetInt("database.port"),
}
viper.Set("database.config", databaseConfig)
}
func init() {
cobra.OnInitialize(initConfig)
rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "environment/public.toml", "config file location")
rootCmd.PersistentFlags().String("database-name", "vulcanize_public", "database name")
rootCmd.PersistentFlags().Int("database-port", 5432, "database port")
rootCmd.PersistentFlags().String("database-hostname", "localhost", "database hostname")
rootCmd.PersistentFlags().String("client-ipcPath", "", "location of geth.ipc file")
viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name"))
viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port"))
viper.BindPFlag("database.hostname", rootCmd.PersistentFlags().Lookup("database-hostname"))
viper.BindPFlag("client.ipcPath", rootCmd.PersistentFlags().Lookup("client-ipcPath"))
}
func initConfig() {
if cfgFile != "" {
viper.SetConfigFile(cfgFile)
} else {
home, err := homedir.Dir()
if err != nil {
fmt.Println(err)
os.Exit(1)
}
viper.AddConfigPath(home)
viper.SetConfigName(".vulcanizedb")
}
viper.AutomaticEnv()
if err := viper.ReadInConfig(); err == nil {
fmt.Printf("Using config file: %s\n\n", viper.ConfigFileUsed())
}
}

View File

@ -1,34 +0,0 @@
package main
import (
"flag"
"time"
"os"
"github.com/vulcanize/vulcanizedb/cmd"
"github.com/vulcanize/vulcanizedb/pkg/geth"
"github.com/vulcanize/vulcanizedb/pkg/history"
)
const (
pollingInterval = 7 * time.Second
)
func main() {
ticker := time.NewTicker(pollingInterval)
defer ticker.Stop()
environment := flag.String("environment", "", "Environment name")
flag.Parse()
config := cmd.LoadConfig(*environment)
blockchain := geth.NewBlockchain(config.Client.IPCPath)
repository := cmd.LoadPostgres(config.Database, blockchain.Node())
validator := history.NewBlockValidator(blockchain, repository, 15)
for range ticker.C {
window := validator.ValidateBlocks()
validator.Log(os.Stdout, window)
}
}

View File

@ -1,31 +0,0 @@
package main
import (
"flag"
"log"
"fmt"
"github.com/vulcanize/vulcanizedb/cmd"
"github.com/vulcanize/vulcanizedb/pkg/contract_summary"
"github.com/vulcanize/vulcanizedb/pkg/geth"
)
func main() {
environment := flag.String("environment", "", "Environment name")
contractHash := flag.String("contract-hash", "", "Contract hash to show summary")
_blockNumber := flag.Int64("block-number", -1, "Block number of summary")
flag.Parse()
config := cmd.LoadConfig(*environment)
blockchain := geth.NewBlockchain(config.Client.IPCPath)
repository := cmd.LoadPostgres(config.Database, blockchain.Node())
blockNumber := cmd.RequestedBlockNumber(_blockNumber)
contractSummary, err := contract_summary.NewSummary(blockchain, repository, *contractHash, blockNumber)
if err != nil {
log.Fatalln(err)
}
output := contract_summary.GenerateConsoleOutput(contractSummary)
fmt.Println(output)
}

77
cmd/sync.go Normal file
View File

@ -0,0 +1,77 @@
package cmd
import (
"os"
"time"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/geth"
"github.com/vulcanize/vulcanizedb/pkg/history"
"github.com/vulcanize/vulcanizedb/pkg/repositories"
"github.com/vulcanize/vulcanizedb/utils"
"github.com/spf13/cobra"
)
// syncCmd represents the sync command
var syncCmd = &cobra.Command{
Use: "sync",
Short: "Syncs vulcanizedb with local ethereum node",
Long: `Syncs vulcanizedb with local ethereum node.
vulcanizedb sync --startingBlockNumber 0 --config public.toml
Expects ethereum node to be running and requires a .toml config:
[database]
name = "vulcanize_public"
hostname = "localhost"
port = 5432
[client]
ipcPath = "/Users/mattkrump/Library/Ethereum/geth.ipc"
`,
Run: func(cmd *cobra.Command, args []string) {
sync()
},
}
const (
pollingInterval = 7 * time.Second
)
var startingBlockNumber int
func init() {
rootCmd.AddCommand(syncCmd)
syncCmd.Flags().IntVarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block number to start syncing from")
}
func backFillAllBlocks(blockchain core.Blockchain, repository repositories.Postgres, missingBlocksPopulated chan int, startingBlockNumber int64) {
go func() {
missingBlocksPopulated <- history.PopulateMissingBlocks(blockchain, repository, startingBlockNumber)
}()
}
func sync() {
ticker := time.NewTicker(pollingInterval)
defer ticker.Stop()
blockchain := geth.NewBlockchain(ipc)
repository := utils.LoadPostgres(databaseConfig, blockchain.Node())
validator := history.NewBlockValidator(blockchain, repository, 15)
missingBlocksPopulated := make(chan int)
_startingBlockNumber := int64(startingBlockNumber)
go backFillAllBlocks(blockchain, repository, missingBlocksPopulated, _startingBlockNumber)
for {
select {
case <-ticker.C:
window := validator.ValidateBlocks()
validator.Log(os.Stdout, window)
case <-missingBlocksPopulated:
go backFillAllBlocks(blockchain, repository, missingBlocksPopulated, _startingBlockNumber)
}
}
}

View File

@ -1,50 +0,0 @@
package main
import (
"flag"
"time"
"os"
"github.com/vulcanize/vulcanizedb/cmd"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/geth"
"github.com/vulcanize/vulcanizedb/pkg/history"
"github.com/vulcanize/vulcanizedb/pkg/repositories"
)
const (
pollingInterval = 7 * time.Second
)
func backFillAllBlocks(blockchain core.Blockchain, repository repositories.Postgres, missingBlocksPopulated chan int) {
go func() {
missingBlocksPopulated <- history.PopulateMissingBlocks(blockchain, repository, 0)
}()
}
func main() {
ticker := time.NewTicker(pollingInterval)
defer ticker.Stop()
environment := flag.String("environment", "", "Environment name")
flag.Parse()
config := cmd.LoadConfig(*environment)
blockchain := geth.NewBlockchain(config.Client.IPCPath)
repository := cmd.LoadPostgres(config.Database, blockchain.Node())
validator := history.NewBlockValidator(blockchain, repository, 15)
missingBlocksPopulated := make(chan int)
go backFillAllBlocks(blockchain, repository, missingBlocksPopulated)
for {
select {
case <-ticker.C:
window := validator.ValidateBlocks()
validator.Log(os.Stdout, window)
case <-missingBlocksPopulated:
go backFillAllBlocks(blockchain, repository, missingBlocksPopulated)
}
}
}

View File

@ -1,26 +0,0 @@
package main
import (
"flag"
"github.com/vulcanize/vulcanizedb/cmd"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/geth"
)
func main() {
environment := flag.String("environment", "", "Environment name")
contractHash := flag.String("contract-hash", "", "contract-hash=x1234")
abiFilepath := flag.String("abi-filepath", "", "path/to/abifile.json")
flag.Parse()
contractAbiString := cmd.GetAbi(*abiFilepath, *contractHash)
config := cmd.LoadConfig(*environment)
blockchain := geth.NewBlockchain(config.Client.IPCPath)
repository := cmd.LoadPostgres(config.Database, blockchain.Node())
watchedContract := core.Contract{
Abi: contractAbiString,
Hash: *contractHash,
}
repository.CreateContract(watchedContract)
}

View File

@ -0,0 +1,3 @@
ALTER TABLE nodes
DROP COLUMN node_id,
DROP COLUMN client_name;

View File

@ -0,0 +1,3 @@
ALTER TABLE nodes
ADD COLUMN node_id VARCHAR(128),
ADD COLUMN client_name VARCHAR;

View File

@ -0,0 +1,9 @@
BEGIN;
ALTER TABLE nodes
DROP CONSTRAINT node_uc;
ALTER TABLE nodes
ADD CONSTRAINT node_uc UNIQUE (genesis_block, network_id);
COMMIT;

View File

@ -0,0 +1,9 @@
BEGIN;
ALTER TABLE nodes
DROP CONSTRAINT node_uc;
ALTER TABLE nodes
ADD CONSTRAINT node_uc UNIQUE (genesis_block, network_id, node_id);
COMMIT;

View File

@ -0,0 +1,12 @@
BEGIN;
ALTER TABLE logs
DROP CONSTRAINT receipts_fk;
ALTER TABLE logs
DROP COLUMN receipt_id;
ALTER TABLE logs
ADD CONSTRAINT log_uc UNIQUE (block_number, index);
COMMIT;

View File

@ -0,0 +1,14 @@
BEGIN;
ALTER TABLE logs
DROP CONSTRAINT log_uc;
ALTER TABLE logs
ADD COLUMN receipt_id INT;
ALTER TABLE logs
ADD CONSTRAINT receipts_fk
FOREIGN KEY (receipt_id)
REFERENCES receipts (id)
ON DELETE CASCADE;
COMMIT;

View File

@ -0,0 +1 @@
DROP TABLE log_filters;

View File

@ -0,0 +1,12 @@
CREATE TABLE log_filters (
id SERIAL,
name VARCHAR NOT NULL CHECK (name <> ''),
from_block BIGINT CHECK (from_block >= 0),
to_block BIGINT CHECK (from_block >= 0),
address VARCHAR(66),
topic0 VARCHAR(66),
topic1 VARCHAR(66),
topic2 VARCHAR(66),
topic3 VARCHAR(66),
CONSTRAINT name_uc UNIQUE (name)
);

View File

@ -0,0 +1,2 @@
DROP VIEW watched_event_logs;
DROP VIEW block_stats;

View File

@ -0,0 +1,29 @@
CREATE VIEW block_stats AS
SELECT
max(block_number) AS max_block,
min(block_number) AS min_block
FROM logs;
CREATE VIEW watched_event_logs AS
SELECT
log_filters.name,
logs.id,
block_number,
logs.address,
tx_hash,
index,
logs.topic0,
logs.topic1,
logs.topic2,
logs.topic3,
data,
receipt_id
FROM log_filters
CROSS JOIN block_stats
JOIN logs ON logs.address = log_filters.address
AND logs.block_number >= coalesce(log_filters.from_block, block_stats.min_block)
AND logs.block_number <= coalesce(log_filters.to_block, block_stats.max_block)
WHERE (log_filters.topic0 = logs.topic0 OR log_filters.topic0 ISNULL)
AND (log_filters.topic1 = logs.topic1 OR log_filters.topic1 ISNULL)
AND (log_filters.topic2 = logs.topic2 OR log_filters.topic2 ISNULL)
AND (log_filters.topic3 = logs.topic3 OR log_filters.topic3 ISNULL);

564
db/schema.sql Normal file
View File

@ -0,0 +1,564 @@
--
-- PostgreSQL database dump
--
-- Dumped from database version 10.1
-- Dumped by pg_dump version 10.1
SET statement_timeout = 0;
SET lock_timeout = 0;
SET idle_in_transaction_session_timeout = 0;
SET client_encoding = 'UTF8';
SET standard_conforming_strings = on;
SET check_function_bodies = false;
SET client_min_messages = warning;
SET row_security = off;
--
-- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: -
--
CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog;
--
-- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: -
--
COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language';
SET search_path = public, pg_catalog;
SET default_tablespace = '';
SET default_with_oids = false;
--
-- Name: logs; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE logs (
id integer NOT NULL,
block_number bigint,
address character varying(66),
tx_hash character varying(66),
index bigint,
topic0 character varying(66),
topic1 character varying(66),
topic2 character varying(66),
topic3 character varying(66),
data text,
receipt_id integer
);
--
-- Name: block_stats; Type: VIEW; Schema: public; Owner: -
--
CREATE VIEW block_stats AS
SELECT max(logs.block_number) AS max_block,
min(logs.block_number) AS min_block
FROM logs;
--
-- Name: blocks; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE blocks (
block_number bigint,
block_gaslimit double precision,
block_gasused double precision,
block_time double precision,
id integer NOT NULL,
block_difficulty bigint,
block_hash character varying(66),
block_nonce character varying(20),
block_parenthash character varying(66),
block_size bigint,
uncle_hash character varying(66),
node_id integer NOT NULL,
is_final boolean,
block_miner character varying(42),
block_extra_data character varying,
block_reward numeric,
block_uncles_reward numeric
);
--
-- Name: blocks_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE blocks_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: blocks_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE blocks_id_seq OWNED BY blocks.id;
--
-- Name: log_filters; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE log_filters (
id integer NOT NULL,
name character varying NOT NULL,
from_block bigint,
to_block bigint,
address character varying(66),
topic0 character varying(66),
topic1 character varying(66),
topic2 character varying(66),
topic3 character varying(66),
CONSTRAINT log_filters_from_block_check CHECK ((from_block >= 0)),
CONSTRAINT log_filters_from_block_check1 CHECK ((from_block >= 0)),
CONSTRAINT log_filters_name_check CHECK (((name)::text <> ''::text))
);
--
-- Name: log_filters_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE log_filters_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: log_filters_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE log_filters_id_seq OWNED BY log_filters.id;
--
-- Name: logs_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE logs_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: logs_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE logs_id_seq OWNED BY logs.id;
--
-- Name: nodes; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE nodes (
id integer NOT NULL,
genesis_block character varying(66),
network_id numeric,
node_id character varying(128),
client_name character varying
);
--
-- Name: nodes_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE nodes_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: nodes_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE nodes_id_seq OWNED BY nodes.id;
--
-- Name: receipts; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE receipts (
id integer NOT NULL,
transaction_id integer NOT NULL,
contract_address character varying(42),
cumulative_gas_used numeric,
gas_used numeric,
state_root character varying(66),
status integer,
tx_hash character varying(66)
);
--
-- Name: receipts_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE receipts_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: receipts_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE receipts_id_seq OWNED BY receipts.id;
--
-- Name: schema_migrations; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE schema_migrations (
version bigint NOT NULL,
dirty boolean NOT NULL
);
--
-- Name: transactions; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE transactions (
id integer NOT NULL,
tx_hash character varying(66),
tx_nonce numeric,
tx_to character varying(66),
tx_gaslimit numeric,
tx_gasprice numeric,
tx_value numeric,
block_id integer NOT NULL,
tx_from character varying(66),
tx_input_data character varying
);
--
-- Name: transactions_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE transactions_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: transactions_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE transactions_id_seq OWNED BY transactions.id;
--
-- Name: watched_contracts; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE watched_contracts (
contract_id integer NOT NULL,
contract_hash character varying(66),
contract_abi json
);
--
-- Name: watched_contracts_contract_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE watched_contracts_contract_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: watched_contracts_contract_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE watched_contracts_contract_id_seq OWNED BY watched_contracts.contract_id;
--
-- Name: watched_event_logs; Type: VIEW; Schema: public; Owner: -
--
CREATE VIEW watched_event_logs AS
SELECT log_filters.name,
logs.id,
logs.block_number,
logs.address,
logs.tx_hash,
logs.index,
logs.topic0,
logs.topic1,
logs.topic2,
logs.topic3,
logs.data,
logs.receipt_id
FROM ((log_filters
CROSS JOIN block_stats)
JOIN logs ON ((((logs.address)::text = (log_filters.address)::text) AND (logs.block_number >= COALESCE(log_filters.from_block, block_stats.min_block)) AND (logs.block_number <= COALESCE(log_filters.to_block, block_stats.max_block)))))
WHERE ((((log_filters.topic0)::text = (logs.topic0)::text) OR (log_filters.topic0 IS NULL)) AND (((log_filters.topic1)::text = (logs.topic1)::text) OR (log_filters.topic1 IS NULL)) AND (((log_filters.topic2)::text = (logs.topic2)::text) OR (log_filters.topic2 IS NULL)) AND (((log_filters.topic3)::text = (logs.topic3)::text) OR (log_filters.topic3 IS NULL)));
--
-- Name: blocks id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY blocks ALTER COLUMN id SET DEFAULT nextval('blocks_id_seq'::regclass);
--
-- Name: log_filters id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY log_filters ALTER COLUMN id SET DEFAULT nextval('log_filters_id_seq'::regclass);
--
-- Name: logs id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY logs ALTER COLUMN id SET DEFAULT nextval('logs_id_seq'::regclass);
--
-- Name: nodes id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY nodes ALTER COLUMN id SET DEFAULT nextval('nodes_id_seq'::regclass);
--
-- Name: receipts id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY receipts ALTER COLUMN id SET DEFAULT nextval('receipts_id_seq'::regclass);
--
-- Name: transactions id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY transactions ALTER COLUMN id SET DEFAULT nextval('transactions_id_seq'::regclass);
--
-- Name: watched_contracts contract_id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY watched_contracts ALTER COLUMN contract_id SET DEFAULT nextval('watched_contracts_contract_id_seq'::regclass);
--
-- Name: blocks blocks_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY blocks
ADD CONSTRAINT blocks_pkey PRIMARY KEY (id);
--
-- Name: watched_contracts contract_hash_uc; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY watched_contracts
ADD CONSTRAINT contract_hash_uc UNIQUE (contract_hash);
--
-- Name: logs log_uc; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY logs
ADD CONSTRAINT log_uc UNIQUE (block_number, index);
--
-- Name: logs logs_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY logs
ADD CONSTRAINT logs_pkey PRIMARY KEY (id);
--
-- Name: log_filters name_uc; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY log_filters
ADD CONSTRAINT name_uc UNIQUE (name);
--
-- Name: blocks node_id_block_number_uc; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY blocks
ADD CONSTRAINT node_id_block_number_uc UNIQUE (block_number, node_id);
--
-- Name: nodes node_uc; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY nodes
ADD CONSTRAINT node_uc UNIQUE (genesis_block, network_id, node_id);
--
-- Name: nodes nodes_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY nodes
ADD CONSTRAINT nodes_pkey PRIMARY KEY (id);
--
-- Name: receipts receipts_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY receipts
ADD CONSTRAINT receipts_pkey PRIMARY KEY (id);
--
-- Name: schema_migrations schema_migrations_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY schema_migrations
ADD CONSTRAINT schema_migrations_pkey PRIMARY KEY (version);
--
-- Name: transactions transactions_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY transactions
ADD CONSTRAINT transactions_pkey PRIMARY KEY (id);
--
-- Name: watched_contracts watched_contracts_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY watched_contracts
ADD CONSTRAINT watched_contracts_pkey PRIMARY KEY (contract_id);
--
-- Name: block_id_index; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX block_id_index ON transactions USING btree (block_id);
--
-- Name: block_number_index; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX block_number_index ON blocks USING btree (block_number);
--
-- Name: node_id_index; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX node_id_index ON blocks USING btree (node_id);
--
-- Name: transaction_id_index; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX transaction_id_index ON receipts USING btree (transaction_id);
--
-- Name: tx_from_index; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX tx_from_index ON transactions USING btree (tx_from);
--
-- Name: tx_to_index; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX tx_to_index ON transactions USING btree (tx_to);
--
-- Name: transactions blocks_fk; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY transactions
ADD CONSTRAINT blocks_fk FOREIGN KEY (block_id) REFERENCES blocks(id) ON DELETE CASCADE;
--
-- Name: blocks node_fk; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY blocks
ADD CONSTRAINT node_fk FOREIGN KEY (node_id) REFERENCES nodes(id) ON DELETE CASCADE;
--
-- Name: logs receipts_fk; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY logs
ADD CONSTRAINT receipts_fk FOREIGN KEY (receipt_id) REFERENCES receipts(id) ON DELETE CASCADE;
--
-- Name: receipts transaction_fk; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY receipts
ADD CONSTRAINT transaction_fk FOREIGN KEY (transaction_id) REFERENCES transactions(id) ON DELETE CASCADE;
--
-- PostgreSQL database dump complete
--

View File

@ -0,0 +1,15 @@
[{
"name": "TransferFilter",
"fromBlock": "0x488290",
"toBlock": "0x488678",
"address": "0x06012c8cf97bead5deae237070f9587f8e7a266d",
"topics": ["0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"]
},
{
"name": "NewFilter",
"toBlock": "0x4B34AA",
"fromBlock": "0x4B34AD",
"address": "0x06012c8cf97bead5deae237070f9587f8e7a266d",
"topics": ["0x241ea03ca20251805084d27d4440371c34a0b85ff108f6bb5611248f73818b80"]
}]

View File

@ -15,8 +15,7 @@ import (
var _ = Describe("Reading contracts", func() {
//TODO was experiencing Infura issue (I suspect) on 1/5. Unignore these and revisit if persists on next commit
XDescribe("Reading the list of attributes", func() {
Describe("Reading the list of attributes", func() {
It("returns a string attribute for a real contract", func() {
config, err := cfg.NewConfig("infura")
if err != nil {
@ -59,8 +58,7 @@ var _ = Describe("Reading contracts", func() {
})
})
//TODO was experiencing Infura issue (I suspect) on 1/5. Unignore these and revisit if persists on next commit
XDescribe("Getting a contract attribute", func() {
Describe("Getting a contract attribute", func() {
It("returns the correct attribute for a real contract", func() {
config, _ := cfg.NewConfig("infura")
blockchain := geth.NewBlockchain(config.Client.IPCPath)
@ -109,8 +107,8 @@ var _ = Describe("Reading contracts", func() {
expectedLogZero := core.Log{
BlockNumber: 4703824,
TxHash: "0xf896bfd1eb539d881a1a31102b78de9f25cd591bf1fe1924b86148c0b205fd5d",
Address: "0xd26114cd6EE289AccF82350c8d8487fedB8A0C07",
Topics: map[int]string{
Address: "0xd26114cd6ee289accf82350c8d8487fedb8a0c07",
Topics: core.Topics{
0: "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef",
1: "0x000000000000000000000000fbb1b73c4f0bda4f67dca266ce6ef42f520fbb98",
2: "0x000000000000000000000000d26114cd6ee289accf82350c8d8487fedb8a0c07",

View File

@ -52,6 +52,8 @@ var _ = Describe("Reading from the Geth blockchain", func() {
Expect(node.GenesisBlock).To(Equal(devNetworkGenesisBlock))
Expect(node.NetworkId).To(Equal(devNetworkNodeId))
Expect(len(node.Id)).To(Equal(128))
Expect(node.ClientName).To(ContainSubstring("Geth"))
close(done)
}, 15)

9
main.go Normal file
View File

@ -0,0 +1,9 @@
package main
import (
"github.com/vulcanize/vulcanizedb/cmd"
)
func main() {
cmd.Execute()
}

View File

@ -4,7 +4,7 @@ type Log struct {
BlockNumber int64
TxHash string
Address string
Topics map[int]string
Topics
Index int64
Data string
}

View File

@ -3,4 +3,6 @@ package core
type Node struct {
GenesisBlock string
NetworkId float64
Id string
ClientName string
}

3
pkg/core/topics.go Normal file
View File

@ -0,0 +1,3 @@
package core
type Topics [4]string

View File

@ -9,5 +9,5 @@ type Transaction struct {
GasLimit int64
GasPrice int64
Receipt
Value int64
Value string
}

View File

@ -50,7 +50,7 @@ func NewBlockchain() *Blockchain {
blocks: make(map[int64]core.Block),
logs: make(map[string][]core.Log),
contractAttributes: make(map[string]map[string]string),
node: core.Node{GenesisBlock: "GENESIS"},
node: core.Node{GenesisBlock: "GENESIS", NetworkId: 1, Id: "x123", ClientName: "Geth"},
}
}

View File

@ -0,0 +1,64 @@
package filters
import (
"encoding/json"
"errors"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
)
type LogFilters []LogFilter
type LogFilter struct {
Name string `json:"name"`
FromBlock int64 `json:"fromBlock"`
ToBlock int64 `json:"toBlock"`
Address string `json:"address"`
core.Topics `json:"topics"`
}
func (filterQuery *LogFilter) UnmarshalJSON(input []byte) error {
type Alias LogFilter
var err error
aux := &struct {
ToBlock string `json:"toBlock"`
FromBlock string `json:"fromBlock"`
*Alias
}{
Alias: (*Alias)(filterQuery),
}
if err := json.Unmarshal(input, &aux); err != nil {
return err
}
if filterQuery.Name == "" {
return errors.New("filters: must provide name for logfilter")
}
filterQuery.ToBlock, err = filterQuery.unmarshalFromToBlock(aux.ToBlock)
if err != nil {
return errors.New("filters: invalid fromBlock")
}
filterQuery.FromBlock, err = filterQuery.unmarshalFromToBlock(aux.FromBlock)
if err != nil {
return errors.New("filters: invalid fromBlock")
}
if !common.IsHexAddress(filterQuery.Address) {
return errors.New("filters: invalid address")
}
return nil
}
func (filterQuery *LogFilter) unmarshalFromToBlock(auxBlock string) (int64, error) {
if auxBlock == "" {
return -1, nil
}
block, err := hexutil.DecodeUint64(auxBlock)
if err != nil {
return 0, errors.New("filters: invalid block arg")
}
return int64(block), nil
}

125
pkg/filters/filter_test.go Normal file
View File

@ -0,0 +1,125 @@
package filters_test
import (
"encoding/json"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/filters"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
var _ = Describe("Log filters", func() {
It("decodes web3 filter to LogFilter", func() {
var logFilter filters.LogFilter
jsonFilter := []byte(
`{
"name": "TestEvent",
"fromBlock": "0x1",
"toBlock": "0x488290",
"address": "0x8888f1f195afa192cfee860698584c030f4c9db1",
"topics": ["0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", null, "0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", null]
}`)
err := json.Unmarshal(jsonFilter, &logFilter)
Expect(err).ToNot(HaveOccurred())
Expect(logFilter.Name).To(Equal("TestEvent"))
Expect(logFilter.FromBlock).To(Equal(int64(1)))
Expect(logFilter.ToBlock).To(Equal(int64(4752016)))
Expect(logFilter.Address).To(Equal("0x8888f1f195afa192cfee860698584c030f4c9db1"))
Expect(logFilter.Topics).To(Equal(
core.Topics{
"0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b",
"",
"0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b",
""}))
})
It("decodes array of web3 filters to []LogFilter", func() {
logFilters := make([]filters.LogFilter, 0)
jsonFilter := []byte(
`[{
"name": "TestEvent",
"fromBlock": "0x1",
"toBlock": "0x488290",
"address": "0x8888f1f195afa192cfee860698584c030f4c9db1",
"topics": ["0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", null, "0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", null]
},
{
"name": "TestEvent2",
"fromBlock": "0x3",
"toBlock": "0x4",
"address": "0xd26114cd6EE289AccF82350c8d8487fedB8A0C07",
"topics": ["0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", "0x0000000000000000000000006b0949d4c6edfe467db78241b7d5566f3c2bb43e", "0x0000000000000000000000005e44c3e467a49c9ca0296a9f130fc433041aaa28"]
}]`)
err := json.Unmarshal(jsonFilter, &logFilters)
Expect(err).ToNot(HaveOccurred())
Expect(len(logFilters)).To(Equal(2))
Expect(logFilters[0].Name).To(Equal("TestEvent"))
Expect(logFilters[1].Name).To(Equal("TestEvent2"))
})
It("requires valid ethereum address", func() {
var logFilter filters.LogFilter
jsonFilter := []byte(
`{
"name": "TestEvent",
"fromBlock": "0x1",
"toBlock": "0x2",
"address": "0x8888f1f195afa192cf84c030f4c9db1",
"topics": ["0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", null, "0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", null]
}`)
err := json.Unmarshal(jsonFilter, &logFilter)
Expect(err).To(HaveOccurred())
})
It("requires name", func() {
var logFilter filters.LogFilter
jsonFilter := []byte(
`{
"fromBlock": "0x1",
"toBlock": "0x2",
"address": "0x8888f1f195afa192cfee860698584c030f4c9db1",
"topics": ["0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", null, "0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", null]
}`)
err := json.Unmarshal(jsonFilter, &logFilter)
Expect(err).To(HaveOccurred())
})
It("maps missing fromBlock to -1", func() {
var logFilter filters.LogFilter
jsonFilter := []byte(
`{
"name": "TestEvent",
"toBlock": "0x2",
"address": "0x8888f1f195afa192cfee860698584c030f4c9db1",
"topics": ["0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", null, "0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", null]
}`)
err := json.Unmarshal(jsonFilter, &logFilter)
Expect(err).ToNot(HaveOccurred())
Expect(logFilter.FromBlock).To(Equal(int64(-1)))
})
It("maps missing toBlock to -1", func() {
var logFilter filters.LogFilter
jsonFilter := []byte(
`{
"name": "TestEvent",
"address": "0x8888f1f195afa192cfee860698584c030f4c9db1",
"topics": ["0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", null, "0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", null]
}`)
err := json.Unmarshal(jsonFilter, &logFilter)
Expect(err).ToNot(HaveOccurred())
Expect(logFilter.ToBlock).To(Equal(int64(-1)))
})
})

View File

@ -0,0 +1,13 @@
package filters_test
import (
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
func TestQueryBuilder(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "QueryBuilder Suite")
}

View File

@ -38,6 +38,19 @@ func NewEtherScanClient(url string) *EtherScanApi {
}
func GenUrl(network string) string {
switch network {
case "ropsten":
return "https://ropsten.etherscan.io"
case "kovan":
return "https://kovan.etherscan.io"
case "rinkeby":
return "https://rinkeby.etherscan.io"
default:
return "https://api.etherscan.io"
}
}
//https://api.etherscan.io/api?module=contract&action=getabi&address=%s
func (e *EtherScanApi) GetAbi(contractHash string) (string, error) {
target := new(Response)

View File

@ -101,5 +101,22 @@ var _ = Describe("ABI files", func() {
})
})
})
Describe("Generating etherscan endpoints based on network", func() {
It("should return the main endpoint as the default", func() {
url := geth.GenUrl("")
Expect(url).To(Equal("https://api.etherscan.io"))
})
It("generates various test network endpoint if test network is supplied", func() {
ropstenUrl := geth.GenUrl("ropsten")
rinkebyUrl := geth.GenUrl("rinkeby")
kovanUrl := geth.GenUrl("kovan")
Expect(ropstenUrl).To(Equal("https://ropsten.etherscan.io"))
Expect(kovanUrl).To(Equal("https://kovan.etherscan.io"))
Expect(rinkebyUrl).To(Equal("https://rinkeby.etherscan.io"))
})
})
})
})

View File

@ -25,7 +25,7 @@ func ToCoreBlock(gethBlock *types.Block, client GethClient) core.Block {
GasLimit: gethBlock.GasLimit().Int64(),
GasUsed: gethBlock.GasUsed().Int64(),
Hash: gethBlock.Hash().Hex(),
Miner: gethBlock.Coinbase().Hex(),
Miner: strings.ToLower(gethBlock.Coinbase().Hex()),
Nonce: hexutil.Encode(gethBlock.Header().Nonce[:]),
Number: gethBlock.Number().Int64(),
ParentHash: gethBlock.ParentHash().Hex(),
@ -58,6 +58,10 @@ func convertTransactionsToCore(gethBlock *types.Block, client GethClient) []core
func appendReceiptToTransaction(client GethClient, transaction core.Transaction) (core.Transaction, error) {
gethReceipt, err := client.TransactionReceipt(context.Background(), common.HexToHash(transaction.Hash))
if err != nil {
log.Println(err)
return transaction, err
}
receipt := ReceiptToCoreReceipt(gethReceipt)
transaction.Receipt = receipt
return transaction, err
@ -72,7 +76,7 @@ func transToCoreTrans(transaction *types.Transaction, from *common.Address) core
From: strings.ToLower(addressToHex(from)),
GasLimit: transaction.Gas().Int64(),
GasPrice: transaction.GasPrice().Int64(),
Value: transaction.Value().Int64(),
Value: transaction.Value().String(),
Data: data,
}
}

View File

@ -246,7 +246,7 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() {
Expect(coreTransaction.From).To(Equal("0x0000000000000000000000000000000000000123"))
Expect(coreTransaction.GasLimit).To(Equal(gethTransaction.Gas().Int64()))
Expect(coreTransaction.GasPrice).To(Equal(gethTransaction.GasPrice().Int64()))
Expect(coreTransaction.Value).To(Equal(gethTransaction.Value().Int64()))
Expect(coreTransaction.Value).To(Equal(gethTransaction.Value().String()))
Expect(coreTransaction.Nonce).To(Equal(gethTransaction.Nonce()))
coreReceipt := coreTransaction.Receipt

View File

@ -3,6 +3,10 @@ package geth
import (
"math/big"
"strings"
"log"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/geth/node"
"github.com/ethereum/go-ethereum"
@ -23,13 +27,27 @@ type Blockchain struct {
func NewBlockchain(ipcPath string) *Blockchain {
blockchain := Blockchain{}
rpcClient, _ := rpc.Dial(ipcPath)
rpcClient, err := rpc.Dial(ipcPath)
if err != nil {
log.Fatal(err)
}
client := ethclient.NewClient(rpcClient)
blockchain.node = node.Retrieve(rpcClient)
blockchain.node = node.Info(rpcClient)
if infura := isInfuraNode(ipcPath); infura {
blockchain.node.Id = "infura"
blockchain.node.ClientName = "infura"
}
blockchain.client = client
return &blockchain
}
func isInfuraNode(ipcPath string) bool {
if strings.Contains(ipcPath, "infura") {
return true
}
return false
}
func (blockchain *Blockchain) GetLogs(contract core.Contract, startingBlockNumber *big.Int, endingBlockNumber *big.Int) ([]core.Log, error) {
if endingBlockNumber == nil {
endingBlockNumber = startingBlockNumber
@ -44,7 +62,7 @@ func (blockchain *Blockchain) GetLogs(contract core.Contract, startingBlockNumbe
if err != nil {
return []core.Log{}, err
}
logs := GethLogsToCoreLogs(gethLogs)
logs := ToCoreLogs(gethLogs)
return logs, nil
}

View File

@ -1,19 +1,36 @@
package geth
import (
"strings"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/core/types"
)
func LogToCoreLog(gethLog types.Log) core.Log {
topics := gethLog.Topics
var hexTopics = make(map[int]string)
func ToCoreLogs(gethLogs []types.Log) []core.Log {
var logs []core.Log
for _, log := range gethLogs {
log := ToCoreLog(log)
logs = append(logs, log)
}
return logs
}
func makeTopics(topics []common.Hash) core.Topics {
var hexTopics core.Topics
for i, topic := range topics {
hexTopics[i] = topic.Hex()
}
return hexTopics
}
func ToCoreLog(gethLog types.Log) core.Log {
topics := gethLog.Topics
hexTopics := makeTopics(topics)
return core.Log{
Address: gethLog.Address.Hex(),
Address: strings.ToLower(gethLog.Address.Hex()),
BlockNumber: int64(gethLog.BlockNumber),
Topics: hexTopics,
@ -22,12 +39,3 @@ func LogToCoreLog(gethLog types.Log) core.Log {
Data: hexutil.Encode(gethLog.Data),
}
}
func GethLogsToCoreLogs(gethLogs []types.Log) []core.Log {
var logs []core.Log
for _, log := range gethLogs {
log := LogToCoreLog(log)
logs = append(logs, log)
}
return logs
}

View File

@ -1,6 +1,8 @@
package geth_test
import (
"strings"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/geth"
"github.com/ethereum/go-ethereum/common"
@ -14,7 +16,7 @@ var _ = Describe("Conversion of GethLog to core.Log", func() {
It("converts geth log to internal log format", func() {
gethLog := types.Log{
Address: common.HexToAddress("0xecf8f87f810ecf450940c9f60066b4a7a501d6a7"),
Address: common.HexToAddress("0x448a5065aeBB8E423F0896E6c5D525C040f59af3"),
BlockHash: common.HexToHash("0x656c34545f90a730a19008c0e7a7cd4fb3895064b48d6d69761bd5abad681056"),
BlockNumber: 2019236,
Data: hexutil.MustDecode("0x000000000000000000000000000000000000000000000001a055690d9db80000"),
@ -28,18 +30,18 @@ var _ = Describe("Conversion of GethLog to core.Log", func() {
}
expected := core.Log{
Address: gethLog.Address.Hex(),
Address: strings.ToLower(gethLog.Address.Hex()),
BlockNumber: int64(gethLog.BlockNumber),
Data: hexutil.Encode(gethLog.Data),
TxHash: gethLog.TxHash.Hex(),
Index: 2,
Topics: map[int]string{
0: common.HexToHash("0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef").Hex(),
1: common.HexToHash("0x00000000000000000000000080b2c9d7cbbf30a1b0fc8983c647d754c6525615").Hex(),
Topics: core.Topics{
gethLog.Topics[0].Hex(),
gethLog.Topics[1].Hex(),
},
}
coreLog := geth.LogToCoreLog(gethLog)
coreLog := geth.ToCoreLog(gethLog)
Expect(coreLog.Address).To(Equal(expected.Address))
Expect(coreLog.BlockNumber).To(Equal(expected.BlockNumber))
@ -79,10 +81,10 @@ var _ = Describe("Conversion of GethLog to core.Log", func() {
},
}
expectedOne := geth.LogToCoreLog(gethLogOne)
expectedTwo := geth.LogToCoreLog(gethLogTwo)
expectedOne := geth.ToCoreLog(gethLogOne)
expectedTwo := geth.ToCoreLog(gethLogTwo)
coreLogs := geth.GethLogsToCoreLogs([]types.Log{gethLogOne, gethLogTwo})
coreLogs := geth.ToCoreLogs([]types.Log{gethLogOne, gethLogTwo})
Expect(len(coreLogs)).To(Equal(2))
Expect(coreLogs[0]).To(Equal(expectedOne))

View File

@ -3,30 +3,47 @@ package node
import (
"context"
"strconv"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/rpc"
)
func Retrieve(client *rpc.Client) core.Node {
var info p2p.NodeInfo
func Info(client *rpc.Client) core.Node {
node := core.Node{}
client.CallContext(context.Background(), &info, "admin_nodeInfo")
for protocolName, protocol := range info.Protocols {
if protocolName == "eth" {
protocolMap, _ := protocol.(map[string]interface{})
node.GenesisBlock = getAttribute(protocolMap, "genesis").(string)
node.NetworkId = getAttribute(protocolMap, "network").(float64)
}
}
node.NetworkId = NetworkId(client)
node.GenesisBlock = GenesisBlock(client)
node.Id, node.ClientName = IdClientName(client)
return node
}
func getAttribute(protocolMap map[string]interface{}, protocol string) interface{} {
for key, val := range protocolMap {
if key == protocol {
return val
func IdClientName(client *rpc.Client) (string, string) {
var info p2p.NodeInfo
modules, _ := client.SupportedModules()
if _, ok := modules["admin"]; ok {
client.CallContext(context.Background(), &info, "admin_nodeInfo")
return info.ID, info.Name
}
}
return nil
return "", ""
}
func NetworkId(client *rpc.Client) float64 {
var version string
client.CallContext(context.Background(), &version, "net_version")
networkId, _ := strconv.ParseFloat(version, 64)
return networkId
}
func ProtocolVersion(client *rpc.Client) string {
var protocolVersion string
client.CallContext(context.Background(), &protocolVersion, "eth_protocolVersion")
return protocolVersion
}
func GenesisBlock(client *rpc.Client) string {
var header *types.Header
client.CallContext(context.Background(), &header, "eth_getBlockByNumber", "0x0", false)
return header.Hash().Hex()
}

View File

@ -49,7 +49,7 @@ func setContractAddress(gethReceipt *types.Receipt) string {
func dereferenceLogs(gethReceipt *types.Receipt) []core.Log {
logs := []core.Log{}
for _, log := range gethReceipt.Logs {
logs = append(logs, LogToCoreLog(*log))
logs = append(logs, ToCoreLog(*log))
}
return logs
}

View File

@ -3,7 +3,10 @@ package repositories
import (
"fmt"
"errors"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/filters"
)
type InMemory struct {
@ -11,9 +14,19 @@ type InMemory struct {
receipts map[string]core.Receipt
contracts map[string]core.Contract
logs map[string][]core.Log
logFilters map[string]filters.LogFilter
CreateOrUpdateBlockCallCount int
}
func (repository *InMemory) AddFilter(filter filters.LogFilter) error {
key := filter.Name
if _, ok := repository.logFilters[key]; ok || key == "" {
return errors.New("filter name not unique")
}
repository.logFilters[key] = filter
return nil
}
func NewInMemory() *InMemory {
return &InMemory{
CreateOrUpdateBlockCallCount: 0,
@ -21,6 +34,7 @@ func NewInMemory() *InMemory {
receipts: make(map[string]core.Receipt),
contracts: make(map[string]core.Contract),
logs: make(map[string][]core.Log),
logFilters: make(map[string]filters.LogFilter),
}
}
@ -102,6 +116,7 @@ func (repository *InMemory) CreateOrUpdateBlock(block core.Block) error {
repository.blocks[block.Number] = block
for _, transaction := range block.Transactions {
repository.receipts[transaction.Hash] = transaction.Receipt
repository.logs[transaction.TxHash] = transaction.Logs
}
return nil
}

View File

@ -11,6 +11,7 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/filters"
"github.com/jmoiron/sqlx"
_ "github.com/lib/pq"
)
@ -62,35 +63,6 @@ func (repository Postgres) SetBlocksStatus(chainHead int64) {
cutoff)
}
func (repository Postgres) CreateLogs(logs []core.Log) error {
tx, _ := repository.Db.BeginTx(context.Background(), nil)
for _, tlog := range logs {
_, err := tx.Exec(
`INSERT INTO logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
ON CONFLICT (index, block_number)
DO UPDATE
SET block_number = $1,
address = $2,
tx_hash = $3,
index = $4,
topic0 = $5,
topic1 = $6,
topic2 = $7,
topic3 = $8,
data = $9
`,
tlog.BlockNumber, tlog.Address, tlog.TxHash, tlog.Index, tlog.Topics[0], tlog.Topics[1], tlog.Topics[2], tlog.Topics[3], tlog.Data,
)
if err != nil {
tx.Rollback()
return ErrDBInsertFailed
}
}
tx.Commit()
return nil
}
func (repository Postgres) FindLogs(address string, blockNumber int64) []core.Log {
logRows, _ := repository.Db.Query(
`SELECT block_number,
@ -111,13 +83,16 @@ func (repository Postgres) FindLogs(address string, blockNumber int64) []core.Lo
func (repository *Postgres) CreateNode(node *core.Node) error {
var nodeId int64
err := repository.Db.QueryRow(
`INSERT INTO nodes (genesis_block, network_id)
VALUES ($1, $2)
ON CONFLICT (genesis_block, network_id)
`INSERT INTO nodes (genesis_block, network_id, node_id, client_name)
VALUES ($1, $2, $3, $4)
ON CONFLICT (genesis_block, network_id, node_id)
DO UPDATE
SET genesis_block = $1, network_id = $2
SET genesis_block = $1,
network_id = $2,
node_id = $3,
client_name = $4
RETURNING id`,
node.GenesisBlock, node.NetworkId).Scan(&nodeId)
node.GenesisBlock, node.NetworkId, node.Id, node.ClientName).Scan(&nodeId)
if err != nil {
return ErrUnableToSetNode
}
@ -329,29 +304,90 @@ func (repository Postgres) createTransaction(tx *sql.Tx, blockId int64, transact
err := tx.QueryRow(
`INSERT INTO transactions
(block_id, tx_hash, tx_nonce, tx_to, tx_from, tx_gaslimit, tx_gasprice, tx_value, tx_input_data)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
VALUES ($1, $2, $3, $4, $5, $6, $7, cast(NULLIF($8, '') AS NUMERIC), $9)
RETURNING id`,
blockId, transaction.Hash, transaction.Nonce, transaction.To, transaction.From, transaction.GasLimit, transaction.GasPrice, transaction.Value, transaction.Data).
Scan(&transactionId)
if err != nil {
return err
}
if transaction.Receipt.TxHash != "" {
err = repository.createReceipt(tx, transactionId, transaction.Receipt)
if hasReceipt(transaction) {
receiptId, err := repository.createReceipt(tx, transactionId, transaction.Receipt)
if err != nil {
return err
}
if hasLogs(transaction) {
err = repository.createLogs(tx, transaction.Receipt.Logs, receiptId)
if err != nil {
return err
}
}
}
return nil
}
func (repository Postgres) createReceipt(tx *sql.Tx, transactionId int, receipt core.Receipt) error {
func hasLogs(transaction core.Transaction) bool {
return len(transaction.Receipt.Logs) > 0
}
func hasReceipt(transaction core.Transaction) bool {
return transaction.Receipt.TxHash != ""
}
func (repository Postgres) createReceipt(tx *sql.Tx, transactionId int, receipt core.Receipt) (int, error) {
//Not currently persisting log bloom filters
_, err := tx.Exec(
var receiptId int
err := tx.QueryRow(
`INSERT INTO receipts
(contract_address, tx_hash, cumulative_gas_used, gas_used, state_root, status, transaction_id)
VALUES ($1, $2, $3, $4, $5, $6, $7)`,
receipt.ContractAddress, receipt.TxHash, receipt.CumulativeGasUsed, receipt.GasUsed, receipt.StateRoot, receipt.Status, transactionId)
VALUES ($1, $2, $3, $4, $5, $6, $7)
RETURNING id`,
receipt.ContractAddress, receipt.TxHash, receipt.CumulativeGasUsed, receipt.GasUsed, receipt.StateRoot, receipt.Status, transactionId).Scan(&receiptId)
if err != nil {
return receiptId, err
}
return receiptId, nil
}
func (repository Postgres) createLogs(tx *sql.Tx, logs []core.Log, receiptId int) error {
for _, tlog := range logs {
_, err := tx.Exec(
`INSERT INTO logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data, receipt_id)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
`,
tlog.BlockNumber, tlog.Address, tlog.TxHash, tlog.Index, tlog.Topics[0], tlog.Topics[1], tlog.Topics[2], tlog.Topics[3], tlog.Data, receiptId,
)
if err != nil {
return ErrDBInsertFailed
}
}
return nil
}
func (repository Postgres) CreateLogs(logs []core.Log) error {
tx, _ := repository.Db.BeginTx(context.Background(), nil)
for _, tlog := range logs {
_, err := tx.Exec(
`INSERT INTO logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
`,
tlog.BlockNumber, tlog.Address, tlog.TxHash, tlog.Index, tlog.Topics[0], tlog.Topics[1], tlog.Topics[2], tlog.Topics[3], tlog.Data,
)
if err != nil {
tx.Rollback()
return ErrDBInsertFailed
}
}
tx.Commit()
return nil
}
func (repository Postgres) AddFilter(query filters.LogFilter) error {
_, err := repository.Db.Exec(
`INSERT INTO log_filters
(name, from_block, to_block, address, topic0, topic1, topic2, topic3)
VALUES ($1, NULLIF($2, -1), NULLIF($3, -1), $4, NULLIF($5, ''), NULLIF($6, ''), NULLIF($7, ''), NULLIF($8, ''))`,
query.Name, query.FromBlock, query.ToBlock, query.Address, query.Topics[0], query.Topics[1], query.Topics[2], query.Topics[3])
if err != nil {
return err
}
@ -439,7 +475,7 @@ func (repository Postgres) loadLogs(logsRows *sql.Rows) []core.Log {
var txHash string
var index int64
var data string
topics := make([]string, 4)
var topics core.Topics
logsRows.Scan(&blockNumber, &address, &txHash, &index, &topics[0], &topics[1], &topics[2], &topics[3], &data)
log := core.Log{
BlockNumber: blockNumber,
@ -448,7 +484,6 @@ func (repository Postgres) loadLogs(logsRows *sql.Rows) []core.Log {
Index: index,
Data: data,
}
log.Topics = make(map[int]string)
for i, topic := range topics {
log.Topics[i] = topic
}
@ -467,7 +502,7 @@ func (repository Postgres) loadTransactions(transactionRows *sql.Rows) []core.Tr
var gasLimit int64
var gasPrice int64
var inputData string
var value int64
var value string
transactionRows.Scan(&hash, &nonce, &to, &from, &gasLimit, &gasPrice, &value, &inputData)
transaction := core.Transaction{
Hash: hash,

View File

@ -7,6 +7,8 @@ import (
"io/ioutil"
"log"
"math/big"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/repositories"
@ -38,6 +40,42 @@ var _ = Describe("Postgres repository", func() {
return repository
})
It("serializes big.Int to db", func() {
// postgres driver doesn't support go big.Int type
// various casts in golang uint64, int64, overflow for
// transaction value (in wei) even though
// postgres numeric can handle an arbitrary
// sized int, so use string representation of big.Int
// and cast on insert
cfg, _ := config.NewConfig("private")
pgConfig := config.DbConnectionString(cfg.Database)
db, err := sqlx.Connect("postgres", pgConfig)
bi := new(big.Int)
bi.SetString("34940183920000000000", 10)
Expect(bi.String()).To(Equal("34940183920000000000"))
defer db.Exec(`DROP TABLE IF EXISTS example`)
_, err = db.Exec("CREATE TABLE example ( id INTEGER, data NUMERIC )")
Expect(err).ToNot(HaveOccurred())
sqlStatement := `
INSERT INTO example (id, data)
VALUES (1, cast($1 AS NUMERIC))`
_, err = db.Exec(sqlStatement, bi.String())
Expect(err).ToNot(HaveOccurred())
var data string
err = db.QueryRow(`SELECT data FROM example WHERE id = 1`).Scan(&data)
Expect(err).ToNot(HaveOccurred())
Expect(bi.String()).To(Equal(data))
actual := new(big.Int)
actual.SetString(data, 10)
Expect(actual).To(Equal(bi))
})
It("does not commit block if block is invalid", func() {
//badNonce violates db Nonce field length
badNonce := fmt.Sprintf("x %s", strings.Repeat("1", 100))
@ -47,7 +85,7 @@ var _ = Describe("Postgres repository", func() {
Transactions: []core.Transaction{},
}
cfg, _ := config.NewConfig("private")
node := core.Node{GenesisBlock: "GENESIS", NetworkId: 1}
node := core.Node{GenesisBlock: "GENESIS", NetworkId: 1, Id: "x123", ClientName: "geth"}
repository, _ := repositories.NewPostgres(cfg.Database, node)
err1 := repository.CreateOrUpdateBlock(badBlock)
@ -60,7 +98,7 @@ var _ = Describe("Postgres repository", func() {
It("throws error when can't connect to the database", func() {
invalidDatabase := config.Database{}
node := core.Node{GenesisBlock: "GENESIS", NetworkId: 1}
node := core.Node{GenesisBlock: "GENESIS", NetworkId: 1, Id: "x123", ClientName: "geth"}
_, err := repositories.NewPostgres(invalidDatabase, node)
Expect(err).To(Equal(repositories.ErrDBConnectionFailed))
})
@ -68,7 +106,7 @@ var _ = Describe("Postgres repository", func() {
It("throws error when can't create node", func() {
cfg, _ := config.NewConfig("private")
badHash := fmt.Sprintf("x %s", strings.Repeat("1", 100))
node := core.Node{GenesisBlock: badHash, NetworkId: 1}
node := core.Node{GenesisBlock: badHash, NetworkId: 1, Id: "x123", ClientName: "geth"}
_, err := repositories.NewPostgres(cfg.Database, node)
Expect(err).To(Equal(repositories.ErrUnableToSetNode))
})
@ -82,7 +120,7 @@ var _ = Describe("Postgres repository", func() {
TxHash: badTxHash,
}
cfg, _ := config.NewConfig("private")
node := core.Node{GenesisBlock: "GENESIS", NetworkId: 1}
node := core.Node{GenesisBlock: "GENESIS", NetworkId: 1, Id: "x123", ClientName: "geth"}
repository, _ := repositories.NewPostgres(cfg.Database, node)
err := repository.CreateLogs([]core.Log{badLog})
@ -101,7 +139,7 @@ var _ = Describe("Postgres repository", func() {
Transactions: []core.Transaction{badTransaction},
}
cfg, _ := config.NewConfig("private")
node := core.Node{GenesisBlock: "GENESIS", NetworkId: 1}
node := core.Node{GenesisBlock: "GENESIS", NetworkId: 1, Id: "x123", ClientName: "geth"}
repository, _ := repositories.NewPostgres(cfg.Database, node)
err1 := repository.CreateOrUpdateBlock(block)

View File

@ -1,6 +1,9 @@
package repositories
import "github.com/vulcanize/vulcanizedb/pkg/core"
import (
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/filters"
)
const (
blocksFromHeadBeforeFinal = 20
@ -19,4 +22,5 @@ type Repository interface {
CreateLogs(log []core.Log) error
FindLogs(address string, blockNumber int64) []core.Log
SetBlocksStatus(chainHead int64)
AddFilter(filter filters.LogFilter) error
}

View File

@ -4,7 +4,10 @@ import (
"sort"
"strconv"
"math/big"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/filters"
"github.com/vulcanize/vulcanizedb/pkg/repositories"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
@ -16,13 +19,19 @@ func ClearData(postgres repositories.Postgres) {
postgres.Db.MustExec("DELETE FROM blocks")
postgres.Db.MustExec("DELETE FROM logs")
postgres.Db.MustExec("DELETE FROM receipts")
postgres.Db.MustExec("DELETE FROM log_filters")
}
func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories.Repository) {
var repository repositories.Repository
BeforeEach(func() {
node := core.Node{GenesisBlock: "GENESIS", NetworkId: 1}
node := core.Node{
GenesisBlock: "GENESIS",
NetworkId: 1,
Id: "b6f90c0fdd8ec9607aed8ee45c69322e47b7063f0bfb7a29c8ecafab24d0a22d24dd2329b5ee6ed4125a03cb14e57fd584e67f9e53e6c631055cbbd82f080845",
ClientName: "Geth/v1.7.2-stable-1db4ecdc/darwin-amd64/go1.9",
}
repository = buildRepository(node)
})
@ -48,6 +57,8 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories.
nodeTwo := core.Node{
GenesisBlock: "0x456",
NetworkId: 1,
Id: "x123456",
ClientName: "Geth",
}
repositoryTwo := buildRepository(nodeTwo)
@ -191,7 +202,8 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories.
nonce := uint64(10000)
to := "1234567890"
from := "0987654321"
value := int64(10)
var value = new(big.Int)
value.SetString("34940183920000000000", 10)
inputData := "0xf7d8c8830000000000000000000000000000000000000000000000000000000000037788000000000000000000000000000000000000000000000000000000000003bd14"
transaction := core.Transaction{
Hash: "x1234",
@ -200,7 +212,7 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories.
Nonce: nonce,
To: to,
From: from,
Value: value,
Value: value.String(),
Data: inputData,
}
block := core.Block{
@ -220,7 +232,7 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories.
Expect(savedTransaction.Nonce).To(Equal(nonce))
Expect(savedTransaction.GasLimit).To(Equal(gasLimit))
Expect(savedTransaction.GasPrice).To(Equal(gasPrice))
Expect(savedTransaction.Value).To(Equal(value))
Expect(savedTransaction.Value).To(Equal(value.String()))
})
})
@ -392,7 +404,7 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories.
Index: 0,
Address: "x123",
TxHash: "x456",
Topics: map[int]string{0: "x777", 1: "x888", 2: "x999"},
Topics: core.Topics{0: "x777", 1: "x888", 2: "x999"},
Data: "xabc",
}},
)
@ -415,37 +427,13 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories.
Expect(log).To(BeNil())
})
It("updates the log when log with when log with same block number and index is already present", func() {
repository.CreateLogs([]core.Log{{
BlockNumber: 1,
Index: 0,
Address: "x123",
TxHash: "x456",
Topics: map[int]string{0: "x777", 1: "x888", 2: "x999"},
Data: "xABC",
},
})
repository.CreateLogs([]core.Log{{
BlockNumber: 1,
Index: 0,
Address: "x123",
TxHash: "x456",
Topics: map[int]string{0: "x777", 1: "x888", 2: "x999"},
Data: "xXYZ",
},
})
log := repository.FindLogs("x123", 1)
Expect(log[0].Data).To(Equal("xXYZ"))
})
It("filters to the correct block number and address", func() {
repository.CreateLogs([]core.Log{{
BlockNumber: 1,
Index: 0,
Address: "x123",
TxHash: "x456",
Topics: map[int]string{0: "x777", 1: "x888", 2: "x999"},
Topics: core.Topics{0: "x777", 1: "x888", 2: "x999"},
Data: "xabc",
}},
)
@ -454,7 +442,7 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories.
Index: 1,
Address: "x123",
TxHash: "x789",
Topics: map[int]string{0: "x111", 1: "x222", 2: "x333"},
Topics: core.Topics{0: "x111", 1: "x222", 2: "x333"},
Data: "xdef",
}},
)
@ -463,7 +451,7 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories.
Index: 0,
Address: "x123",
TxHash: "x456",
Topics: map[int]string{0: "x777", 1: "x888", 2: "x999"},
Topics: core.Topics{0: "x777", 1: "x888", 2: "x999"},
Data: "xabc",
}},
)
@ -497,6 +485,85 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories.
{blockNumber: 1, Index: 1}},
))
})
It("saves the logs attached to a receipt", func() {
logs := []core.Log{{
Address: "0x8a4774fe82c63484afef97ca8d89a6ea5e21f973",
BlockNumber: 4745407,
Data: "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000645a68669900000000000000000000000000000000000000000000003397684ab5869b0000000000000000000000000000000000000000000000000000000000005a36053200000000000000000000000099041f808d598b782d5a3e498681c2452a31da08",
Index: 86,
Topics: core.Topics{
0: "0x5a68669900000000000000000000000000000000000000000000000000000000",
1: "0x000000000000000000000000d0148dad63f73ce6f1b6c607e3413dcf1ff5f030",
2: "0x00000000000000000000000000000000000000000000003397684ab5869b0000",
3: "0x000000000000000000000000000000000000000000000000000000005a360532",
},
TxHash: "0x002c4799161d809b23f67884eb6598c9df5894929fe1a9ead97ca175d360f547",
}, {
Address: "0x99041f808d598b782d5a3e498681c2452a31da08",
BlockNumber: 4745407,
Data: "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000000418178358",
Index: 87,
Topics: core.Topics{
0: "0x1817835800000000000000000000000000000000000000000000000000000000",
1: "0x0000000000000000000000008a4774fe82c63484afef97ca8d89a6ea5e21f973",
2: "0x0000000000000000000000000000000000000000000000000000000000000000",
3: "0x0000000000000000000000000000000000000000000000000000000000000000",
},
TxHash: "0x002c4799161d809b23f67884eb6598c9df5894929fe1a9ead97ca175d360f547",
}, {
Address: "0x99041f808d598b782d5a3e498681c2452a31da08",
BlockNumber: 4745407,
Data: "0x00000000000000000000000000000000000000000000003338f64c8423af4000",
Index: 88,
Topics: core.Topics{
0: "0x296ba4ca62c6c21c95e828080cb8aec7481b71390585605300a8a76f9e95b527",
},
TxHash: "0x002c4799161d809b23f67884eb6598c9df5894929fe1a9ead97ca175d360f547",
},
}
receipt := core.Receipt{
ContractAddress: "",
CumulativeGasUsed: 7481414,
GasUsed: 60711,
Logs: logs,
Bloom: "0x00000800000000000000001000000000000000400000000080000000000000000000400000010000000000000000000000000000040000000000000002000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000000000800004000000000000001000000000000000000000000000002000000480000000000000002000000000000000020000000000000000000000000000000000000000080000000000180000c00000000000000002000002000000040000000000000000000000000000010000000000020000000000000000000002000000000000000000000000400800000000000000000",
Status: 1,
TxHash: "0x002c4799161d809b23f67884eb6598c9df5894929fe1a9ead97ca175d360f547",
}
transaction :=
core.Transaction{
Hash: receipt.TxHash,
Receipt: receipt,
}
block := core.Block{Transactions: []core.Transaction{transaction}}
err := repository.CreateOrUpdateBlock(block)
Expect(err).To(Not(HaveOccurred()))
retrievedLogs := repository.FindLogs("0x99041f808d598b782d5a3e498681c2452a31da08", 4745407)
expected := logs[1:]
Expect(retrievedLogs).To(Equal(expected))
})
It("still saves receipts without logs", func() {
receipt := core.Receipt{
TxHash: "0x002c4799161d809b23f67884eb6598c9df5894929fe1a9ead97ca175d360f547",
}
transaction := core.Transaction{
Hash: receipt.TxHash,
Receipt: receipt,
}
block := core.Block{
Transactions: []core.Transaction{transaction},
}
repository.CreateOrUpdateBlock(block)
_, err := repository.FindReceipt(receipt.TxHash)
Expect(err).To(Not(HaveOccurred()))
})
})
Describe("Saving receipts", func() {
@ -535,6 +602,43 @@ func AssertRepositoryBehavior(buildRepository func(node core.Node) repositories.
Expect(err).To(HaveOccurred())
Expect(receipt).To(BeZero())
})
})
Describe("LogFilter", func() {
It("inserts filter into watched events", func() {
logFilter := filters.LogFilter{
Name: "TestFilter",
FromBlock: 1,
ToBlock: 2,
Address: "0x8888f1f195afa192cfee860698584c030f4c9db1",
Topics: core.Topics{
"0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b",
"",
"0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b",
"",
},
}
err := repository.AddFilter(logFilter)
Expect(err).ToNot(HaveOccurred())
})
It("returns error if name is not provided", func() {
logFilter := filters.LogFilter{
FromBlock: 1,
ToBlock: 2,
Address: "0x8888f1f195afa192cfee860698584c030f4c9db1",
Topics: core.Topics{
"0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b",
"",
"0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b",
"",
},
}
err := repository.AddFilter(logFilter)
Expect(err).To(HaveOccurred())
})
})
}

View File

@ -1,14 +1,14 @@
package cmd
package utils
import (
"log"
"path/filepath"
"fmt"
"math/big"
"os"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/geth"
@ -32,9 +32,7 @@ func LoadPostgres(database config.Database, node core.Node) repositories.Postgre
}
func ReadAbiFile(abiFilepath string) string {
if !filepath.IsAbs(abiFilepath) {
abiFilepath = filepath.Join(config.ProjectRoot(), abiFilepath)
}
abiFilepath = AbsFilePath(abiFilepath)
abi, err := geth.ReadAbiFile(abiFilepath)
if err != nil {
log.Fatalf("Error reading ABI file at \"%s\"\n %v", abiFilepath, err)
@ -42,13 +40,22 @@ func ReadAbiFile(abiFilepath string) string {
return abi
}
func GetAbi(abiFilepath string, contractHash string) string {
func AbsFilePath(filePath string) string {
if !filepath.IsAbs(filePath) {
cwd, _ := os.Getwd()
filePath = filepath.Join(cwd, filePath)
}
return filePath
}
func GetAbi(abiFilepath string, contractHash string, network string) string {
var contractAbiString string
if abiFilepath != "" {
contractAbiString = ReadAbiFile(abiFilepath)
} else {
etherscan := geth.NewEtherScanClient("https://api.etherscan.io")
fmt.Println("No ABI supplied. Retrieving ABI from Etherscan")
url := geth.GenUrl(network)
etherscan := geth.NewEtherScanClient(url)
log.Printf("No ABI supplied. Retrieving ABI from Etherscan: %s", url)
contractAbiString, _ = etherscan.GetAbi(contractHash)
}
_, err := geth.ParseAbi(contractAbiString)