From 18163f970eff5dd547dde5e56c6e50d646c93a67 Mon Sep 17 00:00:00 2001 From: Matt K <1036969+mkrump@users.noreply.github.com> Date: Thu, 7 Dec 2017 09:58:06 -0600 Subject: [PATCH] Get ABI via etherscan API (#96) - Added ABI request - Add unique constraint on contract hash for watched contracts --- .travis.yml | 2 + Gododir/main.go | 11 +- Gopkg.lock | 38 ++---- cmd/subscribe_contract/main.go | 4 +- cmd/utils.go | 18 +++ ...h_constraint_to_watched_contracts.down.sql | 2 + ...ash_constraint_to_watched_contracts.up.sql | 2 + db/schema.sql | 12 +- pkg/geth/abi.go | 42 +++++- pkg/geth/abi_test.go | 122 +++++++++++++----- pkg/geth/contract.go | 17 --- pkg/repositories/postgres.go | 37 +++--- pkg/repositories/testing/helpers.go | 14 ++ 13 files changed, 213 insertions(+), 108 deletions(-) create mode 100644 db/migrations/1512595007_add_contract_hash_constraint_to_watched_contracts.down.sql create mode 100644 db/migrations/1512595007_add_contract_hash_constraint_to_watched_contracts.up.sql diff --git a/.travis.yml b/.travis.yml index cff6868e..08118e55 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,8 @@ go: - 1.9 services: - postgresql +addons: + postgresql: "9.6" before_script: - wget https://gethstore.blob.core.windows.net/builds/geth-linux-amd64-1.7.2-1db4ecdc.tar.gz - tar -xzf geth-linux-amd64-1.7.2-1db4ecdc.tar.gz diff --git a/Gododir/main.go b/Gododir/main.go index 7d3ab6f8..29277924 100644 --- a/Gododir/main.go +++ b/Gododir/main.go @@ -43,9 +43,6 @@ func tasks(p *do.Project) { if contractHash == "" { log.Fatalln("--contract-hash required") } - if abiFilepath == "" { - log.Fatalln("--abi-filepath required") - } context.Start(`go run main.go --environment={{.environment}} --contract-hash={{.contractHash}} --abi-filepath={{.abiFilepath}}`, do.M{ "environment": environment, @@ -78,11 +75,15 @@ func tasks(p *do.Project) { p.Task("showContractSummary", nil, func(context *do.Context) { environment := parseEnvironment(context) contractHash := context.Args.MayString("", "contract-hash", "c") + blockNumber := context.Args.MayInt(-1, "block-number", "b") if contractHash == "" { log.Fatalln("--contract-hash required") } - context.Start(`go run main.go --environment={{.environment}} --contract-hash={{.contractHash}}`, - do.M{"environment": environment, "contractHash": contractHash, "$in": "cmd/show_contract_summary"}) + context.Start(`go run main.go --environment={{.environment}} --contract-hash={{.contractHash}} --block-number={{.blockNumber}}`, + do.M{"environment": environment, + "contractHash": contractHash, + "blockNumber": blockNumber, + "$in": "cmd/show_contract_summary"}) }) } diff --git a/Gopkg.lock b/Gopkg.lock index ed3ad2cc..f997d50f 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -13,12 +13,6 @@ packages = ["."] revision = "4748e29d5718c2df4028a6543edf86fd8cc0f881" -[[projects]] - branch = "master" - name = "github.com/aristanetworks/goarista" - packages = ["monotime"] - revision = "54fadd0c513d502544edf098480238dc9da50f9e" - [[projects]] branch = "master" name = "github.com/btcsuite/btcd" @@ -27,7 +21,7 @@ [[projects]] name = "github.com/ethereum/go-ethereum" - packages = [".","accounts","accounts/abi","accounts/abi/bind","accounts/keystore","common","common/hexutil","common/math","common/mclock","core/types","crypto","crypto/randentropy","crypto/secp256k1","crypto/sha3","ethclient","event","log","params","rlp","rpc","trie"] + packages = [".","accounts/abi","common","common/hexutil","common/math","core/types","crypto","crypto/secp256k1","crypto/sha3","ethclient","log","params","rlp","rpc","trie"] revision = "1db4ecdc0b9e828ff65777fb466fc7c1d04e0de9" version = "v1.7.2" @@ -37,6 +31,12 @@ revision = "817915b46b97fd7bb80e8ab6b69f01a53ac3eebf" version = "v1.6.0" +[[projects]] + branch = "master" + name = "github.com/golang/protobuf" + packages = ["proto"] + revision = "1e59b77b52bf8e4b449a57e6f79f21226d571845" + [[projects]] branch = "master" name = "github.com/howeyc/gopass" @@ -105,28 +105,16 @@ [[projects]] name = "github.com/onsi/gomega" - packages = [".","format","internal/assertion","internal/asyncassertion","internal/oraclematcher","internal/testingtsupport","matchers","matchers/support/goraph/bipartitegraph","matchers/support/goraph/edge","matchers/support/goraph/node","matchers/support/goraph/util","types"] + packages = [".","format","ghttp","internal/assertion","internal/asyncassertion","internal/oraclematcher","internal/testingtsupport","matchers","matchers/support/goraph/bipartitegraph","matchers/support/goraph/edge","matchers/support/goraph/node","matchers/support/goraph/util","types"] revision = "c893efa28eb45626cdaa76c9f653b62488858837" version = "v1.2.0" -[[projects]] - name = "github.com/pborman/uuid" - packages = ["."] - revision = "e790cca94e6cc75c7064b1332e63811d4aae1a53" - version = "v1.1" - [[projects]] branch = "master" name = "github.com/rcrowley/go-metrics" packages = ["."] revision = "1f30fe9094a513ce4c700b9a54458bbb0c96996c" -[[projects]] - branch = "master" - name = "github.com/rjeczalik/notify" - packages = ["."] - revision = "767eb674ef14b09119b2fff3601e64558d530c47" - [[projects]] name = "github.com/rs/cors" packages = ["."] @@ -136,7 +124,7 @@ [[projects]] branch = "master" name = "golang.org/x/crypto" - packages = ["pbkdf2","scrypt","ssh/terminal"] + packages = ["ssh/terminal"] revision = "bd6f299fb381e4c3393d1c4b1f0b94f5e77650c8" [[projects]] @@ -157,12 +145,6 @@ packages = ["encoding","encoding/charmap","encoding/htmlindex","encoding/internal","encoding/internal/identifier","encoding/japanese","encoding/korean","encoding/simplifiedchinese","encoding/traditionalchinese","encoding/unicode","internal/gen","internal/tag","internal/utf8internal","language","runes","transform","unicode/cldr"] revision = "c01e4764d870b77f8abe5096ee19ad20d80e8075" -[[projects]] - branch = "master" - name = "golang.org/x/tools" - packages = ["go/ast/astutil","imports"] - revision = "6d70fb2e85323e81c89374331d3d2b93304faa36" - [[projects]] name = "gopkg.in/fatih/set.v0" packages = ["."] @@ -196,6 +178,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "d2aa2bdc1442319cde6fe38b39e0ff8e25b1a0b0c120a7e2fab8065324c98693" + inputs-digest = "90af18ee127c0b2099f47adc5d33fb6ce9b98630278ec66648ef3e1c5ad9063f" solver-name = "gps-cdcl" solver-version = 1 diff --git a/cmd/subscribe_contract/main.go b/cmd/subscribe_contract/main.go index 3777afd3..5b74e58d 100644 --- a/cmd/subscribe_contract/main.go +++ b/cmd/subscribe_contract/main.go @@ -12,10 +12,12 @@ func main() { contractHash := flag.String("contract-hash", "", "contract-hash=x1234") abiFilepath := flag.String("abi-filepath", "", "path/to/abifile.json") flag.Parse() + + contractAbiString := cmd.GetAbi(*abiFilepath, *contractHash) config := cmd.LoadConfig(*environment) repository := cmd.LoadPostgres(config.Database) watchedContract := core.Contract{ - Abi: cmd.ReadAbiFile(*abiFilepath), + Abi: contractAbiString, Hash: *contractHash, } repository.CreateContract(watchedContract) diff --git a/cmd/utils.go b/cmd/utils.go index 3dff2da8..fc388ade 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -5,6 +5,8 @@ import ( "path/filepath" + "fmt" + "github.com/8thlight/vulcanizedb/pkg/config" "github.com/8thlight/vulcanizedb/pkg/geth" "github.com/8thlight/vulcanizedb/pkg/repositories" @@ -36,3 +38,19 @@ func ReadAbiFile(abiFilepath string) string { } return abi } + +func GetAbi(abiFilepath string, contractHash string) string { + var contractAbiString string + if abiFilepath != "" { + contractAbiString = ReadAbiFile(abiFilepath) + } else { + etherscan := geth.NewEtherScanClient("https://api.etherscan.io") + fmt.Println("No ABI supplied. Retrieving ABI from Etherscan") + contractAbiString, _ = etherscan.GetAbi(contractHash) + } + _, err := geth.ParseAbi(contractAbiString) + if err != nil { + log.Fatalln("Invalid ABI") + } + return contractAbiString +} diff --git a/db/migrations/1512595007_add_contract_hash_constraint_to_watched_contracts.down.sql b/db/migrations/1512595007_add_contract_hash_constraint_to_watched_contracts.down.sql new file mode 100644 index 00000000..70672822 --- /dev/null +++ b/db/migrations/1512595007_add_contract_hash_constraint_to_watched_contracts.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE watched_contracts + DROP CONSTRAINT contract_hash_uc; \ No newline at end of file diff --git a/db/migrations/1512595007_add_contract_hash_constraint_to_watched_contracts.up.sql b/db/migrations/1512595007_add_contract_hash_constraint_to_watched_contracts.up.sql new file mode 100644 index 00000000..f27a54c0 --- /dev/null +++ b/db/migrations/1512595007_add_contract_hash_constraint_to_watched_contracts.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE watched_contracts + ADD CONSTRAINT contract_hash_uc UNIQUE (contract_hash); diff --git a/db/schema.sql b/db/schema.sql index c788d169..e90957a0 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -2,8 +2,8 @@ -- PostgreSQL database dump -- --- Dumped from database version 10.0 --- Dumped by pg_dump version 10.0 +-- Dumped from database version 10.1 +-- Dumped by pg_dump version 10.1 SET statement_timeout = 0; SET lock_timeout = 0; @@ -177,6 +177,14 @@ ALTER TABLE ONLY blocks ADD CONSTRAINT blocks_pkey PRIMARY KEY (id); +-- +-- Name: watched_contracts contract_hash_uc; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY watched_contracts + ADD CONSTRAINT contract_hash_uc UNIQUE (contract_hash); + + -- -- Name: schema_migrations schema_migrations_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- diff --git a/pkg/geth/abi.go b/pkg/geth/abi.go index 705f715e..e84523fb 100644 --- a/pkg/geth/abi.go +++ b/pkg/geth/abi.go @@ -5,14 +5,52 @@ import ( "io/ioutil" "strings" + "encoding/json" + "fmt" + "net/http" + "time" + "github.com/ethereum/go-ethereum/accounts/abi" ) var ( - ErrInvalidAbiFile = errors.New("invalid abi") - ErrMissingAbiFile = errors.New("missing abi") + ErrInvalidAbiFile = errors.New("invalid abi") + ErrMissingAbiFile = errors.New("missing abi") + ErrApiRequestFailed = errors.New("etherscan api request failed") ) +type Response struct { + Status string + Message string + Result string +} + +type EtherScanApi struct { + client *http.Client + url string +} + +func NewEtherScanClient(url string) *EtherScanApi { + return &EtherScanApi{ + client: &http.Client{Timeout: 10 * time.Second}, + url: url, + } + +} + +//https://api.etherscan.io/api?module=contract&action=getabi&address=%s +func (e *EtherScanApi) GetAbi(contractHash string) (string, error) { + target := new(Response) + request := fmt.Sprintf("%s/api?module=contract&action=getabi&address=%s", e.url, contractHash) + r, err := e.client.Get(request) + if err != nil { + return "", ErrApiRequestFailed + } + defer r.Body.Close() + json.NewDecoder(r.Body).Decode(&target) + return target.Result, nil +} + func ParseAbiFile(abiFilePath string) (abi.ABI, error) { abiString, err := ReadAbiFile(abiFilePath) if err != nil { diff --git a/pkg/geth/abi_test.go b/pkg/geth/abi_test.go index d638a212..6a89faa9 100644 --- a/pkg/geth/abi_test.go +++ b/pkg/geth/abi_test.go @@ -3,49 +3,103 @@ package geth_test import ( "path/filepath" + "net/http" + + "fmt" + + "log" + cfg "github.com/8thlight/vulcanizedb/pkg/config" "github.com/8thlight/vulcanizedb/pkg/geth" "github.com/ethereum/go-ethereum/accounts/abi" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/onsi/gomega/ghttp" ) -var _ = Describe("Reading ABI files", func() { +var _ = Describe("ABI files", func() { - It("loads a valid ABI file", func() { - path := filepath.Join(cfg.ProjectRoot(), "pkg", "geth", "testing", "valid_abi.json") + Describe("Reading ABI files", func() { - contractAbi, err := geth.ParseAbiFile(path) + It("loads a valid ABI file", func() { + path := filepath.Join(cfg.ProjectRoot(), "pkg", "geth", "testing", "valid_abi.json") - Expect(contractAbi).NotTo(BeNil()) - Expect(err).To(BeNil()) + contractAbi, err := geth.ParseAbiFile(path) + + Expect(contractAbi).NotTo(BeNil()) + Expect(err).To(BeNil()) + }) + + It("reads the contents of a valid ABI file", func() { + path := filepath.Join(cfg.ProjectRoot(), "pkg", "geth", "testing", "valid_abi.json") + + contractAbi, err := geth.ReadAbiFile(path) + + Expect(contractAbi).To(Equal("[{\"foo\": \"bar\"}]")) + Expect(err).To(BeNil()) + }) + + It("returns an error when the file does not exist", func() { + path := filepath.Join(cfg.ProjectRoot(), "pkg", "geth", "testing", "missing_abi.json") + + contractAbi, err := geth.ParseAbiFile(path) + + Expect(contractAbi).To(Equal(abi.ABI{})) + Expect(err).To(Equal(geth.ErrMissingAbiFile)) + }) + + It("returns an error when the file has invalid contents", func() { + path := filepath.Join(cfg.ProjectRoot(), "pkg", "geth", "testing", "invalid_abi.json") + + contractAbi, err := geth.ParseAbiFile(path) + + Expect(contractAbi).To(Equal(abi.ABI{})) + Expect(err).To(Equal(geth.ErrInvalidAbiFile)) + }) + + Describe("Request ABI from endpoint", func() { + + var ( + server *ghttp.Server + client *geth.EtherScanApi + abiString string + ) + + BeforeEach(func() { + server = ghttp.NewServer() + client = geth.NewEtherScanClient(server.URL()) + path := filepath.Join(cfg.ProjectRoot(), "pkg", "geth", "testing", "sample_abi.json") + abiString, err := geth.ReadAbiFile(path) + _, err = geth.ParseAbi(abiString) + if err != nil { + log.Fatalln("Could not parse ABI") + } + }) + + AfterEach(func() { + server.Close() + }) + + Describe("Fetching ABI from api (etherscan)", func() { + BeforeEach(func() { + + response := fmt.Sprintf(`{"status":"1","message":"OK","result":%q}`, abiString) + server.AppendHandlers( + ghttp.CombineHandlers( + ghttp.VerifyRequest("GET", "/api", "module=contract&action=getabi&address=0xd26114cd6EE289AccF82350c8d8487fedB8A0C07"), + ghttp.RespondWith(http.StatusOK, response), + ), + ) + }) + + It("should make a GET request with supplied contract hash", func() { + + abi, err := client.GetAbi("0xd26114cd6EE289AccF82350c8d8487fedB8A0C07") + Expect(server.ReceivedRequests()).Should(HaveLen(1)) + Expect(err).ShouldNot(HaveOccurred()) + Expect(abi).Should(Equal(abiString)) + }) + }) + }) }) - - It("reads the contents of a valid ABI file", func() { - path := filepath.Join(cfg.ProjectRoot(), "pkg", "geth", "testing", "valid_abi.json") - - contractAbi, err := geth.ReadAbiFile(path) - - Expect(contractAbi).To(Equal("[{\"foo\": \"bar\"}]")) - Expect(err).To(BeNil()) - }) - - It("returns an error when the file does not exist", func() { - path := filepath.Join(cfg.ProjectRoot(), "pkg", "geth", "testing", "missing_abi.json") - - contractAbi, err := geth.ParseAbiFile(path) - - Expect(contractAbi).To(Equal(abi.ABI{})) - Expect(err).To(Equal(geth.ErrMissingAbiFile)) - }) - - It("returns an error when the file has invalid contents", func() { - path := filepath.Join(cfg.ProjectRoot(), "pkg", "geth", "testing", "invalid_abi.json") - - contractAbi, err := geth.ParseAbiFile(path) - - Expect(contractAbi).To(Equal(abi.ABI{})) - Expect(err).To(Equal(geth.ErrInvalidAbiFile)) - }) - }) diff --git a/pkg/geth/contract.go b/pkg/geth/contract.go index b9857297..d68541cb 100644 --- a/pkg/geth/contract.go +++ b/pkg/geth/contract.go @@ -2,15 +2,12 @@ package geth import ( "errors" - "fmt" - "path/filepath" "sort" "context" "math/big" - "github.com/8thlight/vulcanizedb/pkg/config" "github.com/8thlight/vulcanizedb/pkg/core" "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" @@ -59,17 +56,3 @@ func (blockchain *GethBlockchain) GetAttributes(contract core.Contract) (core.Co sort.Sort(contractAttributes) return contractAttributes, nil } - -func (blockchain *GethBlockchain) GetContractAttributesOld(contractHash string) (core.ContractAttributes, error) { - abiFilePath := filepath.Join(config.ProjectRoot(), "contracts", "public", fmt.Sprintf("%s.json", contractHash)) - parsed, _ := ParseAbiFile(abiFilePath) - var contractAttributes core.ContractAttributes - for _, abiElement := range parsed.Methods { - if (len(abiElement.Outputs) > 0) && (len(abiElement.Inputs) == 0) && abiElement.Const { - attributeType := abiElement.Outputs[0].Type.String() - contractAttributes = append(contractAttributes, core.ContractAttribute{abiElement.Name, attributeType}) - } - } - sort.Sort(contractAttributes) - return contractAttributes, nil -} diff --git a/pkg/repositories/postgres.go b/pkg/repositories/postgres.go index 946323d2..81a1cd5e 100644 --- a/pkg/repositories/postgres.go +++ b/pkg/repositories/postgres.go @@ -38,7 +38,12 @@ func (repository Postgres) CreateContract(contract core.Contract) error { abiToInsert = &abi } _, err := repository.Db.Exec( - `INSERT INTO watched_contracts (contract_hash, contract_abi) VALUES ($1, $2)`, contract.Hash, abiToInsert) + `INSERT INTO watched_contracts (contract_hash, contract_abi) + VALUES ($1, $2) + ON CONFLICT (contract_hash) + DO UPDATE + SET contract_hash = $1, contract_abi = $2 + `, contract.Hash, abiToInsert) if err != nil { return ErrDBInsertFailed } @@ -53,15 +58,16 @@ func (repository Postgres) ContractExists(contractHash string) bool { } func (repository Postgres) FindContract(contractHash string) *core.Contract { - var savedContracts []core.Contract - contractRows, _ := repository.Db.Query( + var hash string + var abi string + row := repository.Db.QueryRow( `SELECT contract_hash, contract_abi FROM watched_contracts WHERE contract_hash=$1`, contractHash) - savedContracts = repository.loadContract(contractRows) - if len(savedContracts) > 0 { - return &savedContracts[0] - } else { + err := row.Scan(&hash, &abi) + if err == sql.ErrNoRows { return nil } + savedContract := repository.addTransactions(core.Contract{Hash: hash, Abi: abi}) + return &savedContract } func (repository Postgres) MaxBlockNumber() int64 { @@ -197,16 +203,9 @@ func (repository Postgres) loadTransactions(transactionRows *sql.Rows) []core.Tr return transactions } -func (repository Postgres) loadContract(contractRows *sql.Rows) []core.Contract { - var savedContracts []core.Contract - for contractRows.Next() { - var savedContractHash string - var savedContractAbi string - contractRows.Scan(&savedContractHash, &savedContractAbi) - transactionRows, _ := repository.Db.Query(`SELECT tx_hash, tx_nonce, tx_to, tx_from, tx_gaslimit, tx_gasprice, tx_value FROM transactions WHERE tx_to = $1 ORDER BY block_id desc`, savedContractHash) - transactions := repository.loadTransactions(transactionRows) - savedContract := core.Contract{Hash: savedContractHash, Transactions: transactions, Abi: savedContractAbi} - savedContracts = append(savedContracts, savedContract) - } - return savedContracts +func (repository Postgres) addTransactions(contract core.Contract) core.Contract { + transactionRows, _ := repository.Db.Query(`SELECT tx_hash, tx_nonce, tx_to, tx_from, tx_gaslimit, tx_gasprice, tx_value FROM transactions WHERE tx_to = $1 ORDER BY block_id desc`, contract.Hash) + transactions := repository.loadTransactions(transactionRows) + savedContract := core.Contract{Hash: contract.Hash, Transactions: transactions, Abi: contract.Abi} + return savedContract } diff --git a/pkg/repositories/testing/helpers.go b/pkg/repositories/testing/helpers.go index f7684e8e..7d0b7fb2 100644 --- a/pkg/repositories/testing/helpers.go +++ b/pkg/repositories/testing/helpers.go @@ -262,6 +262,20 @@ func AssertRepositoryBehavior(buildRepository func() repositories.Repository) { Expect(contract).ToNot(BeNil()) Expect(contract.Abi).To(Equal("{\"some\": \"json\"}")) }) + + It("updates the ABI of the contract if hash already present", func() { + repository.CreateContract(core.Contract{ + Abi: "{\"some\": \"json\"}", + Hash: "x123", + }) + repository.CreateContract(core.Contract{ + Abi: "{\"some\": \"different json\"}", + Hash: "x123", + }) + contract := repository.FindContract("x123") + Expect(contract).ToNot(BeNil()) + Expect(contract.Abi).To(Equal("{\"some\": \"different json\"}")) + }) }) }