Get ABI via etherscan API (#96)

- Added ABI request
- Add unique constraint on contract hash for watched contracts

.travis.yml

@@ -4,6 +4,8 @@ go:
   - 1.9
 services:
   - postgresql
+addons:
+  postgresql: "9.6"
 before_script:
   - wget https://gethstore.blob.core.windows.net/builds/geth-linux-amd64-1.7.2-1db4ecdc.tar.gz
   - tar -xzf geth-linux-amd64-1.7.2-1db4ecdc.tar.gz

Gododir/main.go

@@ -43,9 +43,6 @@ func tasks(p *do.Project) {
 		if contractHash == "" {
 			log.Fatalln("--contract-hash required")
 		}
-		if abiFilepath == "" {
-			log.Fatalln("--abi-filepath required")
-		}
 		context.Start(`go run main.go --environment={{.environment}} --contract-hash={{.contractHash}} --abi-filepath={{.abiFilepath}}`,
 			do.M{
 				"environment":  environment,
@@ -78,11 +75,15 @@ func tasks(p *do.Project) {
 	p.Task("showContractSummary", nil, func(context *do.Context) {
 		environment := parseEnvironment(context)
 		contractHash := context.Args.MayString("", "contract-hash", "c")
+		blockNumber := context.Args.MayInt(-1, "block-number", "b")
 		if contractHash == "" {
 			log.Fatalln("--contract-hash required")
 		}
-		context.Start(`go run main.go --environment={{.environment}} --contract-hash={{.contractHash}}`,
+		context.Start(`go run main.go --environment={{.environment}} --contract-hash={{.contractHash}} --block-number={{.blockNumber}}`,
-			do.M{"environment": environment, "contractHash": contractHash, "$in": "cmd/show_contract_summary"})
+			do.M{"environment": environment,
+				"contractHash": contractHash,
+				"blockNumber":  blockNumber,
+				"$in":          "cmd/show_contract_summary"})
 	})
 
 }

Gopkg.lock

@@ -13,12 +13,6 @@
   packages = ["."]
   revision = "4748e29d5718c2df4028a6543edf86fd8cc0f881"
 
-[[projects]]
-  branch = "master"
-  name = "github.com/aristanetworks/goarista"
-  packages = ["monotime"]
-  revision = "54fadd0c513d502544edf098480238dc9da50f9e"
-
 [[projects]]
   branch = "master"
   name = "github.com/btcsuite/btcd"
@@ -27,7 +21,7 @@
 
 [[projects]]
   name = "github.com/ethereum/go-ethereum"
-  packages = [".","accounts","accounts/abi","accounts/abi/bind","accounts/keystore","common","common/hexutil","common/math","common/mclock","core/types","crypto","crypto/randentropy","crypto/secp256k1","crypto/sha3","ethclient","event","log","params","rlp","rpc","trie"]
+  packages = [".","accounts/abi","common","common/hexutil","common/math","core/types","crypto","crypto/secp256k1","crypto/sha3","ethclient","log","params","rlp","rpc","trie"]
   revision = "1db4ecdc0b9e828ff65777fb466fc7c1d04e0de9"
   version = "v1.7.2"
 
@@ -37,6 +31,12 @@
   revision = "817915b46b97fd7bb80e8ab6b69f01a53ac3eebf"
   version = "v1.6.0"
 
+[[projects]]
+  branch = "master"
+  name = "github.com/golang/protobuf"
+  packages = ["proto"]
+  revision = "1e59b77b52bf8e4b449a57e6f79f21226d571845"
+
 [[projects]]
   branch = "master"
   name = "github.com/howeyc/gopass"
@@ -105,28 +105,16 @@
 
 [[projects]]
   name = "github.com/onsi/gomega"
-  packages = [".","format","internal/assertion","internal/asyncassertion","internal/oraclematcher","internal/testingtsupport","matchers","matchers/support/goraph/bipartitegraph","matchers/support/goraph/edge","matchers/support/goraph/node","matchers/support/goraph/util","types"]
+  packages = [".","format","ghttp","internal/assertion","internal/asyncassertion","internal/oraclematcher","internal/testingtsupport","matchers","matchers/support/goraph/bipartitegraph","matchers/support/goraph/edge","matchers/support/goraph/node","matchers/support/goraph/util","types"]
   revision = "c893efa28eb45626cdaa76c9f653b62488858837"
   version = "v1.2.0"
 
-[[projects]]
-  name = "github.com/pborman/uuid"
-  packages = ["."]
-  revision = "e790cca94e6cc75c7064b1332e63811d4aae1a53"
-  version = "v1.1"
-
 [[projects]]
   branch = "master"
   name = "github.com/rcrowley/go-metrics"
   packages = ["."]
   revision = "1f30fe9094a513ce4c700b9a54458bbb0c96996c"
 
-[[projects]]
-  branch = "master"
-  name = "github.com/rjeczalik/notify"
-  packages = ["."]
-  revision = "767eb674ef14b09119b2fff3601e64558d530c47"
-
 [[projects]]
   name = "github.com/rs/cors"
   packages = ["."]
@@ -136,7 +124,7 @@
 [[projects]]
   branch = "master"
   name = "golang.org/x/crypto"
-  packages = ["pbkdf2","scrypt","ssh/terminal"]
+  packages = ["ssh/terminal"]
   revision = "bd6f299fb381e4c3393d1c4b1f0b94f5e77650c8"
 
 [[projects]]
@@ -157,12 +145,6 @@
   packages = ["encoding","encoding/charmap","encoding/htmlindex","encoding/internal","encoding/internal/identifier","encoding/japanese","encoding/korean","encoding/simplifiedchinese","encoding/traditionalchinese","encoding/unicode","internal/gen","internal/tag","internal/utf8internal","language","runes","transform","unicode/cldr"]
   revision = "c01e4764d870b77f8abe5096ee19ad20d80e8075"
 
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/tools"
-  packages = ["go/ast/astutil","imports"]
-  revision = "6d70fb2e85323e81c89374331d3d2b93304faa36"
-
 [[projects]]
   name = "gopkg.in/fatih/set.v0"
   packages = ["."]
@@ -196,6 +178,6 @@
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "d2aa2bdc1442319cde6fe38b39e0ff8e25b1a0b0c120a7e2fab8065324c98693"
+  inputs-digest = "90af18ee127c0b2099f47adc5d33fb6ce9b98630278ec66648ef3e1c5ad9063f"
   solver-name = "gps-cdcl"
   solver-version = 1

cmd/subscribe_contract/main.go

@@ -12,10 +12,12 @@ func main() {
 	contractHash := flag.String("contract-hash", "", "contract-hash=x1234")
 	abiFilepath := flag.String("abi-filepath", "", "path/to/abifile.json")
 	flag.Parse()
+
+	contractAbiString := cmd.GetAbi(*abiFilepath, *contractHash)
 	config := cmd.LoadConfig(*environment)
 	repository := cmd.LoadPostgres(config.Database)
 	watchedContract := core.Contract{
-		Abi:  cmd.ReadAbiFile(*abiFilepath),
+		Abi:  contractAbiString,
 		Hash: *contractHash,
 	}
 	repository.CreateContract(watchedContract)

cmd/utils.go

@@ -5,6 +5,8 @@ import (
 
 	"path/filepath"
 
+	"fmt"
+
 	"github.com/8thlight/vulcanizedb/pkg/config"
 	"github.com/8thlight/vulcanizedb/pkg/geth"
 	"github.com/8thlight/vulcanizedb/pkg/repositories"
@@ -36,3 +38,19 @@ func ReadAbiFile(abiFilepath string) string {
 	}
 	return abi
 }
+
+func GetAbi(abiFilepath string, contractHash string) string {
+	var contractAbiString string
+	if abiFilepath != "" {
+		contractAbiString = ReadAbiFile(abiFilepath)
+	} else {
+		etherscan := geth.NewEtherScanClient("https://api.etherscan.io")
+		fmt.Println("No ABI supplied. Retrieving ABI from Etherscan")
+		contractAbiString, _ = etherscan.GetAbi(contractHash)
+	}
+	_, err := geth.ParseAbi(contractAbiString)
+	if err != nil {
+		log.Fatalln("Invalid ABI")
+	}
+	return contractAbiString
+}

db/migrations/1512595007_add_contract_hash_constraint_to_watched_contracts.down.sql

@@ -0,0 +1,2 @@
+ALTER TABLE watched_contracts
+  DROP CONSTRAINT contract_hash_uc;

db/migrations/1512595007_add_contract_hash_constraint_to_watched_contracts.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE watched_contracts
+  ADD CONSTRAINT contract_hash_uc UNIQUE (contract_hash);

db/schema.sql

@@ -2,8 +2,8 @@
 -- PostgreSQL database dump
 --
 
--- Dumped from database version 10.0
+-- Dumped from database version 10.1
--- Dumped by pg_dump version 10.0
+-- Dumped by pg_dump version 10.1
 
 SET statement_timeout = 0;
 SET lock_timeout = 0;
@@ -177,6 +177,14 @@ ALTER TABLE ONLY blocks
     ADD CONSTRAINT blocks_pkey PRIMARY KEY (id);
 
 
+--
+-- Name: watched_contracts contract_hash_uc; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY watched_contracts
+    ADD CONSTRAINT contract_hash_uc UNIQUE (contract_hash);
+
+
 --
 -- Name: schema_migrations schema_migrations_pkey; Type: CONSTRAINT; Schema: public; Owner: -
 --

pkg/geth/abi.go

@@ -5,14 +5,52 @@ import (
 	"io/ioutil"
 	"strings"
 
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"time"
+
 	"github.com/ethereum/go-ethereum/accounts/abi"
 )
 
 var (
-	ErrInvalidAbiFile = errors.New("invalid abi")
+	ErrInvalidAbiFile   = errors.New("invalid abi")
-	ErrMissingAbiFile = errors.New("missing abi")
+	ErrMissingAbiFile   = errors.New("missing abi")
+	ErrApiRequestFailed = errors.New("etherscan api request failed")
 )
 
+type Response struct {
+	Status  string
+	Message string
+	Result  string
+}
+
+type EtherScanApi struct {
+	client *http.Client
+	url    string
+}
+
+func NewEtherScanClient(url string) *EtherScanApi {
+	return &EtherScanApi{
+		client: &http.Client{Timeout: 10 * time.Second},
+		url:    url,
+	}
+
+}
+
+//https://api.etherscan.io/api?module=contract&action=getabi&address=%s
+func (e *EtherScanApi) GetAbi(contractHash string) (string, error) {
+	target := new(Response)
+	request := fmt.Sprintf("%s/api?module=contract&action=getabi&address=%s", e.url, contractHash)
+	r, err := e.client.Get(request)
+	if err != nil {
+		return "", ErrApiRequestFailed
+	}
+	defer r.Body.Close()
+	json.NewDecoder(r.Body).Decode(&target)
+	return target.Result, nil
+}
+
 func ParseAbiFile(abiFilePath string) (abi.ABI, error) {
 	abiString, err := ReadAbiFile(abiFilePath)
 	if err != nil {

pkg/geth/abi_test.go

@@ -3,49 +3,103 @@ package geth_test
 import (
 	"path/filepath"
 
+	"net/http"
+
+	"fmt"
+
+	"log"
+
 	cfg "github.com/8thlight/vulcanizedb/pkg/config"
 	"github.com/8thlight/vulcanizedb/pkg/geth"
 	"github.com/ethereum/go-ethereum/accounts/abi"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+	"github.com/onsi/gomega/ghttp"
 )
 
-var _ = Describe("Reading ABI files", func() {
+var _ = Describe("ABI files", func() {
 
-	It("loads a valid ABI file", func() {
+	Describe("Reading ABI files", func() {
-		path := filepath.Join(cfg.ProjectRoot(), "pkg", "geth", "testing", "valid_abi.json")
 
-		contractAbi, err := geth.ParseAbiFile(path)
+		It("loads a valid ABI file", func() {
+			path := filepath.Join(cfg.ProjectRoot(), "pkg", "geth", "testing", "valid_abi.json")
 
-		Expect(contractAbi).NotTo(BeNil())
+			contractAbi, err := geth.ParseAbiFile(path)
-		Expect(err).To(BeNil())
+
+			Expect(contractAbi).NotTo(BeNil())
+			Expect(err).To(BeNil())
+		})
+
+		It("reads the contents of a valid ABI file", func() {
+			path := filepath.Join(cfg.ProjectRoot(), "pkg", "geth", "testing", "valid_abi.json")
+
+			contractAbi, err := geth.ReadAbiFile(path)
+
+			Expect(contractAbi).To(Equal("[{\"foo\": \"bar\"}]"))
+			Expect(err).To(BeNil())
+		})
+
+		It("returns an error when the file does not exist", func() {
+			path := filepath.Join(cfg.ProjectRoot(), "pkg", "geth", "testing", "missing_abi.json")
+
+			contractAbi, err := geth.ParseAbiFile(path)
+
+			Expect(contractAbi).To(Equal(abi.ABI{}))
+			Expect(err).To(Equal(geth.ErrMissingAbiFile))
+		})
+
+		It("returns an error when the file has invalid contents", func() {
+			path := filepath.Join(cfg.ProjectRoot(), "pkg", "geth", "testing", "invalid_abi.json")
+
+			contractAbi, err := geth.ParseAbiFile(path)
+
+			Expect(contractAbi).To(Equal(abi.ABI{}))
+			Expect(err).To(Equal(geth.ErrInvalidAbiFile))
+		})
+
+		Describe("Request ABI from endpoint", func() {
+
+			var (
+				server    *ghttp.Server
+				client    *geth.EtherScanApi
+				abiString string
+			)
+
+			BeforeEach(func() {
+				server = ghttp.NewServer()
+				client = geth.NewEtherScanClient(server.URL())
+				path := filepath.Join(cfg.ProjectRoot(), "pkg", "geth", "testing", "sample_abi.json")
+				abiString, err := geth.ReadAbiFile(path)
+				_, err = geth.ParseAbi(abiString)
+				if err != nil {
+					log.Fatalln("Could not parse ABI")
+				}
+			})
+
+			AfterEach(func() {
+				server.Close()
+			})
+
+			Describe("Fetching ABI from api (etherscan)", func() {
+				BeforeEach(func() {
+
+					response := fmt.Sprintf(`{"status":"1","message":"OK","result":%q}`, abiString)
+					server.AppendHandlers(
+						ghttp.CombineHandlers(
+							ghttp.VerifyRequest("GET", "/api", "module=contract&action=getabi&address=0xd26114cd6EE289AccF82350c8d8487fedB8A0C07"),
+							ghttp.RespondWith(http.StatusOK, response),
+						),
+					)
+				})
+
+				It("should make a GET request with supplied contract hash", func() {
+
+					abi, err := client.GetAbi("0xd26114cd6EE289AccF82350c8d8487fedB8A0C07")
+					Expect(server.ReceivedRequests()).Should(HaveLen(1))
+					Expect(err).ShouldNot(HaveOccurred())
+					Expect(abi).Should(Equal(abiString))
+				})
+			})
+		})
 	})
-
-	It("reads the contents of a valid ABI file", func() {
-		path := filepath.Join(cfg.ProjectRoot(), "pkg", "geth", "testing", "valid_abi.json")
-
-		contractAbi, err := geth.ReadAbiFile(path)
-
-		Expect(contractAbi).To(Equal("[{\"foo\": \"bar\"}]"))
-		Expect(err).To(BeNil())
-	})
-
-	It("returns an error when the file does not exist", func() {
-		path := filepath.Join(cfg.ProjectRoot(), "pkg", "geth", "testing", "missing_abi.json")
-
-		contractAbi, err := geth.ParseAbiFile(path)
-
-		Expect(contractAbi).To(Equal(abi.ABI{}))
-		Expect(err).To(Equal(geth.ErrMissingAbiFile))
-	})
-
-	It("returns an error when the file has invalid contents", func() {
-		path := filepath.Join(cfg.ProjectRoot(), "pkg", "geth", "testing", "invalid_abi.json")
-
-		contractAbi, err := geth.ParseAbiFile(path)
-
-		Expect(contractAbi).To(Equal(abi.ABI{}))
-		Expect(err).To(Equal(geth.ErrInvalidAbiFile))
-	})
-
 })

pkg/geth/contract.go

@@ -2,15 +2,12 @@ package geth
 
 import (
 	"errors"
-	"fmt"
-	"path/filepath"
 
 	"sort"
 
 	"context"
 	"math/big"
 
-	"github.com/8thlight/vulcanizedb/pkg/config"
 	"github.com/8thlight/vulcanizedb/pkg/core"
 	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/common"
@@ -59,17 +56,3 @@ func (blockchain *GethBlockchain) GetAttributes(contract core.Contract) (core.Co
 	sort.Sort(contractAttributes)
 	return contractAttributes, nil
 }
-
-func (blockchain *GethBlockchain) GetContractAttributesOld(contractHash string) (core.ContractAttributes, error) {
-	abiFilePath := filepath.Join(config.ProjectRoot(), "contracts", "public", fmt.Sprintf("%s.json", contractHash))
-	parsed, _ := ParseAbiFile(abiFilePath)
-	var contractAttributes core.ContractAttributes
-	for _, abiElement := range parsed.Methods {
-		if (len(abiElement.Outputs) > 0) && (len(abiElement.Inputs) == 0) && abiElement.Const {
-			attributeType := abiElement.Outputs[0].Type.String()
-			contractAttributes = append(contractAttributes, core.ContractAttribute{abiElement.Name, attributeType})
-		}
-	}
-	sort.Sort(contractAttributes)
-	return contractAttributes, nil
-}

pkg/repositories/postgres.go

@@ -38,7 +38,12 @@ func (repository Postgres) CreateContract(contract core.Contract) error {
 		abiToInsert = &abi
 	}
 	_, err := repository.Db.Exec(
-		`INSERT INTO watched_contracts (contract_hash, contract_abi) VALUES ($1, $2)`, contract.Hash, abiToInsert)
+		`INSERT INTO watched_contracts (contract_hash, contract_abi)
+				VALUES ($1, $2)
+				ON CONFLICT (contract_hash)
+				  DO UPDATE
+					SET contract_hash = $1, contract_abi = $2
+				`, contract.Hash, abiToInsert)
 	if err != nil {
 		return ErrDBInsertFailed
 	}
@@ -53,15 +58,16 @@ func (repository Postgres) ContractExists(contractHash string) bool {
 }
 
 func (repository Postgres) FindContract(contractHash string) *core.Contract {
-	var savedContracts []core.Contract
+	var hash string
-	contractRows, _ := repository.Db.Query(
+	var abi string
+	row := repository.Db.QueryRow(
 		`SELECT contract_hash, contract_abi FROM watched_contracts WHERE contract_hash=$1`, contractHash)
-	savedContracts = repository.loadContract(contractRows)
+	err := row.Scan(&hash, &abi)
-	if len(savedContracts) > 0 {
+	if err == sql.ErrNoRows {
-		return &savedContracts[0]
-	} else {
 		return nil
 	}
+	savedContract := repository.addTransactions(core.Contract{Hash: hash, Abi: abi})
+	return &savedContract
 }
 
 func (repository Postgres) MaxBlockNumber() int64 {
@@ -197,16 +203,9 @@ func (repository Postgres) loadTransactions(transactionRows *sql.Rows) []core.Tr
 	return transactions
 }
 
-func (repository Postgres) loadContract(contractRows *sql.Rows) []core.Contract {
+func (repository Postgres) addTransactions(contract core.Contract) core.Contract {
-	var savedContracts []core.Contract
+	transactionRows, _ := repository.Db.Query(`SELECT tx_hash, tx_nonce, tx_to, tx_from, tx_gaslimit, tx_gasprice, tx_value FROM transactions WHERE tx_to = $1 ORDER BY block_id desc`, contract.Hash)
-	for contractRows.Next() {
+	transactions := repository.loadTransactions(transactionRows)
-		var savedContractHash string
+	savedContract := core.Contract{Hash: contract.Hash, Transactions: transactions, Abi: contract.Abi}
-		var savedContractAbi string
+	return savedContract
-		contractRows.Scan(&savedContractHash, &savedContractAbi)
-		transactionRows, _ := repository.Db.Query(`SELECT tx_hash, tx_nonce, tx_to, tx_from, tx_gaslimit, tx_gasprice, tx_value FROM transactions WHERE tx_to = $1 ORDER BY block_id desc`, savedContractHash)
-		transactions := repository.loadTransactions(transactionRows)
-		savedContract := core.Contract{Hash: savedContractHash, Transactions: transactions, Abi: savedContractAbi}
-		savedContracts = append(savedContracts, savedContract)
-	}
-	return savedContracts
 }

pkg/repositories/testing/helpers.go

@@ -262,6 +262,20 @@ func AssertRepositoryBehavior(buildRepository func() repositories.Repository) {
 			Expect(contract).ToNot(BeNil())
 			Expect(contract.Abi).To(Equal("{\"some\": \"json\"}"))
 		})
+
+		It("updates the ABI of the contract if hash already present", func() {
+			repository.CreateContract(core.Contract{
+				Abi:  "{\"some\": \"json\"}",
+				Hash: "x123",
+			})
+			repository.CreateContract(core.Contract{
+				Abi:  "{\"some\": \"different json\"}",
+				Hash: "x123",
+			})
+			contract := repository.FindContract("x123")
+			Expect(contract).ToNot(BeNil())
+			Expect(contract.Abi).To(Equal("{\"some\": \"different json\"}"))
+		})
 	})
 
 }