major refactor part 2: remove cold import, full sync, generalize node table

pkg/eth/blockchain.go

@@ -78,7 +78,7 @@ func (blockChain *BlockChain) GetEthLogsWithCustomQuery(query ethereum.FilterQue
 
 func (blockChain *BlockChain) GetHeaderByNumber(blockNumber int64) (header core.Header, err error) {
 	logrus.Debugf("GetHeaderByNumber called with block %d", blockNumber)
-	if blockChain.node.NetworkID == core.KOVAN_NETWORK_ID {
+	if blockChain.node.NetworkID == string(core.KOVAN_NETWORK_ID) {
 		return blockChain.getPOAHeader(blockNumber)
 	}
 	return blockChain.getPOWHeader(blockNumber)
@@ -86,7 +86,7 @@ func (blockChain *BlockChain) GetHeaderByNumber(blockNumber int64) (header core.
 
 func (blockChain *BlockChain) GetHeadersByNumbers(blockNumbers []int64) (header []core.Header, err error) {
 	logrus.Debug("GetHeadersByNumbers called")
-	if blockChain.node.NetworkID == core.KOVAN_NETWORK_ID {
+	if blockChain.node.NetworkID == string(core.KOVAN_NETWORK_ID) {
 		return blockChain.getPOAHeaders(blockNumbers)
 	}
 	return blockChain.getPOWHeaders(blockNumbers)

pkg/eth/blockchain_test.go

@@ -102,7 +102,7 @@ var _ = Describe("Geth blockchain", func() {
 
 		Describe("POA/Kovan", func() {
 			It("fetches header from rpcClient", func() {
-				node.NetworkID = vulcCore.KOVAN_NETWORK_ID
+				node.NetworkID = string(vulcCore.KOVAN_NETWORK_ID)
 				blockNumber := hexutil.Big(*big.NewInt(100))
 				mockRpcClient.SetReturnPOAHeader(vulcCore.POAHeader{Number: &blockNumber})
 				blockChain = eth.NewBlockChain(mockClient, mockRpcClient, node, fakes.NewMockTransactionConverter())
@@ -114,7 +114,7 @@ var _ = Describe("Geth blockchain", func() {
 			})
 
 			It("returns err if rpcClient returns err", func() {
-				node.NetworkID = vulcCore.KOVAN_NETWORK_ID
+				node.NetworkID = string(vulcCore.KOVAN_NETWORK_ID)
 				mockRpcClient.SetCallContextErr(fakes.FakeError)
 				blockChain = eth.NewBlockChain(mockClient, mockRpcClient, node, fakes.NewMockTransactionConverter())
 
@@ -125,7 +125,7 @@ var _ = Describe("Geth blockchain", func() {
 			})
 
 			It("returns error if returned header is empty", func() {
-				node.NetworkID = vulcCore.KOVAN_NETWORK_ID
+				node.NetworkID = string(vulcCore.KOVAN_NETWORK_ID)
 				blockChain = eth.NewBlockChain(mockClient, mockRpcClient, node, fakes.NewMockTransactionConverter())
 
 				_, err := blockChain.GetHeaderByNumber(100)
@@ -135,7 +135,7 @@ var _ = Describe("Geth blockchain", func() {
 			})
 
 			It("returns multiple headers with multiple blocknumbers", func() {
-				node.NetworkID = vulcCore.KOVAN_NETWORK_ID
+				node.NetworkID = string(vulcCore.KOVAN_NETWORK_ID)
 				blockNumber := hexutil.Big(*big.NewInt(100))
 				mockRpcClient.SetReturnPOAHeaders([]vulcCore.POAHeader{{Number: &blockNumber}})
 

pkg/eth/cold_import/cold_import_suite_test.go

@@ -1,29 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cold_import_test
-
-import (
-	"testing"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-)
-
-func TestColdImport(t *testing.T) {
-	RegisterFailHandler(Fail)
-	RunSpecs(t, "ColdImport Suite")
-}

pkg/eth/cold_import/importer.go

@@ -1,75 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cold_import
-
-import (
-	"github.com/vulcanize/vulcanizedb/pkg/eth/converters/common"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/ethereum"
-)
-
-type ColdImporter struct {
-	blockRepository   datastore.BlockRepository
-	converter         common.BlockConverter
-	ethDB             ethereum.Database
-	receiptRepository datastore.FullSyncReceiptRepository
-}
-
-func NewColdImporter(ethDB ethereum.Database, blockRepository datastore.BlockRepository, receiptRepository datastore.FullSyncReceiptRepository, converter common.BlockConverter) *ColdImporter {
-	return &ColdImporter{
-		blockRepository:   blockRepository,
-		converter:         converter,
-		ethDB:             ethDB,
-		receiptRepository: receiptRepository,
-	}
-}
-
-func (ci *ColdImporter) Execute(startingBlockNumber int64, endingBlockNumber int64, nodeID string) error {
-	missingBlocks := ci.blockRepository.MissingBlockNumbers(startingBlockNumber, endingBlockNumber, nodeID)
-	for _, n := range missingBlocks {
-		hash := ci.ethDB.GetBlockHash(n)
-
-		blockID, err := ci.createBlocksAndTransactions(hash, n)
-		if err != nil {
-			return err
-		}
-		err = ci.createReceiptsAndLogs(hash, n, blockID)
-		if err != nil {
-			return err
-		}
-	}
-	ci.blockRepository.SetBlocksStatus(endingBlockNumber)
-	return nil
-}
-
-func (ci *ColdImporter) createBlocksAndTransactions(hash []byte, i int64) (int64, error) {
-	block := ci.ethDB.GetBlock(hash, i)
-	coreBlock, err := ci.converter.ToCoreBlock(block)
-	if err != nil {
-		return 0, err
-	}
-	return ci.blockRepository.CreateOrUpdateBlock(coreBlock)
-}
-
-func (ci *ColdImporter) createReceiptsAndLogs(hash []byte, number int64, blockID int64) error {
-	receipts := ci.ethDB.GetBlockReceipts(hash, number)
-	coreReceipts, err := common.ToCoreReceipts(receipts)
-	if err != nil {
-		return err
-	}
-	return ci.receiptRepository.CreateReceiptsAndLogs(blockID, coreReceipts)
-}

pkg/eth/cold_import/importer_test.go

@@ -1,152 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cold_import_test
-
-import (
-	"github.com/ethereum/go-ethereum/core/types"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/cold_import"
-	vulcCommon "github.com/vulcanize/vulcanizedb/pkg/eth/converters/common"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres/repositories"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/fakes"
-)
-
-var _ = Describe("Geth cold importer", func() {
-	var fakeGethBlock *types.Block
-
-	BeforeEach(func() {
-		header := &types.Header{}
-		transactions := []*types.Transaction{}
-		uncles := []*types.Header{}
-		receipts := []*types.Receipt{}
-		fakeGethBlock = types.NewBlock(header, transactions, uncles, receipts)
-	})
-
-	It("only populates missing blocks", func() {
-		mockEthereumDatabase := fakes.NewMockEthereumDatabase()
-		mockBlockRepository := fakes.NewMockBlockRepository()
-		mockReceiptRepository := fakes.NewMockReceiptRepository()
-		mockTransactionConverter := fakes.NewMockTransactionConverter()
-		blockConverter := vulcCommon.NewBlockConverter(mockTransactionConverter)
-
-		nodeId := "node_id"
-		startingBlockNumber := int64(120)
-		missingBlockNumber := int64(123)
-		endingBlockNumber := int64(125)
-		fakeHash := []byte{1, 2, 3, 4, 5}
-		mockBlockRepository.SetMissingBlockNumbersReturnArray([]int64{missingBlockNumber})
-		mockEthereumDatabase.SetReturnHash(fakeHash)
-		mockEthereumDatabase.SetReturnBlock(fakeGethBlock)
-		importer := cold_import.NewColdImporter(mockEthereumDatabase, mockBlockRepository, mockReceiptRepository, blockConverter)
-
-		err := importer.Execute(startingBlockNumber, endingBlockNumber, nodeId)
-		Expect(err).NotTo(HaveOccurred())
-
-		mockBlockRepository.AssertMissingBlockNumbersCalledWith(startingBlockNumber, endingBlockNumber, nodeId)
-		mockEthereumDatabase.AssertGetBlockHashCalledWith(missingBlockNumber)
-		mockEthereumDatabase.AssertGetBlockCalledWith(fakeHash, missingBlockNumber)
-	})
-
-	It("fetches missing blocks from level db and persists them to pg", func() {
-		mockEthereumDatabase := fakes.NewMockEthereumDatabase()
-		mockBlockRepository := fakes.NewMockBlockRepository()
-		mockReceiptRepository := fakes.NewMockReceiptRepository()
-		mockTransactionConverter := fakes.NewMockTransactionConverter()
-		blockConverter := vulcCommon.NewBlockConverter(mockTransactionConverter)
-
-		blockNumber := int64(123)
-		fakeHash := []byte{1, 2, 3, 4, 5}
-		mockBlockRepository.SetMissingBlockNumbersReturnArray([]int64{blockNumber})
-		mockEthereumDatabase.SetReturnHash(fakeHash)
-		mockEthereumDatabase.SetReturnBlock(fakeGethBlock)
-		importer := cold_import.NewColdImporter(mockEthereumDatabase, mockBlockRepository, mockReceiptRepository, blockConverter)
-
-		err := importer.Execute(blockNumber, blockNumber, "node_id")
-		Expect(err).NotTo(HaveOccurred())
-
-		mockEthereumDatabase.AssertGetBlockHashCalledWith(blockNumber)
-		mockEthereumDatabase.AssertGetBlockCalledWith(fakeHash, blockNumber)
-		Expect(mockTransactionConverter.ConvertBlockTransactionsToCoreCalled).To(BeTrue())
-		Expect(mockTransactionConverter.ConvertBlockTransactionsToCorePassedBlock).To(Equal(fakeGethBlock))
-		convertedBlock, err := blockConverter.ToCoreBlock(fakeGethBlock)
-		Expect(err).NotTo(HaveOccurred())
-		mockBlockRepository.AssertCreateOrUpdateBlockCalledWith(convertedBlock)
-	})
-
-	It("sets is_final status on populated blocks", func() {
-		mockEthereumDatabase := fakes.NewMockEthereumDatabase()
-		mockBlockRepository := fakes.NewMockBlockRepository()
-		mockReceiptRepository := fakes.NewMockReceiptRepository()
-		mockTransactionConverter := fakes.NewMockTransactionConverter()
-		blockConverter := vulcCommon.NewBlockConverter(mockTransactionConverter)
-
-		startingBlockNumber := int64(120)
-		endingBlockNumber := int64(125)
-		fakeHash := []byte{1, 2, 3, 4, 5}
-		mockBlockRepository.SetMissingBlockNumbersReturnArray([]int64{startingBlockNumber})
-		mockEthereumDatabase.SetReturnHash(fakeHash)
-		mockEthereumDatabase.SetReturnBlock(fakeGethBlock)
-		importer := cold_import.NewColdImporter(mockEthereumDatabase, mockBlockRepository, mockReceiptRepository, blockConverter)
-
-		err := importer.Execute(startingBlockNumber, endingBlockNumber, "node_id")
-		Expect(err).NotTo(HaveOccurred())
-		mockBlockRepository.AssertSetBlockStatusCalledWith(endingBlockNumber)
-	})
-
-	It("fetches receipts from level db and persists them to pg", func() {
-		mockEthereumDatabase := fakes.NewMockEthereumDatabase()
-		mockBlockRepository := fakes.NewMockBlockRepository()
-		mockReceiptRepository := fakes.NewMockReceiptRepository()
-		mockTransactionConverter := fakes.NewMockTransactionConverter()
-		blockConverter := vulcCommon.NewBlockConverter(mockTransactionConverter)
-
-		blockNumber := int64(123)
-		blockId := int64(999)
-		mockBlockRepository.SetCreateOrUpdateBlockReturnVals(blockId, nil)
-		fakeReceipts := types.Receipts{{}}
-		mockBlockRepository.SetMissingBlockNumbersReturnArray([]int64{blockNumber})
-		mockEthereumDatabase.SetReturnBlock(fakeGethBlock)
-		mockEthereumDatabase.SetReturnReceipts(fakeReceipts)
-		importer := cold_import.NewColdImporter(mockEthereumDatabase, mockBlockRepository, mockReceiptRepository, blockConverter)
-
-		err := importer.Execute(blockNumber, blockNumber, "node_id")
-		Expect(err).NotTo(HaveOccurred())
-		expectedReceipts, err := vulcCommon.ToCoreReceipts(fakeReceipts)
-		Expect(err).ToNot(HaveOccurred())
-		mockReceiptRepository.AssertCreateReceiptsAndLogsCalledWith(blockId, expectedReceipts)
-	})
-
-	It("does not fetch receipts if block already exists", func() {
-		mockEthereumDatabase := fakes.NewMockEthereumDatabase()
-		mockBlockRepository := fakes.NewMockBlockRepository()
-		mockReceiptRepository := fakes.NewMockReceiptRepository()
-		mockTransactionConverter := fakes.NewMockTransactionConverter()
-		blockConverter := vulcCommon.NewBlockConverter(mockTransactionConverter)
-
-		blockNumber := int64(123)
-		mockBlockRepository.SetMissingBlockNumbersReturnArray([]int64{})
-		mockEthereumDatabase.SetReturnBlock(fakeGethBlock)
-		mockBlockRepository.SetCreateOrUpdateBlockReturnVals(0, repositories.ErrBlockExists)
-		importer := cold_import.NewColdImporter(mockEthereumDatabase, mockBlockRepository, mockReceiptRepository, blockConverter)
-
-		err := importer.Execute(blockNumber, blockNumber, "node_id")
-
-		Expect(err).NotTo(HaveOccurred())
-		mockReceiptRepository.AssertCreateReceiptsAndLogsNotCalled()
-	})
-})

pkg/eth/cold_import/node_builder.go

@@ -1,85 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cold_import
-
-import (
-	"errors"
-	"strings"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/crypto"
-	"github.com/vulcanize/vulcanizedb/pkg/fs"
-)
-
-const (
-	ColdImportClientName         = "LevelDbColdImport"
-	ColdImportNetworkID  float64 = 1
-)
-
-var (
-	NoChainDataErr = errors.New("Level DB path does not include chaindata extension.")
-	NoGethRootErr  = errors.New("Level DB path does not include root path to geth.")
-)
-
-type ColdImportNodeBuilder struct {
-	reader fs.Reader
-	parser crypto.PublicKeyParser
-}
-
-func NewColdImportNodeBuilder(reader fs.Reader, parser crypto.PublicKeyParser) ColdImportNodeBuilder {
-	return ColdImportNodeBuilder{reader: reader, parser: parser}
-}
-
-func (cinb ColdImportNodeBuilder) GetNode(genesisBlock []byte, levelPath string) (core.Node, error) {
-	var coldNode core.Node
-	nodeKeyPath, err := getNodeKeyPath(levelPath)
-	if err != nil {
-		return coldNode, err
-	}
-	nodeKey, err := cinb.reader.Read(nodeKeyPath)
-	if err != nil {
-		return coldNode, err
-	}
-	nodeID, err := cinb.parser.ParsePublicKey(string(nodeKey))
-	if err != nil {
-		return coldNode, err
-	}
-	genesisBlockHash := common.BytesToHash(genesisBlock).String()
-	coldNode = core.Node{
-		GenesisBlock: genesisBlockHash,
-		NetworkID:    ColdImportNetworkID,
-		ID:           nodeID,
-		ClientName:   ColdImportClientName,
-	}
-	return coldNode, nil
-}
-
-func getNodeKeyPath(levelPath string) (string, error) {
-	chaindataExtension := "chaindata"
-	if !strings.Contains(levelPath, chaindataExtension) {
-		return "", NoChainDataErr
-	}
-	chaindataExtensionLength := len(chaindataExtension)
-	gethRootPathLength := len(levelPath) - chaindataExtensionLength
-	if gethRootPathLength <= chaindataExtensionLength {
-		return "", NoGethRootErr
-	}
-	gethRootPath := levelPath[:gethRootPathLength]
-	nodeKeyPath := gethRootPath + "nodekey"
-	return nodeKeyPath, nil
-}

pkg/eth/cold_import/node_builder_test.go

@@ -1,114 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cold_import_test
-
-import (
-	"errors"
-
-	"github.com/ethereum/go-ethereum/common"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/cold_import"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/fakes"
-)
-
-var _ = Describe("Cold importer node builder", func() {
-	Describe("when level path is not valid", func() {
-		It("returns error if no chaindata extension", func() {
-			gethPath := "path/to/geth"
-			mockReader := fakes.NewMockFsReader()
-			mockParser := fakes.NewMockCryptoParser()
-			nodeBuilder := cold_import.NewColdImportNodeBuilder(mockReader, mockParser)
-
-			_, err := nodeBuilder.GetNode([]byte{1, 2, 3, 4, 5}, gethPath)
-
-			Expect(err).To(HaveOccurred())
-			Expect(err).To(MatchError(cold_import.NoChainDataErr))
-		})
-
-		It("returns error if no root geth path", func() {
-			chaindataPath := "chaindata"
-			mockReader := fakes.NewMockFsReader()
-			mockParser := fakes.NewMockCryptoParser()
-			nodeBuilder := cold_import.NewColdImportNodeBuilder(mockReader, mockParser)
-
-			_, err := nodeBuilder.GetNode([]byte{1, 2, 3, 4, 5}, chaindataPath)
-
-			Expect(err).To(HaveOccurred())
-			Expect(err).To(MatchError(cold_import.NoGethRootErr))
-		})
-	})
-
-	Describe("when reader fails", func() {
-		It("returns err", func() {
-			mockReader := fakes.NewMockFsReader()
-			fakeError := errors.New("Failed")
-			mockReader.SetReturnErr(fakeError)
-			mockParser := fakes.NewMockCryptoParser()
-			nodeBuilder := cold_import.NewColdImportNodeBuilder(mockReader, mockParser)
-
-			_, err := nodeBuilder.GetNode([]byte{1, 2, 3, 4, 5}, "path/to/geth/chaindata")
-
-			Expect(err).To(HaveOccurred())
-			Expect(err).To(MatchError(fakeError))
-		})
-	})
-
-	Describe("when parser fails", func() {
-		It("returns err", func() {
-			mockReader := fakes.NewMockFsReader()
-			mockParser := fakes.NewMockCryptoParser()
-			fakeErr := errors.New("Failed")
-			mockParser.SetReturnErr(fakeErr)
-			nodeBuilder := cold_import.NewColdImportNodeBuilder(mockReader, mockParser)
-
-			_, err := nodeBuilder.GetNode([]byte{1, 2, 3, 4, 5}, "path/to/geth/chaindata")
-
-			Expect(err).To(HaveOccurred())
-			Expect(err).To(MatchError(fakeErr))
-		})
-	})
-
-	Describe("when path is valid and reader and parser succeed", func() {
-		It("builds a node", func() {
-			fakeGenesisBlock := []byte{1, 2, 3, 4, 5}
-			fakeRootGethPath := "root/path/to/geth/"
-			fakeLevelPath := fakeRootGethPath + "chaindata"
-			fakeNodeKeyPath := fakeRootGethPath + "nodekey"
-			fakePublicKeyBytes := []byte{5, 4, 3, 2, 1}
-			fakePublicKeyString := "public_key"
-			mockReader := fakes.NewMockFsReader()
-			mockReader.SetReturnBytes(fakePublicKeyBytes)
-			mockParser := fakes.NewMockCryptoParser()
-			mockParser.SetReturnVal(fakePublicKeyString)
-			nodeBuilder := cold_import.NewColdImportNodeBuilder(mockReader, mockParser)
-
-			result, err := nodeBuilder.GetNode(fakeGenesisBlock, fakeLevelPath)
-
-			Expect(err).NotTo(HaveOccurred())
-			mockReader.AssertReadCalledWith(fakeNodeKeyPath)
-			mockParser.AssertParsePublicKeyCalledWith(string(fakePublicKeyBytes))
-			Expect(result).NotTo(BeNil())
-			Expect(result.ClientName).To(Equal(cold_import.ColdImportClientName))
-			expectedGenesisBlock := common.BytesToHash(fakeGenesisBlock).String()
-			Expect(result.GenesisBlock).To(Equal(expectedGenesisBlock))
-			Expect(result.ID).To(Equal(fakePublicKeyString))
-			Expect(result.NetworkID).To(Equal(cold_import.ColdImportNetworkID))
-		})
-	})
-
-})

pkg/eth/contract_watcher/full/converter/converter.go

@@ -1,117 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package converter
-
-import (
-	"fmt"
-	"math/big"
-	"strconv"
-
-	"github.com/ethereum/go-ethereum/accounts/abi/bind"
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/common/hexutil"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/contract"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/helpers"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/types"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-)
-
-// ConverterInterface is used to convert watched event logs to
-// custom logs containing event input name => value maps
-type ConverterInterface interface {
-	Convert(watchedEvent core.WatchedEvent, event types.Event) (*types.Log, error)
-	Update(info *contract.Contract)
-}
-
-// Converter is the underlying struct for the ConverterInterface
-type Converter struct {
-	ContractInfo *contract.Contract
-}
-
-// Update configures the converter for a specific contract
-func (c *Converter) Update(info *contract.Contract) {
-	c.ContractInfo = info
-}
-
-// Convert converts the given watched event log into a types.Log for the given event
-func (c *Converter) Convert(watchedEvent core.WatchedEvent, event types.Event) (*types.Log, error) {
-	boundContract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil)
-	values := make(map[string]interface{})
-	log := helpers.ConvertToLog(watchedEvent)
-	err := boundContract.UnpackLogIntoMap(values, event.Name, log)
-	if err != nil {
-		return nil, err
-	}
-
-	strValues := make(map[string]string, len(values))
-	seenAddrs := make([]interface{}, 0, len(values))
-	seenHashes := make([]interface{}, 0, len(values))
-	for fieldName, input := range values {
-		// Postgres cannot handle custom types, resolve to strings
-		switch input.(type) {
-		case *big.Int:
-			b := input.(*big.Int)
-			strValues[fieldName] = b.String()
-		case common.Address:
-			a := input.(common.Address)
-			strValues[fieldName] = a.String()
-			seenAddrs = append(seenAddrs, a)
-		case common.Hash:
-			h := input.(common.Hash)
-			strValues[fieldName] = h.String()
-			seenHashes = append(seenHashes, h)
-		case string:
-			strValues[fieldName] = input.(string)
-		case bool:
-			strValues[fieldName] = strconv.FormatBool(input.(bool))
-		case []byte:
-			b := input.([]byte)
-			strValues[fieldName] = hexutil.Encode(b)
-			if len(b) == 32 { // collect byte arrays of size 32 as hashes
-				seenHashes = append(seenHashes, common.HexToHash(strValues[fieldName]))
-			}
-		case byte:
-			b := input.(byte)
-			strValues[fieldName] = string(b)
-		default:
-			return nil, fmt.Errorf("error: unhandled abi type %T", input)
-		}
-	}
-
-	// Only hold onto logs that pass our address filter, if any
-	if c.ContractInfo.PassesEventFilter(strValues) {
-		eventLog := &types.Log{
-			ID:     watchedEvent.LogID,
-			Values: strValues,
-			Block:  watchedEvent.BlockNumber,
-			Tx:     watchedEvent.TxHash,
-		}
-
-		// Cache emitted values if their caching is turned on
-		if c.ContractInfo.EmittedAddrs != nil {
-			c.ContractInfo.AddEmittedAddr(seenAddrs...)
-		}
-		if c.ContractInfo.EmittedHashes != nil {
-			c.ContractInfo.AddEmittedHash(seenHashes...)
-		}
-
-		return eventLog, nil
-	}
-
-	return nil, nil
-}

pkg/eth/contract_watcher/full/converter/converter_suite_test.go

@@ -1,35 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package converter_test
-
-import (
-	"io/ioutil"
-	"log"
-	"testing"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-)
-
-func TestConverter(t *testing.T) {
-	RegisterFailHandler(Fail)
-	RunSpecs(t, "Full Converter Suite Test")
-}
-
-var _ = BeforeSuite(func() {
-	log.SetOutput(ioutil.Discard)
-})

pkg/eth/contract_watcher/full/converter/converter_test.go

@@ -1,113 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package converter_test
-
-import (
-	"github.com/ethereum/go-ethereum/common"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/full/converter"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/contract"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/helpers"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/helpers/test_helpers"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/helpers/test_helpers/mocks"
-)
-
-var _ = Describe("Converter", func() {
-	var con *contract.Contract
-	var wantedEvents = []string{"Transfer"}
-	var err error
-
-	BeforeEach(func() {
-		con = test_helpers.SetupTusdContract(wantedEvents, []string{"balanceOf"})
-	})
-
-	Describe("Update", func() {
-		It("Updates contract con held by the converter", func() {
-			c := converter.Converter{}
-			c.Update(con)
-			Expect(c.ContractInfo).To(Equal(con))
-
-			con := test_helpers.SetupTusdContract([]string{}, []string{})
-			c.Update(con)
-			Expect(c.ContractInfo).To(Equal(con))
-		})
-	})
-
-	Describe("Convert", func() {
-		It("Converts a watched event log to mapping of event input names to values", func() {
-			_, ok := con.Events["Approval"]
-			Expect(ok).To(Equal(false))
-
-			event, ok := con.Events["Transfer"]
-			Expect(ok).To(Equal(true))
-			err = con.GenerateFilters()
-			Expect(err).ToNot(HaveOccurred())
-
-			c := converter.Converter{}
-			c.Update(con)
-			log, err := c.Convert(mocks.MockTranferEvent, event)
-			Expect(err).ToNot(HaveOccurred())
-
-			from := common.HexToAddress("0x000000000000000000000000000000000000000000000000000000000000af21")
-			to := common.HexToAddress("0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391")
-			value := helpers.BigFromString("1097077688018008265106216665536940668749033598146")
-
-			v := log.Values["value"]
-
-			Expect(log.Values["to"]).To(Equal(to.String()))
-			Expect(log.Values["from"]).To(Equal(from.String()))
-			Expect(v).To(Equal(value.String()))
-		})
-
-		It("Keeps track of addresses it sees to grow a token holder address list for the contract", func() {
-			event, ok := con.Events["Transfer"]
-			Expect(ok).To(Equal(true))
-
-			c := converter.Converter{}
-			c.Update(con)
-			_, err := c.Convert(mocks.MockTranferEvent, event)
-			Expect(err).ToNot(HaveOccurred())
-
-			b, ok := con.EmittedAddrs[common.HexToAddress("0x000000000000000000000000000000000000Af21")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			b, ok = con.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843bCE061BA391")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			_, ok = con.EmittedAddrs[common.HexToAddress("0x")]
-			Expect(ok).To(Equal(false))
-
-			_, ok = con.EmittedAddrs[""]
-			Expect(ok).To(Equal(false))
-
-			_, ok = con.EmittedAddrs[common.HexToAddress("0x09THISE21a5IS5cFAKE1D82fAND43bCE06MADEUP")]
-			Expect(ok).To(Equal(false))
-		})
-
-		It("Fails with an empty contract", func() {
-			event := con.Events["Transfer"]
-			c := converter.Converter{}
-			c.Update(&contract.Contract{})
-			_, err = c.Convert(mocks.MockTranferEvent, event)
-			Expect(err).To(HaveOccurred())
-		})
-	})
-})

pkg/eth/contract_watcher/full/retriever/block_retriever.go

@@ -1,99 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package retriever
-
-import (
-	"database/sql"
-
-	"github.com/vulcanize/vulcanizedb/libraries/shared/repository"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
-)
-
-// BlockRetriever is used to retrieve the first block for a given contract and the most recent block
-// It requires a vDB synced database with blocks, transactions, receipts, and logs
-type BlockRetriever interface {
-	RetrieveFirstBlock(contractAddr string) (int64, error)
-	RetrieveMostRecentBlock() (int64, error)
-}
-
-type blockRetriever struct {
-	db *postgres.DB
-}
-
-// NewBlockRetriever returns a new BlockRetriever
-func NewBlockRetriever(db *postgres.DB) BlockRetriever {
-	return &blockRetriever{
-		db: db,
-	}
-}
-
-// RetrieveFirstBlock fetches the block number for the earliest block in the db
-// Tries both methods of finding the first block, with the receipt method taking precedence
-func (r *blockRetriever) RetrieveFirstBlock(contractAddr string) (int64, error) {
-	i, err := r.retrieveFirstBlockFromReceipts(contractAddr)
-	if err != nil {
-		if err == sql.ErrNoRows {
-			i, err = r.retrieveFirstBlockFromLogs(contractAddr)
-		}
-		return i, err
-	}
-
-	return i, err
-}
-
-// For some contracts the contract creation transaction receipt doesn't have the contract address so this doesn't work (e.g. Sai)
-func (r *blockRetriever) retrieveFirstBlockFromReceipts(contractAddr string) (int64, error) {
-	var firstBlock int64
-	addressID, getAddressErr := repository.GetOrCreateAddress(r.db, contractAddr)
-	if getAddressErr != nil {
-		return firstBlock, getAddressErr
-	}
-	err := r.db.Get(
-		&firstBlock,
-		`SELECT number FROM eth_blocks
-		       WHERE id = (SELECT block_id FROM full_sync_receipts
-                           WHERE contract_address_id = $1
-		                   ORDER BY block_id ASC
-					       LIMIT 1)`,
-		addressID,
-	)
-
-	return firstBlock, err
-}
-
-// In which case this servers as a heuristic to find the first block by finding the first contract event log
-func (r *blockRetriever) retrieveFirstBlockFromLogs(contractAddr string) (int64, error) {
-	var firstBlock int
-	err := r.db.Get(
-		&firstBlock,
-		"SELECT block_number FROM full_sync_logs WHERE lower(address) = $1 ORDER BY block_number ASC LIMIT 1",
-		contractAddr,
-	)
-
-	return int64(firstBlock), err
-}
-
-// RetrieveMostRecentBlock retrieves the most recent block number in vDB
-func (r *blockRetriever) RetrieveMostRecentBlock() (int64, error) {
-	var lastBlock int64
-	err := r.db.Get(
-		&lastBlock,
-		"SELECT number FROM eth_blocks ORDER BY number DESC LIMIT 1",
-	)
-
-	return lastBlock, err
-}

pkg/eth/contract_watcher/full/retriever/block_retriever_test.go

@@ -1,259 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package retriever_test
-
-import (
-	"strings"
-
-	"github.com/ethereum/go-ethereum/core/types"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/full/retriever"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/constants"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/helpers/test_helpers"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres/repositories"
-)
-
-var _ = Describe("Block Retriever", func() {
-	var db *postgres.DB
-	var r retriever.BlockRetriever
-	var rawTransaction []byte
-	var blockRepository repositories.BlockRepository
-
-	// Contains no contract address
-	var block1 = core.Block{
-		Hash:         "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad123ert",
-		Number:       1,
-		Transactions: []core.TransactionModel{},
-	}
-
-	BeforeEach(func() {
-		db, _ = test_helpers.SetupDBandBC()
-		blockRepository = *repositories.NewBlockRepository(db)
-		r = retriever.NewBlockRetriever(db)
-		gethTransaction := types.Transaction{}
-		var err error
-		rawTransaction, err = gethTransaction.MarshalJSON()
-		Expect(err).NotTo(HaveOccurred())
-	})
-
-	AfterEach(func() {
-		test_helpers.TearDown(db)
-	})
-
-	Describe("RetrieveFirstBlock", func() {
-		It("Retrieves block number where contract first appears in receipt, if available", func() {
-			// Contains the address in the receipt
-			block2 := core.Block{
-				Hash:   "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad123ert",
-				Number: 2,
-				Transactions: []core.TransactionModel{{
-					Hash:     "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae",
-					GasPrice: 0,
-					GasLimit: 0,
-					Nonce:    0,
-					Raw:      rawTransaction,
-					Receipt: core.Receipt{
-						TxHash:          "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae",
-						ContractAddress: constants.TusdContractAddress,
-						Logs:            []core.FullSyncLog{},
-					},
-					TxIndex: 0,
-					Value:   "0",
-				}},
-			}
-
-			// Contains address in logs
-			block3 := core.Block{
-				Hash:   "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad456yui",
-				Number: 3,
-				Transactions: []core.TransactionModel{{
-					Hash:     "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad234hfs",
-					GasPrice: 0,
-					GasLimit: 0,
-					Nonce:    0,
-					Raw:      rawTransaction,
-					Receipt: core.Receipt{
-						TxHash:          "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad234hfs",
-						ContractAddress: constants.TusdContractAddress,
-						Logs: []core.FullSyncLog{{
-							BlockNumber: 3,
-							TxHash:      "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad234hfs",
-							Address:     constants.TusdContractAddress,
-							Topics: core.Topics{
-								constants.TransferEvent.Signature(),
-								"0x000000000000000000000000000000000000000000000000000000000000af21",
-								"0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391",
-								"",
-							},
-							Index: 1,
-							Data:  "0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe",
-						}},
-					},
-					TxIndex: 0,
-					Value:   "0",
-				}},
-			}
-
-			_, insertErrOne := blockRepository.CreateOrUpdateBlock(block1)
-			Expect(insertErrOne).NotTo(HaveOccurred())
-			_, insertErrTwo := blockRepository.CreateOrUpdateBlock(block2)
-			Expect(insertErrTwo).NotTo(HaveOccurred())
-			_, insertErrThree := blockRepository.CreateOrUpdateBlock(block3)
-			Expect(insertErrThree).NotTo(HaveOccurred())
-
-			i, err := r.RetrieveFirstBlock(strings.ToLower(constants.TusdContractAddress))
-			Expect(err).NotTo(HaveOccurred())
-			Expect(i).To(Equal(int64(2)))
-		})
-
-		It("Retrieves block number where contract first appears in event logs if it cannot find the address in a receipt", func() {
-			block2 := core.Block{
-				Hash:   "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad123ert",
-				Number: 2,
-				Transactions: []core.TransactionModel{{
-					Hash:     "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae",
-					GasPrice: 0,
-					GasLimit: 0,
-					Nonce:    0,
-					Raw:      rawTransaction,
-					Receipt: core.Receipt{
-						TxHash:          "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae",
-						ContractAddress: "",
-						Logs: []core.FullSyncLog{{
-							BlockNumber: 2,
-							TxHash:      "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae",
-							Address:     constants.DaiContractAddress,
-							Topics: core.Topics{
-								constants.TransferEvent.Signature(),
-								"0x000000000000000000000000000000000000000000000000000000000000af21",
-								"0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391",
-								"",
-							},
-							Index: 1,
-							Data:  "0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe",
-						}},
-					},
-					TxIndex: 0,
-					Value:   "0",
-				}},
-			}
-
-			block3 := core.Block{
-				Hash:   "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad456yui",
-				Number: 3,
-				Transactions: []core.TransactionModel{{
-					Hash:     "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad234hfs",
-					GasPrice: 0,
-					GasLimit: 0,
-					Nonce:    0,
-					Raw:      rawTransaction,
-					Receipt: core.Receipt{
-						TxHash:          "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad234hfs",
-						ContractAddress: "",
-						Logs: []core.FullSyncLog{{
-							BlockNumber: 3,
-							TxHash:      "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad234hfs",
-							Address:     constants.DaiContractAddress,
-							Topics: core.Topics{
-								constants.TransferEvent.Signature(),
-								"0x000000000000000000000000000000000000000000000000000000000000af21",
-								"0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391",
-								"",
-							},
-							Index: 1,
-							Data:  "0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe",
-						}},
-					},
-					TxIndex: 0,
-					Value:   "0",
-				}},
-			}
-
-			_, insertErrOne := blockRepository.CreateOrUpdateBlock(block1)
-			Expect(insertErrOne).NotTo(HaveOccurred())
-			_, insertErrTwo := blockRepository.CreateOrUpdateBlock(block2)
-			Expect(insertErrTwo).NotTo(HaveOccurred())
-			_, insertErrThree := blockRepository.CreateOrUpdateBlock(block3)
-			Expect(insertErrThree).NotTo(HaveOccurred())
-
-			i, err := r.RetrieveFirstBlock(constants.DaiContractAddress)
-			Expect(err).NotTo(HaveOccurred())
-			Expect(i).To(Equal(int64(2)))
-		})
-
-		It("Fails if the contract address cannot be found in any blocks", func() {
-			block2 := core.Block{
-				Hash:         "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad123ert",
-				Number:       2,
-				Transactions: []core.TransactionModel{},
-			}
-
-			block3 := core.Block{
-				Hash:         "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad456yui",
-				Number:       3,
-				Transactions: []core.TransactionModel{},
-			}
-
-			_, insertErrOne := blockRepository.CreateOrUpdateBlock(block1)
-			Expect(insertErrOne).NotTo(HaveOccurred())
-			_, insertErrTwo := blockRepository.CreateOrUpdateBlock(block2)
-			Expect(insertErrTwo).NotTo(HaveOccurred())
-			_, insertErrThree := blockRepository.CreateOrUpdateBlock(block3)
-			Expect(insertErrThree).NotTo(HaveOccurred())
-
-			_, err := r.RetrieveFirstBlock(constants.DaiContractAddress)
-			Expect(err).To(HaveOccurred())
-		})
-	})
-
-	Describe("RetrieveMostRecentBlock", func() {
-		It("Retrieves the latest block", func() {
-			block2 := core.Block{
-				Hash:         "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad123ert",
-				Number:       2,
-				Transactions: []core.TransactionModel{},
-			}
-
-			block3 := core.Block{
-				Hash:         "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad456yui",
-				Number:       3,
-				Transactions: []core.TransactionModel{},
-			}
-
-			_, insertErrOne := blockRepository.CreateOrUpdateBlock(block1)
-			Expect(insertErrOne).NotTo(HaveOccurred())
-			_, insertErrTwo := blockRepository.CreateOrUpdateBlock(block2)
-			Expect(insertErrTwo).NotTo(HaveOccurred())
-			_, insertErrThree := blockRepository.CreateOrUpdateBlock(block3)
-			Expect(insertErrThree).NotTo(HaveOccurred())
-
-			i, err := r.RetrieveMostRecentBlock()
-			Expect(err).ToNot(HaveOccurred())
-			Expect(i).To(Equal(int64(3)))
-		})
-
-		It("Fails if it cannot retrieve the latest block", func() {
-			i, err := r.RetrieveMostRecentBlock()
-			Expect(err).To(HaveOccurred())
-			Expect(i).To(Equal(int64(0)))
-		})
-	})
-})

pkg/eth/contract_watcher/full/retriever/retriever_suite_test.go

@@ -1,35 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package retriever_test
-
-import (
-	"io/ioutil"
-	"testing"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-	"github.com/sirupsen/logrus"
-)
-
-func TestRetriever(t *testing.T) {
-	RegisterFailHandler(Fail)
-	RunSpecs(t, "Full Block Number Retriever Suite Test")
-}
-
-var _ = BeforeSuite(func() {
-	logrus.SetOutput(ioutil.Discard)
-})

pkg/eth/contract_watcher/full/transformer/transformer.go

@@ -1,236 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package transformer
-
-import (
-	"errors"
-
-	"github.com/sirupsen/logrus"
-
-	"github.com/vulcanize/vulcanizedb/pkg/config"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/full/converter"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/full/retriever"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/contract"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/parser"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/poller"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/repository"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/types"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres/repositories"
-)
-
-// Transformer is the top level struct for transforming watched contract data
-// Requires a fully synced vDB and a running eth node (or infura)
-type Transformer struct {
-	// Database interfaces
-	FilterRepository           datastore.FilterRepository       // Log filters repo; accepts filters generated by Contract.GenerateFilters()
-	WatchedEventRepository     datastore.WatchedEventRepository // Watched event log views, created by the log filters
-	TransformedEventRepository repository.EventRepository       // Holds transformed watched event log data
-
-	// Pre-processing interfaces
-	Parser    parser.Parser            // Parses events and methods out of contract abi fetched using contract address
-	Retriever retriever.BlockRetriever // Retrieves first block for contract and current block height
-
-	// Processing interfaces
-	Converter converter.ConverterInterface // Converts watched event logs into custom log
-	Poller    poller.Poller                // Polls methods using contract's token holder addresses and persists them using method datastore
-
-	// Store contract configuration information
-	Config config.ContractConfig
-
-	// Store contract info as mapping to contract address
-	Contracts map[string]*contract.Contract
-
-	// Latest block in the block repository
-	LastBlock int64
-}
-
-// NewTransformer takes in contract config, blockchain, and database, and returns a new Transformer
-func NewTransformer(con config.ContractConfig, BC core.BlockChain, DB *postgres.DB) *Transformer {
-	return &Transformer{
-		Poller:                     poller.NewPoller(BC, DB, types.FullSync),
-		Parser:                     parser.NewParser(con.Network),
-		Retriever:                  retriever.NewBlockRetriever(DB),
-		Converter:                  &converter.Converter{},
-		Contracts:                  map[string]*contract.Contract{},
-		WatchedEventRepository:     repositories.WatchedEventRepository{DB: DB},
-		FilterRepository:           repositories.FilterRepository{DB: DB},
-		TransformedEventRepository: repository.NewEventRepository(DB, types.FullSync),
-		Config:                     con,
-	}
-}
-
-// Init initializes the transformer
-// Use after creating and setting transformer
-// Loops over all of the addr => filter sets
-// Uses parser to pull event info from abi
-// Use this info to generate event filters
-func (tr *Transformer) Init() error {
-	for contractAddr := range tr.Config.Addresses {
-		// Configure Abi
-		if tr.Config.Abis[contractAddr] == "" {
-			// If no abi is given in the config, this method will try fetching from internal look-up table and etherscan
-			err := tr.Parser.Parse(contractAddr)
-			if err != nil {
-				return err
-			}
-		} else {
-			// If we have an abi from the config, load that into the parser
-			err := tr.Parser.ParseAbiStr(tr.Config.Abis[contractAddr])
-			if err != nil {
-				return err
-			}
-		}
-
-		// Get first block and most recent block number in the header repo
-		firstBlock, err := tr.Retriever.RetrieveFirstBlock(contractAddr)
-		if err != nil {
-			return err
-		}
-		// Set to specified range if it falls within the bounds
-		if firstBlock < tr.Config.StartingBlocks[contractAddr] {
-			firstBlock = tr.Config.StartingBlocks[contractAddr]
-		}
-
-		// Get contract name if it has one
-		var name = new(string)
-		pollingErr := tr.Poller.FetchContractData(tr.Parser.Abi(), contractAddr, "name", nil, name, tr.LastBlock)
-		if pollingErr != nil {
-			// can't return this error because "name" might not exist on the contract
-			logrus.Warnf("error fetching contract data: %s", pollingErr.Error())
-		}
-
-		// Remove any potential accidental duplicate inputs in arg filter values
-		eventArgs := map[string]bool{}
-		for _, arg := range tr.Config.EventArgs[contractAddr] {
-			eventArgs[arg] = true
-		}
-		methodArgs := map[string]bool{}
-		for _, arg := range tr.Config.MethodArgs[contractAddr] {
-			methodArgs[arg] = true
-		}
-
-		// Aggregate info into contract object
-		info := contract.Contract{
-			Name:          *name,
-			Network:       tr.Config.Network,
-			Address:       contractAddr,
-			Abi:           tr.Parser.Abi(),
-			ParsedAbi:     tr.Parser.ParsedAbi(),
-			StartingBlock: firstBlock,
-			Events:        tr.Parser.GetEvents(tr.Config.Events[contractAddr]),
-			Methods:       tr.Parser.GetSelectMethods(tr.Config.Methods[contractAddr]),
-			FilterArgs:    eventArgs,
-			MethodArgs:    methodArgs,
-			Piping:        tr.Config.Piping[contractAddr],
-		}.Init()
-
-		// Use info to create filters
-		err = info.GenerateFilters()
-		if err != nil {
-			return err
-		}
-
-		// Iterate over filters and push them to the repo using filter repository interface
-		for _, filter := range info.Filters {
-			err = tr.FilterRepository.CreateFilter(filter)
-			if err != nil {
-				return err
-			}
-		}
-
-		// Store contract info for further processing
-		tr.Contracts[contractAddr] = info
-	}
-
-	// Get the most recent block number in the block repo
-	var err error
-	tr.LastBlock, err = tr.Retriever.RetrieveMostRecentBlock()
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// Execute runs the transformation processes
-// Iterates through stored, initialized contract objects
-// Iterates through contract's event filters, grabbing watched event logs
-// Uses converter to convert logs into custom log type
-// Persists converted logs into custom postgres tables
-// Calls selected methods, using token holder address generated during event log conversion
-func (tr *Transformer) Execute() error {
-	if len(tr.Contracts) == 0 {
-		return errors.New("error: transformer has no initialized contracts to work with")
-	}
-	// Iterate through all internal contracts
-	for _, con := range tr.Contracts {
-		// Update converter with current contract
-		tr.Converter.Update(con)
-
-		// Iterate through contract filters and get watched event logs
-		for eventSig, filter := range con.Filters {
-			watchedEvents, err := tr.WatchedEventRepository.GetWatchedEvents(filter.Name)
-			if err != nil {
-				return err
-			}
-
-			// Iterate over watched event logs
-			for _, we := range watchedEvents {
-				// Convert them to our custom log type
-				cstm, err := tr.Converter.Convert(*we, con.Events[eventSig])
-				if err != nil {
-					return err
-				}
-				if cstm == nil {
-					continue
-				}
-
-				// If log is not empty, immediately persist in repo
-				// Run this in seperate goroutine?
-				err = tr.TransformedEventRepository.PersistLogs([]types.Log{*cstm}, con.Events[eventSig], con.Address, con.Name)
-				if err != nil {
-					return err
-				}
-			}
-		}
-
-		// After persisting all watched event logs
-		// poller polls select contract methods
-		// and persists the results into custom pg tables
-		if err := tr.Poller.PollContract(*con, tr.LastBlock); err != nil {
-			return err
-		}
-	}
-
-	// At the end of a transformation cycle, and before the next
-	// update the latest block from the block repo
-	var err error
-	tr.LastBlock, err = tr.Retriever.RetrieveMostRecentBlock()
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// GetConfig returns the transformers config; satisfies the transformer interface
-func (tr *Transformer) GetConfig() config.ContractConfig {
-	return tr.Config
-}

pkg/eth/contract_watcher/full/transformer/transformer_suite_test.go

@@ -1,35 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package transformer_test
-
-import (
-	"io/ioutil"
-	"testing"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-	"github.com/sirupsen/logrus"
-)
-
-func TestTransformer(t *testing.T) {
-	RegisterFailHandler(Fail)
-	RunSpecs(t, "Full Transformer Suite Test")
-}
-
-var _ = BeforeSuite(func() {
-	logrus.SetOutput(ioutil.Discard)
-})

pkg/eth/contract_watcher/full/transformer/transformer_test.go

@@ -1,98 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package transformer_test
-
-import (
-	"math/rand"
-	"time"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/full/retriever"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/full/transformer"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/contract"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/helpers/test_helpers/mocks"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/parser"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/poller"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/types"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/fakes"
-)
-
-var _ = Describe("Transformer", func() {
-	var fakeAddress = "0x1234567890abcdef"
-	rand.Seed(time.Now().UnixNano())
-
-	Describe("Init", func() {
-		It("Initializes transformer's contract objects", func() {
-			blockRetriever := &fakes.MockFullSyncBlockRetriever{}
-			firstBlock := int64(1)
-			mostRecentBlock := int64(2)
-			blockRetriever.FirstBlock = firstBlock
-			blockRetriever.MostRecentBlock = mostRecentBlock
-
-			parsr := &fakes.MockParser{}
-			fakeAbi := "fake_abi"
-			eventName := "Transfer"
-			event := types.Event{}
-			parsr.AbiToReturn = fakeAbi
-			parsr.EventName = eventName
-			parsr.Event = event
-
-			pollr := &fakes.MockPoller{}
-			fakeContractName := "fake_contract_name"
-			pollr.ContractName = fakeContractName
-
-			t := getTransformer(blockRetriever, parsr, pollr)
-
-			err := t.Init()
-
-			Expect(err).ToNot(HaveOccurred())
-
-			c, ok := t.Contracts[fakeAddress]
-			Expect(ok).To(Equal(true))
-
-			Expect(c.StartingBlock).To(Equal(firstBlock))
-			Expect(t.LastBlock).To(Equal(mostRecentBlock))
-			Expect(c.Abi).To(Equal(fakeAbi))
-			Expect(c.Name).To(Equal(fakeContractName))
-			Expect(c.Address).To(Equal(fakeAddress))
-		})
-
-		It("Fails to initialize if first and most recent blocks cannot be fetched from vDB", func() {
-			blockRetriever := &fakes.MockFullSyncBlockRetriever{}
-			blockRetriever.FirstBlockErr = fakes.FakeError
-			t := getTransformer(blockRetriever, &fakes.MockParser{}, &fakes.MockPoller{})
-
-			err := t.Init()
-
-			Expect(err).To(HaveOccurred())
-			Expect(err).To(MatchError(fakes.FakeError))
-		})
-	})
-})
-
-func getTransformer(blockRetriever retriever.BlockRetriever, parsr parser.Parser, pollr poller.Poller) transformer.Transformer {
-	return transformer.Transformer{
-		FilterRepository: &fakes.MockFilterRepository{},
-		Parser:           parsr,
-		Retriever:        blockRetriever,
-		Poller:           pollr,
-		Contracts:        map[string]*contract.Contract{},
-		Config:           mocks.MockConfig,
-	}
-}

pkg/eth/contract_watcher/header/repository/header_repository.go

@@ -23,7 +23,7 @@ import (
 	"github.com/sirupsen/logrus"
 
 	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/postgres"
 )
 
 const columnCacheSize = 1000

pkg/eth/contract_watcher/header/repository/header_repository_test.go

@@ -26,8 +26,8 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/helpers/test_helpers"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/helpers/test_helpers/mocks"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres/repositories"
+	"github.com/vulcanize/vulcanizedb/pkg/postgres"
 )
 
 var _ = Describe("Repository", func() {

pkg/eth/contract_watcher/header/retriever/block_retriever.go

@@ -17,7 +17,7 @@
 package retriever
 
 import (
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/postgres"
 )
 
 // BlockRetriever is used to retrieve the first block for a given contract and the most recent block

pkg/eth/contract_watcher/header/retriever/block_retriever_test.go

@@ -23,8 +23,8 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/header/retriever"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/helpers/test_helpers"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/helpers/test_helpers/mocks"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres/repositories"
+	"github.com/vulcanize/vulcanizedb/pkg/postgres"
 )
 
 var _ = Describe("Block Retriever", func() {

pkg/eth/contract_watcher/header/transformer/transformer.go

@@ -38,7 +38,7 @@ import (
 	srep "github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/repository"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/types"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/postgres"
 )
 
 // Transformer is the top level struct for transforming watched contract data

pkg/eth/contract_watcher/shared/helpers/test_helpers/database.go

@@ -17,8 +17,6 @@
 package test_helpers
 
 import (
-	"math/rand"
-
 	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/ethereum/go-ethereum/rpc"
 	. "github.com/onsi/gomega"
@@ -31,9 +29,8 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/helpers/test_helpers/mocks"
 	rpc2 "github.com/vulcanize/vulcanizedb/pkg/eth/converters/rpc"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/node"
+	"github.com/vulcanize/vulcanizedb/pkg/postgres"
 	"github.com/vulcanize/vulcanizedb/test_config"
 )
 
@@ -129,34 +126,6 @@ func SetupDBandBC() (*postgres.DB, core.BlockChain) {
 	return db, blockChain
 }
 
-func SetupTusdRepo(vulcanizeLogID *int64, wantedEvents, wantedMethods []string) (*postgres.DB, *contract.Contract) {
-	db, err := postgres.NewDB(config.Database{
-		Hostname: "localhost",
-		Name:     "vulcanize_testing",
-		Port:     5432,
-	}, core.Node{})
-	Expect(err).NotTo(HaveOccurred())
-
-	receiptRepository := repositories.FullSyncReceiptRepository{DB: db}
-	logRepository := repositories.FullSyncLogRepository{DB: db}
-	blockRepository := *repositories.NewBlockRepository(db)
-
-	blockNumber := rand.Int63()
-	blockID := CreateBlock(blockNumber, blockRepository)
-
-	receipts := []core.Receipt{{Logs: []core.FullSyncLog{{}}}}
-
-	err = receiptRepository.CreateReceiptsAndLogs(blockID, receipts)
-	Expect(err).ToNot(HaveOccurred())
-
-	err = logRepository.Get(vulcanizeLogID, `SELECT id FROM full_sync_logs`)
-	Expect(err).ToNot(HaveOccurred())
-
-	info := SetupTusdContract(wantedEvents, wantedMethods)
-
-	return db, info
-}
-
 func SetupTusdContract(wantedEvents, wantedMethods []string) *contract.Contract {
 	p := mocks.NewParser(constants.TusdAbiString)
 	err := p.Parse(constants.TusdContractAddress)
@@ -237,27 +206,12 @@ func TearDown(db *postgres.DB) {
 	_, err = tx.Exec(`DELETE FROM addresses`)
 	Expect(err).NotTo(HaveOccurred())
 
-	_, err = tx.Exec(`DELETE FROM eth_blocks`)
-	Expect(err).NotTo(HaveOccurred())
-
 	_, err = tx.Exec(`DELETE FROM headers`)
 	Expect(err).NotTo(HaveOccurred())
 
-	_, err = tx.Exec(`DELETE FROM full_sync_logs`)
-	Expect(err).NotTo(HaveOccurred())
-
-	_, err = tx.Exec(`DELETE FROM log_filters`)
-	Expect(err).NotTo(HaveOccurred())
-
-	_, err = tx.Exec(`DELETE FROM full_sync_transactions`)
-	Expect(err).NotTo(HaveOccurred())
-
 	_, err = tx.Exec("DELETE FROM header_sync_transactions")
 	Expect(err).NotTo(HaveOccurred())
 
-	_, err = tx.Exec(`DELETE FROM full_sync_receipts`)
-	Expect(err).NotTo(HaveOccurred())
-
 	_, err = tx.Exec(`DELETE FROM header_sync_receipts`)
 	Expect(err).NotTo(HaveOccurred())
 
@@ -287,10 +241,3 @@ func TearDown(db *postgres.DB) {
 	_, err = db.Exec(`VACUUM checked_headers`)
 	Expect(err).NotTo(HaveOccurred())
 }
-
-func CreateBlock(blockNumber int64, repository repositories.BlockRepository) int64 {
-	blockID, err := repository.CreateOrUpdateBlock(core.Block{Number: blockNumber})
-	Expect(err).NotTo(HaveOccurred())
-
-	return blockID
-}

pkg/eth/contract_watcher/shared/poller/poller.go

@@ -30,7 +30,7 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/repository"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/types"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/postgres"
 )
 
 // Poller is the interface for polling public contract methods

pkg/eth/contract_watcher/shared/repository/event_repository.go

@@ -25,7 +25,7 @@ import (
 	"github.com/sirupsen/logrus"
 
 	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/types"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/postgres"
 )
 
 const (

pkg/eth/contract_watcher/shared/repository/event_repository_test.go

@@ -1,364 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package repository_test
-
-import (
-	"encoding/json"
-	"fmt"
-	"strings"
-
-	"github.com/ethereum/go-ethereum/common"
-	geth "github.com/ethereum/go-ethereum/core/types"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	fc "github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/full/converter"
-	lc "github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/header/converter"
-	lr "github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/header/repository"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/constants"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/contract"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/helpers/test_helpers"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/helpers/test_helpers/mocks"
-	sr "github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/repository"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/types"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres/repositories"
-)
-
-var _ = Describe("Repository", func() {
-	var db *postgres.DB
-	var dataStore sr.EventRepository
-	var err error
-	var log *types.Log
-	var logs []types.Log
-	var con *contract.Contract
-	var vulcanizeLogId int64
-	var wantedEvents = []string{"Transfer"}
-	var wantedMethods = []string{"balanceOf"}
-	var event types.Event
-	var headerID int64
-	var mockEvent = mocks.MockTranferEvent
-	var mockLog1 = mocks.MockTransferLog1
-	var mockLog2 = mocks.MockTransferLog2
-
-	BeforeEach(func() {
-		db, con = test_helpers.SetupTusdRepo(&vulcanizeLogId, wantedEvents, wantedMethods)
-		mockEvent.LogID = vulcanizeLogId
-
-		event = con.Events["Transfer"]
-		err = con.GenerateFilters()
-		Expect(err).ToNot(HaveOccurred())
-	})
-
-	AfterEach(func() {
-		test_helpers.TearDown(db)
-	})
-
-	Describe("Full sync mode", func() {
-		BeforeEach(func() {
-			dataStore = sr.NewEventRepository(db, types.FullSync)
-		})
-
-		Describe("CreateContractSchema", func() {
-			It("Creates schema if it doesn't exist", func() {
-				created, err := dataStore.CreateContractSchema(con.Address)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(created).To(Equal(true))
-
-				created, err = dataStore.CreateContractSchema(con.Address)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(created).To(Equal(false))
-			})
-
-			It("Caches schema it creates so that it does not need to repeatedly query the database to check for it's existence", func() {
-				_, ok := dataStore.CheckSchemaCache(con.Address)
-				Expect(ok).To(Equal(false))
-
-				created, err := dataStore.CreateContractSchema(con.Address)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(created).To(Equal(true))
-
-				v, ok := dataStore.CheckSchemaCache(con.Address)
-				Expect(ok).To(Equal(true))
-				Expect(v).To(Equal(true))
-			})
-		})
-
-		Describe("CreateEventTable", func() {
-			It("Creates table if it doesn't exist", func() {
-				created, err := dataStore.CreateContractSchema(con.Address)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(created).To(Equal(true))
-
-				created, err = dataStore.CreateEventTable(con.Address, event)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(created).To(Equal(true))
-
-				created, err = dataStore.CreateEventTable(con.Address, event)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(created).To(Equal(false))
-			})
-
-			It("Caches table it creates so that it does not need to repeatedly query the database to check for it's existence", func() {
-				created, err := dataStore.CreateContractSchema(con.Address)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(created).To(Equal(true))
-
-				tableID := fmt.Sprintf("%s_%s.%s_event", types.FullSync, strings.ToLower(con.Address), strings.ToLower(event.Name))
-				_, ok := dataStore.CheckTableCache(tableID)
-				Expect(ok).To(Equal(false))
-
-				created, err = dataStore.CreateEventTable(con.Address, event)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(created).To(Equal(true))
-
-				v, ok := dataStore.CheckTableCache(tableID)
-				Expect(ok).To(Equal(true))
-				Expect(v).To(Equal(true))
-			})
-		})
-
-		Describe("PersistLogs", func() {
-			BeforeEach(func() {
-				c := fc.Converter{}
-				c.Update(con)
-				log, err = c.Convert(mockEvent, event)
-				Expect(err).ToNot(HaveOccurred())
-			})
-
-			It("Persists contract event log values into custom tables", func() {
-				err = dataStore.PersistLogs([]types.Log{*log}, event, con.Address, con.Name)
-				Expect(err).ToNot(HaveOccurred())
-
-				b, ok := con.EmittedAddrs[common.HexToAddress("0x000000000000000000000000000000000000Af21")]
-				Expect(ok).To(Equal(true))
-				Expect(b).To(Equal(true))
-
-				b, ok = con.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843bCE061BA391")]
-				Expect(ok).To(Equal(true))
-				Expect(b).To(Equal(true))
-
-				scanLog := test_helpers.TransferLog{}
-
-				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.transfer_event", constants.TusdContractAddress)).StructScan(&scanLog)
-				Expect(err).ToNot(HaveOccurred())
-				expectedLog := test_helpers.TransferLog{
-					ID:             1,
-					VulcanizeLogID: vulcanizeLogId,
-					TokenName:      "TrueUSD",
-					Block:          5488076,
-					Tx:             "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae",
-					From:           "0x000000000000000000000000000000000000Af21",
-					To:             "0x09BbBBE21a5975cAc061D82f7b843bCE061BA391",
-					Value:          "1097077688018008265106216665536940668749033598146",
-				}
-				Expect(scanLog).To(Equal(expectedLog))
-			})
-
-			It("Doesn't persist duplicate event logs", func() {
-				// Try to persist the same log twice in a single call
-				err = dataStore.PersistLogs([]types.Log{*log, *log}, event, con.Address, con.Name)
-				Expect(err).ToNot(HaveOccurred())
-
-				scanLog := test_helpers.TransferLog{}
-
-				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.transfer_event", constants.TusdContractAddress)).StructScan(&scanLog)
-				Expect(err).ToNot(HaveOccurred())
-				expectedLog := test_helpers.TransferLog{
-					ID:             1,
-					VulcanizeLogID: vulcanizeLogId,
-					TokenName:      "TrueUSD",
-					Block:          5488076,
-					Tx:             "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae",
-					From:           "0x000000000000000000000000000000000000Af21",
-					To:             "0x09BbBBE21a5975cAc061D82f7b843bCE061BA391",
-					Value:          "1097077688018008265106216665536940668749033598146",
-				}
-				Expect(scanLog).To(Equal(expectedLog))
-
-				// Attempt to persist the same log again in separate call
-				err = dataStore.PersistLogs([]types.Log{*log}, event, con.Address, con.Name)
-				Expect(err).ToNot(HaveOccurred())
-
-				// Show that no new logs were entered
-				var count int
-				err = db.Get(&count, fmt.Sprintf("SELECT COUNT(*) FROM full_%s.transfer_event", constants.TusdContractAddress))
-				Expect(err).ToNot(HaveOccurred())
-				Expect(count).To(Equal(1))
-			})
-
-			It("Fails with empty log", func() {
-				err = dataStore.PersistLogs([]types.Log{}, event, con.Address, con.Name)
-				Expect(err).To(HaveOccurred())
-			})
-		})
-	})
-
-	Describe("Header sync mode", func() {
-		BeforeEach(func() {
-			dataStore = sr.NewEventRepository(db, types.HeaderSync)
-		})
-
-		Describe("CreateContractSchema", func() {
-			It("Creates schema if it doesn't exist", func() {
-				created, err := dataStore.CreateContractSchema(con.Address)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(created).To(Equal(true))
-
-				created, err = dataStore.CreateContractSchema(con.Address)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(created).To(Equal(false))
-			})
-
-			It("Caches schema it creates so that it does not need to repeatedly query the database to check for it's existence", func() {
-				_, ok := dataStore.CheckSchemaCache(con.Address)
-				Expect(ok).To(Equal(false))
-
-				created, err := dataStore.CreateContractSchema(con.Address)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(created).To(Equal(true))
-
-				v, ok := dataStore.CheckSchemaCache(con.Address)
-				Expect(ok).To(Equal(true))
-				Expect(v).To(Equal(true))
-			})
-
-			It("Caches table it creates so that it does not need to repeatedly query the database to check for it's existence", func() {
-				created, err := dataStore.CreateContractSchema(con.Address)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(created).To(Equal(true))
-
-				tableID := fmt.Sprintf("%s_%s.%s_event", types.HeaderSync, strings.ToLower(con.Address), strings.ToLower(event.Name))
-				_, ok := dataStore.CheckTableCache(tableID)
-				Expect(ok).To(Equal(false))
-
-				created, err = dataStore.CreateEventTable(con.Address, event)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(created).To(Equal(true))
-
-				v, ok := dataStore.CheckTableCache(tableID)
-				Expect(ok).To(Equal(true))
-				Expect(v).To(Equal(true))
-			})
-		})
-
-		Describe("CreateEventTable", func() {
-			It("Creates table if it doesn't exist", func() {
-				created, err := dataStore.CreateContractSchema(con.Address)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(created).To(Equal(true))
-
-				created, err = dataStore.CreateEventTable(con.Address, event)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(created).To(Equal(true))
-
-				created, err = dataStore.CreateEventTable(con.Address, event)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(created).To(Equal(false))
-			})
-		})
-
-		Describe("PersistLogs", func() {
-			BeforeEach(func() {
-				headerRepository := repositories.NewHeaderRepository(db)
-				headerID, err = headerRepository.CreateOrUpdateHeader(mocks.MockHeader1)
-				Expect(err).ToNot(HaveOccurred())
-				c := lc.Converter{}
-				c.Update(con)
-				logs, err = c.Convert([]geth.Log{mockLog1, mockLog2}, event, headerID)
-				Expect(err).ToNot(HaveOccurred())
-			})
-
-			It("Persists contract event log values into custom tables", func() {
-				hr := lr.NewHeaderRepository(db)
-				err = hr.AddCheckColumn(event.Name + "_" + con.Address)
-				Expect(err).ToNot(HaveOccurred())
-
-				err = dataStore.PersistLogs(logs, event, con.Address, con.Name)
-				Expect(err).ToNot(HaveOccurred())
-
-				var count int
-				err = db.Get(&count, fmt.Sprintf("SELECT COUNT(*) FROM header_%s.transfer_event", constants.TusdContractAddress))
-				Expect(err).ToNot(HaveOccurred())
-				Expect(count).To(Equal(2))
-
-				scanLog := test_helpers.HeaderSyncTransferLog{}
-				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.transfer_event LIMIT 1", constants.TusdContractAddress)).StructScan(&scanLog)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(scanLog.HeaderID).To(Equal(headerID))
-				Expect(scanLog.TokenName).To(Equal("TrueUSD"))
-				Expect(scanLog.TxIndex).To(Equal(int64(110)))
-				Expect(scanLog.LogIndex).To(Equal(int64(1)))
-				Expect(scanLog.From).To(Equal("0x000000000000000000000000000000000000Af21"))
-				Expect(scanLog.To).To(Equal("0x09BbBBE21a5975cAc061D82f7b843bCE061BA391"))
-				Expect(scanLog.Value).To(Equal("1097077688018008265106216665536940668749033598146"))
-
-				var expectedRawLog, rawLog geth.Log
-				err = json.Unmarshal(logs[0].Raw, &expectedRawLog)
-				Expect(err).ToNot(HaveOccurred())
-				err = json.Unmarshal(scanLog.RawLog, &rawLog)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(rawLog).To(Equal(expectedRawLog))
-			})
-
-			It("Doesn't persist duplicate event logs", func() {
-				hr := lr.NewHeaderRepository(db)
-				err = hr.AddCheckColumn(event.Name + "_" + con.Address)
-				Expect(err).ToNot(HaveOccurred())
-
-				// Successfully persist the two unique logs
-				err = dataStore.PersistLogs(logs, event, con.Address, con.Name)
-				Expect(err).ToNot(HaveOccurred())
-
-				// Try to insert the same logs again
-				err = dataStore.PersistLogs(logs, event, con.Address, con.Name)
-				Expect(err).ToNot(HaveOccurred())
-
-				// Show that no new logs were entered
-				var count int
-				err = db.Get(&count, fmt.Sprintf("SELECT COUNT(*) FROM header_%s.transfer_event", constants.TusdContractAddress))
-				Expect(err).ToNot(HaveOccurred())
-				Expect(count).To(Equal(2))
-			})
-
-			It("inserts additional log if only some are duplicate", func() {
-				hr := lr.NewHeaderRepository(db)
-				err = hr.AddCheckColumn(event.Name + "_" + con.Address)
-				Expect(err).ToNot(HaveOccurred())
-
-				// Successfully persist first log
-				err = dataStore.PersistLogs([]types.Log{logs[0]}, event, con.Address, con.Name)
-				Expect(err).ToNot(HaveOccurred())
-
-				// Successfully persist second log even though first already persisted
-				err = dataStore.PersistLogs(logs, event, con.Address, con.Name)
-				Expect(err).ToNot(HaveOccurred())
-
-				// Show that both logs were entered
-				var count int
-				err = db.Get(&count, fmt.Sprintf("SELECT COUNT(*) FROM header_%s.transfer_event", constants.TusdContractAddress))
-				Expect(err).ToNot(HaveOccurred())
-				Expect(count).To(Equal(2))
-			})
-
-			It("Fails with empty log", func() {
-				err = dataStore.PersistLogs([]types.Log{}, event, con.Address, con.Name)
-				Expect(err).To(HaveOccurred())
-			})
-		})
-	})
-})

pkg/eth/contract_watcher/shared/repository/method_repository.go

@@ -25,7 +25,7 @@ import (
 	"github.com/sirupsen/logrus"
 
 	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/types"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/postgres"
 )
 
 const methodCacheSize = 1000

pkg/eth/contract_watcher/shared/repository/method_repository_test.go

@@ -28,7 +28,7 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/helpers/test_helpers"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/repository"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/types"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/postgres"
 )
 
 var _ = Describe("Repository", func() {

pkg/eth/contract_watcher/shared/retriever/address_retriever.go

@@ -25,7 +25,7 @@ import (
 
 	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/contract"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/types"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/postgres"
 )
 
 // AddressRetriever is used to retrieve the addresses associated with a contract

pkg/eth/contract_watcher/shared/retriever/address_retriever_test.go

@@ -1,107 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package retriever_test
-
-import (
-	"github.com/ethereum/go-ethereum/common"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/full/converter"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/constants"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/contract"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/helpers/test_helpers"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/repository"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/retriever"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/contract_watcher/shared/types"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
-)
-
-var mockEvent = core.WatchedEvent{
-	Name:        constants.TransferEvent.String(),
-	BlockNumber: 5488076,
-	Address:     constants.TusdContractAddress,
-	TxHash:      "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae",
-	Index:       110,
-	Topic0:      constants.TransferEvent.Signature(),
-	Topic1:      "0x000000000000000000000000000000000000000000000000000000000000af21",
-	Topic2:      "0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391",
-	Topic3:      "",
-	Data:        "0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe",
-}
-
-var _ = Describe("Address Retriever Test", func() {
-	var db *postgres.DB
-	var dataStore repository.EventRepository
-	var info *contract.Contract
-	var vulcanizeLogId int64
-	var log *types.Log
-	var r retriever.AddressRetriever
-	var wantedEvents = []string{"Transfer"}
-
-	BeforeEach(func() {
-		db, info = test_helpers.SetupTusdRepo(&vulcanizeLogId, wantedEvents, []string{})
-		mockEvent.LogID = vulcanizeLogId
-
-		event := info.Events["Transfer"]
-		filterErr := info.GenerateFilters()
-		Expect(filterErr).ToNot(HaveOccurred())
-
-		c := converter.Converter{}
-		c.Update(info)
-		var convertErr error
-		log, convertErr = c.Convert(mockEvent, event)
-		Expect(convertErr).ToNot(HaveOccurred())
-
-		dataStore = repository.NewEventRepository(db, types.FullSync)
-		persistErr := dataStore.PersistLogs([]types.Log{*log}, event, info.Address, info.Name)
-		Expect(persistErr).ToNot(HaveOccurred())
-
-		r = retriever.NewAddressRetriever(db, types.FullSync)
-	})
-
-	AfterEach(func() {
-		test_helpers.TearDown(db)
-	})
-
-	Describe("RetrieveTokenHolderAddresses", func() {
-		It("Retrieves a list of token holder addresses from persisted event logs", func() {
-			addresses, retrieveErr := r.RetrieveTokenHolderAddresses(*info)
-			Expect(retrieveErr).ToNot(HaveOccurred())
-
-			_, ok := addresses[common.HexToAddress("0x000000000000000000000000000000000000000000000000000000000000af21")]
-			Expect(ok).To(Equal(true))
-
-			_, ok = addresses[common.HexToAddress("0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391")]
-			Expect(ok).To(Equal(true))
-
-			_, ok = addresses[common.HexToAddress("0x")]
-			Expect(ok).To(Equal(false))
-
-			_, ok = addresses[common.HexToAddress(constants.TusdContractAddress)]
-			Expect(ok).To(Equal(false))
-
-		})
-
-		It("Returns empty list when empty contract info is used", func() {
-			addresses, retrieveErr := r.RetrieveTokenHolderAddresses(contract.Contract{})
-			Expect(retrieveErr).ToNot(HaveOccurred())
-			Expect(len(addresses)).To(Equal(0))
-		})
-	})
-})

pkg/eth/contract_watcher/shared/retriever/retriever_suite_test.go

@@ -1,35 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package retriever_test
-
-import (
-	"io/ioutil"
-	"testing"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-	"github.com/sirupsen/logrus"
-)
-
-func TestRetriever(t *testing.T) {
-	RegisterFailHandler(Fail)
-	RunSpecs(t, "Address Retriever Suite Test")
-}
-
-var _ = BeforeSuite(func() {
-	logrus.SetOutput(ioutil.Discard)
-})

pkg/eth/core/node_info.go

@@ -35,7 +35,7 @@ const (
 
 type Node struct {
 	GenesisBlock string
-	NetworkID    float64
+	NetworkID    string
 	ID           string
 	ClientName   string
 }

pkg/eth/datastore/postgres/errors.go

@@ -1,37 +0,0 @@
-package postgres
-
-import (
-	"fmt"
-)
-
-const (
-	BeginTransactionFailedMsg = "failed to begin transaction"
-	DbConnectionFailedMsg     = "db connection failed"
-	DeleteQueryFailedMsg      = "delete query failed"
-	InsertQueryFailedMsg      = "insert query failed"
-	SettingNodeFailedMsg      = "unable to set db node"
-)
-
-func ErrBeginTransactionFailed(beginErr error) error {
-	return formatError(BeginTransactionFailedMsg, beginErr.Error())
-}
-
-func ErrDBConnectionFailed(connectErr error) error {
-	return formatError(DbConnectionFailedMsg, connectErr.Error())
-}
-
-func ErrDBDeleteFailed(deleteErr error) error {
-	return formatError(DeleteQueryFailedMsg, deleteErr.Error())
-}
-
-func ErrDBInsertFailed(insertErr error) error {
-	return formatError(InsertQueryFailedMsg, insertErr.Error())
-}
-
-func ErrUnableToSetNode(setErr error) error {
-	return formatError(SettingNodeFailedMsg, setErr.Error())
-}
-
-func formatError(msg, err string) error {
-	return fmt.Errorf("%s: %s", msg, err)
-}

pkg/eth/datastore/postgres/postgres.go

@@ -1,64 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package postgres
-
-import (
-	"github.com/jmoiron/sqlx"
-	_ "github.com/lib/pq" //postgres driver
-	"github.com/vulcanize/vulcanizedb/pkg/config"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-)
-
-type DB struct {
-	*sqlx.DB
-	Node   core.Node
-	NodeID int64
-}
-
-func NewDB(databaseConfig config.Database, node core.Node) (*DB, error) {
-	connectString := config.DbConnectionString(databaseConfig)
-	db, connectErr := sqlx.Connect("postgres", connectString)
-	if connectErr != nil {
-		return &DB{}, ErrDBConnectionFailed(connectErr)
-	}
-	pg := DB{DB: db, Node: node}
-	nodeErr := pg.CreateNode(&node)
-	if nodeErr != nil {
-		return &DB{}, ErrUnableToSetNode(nodeErr)
-	}
-	return &pg, nil
-}
-
-func (db *DB) CreateNode(node *core.Node) error {
-	var nodeID int64
-	err := db.QueryRow(
-		`INSERT INTO eth_nodes (genesis_block, network_id, eth_node_id, client_name)
-                VALUES ($1, $2, $3, $4)
-                ON CONFLICT (genesis_block, network_id, eth_node_id)
-                  DO UPDATE
-                    SET genesis_block = $1,
-                        network_id = $2,
-                        eth_node_id = $3,
-                        client_name = $4
-                RETURNING id`,
-		node.GenesisBlock, node.NetworkID, node.ID, node.ClientName).Scan(&nodeID)
-	if err != nil {
-		return ErrUnableToSetNode(err)
-	}
-	db.NodeID = nodeID
-	return nil
-}

pkg/eth/datastore/postgres/postgres_suite_test.go

@@ -1,36 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package postgres_test
-
-import (
-	"io/ioutil"
-	"testing"
-
-	log "github.com/sirupsen/logrus"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-)
-
-func init() {
-	log.SetOutput(ioutil.Discard)
-}
-
-func TestPostgres(t *testing.T) {
-	RegisterFailHandler(Fail)
-	RunSpecs(t, "Postgres Suite")
-}

pkg/eth/datastore/postgres/postgres_test.go

@@ -1,165 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package postgres_test
-
-import (
-	"fmt"
-	"strings"
-
-	"math/big"
-
-	"github.com/jmoiron/sqlx"
-	_ "github.com/lib/pq"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-	"github.com/vulcanize/vulcanizedb/pkg/config"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres/repositories"
-	"github.com/vulcanize/vulcanizedb/test_config"
-)
-
-var _ = Describe("Postgres DB", func() {
-	var sqlxdb *sqlx.DB
-
-	It("connects to the database", func() {
-		var err error
-		pgConfig := config.DbConnectionString(test_config.DBConfig)
-
-		sqlxdb, err = sqlx.Connect("postgres", pgConfig)
-
-		Expect(err).Should(BeNil())
-		Expect(sqlxdb).ShouldNot(BeNil())
-	})
-
-	It("serializes big.Int to db", func() {
-		// postgres driver doesn't support go big.Int type
-		// various casts in golang uint64, int64, overflow for
-		// transaction value (in wei) even though
-		// postgres numeric can handle an arbitrary
-		// sized int, so use string representation of big.Int
-		// and cast on insert
-
-		pgConnectString := config.DbConnectionString(test_config.DBConfig)
-		db, err := sqlx.Connect("postgres", pgConnectString)
-		Expect(err).NotTo(HaveOccurred())
-
-		bi := new(big.Int)
-		bi.SetString("34940183920000000000", 10)
-		Expect(bi.String()).To(Equal("34940183920000000000"))
-
-		defer db.Exec(`DROP TABLE IF EXISTS example`)
-		_, err = db.Exec("CREATE TABLE example ( id INTEGER, data NUMERIC )")
-		Expect(err).ToNot(HaveOccurred())
-
-		sqlStatement := `  
-			INSERT INTO example (id, data)
-			VALUES (1, cast($1 AS NUMERIC))`
-		_, err = db.Exec(sqlStatement, bi.String())
-		Expect(err).ToNot(HaveOccurred())
-
-		var data string
-		err = db.QueryRow(`SELECT data FROM example WHERE id = 1`).Scan(&data)
-		Expect(err).ToNot(HaveOccurred())
-
-		Expect(bi.String()).To(Equal(data))
-		actual := new(big.Int)
-		actual.SetString(data, 10)
-		Expect(actual).To(Equal(bi))
-	})
-
-	It("does not commit block if block is invalid", func() {
-		//badNonce violates db Nonce field length
-		badNonce := fmt.Sprintf("x %s", strings.Repeat("1", 100))
-		badBlock := core.Block{
-			Number:       123,
-			Nonce:        badNonce,
-			Transactions: []core.TransactionModel{},
-		}
-		node := core.Node{GenesisBlock: "GENESIS", NetworkID: 1, ID: "x123", ClientName: "geth"}
-		db := test_config.NewTestDB(node)
-		test_config.CleanTestDB(db)
-		blocksRepository := repositories.NewBlockRepository(db)
-
-		_, err1 := blocksRepository.CreateOrUpdateBlock(badBlock)
-
-		Expect(err1).To(HaveOccurred())
-		savedBlock, err2 := blocksRepository.GetBlock(123)
-		Expect(err2).To(HaveOccurred())
-		Expect(savedBlock).To(BeZero())
-	})
-
-	It("throws error when can't connect to the database", func() {
-		invalidDatabase := config.Database{}
-		node := core.Node{GenesisBlock: "GENESIS", NetworkID: 1, ID: "x123", ClientName: "geth"}
-
-		_, err := postgres.NewDB(invalidDatabase, node)
-
-		Expect(err).To(HaveOccurred())
-		Expect(err.Error()).To(ContainSubstring(postgres.DbConnectionFailedMsg))
-	})
-
-	It("throws error when can't create node", func() {
-		badHash := fmt.Sprintf("x %s", strings.Repeat("1", 100))
-		node := core.Node{GenesisBlock: badHash, NetworkID: 1, ID: "x123", ClientName: "geth"}
-
-		_, err := postgres.NewDB(test_config.DBConfig, node)
-
-		Expect(err).To(HaveOccurred())
-		Expect(err.Error()).To(ContainSubstring(postgres.SettingNodeFailedMsg))
-	})
-
-	It("does not commit log if log is invalid", func() {
-		//badTxHash violates db tx_hash field length
-		badTxHash := fmt.Sprintf("x %s", strings.Repeat("1", 100))
-		badLog := core.FullSyncLog{
-			Address:     "x123",
-			BlockNumber: 1,
-			TxHash:      badTxHash,
-		}
-		node := core.Node{GenesisBlock: "GENESIS", NetworkID: 1, ID: "x123", ClientName: "geth"}
-		db, _ := postgres.NewDB(test_config.DBConfig, node)
-		logRepository := repositories.FullSyncLogRepository{DB: db}
-
-		err := logRepository.CreateLogs([]core.FullSyncLog{badLog}, 123)
-
-		Expect(err).ToNot(BeNil())
-		savedBlock, err := logRepository.GetLogs("x123", 1)
-		Expect(savedBlock).To(BeNil())
-		Expect(err).To(Not(HaveOccurred()))
-	})
-
-	It("does not commit block or transactions if transaction is invalid", func() {
-		//badHash violates db To field length
-		badHash := fmt.Sprintf("x %s", strings.Repeat("1", 100))
-		badTransaction := core.TransactionModel{To: badHash}
-		block := core.Block{
-			Number:       123,
-			Transactions: []core.TransactionModel{badTransaction},
-		}
-		node := core.Node{GenesisBlock: "GENESIS", NetworkID: 1, ID: "x123", ClientName: "geth"}
-		db, _ := postgres.NewDB(test_config.DBConfig, node)
-		blockRepository := repositories.NewBlockRepository(db)
-
-		_, err1 := blockRepository.CreateOrUpdateBlock(block)
-
-		Expect(err1).To(HaveOccurred())
-		savedBlock, err2 := blockRepository.GetBlock(123)
-		Expect(err2).To(HaveOccurred())
-		Expect(savedBlock).To(BeZero())
-	})
-})

pkg/eth/datastore/postgres/repositories/block_repository.go

@@ -1,344 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package repositories
-
-import (
-	"database/sql"
-	"errors"
-
-	"github.com/jmoiron/sqlx"
-	"github.com/sirupsen/logrus"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
-)
-
-const (
-	blocksFromHeadBeforeFinal = 20
-)
-
-var ErrBlockExists = errors.New("Won't add block that already exists.")
-
-type BlockRepository struct {
-	database *postgres.DB
-}
-
-func NewBlockRepository(database *postgres.DB) *BlockRepository {
-	return &BlockRepository{database: database}
-}
-
-func (blockRepository BlockRepository) SetBlocksStatus(chainHead int64) error {
-	cutoff := chainHead - blocksFromHeadBeforeFinal
-	_, err := blockRepository.database.Exec(`
-                  UPDATE eth_blocks SET is_final = TRUE
-                  WHERE is_final = FALSE AND number < $1`,
-		cutoff)
-
-	return err
-}
-
-func (blockRepository BlockRepository) CreateOrUpdateBlock(block core.Block) (int64, error) {
-	var err error
-	var blockID int64
-	retrievedBlockHash, ok := blockRepository.getBlockHash(block)
-	if !ok {
-		return blockRepository.insertBlock(block)
-	}
-	if ok && retrievedBlockHash != block.Hash {
-		err = blockRepository.removeBlock(block.Number)
-		if err != nil {
-			return 0, err
-		}
-		return blockRepository.insertBlock(block)
-	}
-	return blockID, ErrBlockExists
-}
-
-func (blockRepository BlockRepository) MissingBlockNumbers(startingBlockNumber int64, highestBlockNumber int64, nodeID string) []int64 {
-	numbers := make([]int64, 0)
-	err := blockRepository.database.Select(&numbers,
-		`SELECT all_block_numbers
-          FROM (
-              SELECT generate_series($1::INT, $2::INT) AS all_block_numbers) series
-          WHERE all_block_numbers NOT IN (
-			  SELECT number FROM eth_blocks WHERE eth_node_fingerprint = $3
-		  ) `,
-		startingBlockNumber,
-		highestBlockNumber, nodeID)
-	if err != nil {
-		logrus.Error("MissingBlockNumbers: error getting blocks: ", err)
-	}
-	return numbers
-}
-
-func (blockRepository BlockRepository) GetBlock(blockNumber int64) (core.Block, error) {
-	blockRows := blockRepository.database.QueryRowx(
-		`SELECT id,
-                       number,
-                       gas_limit,
-                       gas_used,
-                       time,
-                       difficulty,
-                       hash,
-                       nonce,
-                       parent_hash,
-                       size,
-                       uncle_hash,
-                       is_final,
-                       miner,
-                       extra_data,
-                       reward,
-                       uncles_reward
-               FROM eth_blocks
-               WHERE eth_node_id = $1 AND number = $2`, blockRepository.database.NodeID, blockNumber)
-	savedBlock, err := blockRepository.loadBlock(blockRows)
-	if err != nil {
-		switch err {
-		case sql.ErrNoRows:
-			return core.Block{}, datastore.ErrBlockDoesNotExist(blockNumber)
-		default:
-			logrus.Error("GetBlock: error loading blocks: ", err)
-			return savedBlock, err
-		}
-	}
-	return savedBlock, nil
-}
-
-func (blockRepository BlockRepository) insertBlock(block core.Block) (int64, error) {
-	var blockID int64
-	tx, beginErr := blockRepository.database.Beginx()
-	if beginErr != nil {
-		return 0, postgres.ErrBeginTransactionFailed(beginErr)
-	}
-	insertBlockErr := tx.QueryRow(
-		`INSERT INTO eth_blocks
-                (eth_node_id, number, gas_limit, gas_used, time, difficulty, hash, nonce, parent_hash, size, uncle_hash, is_final, miner, extra_data, reward, uncles_reward, eth_node_fingerprint)
-                VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)
-                RETURNING id `,
-		blockRepository.database.NodeID,
-		block.Number,
-		block.GasLimit,
-		block.GasUsed,
-		block.Time,
-		block.Difficulty,
-		block.Hash,
-		block.Nonce,
-		block.ParentHash,
-		block.Size,
-		block.UncleHash,
-		block.IsFinal,
-		block.Miner,
-		block.ExtraData,
-		nullStringToZero(block.Reward),
-		nullStringToZero(block.UnclesReward),
-		blockRepository.database.Node.ID).
-		Scan(&blockID)
-	if insertBlockErr != nil {
-		rollbackErr := tx.Rollback()
-		if rollbackErr != nil {
-			logrus.Error("failed to rollback transaction: ", rollbackErr)
-		}
-		return 0, postgres.ErrDBInsertFailed(insertBlockErr)
-	}
-	if len(block.Uncles) > 0 {
-		insertUncleErr := blockRepository.createUncles(tx, blockID, block.Hash, block.Uncles)
-		if insertUncleErr != nil {
-			tx.Rollback()
-			return 0, postgres.ErrDBInsertFailed(insertUncleErr)
-		}
-	}
-	if len(block.Transactions) > 0 {
-		insertTxErr := blockRepository.createTransactions(tx, blockID, block.Transactions)
-		if insertTxErr != nil {
-			rollbackErr := tx.Rollback()
-			if rollbackErr != nil {
-				logrus.Warn("failed to rollback transaction: ", rollbackErr)
-			}
-			return 0, postgres.ErrDBInsertFailed(insertTxErr)
-		}
-	}
-	commitErr := tx.Commit()
-	if commitErr != nil {
-		rollbackErr := tx.Rollback()
-		if rollbackErr != nil {
-			logrus.Warn("failed to rollback transaction: ", rollbackErr)
-		}
-		return 0, commitErr
-	}
-	return blockID, nil
-}
-
-func (blockRepository BlockRepository) createUncles(tx *sqlx.Tx, blockID int64, blockHash string, uncles []core.Uncle) error {
-	for _, uncle := range uncles {
-		err := blockRepository.createUncle(tx, blockID, uncle)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (blockRepository BlockRepository) createUncle(tx *sqlx.Tx, blockID int64, uncle core.Uncle) error {
-	_, err := tx.Exec(
-		`INSERT INTO uncles
-       (hash, block_id, reward, miner, raw, block_timestamp, eth_node_id, eth_node_fingerprint)
-       VALUES ($1, $2, $3, $4, $5, $6, $7::NUMERIC, $8)
-       RETURNING id`,
-		uncle.Hash, blockID, nullStringToZero(uncle.Reward), uncle.Miner, uncle.Raw, uncle.Timestamp, blockRepository.database.NodeID, blockRepository.database.Node.ID)
-	return err
-}
-
-func (blockRepository BlockRepository) createTransactions(tx *sqlx.Tx, blockID int64, transactions []core.TransactionModel) error {
-	for _, transaction := range transactions {
-		err := blockRepository.createTransaction(tx, blockID, transaction)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-//Fields like value lose precision if converted to
-//int64 so convert to string instead. But nil
-//big.Int -> string = "" so convert to "0"
-func nullStringToZero(s string) string {
-	if s == "" {
-		return "0"
-	}
-	return s
-}
-
-func (blockRepository BlockRepository) createTransaction(tx *sqlx.Tx, blockID int64, transaction core.TransactionModel) error {
-	_, err := tx.Exec(
-		`INSERT INTO full_sync_transactions
-       (block_id, gas_limit, gas_price, hash, input_data, nonce, raw, tx_from, tx_index, tx_to, "value")
-       VALUES ($1, $2::NUMERIC, $3::NUMERIC, $4, $5, $6::NUMERIC, $7,  $8, $9::NUMERIC, $10, $11::NUMERIC)
-       RETURNING id`, blockID, transaction.GasLimit, transaction.GasPrice, transaction.Hash, transaction.Data,
-		transaction.Nonce, transaction.Raw, transaction.From, transaction.TxIndex, transaction.To, nullStringToZero(transaction.Value))
-	if err != nil {
-		return err
-	}
-	if hasReceipt(transaction) {
-		receiptRepo := FullSyncReceiptRepository{}
-		receiptID, err := receiptRepo.CreateFullSyncReceiptInTx(blockID, transaction.Receipt, tx)
-		if err != nil {
-			return err
-		}
-		if hasLogs(transaction) {
-			err = blockRepository.createLogs(tx, transaction.Receipt.Logs, receiptID)
-			if err != nil {
-				return err
-			}
-		}
-	}
-	return nil
-}
-
-func hasLogs(transaction core.TransactionModel) bool {
-	return len(transaction.Receipt.Logs) > 0
-}
-
-func hasReceipt(transaction core.TransactionModel) bool {
-	return transaction.Receipt.TxHash != ""
-}
-
-func (blockRepository BlockRepository) getBlockHash(block core.Block) (string, bool) {
-	var retrievedBlockHash string
-	// TODO: handle possible error
-	blockRepository.database.Get(&retrievedBlockHash,
-		`SELECT hash
-               FROM eth_blocks
-               WHERE number = $1 AND eth_node_id = $2`,
-		block.Number, blockRepository.database.NodeID)
-	return retrievedBlockHash, blockExists(retrievedBlockHash)
-}
-
-func (blockRepository BlockRepository) createLogs(tx *sqlx.Tx, logs []core.FullSyncLog, receiptID int64) error {
-	for _, tlog := range logs {
-		_, err := tx.Exec(
-			`INSERT INTO full_sync_logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data, receipt_id)
-                VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
-                `,
-			tlog.BlockNumber, tlog.Address, tlog.TxHash, tlog.Index, tlog.Topics[0], tlog.Topics[1], tlog.Topics[2], tlog.Topics[3], tlog.Data, receiptID,
-		)
-		if err != nil {
-			return postgres.ErrDBInsertFailed(err)
-		}
-	}
-	return nil
-}
-
-func blockExists(retrievedBlockHash string) bool {
-	return retrievedBlockHash != ""
-}
-
-func (blockRepository BlockRepository) removeBlock(blockNumber int64) error {
-	_, err := blockRepository.database.Exec(
-		`DELETE FROM eth_blocks WHERE number=$1 AND eth_node_id=$2`,
-		blockNumber, blockRepository.database.NodeID)
-	if err != nil {
-		return postgres.ErrDBDeleteFailed(err)
-	}
-	return nil
-}
-
-func (blockRepository BlockRepository) loadBlock(blockRows *sqlx.Row) (core.Block, error) {
-	type b struct {
-		ID int
-		core.Block
-	}
-	var block b
-	err := blockRows.StructScan(&block)
-	if err != nil {
-		logrus.Error("loadBlock: error loading block: ", err)
-		return core.Block{}, err
-	}
-	transactionRows, err := blockRepository.database.Queryx(`
-		SELECT hash,
-			gas_limit,
-			gas_price,
-			input_data,
-			nonce,
-			raw,
-			tx_from,
-			tx_index,
-			tx_to,
-			value
-		FROM full_sync_transactions
-		WHERE block_id = $1
-		ORDER BY hash`, block.ID)
-	if err != nil {
-		logrus.Error("loadBlock: error fetting transactions: ", err)
-		return core.Block{}, err
-	}
-	block.Transactions = blockRepository.LoadTransactions(transactionRows)
-	return block.Block, nil
-}
-
-func (blockRepository BlockRepository) LoadTransactions(transactionRows *sqlx.Rows) []core.TransactionModel {
-	var transactions []core.TransactionModel
-	for transactionRows.Next() {
-		var transaction core.TransactionModel
-		err := transactionRows.StructScan(&transaction)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-		transactions = append(transactions, transaction)
-	}
-	return transactions
-}

pkg/eth/datastore/postgres/repositories/block_repository_test.go

@@ -1,468 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package repositories_test
-
-import (
-	"bytes"
-	"math/big"
-	"strconv"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/fakes"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres/repositories"
-	"github.com/vulcanize/vulcanizedb/test_config"
-)
-
-var _ = Describe("Saving blocks", func() {
-	var db *postgres.DB
-	var node core.Node
-	var blockRepository datastore.BlockRepository
-
-	BeforeEach(func() {
-		node = core.Node{
-			GenesisBlock: "GENESIS",
-			NetworkID:    1,
-			ID:           "b6f90c0fdd8ec9607aed8ee45c69322e47b7063f0bfb7a29c8ecafab24d0a22d24dd2329b5ee6ed4125a03cb14e57fd584e67f9e53e6c631055cbbd82f080845",
-			ClientName:   "Geth/v1.7.2-stable-1db4ecdc/darwin-amd64/go1.9",
-		}
-		db = test_config.NewTestDB(node)
-		test_config.CleanTestDB(db)
-		blockRepository = repositories.NewBlockRepository(db)
-
-	})
-
-	It("associates blocks to a node", func() {
-		block := core.Block{
-			Number: 123,
-		}
-		_, insertErr := blockRepository.CreateOrUpdateBlock(block)
-		Expect(insertErr).NotTo(HaveOccurred())
-		nodeTwo := core.Node{
-			GenesisBlock: "0x456",
-			NetworkID:    1,
-			ID:           "x123456",
-			ClientName:   "Geth",
-		}
-		dbTwo := test_config.NewTestDB(nodeTwo)
-		test_config.CleanTestDB(dbTwo)
-		repositoryTwo := repositories.NewBlockRepository(dbTwo)
-
-		_, err := repositoryTwo.GetBlock(123)
-		Expect(err).To(HaveOccurred())
-	})
-
-	It("saves the attributes of the block", func() {
-		blockNumber := int64(123)
-		gasLimit := uint64(1000000)
-		gasUsed := uint64(10)
-		blockHash := "x123"
-		blockParentHash := "x456"
-		blockNonce := "0x881db2ca900682e9a9"
-		miner := "x123"
-		extraData := "xextraData"
-		blockTime := uint64(1508981640)
-		uncleHash := "x789"
-		blockSize := string("1000")
-		difficulty := int64(10)
-		blockReward := "5132000000000000000"
-		unclesReward := "3580000000000000000"
-		block := core.Block{
-			Reward:       blockReward,
-			Difficulty:   difficulty,
-			GasLimit:     gasLimit,
-			GasUsed:      gasUsed,
-			Hash:         blockHash,
-			ExtraData:    extraData,
-			Nonce:        blockNonce,
-			Miner:        miner,
-			Number:       blockNumber,
-			ParentHash:   blockParentHash,
-			Size:         blockSize,
-			Time:         uint64(blockTime),
-			UncleHash:    uncleHash,
-			UnclesReward: unclesReward,
-		}
-
-		_, insertErr := blockRepository.CreateOrUpdateBlock(block)
-
-		Expect(insertErr).NotTo(HaveOccurred())
-		savedBlock, err := blockRepository.GetBlock(blockNumber)
-		Expect(err).NotTo(HaveOccurred())
-		Expect(savedBlock.Reward).To(Equal(blockReward))
-		Expect(savedBlock.Difficulty).To(Equal(difficulty))
-		Expect(savedBlock.GasLimit).To(Equal(gasLimit))
-		Expect(savedBlock.GasUsed).To(Equal(gasUsed))
-		Expect(savedBlock.Hash).To(Equal(blockHash))
-		Expect(savedBlock.Nonce).To(Equal(blockNonce))
-		Expect(savedBlock.Miner).To(Equal(miner))
-		Expect(savedBlock.ExtraData).To(Equal(extraData))
-		Expect(savedBlock.Number).To(Equal(blockNumber))
-		Expect(savedBlock.ParentHash).To(Equal(blockParentHash))
-		Expect(savedBlock.Size).To(Equal(blockSize))
-		Expect(savedBlock.Time).To(Equal(blockTime))
-		Expect(savedBlock.UncleHash).To(Equal(uncleHash))
-		Expect(savedBlock.UnclesReward).To(Equal(unclesReward))
-	})
-
-	It("does not find a block when searching for a number that does not exist", func() {
-		_, err := blockRepository.GetBlock(111)
-
-		Expect(err).To(HaveOccurred())
-	})
-
-	It("saves one transaction associated to the block", func() {
-		block := core.Block{
-			Number:       123,
-			Transactions: []core.TransactionModel{fakes.FakeTransaction},
-		}
-
-		_, insertErr := blockRepository.CreateOrUpdateBlock(block)
-
-		Expect(insertErr).NotTo(HaveOccurred())
-		savedBlock, getErr := blockRepository.GetBlock(123)
-		Expect(getErr).NotTo(HaveOccurred())
-		Expect(len(savedBlock.Transactions)).To(Equal(1))
-	})
-
-	It("saves two transactions associated to the block", func() {
-		block := core.Block{
-			Number:       123,
-			Transactions: []core.TransactionModel{fakes.FakeTransaction, fakes.FakeTransaction},
-		}
-
-		_, insertErr := blockRepository.CreateOrUpdateBlock(block)
-
-		Expect(insertErr).NotTo(HaveOccurred())
-		savedBlock, getErr := blockRepository.GetBlock(123)
-		Expect(getErr).NotTo(HaveOccurred())
-		Expect(len(savedBlock.Transactions)).To(Equal(2))
-	})
-
-	It("saves one uncle associated to the block", func() {
-		fakeUncle := fakes.GetFakeUncle(common.BytesToHash([]byte{1, 2, 3}).String(), "100000")
-		block := core.Block{
-			Hash:         fakes.FakeHash.String(),
-			Number:       123,
-			Transactions: []core.TransactionModel{fakes.FakeTransaction},
-			Uncles:       []core.Uncle{fakeUncle},
-			UnclesReward: "156250000000000000",
-		}
-
-		id, insertErr := blockRepository.CreateOrUpdateBlock(block)
-
-		Expect(insertErr).NotTo(HaveOccurred())
-		savedBlock, getErr := blockRepository.GetBlock(123)
-		Expect(getErr).NotTo(HaveOccurred())
-		Expect(len(savedBlock.Transactions)).To(Equal(1))
-		Expect(savedBlock.UnclesReward).To(Equal(big.NewInt(0).Div(big.NewInt(5000000000000000000), big.NewInt(32)).String()))
-
-		var uncleModel core.Uncle
-		err := db.Get(&uncleModel, `SELECT hash, reward, miner, raw, block_timestamp FROM uncles 
-								WHERE block_id = $1 AND hash = $2`, id, common.BytesToHash([]byte{1, 2, 3}).Hex())
-		Expect(err).ToNot(HaveOccurred())
-		Expect(uncleModel.Hash).To(Equal(fakeUncle.Hash))
-		Expect(uncleModel.Reward).To(Equal(fakeUncle.Reward))
-		Expect(uncleModel.Miner).To(Equal(fakeUncle.Miner))
-		Expect(uncleModel.Timestamp).To(Equal(fakeUncle.Timestamp))
-	})
-
-	It("saves two uncles associated to the block", func() {
-		fakeUncleOne := fakes.GetFakeUncle(common.BytesToHash([]byte{1, 2, 3}).String(), "100000")
-		fakeUncleTwo := fakes.GetFakeUncle(common.BytesToHash([]byte{3, 2, 1}).String(), "90000")
-		block := core.Block{
-			Hash:         fakes.FakeHash.String(),
-			Number:       123,
-			Transactions: []core.TransactionModel{fakes.FakeTransaction},
-			Uncles:       []core.Uncle{fakeUncleOne, fakeUncleTwo},
-			UnclesReward: "312500000000000000",
-		}
-
-		id, insertErr := blockRepository.CreateOrUpdateBlock(block)
-
-		Expect(insertErr).NotTo(HaveOccurred())
-		savedBlock, getErr := blockRepository.GetBlock(123)
-		Expect(getErr).NotTo(HaveOccurred())
-		Expect(len(savedBlock.Transactions)).To(Equal(1))
-		b := new(big.Int)
-		b.SetString("10000000000000000000", 10)
-		Expect(savedBlock.UnclesReward).To(Equal(big.NewInt(0).Div(b, big.NewInt(32)).String()))
-
-		var uncleModel core.Uncle
-		err := db.Get(&uncleModel, `SELECT hash, reward, miner, raw, block_timestamp FROM uncles 
-								WHERE block_id = $1 AND hash = $2`, id, common.BytesToHash([]byte{1, 2, 3}).Hex())
-		Expect(err).ToNot(HaveOccurred())
-		Expect(uncleModel.Hash).To(Equal(fakeUncleOne.Hash))
-		Expect(uncleModel.Reward).To(Equal(fakeUncleOne.Reward))
-		Expect(uncleModel.Miner).To(Equal(fakeUncleOne.Miner))
-		Expect(uncleModel.Timestamp).To(Equal(fakeUncleOne.Timestamp))
-
-		err = db.Get(&uncleModel, `SELECT hash, reward, miner, raw, block_timestamp FROM uncles 
-								WHERE block_id = $1 AND hash = $2`, id, common.BytesToHash([]byte{3, 2, 1}).Hex())
-		Expect(err).ToNot(HaveOccurred())
-		Expect(uncleModel.Hash).To(Equal(fakeUncleTwo.Hash))
-		Expect(uncleModel.Reward).To(Equal(fakeUncleTwo.Reward))
-		Expect(uncleModel.Miner).To(Equal(fakeUncleTwo.Miner))
-		Expect(uncleModel.Timestamp).To(Equal(fakeUncleTwo.Timestamp))
-	})
-
-	It(`replaces blocks and transactions associated to the block
-			when a more new block is in conflict (same block number + nodeid)`, func() {
-		blockOne := core.Block{
-			Number: 123,
-			Hash:   "xabc",
-			Transactions: []core.TransactionModel{
-				fakes.GetFakeTransaction("x123", core.Receipt{}),
-				fakes.GetFakeTransaction("x345", core.Receipt{}),
-			},
-		}
-		blockTwo := core.Block{
-			Number: 123,
-			Hash:   "xdef",
-			Transactions: []core.TransactionModel{
-				fakes.GetFakeTransaction("x678", core.Receipt{}),
-				fakes.GetFakeTransaction("x9ab", core.Receipt{}),
-			},
-		}
-
-		_, insertErrOne := blockRepository.CreateOrUpdateBlock(blockOne)
-		Expect(insertErrOne).NotTo(HaveOccurred())
-		_, insertErrTwo := blockRepository.CreateOrUpdateBlock(blockTwo)
-		Expect(insertErrTwo).NotTo(HaveOccurred())
-
-		savedBlock, _ := blockRepository.GetBlock(123)
-		Expect(len(savedBlock.Transactions)).To(Equal(2))
-		Expect(savedBlock.Transactions[0].Hash).To(Equal("x678"))
-		Expect(savedBlock.Transactions[1].Hash).To(Equal("x9ab"))
-	})
-
-	It(`does not replace blocks when block number is not unique
-			     but block number + node id is`, func() {
-		blockOne := core.Block{
-			Number: 123,
-			Transactions: []core.TransactionModel{
-				fakes.GetFakeTransaction("x123", core.Receipt{}),
-				fakes.GetFakeTransaction("x345", core.Receipt{}),
-			},
-		}
-		blockTwo := core.Block{
-			Number: 123,
-			Transactions: []core.TransactionModel{
-				fakes.GetFakeTransaction("x678", core.Receipt{}),
-				fakes.GetFakeTransaction("x9ab", core.Receipt{}),
-			},
-		}
-		_, insertErrOne := blockRepository.CreateOrUpdateBlock(blockOne)
-		Expect(insertErrOne).NotTo(HaveOccurred())
-		nodeTwo := core.Node{
-			GenesisBlock: "0x456",
-			NetworkID:    1,
-		}
-		dbTwo := test_config.NewTestDB(nodeTwo)
-		test_config.CleanTestDB(dbTwo)
-		repositoryTwo := repositories.NewBlockRepository(dbTwo)
-
-		_, insertErrTwo := blockRepository.CreateOrUpdateBlock(blockOne)
-		Expect(insertErrTwo).NotTo(HaveOccurred())
-		_, insertErrThree := repositoryTwo.CreateOrUpdateBlock(blockTwo)
-		Expect(insertErrThree).NotTo(HaveOccurred())
-		retrievedBlockOne, getErrOne := blockRepository.GetBlock(123)
-		Expect(getErrOne).NotTo(HaveOccurred())
-		retrievedBlockTwo, getErrTwo := repositoryTwo.GetBlock(123)
-		Expect(getErrTwo).NotTo(HaveOccurred())
-
-		Expect(retrievedBlockOne.Transactions[0].Hash).To(Equal("x123"))
-		Expect(retrievedBlockTwo.Transactions[0].Hash).To(Equal("x678"))
-	})
-
-	It("returns 'block exists' error if attempting to add duplicate block", func() {
-		block := core.Block{
-			Number: 12345,
-			Hash:   "0x12345",
-		}
-
-		_, err := blockRepository.CreateOrUpdateBlock(block)
-
-		Expect(err).NotTo(HaveOccurred())
-
-		_, err = blockRepository.CreateOrUpdateBlock(block)
-
-		Expect(err).To(HaveOccurred())
-		Expect(err).To(MatchError(repositories.ErrBlockExists))
-	})
-
-	It("saves the attributes associated to a transaction", func() {
-		gasLimit := uint64(5000)
-		gasPrice := int64(3)
-		nonce := uint64(10000)
-		to := "1234567890"
-		from := "0987654321"
-		var value = new(big.Int)
-		value.SetString("34940183920000000000", 10)
-		inputData := "0xf7d8c8830000000000000000000000000000000000000000000000000000000000037788000000000000000000000000000000000000000000000000000000000003bd14"
-		gethTransaction := types.NewTransaction(nonce, common.HexToAddress(to), value, gasLimit, big.NewInt(gasPrice), common.FromHex(inputData))
-		var raw bytes.Buffer
-		rlpErr := gethTransaction.EncodeRLP(&raw)
-		Expect(rlpErr).NotTo(HaveOccurred())
-		transaction := core.TransactionModel{
-			Data:     common.Hex2Bytes(inputData),
-			From:     from,
-			GasLimit: gasLimit,
-			GasPrice: gasPrice,
-			Hash:     "x1234",
-			Nonce:    nonce,
-			Receipt:  core.Receipt{},
-			To:       to,
-			TxIndex:  2,
-			Value:    value.String(),
-			Raw:      []byte{},
-		}
-		block := core.Block{
-			Number:       123,
-			Transactions: []core.TransactionModel{transaction},
-		}
-
-		_, insertErr := blockRepository.CreateOrUpdateBlock(block)
-		Expect(insertErr).NotTo(HaveOccurred())
-
-		savedBlock, err := blockRepository.GetBlock(123)
-		Expect(err).NotTo(HaveOccurred())
-		Expect(len(savedBlock.Transactions)).To(Equal(1))
-		savedTransaction := savedBlock.Transactions[0]
-		Expect(savedTransaction).To(Equal(transaction))
-	})
-
-	Describe("The missing block numbers", func() {
-		It("is empty the starting block number is the highest known block number", func() {
-			_, insertErr := blockRepository.CreateOrUpdateBlock(core.Block{Number: 1})
-
-			Expect(insertErr).NotTo(HaveOccurred())
-			Expect(len(blockRepository.MissingBlockNumbers(1, 1, node.ID))).To(Equal(0))
-		})
-
-		It("is empty if copies of block exist from both current node and another", func() {
-			_, insertErrOne := blockRepository.CreateOrUpdateBlock(core.Block{Number: 0})
-			Expect(insertErrOne).NotTo(HaveOccurred())
-			_, insertErrTwo := blockRepository.CreateOrUpdateBlock(core.Block{Number: 1})
-			Expect(insertErrTwo).NotTo(HaveOccurred())
-			nodeTwo := core.Node{
-				GenesisBlock: "0x456",
-				NetworkID:    1,
-			}
-			dbTwo, err := postgres.NewDB(test_config.DBConfig, nodeTwo)
-			Expect(err).NotTo(HaveOccurred())
-			repositoryTwo := repositories.NewBlockRepository(dbTwo)
-			_, insertErrThree := repositoryTwo.CreateOrUpdateBlock(core.Block{Number: 0})
-			Expect(insertErrThree).NotTo(HaveOccurred())
-
-			missing := blockRepository.MissingBlockNumbers(0, 1, node.ID)
-
-			Expect(len(missing)).To(BeZero())
-		})
-
-		It("is the only missing block number", func() {
-			_, insertErr := blockRepository.CreateOrUpdateBlock(core.Block{Number: 2})
-			Expect(insertErr).NotTo(HaveOccurred())
-
-			Expect(blockRepository.MissingBlockNumbers(1, 2, node.ID)).To(Equal([]int64{1}))
-		})
-
-		It("is both missing block numbers", func() {
-			_, insertErr := blockRepository.CreateOrUpdateBlock(core.Block{Number: 3})
-			Expect(insertErr).NotTo(HaveOccurred())
-
-			Expect(blockRepository.MissingBlockNumbers(1, 3, node.ID)).To(Equal([]int64{1, 2}))
-		})
-
-		It("goes back to the starting block number", func() {
-			_, insertErr := blockRepository.CreateOrUpdateBlock(core.Block{Number: 6})
-			Expect(insertErr).NotTo(HaveOccurred())
-
-			Expect(blockRepository.MissingBlockNumbers(4, 6, node.ID)).To(Equal([]int64{4, 5}))
-		})
-
-		It("only includes missing block numbers", func() {
-			_, insertErrOne := blockRepository.CreateOrUpdateBlock(core.Block{Number: 4})
-			Expect(insertErrOne).NotTo(HaveOccurred())
-			_, insertErrTwo := blockRepository.CreateOrUpdateBlock(core.Block{Number: 6})
-			Expect(insertErrTwo).NotTo(HaveOccurred())
-
-			Expect(blockRepository.MissingBlockNumbers(4, 6, node.ID)).To(Equal([]int64{5}))
-		})
-
-		It("includes blocks created by a different node", func() {
-			_, insertErrOne := blockRepository.CreateOrUpdateBlock(core.Block{Number: 4})
-			Expect(insertErrOne).NotTo(HaveOccurred())
-			_, insertErrTwo := blockRepository.CreateOrUpdateBlock(core.Block{Number: 6})
-			Expect(insertErrTwo).NotTo(HaveOccurred())
-
-			Expect(blockRepository.MissingBlockNumbers(4, 6, "Different node id")).To(Equal([]int64{4, 5, 6}))
-		})
-
-		It("is a list with multiple gaps", func() {
-			_, insertErrOne := blockRepository.CreateOrUpdateBlock(core.Block{Number: 4})
-			Expect(insertErrOne).NotTo(HaveOccurred())
-			_, insertErrTwo := blockRepository.CreateOrUpdateBlock(core.Block{Number: 5})
-			Expect(insertErrTwo).NotTo(HaveOccurred())
-			_, insertErrThree := blockRepository.CreateOrUpdateBlock(core.Block{Number: 8})
-			Expect(insertErrThree).NotTo(HaveOccurred())
-			_, insertErrFour := blockRepository.CreateOrUpdateBlock(core.Block{Number: 10})
-			Expect(insertErrFour).NotTo(HaveOccurred())
-
-			Expect(blockRepository.MissingBlockNumbers(3, 10, node.ID)).To(Equal([]int64{3, 6, 7, 9}))
-		})
-
-		It("returns empty array when lower bound exceeds upper bound", func() {
-			Expect(blockRepository.MissingBlockNumbers(10000, 1, node.ID)).To(Equal([]int64{}))
-		})
-
-		It("only returns requested range even when other gaps exist", func() {
-			_, insertErrOne := blockRepository.CreateOrUpdateBlock(core.Block{Number: 3})
-			Expect(insertErrOne).NotTo(HaveOccurred())
-			_, insertErrTwo := blockRepository.CreateOrUpdateBlock(core.Block{Number: 8})
-			Expect(insertErrTwo).NotTo(HaveOccurred())
-
-			Expect(blockRepository.MissingBlockNumbers(1, 5, node.ID)).To(Equal([]int64{1, 2, 4, 5}))
-		})
-	})
-
-	Describe("The block status", func() {
-		It("sets the status of blocks within n-20 of chain HEAD as final", func() {
-			blockNumberOfChainHead := 25
-			for i := 0; i < blockNumberOfChainHead; i++ {
-				_, err := blockRepository.CreateOrUpdateBlock(core.Block{Number: int64(i), Hash: strconv.Itoa(i)})
-				Expect(err).NotTo(HaveOccurred())
-			}
-
-			setErr := blockRepository.SetBlocksStatus(int64(blockNumberOfChainHead))
-
-			Expect(setErr).NotTo(HaveOccurred())
-			blockOne, err := blockRepository.GetBlock(1)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(blockOne.IsFinal).To(Equal(true))
-			blockTwo, err := blockRepository.GetBlock(24)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(blockTwo.IsFinal).To(BeFalse())
-		})
-	})
-})

pkg/eth/datastore/postgres/repositories/checked_headers_repository.go

@@ -18,7 +18,7 @@ package repositories
 
 import (
 	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/postgres"
 )
 
 const (

pkg/eth/datastore/postgres/repositories/checked_headers_repository_test.go

@@ -23,9 +23,9 @@ import (
 	. "github.com/onsi/gomega"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/fakes"
+	"github.com/vulcanize/vulcanizedb/pkg/postgres"
 	"github.com/vulcanize/vulcanizedb/test_config"
 )
 

pkg/eth/datastore/postgres/repositories/checked_logs_repository.go

@@ -1,69 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package repositories
-
-import (
-	"github.com/sirupsen/logrus"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
-)
-
-type CheckedLogsRepository struct {
-	db *postgres.DB
-}
-
-func NewCheckedLogsRepository(db *postgres.DB) CheckedLogsRepository {
-	return CheckedLogsRepository{db: db}
-}
-
-// Return whether a given address + topic0 has been fetched on a previous run of vDB
-func (repository CheckedLogsRepository) AlreadyWatchingLog(addresses []string, topic0 string) (bool, error) {
-	for _, address := range addresses {
-		var addressExists bool
-		getAddressExistsErr := repository.db.Get(&addressExists, `SELECT EXISTS(SELECT 1 FROM public.watched_logs WHERE contract_address = $1)`, address)
-		if getAddressExistsErr != nil {
-			return false, getAddressExistsErr
-		}
-		if !addressExists {
-			return false, nil
-		}
-	}
-	var topicZeroExists bool
-	getTopicZeroExistsErr := repository.db.Get(&topicZeroExists, `SELECT EXISTS(SELECT 1 FROM public.watched_logs WHERE topic_zero = $1)`, topic0)
-	if getTopicZeroExistsErr != nil {
-		return false, getTopicZeroExistsErr
-	}
-	return topicZeroExists, nil
-}
-
-// Persist that a given address + topic0 has is being fetched on this run of vDB
-func (repository CheckedLogsRepository) MarkLogWatched(addresses []string, topic0 string) error {
-	tx, txErr := repository.db.Beginx()
-	if txErr != nil {
-		return txErr
-	}
-	for _, address := range addresses {
-		_, insertErr := tx.Exec(`INSERT INTO public.watched_logs (contract_address, topic_zero) VALUES ($1, $2)`, address, topic0)
-		if insertErr != nil {
-			rollbackErr := tx.Rollback()
-			if rollbackErr != nil {
-				logrus.Errorf("error rolling back transaction inserting checked logs: %s", rollbackErr.Error())
-			}
-			return insertErr
-		}
-	}
-	return tx.Commit()
-}

pkg/eth/datastore/postgres/repositories/checked_logs_repository_test.go

@@ -1,115 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package repositories_test
-
-import (
-	"github.com/ethereum/go-ethereum/common"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres/repositories"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/fakes"
-	"github.com/vulcanize/vulcanizedb/test_config"
-)
-
-var _ = Describe("Checked logs repository", func() {
-	var (
-		db            *postgres.DB
-		fakeAddress   = fakes.FakeAddress.Hex()
-		fakeAddresses = []string{fakeAddress}
-		fakeTopicZero = fakes.FakeHash.Hex()
-		repository    datastore.CheckedLogsRepository
-	)
-
-	BeforeEach(func() {
-		db = test_config.NewTestDB(test_config.NewTestNode())
-		test_config.CleanTestDB(db)
-		repository = repositories.NewCheckedLogsRepository(db)
-	})
-
-	AfterEach(func() {
-		closeErr := db.Close()
-		Expect(closeErr).NotTo(HaveOccurred())
-	})
-
-	Describe("AlreadyWatchingLog", func() {
-		It("returns true if all addresses and the topic0 are already present in the db", func() {
-			_, insertErr := db.Exec(`INSERT INTO public.watched_logs (contract_address, topic_zero) VALUES ($1, $2)`, fakeAddress, fakeTopicZero)
-			Expect(insertErr).NotTo(HaveOccurred())
-
-			hasBeenChecked, err := repository.AlreadyWatchingLog(fakeAddresses, fakeTopicZero)
-
-			Expect(err).NotTo(HaveOccurred())
-			Expect(hasBeenChecked).To(BeTrue())
-		})
-
-		It("returns true if addresses and topic0 were fetched because of a combination of other transformers", func() {
-			anotherFakeAddress := common.HexToAddress("0x" + fakes.RandomString(40)).Hex()
-			anotherFakeTopicZero := common.HexToHash("0x" + fakes.RandomString(64)).Hex()
-			// insert row with matching address but different topic0
-			_, insertOneErr := db.Exec(`INSERT INTO public.watched_logs (contract_address, topic_zero) VALUES ($1, $2)`, fakeAddress, anotherFakeTopicZero)
-			Expect(insertOneErr).NotTo(HaveOccurred())
-			// insert row with matching topic0 but different address
-			_, insertTwoErr := db.Exec(`INSERT INTO public.watched_logs (contract_address, topic_zero) VALUES ($1, $2)`, anotherFakeAddress, fakeTopicZero)
-			Expect(insertTwoErr).NotTo(HaveOccurred())
-
-			hasBeenChecked, err := repository.AlreadyWatchingLog(fakeAddresses, fakeTopicZero)
-
-			Expect(err).NotTo(HaveOccurred())
-			Expect(hasBeenChecked).To(BeTrue())
-		})
-
-		It("returns false if any address has not been checked", func() {
-			anotherFakeAddress := common.HexToAddress("0x" + fakes.RandomString(40)).Hex()
-			_, insertErr := db.Exec(`INSERT INTO public.watched_logs (contract_address, topic_zero) VALUES ($1, $2)`, fakeAddress, fakeTopicZero)
-			Expect(insertErr).NotTo(HaveOccurred())
-
-			hasBeenChecked, err := repository.AlreadyWatchingLog(append(fakeAddresses, anotherFakeAddress), fakeTopicZero)
-
-			Expect(err).NotTo(HaveOccurred())
-			Expect(hasBeenChecked).To(BeFalse())
-		})
-
-		It("returns false if topic0 has not been checked", func() {
-			anotherFakeTopicZero := common.HexToHash("0x" + fakes.RandomString(64)).Hex()
-			_, insertErr := db.Exec(`INSERT INTO public.watched_logs (contract_address, topic_zero) VALUES ($1, $2)`, fakeAddress, anotherFakeTopicZero)
-			Expect(insertErr).NotTo(HaveOccurred())
-
-			hasBeenChecked, err := repository.AlreadyWatchingLog(fakeAddresses, fakeTopicZero)
-
-			Expect(err).NotTo(HaveOccurred())
-			Expect(hasBeenChecked).To(BeFalse())
-		})
-	})
-
-	Describe("MarkLogWatched", func() {
-		It("adds a row for all of transformer's addresses + topic0", func() {
-			anotherFakeAddress := common.HexToAddress("0x" + fakes.RandomString(40)).Hex()
-			err := repository.MarkLogWatched(append(fakeAddresses, anotherFakeAddress), fakeTopicZero)
-
-			Expect(err).NotTo(HaveOccurred())
-			var comboOneExists, comboTwoExists bool
-			getComboOneErr := db.Get(&comboOneExists, `SELECT EXISTS(SELECT 1 FROM public.watched_logs WHERE contract_address = $1 AND topic_zero = $2)`, fakeAddress, fakeTopicZero)
-			Expect(getComboOneErr).NotTo(HaveOccurred())
-			Expect(comboOneExists).To(BeTrue())
-			getComboTwoErr := db.Get(&comboTwoExists, `SELECT EXISTS(SELECT 1 FROM public.watched_logs WHERE contract_address = $1 AND topic_zero = $2)`, anotherFakeAddress, fakeTopicZero)
-			Expect(getComboTwoErr).NotTo(HaveOccurred())
-			Expect(comboTwoExists).To(BeTrue())
-		})
-	})
-})

pkg/eth/datastore/postgres/repositories/contract_repository.go

@@ -1,99 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package repositories
-
-import (
-	"database/sql"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
-)
-
-type ContractRepository struct {
-	*postgres.DB
-}
-
-func (contractRepository ContractRepository) CreateContract(contract core.Contract) error {
-	abi := contract.Abi
-	var abiToInsert *string
-	if abi != "" {
-		abiToInsert = &abi
-	}
-	_, err := contractRepository.DB.Exec(
-		`INSERT INTO watched_contracts (contract_hash, contract_abi)
-				VALUES ($1, $2)
-				ON CONFLICT (contract_hash)
-				  DO UPDATE
-					SET contract_hash = $1, contract_abi = $2
-				`, contract.Hash, abiToInsert)
-	if err != nil {
-		return postgres.ErrDBInsertFailed(err)
-	}
-	return nil
-}
-
-func (contractRepository ContractRepository) ContractExists(contractHash string) (bool, error) {
-	var exists bool
-	err := contractRepository.DB.QueryRow(
-		`SELECT exists(
-                   SELECT 1
-                   FROM watched_contracts
-                   WHERE contract_hash = $1)`, contractHash).Scan(&exists)
-	if err != nil {
-		return false, err
-	}
-	return exists, nil
-}
-
-func (contractRepository ContractRepository) GetContract(contractHash string) (core.Contract, error) {
-	var hash string
-	var abi string
-	contract := contractRepository.DB.QueryRow(
-		`SELECT contract_hash, contract_abi FROM watched_contracts WHERE contract_hash=$1`, contractHash)
-	err := contract.Scan(&hash, &abi)
-	if err == sql.ErrNoRows {
-		return core.Contract{}, datastore.ErrContractDoesNotExist(contractHash)
-	}
-	savedContract, err := contractRepository.addTransactions(core.Contract{Hash: hash, Abi: abi})
-	if err != nil {
-		return core.Contract{}, err
-	}
-	return savedContract, nil
-}
-
-func (contractRepository ContractRepository) addTransactions(contract core.Contract) (core.Contract, error) {
-	transactionRows, err := contractRepository.DB.Queryx(`
-            SELECT hash,
-                   nonce,
-                   tx_to,
-                   tx_from,
-                   gas_limit,
-                   gas_price,
-                   value,
-                   input_data
-            FROM full_sync_transactions
-            WHERE tx_to = $1
-            ORDER BY block_id DESC`, contract.Hash)
-	if err != nil {
-		return core.Contract{}, err
-	}
-	blockRepository := &BlockRepository{contractRepository.DB}
-	transactions := blockRepository.LoadTransactions(transactionRows)
-	savedContract := core.Contract{Hash: contract.Hash, Transactions: transactions, Abi: contract.Abi}
-	return savedContract, nil
-}

pkg/eth/datastore/postgres/repositories/contract_repository_test.go

@@ -1,122 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package repositories_test
-
-import (
-	"sort"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres/repositories"
-	"github.com/vulcanize/vulcanizedb/test_config"
-)
-
-var _ = Describe("Creating contracts", func() {
-	var db *postgres.DB
-	var contractRepository datastore.ContractRepository
-	var node core.Node
-
-	BeforeEach(func() {
-		node = core.Node{
-			GenesisBlock: "GENESIS",
-			NetworkID:    1,
-			ID:           "b6f90c0fdd8ec9607aed8ee45c69322e47b7063f0bfb7a29c8ecafab24d0a22d24dd2329b5ee6ed4125a03cb14e57fd584e67f9e53e6c631055cbbd82f080845",
-			ClientName:   "Geth/v1.7.2-stable-1db4ecdc/darwin-amd64/go1.9",
-		}
-		db = test_config.NewTestDB(node)
-		test_config.CleanTestDB(db)
-		contractRepository = repositories.ContractRepository{DB: db}
-	})
-
-	It("returns the contract when it exists", func() {
-		contractRepository.CreateContract(core.Contract{Hash: "x123"})
-
-		contract, err := contractRepository.GetContract("x123")
-		Expect(err).NotTo(HaveOccurred())
-		Expect(contract.Hash).To(Equal("x123"))
-
-		Expect(contractRepository.ContractExists("x123")).To(BeTrue())
-		Expect(contractRepository.ContractExists("x456")).To(BeFalse())
-	})
-
-	It("returns err if contract does not exist", func() {
-		_, err := contractRepository.GetContract("x123")
-		Expect(err).To(HaveOccurred())
-	})
-
-	It("returns empty array when no transactions 'To' a contract", func() {
-		contractRepository.CreateContract(core.Contract{Hash: "x123"})
-		contract, err := contractRepository.GetContract("x123")
-		Expect(err).ToNot(HaveOccurred())
-		Expect(contract.Transactions).To(BeEmpty())
-	})
-
-	It("returns transactions 'To' a contract", func() {
-		var blockRepository datastore.BlockRepository
-		blockRepository = repositories.NewBlockRepository(db)
-		block := core.Block{
-			Number: 123,
-			Transactions: []core.TransactionModel{
-				{Hash: "TRANSACTION1", To: "x123", Value: "0"},
-				{Hash: "TRANSACTION2", To: "x345", Value: "0"},
-				{Hash: "TRANSACTION3", To: "x123", Value: "0"},
-			},
-		}
-		_, insertBlockErr := blockRepository.CreateOrUpdateBlock(block)
-		Expect(insertBlockErr).NotTo(HaveOccurred())
-
-		insertContractErr := contractRepository.CreateContract(core.Contract{Hash: "x123"})
-		Expect(insertContractErr).NotTo(HaveOccurred())
-		contract, err := contractRepository.GetContract("x123")
-		Expect(err).ToNot(HaveOccurred())
-		sort.Slice(contract.Transactions, func(i, j int) bool {
-			return contract.Transactions[i].Hash < contract.Transactions[j].Hash
-		})
-		Expect(contract.Transactions).To(
-			Equal([]core.TransactionModel{
-				{Data: []byte{}, Hash: "TRANSACTION1", To: "x123", Value: "0"},
-				{Data: []byte{}, Hash: "TRANSACTION3", To: "x123", Value: "0"},
-			}))
-	})
-
-	It("stores the ABI of the contract", func() {
-		contractRepository.CreateContract(core.Contract{
-			Abi:  "{\"some\": \"json\"}",
-			Hash: "x123",
-		})
-		contract, err := contractRepository.GetContract("x123")
-		Expect(err).ToNot(HaveOccurred())
-		Expect(contract.Abi).To(Equal("{\"some\": \"json\"}"))
-	})
-
-	It("updates the ABI of the contract if hash already present", func() {
-		contractRepository.CreateContract(core.Contract{
-			Abi:  "{\"some\": \"json\"}",
-			Hash: "x123",
-		})
-		contractRepository.CreateContract(core.Contract{
-			Abi:  "{\"some\": \"different json\"}",
-			Hash: "x123",
-		})
-		contract, err := contractRepository.GetContract("x123")
-		Expect(err).ToNot(HaveOccurred())
-		Expect(contract.Abi).To(Equal("{\"some\": \"different json\"}"))
-	})
-})

pkg/eth/datastore/postgres/repositories/full_sync_log_repository.go

@@ -1,106 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package repositories
-
-import (
-	"database/sql"
-
-	"github.com/sirupsen/logrus"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
-)
-
-type FullSyncLogRepository struct {
-	*postgres.DB
-}
-
-func (repository FullSyncLogRepository) CreateLogs(lgs []core.FullSyncLog, receiptID int64) error {
-	tx, _ := repository.DB.Beginx()
-	for _, tlog := range lgs {
-		_, insertLogErr := tx.Exec(
-			`INSERT INTO full_sync_logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data, receipt_id)
-                VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
-                `,
-			tlog.BlockNumber, tlog.Address, tlog.TxHash, tlog.Index, tlog.Topics[0], tlog.Topics[1], tlog.Topics[2], tlog.Topics[3], tlog.Data, receiptID,
-		)
-		if insertLogErr != nil {
-			rollbackErr := tx.Rollback()
-			if rollbackErr != nil {
-				logrus.Error("CreateLogs: could not perform rollback: ", rollbackErr)
-			}
-			return postgres.ErrDBInsertFailed(insertLogErr)
-		}
-	}
-	err := tx.Commit()
-	if err != nil {
-		err = tx.Rollback()
-		if err != nil {
-			logrus.Error("CreateLogs: could not perform rollback: ", err)
-		}
-		return postgres.ErrDBInsertFailed(err)
-	}
-	return nil
-}
-
-func (repository FullSyncLogRepository) GetLogs(address string, blockNumber int64) ([]core.FullSyncLog, error) {
-	logRows, err := repository.DB.Query(
-		`SELECT block_number,
-					  address,
-					  tx_hash,
-					  index,
-					  topic0,
-					  topic1,
-					  topic2,
-					  topic3,
-					  data
-				FROM full_sync_logs
-				WHERE address = $1 AND block_number = $2
-				ORDER BY block_number DESC`, address, blockNumber)
-	if err != nil {
-		return []core.FullSyncLog{}, err
-	}
-	return repository.loadLogs(logRows)
-}
-
-func (repository FullSyncLogRepository) loadLogs(logsRows *sql.Rows) ([]core.FullSyncLog, error) {
-	var lgs []core.FullSyncLog
-	for logsRows.Next() {
-		var blockNumber int64
-		var address string
-		var txHash string
-		var index int64
-		var data string
-		var topics core.Topics
-		err := logsRows.Scan(&blockNumber, &address, &txHash, &index, &topics[0], &topics[1], &topics[2], &topics[3], &data)
-		if err != nil {
-			logrus.Error("loadLogs: Error scanning a row in logRows: ", err)
-			return []core.FullSyncLog{}, err
-		}
-		lg := core.FullSyncLog{
-			BlockNumber: blockNumber,
-			TxHash:      txHash,
-			Address:     address,
-			Index:       index,
-			Data:        data,
-		}
-		for i, topic := range topics {
-			lg.Topics[i] = topic
-		}
-		lgs = append(lgs, lg)
-	}
-	return lgs, nil
-}

pkg/eth/datastore/postgres/repositories/full_sync_log_repository_test.go

@@ -1,221 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package repositories_test
-
-import (
-	"sort"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth/fakes"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres/repositories"
-	"github.com/vulcanize/vulcanizedb/test_config"
-)
-
-var _ = Describe("Full sync log Repository", func() {
-	Describe("Saving logs", func() {
-		var db *postgres.DB
-		var blockRepository datastore.BlockRepository
-		var logsRepository datastore.FullSyncLogRepository
-		var receiptRepository datastore.FullSyncReceiptRepository
-		var node core.Node
-
-		BeforeEach(func() {
-			node = core.Node{
-				GenesisBlock: "GENESIS",
-				NetworkID:    1,
-				ID:           "b6f90c0fdd8ec9607aed8ee45c69322e47b7063f0bfb7a29c8ecafab24d0a22d24dd2329b5ee6ed4125a03cb14e57fd584e67f9e53e6c631055cbbd82f080845",
-				ClientName:   "Geth/v1.7.2-stable-1db4ecdc/darwin-amd64/go1.9",
-			}
-			db = test_config.NewTestDB(node)
-			test_config.CleanTestDB(db)
-			blockRepository = repositories.NewBlockRepository(db)
-			logsRepository = repositories.FullSyncLogRepository{DB: db}
-			receiptRepository = repositories.FullSyncReceiptRepository{DB: db}
-		})
-
-		It("returns the log when it exists", func() {
-			blockNumber := int64(12345)
-			blockId, err := blockRepository.CreateOrUpdateBlock(core.Block{Number: blockNumber})
-			Expect(err).NotTo(HaveOccurred())
-			tx, _ := db.Beginx()
-			receiptId, err := receiptRepository.CreateFullSyncReceiptInTx(blockId, core.Receipt{}, tx)
-			tx.Commit()
-			Expect(err).NotTo(HaveOccurred())
-			err = logsRepository.CreateLogs([]core.FullSyncLog{{
-				BlockNumber: blockNumber,
-				Index:       0,
-				Address:     "x123",
-				TxHash:      "x456",
-				Topics:      core.Topics{0: "x777", 1: "x888", 2: "x999"},
-				Data:        "xabc",
-			}}, receiptId)
-			Expect(err).NotTo(HaveOccurred())
-
-			log, err := logsRepository.GetLogs("x123", blockNumber)
-
-			Expect(err).NotTo(HaveOccurred())
-			Expect(log).NotTo(BeNil())
-			Expect(log[0].BlockNumber).To(Equal(blockNumber))
-			Expect(log[0].Address).To(Equal("x123"))
-			Expect(log[0].Index).To(Equal(int64(0)))
-			Expect(log[0].TxHash).To(Equal("x456"))
-			Expect(log[0].Topics[0]).To(Equal("x777"))
-			Expect(log[0].Topics[1]).To(Equal("x888"))
-			Expect(log[0].Topics[2]).To(Equal("x999"))
-			Expect(log[0].Data).To(Equal("xabc"))
-		})
-
-		It("returns nil if log does not exist", func() {
-			log, err := logsRepository.GetLogs("x123", 1)
-			Expect(err).NotTo(HaveOccurred())
-			Expect(log).To(BeNil())
-		})
-
-		It("filters to the correct block number and address", func() {
-			blockNumber := int64(12345)
-			blockId, err := blockRepository.CreateOrUpdateBlock(core.Block{Number: blockNumber})
-			Expect(err).NotTo(HaveOccurred())
-			tx, _ := db.Beginx()
-			receiptId, err := receiptRepository.CreateFullSyncReceiptInTx(blockId, core.Receipt{}, tx)
-			tx.Commit()
-			Expect(err).NotTo(HaveOccurred())
-
-			err = logsRepository.CreateLogs([]core.FullSyncLog{{
-				BlockNumber: blockNumber,
-				Index:       0,
-				Address:     "x123",
-				TxHash:      "x456",
-				Topics:      core.Topics{0: "x777", 1: "x888", 2: "x999"},
-				Data:        "xabc",
-			}}, receiptId)
-			Expect(err).NotTo(HaveOccurred())
-
-			err = logsRepository.CreateLogs([]core.FullSyncLog{{
-				BlockNumber: blockNumber,
-				Index:       1,
-				Address:     "x123",
-				TxHash:      "x789",
-				Topics:      core.Topics{0: "x111", 1: "x222", 2: "x333"},
-				Data:        "xdef",
-			}}, receiptId)
-			Expect(err).NotTo(HaveOccurred())
-
-			err = logsRepository.CreateLogs([]core.FullSyncLog{{
-				BlockNumber: 2,
-				Index:       0,
-				Address:     "x123",
-				TxHash:      "x456",
-				Topics:      core.Topics{0: "x777", 1: "x888", 2: "x999"},
-				Data:        "xabc",
-			}}, receiptId)
-			Expect(err).NotTo(HaveOccurred())
-
-			log, err := logsRepository.GetLogs("x123", blockNumber)
-			Expect(err).NotTo(HaveOccurred())
-
-			type logIndex struct {
-				blockNumber int64
-				Index       int64
-			}
-
-			var uniqueBlockNumbers []logIndex
-			for _, log := range log {
-				uniqueBlockNumbers = append(uniqueBlockNumbers,
-					logIndex{log.BlockNumber, log.Index})
-			}
-			sort.Slice(uniqueBlockNumbers, func(i, j int) bool {
-				if uniqueBlockNumbers[i].blockNumber < uniqueBlockNumbers[j].blockNumber {
-					return true
-				}
-				if uniqueBlockNumbers[i].blockNumber > uniqueBlockNumbers[j].blockNumber {
-					return false
-				}
-				return uniqueBlockNumbers[i].Index < uniqueBlockNumbers[j].Index
-			})
-
-			Expect(log).NotTo(BeNil())
-			Expect(len(log)).To(Equal(2))
-			Expect(uniqueBlockNumbers).To(Equal(
-				[]logIndex{
-					{blockNumber: blockNumber, Index: 0},
-					{blockNumber: blockNumber, Index: 1}},
-			))
-		})
-
-		It("saves the logs attached to a receipt", func() {
-			logs := []core.FullSyncLog{{
-				Address:     "0x8a4774fe82c63484afef97ca8d89a6ea5e21f973",
-				BlockNumber: 4745407,
-				Data:        "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000645a68669900000000000000000000000000000000000000000000003397684ab5869b0000000000000000000000000000000000000000000000000000000000005a36053200000000000000000000000099041f808d598b782d5a3e498681c2452a31da08",
-				Index:       86,
-				Topics: core.Topics{
-					0: "0x5a68669900000000000000000000000000000000000000000000000000000000",
-					1: "0x000000000000000000000000d0148dad63f73ce6f1b6c607e3413dcf1ff5f030",
-					2: "0x00000000000000000000000000000000000000000000003397684ab5869b0000",
-					3: "0x000000000000000000000000000000000000000000000000000000005a360532",
-				},
-				TxHash: "0x002c4799161d809b23f67884eb6598c9df5894929fe1a9ead97ca175d360f547",
-			}, {
-				Address:     "0x99041f808d598b782d5a3e498681c2452a31da08",
-				BlockNumber: 4745407,
-				Data:        "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000000418178358",
-				Index:       87,
-				Topics: core.Topics{
-					0: "0x1817835800000000000000000000000000000000000000000000000000000000",
-					1: "0x0000000000000000000000008a4774fe82c63484afef97ca8d89a6ea5e21f973",
-					2: "0x0000000000000000000000000000000000000000000000000000000000000000",
-					3: "0x0000000000000000000000000000000000000000000000000000000000000000",
-				},
-				TxHash: "0x002c4799161d809b23f67884eb6598c9df5894929fe1a9ead97ca175d360f547",
-			}, {
-				Address:     "0x99041f808d598b782d5a3e498681c2452a31da08",
-				BlockNumber: 4745407,
-				Data:        "0x00000000000000000000000000000000000000000000003338f64c8423af4000",
-				Index:       88,
-				Topics: core.Topics{
-					0: "0x296ba4ca62c6c21c95e828080cb8aec7481b71390585605300a8a76f9e95b527",
-				},
-				TxHash: "0x002c4799161d809b23f67884eb6598c9df5894929fe1a9ead97ca175d360f547",
-			},
-			}
-			receipt := core.Receipt{
-				ContractAddress:   "",
-				CumulativeGasUsed: 7481414,
-				GasUsed:           60711,
-				Logs:              logs,
-				Bloom:             "0x00000800000000000000001000000000000000400000000080000000000000000000400000010000000000000000000000000000040000000000000002000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000000000800004000000000000001000000000000000000000000000002000000480000000000000002000000000000000020000000000000000000000000000000000000000080000000000180000c00000000000000002000002000000040000000000000000000000000000010000000000020000000000000000000002000000000000000000000000400800000000000000000",
-				Status:            1,
-				TxHash:            "0x002c4799161d809b23f67884eb6598c9df5894929fe1a9ead97ca175d360f547",
-			}
-			transaction := fakes.GetFakeTransaction(receipt.TxHash, receipt)
-
-			block := core.Block{Transactions: []core.TransactionModel{transaction}}
-			_, err := blockRepository.CreateOrUpdateBlock(block)
-			Expect(err).To(Not(HaveOccurred()))
-			retrievedLogs, err := logsRepository.GetLogs("0x99041f808d598b782d5a3e498681c2452a31da08", 4745407)
-
-			Expect(err).NotTo(HaveOccurred())
-			expected := logs[1:]
-			Expect(retrievedLogs).To(ConsistOf(expected))
-		})
-	})
-})

pkg/eth/datastore/postgres/repositories/full_sync_receipt_repository.go

@@ -1,147 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package repositories
-
-import (
-	"database/sql"
-
-	"github.com/jmoiron/sqlx"
-	"github.com/sirupsen/logrus"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/repository"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
-)
-
-type FullSyncReceiptRepository struct {
-	*postgres.DB
-}
-
-func (receiptRepository FullSyncReceiptRepository) CreateReceiptsAndLogs(blockID int64, receipts []core.Receipt) error {
-	tx, err := receiptRepository.DB.Beginx()
-	if err != nil {
-		return err
-	}
-	for _, receipt := range receipts {
-		receiptID, err := receiptRepository.CreateFullSyncReceiptInTx(blockID, receipt, tx)
-		if err != nil {
-			tx.Rollback()
-			return err
-		}
-		if len(receipt.Logs) > 0 {
-			err = createLogs(receipt.Logs, receiptID, tx)
-			if err != nil {
-				tx.Rollback()
-				return err
-			}
-		}
-	}
-	tx.Commit()
-	return nil
-}
-
-func createReceipt(receipt core.Receipt, blockID int64, tx *sqlx.Tx) (int64, error) {
-	var receiptID int64
-	err := tx.QueryRow(
-		`INSERT INTO full_sync_receipts
-		               (contract_address, tx_hash, cumulative_gas_used, gas_used, state_root, status, block_id)
-		               VALUES ($1, $2, $3, $4, $5, $6, $7)
-		               RETURNING id`,
-		receipt.ContractAddress, receipt.TxHash, receipt.CumulativeGasUsed, receipt.GasUsed, receipt.StateRoot, receipt.Status, blockID,
-	).Scan(&receiptID)
-	if err != nil {
-		logrus.Error("createReceipt: Error inserting: ", err)
-	}
-	return receiptID, err
-}
-
-func createLogs(logs []core.FullSyncLog, receiptID int64, tx *sqlx.Tx) error {
-	for _, log := range logs {
-		_, err := tx.Exec(
-			`INSERT INTO full_sync_logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data, receipt_id)
-                VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
-                `,
-			log.BlockNumber, log.Address, log.TxHash, log.Index, log.Topics[0], log.Topics[1], log.Topics[2], log.Topics[3], log.Data, receiptID,
-		)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (FullSyncReceiptRepository) CreateFullSyncReceiptInTx(blockID int64, receipt core.Receipt, tx *sqlx.Tx) (int64, error) {
-	var receiptID int64
-	addressID, getAddressErr := repository.GetOrCreateAddressInTransaction(tx, receipt.ContractAddress)
-	if getAddressErr != nil {
-		logrus.Error("createReceipt: Error getting address id: ", getAddressErr)
-		return receiptID, getAddressErr
-	}
-	err := tx.QueryRow(
-		`INSERT INTO full_sync_receipts
-               (contract_address_id, tx_hash, cumulative_gas_used, gas_used, state_root, status, block_id)
-               VALUES ($1, $2, $3, $4, $5, $6, $7)
-               RETURNING id`,
-		addressID, receipt.TxHash, receipt.CumulativeGasUsed, receipt.GasUsed, receipt.StateRoot, receipt.Status, blockID).Scan(&receiptID)
-	if err != nil {
-		tx.Rollback()
-		logrus.Warning("CreateReceipt: error inserting receipt: ", err)
-		return receiptID, err
-	}
-	return receiptID, nil
-}
-
-func (receiptRepository FullSyncReceiptRepository) GetFullSyncReceipt(txHash string) (core.Receipt, error) {
-	row := receiptRepository.DB.QueryRow(
-		`SELECT contract_address_id,
-                       tx_hash,
-                       cumulative_gas_used,
-                       gas_used,
-                       state_root,
-                       status
-                FROM full_sync_receipts
-                WHERE tx_hash = $1`, txHash)
-	receipt, err := loadReceipt(row)
-	if err != nil {
-		switch err {
-		case sql.ErrNoRows:
-			return core.Receipt{}, datastore.ErrReceiptDoesNotExist(txHash)
-		default:
-			return core.Receipt{}, err
-		}
-	}
-	return receipt, nil
-}
-
-func loadReceipt(receiptsRow *sql.Row) (core.Receipt, error) {
-	var contractAddress string
-	var txHash string
-	var cumulativeGasUsed uint64
-	var gasUsed uint64
-	var stateRoot string
-	var status int
-
-	err := receiptsRow.Scan(&contractAddress, &txHash, &cumulativeGasUsed, &gasUsed, &stateRoot, &status)
-	return core.Receipt{
-		TxHash:            txHash,
-		ContractAddress:   contractAddress,
-		CumulativeGasUsed: cumulativeGasUsed,
-		GasUsed:           gasUsed,
-		StateRoot:         stateRoot,
-		Status:            status,
-	}, err
-}

pkg/eth/datastore/postgres/repositories/full_sync_receipt_repository_test.go

@@ -1,161 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package repositories_test
-
-import (
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres/repositories"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/fakes"
-	"github.com/vulcanize/vulcanizedb/test_config"
-)
-
-var _ = Describe("Receipt Repository", func() {
-	var blockRepository datastore.BlockRepository
-	var logRepository datastore.FullSyncLogRepository
-	var receiptRepository datastore.FullSyncReceiptRepository
-	var db *postgres.DB
-	var node core.Node
-	BeforeEach(func() {
-		node = core.Node{
-			GenesisBlock: "GENESIS",
-			NetworkID:    1,
-			ID:           "b6f90c0fdd8ec9607aed8ee45c69322e47b7063f0bfb7a29c8ecafab24d0a22d24dd2329b5ee6ed4125a03cb14e57fd584e67f9e53e6c631055cbbd82f080845",
-			ClientName:   "Geth/v1.7.2-stable-1db4ecdc/darwin-amd64/go1.9",
-		}
-		db = test_config.NewTestDB(node)
-		test_config.CleanTestDB(db)
-		blockRepository = repositories.NewBlockRepository(db)
-		logRepository = repositories.FullSyncLogRepository{DB: db}
-		receiptRepository = repositories.FullSyncReceiptRepository{DB: db}
-	})
-
-	Describe("Saving multiple receipts", func() {
-		It("persists each receipt and its logs", func() {
-			blockNumber := int64(1234567)
-			blockId, err := blockRepository.CreateOrUpdateBlock(core.Block{Number: blockNumber})
-			Expect(err).NotTo(HaveOccurred())
-			txHashOne := "0xTxHashOne"
-			txHashTwo := "0xTxHashTwo"
-			addressOne := "0xAddressOne"
-			addressTwo := "0xAddressTwo"
-			logsOne := []core.FullSyncLog{{
-				Address:     addressOne,
-				BlockNumber: blockNumber,
-				TxHash:      txHashOne,
-			}, {
-				Address:     addressOne,
-				BlockNumber: blockNumber,
-				TxHash:      txHashOne,
-			}}
-			logsTwo := []core.FullSyncLog{{
-				BlockNumber: blockNumber,
-				TxHash:      txHashTwo,
-				Address:     addressTwo,
-			}}
-			receiptOne := core.Receipt{
-				Logs:   logsOne,
-				TxHash: txHashOne,
-			}
-			receiptTwo := core.Receipt{
-				Logs:   logsTwo,
-				TxHash: txHashTwo,
-			}
-			receipts := []core.Receipt{receiptOne, receiptTwo}
-
-			err = receiptRepository.CreateReceiptsAndLogs(blockId, receipts)
-
-			Expect(err).NotTo(HaveOccurred())
-
-			persistedReceiptOne, err := receiptRepository.GetFullSyncReceipt(txHashOne)
-			Expect(err).NotTo(HaveOccurred())
-			Expect(persistedReceiptOne).NotTo(BeNil())
-			Expect(persistedReceiptOne.TxHash).To(Equal(txHashOne))
-
-			persistedReceiptTwo, err := receiptRepository.GetFullSyncReceipt(txHashTwo)
-			Expect(err).NotTo(HaveOccurred())
-			Expect(persistedReceiptTwo).NotTo(BeNil())
-			Expect(persistedReceiptTwo.TxHash).To(Equal(txHashTwo))
-
-			persistedAddressOneLogs, err := logRepository.GetLogs(addressOne, blockNumber)
-			Expect(err).NotTo(HaveOccurred())
-			Expect(persistedAddressOneLogs).NotTo(BeNil())
-			Expect(len(persistedAddressOneLogs)).To(Equal(2))
-
-			persistedAddressTwoLogs, err := logRepository.GetLogs(addressTwo, blockNumber)
-			Expect(err).NotTo(HaveOccurred())
-			Expect(persistedAddressTwoLogs).NotTo(BeNil())
-			Expect(len(persistedAddressTwoLogs)).To(Equal(1))
-		})
-	})
-
-	Describe("Saving receipts on a block's transactions", func() {
-		It("returns the receipt when it exists", func() {
-			expected := core.Receipt{
-				ContractAddress:   "0xde0b295669a9fd93d5f28d9ec85e40f4cb697bae",
-				CumulativeGasUsed: 7996119,
-				GasUsed:           21000,
-				Logs:              []core.FullSyncLog{},
-				StateRoot:         "0x88abf7e73128227370aa7baa3dd4e18d0af70e92ef1f9ef426942fbe2dddb733",
-				Status:            1,
-				TxHash:            "0xe340558980f89d5f86045ac11e5cc34e4bcec20f9f1e2a427aa39d87114e8223",
-			}
-
-			transaction := fakes.GetFakeTransaction(expected.TxHash, expected)
-			block := core.Block{Transactions: []core.TransactionModel{transaction}}
-
-			_, err := blockRepository.CreateOrUpdateBlock(block)
-
-			Expect(err).NotTo(HaveOccurred())
-			receipt, err := receiptRepository.GetFullSyncReceipt("0xe340558980f89d5f86045ac11e5cc34e4bcec20f9f1e2a427aa39d87114e8223")
-			Expect(err).ToNot(HaveOccurred())
-			//Not currently serializing bloom logs
-			Expect(receipt.Bloom).To(Equal(core.Receipt{}.Bloom))
-			Expect(receipt.TxHash).To(Equal(expected.TxHash))
-			Expect(receipt.CumulativeGasUsed).To(Equal(expected.CumulativeGasUsed))
-			Expect(receipt.GasUsed).To(Equal(expected.GasUsed))
-			Expect(receipt.StateRoot).To(Equal(expected.StateRoot))
-			Expect(receipt.Status).To(Equal(expected.Status))
-		})
-
-		It("returns ErrReceiptDoesNotExist when receipt does not exist", func() {
-			receipt, err := receiptRepository.GetFullSyncReceipt("DOES NOT EXIST")
-			Expect(err).To(HaveOccurred())
-			Expect(receipt).To(BeZero())
-		})
-
-		It("still saves receipts without logs", func() {
-			receipt := core.Receipt{
-				TxHash: "0x002c4799161d809b23f67884eb6598c9df5894929fe1a9ead97ca175d360f547",
-			}
-			transaction := fakes.GetFakeTransaction(receipt.TxHash, receipt)
-
-			block := core.Block{
-				Transactions: []core.TransactionModel{transaction},
-			}
-
-			_, err := blockRepository.CreateOrUpdateBlock(block)
-
-			Expect(err).NotTo(HaveOccurred())
-			_, err = receiptRepository.GetFullSyncReceipt(receipt.TxHash)
-			Expect(err).To(Not(HaveOccurred()))
-		})
-	})
-})

pkg/eth/datastore/postgres/repositories/header_repository.go

@@ -24,7 +24,7 @@ import (
 	log "github.com/sirupsen/logrus"
 
 	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/postgres"
 )
 
 var ErrValidHeaderExists = errors.New("valid header already exists")
@@ -134,7 +134,7 @@ func (repository HeaderRepository) getHeaderHash(header core.Header) (string, er
 func (repository HeaderRepository) InternalInsertHeader(header core.Header) (int64, error) {
 	var headerID int64
 	row := repository.database.QueryRowx(
-		`INSERT INTO public.headers (block_number, hash, block_timestamp, raw, eth_node_id, eth_node_fingerprint)
+		`INSERT INTO public.headers (block_number, hash, block_timestamp, raw, node_id, eth_node_fingerprint)
 		VALUES ($1, $2, $3::NUMERIC, $4, $5, $6) ON CONFLICT DO NOTHING RETURNING id`,
 		header.BlockNumber, header.Hash, header.Timestamp, header.Raw, repository.database.NodeID, repository.database.Node.ID)
 	err := row.Scan(&headerID)

pkg/eth/datastore/postgres/repositories/header_repository_test.go

@@ -27,8 +27,8 @@ import (
 	. "github.com/onsi/gomega"
 
 	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres/repositories"
+	"github.com/vulcanize/vulcanizedb/pkg/postgres"
 	"github.com/vulcanize/vulcanizedb/test_config"
 )
 
@@ -75,7 +75,7 @@ var _ = Describe("Block header repository", func() {
 			_, err = repo.CreateOrUpdateHeader(header)
 			Expect(err).NotTo(HaveOccurred())
 			var ethNodeId int64
-			err = db.Get(&ethNodeId, `SELECT eth_node_id FROM public.headers WHERE block_number = $1`, header.BlockNumber)
+			err = db.Get(&ethNodeId, `SELECT node_id FROM public.headers WHERE block_number = $1`, header.BlockNumber)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(ethNodeId).To(Equal(db.NodeID))
 			var ethNodeFingerprint string

pkg/eth/datastore/postgres/repositories/header_sync_log_repository.go

@@ -24,7 +24,7 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/repository"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/postgres"
 )
 
 const insertHeaderSyncLogQuery = `INSERT INTO header_sync_logs

pkg/eth/datastore/postgres/repositories/header_sync_log_repository_test.go

@@ -25,9 +25,9 @@ import (
 	repository2 "github.com/vulcanize/vulcanizedb/libraries/shared/repository"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/test_data"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/fakes"
+	"github.com/vulcanize/vulcanizedb/pkg/postgres"
 	"github.com/vulcanize/vulcanizedb/test_config"
 )
 

pkg/eth/datastore/postgres/repositories/header_sync_receipt_repository_test.go

@@ -26,8 +26,8 @@ import (
 	. "github.com/onsi/gomega"
 
 	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres/repositories"
+	"github.com/vulcanize/vulcanizedb/pkg/postgres"
 	"github.com/vulcanize/vulcanizedb/test_config"
 )
 

pkg/eth/datastore/postgres/repositories/log_filter_repository.go

@@ -1,92 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package repositories
-
-import (
-	"database/sql"
-
-	"encoding/json"
-	"errors"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/filters"
-)
-
-type FilterRepository struct {
-	*postgres.DB
-}
-
-func (filterRepository FilterRepository) CreateFilter(query filters.LogFilter) error {
-	_, err := filterRepository.DB.Exec(
-		`INSERT INTO log_filters 
-        (name, from_block, to_block, address, topic0, topic1, topic2, topic3)
-        VALUES ($1, NULLIF($2, -1), NULLIF($3, -1), $4, NULLIF($5, ''), NULLIF($6, ''), NULLIF($7, ''), NULLIF($8, ''))`,
-		query.Name, query.FromBlock, query.ToBlock, query.Address, query.Topics[0], query.Topics[1], query.Topics[2], query.Topics[3])
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func (filterRepository FilterRepository) GetFilter(name string) (filters.LogFilter, error) {
-	lf := DBLogFilter{}
-	err := filterRepository.DB.Get(&lf,
-		`SELECT
-                  id,
-                  name,
-                  from_block,
-                  to_block,
-                  address,
-                  json_build_array(topic0, topic1, topic2, topic3) AS topics
-                FROM log_filters
-                WHERE name = $1`, name)
-	if err != nil {
-		switch err {
-		case sql.ErrNoRows:
-			return filters.LogFilter{}, datastore.ErrFilterDoesNotExist(name)
-		default:
-			return filters.LogFilter{}, err
-		}
-	}
-	dbLogFilterToCoreLogFilter(lf)
-	return *lf.LogFilter, nil
-}
-
-type DBTopics []*string
-
-func (t *DBTopics) Scan(src interface{}) error {
-	asBytes, ok := src.([]byte)
-	if !ok {
-		return error(errors.New("scan source was not []byte"))
-	}
-	return json.Unmarshal(asBytes, &t)
-}
-
-type DBLogFilter struct {
-	ID int
-	*filters.LogFilter
-	Topics DBTopics
-}
-
-func dbLogFilterToCoreLogFilter(lf DBLogFilter) {
-	for i, v := range lf.Topics {
-		if v != nil {
-			lf.LogFilter.Topics[i] = *v
-		}
-	}
-}

pkg/eth/datastore/postgres/repositories/log_filter_repository_test.go

@@ -1,127 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package repositories_test
-
-import (
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres/repositories"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/filters"
-	"github.com/vulcanize/vulcanizedb/test_config"
-)
-
-var _ = Describe("Log Filters Repository", func() {
-	var db *postgres.DB
-	var filterRepository datastore.FilterRepository
-	var node core.Node
-	BeforeEach(func() {
-		node = core.Node{
-			GenesisBlock: "GENESIS",
-			NetworkID:    1,
-			ID:           "b6f90c0fdd8ec9607aed8ee45c69322e47b7063f0bfb7a29c8ecafab24d0a22d24dd2329b5ee6ed4125a03cb14e57fd584e67f9e53e6c631055cbbd82f080845",
-			ClientName:   "Geth/v1.7.2-stable-1db4ecdc/darwin-amd64/go1.9",
-		}
-		db = test_config.NewTestDB(node)
-		test_config.CleanTestDB(db)
-		filterRepository = repositories.FilterRepository{DB: db}
-	})
-
-	Describe("LogFilter", func() {
-
-		It("inserts filter into watched events", func() {
-
-			logFilter := filters.LogFilter{
-				Name:      "TestFilter",
-				FromBlock: 1,
-				ToBlock:   2,
-				Address:   "0x8888f1f195afa192cfee860698584c030f4c9db1",
-				Topics: core.Topics{
-					"0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b",
-					"",
-					"0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b",
-					"",
-				},
-			}
-			err := filterRepository.CreateFilter(logFilter)
-			Expect(err).ToNot(HaveOccurred())
-		})
-
-		It("returns error if name is not provided", func() {
-
-			logFilter := filters.LogFilter{
-				FromBlock: 1,
-				ToBlock:   2,
-				Address:   "0x8888f1f195afa192cfee860698584c030f4c9db1",
-				Topics: core.Topics{
-					"0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b",
-					"",
-					"0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b",
-					"",
-				},
-			}
-			err := filterRepository.CreateFilter(logFilter)
-			Expect(err).To(HaveOccurred())
-		})
-
-		It("gets a log filter", func() {
-
-			expectedLogFilter1 := filters.LogFilter{
-				Name:      "TestFilter1",
-				FromBlock: 1,
-				ToBlock:   2,
-				Address:   "0x8888f1f195afa192cfee860698584c030f4c9db1",
-				Topics: core.Topics{
-					"0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b",
-					"",
-					"0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b",
-					"",
-				},
-			}
-			err := filterRepository.CreateFilter(expectedLogFilter1)
-			Expect(err).ToNot(HaveOccurred())
-			expectedLogFilter2 := filters.LogFilter{
-				Name:      "TestFilter2",
-				FromBlock: 10,
-				ToBlock:   20,
-				Address:   "0x8888f1f195afa192cfee860698584c030f4c9db1",
-				Topics: core.Topics{
-					"0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b",
-					"",
-					"0x000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b",
-					"",
-				},
-			}
-			err = filterRepository.CreateFilter(expectedLogFilter2)
-			Expect(err).ToNot(HaveOccurred())
-
-			logFilter1, err := filterRepository.GetFilter("TestFilter1")
-			Expect(err).ToNot(HaveOccurred())
-			Expect(logFilter1).To(Equal(expectedLogFilter1))
-			logFilter2, err := filterRepository.GetFilter("TestFilter2")
-			Expect(err).ToNot(HaveOccurred())
-			Expect(logFilter2).To(Equal(expectedLogFilter2))
-		})
-
-		It("returns ErrFilterDoesNotExist error when log does not exist", func() {
-			_, err := filterRepository.GetFilter("TestFilter1")
-			Expect(err).To(Equal(datastore.ErrFilterDoesNotExist("TestFilter1")))
-		})
-	})
-})

pkg/eth/datastore/postgres/repositories/storage_diff_repository.go

@@ -20,7 +20,7 @@ import (
 	"database/sql"
 
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/postgres"
 )
 
 var ErrDuplicateDiff = sql.ErrNoRows

pkg/eth/datastore/postgres/repositories/storage_diff_repository_test.go

@@ -24,8 +24,8 @@ import (
 	. "github.com/onsi/gomega"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/test_data"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres/repositories"
+	"github.com/vulcanize/vulcanizedb/pkg/postgres"
 	"github.com/vulcanize/vulcanizedb/test_config"
 )
 

pkg/eth/datastore/postgres/repositories/watched_events_repository.go

@@ -1,52 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package repositories
-
-import (
-	"github.com/sirupsen/logrus"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
-)
-
-type WatchedEventRepository struct {
-	*postgres.DB
-}
-
-func (watchedEventRepository WatchedEventRepository) GetWatchedEvents(name string) ([]*core.WatchedEvent, error) {
-	rows, err := watchedEventRepository.DB.Queryx(`SELECT id, name, block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data FROM watched_event_logs where name=$1`, name)
-	if err != nil {
-		logrus.Error("GetWatchedEvents: error getting watched events: ", err)
-		return nil, err
-	}
-	defer rows.Close()
-
-	lgs := make([]*core.WatchedEvent, 0)
-	for rows.Next() {
-		lg := new(core.WatchedEvent)
-		err = rows.StructScan(lg)
-		if err != nil {
-			logrus.Warn("GetWatchedEvents: error scanning log: ", err)
-			return nil, err
-		}
-		lgs = append(lgs, lg)
-	}
-	if err = rows.Err(); err != nil {
-		logrus.Warn("GetWatchedEvents: error scanning logs: ", err)
-		return nil, err
-	}
-	return lgs, nil
-}

pkg/eth/datastore/postgres/repositories/watched_events_repository_test.go

@@ -1,161 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package repositories_test
-
-import (
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore/postgres/repositories"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/filters"
-	"github.com/vulcanize/vulcanizedb/test_config"
-)
-
-var _ = Describe("Watched Events Repository", func() {
-	var db *postgres.DB
-	var blocksRepository datastore.BlockRepository
-	var filterRepository datastore.FilterRepository
-	var logRepository datastore.FullSyncLogRepository
-	var receiptRepository datastore.FullSyncReceiptRepository
-	var watchedEventRepository datastore.WatchedEventRepository
-
-	BeforeEach(func() {
-		db = test_config.NewTestDB(core.Node{})
-		test_config.CleanTestDB(db)
-		blocksRepository = repositories.NewBlockRepository(db)
-		filterRepository = repositories.FilterRepository{DB: db}
-		logRepository = repositories.FullSyncLogRepository{DB: db}
-		receiptRepository = repositories.FullSyncReceiptRepository{DB: db}
-		watchedEventRepository = repositories.WatchedEventRepository{DB: db}
-	})
-
-	It("retrieves watched event logs that match the event filter", func() {
-		filter := filters.LogFilter{
-			Name:      "Filter1",
-			FromBlock: 0,
-			ToBlock:   10,
-			Address:   "0x123",
-			Topics:    core.Topics{0: "event1=10", 2: "event3=hello"},
-		}
-		logs := []core.FullSyncLog{
-			{
-				BlockNumber: 0,
-				TxHash:      "0x1",
-				Address:     "0x123",
-				Topics:      core.Topics{0: "event1=10", 2: "event3=hello"},
-				Index:       0,
-				Data:        "",
-			},
-		}
-		expectedWatchedEventLog := []*core.WatchedEvent{
-			{
-				Name:        "Filter1",
-				BlockNumber: 0,
-				TxHash:      "0x1",
-				Address:     "0x123",
-				Topic0:      "event1=10",
-				Topic2:      "event3=hello",
-				Index:       0,
-				Data:        "",
-			},
-		}
-		err := filterRepository.CreateFilter(filter)
-		Expect(err).ToNot(HaveOccurred())
-		blockId, err := blocksRepository.CreateOrUpdateBlock(core.Block{})
-		Expect(err).NotTo(HaveOccurred())
-		tx, txBeginErr := db.Beginx()
-		Expect(txBeginErr).NotTo(HaveOccurred())
-		receiptId, err := receiptRepository.CreateFullSyncReceiptInTx(blockId, core.Receipt{}, tx)
-		tx.Commit()
-		Expect(err).NotTo(HaveOccurred())
-		err = logRepository.CreateLogs(logs, receiptId)
-		Expect(err).ToNot(HaveOccurred())
-		matchingLogs, err := watchedEventRepository.GetWatchedEvents("Filter1")
-		Expect(err).ToNot(HaveOccurred())
-		Expect(len(matchingLogs)).To(Equal(1))
-		Expect(matchingLogs[0].Name).To(Equal(expectedWatchedEventLog[0].Name))
-		Expect(matchingLogs[0].BlockNumber).To(Equal(expectedWatchedEventLog[0].BlockNumber))
-		Expect(matchingLogs[0].TxHash).To(Equal(expectedWatchedEventLog[0].TxHash))
-		Expect(matchingLogs[0].Address).To(Equal(expectedWatchedEventLog[0].Address))
-		Expect(matchingLogs[0].Topic0).To(Equal(expectedWatchedEventLog[0].Topic0))
-		Expect(matchingLogs[0].Topic1).To(Equal(expectedWatchedEventLog[0].Topic1))
-		Expect(matchingLogs[0].Topic2).To(Equal(expectedWatchedEventLog[0].Topic2))
-		Expect(matchingLogs[0].Data).To(Equal(expectedWatchedEventLog[0].Data))
-	})
-
-	It("retrieves a watched event log by name", func() {
-		filter := filters.LogFilter{
-			Name:      "Filter1",
-			FromBlock: 0,
-			ToBlock:   10,
-			Address:   "0x123",
-			Topics:    core.Topics{0: "event1=10", 2: "event3=hello"},
-		}
-		logs := []core.FullSyncLog{
-			{
-				BlockNumber: 0,
-				TxHash:      "0x1",
-				Address:     "0x123",
-				Topics:      core.Topics{0: "event1=10", 2: "event3=hello"},
-				Index:       0,
-				Data:        "",
-			},
-			{
-				BlockNumber: 100,
-				TxHash:      "",
-				Address:     "",
-				Topics:      core.Topics{},
-				Index:       0,
-				Data:        "",
-			},
-		}
-		expectedWatchedEventLog := []*core.WatchedEvent{{
-			Name:        "Filter1",
-			BlockNumber: 0,
-			TxHash:      "0x1",
-			Address:     "0x123",
-			Topic0:      "event1=10",
-			Topic2:      "event3=hello",
-			Index:       0,
-			Data:        "",
-		}}
-		err := filterRepository.CreateFilter(filter)
-		Expect(err).ToNot(HaveOccurred())
-		blockId, err := blocksRepository.CreateOrUpdateBlock(core.Block{Hash: "Ox123"})
-		Expect(err).NotTo(HaveOccurred())
-		tx, _ := db.Beginx()
-		receiptId, err := receiptRepository.CreateFullSyncReceiptInTx(blockId, core.Receipt{}, tx)
-		tx.Commit()
-		Expect(err).NotTo(HaveOccurred())
-		err = logRepository.CreateLogs(logs, receiptId)
-		Expect(err).ToNot(HaveOccurred())
-		matchingLogs, err := watchedEventRepository.GetWatchedEvents("Filter1")
-		Expect(err).ToNot(HaveOccurred())
-		Expect(len(matchingLogs)).To(Equal(1))
-		Expect(matchingLogs[0].Name).To(Equal(expectedWatchedEventLog[0].Name))
-		Expect(matchingLogs[0].BlockNumber).To(Equal(expectedWatchedEventLog[0].BlockNumber))
-		Expect(matchingLogs[0].TxHash).To(Equal(expectedWatchedEventLog[0].TxHash))
-		Expect(matchingLogs[0].Address).To(Equal(expectedWatchedEventLog[0].Address))
-		Expect(matchingLogs[0].Topic0).To(Equal(expectedWatchedEventLog[0].Topic0))
-		Expect(matchingLogs[0].Topic1).To(Equal(expectedWatchedEventLog[0].Topic1))
-		Expect(matchingLogs[0].Topic2).To(Equal(expectedWatchedEventLog[0].Topic2))
-		Expect(matchingLogs[0].Data).To(Equal(expectedWatchedEventLog[0].Data))
-	})
-})

pkg/eth/datastore/repository.go

@@ -29,7 +29,6 @@ type AddressRepository interface {
 }
 
 type BlockRepository interface {
-	CreateOrUpdateBlock(block core.Block) (int64, error)
 	GetBlock(blockNumber int64) (core.Block, error)
 	MissingBlockNumbers(startingBlockNumber, endingBlockNumber int64, nodeID string) []int64
 	SetBlocksStatus(chainHead int64) error

pkg/eth/fakes/mock_blockchain.go

@@ -52,7 +52,7 @@ type MockBlockChain struct {
 
 func NewMockBlockChain() *MockBlockChain {
 	return &MockBlockChain{
-		node: core.Node{GenesisBlock: "GENESIS", NetworkID: 1, ID: "x123", ClientName: "Geth"},
+		node: core.Node{GenesisBlock: "GENESIS", NetworkID: "1", ID: "x123", ClientName: "Geth"},
 	}
 }
 

pkg/eth/history/block_validator.go

@@ -1,65 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package history
-
-import (
-	"github.com/sirupsen/logrus"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore"
-)
-
-type BlockValidator struct {
-	blockchain      core.BlockChain
-	blockRepository datastore.BlockRepository
-	windowSize      int
-}
-
-func NewBlockValidator(blockchain core.BlockChain, blockRepository datastore.BlockRepository, windowSize int) *BlockValidator {
-	return &BlockValidator{
-		blockchain:      blockchain,
-		blockRepository: blockRepository,
-		windowSize:      windowSize,
-	}
-}
-
-func (bv BlockValidator) ValidateBlocks() (ValidationWindow, error) {
-	window, err := MakeValidationWindow(bv.blockchain, bv.windowSize)
-	if err != nil {
-		logrus.Error("ValidateBlocks: error creating validation window: ", err)
-		return ValidationWindow{}, err
-	}
-
-	blockNumbers := MakeRange(window.LowerBound, window.UpperBound)
-	_, err = RetrieveAndUpdateBlocks(bv.blockchain, bv.blockRepository, blockNumbers)
-	if err != nil {
-		logrus.Error("ValidateBlocks: error getting and updating blocks: ", err)
-		return ValidationWindow{}, err
-	}
-
-	lastBlock, err := bv.blockchain.LastBlock()
-	if err != nil {
-		logrus.Error("ValidateBlocks: error getting last block: ", err)
-		return ValidationWindow{}, err
-	}
-
-	err = bv.blockRepository.SetBlocksStatus(lastBlock.Int64())
-	if err != nil {
-		logrus.Error("ValidateBlocks: error setting block status: ", err)
-		return ValidationWindow{}, err
-	}
-	return window, nil
-}

pkg/eth/history/block_validator_test.go

@@ -1,51 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package history_test
-
-import (
-	"math/big"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth/fakes"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/history"
-)
-
-var _ = Describe("Blocks validator", func() {
-
-	It("calls create or update for all blocks within the window", func() {
-		blockChain := fakes.NewMockBlockChain()
-		blockChain.SetLastBlock(big.NewInt(7))
-		blocksRepository := fakes.NewMockBlockRepository()
-		validator := history.NewBlockValidator(blockChain, blocksRepository, 2)
-
-		window, err := validator.ValidateBlocks()
-		Expect(err).NotTo(HaveOccurred())
-
-		Expect(window).To(Equal(history.ValidationWindow{LowerBound: 5, UpperBound: 7}))
-		blocksRepository.AssertCreateOrUpdateBlockCallCountEquals(3)
-	})
-
-	It("returns the number of largest block", func() {
-		blockChain := fakes.NewMockBlockChain()
-		blockChain.SetLastBlock(big.NewInt(3))
-		maxBlockNumber, _ := blockChain.LastBlock()
-
-		Expect(maxBlockNumber.Int64()).To(Equal(int64(3)))
-	})
-})

pkg/eth/history/populate_blocks.go

@@ -1,69 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package history
-
-import (
-	"fmt"
-
-	log "github.com/sirupsen/logrus"
-
-	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/datastore"
-)
-
-func PopulateMissingBlocks(blockchain core.BlockChain, blockRepository datastore.BlockRepository, startingBlockNumber int64) (int, error) {
-	lastBlock, err := blockchain.LastBlock()
-	if err != nil {
-		log.Error("PopulateMissingBlocks: error getting last block: ", err)
-		return 0, err
-	}
-	blockRange := blockRepository.MissingBlockNumbers(startingBlockNumber, lastBlock.Int64(), blockchain.Node().ID)
-
-	if len(blockRange) == 0 {
-		return 0, nil
-	}
-
-	log.Debug(getBlockRangeString(blockRange))
-	_, err = RetrieveAndUpdateBlocks(blockchain, blockRepository, blockRange)
-	if err != nil {
-		log.Error("PopulateMissingBlocks: error gettings/updating blocks: ", err)
-		return 0, err
-	}
-	return len(blockRange), nil
-}
-
-func RetrieveAndUpdateBlocks(blockchain core.BlockChain, blockRepository datastore.BlockRepository, blockNumbers []int64) (int, error) {
-	for _, blockNumber := range blockNumbers {
-		block, err := blockchain.GetBlockByNumber(blockNumber)
-		if err != nil {
-			log.Error("RetrieveAndUpdateBlocks: error getting block: ", err)
-			return 0, err
-		}
-
-		_, err = blockRepository.CreateOrUpdateBlock(block)
-		if err != nil {
-			log.Error("RetrieveAndUpdateBlocks: error creating/updating block: ", err)
-			return 0, err
-		}
-
-	}
-	return len(blockNumbers), nil
-}
-
-func getBlockRangeString(blockRange []int64) string {
-	return fmt.Sprintf("Backfilling |%v| blocks", len(blockRange))
-}

pkg/eth/history/populate_blocks_test.go

@@ -1,90 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package history_test
-
-import (
-	"math/big"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/fakes"
-	"github.com/vulcanize/vulcanizedb/pkg/eth/history"
-)
-
-var _ = Describe("Populating blocks", func() {
-	var blockRepository *fakes.MockBlockRepository
-
-	BeforeEach(func() {
-		blockRepository = fakes.NewMockBlockRepository()
-	})
-
-	It("fills in the only missing block (BlockNumber 1)", func() {
-		blockChain := fakes.NewMockBlockChain()
-		blockChain.SetLastBlock(big.NewInt(2))
-		blockRepository.SetMissingBlockNumbersReturnArray([]int64{2})
-
-		blocksAdded, err := history.PopulateMissingBlocks(blockChain, blockRepository, 1)
-		Expect(err).NotTo(HaveOccurred())
-		_, err = blockRepository.GetBlock(1)
-
-		Expect(blocksAdded).To(Equal(1))
-		Expect(err).ToNot(HaveOccurred())
-	})
-
-	It("fills in the three missing blocks (Numbers: 5,8,10)", func() {
-		blockChain := fakes.NewMockBlockChain()
-		blockChain.SetLastBlock(big.NewInt(13))
-		blockRepository.SetMissingBlockNumbersReturnArray([]int64{5, 8, 10})
-
-		blocksAdded, err := history.PopulateMissingBlocks(blockChain, blockRepository, 5)
-
-		Expect(err).NotTo(HaveOccurred())
-		Expect(blocksAdded).To(Equal(3))
-		blockRepository.AssertCreateOrUpdateBlocksCallCountAndBlockNumbersEquals(3, []int64{5, 8, 10})
-	})
-
-	It("returns the number of blocks created", func() {
-		blockChain := fakes.NewMockBlockChain()
-		blockChain.SetLastBlock(big.NewInt(6))
-		blockRepository.SetMissingBlockNumbersReturnArray([]int64{4, 5})
-
-		numberOfBlocksCreated, err := history.PopulateMissingBlocks(blockChain, blockRepository, 3)
-
-		Expect(err).NotTo(HaveOccurred())
-		Expect(numberOfBlocksCreated).To(Equal(2))
-	})
-
-	It("updates the repository with a range of blocks w/in the range ", func() {
-		blockChain := fakes.NewMockBlockChain()
-
-		_, err := history.RetrieveAndUpdateBlocks(blockChain, blockRepository, history.MakeRange(2, 5))
-
-		Expect(err).NotTo(HaveOccurred())
-		blockRepository.AssertCreateOrUpdateBlocksCallCountAndBlockNumbersEquals(4, []int64{2, 3, 4, 5})
-	})
-
-	It("does not call repository create block when there is an error", func() {
-		blockChain := fakes.NewMockBlockChain()
-		blockChain.SetGetBlockByNumberErr(fakes.FakeError)
-		blocks := history.MakeRange(1, 10)
-
-		_, err := history.RetrieveAndUpdateBlocks(blockChain, blockRepository, blocks)
-
-		Expect(err).To(HaveOccurred())
-		blockRepository.AssertCreateOrUpdateBlockCallCountEquals(0)
-	})
-})

pkg/eth/history/populate_headers.go

@@ -17,6 +17,7 @@
 package history
 
 import (
+	"fmt"
 	"github.com/sirupsen/logrus"
 
 	"github.com/vulcanize/vulcanizedb/pkg/eth/core"
@@ -61,3 +62,7 @@ func RetrieveAndUpdateHeaders(blockChain core.BlockChain, headerRepository datas
 	}
 	return len(blockNumbers), nil
 }
+
+func getBlockRangeString(blockRange []int64) string {
+	return fmt.Sprintf("Backfilling |%v| blocks", len(blockRange))
+}

pkg/eth/node/node.go

@@ -18,6 +18,7 @@ package node
 
 import (
 	"context"
+	"fmt"
 	"regexp"
 	"strconv"
 	"strings"
@@ -60,7 +61,7 @@ func MakeNode(rpcClient core.RPCClient) core.Node {
 	id, name := pr.NodeInfo()
 	return core.Node{
 		GenesisBlock: pr.GenesisBlock(),
-		NetworkID:    pr.NetworkID(),
+		NetworkID:    fmt.Sprintf("%f", pr.NetworkID()),
 		ID:           id,
 		ClientName:   name,
 	}

pkg/eth/node/node_test.go

@@ -82,7 +82,7 @@ var _ = Describe("Node Info", func() {
 	It("returns the network id for any client", func() {
 		client := fakes.NewMockRPCClient()
 		n := node.MakeNode(client)
-		Expect(n.NetworkID).To(Equal(float64(1234)))
+		Expect(n.NetworkID).To(Equal("1234.000000"))
 	})
 
 	It("returns geth ID and client name for geth node", func() {