remove option for publishing through ipfs node interface

pkg/eth/api_test.go

@@ -84,8 +84,8 @@ var _ = Describe("API", func() {
 	var (
 		db                *postgres.DB
 		retriever         *eth.CIDRetriever
-		fetcher           *eth.IPLDPGFetcher
-		indexAndPublisher *eth.IPLDPublisherAndIndexer
+		fetcher           *eth.IPLDFetcher
+		indexAndPublisher *eth.IPLDPublisher
 		backend           *eth.Backend
 		api               *eth.PublicEthAPI
 	)
@@ -94,15 +94,15 @@ var _ = Describe("API", func() {
 		db, err = shared.SetupDB()
 		Expect(err).ToNot(HaveOccurred())
 		retriever = eth.NewCIDRetriever(db)
-		fetcher = eth.NewIPLDPGFetcher(db)
-		indexAndPublisher = eth.NewIPLDPublisherAndIndexer(db)
+		fetcher = eth.NewIPLDFetcher(db)
+		indexAndPublisher = eth.NewIPLDPublisher(db)
 		backend = &eth.Backend{
 			Retriever: retriever,
 			Fetcher:   fetcher,
 			DB:        db,
 		}
 		api = eth.NewPublicEthAPI(backend)
-		_, err = indexAndPublisher.Publish(mocks.MockConvertedPayload)
+		err = indexAndPublisher.Publish(mocks.MockConvertedPayload)
 		Expect(err).ToNot(HaveOccurred())
 		uncles := mocks.MockBlock.Uncles()
 		uncleHashes := make([]common.Hash, len(uncles))

pkg/eth/backend.go

@@ -39,7 +39,7 @@ var (
 
 type Backend struct {
 	Retriever *CIDRetriever
-	Fetcher   *IPLDPGFetcher
+	Fetcher   *IPLDFetcher
 	DB        *postgres.DB
 }
 

pkg/eth/cid_retriever_test.go

@@ -211,14 +211,14 @@ var (
 var _ = Describe("Retriever", func() {
 	var (
 		db        *postgres.DB
-		repo      *eth2.IPLDPublisherAndIndexer
+		repo      *eth2.IPLDPublisher
 		retriever *eth2.CIDRetriever
 	)
 	BeforeEach(func() {
 		var err error
 		db, err = shared.SetupDB()
 		Expect(err).ToNot(HaveOccurred())
-		repo = eth2.NewIPLDPublisherAndIndexer(db)
+		repo = eth2.NewIPLDPublisher(db)
 		retriever = eth2.NewCIDRetriever(db)
 	})
 	AfterEach(func() {
@@ -227,7 +227,7 @@ var _ = Describe("Retriever", func() {
 
 	Describe("Retrieve", func() {
 		BeforeEach(func() {
-			_, err := repo.Publish(mocks.MockConvertedPayload)
+			err := repo.Publish(mocks.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
 		})
 		It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() {
@@ -413,7 +413,7 @@ var _ = Describe("Retriever", func() {
 			Expect(err).To(HaveOccurred())
 		})
 		It("Gets the number of the first block that has data in the database", func() {
-			_, err := repo.Publish(mocks.MockConvertedPayload)
+			err := repo.Publish(mocks.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
 			num, err := retriever.RetrieveFirstBlockNumber()
 			Expect(err).ToNot(HaveOccurred())
@@ -423,7 +423,7 @@ var _ = Describe("Retriever", func() {
 		It("Gets the number of the first block that has data in the database", func() {
 			payload := mocks.MockConvertedPayload
 			payload.Block = newMockBlock(1010101)
-			_, err := repo.Publish(payload)
+			err := repo.Publish(payload)
 			Expect(err).ToNot(HaveOccurred())
 			num, err := retriever.RetrieveFirstBlockNumber()
 			Expect(err).ToNot(HaveOccurred())
@@ -435,9 +435,9 @@ var _ = Describe("Retriever", func() {
 			payload1.Block = newMockBlock(1010101)
 			payload2 := payload1
 			payload2.Block = newMockBlock(5)
-			_, err := repo.Publish(payload1)
+			err := repo.Publish(payload1)
 			Expect(err).ToNot(HaveOccurred())
-			_, err = repo.Publish(payload2)
+			err = repo.Publish(payload2)
 			Expect(err).ToNot(HaveOccurred())
 			num, err := retriever.RetrieveFirstBlockNumber()
 			Expect(err).ToNot(HaveOccurred())
@@ -451,7 +451,7 @@ var _ = Describe("Retriever", func() {
 			Expect(err).To(HaveOccurred())
 		})
 		It("Gets the number of the latest block that has data in the database", func() {
-			_, err := repo.Publish(mocks.MockConvertedPayload)
+			err := repo.Publish(mocks.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
 			num, err := retriever.RetrieveLastBlockNumber()
 			Expect(err).ToNot(HaveOccurred())
@@ -461,7 +461,7 @@ var _ = Describe("Retriever", func() {
 		It("Gets the number of the latest block that has data in the database", func() {
 			payload := mocks.MockConvertedPayload
 			payload.Block = newMockBlock(1010101)
-			_, err := repo.Publish(payload)
+			err := repo.Publish(payload)
 			Expect(err).ToNot(HaveOccurred())
 			num, err := retriever.RetrieveLastBlockNumber()
 			Expect(err).ToNot(HaveOccurred())
@@ -473,9 +473,9 @@ var _ = Describe("Retriever", func() {
 			payload1.Block = newMockBlock(1010101)
 			payload2 := payload1
 			payload2.Block = newMockBlock(5)
-			_, err := repo.Publish(payload1)
+			err := repo.Publish(payload1)
 			Expect(err).ToNot(HaveOccurred())
-			_, err = repo.Publish(payload2)
+			err = repo.Publish(payload2)
 			Expect(err).ToNot(HaveOccurred())
 			num, err := retriever.RetrieveLastBlockNumber()
 			Expect(err).ToNot(HaveOccurred())
@@ -492,13 +492,13 @@ var _ = Describe("Retriever", func() {
 			payload2.Block = newMockBlock(2)
 			payload3 := payload2
 			payload3.Block = newMockBlock(3)
-			_, err := repo.Publish(payload0)
+			err := repo.Publish(payload0)
 			Expect(err).ToNot(HaveOccurred())
-			_, err = repo.Publish(payload1)
+			err = repo.Publish(payload1)
 			Expect(err).ToNot(HaveOccurred())
-			_, err = repo.Publish(payload2)
+			err = repo.Publish(payload2)
 			Expect(err).ToNot(HaveOccurred())
-			_, err = repo.Publish(payload3)
+			err = repo.Publish(payload3)
 			Expect(err).ToNot(HaveOccurred())
 			gaps, err := retriever.RetrieveGapsInData(1)
 			Expect(err).ToNot(HaveOccurred())
@@ -508,7 +508,7 @@ var _ = Describe("Retriever", func() {
 		It("Returns the gap from 0 to the earliest block", func() {
 			payload := mocks.MockConvertedPayload
 			payload.Block = newMockBlock(5)
-			_, err := repo.Publish(payload)
+			err := repo.Publish(payload)
 			Expect(err).ToNot(HaveOccurred())
 			gaps, err := retriever.RetrieveGapsInData(1)
 			Expect(err).ToNot(HaveOccurred())
@@ -523,11 +523,11 @@ var _ = Describe("Retriever", func() {
 			payload1 := mocks.MockConvertedPayload
 			payload3 := payload1
 			payload3.Block = newMockBlock(3)
-			_, err := repo.Publish(payload0)
+			err := repo.Publish(payload0)
 			Expect(err).ToNot(HaveOccurred())
-			_, err = repo.Publish(payload1)
+			err = repo.Publish(payload1)
 			Expect(err).ToNot(HaveOccurred())
-			_, err = repo.Publish(payload3)
+			err = repo.Publish(payload3)
 			Expect(err).ToNot(HaveOccurred())
 			gaps, err := retriever.RetrieveGapsInData(1)
 			Expect(err).ToNot(HaveOccurred())
@@ -541,9 +541,9 @@ var _ = Describe("Retriever", func() {
 			payload1.Block = newMockBlock(1010101)
 			payload2 := payload1
 			payload2.Block = newMockBlock(0)
-			_, err := repo.Publish(payload1)
+			err := repo.Publish(payload1)
 			Expect(err).ToNot(HaveOccurred())
-			_, err = repo.Publish(payload2)
+			err = repo.Publish(payload2)
 			Expect(err).ToNot(HaveOccurred())
 			gaps, err := retriever.RetrieveGapsInData(1)
 			Expect(err).ToNot(HaveOccurred())
@@ -576,27 +576,27 @@ var _ = Describe("Retriever", func() {
 			payload11 := mocks.MockConvertedPayload
 			payload11.Block = newMockBlock(1000)
 
-			_, err := repo.Publish(payload1)
+			err := repo.Publish(payload1)
 			Expect(err).ToNot(HaveOccurred())
-			_, err = repo.Publish(payload2)
+			err = repo.Publish(payload2)
 			Expect(err).ToNot(HaveOccurred())
-			_, err = repo.Publish(payload3)
+			err = repo.Publish(payload3)
 			Expect(err).ToNot(HaveOccurred())
-			_, err = repo.Publish(payload4)
+			err = repo.Publish(payload4)
 			Expect(err).ToNot(HaveOccurred())
-			_, err = repo.Publish(payload5)
+			err = repo.Publish(payload5)
 			Expect(err).ToNot(HaveOccurred())
-			_, err = repo.Publish(payload6)
+			err = repo.Publish(payload6)
 			Expect(err).ToNot(HaveOccurred())
-			_, err = repo.Publish(payload7)
+			err = repo.Publish(payload7)
 			Expect(err).ToNot(HaveOccurred())
-			_, err = repo.Publish(payload8)
+			err = repo.Publish(payload8)
 			Expect(err).ToNot(HaveOccurred())
-			_, err = repo.Publish(payload9)
+			err = repo.Publish(payload9)
 			Expect(err).ToNot(HaveOccurred())
-			_, err = repo.Publish(payload10)
+			err = repo.Publish(payload10)
 			Expect(err).ToNot(HaveOccurred())
-			_, err = repo.Publish(payload11)
+			err = repo.Publish(payload11)
 			Expect(err).ToNot(HaveOccurred())
 
 			gaps, err := retriever.RetrieveGapsInData(1)
@@ -640,33 +640,33 @@ var _ = Describe("Retriever", func() {
 			payload14 := mocks.MockConvertedPayload
 			payload14.Block = newMockBlock(1000)
 
-			_, err := repo.Publish(payload1)
+			err := repo.Publish(payload1)
 			Expect(err).ToNot(HaveOccurred())
-			_, err = repo.Publish(payload2)
+			err = repo.Publish(payload2)
 			Expect(err).ToNot(HaveOccurred())
-			_, err = repo.Publish(payload3)
+			err = repo.Publish(payload3)
 			Expect(err).ToNot(HaveOccurred())
-			_, err = repo.Publish(payload4)
+			err = repo.Publish(payload4)
 			Expect(err).ToNot(HaveOccurred())
-			_, err = repo.Publish(payload5)
+			err = repo.Publish(payload5)
 			Expect(err).ToNot(HaveOccurred())
-			_, err = repo.Publish(payload6)
+			err = repo.Publish(payload6)
 			Expect(err).ToNot(HaveOccurred())
-			_, err = repo.Publish(payload7)
+			err = repo.Publish(payload7)
 			Expect(err).ToNot(HaveOccurred())
-			_, err = repo.Publish(payload8)
+			err = repo.Publish(payload8)
 			Expect(err).ToNot(HaveOccurred())
-			_, err = repo.Publish(payload9)
+			err = repo.Publish(payload9)
 			Expect(err).ToNot(HaveOccurred())
-			_, err = repo.Publish(payload10)
+			err = repo.Publish(payload10)
 			Expect(err).ToNot(HaveOccurred())
-			_, err = repo.Publish(payload11)
+			err = repo.Publish(payload11)
 			Expect(err).ToNot(HaveOccurred())
-			_, err = repo.Publish(payload12)
+			err = repo.Publish(payload12)
 			Expect(err).ToNot(HaveOccurred())
-			_, err = repo.Publish(payload13)
+			err = repo.Publish(payload13)
 			Expect(err).ToNot(HaveOccurred())
-			_, err = repo.Publish(payload14)
+			err = repo.Publish(payload14)
 			Expect(err).ToNot(HaveOccurred())
 
 			cleaner := eth.NewCleaner(db)

pkg/eth/helpers.go

@@ -18,6 +18,7 @@ package eth
 
 import (
 	"fmt"
+
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/statediff"
 )
@@ -66,4 +67,4 @@ func ChainConfig(chainID uint64) (*params.ChainConfig, error) {
 	default:
 		return nil, fmt.Errorf("chain config for chainid %d not available", chainID)
 	}
-}
+}

pkg/eth/ipld_fetcher.go

@@ -30,21 +30,21 @@ import (
 	"github.com/vulcanize/ipfs-blockchain-watcher/pkg/shared"
 )
 
-// IPLDPGFetcher satisfies the IPLDFetcher interface for ethereum
+// IPLDFetcher satisfies the IPLDFetcher interface for ethereum
 // It interfaces directly with PG-IPFS
-type IPLDPGFetcher struct {
+type IPLDFetcher struct {
 	db *postgres.DB
 }
 
-// NewIPLDPGFetcher creates a pointer to a new IPLDPGFetcher
-func NewIPLDPGFetcher(db *postgres.DB) *IPLDPGFetcher {
-	return &IPLDPGFetcher{
+// NewIPLDFetcher creates a pointer to a new IPLDFetcher
+func NewIPLDFetcher(db *postgres.DB) *IPLDFetcher {
+	return &IPLDFetcher{
 		db: db,
 	}
 }
 
 // Fetch is the exported method for fetching and returning all the IPLDS specified in the CIDWrapper
-func (f *IPLDPGFetcher) Fetch(cids shared.CIDsForFetching) (shared.IPLDs, error) {
+func (f *IPLDFetcher) Fetch(cids shared.CIDsForFetching) (shared.IPLDs, error) {
 	cidWrapper, ok := cids.(*CIDWrapper)
 	if !ok {
 		return nil, fmt.Errorf("eth fetcher: expected cids type %T got %T", &CIDWrapper{}, cids)
@@ -100,7 +100,7 @@ func (f *IPLDPGFetcher) Fetch(cids shared.CIDsForFetching) (shared.IPLDs, error)
 }
 
 // FetchHeaders fetches headers
-func (f *IPLDPGFetcher) FetchHeader(tx *sqlx.Tx, c HeaderModel) (ipfs.BlockModel, error) {
+func (f *IPLDFetcher) FetchHeader(tx *sqlx.Tx, c HeaderModel) (ipfs.BlockModel, error) {
 	log.Debug("fetching header ipld")
 	headerBytes, err := shared.FetchIPLDByMhKey(tx, c.MhKey)
 	if err != nil {
@@ -113,7 +113,7 @@ func (f *IPLDPGFetcher) FetchHeader(tx *sqlx.Tx, c HeaderModel) (ipfs.BlockModel
 }
 
 // FetchUncles fetches uncles
-func (f *IPLDPGFetcher) FetchUncles(tx *sqlx.Tx, cids []UncleModel) ([]ipfs.BlockModel, error) {
+func (f *IPLDFetcher) FetchUncles(tx *sqlx.Tx, cids []UncleModel) ([]ipfs.BlockModel, error) {
 	log.Debug("fetching uncle iplds")
 	uncleIPLDs := make([]ipfs.BlockModel, len(cids))
 	for i, c := range cids {
@@ -130,7 +130,7 @@ func (f *IPLDPGFetcher) FetchUncles(tx *sqlx.Tx, cids []UncleModel) ([]ipfs.Bloc
 }
 
 // FetchTrxs fetches transactions
-func (f *IPLDPGFetcher) FetchTrxs(tx *sqlx.Tx, cids []TxModel) ([]ipfs.BlockModel, error) {
+func (f *IPLDFetcher) FetchTrxs(tx *sqlx.Tx, cids []TxModel) ([]ipfs.BlockModel, error) {
 	log.Debug("fetching transaction iplds")
 	trxIPLDs := make([]ipfs.BlockModel, len(cids))
 	for i, c := range cids {
@@ -147,7 +147,7 @@ func (f *IPLDPGFetcher) FetchTrxs(tx *sqlx.Tx, cids []TxModel) ([]ipfs.BlockMode
 }
 
 // FetchRcts fetches receipts
-func (f *IPLDPGFetcher) FetchRcts(tx *sqlx.Tx, cids []ReceiptModel) ([]ipfs.BlockModel, error) {
+func (f *IPLDFetcher) FetchRcts(tx *sqlx.Tx, cids []ReceiptModel) ([]ipfs.BlockModel, error) {
 	log.Debug("fetching receipt iplds")
 	rctIPLDs := make([]ipfs.BlockModel, len(cids))
 	for i, c := range cids {
@@ -164,7 +164,7 @@ func (f *IPLDPGFetcher) FetchRcts(tx *sqlx.Tx, cids []ReceiptModel) ([]ipfs.Bloc
 }
 
 // FetchState fetches state nodes
-func (f *IPLDPGFetcher) FetchState(tx *sqlx.Tx, cids []StateNodeModel) ([]StateNode, error) {
+func (f *IPLDFetcher) FetchState(tx *sqlx.Tx, cids []StateNodeModel) ([]StateNode, error) {
 	log.Debug("fetching state iplds")
 	stateNodes := make([]StateNode, 0, len(cids))
 	for _, stateNode := range cids {
@@ -189,7 +189,7 @@ func (f *IPLDPGFetcher) FetchState(tx *sqlx.Tx, cids []StateNodeModel) ([]StateN
 }
 
 // FetchStorage fetches storage nodes
-func (f *IPLDPGFetcher) FetchStorage(tx *sqlx.Tx, cids []StorageNodeWithStateKeyModel) ([]StorageNode, error) {
+func (f *IPLDFetcher) FetchStorage(tx *sqlx.Tx, cids []StorageNodeWithStateKeyModel) ([]StorageNode, error) {
 	log.Debug("fetching storage iplds")
 	storageNodes := make([]StorageNode, 0, len(cids))
 	for _, storageNode := range cids {

pkg/eth/ipld_fetcher_test.go

@@ -28,20 +28,20 @@ import (
 
 var (
 	db            *postgres.DB
-	pubAndIndexer *eth.IPLDPublisherAndIndexer
-	fetcher       *eth.IPLDPGFetcher
+	pubAndIndexer *eth.IPLDPublisher
+	fetcher       *eth.IPLDFetcher
 )
 
-var _ = Describe("IPLDPGFetcher", func() {
+var _ = Describe("IPLDFetcher", func() {
 	Describe("Fetch", func() {
 		BeforeEach(func() {
 			var err error
 			db, err = shared.SetupDB()
 			Expect(err).ToNot(HaveOccurred())
-			pubAndIndexer = eth.NewIPLDPublisherAndIndexer(db)
-			_, err = pubAndIndexer.Publish(mocks.MockConvertedPayload)
+			pubAndIndexer = eth.NewIPLDPublisher(db)
+			err = pubAndIndexer.Publish(mocks.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
-			fetcher = eth.NewIPLDPGFetcher(db)
+			fetcher = eth.NewIPLDFetcher(db)
 		})
 		AfterEach(func() {
 			eth.TearDownDB(db)

pkg/eth/ipld_pg_fetcher.go

@@ -1,215 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package eth
-
-import (
-	"errors"
-	"fmt"
-	"math/big"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/jmoiron/sqlx"
-	log "github.com/sirupsen/logrus"
-
-	"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs"
-	"github.com/vulcanize/ipfs-blockchain-watcher/pkg/postgres"
-	"github.com/vulcanize/ipfs-blockchain-watcher/pkg/shared"
-)
-
-// IPLDPGFetcher satisfies the IPLDFetcher interface for ethereum
-// It interfaces directly with PG-IPFS
-type IPLDPGFetcher struct {
-	db *postgres.DB
-}
-
-// NewIPLDPGFetcher creates a pointer to a new IPLDPGFetcher
-func NewIPLDPGFetcher(db *postgres.DB) *IPLDPGFetcher {
-	return &IPLDPGFetcher{
-		db: db,
-	}
-}
-
-// Fetch is the exported method for fetching and returning all the IPLDS specified in the CIDWrapper
-func (f *IPLDPGFetcher) Fetch(cids shared.CIDsForFetching) (shared.IPLDs, error) {
-	cidWrapper, ok := cids.(*CIDWrapper)
-	if !ok {
-		return nil, fmt.Errorf("eth fetcher: expected cids type %T got %T", &CIDWrapper{}, cids)
-	}
-	log.Debug("fetching iplds")
-	iplds := IPLDs{}
-	iplds.TotalDifficulty, ok = new(big.Int).SetString(cidWrapper.Header.TotalDifficulty, 10)
-	if !ok {
-		return nil, errors.New("eth fetcher: unable to set total difficulty")
-	}
-	iplds.BlockNumber = cidWrapper.BlockNumber
-
-	tx, err := f.db.Beginx()
-	if err != nil {
-		return nil, err
-	}
-	defer func() {
-		if p := recover(); p != nil {
-			shared.Rollback(tx)
-			panic(p)
-		} else if err != nil {
-			shared.Rollback(tx)
-		} else {
-			err = tx.Commit()
-		}
-	}()
-
-	iplds.Header, err = f.FetchHeader(tx, cidWrapper.Header)
-	if err != nil {
-		return nil, fmt.Errorf("eth pg fetcher: header fetching error: %s", err.Error())
-	}
-	iplds.Uncles, err = f.FetchUncles(tx, cidWrapper.Uncles)
-	if err != nil {
-		return nil, fmt.Errorf("eth pg fetcher: uncle fetching error: %s", err.Error())
-	}
-	iplds.Transactions, err = f.FetchTrxs(tx, cidWrapper.Transactions)
-	if err != nil {
-		return nil, fmt.Errorf("eth pg fetcher: transaction fetching error: %s", err.Error())
-	}
-	iplds.Receipts, err = f.FetchRcts(tx, cidWrapper.Receipts)
-	if err != nil {
-		return nil, fmt.Errorf("eth pg fetcher: receipt fetching error: %s", err.Error())
-	}
-	iplds.StateNodes, err = f.FetchState(tx, cidWrapper.StateNodes)
-	if err != nil {
-		return nil, fmt.Errorf("eth pg fetcher: state fetching error: %s", err.Error())
-	}
-	iplds.StorageNodes, err = f.FetchStorage(tx, cidWrapper.StorageNodes)
-	if err != nil {
-		return nil, fmt.Errorf("eth pg fetcher: storage fetching error: %s", err.Error())
-	}
-	return iplds, err
-}
-
-// FetchHeaders fetches headers
-func (f *IPLDPGFetcher) FetchHeader(tx *sqlx.Tx, c HeaderModel) (ipfs.BlockModel, error) {
-	log.Debug("fetching header ipld")
-	headerBytes, err := shared.FetchIPLDByMhKey(tx, c.MhKey)
-	if err != nil {
-		return ipfs.BlockModel{}, err
-	}
-	return ipfs.BlockModel{
-		Data: headerBytes,
-		CID:  c.CID,
-	}, nil
-}
-
-// FetchUncles fetches uncles
-func (f *IPLDPGFetcher) FetchUncles(tx *sqlx.Tx, cids []UncleModel) ([]ipfs.BlockModel, error) {
-	log.Debug("fetching uncle iplds")
-	uncleIPLDs := make([]ipfs.BlockModel, len(cids))
-	for i, c := range cids {
-		uncleBytes, err := shared.FetchIPLDByMhKey(tx, c.MhKey)
-		if err != nil {
-			return nil, err
-		}
-		uncleIPLDs[i] = ipfs.BlockModel{
-			Data: uncleBytes,
-			CID:  c.CID,
-		}
-	}
-	return uncleIPLDs, nil
-}
-
-// FetchTrxs fetches transactions
-func (f *IPLDPGFetcher) FetchTrxs(tx *sqlx.Tx, cids []TxModel) ([]ipfs.BlockModel, error) {
-	log.Debug("fetching transaction iplds")
-	trxIPLDs := make([]ipfs.BlockModel, len(cids))
-	for i, c := range cids {
-		txBytes, err := shared.FetchIPLDByMhKey(tx, c.MhKey)
-		if err != nil {
-			return nil, err
-		}
-		trxIPLDs[i] = ipfs.BlockModel{
-			Data: txBytes,
-			CID:  c.CID,
-		}
-	}
-	return trxIPLDs, nil
-}
-
-// FetchRcts fetches receipts
-func (f *IPLDPGFetcher) FetchRcts(tx *sqlx.Tx, cids []ReceiptModel) ([]ipfs.BlockModel, error) {
-	log.Debug("fetching receipt iplds")
-	rctIPLDs := make([]ipfs.BlockModel, len(cids))
-	for i, c := range cids {
-		rctBytes, err := shared.FetchIPLDByMhKey(tx, c.MhKey)
-		if err != nil {
-			return nil, err
-		}
-		rctIPLDs[i] = ipfs.BlockModel{
-			Data: rctBytes,
-			CID:  c.CID,
-		}
-	}
-	return rctIPLDs, nil
-}
-
-// FetchState fetches state nodes
-func (f *IPLDPGFetcher) FetchState(tx *sqlx.Tx, cids []StateNodeModel) ([]StateNode, error) {
-	log.Debug("fetching state iplds")
-	stateNodes := make([]StateNode, 0, len(cids))
-	for _, stateNode := range cids {
-		if stateNode.CID == "" {
-			continue
-		}
-		stateBytes, err := shared.FetchIPLDByMhKey(tx, stateNode.MhKey)
-		if err != nil {
-			return nil, err
-		}
-		stateNodes = append(stateNodes, StateNode{
-			IPLD: ipfs.BlockModel{
-				Data: stateBytes,
-				CID:  stateNode.CID,
-			},
-			StateLeafKey: common.HexToHash(stateNode.StateKey),
-			Type:         ResolveToNodeType(stateNode.NodeType),
-			Path:         stateNode.Path,
-		})
-	}
-	return stateNodes, nil
-}
-
-// FetchStorage fetches storage nodes
-func (f *IPLDPGFetcher) FetchStorage(tx *sqlx.Tx, cids []StorageNodeWithStateKeyModel) ([]StorageNode, error) {
-	log.Debug("fetching storage iplds")
-	storageNodes := make([]StorageNode, 0, len(cids))
-	for _, storageNode := range cids {
-		if storageNode.CID == "" || storageNode.StateKey == "" {
-			continue
-		}
-		storageBytes, err := shared.FetchIPLDByMhKey(tx, storageNode.MhKey)
-		if err != nil {
-			return nil, err
-		}
-		storageNodes = append(storageNodes, StorageNode{
-			IPLD: ipfs.BlockModel{
-				Data: storageBytes,
-				CID:  storageNode.CID,
-			},
-			StateLeafKey:   common.HexToHash(storageNode.StateKey),
-			StorageLeafKey: common.HexToHash(storageNode.StorageKey),
-			Type:           ResolveToNodeType(storageNode.NodeType),
-			Path:           storageNode.Path,
-		})
-	}
-	return storageNodes, nil
-}

pkg/eth/ipld_pg_fetcher_test.go

@@ -1,65 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package eth_test
-
-import (
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	"github.com/vulcanize/ipfs-blockchain-watcher/pkg/eth"
-	"github.com/vulcanize/ipfs-blockchain-watcher/pkg/eth/mocks"
-	"github.com/vulcanize/ipfs-blockchain-watcher/pkg/postgres"
-	"github.com/vulcanize/ipfs-blockchain-watcher/pkg/shared"
-)
-
-var (
-	db            *postgres.DB
-	pubAndIndexer *eth.IPLDPublisherAndIndexer
-	fetcher       *eth.IPLDPGFetcher
-)
-
-var _ = Describe("IPLDPGFetcher", func() {
-	Describe("Fetch", func() {
-		BeforeEach(func() {
-			var err error
-			db, err = shared.SetupDB()
-			Expect(err).ToNot(HaveOccurred())
-			pubAndIndexer = eth.NewIPLDPublisherAndIndexer(db)
-			_, err = pubAndIndexer.Publish(mocks.MockConvertedPayload)
-			Expect(err).ToNot(HaveOccurred())
-			fetcher = eth.NewIPLDPGFetcher(db)
-		})
-		AfterEach(func() {
-			eth.TearDownDB(db)
-		})
-
-		It("Fetches and returns IPLDs for the CIDs provided in the CIDWrapper", func() {
-			i, err := fetcher.Fetch(mocks.MockCIDWrapper)
-			Expect(err).ToNot(HaveOccurred())
-			iplds, ok := i.(eth.IPLDs)
-			Expect(ok).To(BeTrue())
-			Expect(iplds.TotalDifficulty).To(Equal(mocks.MockConvertedPayload.TotalDifficulty))
-			Expect(iplds.BlockNumber).To(Equal(mocks.MockConvertedPayload.Block.Number()))
-			Expect(iplds.Header).To(Equal(mocks.MockIPLDs.Header))
-			Expect(len(iplds.Uncles)).To(Equal(0))
-			Expect(iplds.Transactions).To(Equal(mocks.MockIPLDs.Transactions))
-			Expect(iplds.Receipts).To(Equal(mocks.MockIPLDs.Receipts))
-			Expect(iplds.StateNodes).To(Equal(mocks.MockIPLDs.StateNodes))
-			Expect(iplds.StorageNodes).To(Equal(mocks.MockIPLDs.StorageNodes))
-		})
-	})
-})

pkg/eth/mocks/publisher.go

@@ -32,13 +32,13 @@ type IPLDPublisher struct {
 }
 
 // Publish publishes an IPLDPayload to IPFS and returns the corresponding CIDPayload
-func (pub *IPLDPublisher) Publish(payload shared.ConvertedData) (shared.CIDsForIndexing, error) {
+func (pub *IPLDPublisher) Publish(payload shared.ConvertedData) error {
 	ipldPayload, ok := payload.(eth.ConvertedPayload)
 	if !ok {
-		return nil, fmt.Errorf("publish expected payload type %T got %T", &eth.ConvertedPayload{}, payload)
+		return fmt.Errorf("publish expected payload type %T got %T", &eth.ConvertedPayload{}, payload)
 	}
 	pub.PassedIPLDPayload = ipldPayload
-	return pub.ReturnCIDPayload, pub.ReturnErr
+	return pub.ReturnErr
 }
 
 // IterativeIPLDPublisher is the underlying struct for the Publisher interface; used in testing
@@ -50,16 +50,12 @@ type IterativeIPLDPublisher struct {
 }
 
 // Publish publishes an IPLDPayload to IPFS and returns the corresponding CIDPayload
-func (pub *IterativeIPLDPublisher) Publish(payload shared.ConvertedData) (shared.CIDsForIndexing, error) {
+func (pub *IterativeIPLDPublisher) Publish(payload shared.ConvertedData) error {
 	ipldPayload, ok := payload.(eth.ConvertedPayload)
 	if !ok {
-		return nil, fmt.Errorf("publish expected payload type %T got %T", &eth.ConvertedPayload{}, payload)
+		return fmt.Errorf("publish expected payload type %T got %T", &eth.ConvertedPayload{}, payload)
 	}
 	pub.PassedIPLDPayload = append(pub.PassedIPLDPayload, ipldPayload)
-	if len(pub.ReturnCIDPayload) < pub.iteration+1 {
-		return nil, fmt.Errorf("IterativeIPLDPublisher does not have a payload to return at iteration %d", pub.iteration)
-	}
-	returnPayload := pub.ReturnCIDPayload[pub.iteration]
 	pub.iteration++
-	return returnPayload, pub.ReturnErr
+	return pub.ReturnErr
 }

pkg/eth/mocks/test_data.go

@@ -63,6 +63,7 @@ var (
 	AnotherAddress                             = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476593")
 	ContractAddress                            = crypto.CreateAddress(SenderAddr, MockTransactions[2].Nonce())
 	ContractHash                               = crypto.Keccak256Hash(ContractAddress.Bytes()).String()
+	MockContractByteCode                       = []byte{0, 1, 2, 3, 4, 5}
 	mockTopic11                                = common.HexToHash("0x04")
 	mockTopic12                                = common.HexToHash("0x06")
 	mockTopic21                                = common.HexToHash("0x05")
@@ -99,28 +100,34 @@ var (
 	StorageMhKey  = shared.MultihashKeyFromCID(StorageCID)
 	MockTrxMeta   = []eth.TxModel{
 		{
-			CID:    "", // This is empty until we go to publish to ipfs
-			MhKey:  "",
-			Src:    SenderAddr.Hex(),
-			Dst:    Address.String(),
-			Index:  0,
-			TxHash: MockTransactions[0].Hash().String(),
+			CID:        "", // This is empty until we go to publish to ipfs
+			MhKey:      "",
+			Src:        SenderAddr.Hex(),
+			Dst:        Address.String(),
+			Index:      0,
+			TxHash:     MockTransactions[0].Hash().String(),
+			Data:       []byte{},
+			Deployment: false,
 		},
 		{
-			CID:    "",
-			MhKey:  "",
-			Src:    SenderAddr.Hex(),
-			Dst:    AnotherAddress.String(),
-			Index:  1,
-			TxHash: MockTransactions[1].Hash().String(),
+			CID:        "",
+			MhKey:      "",
+			Src:        SenderAddr.Hex(),
+			Dst:        AnotherAddress.String(),
+			Index:      1,
+			TxHash:     MockTransactions[1].Hash().String(),
+			Data:       []byte{},
+			Deployment: false,
 		},
 		{
-			CID:    "",
-			MhKey:  "",
-			Src:    SenderAddr.Hex(),
-			Dst:    "",
-			Index:  2,
-			TxHash: MockTransactions[2].Hash().String(),
+			CID:        "",
+			MhKey:      "",
+			Src:        SenderAddr.Hex(),
+			Dst:        "",
+			Index:      2,
+			TxHash:     MockTransactions[2].Hash().String(),
+			Data:       MockContractByteCode,
+			Deployment: true,
 		},
 	}
 	MockTrxMetaPostPublsh = []eth.TxModel{
@@ -532,7 +539,7 @@ func createTransactionsAndReceipts() (types.Transactions, types.Receipts, common
 	// make transactions
 	trx1 := types.NewTransaction(0, Address, big.NewInt(1000), 50, big.NewInt(100), []byte{})
 	trx2 := types.NewTransaction(1, AnotherAddress, big.NewInt(2000), 100, big.NewInt(200), []byte{})
-	trx3 := types.NewContractCreation(2, big.NewInt(1500), 75, big.NewInt(150), []byte{0, 1, 2, 3, 4, 5})
+	trx3 := types.NewContractCreation(2, big.NewInt(1500), 75, big.NewInt(150), MockContractByteCode)
 	transactionSigner := types.MakeSigner(params.MainnetChainConfig, new(big.Int).Set(BlockNumber))
 	mockCurve := elliptic.P256()
 	mockPrvKey, err := ecdsa.GenerateKey(mockCurve, rand.Reader)

pkg/eth/models.go

@@ -51,16 +51,16 @@ type UncleModel struct {
 
 // TxModel is the db model for eth.transaction_cids
 type TxModel struct {
-	ID       int64  `db:"id"`
-	HeaderID int64  `db:"header_id"`
-	Index    int64  `db:"index"`
-	TxHash   string `db:"tx_hash"`
-	CID      string `db:"cid"`
-	MhKey    string `db:"mh_key"`
-	Dst      string `db:"dst"`
-	Src      string `db:"src"`
-	Data 	 []byte `db:"data"`
-	Deployment bool `db:"deployment"`
+	ID         int64  `db:"id"`
+	HeaderID   int64  `db:"header_id"`
+	Index      int64  `db:"index"`
+	TxHash     string `db:"tx_hash"`
+	CID        string `db:"cid"`
+	MhKey      string `db:"mh_key"`
+	Dst        string `db:"dst"`
+	Src        string `db:"src"`
+	Data       []byte `db:"data"`
+	Deployment bool   `db:"deployment"`
 }
 
 // ReceiptModel is the db model for eth.receipt_cids

pkg/eth/publish_and_indexer.go

@@ -1,228 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package eth
-
-import (
-	"fmt"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/state"
-	"github.com/ethereum/go-ethereum/rlp"
-	"github.com/ethereum/go-ethereum/statediff"
-	"github.com/jmoiron/sqlx"
-	"github.com/multiformats/go-multihash"
-
-	"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs/ipld"
-	"github.com/vulcanize/ipfs-blockchain-watcher/pkg/postgres"
-	"github.com/vulcanize/ipfs-blockchain-watcher/pkg/shared"
-)
-
-// IPLDPublisherAndIndexer satisfies the IPLDPublisher interface for ethereum
-// It interfaces directly with the public.blocks table of PG-IPFS rather than going through an ipfs intermediary
-// It publishes and indexes IPLDs together in a single sqlx.Tx
-type IPLDPublisherAndIndexer struct {
-	indexer *CIDIndexer
-}
-
-// NewIPLDPublisherAndIndexer creates a pointer to a new IPLDPublisherAndIndexer which satisfies the IPLDPublisher interface
-func NewIPLDPublisherAndIndexer(db *postgres.DB) *IPLDPublisherAndIndexer {
-	return &IPLDPublisherAndIndexer{
-		indexer: NewCIDIndexer(db),
-	}
-}
-
-// Publish publishes an IPLDPayload to IPFS and returns the corresponding CIDPayload
-func (pub *IPLDPublisherAndIndexer) Publish(payload shared.ConvertedData) (shared.CIDsForIndexing, error) {
-	ipldPayload, ok := payload.(ConvertedPayload)
-	if !ok {
-		return nil, fmt.Errorf("eth IPLDPublisherAndIndexer expected payload type %T got %T", ConvertedPayload{}, payload)
-	}
-	// Generate the iplds
-	headerNode, uncleNodes, txNodes, txTrieNodes, rctNodes, rctTrieNodes, err := ipld.FromBlockAndReceipts(ipldPayload.Block, ipldPayload.Receipts)
-	if err != nil {
-		return nil, err
-	}
-
-	// Begin new db tx
-	tx, err := pub.indexer.db.Beginx()
-	if err != nil {
-		return nil, err
-	}
-	defer func() {
-		if p := recover(); p != nil {
-			shared.Rollback(tx)
-			panic(p)
-		} else if err != nil {
-			shared.Rollback(tx)
-		} else {
-			err = tx.Commit()
-		}
-	}()
-
-	// Publish trie nodes
-	for _, node := range txTrieNodes {
-		if err := shared.PublishIPLD(tx, node); err != nil {
-			return nil, err
-		}
-	}
-	for _, node := range rctTrieNodes {
-		if err := shared.PublishIPLD(tx, node); err != nil {
-			return nil, err
-		}
-	}
-
-	// Publish and index header
-	if err := shared.PublishIPLD(tx, headerNode); err != nil {
-		return nil, err
-	}
-	reward := CalcEthBlockReward(ipldPayload.Block.Header(), ipldPayload.Block.Uncles(), ipldPayload.Block.Transactions(), ipldPayload.Receipts)
-	header := HeaderModel{
-		CID:             headerNode.Cid().String(),
-		MhKey:           shared.MultihashKeyFromCID(headerNode.Cid()),
-		ParentHash:      ipldPayload.Block.ParentHash().String(),
-		BlockNumber:     ipldPayload.Block.Number().String(),
-		BlockHash:       ipldPayload.Block.Hash().String(),
-		TotalDifficulty: ipldPayload.TotalDifficulty.String(),
-		Reward:          reward.String(),
-		Bloom:           ipldPayload.Block.Bloom().Bytes(),
-		StateRoot:       ipldPayload.Block.Root().String(),
-		RctRoot:         ipldPayload.Block.ReceiptHash().String(),
-		TxRoot:          ipldPayload.Block.TxHash().String(),
-		UncleRoot:       ipldPayload.Block.UncleHash().String(),
-		Timestamp:       ipldPayload.Block.Time(),
-	}
-	headerID, err := pub.indexer.indexHeaderCID(tx, header)
-	if err != nil {
-		return nil, err
-	}
-
-	// Publish and index uncles
-	for _, uncleNode := range uncleNodes {
-		if err := shared.PublishIPLD(tx, uncleNode); err != nil {
-			return nil, err
-		}
-		uncleReward := CalcUncleMinerReward(ipldPayload.Block.Number().Int64(), uncleNode.Number.Int64())
-		uncle := UncleModel{
-			CID:        uncleNode.Cid().String(),
-			MhKey:      shared.MultihashKeyFromCID(uncleNode.Cid()),
-			ParentHash: uncleNode.ParentHash.String(),
-			BlockHash:  uncleNode.Hash().String(),
-			Reward:     uncleReward.String(),
-		}
-		if err := pub.indexer.indexUncleCID(tx, uncle, headerID); err != nil {
-			return nil, err
-		}
-	}
-
-	// Publish and index txs and receipts
-	for i, txNode := range txNodes {
-		if err := shared.PublishIPLD(tx, txNode); err != nil {
-			return nil, err
-		}
-		rctNode := rctNodes[i]
-		if err := shared.PublishIPLD(tx, rctNode); err != nil {
-			return nil, err
-		}
-		txModel := ipldPayload.TxMetaData[i]
-		txModel.CID = txNode.Cid().String()
-		txModel.MhKey = shared.MultihashKeyFromCID(txNode.Cid())
-		txID, err := pub.indexer.indexTransactionCID(tx, txModel, headerID)
-		if err != nil {
-			return nil, err
-		}
-		rctModel := ipldPayload.ReceiptMetaData[i]
-		rctModel.CID = rctNode.Cid().String()
-		rctModel.MhKey = shared.MultihashKeyFromCID(rctNode.Cid())
-		if err := pub.indexer.indexReceiptCID(tx, rctModel, txID); err != nil {
-			return nil, err
-		}
-	}
-
-	// Publish and index state and storage
-	err = pub.publishAndIndexStateAndStorage(tx, ipldPayload, headerID)
-
-	// This IPLDPublisher does both publishing and indexing, we do not need to pass anything forward to the indexer
-	return nil, err // return err variable explicitly so that we return the err = tx.Commit() assignment in the defer
-}
-
-func (pub *IPLDPublisherAndIndexer) publishAndIndexStateAndStorage(tx *sqlx.Tx, ipldPayload ConvertedPayload, headerID int64) error {
-	// Publish and index state and storage
-	for _, stateNode := range ipldPayload.StateNodes {
-		stateCIDStr, err := shared.PublishRaw(tx, ipld.MEthStateTrie, multihash.KECCAK_256, stateNode.Value)
-		if err != nil {
-			return err
-		}
-		mhKey, _ := shared.MultihashKeyFromCIDString(stateCIDStr)
-		stateModel := StateNodeModel{
-			Path:     stateNode.Path,
-			StateKey: stateNode.LeafKey.String(),
-			CID:      stateCIDStr,
-			MhKey:    mhKey,
-			NodeType: ResolveFromNodeType(stateNode.Type),
-		}
-		stateID, err := pub.indexer.indexStateCID(tx, stateModel, headerID)
-		if err != nil {
-			return err
-		}
-		// If we have a leaf, decode and index the account data and any associated storage diffs
-		if stateNode.Type == statediff.Leaf {
-			var i []interface{}
-			if err := rlp.DecodeBytes(stateNode.Value, &i); err != nil {
-				return err
-			}
-			if len(i) != 2 {
-				return fmt.Errorf("eth IPLDPublisherAndIndexer expected state leaf node rlp to decode into two elements")
-			}
-			var account state.Account
-			if err := rlp.DecodeBytes(i[1].([]byte), &account); err != nil {
-				return err
-			}
-			accountModel := StateAccountModel{
-				Balance:     account.Balance.String(),
-				Nonce:       account.Nonce,
-				CodeHash:    account.CodeHash,
-				StorageRoot: account.Root.String(),
-			}
-			if err := pub.indexer.indexStateAccount(tx, accountModel, stateID); err != nil {
-				return err
-			}
-			for _, storageNode := range ipldPayload.StorageNodes[common.Bytes2Hex(stateNode.Path)] {
-				storageCIDStr, err := shared.PublishRaw(tx, ipld.MEthStorageTrie, multihash.KECCAK_256, storageNode.Value)
-				if err != nil {
-					return err
-				}
-				mhKey, _ := shared.MultihashKeyFromCIDString(storageCIDStr)
-				storageModel := StorageNodeModel{
-					Path:       storageNode.Path,
-					StorageKey: storageNode.LeafKey.Hex(),
-					CID:        storageCIDStr,
-					MhKey:      mhKey,
-					NodeType:   ResolveFromNodeType(storageNode.Type),
-				}
-				if err := pub.indexer.indexStorageCID(tx, storageModel, stateID); err != nil {
-					return err
-				}
-			}
-		}
-	}
-	return nil
-}
-
-// Index satisfies the shared.CIDIndexer interface
-func (pub *IPLDPublisherAndIndexer) Index(cids shared.CIDsForIndexing) error {
-	return nil
-}

pkg/eth/publish_and_indexer_test.go

@@ -1,238 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package eth_test
-
-import (
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ipfs/go-cid"
-	"github.com/ipfs/go-ipfs-blockstore"
-	"github.com/ipfs/go-ipfs-ds-help"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	"github.com/vulcanize/ipfs-blockchain-watcher/pkg/eth"
-	"github.com/vulcanize/ipfs-blockchain-watcher/pkg/eth/mocks"
-	"github.com/vulcanize/ipfs-blockchain-watcher/pkg/postgres"
-	"github.com/vulcanize/ipfs-blockchain-watcher/pkg/shared"
-)
-
-var _ = Describe("PublishAndIndexer", func() {
-	var (
-		db        *postgres.DB
-		err       error
-		repo      *eth.IPLDPublisherAndIndexer
-		ipfsPgGet = `SELECT data FROM public.blocks
-					WHERE key = $1`
-	)
-	BeforeEach(func() {
-		db, err = shared.SetupDB()
-		Expect(err).ToNot(HaveOccurred())
-		repo = eth.NewIPLDPublisherAndIndexer(db)
-	})
-	AfterEach(func() {
-		eth.TearDownDB(db)
-	})
-
-	Describe("Publish", func() {
-		It("Published and indexes header IPLDs in a single tx", func() {
-			emptyReturn, err := repo.Publish(mocks.MockConvertedPayload)
-			Expect(emptyReturn).To(BeNil())
-			Expect(err).ToNot(HaveOccurred())
-			pgStr := `SELECT cid, td, reward, id
-				FROM eth.header_cids
-				WHERE block_number = $1`
-			// check header was properly indexed
-			type res struct {
-				CID    string
-				TD     string
-				Reward string
-				ID     int
-			}
-			header := new(res)
-			err = db.QueryRowx(pgStr, 1).StructScan(header)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(header.CID).To(Equal(mocks.HeaderCID.String()))
-			Expect(header.TD).To(Equal(mocks.MockBlock.Difficulty().String()))
-			Expect(header.Reward).To(Equal("5000000000000000000"))
-			dc, err := cid.Decode(header.CID)
-			Expect(err).ToNot(HaveOccurred())
-			mhKey := dshelp.MultihashToDsKey(dc.Hash())
-			prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
-			var data []byte
-			err = db.Get(&data, ipfsPgGet, prefixedKey)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(data).To(Equal(mocks.MockHeaderRlp))
-		})
-
-		It("Publishes and indexes transaction IPLDs in a single tx", func() {
-			emptyReturn, err := repo.Publish(mocks.MockConvertedPayload)
-			Expect(emptyReturn).To(BeNil())
-			Expect(err).ToNot(HaveOccurred())
-			// check that txs were properly indexed
-			trxs := make([]string, 0)
-			pgStr := `SELECT transaction_cids.cid FROM eth.transaction_cids INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id)
-				WHERE header_cids.block_number = $1`
-			err = db.Select(&trxs, pgStr, 1)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(trxs)).To(Equal(3))
-			Expect(shared.ListContainsString(trxs, mocks.Trx1CID.String())).To(BeTrue())
-			Expect(shared.ListContainsString(trxs, mocks.Trx2CID.String())).To(BeTrue())
-			Expect(shared.ListContainsString(trxs, mocks.Trx3CID.String())).To(BeTrue())
-			// and published
-			for _, c := range trxs {
-				dc, err := cid.Decode(c)
-				Expect(err).ToNot(HaveOccurred())
-				mhKey := dshelp.MultihashToDsKey(dc.Hash())
-				prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
-				var data []byte
-				err = db.Get(&data, ipfsPgGet, prefixedKey)
-				Expect(err).ToNot(HaveOccurred())
-				switch c {
-				case mocks.Trx1CID.String():
-					Expect(data).To(Equal(mocks.MockTransactions.GetRlp(0)))
-				case mocks.Trx2CID.String():
-					Expect(data).To(Equal(mocks.MockTransactions.GetRlp(1)))
-				case mocks.Trx3CID.String():
-					Expect(data).To(Equal(mocks.MockTransactions.GetRlp(2)))
-				}
-			}
-		})
-
-		It("Publishes and indexes receipt IPLDs in a single tx", func() {
-			emptyReturn, err := repo.Publish(mocks.MockConvertedPayload)
-			Expect(emptyReturn).To(BeNil())
-			Expect(err).ToNot(HaveOccurred())
-			// check receipts were properly indexed
-			rcts := make([]string, 0)
-			pgStr := `SELECT receipt_cids.cid FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
-				WHERE receipt_cids.tx_id = transaction_cids.id 
-				AND transaction_cids.header_id = header_cids.id
-				AND header_cids.block_number = $1`
-			err = db.Select(&rcts, pgStr, 1)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(rcts)).To(Equal(3))
-			Expect(shared.ListContainsString(rcts, mocks.Rct1CID.String())).To(BeTrue())
-			Expect(shared.ListContainsString(rcts, mocks.Rct2CID.String())).To(BeTrue())
-			Expect(shared.ListContainsString(rcts, mocks.Rct3CID.String())).To(BeTrue())
-			// and published
-			for _, c := range rcts {
-				dc, err := cid.Decode(c)
-				Expect(err).ToNot(HaveOccurred())
-				mhKey := dshelp.MultihashToDsKey(dc.Hash())
-				prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
-				var data []byte
-				err = db.Get(&data, ipfsPgGet, prefixedKey)
-				Expect(err).ToNot(HaveOccurred())
-				switch c {
-				case mocks.Rct1CID.String():
-					Expect(data).To(Equal(mocks.MockReceipts.GetRlp(0)))
-				case mocks.Rct2CID.String():
-					Expect(data).To(Equal(mocks.MockReceipts.GetRlp(1)))
-				case mocks.Rct3CID.String():
-					Expect(data).To(Equal(mocks.MockReceipts.GetRlp(2)))
-				}
-			}
-		})
-
-		It("Publishes and indexes state IPLDs in a single tx", func() {
-			emptyReturn, err := repo.Publish(mocks.MockConvertedPayload)
-			Expect(emptyReturn).To(BeNil())
-			Expect(err).ToNot(HaveOccurred())
-			// check that state nodes were properly indexed and published
-			stateNodes := make([]eth.StateNodeModel, 0)
-			pgStr := `SELECT state_cids.id, state_cids.cid, state_cids.state_leaf_key, state_cids.node_type, state_cids.state_path, state_cids.header_id
-				FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
-				WHERE header_cids.block_number = $1`
-			err = db.Select(&stateNodes, pgStr, 1)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(stateNodes)).To(Equal(2))
-			for _, stateNode := range stateNodes {
-				var data []byte
-				dc, err := cid.Decode(stateNode.CID)
-				Expect(err).ToNot(HaveOccurred())
-				mhKey := dshelp.MultihashToDsKey(dc.Hash())
-				prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
-				err = db.Get(&data, ipfsPgGet, prefixedKey)
-				Expect(err).ToNot(HaveOccurred())
-				pgStr = `SELECT * from eth.state_accounts WHERE state_id = $1`
-				var account eth.StateAccountModel
-				err = db.Get(&account, pgStr, stateNode.ID)
-				Expect(err).ToNot(HaveOccurred())
-				if stateNode.CID == mocks.State1CID.String() {
-					Expect(stateNode.NodeType).To(Equal(2))
-					Expect(stateNode.StateKey).To(Equal(common.BytesToHash(mocks.ContractLeafKey).Hex()))
-					Expect(stateNode.Path).To(Equal([]byte{'\x06'}))
-					Expect(data).To(Equal(mocks.ContractLeafNode))
-					Expect(account).To(Equal(eth.StateAccountModel{
-						ID:          account.ID,
-						StateID:     stateNode.ID,
-						Balance:     "0",
-						CodeHash:    mocks.ContractCodeHash.Bytes(),
-						StorageRoot: mocks.ContractRoot,
-						Nonce:       1,
-					}))
-				}
-				if stateNode.CID == mocks.State2CID.String() {
-					Expect(stateNode.NodeType).To(Equal(2))
-					Expect(stateNode.StateKey).To(Equal(common.BytesToHash(mocks.AccountLeafKey).Hex()))
-					Expect(stateNode.Path).To(Equal([]byte{'\x0c'}))
-					Expect(data).To(Equal(mocks.AccountLeafNode))
-					Expect(account).To(Equal(eth.StateAccountModel{
-						ID:          account.ID,
-						StateID:     stateNode.ID,
-						Balance:     "1000",
-						CodeHash:    mocks.AccountCodeHash.Bytes(),
-						StorageRoot: mocks.AccountRoot,
-						Nonce:       0,
-					}))
-				}
-			}
-			pgStr = `SELECT * from eth.state_accounts WHERE state_id = $1`
-		})
-
-		It("Publishes and indexes storage IPLDs in a single tx", func() {
-			emptyReturn, err := repo.Publish(mocks.MockConvertedPayload)
-			Expect(emptyReturn).To(BeNil())
-			Expect(err).ToNot(HaveOccurred())
-			// check that storage nodes were properly indexed
-			storageNodes := make([]eth.StorageNodeWithStateKeyModel, 0)
-			pgStr := `SELECT storage_cids.cid, state_cids.state_leaf_key, storage_cids.storage_leaf_key, storage_cids.node_type, storage_cids.storage_path 
-				FROM eth.storage_cids, eth.state_cids, eth.header_cids
-				WHERE storage_cids.state_id = state_cids.id 
-				AND state_cids.header_id = header_cids.id
-				AND header_cids.block_number = $1`
-			err = db.Select(&storageNodes, pgStr, 1)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(storageNodes)).To(Equal(1))
-			Expect(storageNodes[0]).To(Equal(eth.StorageNodeWithStateKeyModel{
-				CID:        mocks.StorageCID.String(),
-				NodeType:   2,
-				StorageKey: common.BytesToHash(mocks.StorageLeafKey).Hex(),
-				StateKey:   common.BytesToHash(mocks.ContractLeafKey).Hex(),
-				Path:       []byte{},
-			}))
-			var data []byte
-			dc, err := cid.Decode(storageNodes[0].CID)
-			Expect(err).ToNot(HaveOccurred())
-			mhKey := dshelp.MultihashToDsKey(dc.Hash())
-			prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
-			err = db.Get(&data, ipfsPgGet, prefixedKey)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(data).To(Equal(mocks.StorageLeafNode))
-		})
-	})
-})

pkg/eth/publisher.go

@@ -31,36 +31,36 @@ import (
 	"github.com/vulcanize/ipfs-blockchain-watcher/pkg/shared"
 )
 
-// IPLDPublisherAndIndexer satisfies the IPLDPublisher interface for ethereum
+// IPLDPublisher satisfies the IPLDPublisher interface for ethereum
 // It interfaces directly with the public.blocks table of PG-IPFS rather than going through an ipfs intermediary
 // It publishes and indexes IPLDs together in a single sqlx.Tx
-type IPLDPublisherAndIndexer struct {
+type IPLDPublisher struct {
 	indexer *CIDIndexer
 }
 
-// NewIPLDPublisherAndIndexer creates a pointer to a new IPLDPublisherAndIndexer which satisfies the IPLDPublisher interface
-func NewIPLDPublisherAndIndexer(db *postgres.DB) *IPLDPublisherAndIndexer {
-	return &IPLDPublisherAndIndexer{
+// NewIPLDPublisher creates a pointer to a new IPLDPublisher which satisfies the IPLDPublisher interface
+func NewIPLDPublisher(db *postgres.DB) *IPLDPublisher {
+	return &IPLDPublisher{
 		indexer: NewCIDIndexer(db),
 	}
 }
 
 // Publish publishes an IPLDPayload to IPFS and returns the corresponding CIDPayload
-func (pub *IPLDPublisherAndIndexer) Publish(payload shared.ConvertedData) (shared.CIDsForIndexing, error) {
+func (pub *IPLDPublisher) Publish(payload shared.ConvertedData) error {
 	ipldPayload, ok := payload.(ConvertedPayload)
 	if !ok {
-		return nil, fmt.Errorf("eth IPLDPublisherAndIndexer expected payload type %T got %T", ConvertedPayload{}, payload)
+		return fmt.Errorf("eth IPLDPublisher expected payload type %T got %T", ConvertedPayload{}, payload)
 	}
 	// Generate the iplds
 	headerNode, uncleNodes, txNodes, txTrieNodes, rctNodes, rctTrieNodes, err := ipld.FromBlockAndReceipts(ipldPayload.Block, ipldPayload.Receipts)
 	if err != nil {
-		return nil, err
+		return err
 	}
 
 	// Begin new db tx
 	tx, err := pub.indexer.db.Beginx()
 	if err != nil {
-		return nil, err
+		return err
 	}
 	defer func() {
 		if p := recover(); p != nil {
@@ -76,18 +76,18 @@ func (pub *IPLDPublisherAndIndexer) Publish(payload shared.ConvertedData) (share
 	// Publish trie nodes
 	for _, node := range txTrieNodes {
 		if err := shared.PublishIPLD(tx, node); err != nil {
-			return nil, err
+			return err
 		}
 	}
 	for _, node := range rctTrieNodes {
 		if err := shared.PublishIPLD(tx, node); err != nil {
-			return nil, err
+			return err
 		}
 	}
 
 	// Publish and index header
 	if err := shared.PublishIPLD(tx, headerNode); err != nil {
-		return nil, err
+		return err
 	}
 	reward := CalcEthBlockReward(ipldPayload.Block.Header(), ipldPayload.Block.Uncles(), ipldPayload.Block.Transactions(), ipldPayload.Receipts)
 	header := HeaderModel{
@@ -107,13 +107,13 @@ func (pub *IPLDPublisherAndIndexer) Publish(payload shared.ConvertedData) (share
 	}
 	headerID, err := pub.indexer.indexHeaderCID(tx, header)
 	if err != nil {
-		return nil, err
+		return err
 	}
 
 	// Publish and index uncles
 	for _, uncleNode := range uncleNodes {
 		if err := shared.PublishIPLD(tx, uncleNode); err != nil {
-			return nil, err
+			return err
 		}
 		uncleReward := CalcUncleMinerReward(ipldPayload.Block.Number().Int64(), uncleNode.Number.Int64())
 		uncle := UncleModel{
@@ -124,47 +124,47 @@ func (pub *IPLDPublisherAndIndexer) Publish(payload shared.ConvertedData) (share
 			Reward:     uncleReward.String(),
 		}
 		if err := pub.indexer.indexUncleCID(tx, uncle, headerID); err != nil {
-			return nil, err
+			return err
 		}
 	}
 
 	// Publish and index txs and receipts
 	for i, txNode := range txNodes {
 		if err := shared.PublishIPLD(tx, txNode); err != nil {
-			return nil, err
+			return err
 		}
 		rctNode := rctNodes[i]
 		if err := shared.PublishIPLD(tx, rctNode); err != nil {
-			return nil, err
+			return err
 		}
 		txModel := ipldPayload.TxMetaData[i]
 		txModel.CID = txNode.Cid().String()
 		txModel.MhKey = shared.MultihashKeyFromCID(txNode.Cid())
 		txID, err := pub.indexer.indexTransactionCID(tx, txModel, headerID)
 		if err != nil {
-			return nil, err
+			return err
 		}
+		// If tx is a contract deployment, publish the data (code)
 		if txModel.Deployment {
 			if _, err = shared.PublishRaw(tx, ipld.MEthStorageTrie, multihash.KECCAK_256, txModel.Data); err != nil {
-				return nil, err
+				return err
 			}
 		}
 		rctModel := ipldPayload.ReceiptMetaData[i]
 		rctModel.CID = rctNode.Cid().String()
 		rctModel.MhKey = shared.MultihashKeyFromCID(rctNode.Cid())
 		if err := pub.indexer.indexReceiptCID(tx, rctModel, txID); err != nil {
-			return nil, err
+			return err
 		}
 	}
 
 	// Publish and index state and storage
 	err = pub.publishAndIndexStateAndStorage(tx, ipldPayload, headerID)
 
-	// This IPLDPublisher does both publishing and indexing, we do not need to pass anything forward to the indexer
-	return nil, err // return err variable explicitly so that we return the err = tx.Commit() assignment in the defer
+	return err // return err variable explicitly so that we return the err = tx.Commit() assignment in the defer
 }
 
-func (pub *IPLDPublisherAndIndexer) publishAndIndexStateAndStorage(tx *sqlx.Tx, ipldPayload ConvertedPayload, headerID int64) error {
+func (pub *IPLDPublisher) publishAndIndexStateAndStorage(tx *sqlx.Tx, ipldPayload ConvertedPayload, headerID int64) error {
 	// Publish and index state and storage
 	for _, stateNode := range ipldPayload.StateNodes {
 		stateCIDStr, err := shared.PublishRaw(tx, ipld.MEthStateTrie, multihash.KECCAK_256, stateNode.Value)
@@ -190,7 +190,7 @@ func (pub *IPLDPublisherAndIndexer) publishAndIndexStateAndStorage(tx *sqlx.Tx,
 				return err
 			}
 			if len(i) != 2 {
-				return fmt.Errorf("eth IPLDPublisherAndIndexer expected state leaf node rlp to decode into two elements")
+				return fmt.Errorf("eth IPLDPublisher expected state leaf node rlp to decode into two elements")
 			}
 			var account state.Account
 			if err := rlp.DecodeBytes(i[1].([]byte), &account); err != nil {
@@ -226,8 +226,3 @@ func (pub *IPLDPublisherAndIndexer) publishAndIndexStateAndStorage(tx *sqlx.Tx,
 	}
 	return nil
 }
-
-// Index satisfies the shared.CIDIndexer interface
-func (pub *IPLDPublisherAndIndexer) Index(cids shared.CIDsForIndexing) error {
-	return nil
-}

pkg/eth/publisher_test.go

@@ -34,14 +34,14 @@ var _ = Describe("PublishAndIndexer", func() {
 	var (
 		db        *postgres.DB
 		err       error
-		repo      *eth.IPLDPublisherAndIndexer
+		repo      *eth.IPLDPublisher
 		ipfsPgGet = `SELECT data FROM public.blocks
 					WHERE key = $1`
 	)
 	BeforeEach(func() {
 		db, err = shared.SetupDB()
 		Expect(err).ToNot(HaveOccurred())
-		repo = eth.NewIPLDPublisherAndIndexer(db)
+		repo = eth.NewIPLDPublisher(db)
 	})
 	AfterEach(func() {
 		eth.TearDownDB(db)
@@ -49,8 +49,7 @@ var _ = Describe("PublishAndIndexer", func() {
 
 	Describe("Publish", func() {
 		It("Published and indexes header IPLDs in a single tx", func() {
-			emptyReturn, err := repo.Publish(mocks.MockConvertedPayload)
-			Expect(emptyReturn).To(BeNil())
+			err = repo.Publish(mocks.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
 			pgStr := `SELECT cid, td, reward, id
 				FROM eth.header_cids
@@ -79,8 +78,7 @@ var _ = Describe("PublishAndIndexer", func() {
 		})
 
 		It("Publishes and indexes transaction IPLDs in a single tx", func() {
-			emptyReturn, err := repo.Publish(mocks.MockConvertedPayload)
-			Expect(emptyReturn).To(BeNil())
+			err = repo.Publish(mocks.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
 			// check that txs were properly indexed
 			trxs := make([]string, 0)
@@ -113,8 +111,7 @@ var _ = Describe("PublishAndIndexer", func() {
 		})
 
 		It("Publishes and indexes receipt IPLDs in a single tx", func() {
-			emptyReturn, err := repo.Publish(mocks.MockConvertedPayload)
-			Expect(emptyReturn).To(BeNil())
+			err = repo.Publish(mocks.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
 			// check receipts were properly indexed
 			rcts := make([]string, 0)
@@ -149,8 +146,7 @@ var _ = Describe("PublishAndIndexer", func() {
 		})
 
 		It("Publishes and indexes state IPLDs in a single tx", func() {
-			emptyReturn, err := repo.Publish(mocks.MockConvertedPayload)
-			Expect(emptyReturn).To(BeNil())
+			err = repo.Publish(mocks.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
 			// check that state nodes were properly indexed and published
 			stateNodes := make([]eth.StateNodeModel, 0)
@@ -205,8 +201,7 @@ var _ = Describe("PublishAndIndexer", func() {
 		})
 
 		It("Publishes and indexes storage IPLDs in a single tx", func() {
-			emptyReturn, err := repo.Publish(mocks.MockConvertedPayload)
-			Expect(emptyReturn).To(BeNil())
+			err = repo.Publish(mocks.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
 			// check that storage nodes were properly indexed
 			storageNodes := make([]eth.StorageNodeWithStateKeyModel, 0)