deep-stack/ipld-eth-server
6
0
Fork 0
forked from cerc-io/ipld-eth-server
Code Pull Requests Activity

Merge pull request #11 from vulcanize/doc

Browse Source 
Testnet fixes
This commit is contained in:
Ian Norden 2020-08-12 09:36:16 -05:00 committed by GitHub
commit d842a5e796
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
63 changed files with 793 additions and 2895 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
README.md
View File

@ -161,10 +161,6 @@ This set of parameters needs to be set no matter the chain type.
user = "vdbm" # $DATABASE_USER user = "vdbm" # $DATABASE_USER
password = "" # $DATABASE_PASSWORD password = "" # $DATABASE_PASSWORD
[ipfs]
path = "~/.ipfs" # $IPFS_PATH
mode = "postgres" # $IPFS_MODE
[watcher] [watcher]
chain = "bitcoin" # $SUPERNODE_CHAIN chain = "bitcoin" # $SUPERNODE_CHAIN
server = true # $SUPERNODE_SERVER server = true # $SUPERNODE_SERVER
@ -207,6 +203,7 @@ For Ethereum:
clientName = "Geth" # $ETH_CLIENT_NAME clientName = "Geth" # $ETH_CLIENT_NAME
genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" # $ETH_GENESIS_BLOCK genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" # $ETH_GENESIS_BLOCK
networkID = "1" # $ETH_NETWORK_ID networkID = "1" # $ETH_NETWORK_ID
chainID = "1" # $ETH_CHAIN_ID
``` ```
### Exposing the data ### Exposing the data

8
cmd/resync.go
View File

@ -19,9 +19,6 @@ import (
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/spf13/viper" "github.com/spf13/viper"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/shared"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/resync" "github.com/vulcanize/ipfs-blockchain-watcher/pkg/resync"
v "github.com/vulcanize/ipfs-blockchain-watcher/version" v "github.com/vulcanize/ipfs-blockchain-watcher/version"
) )
@ -46,11 +43,6 @@ func rsyncCmdCommand() {
logWithCommand.Fatal(err) logWithCommand.Fatal(err)
} }
logWithCommand.Infof("resync config: %+v", rConfig) logWithCommand.Infof("resync config: %+v", rConfig)
if rConfig.IPFSMode == shared.LocalInterface {
if err := ipfs.InitIPFSPlugins(); err != nil {
logWithCommand.Fatal(err)
}
}
logWithCommand.Debug("initializing new resync service") logWithCommand.Debug("initializing new resync service")
rService, err := resync.NewResyncService(rConfig) rService, err := resync.NewResyncService(rConfig)
if err != nil { if err != nil {

6
cmd/watch.go
View File

@ -26,7 +26,6 @@ import (
"github.com/spf13/viper" "github.com/spf13/viper"
h "github.com/vulcanize/ipfs-blockchain-watcher/pkg/historical" h "github.com/vulcanize/ipfs-blockchain-watcher/pkg/historical"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/shared" "github.com/vulcanize/ipfs-blockchain-watcher/pkg/shared"
w "github.com/vulcanize/ipfs-blockchain-watcher/pkg/watch" w "github.com/vulcanize/ipfs-blockchain-watcher/pkg/watch"
v "github.com/vulcanize/ipfs-blockchain-watcher/version" v "github.com/vulcanize/ipfs-blockchain-watcher/version"
@ -65,11 +64,6 @@ func watch() {
logWithCommand.Fatal(err) logWithCommand.Fatal(err)
} }
logWithCommand.Infof("watcher config: %+v", watcherConfig) logWithCommand.Infof("watcher config: %+v", watcherConfig)
if watcherConfig.IPFSMode == shared.LocalInterface {
if err := ipfs.InitIPFSPlugins(); err != nil {
logWithCommand.Fatal(err)
}
}
logWithCommand.Debug("initializing new watcher service") logWithCommand.Debug("initializing new watcher service")
watcher, err := w.NewWatcher(watcherConfig) watcher, err := w.NewWatcher(watcherConfig)
if err != nil { if err != nil {

2
db/migrations/00006_create_eth_transaction_cids_table.sql
View File

@ -8,6 +8,8 @@ CREATE TABLE eth.transaction_cids (
mh_key TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, mh_key TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
dst VARCHAR(66) NOT NULL, dst VARCHAR(66) NOT NULL,
src VARCHAR(66) NOT NULL, src VARCHAR(66) NOT NULL,
deployment BOOL NOT NULL,
data BYTEA,
UNIQUE (header_id, tx_hash) UNIQUE (header_id, tx_hash)
); );

4
db/schema.sql
View File

@ -415,7 +415,9 @@ CREATE TABLE eth.transaction_cids (
cid text NOT NULL, cid text NOT NULL,
mh_key text NOT NULL, mh_key text NOT NULL,
dst character varying(66) NOT NULL, dst character varying(66) NOT NULL,
src character varying(66) NOT NULL src character varying(66) NOT NULL,
deployment boolean NOT NULL,
data bytea
); );

1
dockerfiles/super_node/entrypoint.sh
View File

@ -13,7 +13,6 @@ set +x
#test $DATABASE_USER #test $DATABASE_USER
#test $DATABASE_PASSWORD #test $DATABASE_PASSWORD
#test $IPFS_INIT #test $IPFS_INIT
#test $IPFS_PATH
VDB_COMMAND=${VDB_COMMAND:-watch} VDB_COMMAND=${VDB_COMMAND:-watch}
set +e set +e

1
dockerfiles/super_node/startup_script.sh
View File

@ -12,7 +12,6 @@ test $DATABASE_PORT
test $DATABASE_USER test $DATABASE_USER
test $DATABASE_PASSWORD test $DATABASE_PASSWORD
test $IPFS_INIT test $IPFS_INIT
test $IPFS_PATH
test $VDB_COMMAND test $VDB_COMMAND
set +e set +e

4
documentation/architecture.md
View File

@ -52,10 +52,6 @@ This set of parameters needs to be set no matter the chain type.
user = "vdbm" # $DATABASE_USER user = "vdbm" # $DATABASE_USER
password = "" # $DATABASE_PASSWORD password = "" # $DATABASE_PASSWORD
[ipfs]
path = "~/.ipfs" # $IPFS_PATH
mode = "direct" # $IPFS_MODE
[watcher] [watcher]
chain = "bitcoin" # $SUPERNODE_CHAIN chain = "bitcoin" # $SUPERNODE_CHAIN
server = true # $SUPERNODE_SERVER server = true # $SUPERNODE_SERVER

3
documentation/resync.md
View File

@ -30,9 +30,6 @@ This set of parameters needs to be set no matter the chain type.
port = 5432 # $DATABASE_PORT port = 5432 # $DATABASE_PORT
user = "vdbm" # $DATABASE_USER user = "vdbm" # $DATABASE_USER
password = "" # $DATABASE_PASSWORD password = "" # $DATABASE_PASSWORD
[ipfs]
path = "~/.ipfs" # $IPFS_PATH
[resync] [resync]
chain = "ethereum" # $RESYNC_CHAIN chain = "ethereum" # $RESYNC_CHAIN

1
environments/superNodeETH.toml
View File

@ -43,3 +43,4 @@
clientName = "Geth" # $ETH_CLIENT_NAME clientName = "Geth" # $ETH_CLIENT_NAME
genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" # $ETH_GENESIS_BLOCK genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" # $ETH_GENESIS_BLOCK
networkID = "1" # $ETH_NETWORK_ID networkID = "1" # $ETH_NETWORK_ID
chainID = "1" # $ETH_CHAIN_ID

100
pkg/btc/ipld_fetcher.go
View File

@ -17,38 +17,27 @@
package btc package btc
import ( import (
"context"
"errors"
"fmt" "fmt"
"github.com/ipfs/go-block-format" "github.com/jmoiron/sqlx"
"github.com/ipfs/go-blockservice"
"github.com/ipfs/go-cid"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs" "github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/postgres"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/shared" "github.com/vulcanize/ipfs-blockchain-watcher/pkg/shared"
) )
var (
errUnexpectedNumberOfIPLDs = errors.New("ipfs batch fetch returned unexpected number of IPLDs")
)
// IPLDFetcher satisfies the IPLDFetcher interface for ethereum // IPLDFetcher satisfies the IPLDFetcher interface for ethereum
// it interfaces directly with PG-IPFS instead of going through a node-interface or remote node
type IPLDFetcher struct { type IPLDFetcher struct {
BlockService blockservice.BlockService db *postgres.DB
} }
// NewIPLDFetcher creates a pointer to a new IPLDFetcher // NewIPLDFetcher creates a pointer to a new IPLDFetcher
// It interfaces with PG-IPFS through an internalized IPFS node interface func NewIPLDFetcher(db *postgres.DB) *IPLDFetcher {
func NewIPLDFetcher(ipfsPath string) (*IPLDFetcher, error) {
blockService, err := ipfs.InitIPFSBlockService(ipfsPath)
if err != nil {
return nil, err
}
return &IPLDFetcher{ return &IPLDFetcher{
BlockService: blockService, db: db,
}, nil }
} }
// Fetch is the exported method for fetching and returning all the IPLDS specified in the CIDWrapper // Fetch is the exported method for fetching and returning all the IPLDS specified in the CIDWrapper
@ -60,76 +49,59 @@ func (f *IPLDFetcher) Fetch(cids shared.CIDsForFetching) (shared.IPLDs, error) {
log.Debug("fetching iplds") log.Debug("fetching iplds")
iplds := IPLDs{} iplds := IPLDs{}
iplds.BlockNumber = cidWrapper.BlockNumber iplds.BlockNumber = cidWrapper.BlockNumber
var err error
iplds.Header, err = f.FetchHeader(cidWrapper.Header) tx, err := f.db.Beginx()
if err != nil { if err != nil {
return nil, err return nil, err
} }
iplds.Transactions, err = f.FetchTrxs(cidWrapper.Transactions) defer func() {
if p := recover(); p != nil {
shared.Rollback(tx)
panic(p)
} else if err != nil {
shared.Rollback(tx)
} else {
err = tx.Commit()
}
}()
iplds.Header, err = f.FetchHeader(tx, cidWrapper.Header)
if err != nil { if err != nil {
return nil, err return nil, fmt.Errorf("btc pg fetcher: header fetching error: %s", err.Error())
} }
return iplds, nil iplds.Transactions, err = f.FetchTrxs(tx, cidWrapper.Transactions)
if err != nil {
return nil, fmt.Errorf("btc pg fetcher: transaction fetching error: %s", err.Error())
}
return iplds, err
} }
// FetchHeaders fetches headers // FetchHeaders fetches headers
// It uses the f.fetch method func (f *IPLDFetcher) FetchHeader(tx *sqlx.Tx, c HeaderModel) (ipfs.BlockModel, error) {
func (f *IPLDFetcher) FetchHeader(c HeaderModel) (ipfs.BlockModel, error) {
log.Debug("fetching header ipld") log.Debug("fetching header ipld")
dc, err := cid.Decode(c.CID) headerBytes, err := shared.FetchIPLDByMhKey(tx, c.MhKey)
if err != nil {
return ipfs.BlockModel{}, err
}
header, err := f.fetch(dc)
if err != nil { if err != nil {
return ipfs.BlockModel{}, err return ipfs.BlockModel{}, err
} }
return ipfs.BlockModel{ return ipfs.BlockModel{
Data: header.RawData(), Data: headerBytes,
CID: header.Cid().String(), CID: c.CID,
}, nil }, nil
} }
// FetchTrxs fetches transactions // FetchTrxs fetches transactions
// It uses the f.fetchBatch method func (f *IPLDFetcher) FetchTrxs(tx *sqlx.Tx, cids []TxModel) ([]ipfs.BlockModel, error) {
func (f *IPLDFetcher) FetchTrxs(cids []TxModel) ([]ipfs.BlockModel, error) {
log.Debug("fetching transaction iplds") log.Debug("fetching transaction iplds")
trxCids := make([]cid.Cid, len(cids)) trxIPLDs := make([]ipfs.BlockModel, len(cids))
for i, c := range cids { for i, c := range cids {
dc, err := cid.Decode(c.CID) trxBytes, err := shared.FetchIPLDByMhKey(tx, c.MhKey)
if err != nil { if err != nil {
return nil, err return nil, err
} }
trxCids[i] = dc
}
trxs := f.fetchBatch(trxCids)
trxIPLDs := make([]ipfs.BlockModel, len(trxs))
for i, trx := range trxs {
trxIPLDs[i] = ipfs.BlockModel{ trxIPLDs[i] = ipfs.BlockModel{
Data: trx.RawData(), Data: trxBytes,
CID: trx.Cid().String(), CID: c.CID,
} }
} }
if len(trxIPLDs) != len(trxCids) {
log.Errorf("ipfs fetcher: number of transaction blocks returned (%d) does not match number expected (%d)", len(trxs), len(trxCids))
return trxIPLDs, errUnexpectedNumberOfIPLDs
}
return trxIPLDs, nil return trxIPLDs, nil
} }
// fetch is used to fetch a single cid
func (f *IPLDFetcher) fetch(cid cid.Cid) (blocks.Block, error) {
return f.BlockService.GetBlock(context.Background(), cid)
}
// fetchBatch is used to fetch a batch of IPFS data blocks by cid
// There is no guarantee all are fetched, and no error in such a case, so
// downstream we will need to confirm which CIDs were fetched in the result set
func (f *IPLDFetcher) fetchBatch(cids []cid.Cid) []blocks.Block {
fetchedBlocks := make([]blocks.Block, 0, len(cids))
blockChan := f.BlockService.GetBlocks(context.Background(), cids)
for block := range blockChan {
fetchedBlocks = append(fetchedBlocks, block)
}
return fetchedBlocks
}

107
pkg/btc/ipld_pg_fetcher.go
View File

@ -1,107 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package btc
import (
"fmt"
"github.com/jmoiron/sqlx"
log "github.com/sirupsen/logrus"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/postgres"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/shared"
)
// IPLDPGFetcher satisfies the IPLDFetcher interface for ethereum
// it interfaces directly with PG-IPFS instead of going through a node-interface or remote node
type IPLDPGFetcher struct {
db *postgres.DB
}
// NewIPLDPGFetcher creates a pointer to a new IPLDPGFetcher
func NewIPLDPGFetcher(db *postgres.DB) *IPLDPGFetcher {
return &IPLDPGFetcher{
db: db,
}
}
// Fetch is the exported method for fetching and returning all the IPLDS specified in the CIDWrapper
func (f *IPLDPGFetcher) Fetch(cids shared.CIDsForFetching) (shared.IPLDs, error) {
cidWrapper, ok := cids.(*CIDWrapper)
if !ok {
return nil, fmt.Errorf("btc fetcher: expected cids type %T got %T", &CIDWrapper{}, cids)
}
log.Debug("fetching iplds")
iplds := IPLDs{}
iplds.BlockNumber = cidWrapper.BlockNumber
tx, err := f.db.Beginx()
if err != nil {
return nil, err
}
defer func() {
if p := recover(); p != nil {
shared.Rollback(tx)
panic(p)
} else if err != nil {
shared.Rollback(tx)
} else {
err = tx.Commit()
}
}()
iplds.Header, err = f.FetchHeader(tx, cidWrapper.Header)
if err != nil {
return nil, fmt.Errorf("btc pg fetcher: header fetching error: %s", err.Error())
}
iplds.Transactions, err = f.FetchTrxs(tx, cidWrapper.Transactions)
if err != nil {
return nil, fmt.Errorf("btc pg fetcher: transaction fetching error: %s", err.Error())
}
return iplds, err
}
// FetchHeaders fetches headers
func (f *IPLDPGFetcher) FetchHeader(tx *sqlx.Tx, c HeaderModel) (ipfs.BlockModel, error) {
log.Debug("fetching header ipld")
headerBytes, err := shared.FetchIPLDByMhKey(tx, c.MhKey)
if err != nil {
return ipfs.BlockModel{}, err
}
return ipfs.BlockModel{
Data: headerBytes,
CID: c.CID,
}, nil
}
// FetchTrxs fetches transactions
func (f *IPLDPGFetcher) FetchTrxs(tx *sqlx.Tx, cids []TxModel) ([]ipfs.BlockModel, error) {
log.Debug("fetching transaction iplds")
trxIPLDs := make([]ipfs.BlockModel, len(cids))
for i, c := range cids {
trxBytes, err := shared.FetchIPLDByMhKey(tx, c.MhKey)
if err != nil {
return nil, err
}
trxIPLDs[i] = ipfs.BlockModel{
Data: trxBytes,
CID: c.CID,
}
}
return trxIPLDs, nil
}

126
pkg/btc/publish_and_indexer.go
View File

@ -1,126 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package btc
import (
"fmt"
"strconv"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs/ipld"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/postgres"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/shared"
)
// IPLDPublisherAndIndexer satisfies the IPLDPublisher interface for bitcoin
// It interfaces directly with the public.blocks table of PG-IPFS rather than going through an ipfs intermediary
// It publishes and indexes IPLDs together in a single sqlx.Tx
type IPLDPublisherAndIndexer struct {
indexer *CIDIndexer
}
// NewIPLDPublisherAndIndexer creates a pointer to a new IPLDPublisherAndIndexer which satisfies the IPLDPublisher interface
func NewIPLDPublisherAndIndexer(db *postgres.DB) *IPLDPublisherAndIndexer {
return &IPLDPublisherAndIndexer{
indexer: NewCIDIndexer(db),
}
}
// Publish publishes an IPLDPayload to IPFS and returns the corresponding CIDPayload
func (pub *IPLDPublisherAndIndexer) Publish(payload shared.ConvertedData) (shared.CIDsForIndexing, error) {
ipldPayload, ok := payload.(ConvertedPayload)
if !ok {
return nil, fmt.Errorf("btc publisher expected payload type %T got %T", ConvertedPayload{}, payload)
}
// Generate the iplds
headerNode, txNodes, txTrieNodes, err := ipld.FromHeaderAndTxs(ipldPayload.Header, ipldPayload.Txs)
if err != nil {
return nil, err
}
// Begin new db tx
tx, err := pub.indexer.db.Beginx()
if err != nil {
return nil, err
}
defer func() {
if p := recover(); p != nil {
shared.Rollback(tx)
panic(p)
} else if err != nil {
shared.Rollback(tx)
} else {
err = tx.Commit()
}
}()
// Publish trie nodes
for _, node := range txTrieNodes {
if err := shared.PublishIPLD(tx, node); err != nil {
return nil, err
}
}
// Publish and index header
if err := shared.PublishIPLD(tx, headerNode); err != nil {
return nil, err
}
header := HeaderModel{
CID: headerNode.Cid().String(),
MhKey: shared.MultihashKeyFromCID(headerNode.Cid()),
ParentHash: ipldPayload.Header.PrevBlock.String(),
BlockNumber: strconv.Itoa(int(ipldPayload.BlockPayload.BlockHeight)),
BlockHash: ipldPayload.Header.BlockHash().String(),
Timestamp: ipldPayload.Header.Timestamp.UnixNano(),
Bits: ipldPayload.Header.Bits,
}
headerID, err := pub.indexer.indexHeaderCID(tx, header)
if err != nil {
return nil, err
}
// Publish and index txs
for i, txNode := range txNodes {
if err := shared.PublishIPLD(tx, txNode); err != nil {
return nil, err
}
txModel := ipldPayload.TxMetaData[i]
txModel.CID = txNode.Cid().String()
txModel.MhKey = shared.MultihashKeyFromCID(txNode.Cid())
txID, err := pub.indexer.indexTransactionCID(tx, txModel, headerID)
if err != nil {
return nil, err
}
for _, input := range txModel.TxInputs {
if err := pub.indexer.indexTxInput(tx, input, txID); err != nil {
return nil, err
}
}
for _, output := range txModel.TxOutputs {
if err := pub.indexer.indexTxOutput(tx, output, txID); err != nil {
return nil, err
}
}
}
// This IPLDPublisher does both publishing and indexing, we do not need to pass anything forward to the indexer
return nil, err
}
// Index satisfies the shared.CIDIndexer interface
func (pub *IPLDPublisherAndIndexer) Index(cids shared.CIDsForIndexing) error {
return nil
}

121
pkg/btc/publish_and_indexer_test.go
View File

@ -1,121 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package btc_test
import (
"bytes"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-ipfs-blockstore"
"github.com/ipfs/go-ipfs-ds-help"
"github.com/multiformats/go-multihash"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs/ipld"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/btc"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/btc/mocks"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/postgres"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/shared"
)
var _ = Describe("PublishAndIndexer", func() {
var (
db *postgres.DB
err error
repo *btc.IPLDPublisherAndIndexer
ipfsPgGet = `SELECT data FROM public.blocks
WHERE key = $1`
)
BeforeEach(func() {
db, err = shared.SetupDB()
Expect(err).ToNot(HaveOccurred())
repo = btc.NewIPLDPublisherAndIndexer(db)
})
AfterEach(func() {
btc.TearDownDB(db)
})
Describe("Publish", func() {
It("Published and indexes header and transaction IPLDs in a single tx", func() {
emptyReturn, err := repo.Publish(mocks.MockConvertedPayload)
Expect(emptyReturn).To(BeNil())
Expect(err).ToNot(HaveOccurred())
pgStr := `SELECT * FROM btc.header_cids
WHERE block_number = $1`
// check header was properly indexed
buf := bytes.NewBuffer(make([]byte, 0, 80))
err = mocks.MockBlock.Header.Serialize(buf)
Expect(err).ToNot(HaveOccurred())
headerBytes := buf.Bytes()
c, _ := ipld.RawdataToCid(ipld.MBitcoinHeader, headerBytes, multihash.DBL_SHA2_256)
header := new(btc.HeaderModel)
err = db.Get(header, pgStr, mocks.MockHeaderMetaData.BlockNumber)
Expect(err).ToNot(HaveOccurred())
Expect(header.CID).To(Equal(c.String()))
Expect(header.BlockNumber).To(Equal(mocks.MockHeaderMetaData.BlockNumber))
Expect(header.Bits).To(Equal(mocks.MockHeaderMetaData.Bits))
Expect(header.Timestamp).To(Equal(mocks.MockHeaderMetaData.Timestamp))
Expect(header.BlockHash).To(Equal(mocks.MockHeaderMetaData.BlockHash))
Expect(header.ParentHash).To(Equal(mocks.MockHeaderMetaData.ParentHash))
dc, err := cid.Decode(header.CID)
Expect(err).ToNot(HaveOccurred())
mhKey := dshelp.MultihashToDsKey(dc.Hash())
prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
var data []byte
err = db.Get(&data, ipfsPgGet, prefixedKey)
Expect(err).ToNot(HaveOccurred())
Expect(data).To(Equal(headerBytes))
// check that txs were properly indexed
trxs := make([]btc.TxModel, 0)
pgStr = `SELECT transaction_cids.id, transaction_cids.header_id, transaction_cids.index,
transaction_cids.tx_hash, transaction_cids.cid, transaction_cids.segwit, transaction_cids.witness_hash
FROM btc.transaction_cids INNER JOIN btc.header_cids ON (transaction_cids.header_id = header_cids.id)
WHERE header_cids.block_number = $1`
err = db.Select(&trxs, pgStr, mocks.MockHeaderMetaData.BlockNumber)
Expect(err).ToNot(HaveOccurred())
Expect(len(trxs)).To(Equal(3))
txData := make([][]byte, len(mocks.MockTransactions))
txCIDs := make([]string, len(mocks.MockTransactions))
for i, m := range mocks.MockTransactions {
buf := bytes.NewBuffer(make([]byte, 0))
err = m.MsgTx().Serialize(buf)
Expect(err).ToNot(HaveOccurred())
tx := buf.Bytes()
txData[i] = tx
c, _ := ipld.RawdataToCid(ipld.MBitcoinTx, tx, multihash.DBL_SHA2_256)
txCIDs[i] = c.String()
}
for _, tx := range trxs {
Expect(tx.SegWit).To(Equal(false))
Expect(tx.HeaderID).To(Equal(header.ID))
Expect(tx.WitnessHash).To(Equal(""))
Expect(tx.CID).To(Equal(txCIDs[tx.Index]))
Expect(tx.TxHash).To(Equal(mocks.MockBlock.Transactions[tx.Index].TxHash().String()))
dc, err := cid.Decode(tx.CID)
Expect(err).ToNot(HaveOccurred())
mhKey := dshelp.MultihashToDsKey(dc.Hash())
prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
var data []byte
err = db.Get(&data, ipfsPgGet, prefixedKey)
Expect(err).ToNot(HaveOccurred())
Expect(data).To(Equal(txData[tx.Index]))
}
})
})
})

129
pkg/btc/publisher.go
View File

@ -20,102 +20,101 @@ import (
"fmt" "fmt"
"strconv" "strconv"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs/dag_putters"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs/ipld" "github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs/ipld"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/postgres"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/shared" "github.com/vulcanize/ipfs-blockchain-watcher/pkg/shared"
) )
// IPLDPublisher satisfies the IPLDPublisher for ethereum // IPLDPublisher satisfies the IPLDPublisher interface for bitcoin
// It interfaces directly with the public.blocks table of PG-IPFS rather than going through an ipfs intermediary
// It publishes and indexes IPLDs together in a single sqlx.Tx
type IPLDPublisher struct { type IPLDPublisher struct {
HeaderPutter ipfs.DagPutter indexer *CIDIndexer
TransactionPutter ipfs.DagPutter
TransactionTriePutter ipfs.DagPutter
} }
// NewIPLDPublisher creates a pointer to a new Publisher which satisfies the IPLDPublisher interface // NewIPLDPublisher creates a pointer to a new eth IPLDPublisher which satisfies the IPLDPublisher interface
func NewIPLDPublisher(ipfsPath string) (*IPLDPublisher, error) { func NewIPLDPublisher(db *postgres.DB) *IPLDPublisher {
node, err := ipfs.InitIPFSNode(ipfsPath)
if err != nil {
return nil, err
}
return &IPLDPublisher{ return &IPLDPublisher{
HeaderPutter: dag_putters.NewBtcHeaderDagPutter(node), indexer: NewCIDIndexer(db),
TransactionPutter: dag_putters.NewBtcTxDagPutter(node), }
TransactionTriePutter: dag_putters.NewBtcTxTrieDagPutter(node),
}, nil
} }
// Publish publishes an IPLDPayload to IPFS and returns the corresponding CIDPayload // Publish publishes an IPLDPayload to IPFS and returns the corresponding CIDPayload
func (pub *IPLDPublisher) Publish(payload shared.ConvertedData) (shared.CIDsForIndexing, error) { func (pub *IPLDPublisher) Publish(payload shared.ConvertedData) error {
ipldPayload, ok := payload.(ConvertedPayload) ipldPayload, ok := payload.(ConvertedPayload)
if !ok { if !ok {
return nil, fmt.Errorf("eth publisher expected payload type %T got %T", &ConvertedPayload{}, payload) return fmt.Errorf("btc publisher expected payload type %T got %T", ConvertedPayload{}, payload)
} }
// Generate nodes // Generate the iplds
headerNode, txNodes, txTrieNodes, err := ipld.FromHeaderAndTxs(ipldPayload.Header, ipldPayload.Txs) headerNode, txNodes, txTrieNodes, err := ipld.FromHeaderAndTxs(ipldPayload.Header, ipldPayload.Txs)
if err != nil { if err != nil {
return nil, err return err
} }
// Process and publish headers
headerCid, err := pub.publishHeader(headerNode) // Begin new db tx
tx, err := pub.indexer.db.Beginx()
if err != nil { if err != nil {
return nil, err return err
}
defer func() {
if p := recover(); p != nil {
shared.Rollback(tx)
panic(p)
} else if err != nil {
shared.Rollback(tx)
} else {
err = tx.Commit()
}
}()
// Publish trie nodes
for _, node := range txTrieNodes {
if err := shared.PublishIPLD(tx, node); err != nil {
return err
}
}
// Publish and index header
if err := shared.PublishIPLD(tx, headerNode); err != nil {
return err
} }
mhKey, _ := shared.MultihashKeyFromCIDString(headerCid)
header := HeaderModel{ header := HeaderModel{
CID: headerCid, CID: headerNode.Cid().String(),
MhKey: mhKey, MhKey: shared.MultihashKeyFromCID(headerNode.Cid()),
ParentHash: ipldPayload.Header.PrevBlock.String(), ParentHash: ipldPayload.Header.PrevBlock.String(),
BlockNumber: strconv.Itoa(int(ipldPayload.BlockPayload.BlockHeight)), BlockNumber: strconv.Itoa(int(ipldPayload.BlockPayload.BlockHeight)),
BlockHash: ipldPayload.Header.BlockHash().String(), BlockHash: ipldPayload.Header.BlockHash().String(),
Timestamp: ipldPayload.Header.Timestamp.UnixNano(), Timestamp: ipldPayload.Header.Timestamp.UnixNano(),
Bits: ipldPayload.Header.Bits, Bits: ipldPayload.Header.Bits,
} }
// Process and publish transactions headerID, err := pub.indexer.indexHeaderCID(tx, header)
transactionCids, err := pub.publishTransactions(txNodes, txTrieNodes, ipldPayload.TxMetaData)
if err != nil { if err != nil {
return nil, err return err
} }
// Package CIDs and their metadata into a single struct
return &CIDPayload{
HeaderCID: header,
TransactionCIDs: transactionCids,
}, nil
}
func (pub *IPLDPublisher) publishHeader(header *ipld.BtcHeader) (string, error) { // Publish and index txs
cid, err := pub.HeaderPutter.DagPut(header) for i, txNode := range txNodes {
if err != nil { if err := shared.PublishIPLD(tx, txNode); err != nil {
return "", err return err
} }
return cid, nil txModel := ipldPayload.TxMetaData[i]
} txModel.CID = txNode.Cid().String()
txModel.MhKey = shared.MultihashKeyFromCID(txNode.Cid())
func (pub *IPLDPublisher) publishTransactions(transactions []*ipld.BtcTx, txTrie []*ipld.BtcTxTrie, trxMeta []TxModelWithInsAndOuts) ([]TxModelWithInsAndOuts, error) { txID, err := pub.indexer.indexTransactionCID(tx, txModel, headerID)
txCids := make([]TxModelWithInsAndOuts, len(transactions))
for i, tx := range transactions {
cid, err := pub.TransactionPutter.DagPut(tx)
if err != nil { if err != nil {
return nil, err return err
} }
mhKey, _ := shared.MultihashKeyFromCIDString(cid) for _, input := range txModel.TxInputs {
txCids[i] = TxModelWithInsAndOuts{ if err := pub.indexer.indexTxInput(tx, input, txID); err != nil {
CID: cid, return err
MhKey: mhKey, }
Index: trxMeta[i].Index, }
TxHash: trxMeta[i].TxHash, for _, output := range txModel.TxOutputs {
SegWit: trxMeta[i].SegWit, if err := pub.indexer.indexTxOutput(tx, output, txID); err != nil {
WitnessHash: trxMeta[i].WitnessHash, return err
TxInputs: trxMeta[i].TxInputs, }
TxOutputs: trxMeta[i].TxOutputs,
} }
} }
for _, txNode := range txTrie {
// We don't do anything with the tx trie cids atm return err
if _, err := pub.TransactionTriePutter.DagPut(txNode); err != nil {
return nil, err
}
}
return txCids, nil
} }

119
pkg/btc/publisher_test.go
View File

@ -19,63 +19,102 @@ package btc_test
import ( import (
"bytes" "bytes"
"github.com/ethereum/go-ethereum/common" "github.com/ipfs/go-cid"
"github.com/ipfs/go-ipfs-blockstore"
"github.com/ipfs/go-ipfs-ds-help"
"github.com/multiformats/go-multihash"
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs/ipld"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/btc" "github.com/vulcanize/ipfs-blockchain-watcher/pkg/btc"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/btc/mocks" "github.com/vulcanize/ipfs-blockchain-watcher/pkg/btc/mocks"
mocks2 "github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs/mocks" "github.com/vulcanize/ipfs-blockchain-watcher/pkg/postgres"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/shared"
) )
var ( var _ = Describe("PublishAndIndexer", func() {
mockHeaderDagPutter *mocks2.MappedDagPutter var (
mockTrxDagPutter *mocks2.MappedDagPutter db *postgres.DB
mockTrxTrieDagPutter *mocks2.DagPutter err error
) repo *btc.IPLDPublisher
ipfsPgGet = `SELECT data FROM public.blocks
var _ = Describe("Publisher", func() { WHERE key = $1`
)
BeforeEach(func() { BeforeEach(func() {
mockHeaderDagPutter = new(mocks2.MappedDagPutter) db, err = shared.SetupDB()
mockTrxDagPutter = new(mocks2.MappedDagPutter) Expect(err).ToNot(HaveOccurred())
mockTrxTrieDagPutter = new(mocks2.DagPutter) repo = btc.NewIPLDPublisher(db)
})
AfterEach(func() {
btc.TearDownDB(db)
}) })
Describe("Publish", func() { Describe("Publish", func() {
It("Publishes the passed IPLDPayload objects to IPFS and returns a CIDPayload for indexing", func() { It("Published and indexes header and transaction IPLDs in a single tx", func() {
by := new(bytes.Buffer) err = repo.Publish(mocks.MockConvertedPayload)
err := mocks.MockConvertedPayload.BlockPayload.Header.Serialize(by)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
headerBytes := by.Bytes() pgStr := `SELECT * FROM btc.header_cids
err = mocks.MockTransactions[0].MsgTx().Serialize(by) WHERE block_number = $1`
// check header was properly indexed
buf := bytes.NewBuffer(make([]byte, 0, 80))
err = mocks.MockBlock.Header.Serialize(buf)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
tx1Bytes := by.Bytes() headerBytes := buf.Bytes()
err = mocks.MockTransactions[1].MsgTx().Serialize(by) c, _ := ipld.RawdataToCid(ipld.MBitcoinHeader, headerBytes, multihash.DBL_SHA2_256)
header := new(btc.HeaderModel)
err = db.Get(header, pgStr, mocks.MockHeaderMetaData.BlockNumber)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
tx2Bytes := by.Bytes() Expect(header.CID).To(Equal(c.String()))
err = mocks.MockTransactions[2].MsgTx().Serialize(by) Expect(header.BlockNumber).To(Equal(mocks.MockHeaderMetaData.BlockNumber))
Expect(header.Bits).To(Equal(mocks.MockHeaderMetaData.Bits))
Expect(header.Timestamp).To(Equal(mocks.MockHeaderMetaData.Timestamp))
Expect(header.BlockHash).To(Equal(mocks.MockHeaderMetaData.BlockHash))
Expect(header.ParentHash).To(Equal(mocks.MockHeaderMetaData.ParentHash))
dc, err := cid.Decode(header.CID)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
tx3Bytes := by.Bytes() mhKey := dshelp.MultihashToDsKey(dc.Hash())
mockHeaderDagPutter.CIDsToReturn = map[common.Hash]string{ prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
common.BytesToHash(headerBytes): mocks.MockHeaderCID.String(), var data []byte
err = db.Get(&data, ipfsPgGet, prefixedKey)
Expect(err).ToNot(HaveOccurred())
Expect(data).To(Equal(headerBytes))
// check that txs were properly indexed
trxs := make([]btc.TxModel, 0)
pgStr = `SELECT transaction_cids.id, transaction_cids.header_id, transaction_cids.index,
transaction_cids.tx_hash, transaction_cids.cid, transaction_cids.segwit, transaction_cids.witness_hash
FROM btc.transaction_cids INNER JOIN btc.header_cids ON (transaction_cids.header_id = header_cids.id)
WHERE header_cids.block_number = $1`
err = db.Select(&trxs, pgStr, mocks.MockHeaderMetaData.BlockNumber)
Expect(err).ToNot(HaveOccurred())
Expect(len(trxs)).To(Equal(3))
txData := make([][]byte, len(mocks.MockTransactions))
txCIDs := make([]string, len(mocks.MockTransactions))
for i, m := range mocks.MockTransactions {
buf := bytes.NewBuffer(make([]byte, 0))
err = m.MsgTx().Serialize(buf)
Expect(err).ToNot(HaveOccurred())
tx := buf.Bytes()
txData[i] = tx
c, _ := ipld.RawdataToCid(ipld.MBitcoinTx, tx, multihash.DBL_SHA2_256)
txCIDs[i] = c.String()
} }
mockTrxDagPutter.CIDsToReturn = map[common.Hash]string{ for _, tx := range trxs {
common.BytesToHash(tx1Bytes): mocks.MockTrxCID1.String(), Expect(tx.SegWit).To(Equal(false))
common.BytesToHash(tx2Bytes): mocks.MockTrxCID2.String(), Expect(tx.HeaderID).To(Equal(header.ID))
common.BytesToHash(tx3Bytes): mocks.MockTrxCID3.String(), Expect(tx.WitnessHash).To(Equal(""))
Expect(tx.CID).To(Equal(txCIDs[tx.Index]))
Expect(tx.TxHash).To(Equal(mocks.MockBlock.Transactions[tx.Index].TxHash().String()))
dc, err := cid.Decode(tx.CID)
Expect(err).ToNot(HaveOccurred())
mhKey := dshelp.MultihashToDsKey(dc.Hash())
prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
var data []byte
err = db.Get(&data, ipfsPgGet, prefixedKey)
Expect(err).ToNot(HaveOccurred())
Expect(data).To(Equal(txData[tx.Index]))
} }
publisher := btc.IPLDPublisher{
HeaderPutter: mockHeaderDagPutter,
TransactionPutter: mockTrxDagPutter,
TransactionTriePutter: mockTrxTrieDagPutter,
}
payload, err := publisher.Publish(mocks.MockConvertedPayload)
Expect(err).ToNot(HaveOccurred())
cidPayload, ok := payload.(*btc.CIDPayload)
Expect(ok).To(BeTrue())
Expect(cidPayload).To(Equal(&mocks.MockCIDPayload))
Expect(cidPayload.HeaderCID).To(Equal(mocks.MockHeaderMetaData))
Expect(cidPayload.TransactionCIDs).To(Equal(mocks.MockTxsMetaDataPostPublish))
}) })
}) })
}) })

81
pkg/builders/builders.go
View File

@ -22,7 +22,6 @@ import (
"github.com/btcsuite/btcd/chaincfg" "github.com/btcsuite/btcd/chaincfg"
"github.com/btcsuite/btcd/rpcclient" "github.com/btcsuite/btcd/rpcclient"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/rpc"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/btc" "github.com/vulcanize/ipfs-blockchain-watcher/pkg/btc"
@ -43,32 +42,6 @@ func NewResponseFilterer(chain shared.ChainType) (shared.ResponseFilterer, error
} }
} }
// NewCIDIndexer constructs a CIDIndexer for the provided chain type
func NewCIDIndexer(chain shared.ChainType, db *postgres.DB, ipfsMode shared.IPFSMode) (shared.CIDIndexer, error) {
switch chain {
case shared.Ethereum:
switch ipfsMode {
case shared.LocalInterface, shared.RemoteClient:
return eth.NewCIDIndexer(db), nil
case shared.DirectPostgres:
return eth.NewIPLDPublisherAndIndexer(db), nil
default:
return nil, fmt.Errorf("ethereum CIDIndexer unexpected ipfs mode %s", ipfsMode.String())
}
case shared.Bitcoin:
switch ipfsMode {
case shared.LocalInterface, shared.RemoteClient:
return btc.NewCIDIndexer(db), nil
case shared.DirectPostgres:
return eth.NewIPLDPublisherAndIndexer(db), nil
default:
return nil, fmt.Errorf("bitcoin CIDIndexer unexpected ipfs mode %s", ipfsMode.String())
}
default:
return nil, fmt.Errorf("invalid chain %s for indexer constructor", chain.String())
}
}
// NewCIDRetriever constructs a CIDRetriever for the provided chain type // NewCIDRetriever constructs a CIDRetriever for the provided chain type
func NewCIDRetriever(chain shared.ChainType, db *postgres.DB) (shared.CIDRetriever, error) { func NewCIDRetriever(chain shared.ChainType, db *postgres.DB) (shared.CIDRetriever, error) {
switch chain { switch chain {
@ -124,71 +97,47 @@ func NewPaylaodFetcher(chain shared.ChainType, client interface{}, timeout time.
} }
// NewPayloadConverter constructs a PayloadConverter for the provided chain type // NewPayloadConverter constructs a PayloadConverter for the provided chain type
func NewPayloadConverter(chain shared.ChainType) (shared.PayloadConverter, error) { func NewPayloadConverter(chainType shared.ChainType, chainID uint64) (shared.PayloadConverter, error) {
switch chain { switch chainType {
case shared.Ethereum: case shared.Ethereum:
return eth.NewPayloadConverter(params.MainnetChainConfig), nil chainConfig, err := eth.ChainConfig(chainID)
if err != nil {
return nil, err
}
return eth.NewPayloadConverter(chainConfig), nil
case shared.Bitcoin: case shared.Bitcoin:
return btc.NewPayloadConverter(&chaincfg.MainNetParams), nil return btc.NewPayloadConverter(&chaincfg.MainNetParams), nil
default: default:
return nil, fmt.Errorf("invalid chain %s for converter constructor", chain.String()) return nil, fmt.Errorf("invalid chain %s for converter constructor", chainType.String())
} }
} }
// NewIPLDFetcher constructs an IPLDFetcher for the provided chain type // NewIPLDFetcher constructs an IPLDFetcher for the provided chain type
func NewIPLDFetcher(chain shared.ChainType, ipfsPath string, db *postgres.DB, ipfsMode shared.IPFSMode) (shared.IPLDFetcher, error) { func NewIPLDFetcher(chain shared.ChainType, db *postgres.DB) (shared.IPLDFetcher, error) {
switch chain { switch chain {
case shared.Ethereum: case shared.Ethereum:
switch ipfsMode { return eth.NewIPLDFetcher(db), nil
case shared.LocalInterface, shared.RemoteClient:
return eth.NewIPLDFetcher(ipfsPath)
case shared.DirectPostgres:
return eth.NewIPLDPGFetcher(db), nil
default:
return nil, fmt.Errorf("ethereum IPLDFetcher unexpected ipfs mode %s", ipfsMode.String())
}
case shared.Bitcoin: case shared.Bitcoin:
switch ipfsMode { return btc.NewIPLDFetcher(db), nil
case shared.LocalInterface, shared.RemoteClient:
return btc.NewIPLDFetcher(ipfsPath)
case shared.DirectPostgres:
return btc.NewIPLDPGFetcher(db), nil
default:
return nil, fmt.Errorf("bitcoin IPLDFetcher unexpected ipfs mode %s", ipfsMode.String())
}
default: default:
return nil, fmt.Errorf("invalid chain %s for IPLD fetcher constructor", chain.String()) return nil, fmt.Errorf("invalid chain %s for IPLD fetcher constructor", chain.String())
} }
} }
// NewIPLDPublisher constructs an IPLDPublisher for the provided chain type // NewIPLDPublisher constructs an IPLDPublisher for the provided chain type
func NewIPLDPublisher(chain shared.ChainType, ipfsPath string, db *postgres.DB, ipfsMode shared.IPFSMode) (shared.IPLDPublisher, error) { func NewIPLDPublisher(chain shared.ChainType, db *postgres.DB) (shared.IPLDPublisher, error) {
switch chain { switch chain {
case shared.Ethereum: case shared.Ethereum:
switch ipfsMode { return eth.NewIPLDPublisher(db), nil
case shared.LocalInterface, shared.RemoteClient:
return eth.NewIPLDPublisher(ipfsPath)
case shared.DirectPostgres:
return eth.NewIPLDPublisherAndIndexer(db), nil
default:
return nil, fmt.Errorf("ethereum IPLDPublisher unexpected ipfs mode %s", ipfsMode.String())
}
case shared.Bitcoin: case shared.Bitcoin:
switch ipfsMode { return btc.NewIPLDPublisher(db), nil
case shared.LocalInterface, shared.RemoteClient:
return btc.NewIPLDPublisher(ipfsPath)
case shared.DirectPostgres:
return btc.NewIPLDPublisherAndIndexer(db), nil
default:
return nil, fmt.Errorf("bitcoin IPLDPublisher unexpected ipfs mode %s", ipfsMode.String())
}
default: default:
return nil, fmt.Errorf("invalid chain %s for publisher constructor", chain.String()) return nil, fmt.Errorf("invalid chain %s for publisher constructor", chain.String())
} }
} }
// NewPublicAPI constructs a PublicAPI for the provided chain type // NewPublicAPI constructs a PublicAPI for the provided chain type
func NewPublicAPI(chain shared.ChainType, db *postgres.DB, ipfsPath string) (rpc.API, error) { func NewPublicAPI(chain shared.ChainType, db *postgres.DB) (rpc.API, error) {
switch chain { switch chain {
case shared.Ethereum: case shared.Ethereum:
backend, err := eth.NewEthBackend(db) backend, err := eth.NewEthBackend(db)

10
pkg/eth/api_test.go
View File

@ -84,8 +84,8 @@ var _ = Describe("API", func() {
var ( var (
db *postgres.DB db *postgres.DB
retriever *eth.CIDRetriever retriever *eth.CIDRetriever
fetcher *eth.IPLDPGFetcher fetcher *eth.IPLDFetcher
indexAndPublisher *eth.IPLDPublisherAndIndexer indexAndPublisher *eth.IPLDPublisher
backend *eth.Backend backend *eth.Backend
api *eth.PublicEthAPI api *eth.PublicEthAPI
) )
@ -94,15 +94,15 @@ var _ = Describe("API", func() {
db, err = shared.SetupDB() db, err = shared.SetupDB()
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
retriever = eth.NewCIDRetriever(db) retriever = eth.NewCIDRetriever(db)
fetcher = eth.NewIPLDPGFetcher(db) fetcher = eth.NewIPLDFetcher(db)
indexAndPublisher = eth.NewIPLDPublisherAndIndexer(db) indexAndPublisher = eth.NewIPLDPublisher(db)
backend = &eth.Backend{ backend = &eth.Backend{
Retriever: retriever, Retriever: retriever,
Fetcher: fetcher, Fetcher: fetcher,
DB: db, DB: db,
} }
api = eth.NewPublicEthAPI(backend) api = eth.NewPublicEthAPI(backend)
_, err = indexAndPublisher.Publish(mocks.MockConvertedPayload) err = indexAndPublisher.Publish(mocks.MockConvertedPayload)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
uncles := mocks.MockBlock.Uncles() uncles := mocks.MockBlock.Uncles()
uncleHashes := make([]common.Hash, len(uncles)) uncleHashes := make([]common.Hash, len(uncles))

2
pkg/eth/backend.go
View File

@ -39,7 +39,7 @@ var (
type Backend struct { type Backend struct {
Retriever *CIDRetriever Retriever *CIDRetriever
Fetcher *IPLDPGFetcher Fetcher *IPLDFetcher
DB *postgres.DB DB *postgres.DB
} }

92
pkg/eth/cid_retriever_test.go
View File

@ -211,14 +211,14 @@ var (
var _ = Describe("Retriever", func() { var _ = Describe("Retriever", func() {
var ( var (
db *postgres.DB db *postgres.DB
repo *eth2.IPLDPublisherAndIndexer repo *eth2.IPLDPublisher
retriever *eth2.CIDRetriever retriever *eth2.CIDRetriever
) )
BeforeEach(func() { BeforeEach(func() {
var err error var err error
db, err = shared.SetupDB() db, err = shared.SetupDB()
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
repo = eth2.NewIPLDPublisherAndIndexer(db) repo = eth2.NewIPLDPublisher(db)
retriever = eth2.NewCIDRetriever(db) retriever = eth2.NewCIDRetriever(db)
}) })
AfterEach(func() { AfterEach(func() {
@ -227,7 +227,7 @@ var _ = Describe("Retriever", func() {
Describe("Retrieve", func() { Describe("Retrieve", func() {
BeforeEach(func() { BeforeEach(func() {
_, err := repo.Publish(mocks.MockConvertedPayload) err := repo.Publish(mocks.MockConvertedPayload)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
}) })
It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() { It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() {
@ -413,7 +413,7 @@ var _ = Describe("Retriever", func() {
Expect(err).To(HaveOccurred()) Expect(err).To(HaveOccurred())
}) })
It("Gets the number of the first block that has data in the database", func() { It("Gets the number of the first block that has data in the database", func() {
_, err := repo.Publish(mocks.MockConvertedPayload) err := repo.Publish(mocks.MockConvertedPayload)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
num, err := retriever.RetrieveFirstBlockNumber() num, err := retriever.RetrieveFirstBlockNumber()
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
@ -423,7 +423,7 @@ var _ = Describe("Retriever", func() {
It("Gets the number of the first block that has data in the database", func() { It("Gets the number of the first block that has data in the database", func() {
payload := mocks.MockConvertedPayload payload := mocks.MockConvertedPayload
payload.Block = newMockBlock(1010101) payload.Block = newMockBlock(1010101)
_, err := repo.Publish(payload) err := repo.Publish(payload)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
num, err := retriever.RetrieveFirstBlockNumber() num, err := retriever.RetrieveFirstBlockNumber()
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
@ -435,9 +435,9 @@ var _ = Describe("Retriever", func() {
payload1.Block = newMockBlock(1010101) payload1.Block = newMockBlock(1010101)
payload2 := payload1 payload2 := payload1
payload2.Block = newMockBlock(5) payload2.Block = newMockBlock(5)
_, err := repo.Publish(payload1) err := repo.Publish(payload1)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
_, err = repo.Publish(payload2) err = repo.Publish(payload2)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
num, err := retriever.RetrieveFirstBlockNumber() num, err := retriever.RetrieveFirstBlockNumber()
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
@ -451,7 +451,7 @@ var _ = Describe("Retriever", func() {
Expect(err).To(HaveOccurred()) Expect(err).To(HaveOccurred())
}) })
It("Gets the number of the latest block that has data in the database", func() { It("Gets the number of the latest block that has data in the database", func() {
_, err := repo.Publish(mocks.MockConvertedPayload) err := repo.Publish(mocks.MockConvertedPayload)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
num, err := retriever.RetrieveLastBlockNumber() num, err := retriever.RetrieveLastBlockNumber()
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
@ -461,7 +461,7 @@ var _ = Describe("Retriever", func() {
It("Gets the number of the latest block that has data in the database", func() { It("Gets the number of the latest block that has data in the database", func() {
payload := mocks.MockConvertedPayload payload := mocks.MockConvertedPayload
payload.Block = newMockBlock(1010101) payload.Block = newMockBlock(1010101)
_, err := repo.Publish(payload) err := repo.Publish(payload)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
num, err := retriever.RetrieveLastBlockNumber() num, err := retriever.RetrieveLastBlockNumber()
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
@ -473,9 +473,9 @@ var _ = Describe("Retriever", func() {
payload1.Block = newMockBlock(1010101) payload1.Block = newMockBlock(1010101)
payload2 := payload1 payload2 := payload1
payload2.Block = newMockBlock(5) payload2.Block = newMockBlock(5)
_, err := repo.Publish(payload1) err := repo.Publish(payload1)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
_, err = repo.Publish(payload2) err = repo.Publish(payload2)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
num, err := retriever.RetrieveLastBlockNumber() num, err := retriever.RetrieveLastBlockNumber()
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
@ -492,13 +492,13 @@ var _ = Describe("Retriever", func() {
payload2.Block = newMockBlock(2) payload2.Block = newMockBlock(2)
payload3 := payload2 payload3 := payload2
payload3.Block = newMockBlock(3) payload3.Block = newMockBlock(3)
_, err := repo.Publish(payload0) err := repo.Publish(payload0)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
_, err = repo.Publish(payload1) err = repo.Publish(payload1)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
_, err = repo.Publish(payload2) err = repo.Publish(payload2)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
_, err = repo.Publish(payload3) err = repo.Publish(payload3)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
gaps, err := retriever.RetrieveGapsInData(1) gaps, err := retriever.RetrieveGapsInData(1)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
@ -508,7 +508,7 @@ var _ = Describe("Retriever", func() {
It("Returns the gap from 0 to the earliest block", func() { It("Returns the gap from 0 to the earliest block", func() {
payload := mocks.MockConvertedPayload payload := mocks.MockConvertedPayload
payload.Block = newMockBlock(5) payload.Block = newMockBlock(5)
_, err := repo.Publish(payload) err := repo.Publish(payload)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
gaps, err := retriever.RetrieveGapsInData(1) gaps, err := retriever.RetrieveGapsInData(1)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
@ -523,11 +523,11 @@ var _ = Describe("Retriever", func() {
payload1 := mocks.MockConvertedPayload payload1 := mocks.MockConvertedPayload
payload3 := payload1 payload3 := payload1
payload3.Block = newMockBlock(3) payload3.Block = newMockBlock(3)
_, err := repo.Publish(payload0) err := repo.Publish(payload0)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
_, err = repo.Publish(payload1) err = repo.Publish(payload1)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
_, err = repo.Publish(payload3) err = repo.Publish(payload3)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
gaps, err := retriever.RetrieveGapsInData(1) gaps, err := retriever.RetrieveGapsInData(1)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
@ -541,9 +541,9 @@ var _ = Describe("Retriever", func() {
payload1.Block = newMockBlock(1010101) payload1.Block = newMockBlock(1010101)
payload2 := payload1 payload2 := payload1
payload2.Block = newMockBlock(0) payload2.Block = newMockBlock(0)
_, err := repo.Publish(payload1) err := repo.Publish(payload1)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
_, err = repo.Publish(payload2) err = repo.Publish(payload2)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
gaps, err := retriever.RetrieveGapsInData(1) gaps, err := retriever.RetrieveGapsInData(1)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
@ -576,27 +576,27 @@ var _ = Describe("Retriever", func() {
payload11 := mocks.MockConvertedPayload payload11 := mocks.MockConvertedPayload
payload11.Block = newMockBlock(1000) payload11.Block = newMockBlock(1000)
_, err := repo.Publish(payload1) err := repo.Publish(payload1)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
_, err = repo.Publish(payload2) err = repo.Publish(payload2)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
_, err = repo.Publish(payload3) err = repo.Publish(payload3)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
_, err = repo.Publish(payload4) err = repo.Publish(payload4)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
_, err = repo.Publish(payload5) err = repo.Publish(payload5)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
_, err = repo.Publish(payload6) err = repo.Publish(payload6)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
_, err = repo.Publish(payload7) err = repo.Publish(payload7)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
_, err = repo.Publish(payload8) err = repo.Publish(payload8)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
_, err = repo.Publish(payload9) err = repo.Publish(payload9)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
_, err = repo.Publish(payload10) err = repo.Publish(payload10)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
_, err = repo.Publish(payload11) err = repo.Publish(payload11)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
gaps, err := retriever.RetrieveGapsInData(1) gaps, err := retriever.RetrieveGapsInData(1)
@ -640,33 +640,33 @@ var _ = Describe("Retriever", func() {
payload14 := mocks.MockConvertedPayload payload14 := mocks.MockConvertedPayload
payload14.Block = newMockBlock(1000) payload14.Block = newMockBlock(1000)
_, err := repo.Publish(payload1) err := repo.Publish(payload1)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
_, err = repo.Publish(payload2) err = repo.Publish(payload2)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
_, err = repo.Publish(payload3) err = repo.Publish(payload3)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
_, err = repo.Publish(payload4) err = repo.Publish(payload4)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
_, err = repo.Publish(payload5) err = repo.Publish(payload5)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
_, err = repo.Publish(payload6) err = repo.Publish(payload6)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
_, err = repo.Publish(payload7) err = repo.Publish(payload7)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
_, err = repo.Publish(payload8) err = repo.Publish(payload8)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
_, err = repo.Publish(payload9) err = repo.Publish(payload9)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
_, err = repo.Publish(payload10) err = repo.Publish(payload10)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
_, err = repo.Publish(payload11) err = repo.Publish(payload11)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
_, err = repo.Publish(payload12) err = repo.Publish(payload12)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
_, err = repo.Publish(payload13) err = repo.Publish(payload13)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
_, err = repo.Publish(payload14) err = repo.Publish(payload14)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
cleaner := eth.NewCleaner(db) cleaner := eth.NewCleaner(db)

5
pkg/eth/converter.go
View File

@ -71,11 +71,13 @@ func (pc *PayloadConverter) Convert(payload shared.RawChainData) (shared.Convert
if err != nil { if err != nil {
return nil, err return nil, err
} }
txMeta := TxModel{ txMeta := TxModel{
Dst: shared.HandleZeroAddrPointer(trx.To()), Dst: shared.HandleZeroAddrPointer(trx.To()),
Src: shared.HandleZeroAddr(from), Src: shared.HandleZeroAddr(from),
TxHash: trx.Hash().String(), TxHash: trx.Hash().String(),
Index: int64(i), Index: int64(i),
Data: trx.Data(),
} }
// txMeta will have same index as its corresponding trx in the convertedPayload.BlockBody // txMeta will have same index as its corresponding trx in the convertedPayload.BlockBody
convertedPayload.TxMetaData = append(convertedPayload.TxMetaData, txMeta) convertedPayload.TxMetaData = append(convertedPayload.TxMetaData, txMeta)
@ -90,7 +92,7 @@ func (pc *PayloadConverter) Convert(payload shared.RawChainData) (shared.Convert
if err := receipts.DeriveFields(pc.chainConfig, block.Hash(), block.NumberU64(), block.Transactions()); err != nil { if err := receipts.DeriveFields(pc.chainConfig, block.Hash(), block.NumberU64(), block.Transactions()); err != nil {
return nil, err return nil, err
} }
for _, receipt := range receipts { for i, receipt := range receipts {
// Extract topic and contract data from the receipt for indexing // Extract topic and contract data from the receipt for indexing
topicSets := make([][]string, 4) topicSets := make([][]string, 4)
mappedContracts := make(map[string]bool) // use map to avoid duplicate addresses mappedContracts := make(map[string]bool) // use map to avoid duplicate addresses
@ -109,6 +111,7 @@ func (pc *PayloadConverter) Convert(payload shared.RawChainData) (shared.Convert
contract := shared.HandleZeroAddr(receipt.ContractAddress) contract := shared.HandleZeroAddr(receipt.ContractAddress)
var contractHash string var contractHash string
if contract != "" { if contract != "" {
convertedPayload.TxMetaData[i].Deployment = true
contractHash = crypto.Keccak256Hash(common.HexToAddress(contract).Bytes()).String() contractHash = crypto.Keccak256Hash(common.HexToAddress(contract).Bytes()).String()
} }
rctMeta := ReceiptModel{ rctMeta := ReceiptModel{

23
pkg/eth/helpers.go
View File

@ -16,7 +16,12 @@
package eth package eth
import "github.com/ethereum/go-ethereum/statediff" import (
"fmt"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/statediff"
)
func ResolveFromNodeType(nodeType statediff.NodeType) int { func ResolveFromNodeType(nodeType statediff.NodeType) int {
switch nodeType { switch nodeType {
@ -47,3 +52,19 @@ func ResolveToNodeType(nodeType int) statediff.NodeType {
return statediff.Unknown return statediff.Unknown
} }
} }
// ChainConfig returns the appropriate ethereum chain config for the provided chain id
func ChainConfig(chainID uint64) (*params.ChainConfig, error) {
switch chainID {
case 1:
return params.MainnetChainConfig, nil
case 3:
return params.TestnetChainConfig, nil // Ropsten
case 4:
return params.RinkebyChainConfig, nil
case 5:
return params.GoerliChainConfig, nil
default:
return nil, fmt.Errorf("chain config for chainid %d not available", chainID)
}
}

12
pkg/eth/indexer.go
View File

@ -109,10 +109,10 @@ func (in *CIDIndexer) indexUncleCID(tx *sqlx.Tx, uncle UncleModel, headerID int6
func (in *CIDIndexer) indexTransactionAndReceiptCIDs(tx *sqlx.Tx, payload *CIDPayload, headerID int64) error { func (in *CIDIndexer) indexTransactionAndReceiptCIDs(tx *sqlx.Tx, payload *CIDPayload, headerID int64) error {
for _, trxCidMeta := range payload.TransactionCIDs { for _, trxCidMeta := range payload.TransactionCIDs {
var txID int64 var txID int64
err := tx.QueryRowx(`INSERT INTO eth.transaction_cids (header_id, tx_hash, cid, dst, src, index, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7) err := tx.QueryRowx(`INSERT INTO eth.transaction_cids (header_id, tx_hash, cid, dst, src, index, mh_key, data, deployment) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
ON CONFLICT (header_id, tx_hash) DO UPDATE SET (cid, dst, src, index, mh_key) = ($3, $4, $5, $6, $7) ON CONFLICT (header_id, tx_hash) DO UPDATE SET (cid, dst, src, index, mh_key, data, deployment) = ($3, $4, $5, $6, $7, $8, $9)
RETURNING id`, RETURNING id`,
headerID, trxCidMeta.TxHash, trxCidMeta.CID, trxCidMeta.Dst, trxCidMeta.Src, trxCidMeta.Index, trxCidMeta.MhKey).Scan(&txID) headerID, trxCidMeta.TxHash, trxCidMeta.CID, trxCidMeta.Dst, trxCidMeta.Src, trxCidMeta.Index, trxCidMeta.MhKey, trxCidMeta.Data, trxCidMeta.Deployment).Scan(&txID)
if err != nil { if err != nil {
return err return err
} }
@ -128,10 +128,10 @@ func (in *CIDIndexer) indexTransactionAndReceiptCIDs(tx *sqlx.Tx, payload *CIDPa
func (in *CIDIndexer) indexTransactionCID(tx *sqlx.Tx, transaction TxModel, headerID int64) (int64, error) { func (in *CIDIndexer) indexTransactionCID(tx *sqlx.Tx, transaction TxModel, headerID int64) (int64, error) {
var txID int64 var txID int64
err := tx.QueryRowx(`INSERT INTO eth.transaction_cids (header_id, tx_hash, cid, dst, src, index, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7) err := tx.QueryRowx(`INSERT INTO eth.transaction_cids (header_id, tx_hash, cid, dst, src, index, mh_key, data, deployment) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
ON CONFLICT (header_id, tx_hash) DO UPDATE SET (cid, dst, src, index, mh_key) = ($3, $4, $5, $6, $7) ON CONFLICT (header_id, tx_hash) DO UPDATE SET (cid, dst, src, index, mh_key, data, deployment) = ($3, $4, $5, $6, $7, $8, $9)
RETURNING id`, RETURNING id`,
headerID, transaction.TxHash, transaction.CID, transaction.Dst, transaction.Src, transaction.Index, transaction.MhKey).Scan(&txID) headerID, transaction.TxHash, transaction.CID, transaction.Dst, transaction.Src, transaction.Index, transaction.MhKey, transaction.Data, transaction.Deployment).Scan(&txID)
return txID, err return txID, err
} }

182
pkg/eth/ipld_fetcher.go
View File

@ -17,39 +17,30 @@
package eth package eth
import ( import (
"context"
"errors" "errors"
"fmt" "fmt"
"math/big" "math/big"
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ipfs/go-block-format" "github.com/jmoiron/sqlx"
"github.com/ipfs/go-blockservice"
"github.com/ipfs/go-cid"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs" "github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/postgres"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/shared" "github.com/vulcanize/ipfs-blockchain-watcher/pkg/shared"
) )
var (
errUnexpectedNumberOfIPLDs = errors.New("ipfs batch fetch returned unexpected number of IPLDs")
)
// IPLDFetcher satisfies the IPLDFetcher interface for ethereum // IPLDFetcher satisfies the IPLDFetcher interface for ethereum
// It interfaces directly with PG-IPFS
type IPLDFetcher struct { type IPLDFetcher struct {
BlockService blockservice.BlockService db *postgres.DB
} }
// NewIPLDFetcher creates a pointer to a new IPLDFetcher // NewIPLDFetcher creates a pointer to a new IPLDFetcher
func NewIPLDFetcher(ipfsPath string) (*IPLDFetcher, error) { func NewIPLDFetcher(db *postgres.DB) *IPLDFetcher {
blockService, err := ipfs.InitIPFSBlockService(ipfsPath)
if err != nil {
return nil, err
}
return &IPLDFetcher{ return &IPLDFetcher{
BlockService: blockService, db: db,
}, nil }
} }
// Fetch is the exported method for fetching and returning all the IPLDS specified in the CIDWrapper // Fetch is the exported method for fetching and returning all the IPLDS specified in the CIDWrapper
@ -59,161 +50,135 @@ func (f *IPLDFetcher) Fetch(cids shared.CIDsForFetching) (shared.IPLDs, error) {
return nil, fmt.Errorf("eth fetcher: expected cids type %T got %T", &CIDWrapper{}, cids) return nil, fmt.Errorf("eth fetcher: expected cids type %T got %T", &CIDWrapper{}, cids)
} }
log.Debug("fetching iplds") log.Debug("fetching iplds")
var err error
iplds := IPLDs{} iplds := IPLDs{}
iplds.TotalDifficulty, ok = new(big.Int).SetString(cidWrapper.Header.TotalDifficulty, 10) iplds.TotalDifficulty, ok = new(big.Int).SetString(cidWrapper.Header.TotalDifficulty, 10)
if !ok { if !ok {
return nil, errors.New("eth fetcher: unable to set total difficulty") return nil, errors.New("eth fetcher: unable to set total difficulty")
} }
iplds.BlockNumber = cidWrapper.BlockNumber iplds.BlockNumber = cidWrapper.BlockNumber
iplds.Header, err = f.FetchHeader(cidWrapper.Header)
tx, err := f.db.Beginx()
if err != nil { if err != nil {
return nil, err return nil, err
} }
iplds.Uncles, err = f.FetchUncles(cidWrapper.Uncles) defer func() {
if p := recover(); p != nil {
shared.Rollback(tx)
panic(p)
} else if err != nil {
shared.Rollback(tx)
} else {
err = tx.Commit()
}
}()
iplds.Header, err = f.FetchHeader(tx, cidWrapper.Header)
if err != nil { if err != nil {
return nil, err return nil, fmt.Errorf("eth pg fetcher: header fetching error: %s", err.Error())
} }
iplds.Transactions, err = f.FetchTrxs(cidWrapper.Transactions) iplds.Uncles, err = f.FetchUncles(tx, cidWrapper.Uncles)
if err != nil { if err != nil {
return nil, err return nil, fmt.Errorf("eth pg fetcher: uncle fetching error: %s", err.Error())
} }
iplds.Receipts, err = f.FetchRcts(cidWrapper.Receipts) iplds.Transactions, err = f.FetchTrxs(tx, cidWrapper.Transactions)
if err != nil { if err != nil {
return nil, err return nil, fmt.Errorf("eth pg fetcher: transaction fetching error: %s", err.Error())
} }
iplds.StateNodes, err = f.FetchState(cidWrapper.StateNodes) iplds.Receipts, err = f.FetchRcts(tx, cidWrapper.Receipts)
if err != nil { if err != nil {
return nil, err return nil, fmt.Errorf("eth pg fetcher: receipt fetching error: %s", err.Error())
} }
iplds.StorageNodes, err = f.FetchStorage(cidWrapper.StorageNodes) iplds.StateNodes, err = f.FetchState(tx, cidWrapper.StateNodes)
if err != nil { if err != nil {
return nil, err return nil, fmt.Errorf("eth pg fetcher: state fetching error: %s", err.Error())
} }
return iplds, nil iplds.StorageNodes, err = f.FetchStorage(tx, cidWrapper.StorageNodes)
if err != nil {
return nil, fmt.Errorf("eth pg fetcher: storage fetching error: %s", err.Error())
}
return iplds, err
} }
// FetchHeaders fetches headers // FetchHeaders fetches headers
// It uses the f.fetch method func (f *IPLDFetcher) FetchHeader(tx *sqlx.Tx, c HeaderModel) (ipfs.BlockModel, error) {
func (f *IPLDFetcher) FetchHeader(c HeaderModel) (ipfs.BlockModel, error) {
log.Debug("fetching header ipld") log.Debug("fetching header ipld")
dc, err := cid.Decode(c.CID) headerBytes, err := shared.FetchIPLDByMhKey(tx, c.MhKey)
if err != nil {
return ipfs.BlockModel{}, err
}
header, err := f.fetch(dc)
if err != nil { if err != nil {
return ipfs.BlockModel{}, err return ipfs.BlockModel{}, err
} }
return ipfs.BlockModel{ return ipfs.BlockModel{
Data: header.RawData(), Data: headerBytes,
CID: header.Cid().String(), CID: c.CID,
}, nil }, nil
} }
// FetchUncles fetches uncles // FetchUncles fetches uncles
// It uses the f.fetchBatch method func (f *IPLDFetcher) FetchUncles(tx *sqlx.Tx, cids []UncleModel) ([]ipfs.BlockModel, error) {
func (f *IPLDFetcher) FetchUncles(cids []UncleModel) ([]ipfs.BlockModel, error) {
log.Debug("fetching uncle iplds") log.Debug("fetching uncle iplds")
uncleCids := make([]cid.Cid, len(cids)) uncleIPLDs := make([]ipfs.BlockModel, len(cids))
for i, c := range cids { for i, c := range cids {
dc, err := cid.Decode(c.CID) uncleBytes, err := shared.FetchIPLDByMhKey(tx, c.MhKey)
if err != nil { if err != nil {
return nil, err return nil, err
} }
uncleCids[i] = dc
}
uncles := f.fetchBatch(uncleCids)
uncleIPLDs := make([]ipfs.BlockModel, len(uncles))
for i, uncle := range uncles {
uncleIPLDs[i] = ipfs.BlockModel{ uncleIPLDs[i] = ipfs.BlockModel{
Data: uncle.RawData(), Data: uncleBytes,
CID: uncle.Cid().String(), CID: c.CID,
} }
} }
if len(uncleIPLDs) != len(uncleCids) {
log.Errorf("ipfs fetcher: number of uncle blocks returned (%d) does not match number expected (%d)", len(uncles), len(uncleCids))
return uncleIPLDs, errUnexpectedNumberOfIPLDs
}
return uncleIPLDs, nil return uncleIPLDs, nil
} }
// FetchTrxs fetches transactions // FetchTrxs fetches transactions
// It uses the f.fetchBatch method func (f *IPLDFetcher) FetchTrxs(tx *sqlx.Tx, cids []TxModel) ([]ipfs.BlockModel, error) {
func (f *IPLDFetcher) FetchTrxs(cids []TxModel) ([]ipfs.BlockModel, error) {
log.Debug("fetching transaction iplds") log.Debug("fetching transaction iplds")
trxCids := make([]cid.Cid, len(cids)) trxIPLDs := make([]ipfs.BlockModel, len(cids))
for i, c := range cids { for i, c := range cids {
dc, err := cid.Decode(c.CID) txBytes, err := shared.FetchIPLDByMhKey(tx, c.MhKey)
if err != nil { if err != nil {
return nil, err return nil, err
} }
trxCids[i] = dc
}
trxs := f.fetchBatch(trxCids)
trxIPLDs := make([]ipfs.BlockModel, len(trxs))
for i, trx := range trxs {
trxIPLDs[i] = ipfs.BlockModel{ trxIPLDs[i] = ipfs.BlockModel{
Data: trx.RawData(), Data: txBytes,
CID: trx.Cid().String(), CID: c.CID,
} }
} }
if len(trxIPLDs) != len(trxCids) {
log.Errorf("ipfs fetcher: number of transaction blocks returned (%d) does not match number expected (%d)", len(trxs), len(trxCids))
return trxIPLDs, errUnexpectedNumberOfIPLDs
}
return trxIPLDs, nil return trxIPLDs, nil
} }
// FetchRcts fetches receipts // FetchRcts fetches receipts
// It uses the f.fetchBatch method func (f *IPLDFetcher) FetchRcts(tx *sqlx.Tx, cids []ReceiptModel) ([]ipfs.BlockModel, error) {
func (f *IPLDFetcher) FetchRcts(cids []ReceiptModel) ([]ipfs.BlockModel, error) {
log.Debug("fetching receipt iplds") log.Debug("fetching receipt iplds")
rctCids := make([]cid.Cid, len(cids)) rctIPLDs := make([]ipfs.BlockModel, len(cids))
for i, c := range cids { for i, c := range cids {
dc, err := cid.Decode(c.CID) rctBytes, err := shared.FetchIPLDByMhKey(tx, c.MhKey)
if err != nil { if err != nil {
return nil, err return nil, err
} }
rctCids[i] = dc
}
rcts := f.fetchBatch(rctCids)
rctIPLDs := make([]ipfs.BlockModel, len(rcts))
for i, rct := range rcts {
rctIPLDs[i] = ipfs.BlockModel{ rctIPLDs[i] = ipfs.BlockModel{
Data: rct.RawData(), Data: rctBytes,
CID: rct.Cid().String(), CID: c.CID,
} }
} }
if len(rctIPLDs) != len(rctCids) {
log.Errorf("ipfs fetcher: number of receipt blocks returned (%d) does not match number expected (%d)", len(rcts), len(rctCids))
return rctIPLDs, errUnexpectedNumberOfIPLDs
}
return rctIPLDs, nil return rctIPLDs, nil
} }
// FetchState fetches state nodes // FetchState fetches state nodes
// It uses the single f.fetch method instead of the batch fetch, because it func (f *IPLDFetcher) FetchState(tx *sqlx.Tx, cids []StateNodeModel) ([]StateNode, error) {
// needs to maintain the data's relation to state keys
func (f *IPLDFetcher) FetchState(cids []StateNodeModel) ([]StateNode, error) {
log.Debug("fetching state iplds") log.Debug("fetching state iplds")
stateNodes := make([]StateNode, 0, len(cids)) stateNodes := make([]StateNode, 0, len(cids))
for _, stateNode := range cids { for _, stateNode := range cids {
if stateNode.CID == "" { if stateNode.CID == "" {
continue continue
} }
dc, err := cid.Decode(stateNode.CID) stateBytes, err := shared.FetchIPLDByMhKey(tx, stateNode.MhKey)
if err != nil {
return nil, err
}
state, err := f.fetch(dc)
if err != nil { if err != nil {
return nil, err return nil, err
} }
stateNodes = append(stateNodes, StateNode{ stateNodes = append(stateNodes, StateNode{
IPLD: ipfs.BlockModel{ IPLD: ipfs.BlockModel{
Data: state.RawData(), Data: stateBytes,
CID: state.Cid().String(), CID: stateNode.CID,
}, },
StateLeafKey: common.HexToHash(stateNode.StateKey), StateLeafKey: common.HexToHash(stateNode.StateKey),
Type: ResolveToNodeType(stateNode.NodeType), Type: ResolveToNodeType(stateNode.NodeType),
@ -224,27 +189,21 @@ func (f *IPLDFetcher) FetchState(cids []StateNodeModel) ([]StateNode, error) {
} }
// FetchStorage fetches storage nodes // FetchStorage fetches storage nodes
// It uses the single f.fetch method instead of the batch fetch, because it func (f *IPLDFetcher) FetchStorage(tx *sqlx.Tx, cids []StorageNodeWithStateKeyModel) ([]StorageNode, error) {
// needs to maintain the data's relation to state and storage keys
func (f *IPLDFetcher) FetchStorage(cids []StorageNodeWithStateKeyModel) ([]StorageNode, error) {
log.Debug("fetching storage iplds") log.Debug("fetching storage iplds")
storageNodes := make([]StorageNode, 0, len(cids)) storageNodes := make([]StorageNode, 0, len(cids))
for _, storageNode := range cids { for _, storageNode := range cids {
if storageNode.CID == "" || storageNode.StateKey == "" { if storageNode.CID == "" || storageNode.StateKey == "" {
continue continue
} }
dc, err := cid.Decode(storageNode.CID) storageBytes, err := shared.FetchIPLDByMhKey(tx, storageNode.MhKey)
if err != nil {
return nil, err
}
storage, err := f.fetch(dc)
if err != nil { if err != nil {
return nil, err return nil, err
} }
storageNodes = append(storageNodes, StorageNode{ storageNodes = append(storageNodes, StorageNode{
IPLD: ipfs.BlockModel{ IPLD: ipfs.BlockModel{
Data: storage.RawData(), Data: storageBytes,
CID: storage.Cid().String(), CID: storageNode.CID,
}, },
StateLeafKey: common.HexToHash(storageNode.StateKey), StateLeafKey: common.HexToHash(storageNode.StateKey),
StorageLeafKey: common.HexToHash(storageNode.StorageKey), StorageLeafKey: common.HexToHash(storageNode.StorageKey),
@ -254,20 +213,3 @@ func (f *IPLDFetcher) FetchStorage(cids []StorageNodeWithStateKeyModel) ([]Stora
} }
return storageNodes, nil return storageNodes, nil
} }
// fetch is used to fetch a single cid
func (f *IPLDFetcher) fetch(cid cid.Cid) (blocks.Block, error) {
return f.BlockService.GetBlock(context.Background(), cid)
}
// fetchBatch is used to fetch a batch of IPFS data blocks by cid
// There is no guarantee all are fetched, and no error in such a case, so
// downstream we will need to confirm which CIDs were fetched in the result set
func (f *IPLDFetcher) fetchBatch(cids []cid.Cid) []blocks.Block {
fetchedBlocks := make([]blocks.Block, 0, len(cids))
blockChan := f.BlockService.GetBlocks(context.Background(), cids)
for block := range blockChan {
fetchedBlocks = append(fetchedBlocks, block)
}
return fetchedBlocks
}

138
pkg/eth/ipld_fetcher_test.go
View File

@ -17,139 +17,49 @@
package eth_test package eth_test
import ( import (
"bytes"
"math/big"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/statediff"
"github.com/ipfs/go-block-format"
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/eth" "github.com/vulcanize/ipfs-blockchain-watcher/pkg/eth"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs" "github.com/vulcanize/ipfs-blockchain-watcher/pkg/eth/mocks"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs/mocks" "github.com/vulcanize/ipfs-blockchain-watcher/pkg/postgres"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/shared"
) )
var ( var (
mockHeaderData = []byte{0, 1, 2, 3, 4} db *postgres.DB
mockUncleData = []byte{1, 2, 3, 4, 5} pubAndIndexer *eth.IPLDPublisher
mockTrxData = []byte{2, 3, 4, 5, 6} fetcher *eth.IPLDFetcher
mockReceiptData = []byte{3, 4, 5, 6, 7}
mockStateData = []byte{4, 5, 6, 7, 8}
mockStorageData = []byte{5, 6, 7, 8, 9}
mockStorageData2 = []byte{6, 7, 8, 9, 1}
mockHeaderBlock = blocks.NewBlock(mockHeaderData)
mockUncleBlock = blocks.NewBlock(mockUncleData)
mockTrxBlock = blocks.NewBlock(mockTrxData)
mockReceiptBlock = blocks.NewBlock(mockReceiptData)
mockStateBlock = blocks.NewBlock(mockStateData)
mockStorageBlock1 = blocks.NewBlock(mockStorageData)
mockStorageBlock2 = blocks.NewBlock(mockStorageData2)
mockBlocks = []blocks.Block{mockHeaderBlock, mockUncleBlock, mockTrxBlock, mockReceiptBlock, mockStateBlock, mockStorageBlock1, mockStorageBlock2}
mockBlockService *mocks.MockIPFSBlockService
mockCIDWrapper = &eth.CIDWrapper{
BlockNumber: big.NewInt(9000),
Header: eth.HeaderModel{
TotalDifficulty: "1337",
CID: mockHeaderBlock.Cid().String(),
},
Uncles: []eth.UncleModel{
{
CID: mockUncleBlock.Cid().String(),
},
},
Transactions: []eth.TxModel{
{
CID: mockTrxBlock.Cid().String(),
},
},
Receipts: []eth.ReceiptModel{
{
CID: mockReceiptBlock.Cid().String(),
},
},
StateNodes: []eth.StateNodeModel{{
CID: mockStateBlock.Cid().String(),
NodeType: 2,
StateKey: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
}},
StorageNodes: []eth.StorageNodeWithStateKeyModel{{
CID: mockStorageBlock1.Cid().String(),
NodeType: 2,
StateKey: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
StorageKey: "0000000000000000000000000000000000000000000000000000000000000001",
},
{
CID: mockStorageBlock2.Cid().String(),
NodeType: 2,
StateKey: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
StorageKey: "0000000000000000000000000000000000000000000000000000000000000002",
}},
}
) )
var _ = Describe("IPLDFetcher", func() { var _ = Describe("IPLDFetcher", func() {
Describe("Fetch", func() { Describe("Fetch", func() {
BeforeEach(func() { BeforeEach(func() {
mockBlockService = new(mocks.MockIPFSBlockService) var err error
err := mockBlockService.AddBlocks(mockBlocks) db, err = shared.SetupDB()
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
Expect(len(mockBlockService.Blocks)).To(Equal(7)) pubAndIndexer = eth.NewIPLDPublisher(db)
err = pubAndIndexer.Publish(mocks.MockConvertedPayload)
Expect(err).ToNot(HaveOccurred())
fetcher = eth.NewIPLDFetcher(db)
})
AfterEach(func() {
eth.TearDownDB(db)
}) })
It("Fetches and returns IPLDs for the CIDs provided in the CIDWrapper", func() { It("Fetches and returns IPLDs for the CIDs provided in the CIDWrapper", func() {
fetcher := new(eth.IPLDFetcher) i, err := fetcher.Fetch(mocks.MockCIDWrapper)
fetcher.BlockService = mockBlockService
i, err := fetcher.Fetch(mockCIDWrapper)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
iplds, ok := i.(eth.IPLDs) iplds, ok := i.(eth.IPLDs)
Expect(ok).To(BeTrue()) Expect(ok).To(BeTrue())
Expect(iplds.TotalDifficulty).To(Equal(big.NewInt(1337))) Expect(iplds.TotalDifficulty).To(Equal(mocks.MockConvertedPayload.TotalDifficulty))
Expect(iplds.BlockNumber).To(Equal(mockCIDWrapper.BlockNumber)) Expect(iplds.BlockNumber).To(Equal(mocks.MockConvertedPayload.Block.Number()))
Expect(iplds.Header).To(Equal(ipfs.BlockModel{ Expect(iplds.Header).To(Equal(mocks.MockIPLDs.Header))
Data: mockHeaderBlock.RawData(), Expect(len(iplds.Uncles)).To(Equal(0))
CID: mockHeaderBlock.Cid().String(), Expect(iplds.Transactions).To(Equal(mocks.MockIPLDs.Transactions))
})) Expect(iplds.Receipts).To(Equal(mocks.MockIPLDs.Receipts))
Expect(len(iplds.Uncles)).To(Equal(1)) Expect(iplds.StateNodes).To(Equal(mocks.MockIPLDs.StateNodes))
Expect(iplds.Uncles[0]).To(Equal(ipfs.BlockModel{ Expect(iplds.StorageNodes).To(Equal(mocks.MockIPLDs.StorageNodes))
Data: mockUncleBlock.RawData(),
CID: mockUncleBlock.Cid().String(),
}))
Expect(len(iplds.Transactions)).To(Equal(1))
Expect(iplds.Transactions[0]).To(Equal(ipfs.BlockModel{
Data: mockTrxBlock.RawData(),
CID: mockTrxBlock.Cid().String(),
}))
Expect(len(iplds.Receipts)).To(Equal(1))
Expect(iplds.Receipts[0]).To(Equal(ipfs.BlockModel{
Data: mockReceiptBlock.RawData(),
CID: mockReceiptBlock.Cid().String(),
}))
Expect(len(iplds.StateNodes)).To(Equal(1))
Expect(iplds.StateNodes[0].StateLeafKey).To(Equal(common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")))
Expect(iplds.StateNodes[0].Type).To(Equal(statediff.Leaf))
Expect(iplds.StateNodes[0].IPLD).To(Equal(ipfs.BlockModel{
Data: mockStateBlock.RawData(),
CID: mockStateBlock.Cid().String(),
}))
Expect(len(iplds.StorageNodes)).To(Equal(2))
for _, storage := range iplds.StorageNodes {
Expect(storage.Type).To(Equal(statediff.Leaf))
Expect(storage.StateLeafKey).To(Equal(common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")))
if bytes.Equal(storage.StorageLeafKey.Bytes(), common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001").Bytes()) {
Expect(storage.IPLD).To(Equal(ipfs.BlockModel{
Data: mockStorageBlock1.RawData(),
CID: mockStorageBlock1.Cid().String(),
}))
}
if bytes.Equal(storage.StorageLeafKey.Bytes(), common.HexToHash("0000000000000000000000000000000000000000000000000000000000000002").Bytes()) {
Expect(storage.IPLD).To(Equal(ipfs.BlockModel{
Data: mockStorageBlock2.RawData(),
CID: mockStorageBlock2.Cid().String(),
}))
}
}
}) })
}) })
}) })

215
pkg/eth/ipld_pg_fetcher.go
View File

@ -1,215 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package eth
import (
"errors"
"fmt"
"math/big"
"github.com/ethereum/go-ethereum/common"
"github.com/jmoiron/sqlx"
log "github.com/sirupsen/logrus"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/postgres"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/shared"
)
// IPLDPGFetcher satisfies the IPLDFetcher interface for ethereum
// It interfaces directly with PG-IPFS
type IPLDPGFetcher struct {
db *postgres.DB
}
// NewIPLDPGFetcher creates a pointer to a new IPLDPGFetcher
func NewIPLDPGFetcher(db *postgres.DB) *IPLDPGFetcher {
return &IPLDPGFetcher{
db: db,
}
}
// Fetch is the exported method for fetching and returning all the IPLDS specified in the CIDWrapper
func (f *IPLDPGFetcher) Fetch(cids shared.CIDsForFetching) (shared.IPLDs, error) {
cidWrapper, ok := cids.(*CIDWrapper)
if !ok {
return nil, fmt.Errorf("eth fetcher: expected cids type %T got %T", &CIDWrapper{}, cids)
}
log.Debug("fetching iplds")
iplds := IPLDs{}
iplds.TotalDifficulty, ok = new(big.Int).SetString(cidWrapper.Header.TotalDifficulty, 10)
if !ok {
return nil, errors.New("eth fetcher: unable to set total difficulty")
}
iplds.BlockNumber = cidWrapper.BlockNumber
tx, err := f.db.Beginx()
if err != nil {
return nil, err
}
defer func() {
if p := recover(); p != nil {
shared.Rollback(tx)
panic(p)
} else if err != nil {
shared.Rollback(tx)
} else {
err = tx.Commit()
}
}()
iplds.Header, err = f.FetchHeader(tx, cidWrapper.Header)
if err != nil {
return nil, fmt.Errorf("eth pg fetcher: header fetching error: %s", err.Error())
}
iplds.Uncles, err = f.FetchUncles(tx, cidWrapper.Uncles)
if err != nil {
return nil, fmt.Errorf("eth pg fetcher: uncle fetching error: %s", err.Error())
}
iplds.Transactions, err = f.FetchTrxs(tx, cidWrapper.Transactions)
if err != nil {
return nil, fmt.Errorf("eth pg fetcher: transaction fetching error: %s", err.Error())
}
iplds.Receipts, err = f.FetchRcts(tx, cidWrapper.Receipts)
if err != nil {
return nil, fmt.Errorf("eth pg fetcher: receipt fetching error: %s", err.Error())
}
iplds.StateNodes, err = f.FetchState(tx, cidWrapper.StateNodes)
if err != nil {
return nil, fmt.Errorf("eth pg fetcher: state fetching error: %s", err.Error())
}
iplds.StorageNodes, err = f.FetchStorage(tx, cidWrapper.StorageNodes)
if err != nil {
return nil, fmt.Errorf("eth pg fetcher: storage fetching error: %s", err.Error())
}
return iplds, err
}
// FetchHeaders fetches headers
func (f *IPLDPGFetcher) FetchHeader(tx *sqlx.Tx, c HeaderModel) (ipfs.BlockModel, error) {
log.Debug("fetching header ipld")
headerBytes, err := shared.FetchIPLDByMhKey(tx, c.MhKey)
if err != nil {
return ipfs.BlockModel{}, err
}
return ipfs.BlockModel{
Data: headerBytes,
CID: c.CID,
}, nil
}
// FetchUncles fetches uncles
func (f *IPLDPGFetcher) FetchUncles(tx *sqlx.Tx, cids []UncleModel) ([]ipfs.BlockModel, error) {
log.Debug("fetching uncle iplds")
uncleIPLDs := make([]ipfs.BlockModel, len(cids))
for i, c := range cids {
uncleBytes, err := shared.FetchIPLDByMhKey(tx, c.MhKey)
if err != nil {
return nil, err
}
uncleIPLDs[i] = ipfs.BlockModel{
Data: uncleBytes,
CID: c.CID,
}
}
return uncleIPLDs, nil
}
// FetchTrxs fetches transactions
func (f *IPLDPGFetcher) FetchTrxs(tx *sqlx.Tx, cids []TxModel) ([]ipfs.BlockModel, error) {
log.Debug("fetching transaction iplds")
trxIPLDs := make([]ipfs.BlockModel, len(cids))
for i, c := range cids {
txBytes, err := shared.FetchIPLDByMhKey(tx, c.MhKey)
if err != nil {
return nil, err
}
trxIPLDs[i] = ipfs.BlockModel{
Data: txBytes,
CID: c.CID,
}
}
return trxIPLDs, nil
}
// FetchRcts fetches receipts
func (f *IPLDPGFetcher) FetchRcts(tx *sqlx.Tx, cids []ReceiptModel) ([]ipfs.BlockModel, error) {
log.Debug("fetching receipt iplds")
rctIPLDs := make([]ipfs.BlockModel, len(cids))
for i, c := range cids {
rctBytes, err := shared.FetchIPLDByMhKey(tx, c.MhKey)
if err != nil {
return nil, err
}
rctIPLDs[i] = ipfs.BlockModel{
Data: rctBytes,
CID: c.CID,
}
}
return rctIPLDs, nil
}
// FetchState fetches state nodes
func (f *IPLDPGFetcher) FetchState(tx *sqlx.Tx, cids []StateNodeModel) ([]StateNode, error) {
log.Debug("fetching state iplds")
stateNodes := make([]StateNode, 0, len(cids))
for _, stateNode := range cids {
if stateNode.CID == "" {
continue
}
stateBytes, err := shared.FetchIPLDByMhKey(tx, stateNode.MhKey)
if err != nil {
return nil, err
}
stateNodes = append(stateNodes, StateNode{
IPLD: ipfs.BlockModel{
Data: stateBytes,
CID: stateNode.CID,
},
StateLeafKey: common.HexToHash(stateNode.StateKey),
Type: ResolveToNodeType(stateNode.NodeType),
Path: stateNode.Path,
})
}
return stateNodes, nil
}
// FetchStorage fetches storage nodes
func (f *IPLDPGFetcher) FetchStorage(tx *sqlx.Tx, cids []StorageNodeWithStateKeyModel) ([]StorageNode, error) {
log.Debug("fetching storage iplds")
storageNodes := make([]StorageNode, 0, len(cids))
for _, storageNode := range cids {
if storageNode.CID == "" || storageNode.StateKey == "" {
continue
}
storageBytes, err := shared.FetchIPLDByMhKey(tx, storageNode.MhKey)
if err != nil {
return nil, err
}
storageNodes = append(storageNodes, StorageNode{
IPLD: ipfs.BlockModel{
Data: storageBytes,
CID: storageNode.CID,
},
StateLeafKey: common.HexToHash(storageNode.StateKey),
StorageLeafKey: common.HexToHash(storageNode.StorageKey),
Type: ResolveToNodeType(storageNode.NodeType),
Path: storageNode.Path,
})
}
return storageNodes, nil
}

65
pkg/eth/ipld_pg_fetcher_test.go
View File

@ -1,65 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package eth_test
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/eth"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/eth/mocks"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/postgres"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/shared"
)
var (
db *postgres.DB
pubAndIndexer *eth.IPLDPublisherAndIndexer
fetcher *eth.IPLDPGFetcher
)
var _ = Describe("IPLDPGFetcher", func() {
Describe("Fetch", func() {
BeforeEach(func() {
var err error
db, err = shared.SetupDB()
Expect(err).ToNot(HaveOccurred())
pubAndIndexer = eth.NewIPLDPublisherAndIndexer(db)
_, err = pubAndIndexer.Publish(mocks.MockConvertedPayload)
Expect(err).ToNot(HaveOccurred())
fetcher = eth.NewIPLDPGFetcher(db)
})
AfterEach(func() {
eth.TearDownDB(db)
})
It("Fetches and returns IPLDs for the CIDs provided in the CIDWrapper", func() {
i, err := fetcher.Fetch(mocks.MockCIDWrapper)
Expect(err).ToNot(HaveOccurred())
iplds, ok := i.(eth.IPLDs)
Expect(ok).To(BeTrue())
Expect(iplds.TotalDifficulty).To(Equal(mocks.MockConvertedPayload.TotalDifficulty))
Expect(iplds.BlockNumber).To(Equal(mocks.MockConvertedPayload.Block.Number()))
Expect(iplds.Header).To(Equal(mocks.MockIPLDs.Header))
Expect(len(iplds.Uncles)).To(Equal(0))
Expect(iplds.Transactions).To(Equal(mocks.MockIPLDs.Transactions))
Expect(iplds.Receipts).To(Equal(mocks.MockIPLDs.Receipts))
Expect(iplds.StateNodes).To(Equal(mocks.MockIPLDs.StateNodes))
Expect(iplds.StorageNodes).To(Equal(mocks.MockIPLDs.StorageNodes))
})
})
})

16
pkg/eth/mocks/publisher.go
View File

@ -32,13 +32,13 @@ type IPLDPublisher struct {
} }
// Publish publishes an IPLDPayload to IPFS and returns the corresponding CIDPayload // Publish publishes an IPLDPayload to IPFS and returns the corresponding CIDPayload
func (pub *IPLDPublisher) Publish(payload shared.ConvertedData) (shared.CIDsForIndexing, error) { func (pub *IPLDPublisher) Publish(payload shared.ConvertedData) error {
ipldPayload, ok := payload.(eth.ConvertedPayload) ipldPayload, ok := payload.(eth.ConvertedPayload)
if !ok { if !ok {
return nil, fmt.Errorf("publish expected payload type %T got %T", &eth.ConvertedPayload{}, payload) return fmt.Errorf("publish expected payload type %T got %T", &eth.ConvertedPayload{}, payload)
} }
pub.PassedIPLDPayload = ipldPayload pub.PassedIPLDPayload = ipldPayload
return pub.ReturnCIDPayload, pub.ReturnErr return pub.ReturnErr
} }
// IterativeIPLDPublisher is the underlying struct for the Publisher interface; used in testing // IterativeIPLDPublisher is the underlying struct for the Publisher interface; used in testing
@ -50,16 +50,12 @@ type IterativeIPLDPublisher struct {
} }
// Publish publishes an IPLDPayload to IPFS and returns the corresponding CIDPayload // Publish publishes an IPLDPayload to IPFS and returns the corresponding CIDPayload
func (pub *IterativeIPLDPublisher) Publish(payload shared.ConvertedData) (shared.CIDsForIndexing, error) { func (pub *IterativeIPLDPublisher) Publish(payload shared.ConvertedData) error {
ipldPayload, ok := payload.(eth.ConvertedPayload) ipldPayload, ok := payload.(eth.ConvertedPayload)
if !ok { if !ok {
return nil, fmt.Errorf("publish expected payload type %T got %T", &eth.ConvertedPayload{}, payload) return fmt.Errorf("publish expected payload type %T got %T", &eth.ConvertedPayload{}, payload)
} }
pub.PassedIPLDPayload = append(pub.PassedIPLDPayload, ipldPayload) pub.PassedIPLDPayload = append(pub.PassedIPLDPayload, ipldPayload)
if len(pub.ReturnCIDPayload) < pub.iteration+1 {
return nil, fmt.Errorf("IterativeIPLDPublisher does not have a payload to return at iteration %d", pub.iteration)
}
returnPayload := pub.ReturnCIDPayload[pub.iteration]
pub.iteration++ pub.iteration++
return returnPayload, pub.ReturnErr return pub.ReturnErr
} }

45
pkg/eth/mocks/test_data.go
View File

@ -63,6 +63,7 @@ var (
AnotherAddress = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476593") AnotherAddress = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476593")
ContractAddress = crypto.CreateAddress(SenderAddr, MockTransactions[2].Nonce()) ContractAddress = crypto.CreateAddress(SenderAddr, MockTransactions[2].Nonce())
ContractHash = crypto.Keccak256Hash(ContractAddress.Bytes()).String() ContractHash = crypto.Keccak256Hash(ContractAddress.Bytes()).String()
MockContractByteCode = []byte{0, 1, 2, 3, 4, 5}
mockTopic11 = common.HexToHash("0x04") mockTopic11 = common.HexToHash("0x04")
mockTopic12 = common.HexToHash("0x06") mockTopic12 = common.HexToHash("0x06")
mockTopic21 = common.HexToHash("0x05") mockTopic21 = common.HexToHash("0x05")
@ -99,28 +100,34 @@ var (
StorageMhKey = shared.MultihashKeyFromCID(StorageCID) StorageMhKey = shared.MultihashKeyFromCID(StorageCID)
MockTrxMeta = []eth.TxModel{ MockTrxMeta = []eth.TxModel{
{ {
CID: "", // This is empty until we go to publish to ipfs CID: "", // This is empty until we go to publish to ipfs
MhKey: "", MhKey: "",
Src: SenderAddr.Hex(), Src: SenderAddr.Hex(),
Dst: Address.String(), Dst: Address.String(),
Index: 0, Index: 0,
TxHash: MockTransactions[0].Hash().String(), TxHash: MockTransactions[0].Hash().String(),
Data: []byte{},
Deployment: false,
}, },
{ {
CID: "", CID: "",
MhKey: "", MhKey: "",
Src: SenderAddr.Hex(), Src: SenderAddr.Hex(),
Dst: AnotherAddress.String(), Dst: AnotherAddress.String(),
Index: 1, Index: 1,
TxHash: MockTransactions[1].Hash().String(), TxHash: MockTransactions[1].Hash().String(),
Data: []byte{},
Deployment: false,
}, },
{ {
CID: "", CID: "",
MhKey: "", MhKey: "",
Src: SenderAddr.Hex(), Src: SenderAddr.Hex(),
Dst: "", Dst: "",
Index: 2, Index: 2,
TxHash: MockTransactions[2].Hash().String(), TxHash: MockTransactions[2].Hash().String(),
Data: MockContractByteCode,
Deployment: true,
}, },
} }
MockTrxMetaPostPublsh = []eth.TxModel{ MockTrxMetaPostPublsh = []eth.TxModel{
@ -532,7 +539,7 @@ func createTransactionsAndReceipts() (types.Transactions, types.Receipts, common
// make transactions // make transactions
trx1 := types.NewTransaction(0, Address, big.NewInt(1000), 50, big.NewInt(100), []byte{}) trx1 := types.NewTransaction(0, Address, big.NewInt(1000), 50, big.NewInt(100), []byte{})
trx2 := types.NewTransaction(1, AnotherAddress, big.NewInt(2000), 100, big.NewInt(200), []byte{}) trx2 := types.NewTransaction(1, AnotherAddress, big.NewInt(2000), 100, big.NewInt(200), []byte{})
trx3 := types.NewContractCreation(2, big.NewInt(1500), 75, big.NewInt(150), []byte{0, 1, 2, 3, 4, 5}) trx3 := types.NewContractCreation(2, big.NewInt(1500), 75, big.NewInt(150), MockContractByteCode)
transactionSigner := types.MakeSigner(params.MainnetChainConfig, new(big.Int).Set(BlockNumber)) transactionSigner := types.MakeSigner(params.MainnetChainConfig, new(big.Int).Set(BlockNumber))
mockCurve := elliptic.P256() mockCurve := elliptic.P256()
mockPrvKey, err := ecdsa.GenerateKey(mockCurve, rand.Reader) mockPrvKey, err := ecdsa.GenerateKey(mockCurve, rand.Reader)

18
pkg/eth/models.go
View File

@ -51,14 +51,16 @@ type UncleModel struct {
// TxModel is the db model for eth.transaction_cids // TxModel is the db model for eth.transaction_cids
type TxModel struct { type TxModel struct {
ID int64 `db:"id"` ID int64 `db:"id"`
HeaderID int64 `db:"header_id"` HeaderID int64 `db:"header_id"`
Index int64 `db:"index"` Index int64 `db:"index"`
TxHash string `db:"tx_hash"` TxHash string `db:"tx_hash"`
CID string `db:"cid"` CID string `db:"cid"`
MhKey string `db:"mh_key"` MhKey string `db:"mh_key"`
Dst string `db:"dst"` Dst string `db:"dst"`
Src string `db:"src"` Src string `db:"src"`
Data []byte `db:"data"`
Deployment bool `db:"deployment"`
} }
// ReceiptModel is the db model for eth.receipt_cids // ReceiptModel is the db model for eth.receipt_cids

228
pkg/eth/publish_and_indexer.go
View File

@ -1,228 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package eth
import (
"fmt"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/statediff"
"github.com/jmoiron/sqlx"
"github.com/multiformats/go-multihash"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs/ipld"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/postgres"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/shared"
)
// IPLDPublisherAndIndexer satisfies the IPLDPublisher interface for ethereum
// It interfaces directly with the public.blocks table of PG-IPFS rather than going through an ipfs intermediary
// It publishes and indexes IPLDs together in a single sqlx.Tx
type IPLDPublisherAndIndexer struct {
indexer *CIDIndexer
}
// NewIPLDPublisherAndIndexer creates a pointer to a new IPLDPublisherAndIndexer which satisfies the IPLDPublisher interface
func NewIPLDPublisherAndIndexer(db *postgres.DB) *IPLDPublisherAndIndexer {
return &IPLDPublisherAndIndexer{
indexer: NewCIDIndexer(db),
}
}
// Publish publishes an IPLDPayload to IPFS and returns the corresponding CIDPayload
func (pub *IPLDPublisherAndIndexer) Publish(payload shared.ConvertedData) (shared.CIDsForIndexing, error) {
ipldPayload, ok := payload.(ConvertedPayload)
if !ok {
return nil, fmt.Errorf("eth IPLDPublisherAndIndexer expected payload type %T got %T", ConvertedPayload{}, payload)
}
// Generate the iplds
headerNode, uncleNodes, txNodes, txTrieNodes, rctNodes, rctTrieNodes, err := ipld.FromBlockAndReceipts(ipldPayload.Block, ipldPayload.Receipts)
if err != nil {
return nil, err
}
// Begin new db tx
tx, err := pub.indexer.db.Beginx()
if err != nil {
return nil, err
}
defer func() {
if p := recover(); p != nil {
shared.Rollback(tx)
panic(p)
} else if err != nil {
shared.Rollback(tx)
} else {
err = tx.Commit()
}
}()
// Publish trie nodes
for _, node := range txTrieNodes {
if err := shared.PublishIPLD(tx, node); err != nil {
return nil, err
}
}
for _, node := range rctTrieNodes {
if err := shared.PublishIPLD(tx, node); err != nil {
return nil, err
}
}
// Publish and index header
if err := shared.PublishIPLD(tx, headerNode); err != nil {
return nil, err
}
reward := CalcEthBlockReward(ipldPayload.Block.Header(), ipldPayload.Block.Uncles(), ipldPayload.Block.Transactions(), ipldPayload.Receipts)
header := HeaderModel{
CID: headerNode.Cid().String(),
MhKey: shared.MultihashKeyFromCID(headerNode.Cid()),
ParentHash: ipldPayload.Block.ParentHash().String(),
BlockNumber: ipldPayload.Block.Number().String(),
BlockHash: ipldPayload.Block.Hash().String(),
TotalDifficulty: ipldPayload.TotalDifficulty.String(),
Reward: reward.String(),
Bloom: ipldPayload.Block.Bloom().Bytes(),
StateRoot: ipldPayload.Block.Root().String(),
RctRoot: ipldPayload.Block.ReceiptHash().String(),
TxRoot: ipldPayload.Block.TxHash().String(),
UncleRoot: ipldPayload.Block.UncleHash().String(),
Timestamp: ipldPayload.Block.Time(),
}
headerID, err := pub.indexer.indexHeaderCID(tx, header)
if err != nil {
return nil, err
}
// Publish and index uncles
for _, uncleNode := range uncleNodes {
if err := shared.PublishIPLD(tx, uncleNode); err != nil {
return nil, err
}
uncleReward := CalcUncleMinerReward(ipldPayload.Block.Number().Int64(), uncleNode.Number.Int64())
uncle := UncleModel{
CID: uncleNode.Cid().String(),
MhKey: shared.MultihashKeyFromCID(uncleNode.Cid()),
ParentHash: uncleNode.ParentHash.String(),
BlockHash: uncleNode.Hash().String(),
Reward: uncleReward.String(),
}
if err := pub.indexer.indexUncleCID(tx, uncle, headerID); err != nil {
return nil, err
}
}
// Publish and index txs and receipts
for i, txNode := range txNodes {
if err := shared.PublishIPLD(tx, txNode); err != nil {
return nil, err
}
rctNode := rctNodes[i]
if err := shared.PublishIPLD(tx, rctNode); err != nil {
return nil, err
}
txModel := ipldPayload.TxMetaData[i]
txModel.CID = txNode.Cid().String()
txModel.MhKey = shared.MultihashKeyFromCID(txNode.Cid())
txID, err := pub.indexer.indexTransactionCID(tx, txModel, headerID)
if err != nil {
return nil, err
}
rctModel := ipldPayload.ReceiptMetaData[i]
rctModel.CID = rctNode.Cid().String()
rctModel.MhKey = shared.MultihashKeyFromCID(rctNode.Cid())
if err := pub.indexer.indexReceiptCID(tx, rctModel, txID); err != nil {
return nil, err
}
}
// Publish and index state and storage
err = pub.publishAndIndexStateAndStorage(tx, ipldPayload, headerID)
// This IPLDPublisher does both publishing and indexing, we do not need to pass anything forward to the indexer
return nil, err // return err variable explicitly so that we return the err = tx.Commit() assignment in the defer
}
func (pub *IPLDPublisherAndIndexer) publishAndIndexStateAndStorage(tx *sqlx.Tx, ipldPayload ConvertedPayload, headerID int64) error {
// Publish and index state and storage
for _, stateNode := range ipldPayload.StateNodes {
stateCIDStr, err := shared.PublishRaw(tx, ipld.MEthStateTrie, multihash.KECCAK_256, stateNode.Value)
if err != nil {
return err
}
mhKey, _ := shared.MultihashKeyFromCIDString(stateCIDStr)
stateModel := StateNodeModel{
Path: stateNode.Path,
StateKey: stateNode.LeafKey.String(),
CID: stateCIDStr,
MhKey: mhKey,
NodeType: ResolveFromNodeType(stateNode.Type),
}
stateID, err := pub.indexer.indexStateCID(tx, stateModel, headerID)
if err != nil {
return err
}
// If we have a leaf, decode and index the account data and any associated storage diffs
if stateNode.Type == statediff.Leaf {
var i []interface{}
if err := rlp.DecodeBytes(stateNode.Value, &i); err != nil {
return err
}
if len(i) != 2 {
return fmt.Errorf("eth IPLDPublisherAndIndexer expected state leaf node rlp to decode into two elements")
}
var account state.Account
if err := rlp.DecodeBytes(i[1].([]byte), &account); err != nil {
return err
}
accountModel := StateAccountModel{
Balance: account.Balance.String(),
Nonce: account.Nonce,
CodeHash: account.CodeHash,
StorageRoot: account.Root.String(),
}
if err := pub.indexer.indexStateAccount(tx, accountModel, stateID); err != nil {
return err
}
for _, storageNode := range ipldPayload.StorageNodes[common.Bytes2Hex(stateNode.Path)] {
storageCIDStr, err := shared.PublishRaw(tx, ipld.MEthStorageTrie, multihash.KECCAK_256, storageNode.Value)
if err != nil {
return err
}
mhKey, _ := shared.MultihashKeyFromCIDString(storageCIDStr)
storageModel := StorageNodeModel{
Path: storageNode.Path,
StorageKey: storageNode.LeafKey.Hex(),
CID: storageCIDStr,
MhKey: mhKey,
NodeType: ResolveFromNodeType(storageNode.Type),
}
if err := pub.indexer.indexStorageCID(tx, storageModel, stateID); err != nil {
return err
}
}
}
}
return nil
}
// Index satisfies the shared.CIDIndexer interface
func (pub *IPLDPublisherAndIndexer) Index(cids shared.CIDsForIndexing) error {
return nil
}

238
pkg/eth/publish_and_indexer_test.go
View File

@ -1,238 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package eth_test
import (
"github.com/ethereum/go-ethereum/common"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-ipfs-blockstore"
"github.com/ipfs/go-ipfs-ds-help"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/eth"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/eth/mocks"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/postgres"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/shared"
)
var _ = Describe("PublishAndIndexer", func() {
var (
db *postgres.DB
err error
repo *eth.IPLDPublisherAndIndexer
ipfsPgGet = `SELECT data FROM public.blocks
WHERE key = $1`
)
BeforeEach(func() {
db, err = shared.SetupDB()
Expect(err).ToNot(HaveOccurred())
repo = eth.NewIPLDPublisherAndIndexer(db)
})
AfterEach(func() {
eth.TearDownDB(db)
})
Describe("Publish", func() {
It("Published and indexes header IPLDs in a single tx", func() {
emptyReturn, err := repo.Publish(mocks.MockConvertedPayload)
Expect(emptyReturn).To(BeNil())
Expect(err).ToNot(HaveOccurred())
pgStr := `SELECT cid, td, reward, id
FROM eth.header_cids
WHERE block_number = $1`
// check header was properly indexed
type res struct {
CID string
TD string
Reward string
ID int
}
header := new(res)
err = db.QueryRowx(pgStr, 1).StructScan(header)
Expect(err).ToNot(HaveOccurred())
Expect(header.CID).To(Equal(mocks.HeaderCID.String()))
Expect(header.TD).To(Equal(mocks.MockBlock.Difficulty().String()))
Expect(header.Reward).To(Equal("5000000000000000000"))
dc, err := cid.Decode(header.CID)
Expect(err).ToNot(HaveOccurred())
mhKey := dshelp.MultihashToDsKey(dc.Hash())
prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
var data []byte
err = db.Get(&data, ipfsPgGet, prefixedKey)
Expect(err).ToNot(HaveOccurred())
Expect(data).To(Equal(mocks.MockHeaderRlp))
})
It("Publishes and indexes transaction IPLDs in a single tx", func() {
emptyReturn, err := repo.Publish(mocks.MockConvertedPayload)
Expect(emptyReturn).To(BeNil())
Expect(err).ToNot(HaveOccurred())
// check that txs were properly indexed
trxs := make([]string, 0)
pgStr := `SELECT transaction_cids.cid FROM eth.transaction_cids INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id)
WHERE header_cids.block_number = $1`
err = db.Select(&trxs, pgStr, 1)
Expect(err).ToNot(HaveOccurred())
Expect(len(trxs)).To(Equal(3))
Expect(shared.ListContainsString(trxs, mocks.Trx1CID.String())).To(BeTrue())
Expect(shared.ListContainsString(trxs, mocks.Trx2CID.String())).To(BeTrue())
Expect(shared.ListContainsString(trxs, mocks.Trx3CID.String())).To(BeTrue())
// and published
for _, c := range trxs {
dc, err := cid.Decode(c)
Expect(err).ToNot(HaveOccurred())
mhKey := dshelp.MultihashToDsKey(dc.Hash())
prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
var data []byte
err = db.Get(&data, ipfsPgGet, prefixedKey)
Expect(err).ToNot(HaveOccurred())
switch c {
case mocks.Trx1CID.String():
Expect(data).To(Equal(mocks.MockTransactions.GetRlp(0)))
case mocks.Trx2CID.String():
Expect(data).To(Equal(mocks.MockTransactions.GetRlp(1)))
case mocks.Trx3CID.String():
Expect(data).To(Equal(mocks.MockTransactions.GetRlp(2)))
}
}
})
It("Publishes and indexes receipt IPLDs in a single tx", func() {
emptyReturn, err := repo.Publish(mocks.MockConvertedPayload)
Expect(emptyReturn).To(BeNil())
Expect(err).ToNot(HaveOccurred())
// check receipts were properly indexed
rcts := make([]string, 0)
pgStr := `SELECT receipt_cids.cid FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
WHERE receipt_cids.tx_id = transaction_cids.id
AND transaction_cids.header_id = header_cids.id
AND header_cids.block_number = $1`
err = db.Select(&rcts, pgStr, 1)
Expect(err).ToNot(HaveOccurred())
Expect(len(rcts)).To(Equal(3))
Expect(shared.ListContainsString(rcts, mocks.Rct1CID.String())).To(BeTrue())
Expect(shared.ListContainsString(rcts, mocks.Rct2CID.String())).To(BeTrue())
Expect(shared.ListContainsString(rcts, mocks.Rct3CID.String())).To(BeTrue())
// and published
for _, c := range rcts {
dc, err := cid.Decode(c)
Expect(err).ToNot(HaveOccurred())
mhKey := dshelp.MultihashToDsKey(dc.Hash())
prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
var data []byte
err = db.Get(&data, ipfsPgGet, prefixedKey)
Expect(err).ToNot(HaveOccurred())
switch c {
case mocks.Rct1CID.String():
Expect(data).To(Equal(mocks.MockReceipts.GetRlp(0)))
case mocks.Rct2CID.String():
Expect(data).To(Equal(mocks.MockReceipts.GetRlp(1)))
case mocks.Rct3CID.String():
Expect(data).To(Equal(mocks.MockReceipts.GetRlp(2)))
}
}
})
It("Publishes and indexes state IPLDs in a single tx", func() {
emptyReturn, err := repo.Publish(mocks.MockConvertedPayload)
Expect(emptyReturn).To(BeNil())
Expect(err).ToNot(HaveOccurred())
// check that state nodes were properly indexed and published
stateNodes := make([]eth.StateNodeModel, 0)
pgStr := `SELECT state_cids.id, state_cids.cid, state_cids.state_leaf_key, state_cids.node_type, state_cids.state_path, state_cids.header_id
FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
WHERE header_cids.block_number = $1`
err = db.Select(&stateNodes, pgStr, 1)
Expect(err).ToNot(HaveOccurred())
Expect(len(stateNodes)).To(Equal(2))
for _, stateNode := range stateNodes {
var data []byte
dc, err := cid.Decode(stateNode.CID)
Expect(err).ToNot(HaveOccurred())
mhKey := dshelp.MultihashToDsKey(dc.Hash())
prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
err = db.Get(&data, ipfsPgGet, prefixedKey)
Expect(err).ToNot(HaveOccurred())
pgStr = `SELECT * from eth.state_accounts WHERE state_id = $1`
var account eth.StateAccountModel
err = db.Get(&account, pgStr, stateNode.ID)
Expect(err).ToNot(HaveOccurred())
if stateNode.CID == mocks.State1CID.String() {
Expect(stateNode.NodeType).To(Equal(2))
Expect(stateNode.StateKey).To(Equal(common.BytesToHash(mocks.ContractLeafKey).Hex()))
Expect(stateNode.Path).To(Equal([]byte{'\x06'}))
Expect(data).To(Equal(mocks.ContractLeafNode))
Expect(account).To(Equal(eth.StateAccountModel{
ID: account.ID,
StateID: stateNode.ID,
Balance: "0",
CodeHash: mocks.ContractCodeHash.Bytes(),
StorageRoot: mocks.ContractRoot,
Nonce: 1,
}))
}
if stateNode.CID == mocks.State2CID.String() {
Expect(stateNode.NodeType).To(Equal(2))
Expect(stateNode.StateKey).To(Equal(common.BytesToHash(mocks.AccountLeafKey).Hex()))
Expect(stateNode.Path).To(Equal([]byte{'\x0c'}))
Expect(data).To(Equal(mocks.AccountLeafNode))
Expect(account).To(Equal(eth.StateAccountModel{
ID: account.ID,
StateID: stateNode.ID,
Balance: "1000",
CodeHash: mocks.AccountCodeHash.Bytes(),
StorageRoot: mocks.AccountRoot,
Nonce: 0,
}))
}
}
pgStr = `SELECT * from eth.state_accounts WHERE state_id = $1`
})
It("Publishes and indexes storage IPLDs in a single tx", func() {
emptyReturn, err := repo.Publish(mocks.MockConvertedPayload)
Expect(emptyReturn).To(BeNil())
Expect(err).ToNot(HaveOccurred())
// check that storage nodes were properly indexed
storageNodes := make([]eth.StorageNodeWithStateKeyModel, 0)
pgStr := `SELECT storage_cids.cid, state_cids.state_leaf_key, storage_cids.storage_leaf_key, storage_cids.node_type, storage_cids.storage_path
FROM eth.storage_cids, eth.state_cids, eth.header_cids
WHERE storage_cids.state_id = state_cids.id
AND state_cids.header_id = header_cids.id
AND header_cids.block_number = $1`
err = db.Select(&storageNodes, pgStr, 1)
Expect(err).ToNot(HaveOccurred())
Expect(len(storageNodes)).To(Equal(1))
Expect(storageNodes[0]).To(Equal(eth.StorageNodeWithStateKeyModel{
CID: mocks.StorageCID.String(),
NodeType: 2,
StorageKey: common.BytesToHash(mocks.StorageLeafKey).Hex(),
StateKey: common.BytesToHash(mocks.ContractLeafKey).Hex(),
Path: []byte{},
}))
var data []byte
dc, err := cid.Decode(storageNodes[0].CID)
Expect(err).ToNot(HaveOccurred())
mhKey := dshelp.MultihashToDsKey(dc.Hash())
prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
err = db.Get(&data, ipfsPgGet, prefixedKey)
Expect(err).ToNot(HaveOccurred())
Expect(data).To(Equal(mocks.StorageLeafNode))
})
})
})

316
pkg/eth/publisher.go
View File

@ -21,64 +21,77 @@ import (
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/statediff" "github.com/ethereum/go-ethereum/statediff"
"github.com/jmoiron/sqlx"
"github.com/multiformats/go-multihash"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs/dag_putters"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs/ipld" "github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs/ipld"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/postgres"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/shared" "github.com/vulcanize/ipfs-blockchain-watcher/pkg/shared"
) )
// IPLDPublisher satisfies the IPLDPublisher for ethereum // IPLDPublisher satisfies the IPLDPublisher interface for ethereum
// It interfaces directly with the public.blocks table of PG-IPFS rather than going through an ipfs intermediary
// It publishes and indexes IPLDs together in a single sqlx.Tx
type IPLDPublisher struct { type IPLDPublisher struct {
HeaderPutter ipfs.DagPutter indexer *CIDIndexer
TransactionPutter ipfs.DagPutter
TransactionTriePutter ipfs.DagPutter
ReceiptPutter ipfs.DagPutter
ReceiptTriePutter ipfs.DagPutter
StatePutter ipfs.DagPutter
StoragePutter ipfs.DagPutter
} }
// NewIPLDPublisher creates a pointer to a new IPLDPublisher which satisfies the IPLDPublisher interface // NewIPLDPublisher creates a pointer to a new IPLDPublisher which satisfies the IPLDPublisher interface
func NewIPLDPublisher(ipfsPath string) (*IPLDPublisher, error) { func NewIPLDPublisher(db *postgres.DB) *IPLDPublisher {
node, err := ipfs.InitIPFSNode(ipfsPath)
if err != nil {
return nil, err
}
return &IPLDPublisher{ return &IPLDPublisher{
HeaderPutter: dag_putters.NewEthBlockHeaderDagPutter(node), indexer: NewCIDIndexer(db),
TransactionPutter: dag_putters.NewEthTxsDagPutter(node), }
TransactionTriePutter: dag_putters.NewEthTxTrieDagPutter(node),
ReceiptPutter: dag_putters.NewEthReceiptDagPutter(node),
ReceiptTriePutter: dag_putters.NewEthRctTrieDagPutter(node),
StatePutter: dag_putters.NewEthStateDagPutter(node),
StoragePutter: dag_putters.NewEthStorageDagPutter(node),
}, nil
} }
// Publish publishes an IPLDPayload to IPFS and returns the corresponding CIDPayload // Publish publishes an IPLDPayload to IPFS and returns the corresponding CIDPayload
func (pub *IPLDPublisher) Publish(payload shared.ConvertedData) (shared.CIDsForIndexing, error) { func (pub *IPLDPublisher) Publish(payload shared.ConvertedData) error {
ipldPayload, ok := payload.(ConvertedPayload) ipldPayload, ok := payload.(ConvertedPayload)
if !ok { if !ok {
return nil, fmt.Errorf("eth publisher expected payload type %T got %T", ConvertedPayload{}, payload) return fmt.Errorf("eth IPLDPublisher expected payload type %T got %T", ConvertedPayload{}, payload)
} }
// Generate the nodes for publishing // Generate the iplds
headerNode, uncleNodes, txNodes, txTrieNodes, rctNodes, rctTrieNodes, err := ipld.FromBlockAndReceipts(ipldPayload.Block, ipldPayload.Receipts) headerNode, uncleNodes, txNodes, txTrieNodes, rctNodes, rctTrieNodes, err := ipld.FromBlockAndReceipts(ipldPayload.Block, ipldPayload.Receipts)
if err != nil { if err != nil {
return nil, err return err
} }
// Process and publish headers // Begin new db tx
headerCid, err := pub.publishHeader(headerNode) tx, err := pub.indexer.db.Beginx()
if err != nil { if err != nil {
return nil, err return err
}
defer func() {
if p := recover(); p != nil {
shared.Rollback(tx)
panic(p)
} else if err != nil {
shared.Rollback(tx)
} else {
err = tx.Commit()
}
}()
// Publish trie nodes
for _, node := range txTrieNodes {
if err := shared.PublishIPLD(tx, node); err != nil {
return err
}
}
for _, node := range rctTrieNodes {
if err := shared.PublishIPLD(tx, node); err != nil {
return err
}
}
// Publish and index header
if err := shared.PublishIPLD(tx, headerNode); err != nil {
return err
} }
reward := CalcEthBlockReward(ipldPayload.Block.Header(), ipldPayload.Block.Uncles(), ipldPayload.Block.Transactions(), ipldPayload.Receipts) reward := CalcEthBlockReward(ipldPayload.Block.Header(), ipldPayload.Block.Uncles(), ipldPayload.Block.Transactions(), ipldPayload.Receipts)
header := HeaderModel{ header := HeaderModel{
CID: headerCid, CID: headerNode.Cid().String(),
MhKey: shared.MultihashKeyFromCID(headerNode.Cid()), MhKey: shared.MultihashKeyFromCID(headerNode.Cid()),
ParentHash: ipldPayload.Block.ParentHash().String(), ParentHash: ipldPayload.Block.ParentHash().String(),
BlockNumber: ipldPayload.Block.Number().String(), BlockNumber: ipldPayload.Block.Number().String(),
@ -92,189 +105,124 @@ func (pub *IPLDPublisher) Publish(payload shared.ConvertedData) (shared.CIDsForI
UncleRoot: ipldPayload.Block.UncleHash().String(), UncleRoot: ipldPayload.Block.UncleHash().String(),
Timestamp: ipldPayload.Block.Time(), Timestamp: ipldPayload.Block.Time(),
} }
headerID, err := pub.indexer.indexHeaderCID(tx, header)
if err != nil {
return err
}
// Process and publish uncles // Publish and index uncles
uncleCids := make([]UncleModel, len(uncleNodes)) for _, uncleNode := range uncleNodes {
for i, uncle := range uncleNodes { if err := shared.PublishIPLD(tx, uncleNode); err != nil {
uncleCid, err := pub.publishHeader(uncle) return err
if err != nil {
return nil, err
} }
uncleReward := CalcUncleMinerReward(ipldPayload.Block.Number().Int64(), uncle.Number.Int64()) uncleReward := CalcUncleMinerReward(ipldPayload.Block.Number().Int64(), uncleNode.Number.Int64())
uncleCids[i] = UncleModel{ uncle := UncleModel{
CID: uncleCid, CID: uncleNode.Cid().String(),
MhKey: shared.MultihashKeyFromCID(uncle.Cid()), MhKey: shared.MultihashKeyFromCID(uncleNode.Cid()),
ParentHash: uncle.ParentHash.String(), ParentHash: uncleNode.ParentHash.String(),
BlockHash: uncle.Hash().String(), BlockHash: uncleNode.Hash().String(),
Reward: uncleReward.String(), Reward: uncleReward.String(),
} }
if err := pub.indexer.indexUncleCID(tx, uncle, headerID); err != nil {
return err
}
} }
// Process and publish transactions // Publish and index txs and receipts
transactionCids, err := pub.publishTransactions(txNodes, txTrieNodes, ipldPayload.TxMetaData) for i, txNode := range txNodes {
if err != nil { if err := shared.PublishIPLD(tx, txNode); err != nil {
return nil, err return err
} }
rctNode := rctNodes[i]
// Process and publish receipts if err := shared.PublishIPLD(tx, rctNode); err != nil {
receiptsCids, err := pub.publishReceipts(rctNodes, rctTrieNodes, ipldPayload.ReceiptMetaData) return err
if err != nil { }
return nil, err txModel := ipldPayload.TxMetaData[i]
} txModel.CID = txNode.Cid().String()
txModel.MhKey = shared.MultihashKeyFromCID(txNode.Cid())
// Process and publish state leafs txID, err := pub.indexer.indexTransactionCID(tx, txModel, headerID)
stateNodeCids, stateAccounts, err := pub.publishStateNodes(ipldPayload.StateNodes)
if err != nil {
return nil, err
}
// Process and publish storage leafs
storageNodeCids, err := pub.publishStorageNodes(ipldPayload.StorageNodes)
if err != nil {
return nil, err
}
// Package CIDs and their metadata into a single struct
return &CIDPayload{
HeaderCID: header,
UncleCIDs: uncleCids,
TransactionCIDs: transactionCids,
ReceiptCIDs: receiptsCids,
StateNodeCIDs: stateNodeCids,
StorageNodeCIDs: storageNodeCids,
StateAccounts: stateAccounts,
}, nil
}
func (pub *IPLDPublisher) generateBlockNodes(body *types.Block, receipts types.Receipts) (*ipld.EthHeader,
[]*ipld.EthHeader, []*ipld.EthTx, []*ipld.EthTxTrie, []*ipld.EthReceipt, []*ipld.EthRctTrie, error) {
return ipld.FromBlockAndReceipts(body, receipts)
}
func (pub *IPLDPublisher) publishHeader(header *ipld.EthHeader) (string, error) {
return pub.HeaderPutter.DagPut(header)
}
func (pub *IPLDPublisher) publishTransactions(transactions []*ipld.EthTx, txTrie []*ipld.EthTxTrie, trxMeta []TxModel) ([]TxModel, error) {
trxCids := make([]TxModel, len(transactions))
for i, tx := range transactions {
cid, err := pub.TransactionPutter.DagPut(tx)
if err != nil { if err != nil {
return nil, err return err
} }
trxCids[i] = TxModel{ // If tx is a contract deployment, publish the data (code)
CID: cid, if txModel.Deployment {
MhKey: shared.MultihashKeyFromCID(tx.Cid()), if _, err = shared.PublishRaw(tx, ipld.MEthStorageTrie, multihash.KECCAK_256, txModel.Data); err != nil {
Index: trxMeta[i].Index, return err
TxHash: trxMeta[i].TxHash, }
Src: trxMeta[i].Src, }
Dst: trxMeta[i].Dst, rctModel := ipldPayload.ReceiptMetaData[i]
rctModel.CID = rctNode.Cid().String()
rctModel.MhKey = shared.MultihashKeyFromCID(rctNode.Cid())
if err := pub.indexer.indexReceiptCID(tx, rctModel, txID); err != nil {
return err
} }
} }
for _, txNode := range txTrie {
// We don't do anything with the tx trie cids atm // Publish and index state and storage
if _, err := pub.TransactionTriePutter.DagPut(txNode); err != nil { err = pub.publishAndIndexStateAndStorage(tx, ipldPayload, headerID)
return nil, err
} return err // return err variable explicitly so that we return the err = tx.Commit() assignment in the defer
}
return trxCids, nil
} }
func (pub *IPLDPublisher) publishReceipts(receipts []*ipld.EthReceipt, receiptTrie []*ipld.EthRctTrie, receiptMeta []ReceiptModel) (map[common.Hash]ReceiptModel, error) { func (pub *IPLDPublisher) publishAndIndexStateAndStorage(tx *sqlx.Tx, ipldPayload ConvertedPayload, headerID int64) error {
rctCids := make(map[common.Hash]ReceiptModel) // Publish and index state and storage
for i, rct := range receipts { for _, stateNode := range ipldPayload.StateNodes {
cid, err := pub.ReceiptPutter.DagPut(rct) stateCIDStr, err := shared.PublishRaw(tx, ipld.MEthStateTrie, multihash.KECCAK_256, stateNode.Value)
if err != nil { if err != nil {
return nil, err return err
} }
rctCids[rct.TxHash] = ReceiptModel{ mhKey, _ := shared.MultihashKeyFromCIDString(stateCIDStr)
CID: cid, stateModel := StateNodeModel{
MhKey: shared.MultihashKeyFromCID(rct.Cid()),
Contract: receiptMeta[i].Contract,
ContractHash: receiptMeta[i].ContractHash,
Topic0s: receiptMeta[i].Topic0s,
Topic1s: receiptMeta[i].Topic1s,
Topic2s: receiptMeta[i].Topic2s,
Topic3s: receiptMeta[i].Topic3s,
LogContracts: receiptMeta[i].LogContracts,
}
}
for _, rctNode := range receiptTrie {
// We don't do anything with the rct trie cids atm
if _, err := pub.ReceiptTriePutter.DagPut(rctNode); err != nil {
return nil, err
}
}
return rctCids, nil
}
func (pub *IPLDPublisher) publishStateNodes(stateNodes []TrieNode) ([]StateNodeModel, map[string]StateAccountModel, error) {
stateNodeCids := make([]StateNodeModel, 0, len(stateNodes))
stateAccounts := make(map[string]StateAccountModel)
for _, stateNode := range stateNodes {
node, err := ipld.FromStateTrieRLP(stateNode.Value)
if err != nil {
return nil, nil, err
}
cid, err := pub.StatePutter.DagPut(node)
if err != nil {
return nil, nil, err
}
stateNodeCids = append(stateNodeCids, StateNodeModel{
Path: stateNode.Path, Path: stateNode.Path,
StateKey: stateNode.LeafKey.String(), StateKey: stateNode.LeafKey.String(),
CID: cid, CID: stateCIDStr,
MhKey: shared.MultihashKeyFromCID(node.Cid()), MhKey: mhKey,
NodeType: ResolveFromNodeType(stateNode.Type), NodeType: ResolveFromNodeType(stateNode.Type),
}) }
// If we have a leaf, decode the account to extract additional metadata for indexing stateID, err := pub.indexer.indexStateCID(tx, stateModel, headerID)
if err != nil {
return err
}
// If we have a leaf, decode and index the account data and any associated storage diffs
if stateNode.Type == statediff.Leaf { if stateNode.Type == statediff.Leaf {
var i []interface{} var i []interface{}
if err := rlp.DecodeBytes(stateNode.Value, &i); err != nil { if err := rlp.DecodeBytes(stateNode.Value, &i); err != nil {
return nil, nil, err return err
} }
if len(i) != 2 { if len(i) != 2 {
return nil, nil, fmt.Errorf("IPLDPublisher expected state leaf node rlp to decode into two elements") return fmt.Errorf("eth IPLDPublisher expected state leaf node rlp to decode into two elements")
} }
var account state.Account var account state.Account
if err := rlp.DecodeBytes(i[1].([]byte), &account); err != nil { if err := rlp.DecodeBytes(i[1].([]byte), &account); err != nil {
return nil, nil, err return err
} }
// Map state account to the state path hash accountModel := StateAccountModel{
statePath := common.Bytes2Hex(stateNode.Path)
stateAccounts[statePath] = StateAccountModel{
Balance: account.Balance.String(), Balance: account.Balance.String(),
Nonce: account.Nonce, Nonce: account.Nonce,
CodeHash: account.CodeHash, CodeHash: account.CodeHash,
StorageRoot: account.Root.String(), StorageRoot: account.Root.String(),
} }
if err := pub.indexer.indexStateAccount(tx, accountModel, stateID); err != nil {
return err
}
for _, storageNode := range ipldPayload.StorageNodes[common.Bytes2Hex(stateNode.Path)] {
storageCIDStr, err := shared.PublishRaw(tx, ipld.MEthStorageTrie, multihash.KECCAK_256, storageNode.Value)
if err != nil {
return err
}
mhKey, _ := shared.MultihashKeyFromCIDString(storageCIDStr)
storageModel := StorageNodeModel{
Path: storageNode.Path,
StorageKey: storageNode.LeafKey.Hex(),
CID: storageCIDStr,
MhKey: mhKey,
NodeType: ResolveFromNodeType(storageNode.Type),
}
if err := pub.indexer.indexStorageCID(tx, storageModel, stateID); err != nil {
return err
}
}
} }
} }
return stateNodeCids, stateAccounts, nil return nil
}
func (pub *IPLDPublisher) publishStorageNodes(storageNodes map[string][]TrieNode) (map[string][]StorageNodeModel, error) {
storageLeafCids := make(map[string][]StorageNodeModel)
for path, storageTrie := range storageNodes {
storageLeafCids[path] = make([]StorageNodeModel, 0, len(storageTrie))
for _, storageNode := range storageTrie {
node, err := ipld.FromStorageTrieRLP(storageNode.Value)
if err != nil {
return nil, err
}
cid, err := pub.StoragePutter.DagPut(node)
if err != nil {
return nil, err
}
// Map storage node cids to the state path hash
storageLeafCids[path] = append(storageLeafCids[path], StorageNodeModel{
Path: storageNode.Path,
StorageKey: storageNode.LeafKey.Hex(),
CID: cid,
MhKey: shared.MultihashKeyFromCID(node.Cid()),
NodeType: ResolveFromNodeType(storageNode.Type),
})
}
}
return storageLeafCids, nil
} }

268
pkg/eth/publisher_test.go
View File

@ -18,88 +18,216 @@ package eth_test
import ( import (
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-ipfs-blockstore"
"github.com/ipfs/go-ipfs-ds-help"
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/eth" "github.com/vulcanize/ipfs-blockchain-watcher/pkg/eth"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/eth/mocks" "github.com/vulcanize/ipfs-blockchain-watcher/pkg/eth/mocks"
mocks2 "github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs/mocks" "github.com/vulcanize/ipfs-blockchain-watcher/pkg/postgres"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/shared"
) )
var ( var _ = Describe("PublishAndIndexer", func() {
mockHeaderDagPutter *mocks2.MappedDagPutter var (
mockTrxDagPutter *mocks2.MappedDagPutter db *postgres.DB
mockTrxTrieDagPutter *mocks2.DagPutter err error
mockRctDagPutter *mocks2.MappedDagPutter repo *eth.IPLDPublisher
mockRctTrieDagPutter *mocks2.DagPutter ipfsPgGet = `SELECT data FROM public.blocks
mockStateDagPutter *mocks2.MappedDagPutter WHERE key = $1`
mockStorageDagPutter *mocks2.MappedDagPutter )
)
var _ = Describe("Publisher", func() {
BeforeEach(func() { BeforeEach(func() {
mockHeaderDagPutter = new(mocks2.MappedDagPutter) db, err = shared.SetupDB()
mockTrxDagPutter = new(mocks2.MappedDagPutter) Expect(err).ToNot(HaveOccurred())
mockTrxTrieDagPutter = new(mocks2.DagPutter) repo = eth.NewIPLDPublisher(db)
mockRctDagPutter = new(mocks2.MappedDagPutter) })
mockRctTrieDagPutter = new(mocks2.DagPutter) AfterEach(func() {
mockStateDagPutter = new(mocks2.MappedDagPutter) eth.TearDownDB(db)
mockStorageDagPutter = new(mocks2.MappedDagPutter)
}) })
Describe("Publish", func() { Describe("Publish", func() {
It("Publishes the passed IPLDPayload objects to IPFS and returns a CIDPayload for indexing", func() { It("Published and indexes header IPLDs in a single tx", func() {
mockHeaderDagPutter.CIDsToReturn = map[common.Hash]string{ err = repo.Publish(mocks.MockConvertedPayload)
common.BytesToHash(mocks.HeaderIPLD.RawData()): mocks.HeaderCID.String(),
}
mockTrxDagPutter.CIDsToReturn = map[common.Hash]string{
common.BytesToHash(mocks.Trx1IPLD.RawData()): mocks.Trx1CID.String(),
common.BytesToHash(mocks.Trx2IPLD.RawData()): mocks.Trx2CID.String(),
common.BytesToHash(mocks.Trx3IPLD.RawData()): mocks.Trx3CID.String(),
}
mockRctDagPutter.CIDsToReturn = map[common.Hash]string{
common.BytesToHash(mocks.Rct1IPLD.RawData()): mocks.Rct1CID.String(),
common.BytesToHash(mocks.Rct2IPLD.RawData()): mocks.Rct2CID.String(),
common.BytesToHash(mocks.Rct3IPLD.RawData()): mocks.Rct3CID.String(),
}
mockStateDagPutter.CIDsToReturn = map[common.Hash]string{
common.BytesToHash(mocks.State1IPLD.RawData()): mocks.State1CID.String(),
common.BytesToHash(mocks.State2IPLD.RawData()): mocks.State2CID.String(),
}
mockStorageDagPutter.CIDsToReturn = map[common.Hash]string{
common.BytesToHash(mocks.StorageIPLD.RawData()): mocks.StorageCID.String(),
}
publisher := eth.IPLDPublisher{
HeaderPutter: mockHeaderDagPutter,
TransactionPutter: mockTrxDagPutter,
TransactionTriePutter: mockTrxTrieDagPutter,
ReceiptPutter: mockRctDagPutter,
ReceiptTriePutter: mockRctTrieDagPutter,
StatePutter: mockStateDagPutter,
StoragePutter: mockStorageDagPutter,
}
payload, err := publisher.Publish(mocks.MockConvertedPayload)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
cidPayload, ok := payload.(*eth.CIDPayload) pgStr := `SELECT cid, td, reward, id
Expect(ok).To(BeTrue()) FROM eth.header_cids
Expect(cidPayload.HeaderCID.TotalDifficulty).To(Equal(mocks.MockConvertedPayload.TotalDifficulty.String())) WHERE block_number = $1`
Expect(cidPayload.HeaderCID.BlockNumber).To(Equal(mocks.MockCIDPayload.HeaderCID.BlockNumber)) // check header was properly indexed
Expect(cidPayload.HeaderCID.BlockHash).To(Equal(mocks.MockCIDPayload.HeaderCID.BlockHash)) type res struct {
Expect(cidPayload.HeaderCID.Reward).To(Equal(mocks.MockCIDPayload.HeaderCID.Reward)) CID string
Expect(cidPayload.UncleCIDs).To(Equal(mocks.MockCIDPayload.UncleCIDs)) TD string
Expect(cidPayload.HeaderCID).To(Equal(mocks.MockCIDPayload.HeaderCID)) Reward string
Expect(len(cidPayload.TransactionCIDs)).To(Equal(3)) ID int
Expect(cidPayload.TransactionCIDs[0]).To(Equal(mocks.MockCIDPayload.TransactionCIDs[0])) }
Expect(cidPayload.TransactionCIDs[1]).To(Equal(mocks.MockCIDPayload.TransactionCIDs[1])) header := new(res)
Expect(cidPayload.TransactionCIDs[2]).To(Equal(mocks.MockCIDPayload.TransactionCIDs[2])) err = db.QueryRowx(pgStr, 1).StructScan(header)
Expect(len(cidPayload.ReceiptCIDs)).To(Equal(3)) Expect(err).ToNot(HaveOccurred())
Expect(cidPayload.ReceiptCIDs[mocks.MockTransactions[0].Hash()]).To(Equal(mocks.MockCIDPayload.ReceiptCIDs[mocks.MockTransactions[0].Hash()])) Expect(header.CID).To(Equal(mocks.HeaderCID.String()))
Expect(cidPayload.ReceiptCIDs[mocks.MockTransactions[1].Hash()]).To(Equal(mocks.MockCIDPayload.ReceiptCIDs[mocks.MockTransactions[1].Hash()])) Expect(header.TD).To(Equal(mocks.MockBlock.Difficulty().String()))
Expect(cidPayload.ReceiptCIDs[mocks.MockTransactions[2].Hash()]).To(Equal(mocks.MockCIDPayload.ReceiptCIDs[mocks.MockTransactions[2].Hash()])) Expect(header.Reward).To(Equal("5000000000000000000"))
Expect(len(cidPayload.StateNodeCIDs)).To(Equal(2)) dc, err := cid.Decode(header.CID)
Expect(cidPayload.StateNodeCIDs[0]).To(Equal(mocks.MockCIDPayload.StateNodeCIDs[0])) Expect(err).ToNot(HaveOccurred())
Expect(cidPayload.StateNodeCIDs[1]).To(Equal(mocks.MockCIDPayload.StateNodeCIDs[1])) mhKey := dshelp.MultihashToDsKey(dc.Hash())
Expect(cidPayload.StorageNodeCIDs).To(Equal(mocks.MockCIDPayload.StorageNodeCIDs)) prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
var data []byte
err = db.Get(&data, ipfsPgGet, prefixedKey)
Expect(err).ToNot(HaveOccurred())
Expect(data).To(Equal(mocks.MockHeaderRlp))
})
It("Publishes and indexes transaction IPLDs in a single tx", func() {
err = repo.Publish(mocks.MockConvertedPayload)
Expect(err).ToNot(HaveOccurred())
// check that txs were properly indexed
trxs := make([]string, 0)
pgStr := `SELECT transaction_cids.cid FROM eth.transaction_cids INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id)
WHERE header_cids.block_number = $1`
err = db.Select(&trxs, pgStr, 1)
Expect(err).ToNot(HaveOccurred())
Expect(len(trxs)).To(Equal(3))
Expect(shared.ListContainsString(trxs, mocks.Trx1CID.String())).To(BeTrue())
Expect(shared.ListContainsString(trxs, mocks.Trx2CID.String())).To(BeTrue())
Expect(shared.ListContainsString(trxs, mocks.Trx3CID.String())).To(BeTrue())
// and published
for _, c := range trxs {
dc, err := cid.Decode(c)
Expect(err).ToNot(HaveOccurred())
mhKey := dshelp.MultihashToDsKey(dc.Hash())
prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
var data []byte
err = db.Get(&data, ipfsPgGet, prefixedKey)
Expect(err).ToNot(HaveOccurred())
switch c {
case mocks.Trx1CID.String():
Expect(data).To(Equal(mocks.MockTransactions.GetRlp(0)))
case mocks.Trx2CID.String():
Expect(data).To(Equal(mocks.MockTransactions.GetRlp(1)))
case mocks.Trx3CID.String():
Expect(data).To(Equal(mocks.MockTransactions.GetRlp(2)))
}
}
})
It("Publishes and indexes receipt IPLDs in a single tx", func() {
err = repo.Publish(mocks.MockConvertedPayload)
Expect(err).ToNot(HaveOccurred())
// check receipts were properly indexed
rcts := make([]string, 0)
pgStr := `SELECT receipt_cids.cid FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
WHERE receipt_cids.tx_id = transaction_cids.id
AND transaction_cids.header_id = header_cids.id
AND header_cids.block_number = $1`
err = db.Select(&rcts, pgStr, 1)
Expect(err).ToNot(HaveOccurred())
Expect(len(rcts)).To(Equal(3))
Expect(shared.ListContainsString(rcts, mocks.Rct1CID.String())).To(BeTrue())
Expect(shared.ListContainsString(rcts, mocks.Rct2CID.String())).To(BeTrue())
Expect(shared.ListContainsString(rcts, mocks.Rct3CID.String())).To(BeTrue())
// and published
for _, c := range rcts {
dc, err := cid.Decode(c)
Expect(err).ToNot(HaveOccurred())
mhKey := dshelp.MultihashToDsKey(dc.Hash())
prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
var data []byte
err = db.Get(&data, ipfsPgGet, prefixedKey)
Expect(err).ToNot(HaveOccurred())
switch c {
case mocks.Rct1CID.String():
Expect(data).To(Equal(mocks.MockReceipts.GetRlp(0)))
case mocks.Rct2CID.String():
Expect(data).To(Equal(mocks.MockReceipts.GetRlp(1)))
case mocks.Rct3CID.String():
Expect(data).To(Equal(mocks.MockReceipts.GetRlp(2)))
}
}
})
It("Publishes and indexes state IPLDs in a single tx", func() {
err = repo.Publish(mocks.MockConvertedPayload)
Expect(err).ToNot(HaveOccurred())
// check that state nodes were properly indexed and published
stateNodes := make([]eth.StateNodeModel, 0)
pgStr := `SELECT state_cids.id, state_cids.cid, state_cids.state_leaf_key, state_cids.node_type, state_cids.state_path, state_cids.header_id
FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
WHERE header_cids.block_number = $1`
err = db.Select(&stateNodes, pgStr, 1)
Expect(err).ToNot(HaveOccurred())
Expect(len(stateNodes)).To(Equal(2))
for _, stateNode := range stateNodes {
var data []byte
dc, err := cid.Decode(stateNode.CID)
Expect(err).ToNot(HaveOccurred())
mhKey := dshelp.MultihashToDsKey(dc.Hash())
prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
err = db.Get(&data, ipfsPgGet, prefixedKey)
Expect(err).ToNot(HaveOccurred())
pgStr = `SELECT * from eth.state_accounts WHERE state_id = $1`
var account eth.StateAccountModel
err = db.Get(&account, pgStr, stateNode.ID)
Expect(err).ToNot(HaveOccurred())
if stateNode.CID == mocks.State1CID.String() {
Expect(stateNode.NodeType).To(Equal(2))
Expect(stateNode.StateKey).To(Equal(common.BytesToHash(mocks.ContractLeafKey).Hex()))
Expect(stateNode.Path).To(Equal([]byte{'\x06'}))
Expect(data).To(Equal(mocks.ContractLeafNode))
Expect(account).To(Equal(eth.StateAccountModel{
ID: account.ID,
StateID: stateNode.ID,
Balance: "0",
CodeHash: mocks.ContractCodeHash.Bytes(),
StorageRoot: mocks.ContractRoot,
Nonce: 1,
}))
}
if stateNode.CID == mocks.State2CID.String() {
Expect(stateNode.NodeType).To(Equal(2))
Expect(stateNode.StateKey).To(Equal(common.BytesToHash(mocks.AccountLeafKey).Hex()))
Expect(stateNode.Path).To(Equal([]byte{'\x0c'}))
Expect(data).To(Equal(mocks.AccountLeafNode))
Expect(account).To(Equal(eth.StateAccountModel{
ID: account.ID,
StateID: stateNode.ID,
Balance: "1000",
CodeHash: mocks.AccountCodeHash.Bytes(),
StorageRoot: mocks.AccountRoot,
Nonce: 0,
}))
}
}
pgStr = `SELECT * from eth.state_accounts WHERE state_id = $1`
})
It("Publishes and indexes storage IPLDs in a single tx", func() {
err = repo.Publish(mocks.MockConvertedPayload)
Expect(err).ToNot(HaveOccurred())
// check that storage nodes were properly indexed
storageNodes := make([]eth.StorageNodeWithStateKeyModel, 0)
pgStr := `SELECT storage_cids.cid, state_cids.state_leaf_key, storage_cids.storage_leaf_key, storage_cids.node_type, storage_cids.storage_path
FROM eth.storage_cids, eth.state_cids, eth.header_cids
WHERE storage_cids.state_id = state_cids.id
AND state_cids.header_id = header_cids.id
AND header_cids.block_number = $1`
err = db.Select(&storageNodes, pgStr, 1)
Expect(err).ToNot(HaveOccurred())
Expect(len(storageNodes)).To(Equal(1))
Expect(storageNodes[0]).To(Equal(eth.StorageNodeWithStateKeyModel{
CID: mocks.StorageCID.String(),
NodeType: 2,
StorageKey: common.BytesToHash(mocks.StorageLeafKey).Hex(),
StateKey: common.BytesToHash(mocks.ContractLeafKey).Hex(),
Path: []byte{},
}))
var data []byte
dc, err := cid.Decode(storageNodes[0].CID)
Expect(err).ToNot(HaveOccurred())
mhKey := dshelp.MultihashToDsKey(dc.Hash())
prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
err = db.Get(&data, ipfsPgGet, prefixedKey)
Expect(err).ToNot(HaveOccurred())
Expect(data).To(Equal(mocks.StorageLeafNode))
}) })
}) })
}) })

14
pkg/historical/config.go
View File

@ -45,8 +45,6 @@ const (
// Config struct // Config struct
type Config struct { type Config struct {
Chain shared.ChainType Chain shared.ChainType
IPFSPath string
IPFSMode shared.IPFSMode
DBConfig config.Database DBConfig config.Database
DB *postgres.DB DB *postgres.DB
@ -71,19 +69,7 @@ func NewConfig() (*Config, error) {
return nil, err return nil, err
} }
c.IPFSMode, err = shared.GetIPFSMode()
if err != nil {
return nil, err
}
if c.IPFSMode == shared.LocalInterface || c.IPFSMode == shared.RemoteClient {
c.IPFSPath, err = shared.GetIPFSPath()
if err != nil {
return nil, err
}
}
c.DBConfig.Init() c.DBConfig.Init()
if err := c.init(); err != nil { if err := c.init(); err != nil {
return nil, err return nil, err
} }

17
pkg/historical/service.go
View File

@ -40,8 +40,6 @@ type BackFillService struct {
Converter shared.PayloadConverter Converter shared.PayloadConverter
// Interface for publishing the IPLD payloads to IPFS // Interface for publishing the IPLD payloads to IPFS
Publisher shared.IPLDPublisher Publisher shared.IPLDPublisher
// Interface for indexing the CIDs of the published IPLDs in Postgres
Indexer shared.CIDIndexer
// Interface for searching and retrieving CIDs from Postgres index // Interface for searching and retrieving CIDs from Postgres index
Retriever shared.CIDRetriever Retriever shared.CIDRetriever
// Interface for fetching payloads over at historical blocks; over http // Interface for fetching payloads over at historical blocks; over http
@ -64,15 +62,11 @@ type BackFillService struct {
// NewBackFillService returns a new BackFillInterface // NewBackFillService returns a new BackFillInterface
func NewBackFillService(settings *Config, screenAndServeChan chan shared.ConvertedData) (BackFillInterface, error) { func NewBackFillService(settings *Config, screenAndServeChan chan shared.ConvertedData) (BackFillInterface, error) {
publisher, err := builders.NewIPLDPublisher(settings.Chain, settings.IPFSPath, settings.DB, settings.IPFSMode) publisher, err := builders.NewIPLDPublisher(settings.Chain, settings.DB)
if err != nil { if err != nil {
return nil, err return nil, err
} }
indexer, err := builders.NewCIDIndexer(settings.Chain, settings.DB, settings.IPFSMode) converter, err := builders.NewPayloadConverter(settings.Chain, settings.NodeInfo.ChainID)
if err != nil {
return nil, err
}
converter, err := builders.NewPayloadConverter(settings.Chain)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -93,7 +87,6 @@ func NewBackFillService(settings *Config, screenAndServeChan chan shared.Convert
batchNumber = shared.DefaultMaxBatchNumber batchNumber = shared.DefaultMaxBatchNumber
} }
return &BackFillService{ return &BackFillService{
Indexer: indexer,
Converter: converter, Converter: converter,
Publisher: publisher, Publisher: publisher,
Retriever: retriever, Retriever: retriever,
@ -183,14 +176,10 @@ func (bfs *BackFillService) backFill(wg *sync.WaitGroup, id int, heightChan chan
default: default:
log.Debugf("%s backFill worker %d unable to forward converted payload to server; no channel ready to receive", bfs.chain.String(), id) log.Debugf("%s backFill worker %d unable to forward converted payload to server; no channel ready to receive", bfs.chain.String(), id)
} }
cidPayload, err := bfs.Publisher.Publish(ipldPayload) if err := bfs.Publisher.Publish(ipldPayload); err != nil {
if err != nil {
log.Errorf("%s backFill worker %d publisher error: %s", bfs.chain.String(), id, err.Error()) log.Errorf("%s backFill worker %d publisher error: %s", bfs.chain.String(), id, err.Error())
continue continue
} }
if err := bfs.Indexer.Index(cidPayload); err != nil {
log.Errorf("%s backFill worker %d indexer error: %s", bfs.chain.String(), id, err.Error())
}
} }
log.Infof("%s backFill worker %d finished section from %d to %d", bfs.chain.String(), id, heights[0], heights[len(heights)-1]) log.Infof("%s backFill worker %d finished section from %d to %d", bfs.chain.String(), id, heights[0], heights[len(heights)-1])
case <-bfs.QuitChan: case <-bfs.QuitChan:

20
pkg/historical/service_test.go
View File

@ -33,9 +33,6 @@ import (
var _ = Describe("BackFiller", func() { var _ = Describe("BackFiller", func() {
Describe("FillGaps", func() { Describe("FillGaps", func() {
It("Periodically checks for and fills in gaps in the watcher's data", func() { It("Periodically checks for and fills in gaps in the watcher's data", func() {
mockCidRepo := &mocks.CIDIndexer{
ReturnErr: nil,
}
mockPublisher := &mocks.IterativeIPLDPublisher{ mockPublisher := &mocks.IterativeIPLDPublisher{
ReturnCIDPayload: []*eth.CIDPayload{mocks.MockCIDPayload, mocks.MockCIDPayload}, ReturnCIDPayload: []*eth.CIDPayload{mocks.MockCIDPayload, mocks.MockCIDPayload},
ReturnErr: nil, ReturnErr: nil,
@ -60,7 +57,6 @@ var _ = Describe("BackFiller", func() {
} }
quitChan := make(chan bool, 1) quitChan := make(chan bool, 1)
backfiller := &historical.BackFillService{ backfiller := &historical.BackFillService{
Indexer: mockCidRepo,
Publisher: mockPublisher, Publisher: mockPublisher,
Converter: mockConverter, Converter: mockConverter,
Fetcher: mockFetcher, Fetcher: mockFetcher,
@ -74,9 +70,6 @@ var _ = Describe("BackFiller", func() {
backfiller.BackFill(wg) backfiller.BackFill(wg)
time.Sleep(time.Second * 3) time.Sleep(time.Second * 3)
quitChan <- true quitChan <- true
Expect(len(mockCidRepo.PassedCIDPayload)).To(Equal(2))
Expect(mockCidRepo.PassedCIDPayload[0]).To(Equal(mocks.MockCIDPayload))
Expect(mockCidRepo.PassedCIDPayload[1]).To(Equal(mocks.MockCIDPayload))
Expect(len(mockPublisher.PassedIPLDPayload)).To(Equal(2)) Expect(len(mockPublisher.PassedIPLDPayload)).To(Equal(2))
Expect(mockPublisher.PassedIPLDPayload[0]).To(Equal(mocks.MockConvertedPayload)) Expect(mockPublisher.PassedIPLDPayload[0]).To(Equal(mocks.MockConvertedPayload))
Expect(mockPublisher.PassedIPLDPayload[1]).To(Equal(mocks.MockConvertedPayload)) Expect(mockPublisher.PassedIPLDPayload[1]).To(Equal(mocks.MockConvertedPayload))
@ -89,9 +82,6 @@ var _ = Describe("BackFiller", func() {
}) })
It("Works for single block `ranges`", func() { It("Works for single block `ranges`", func() {
mockCidRepo := &mocks.CIDIndexer{
ReturnErr: nil,
}
mockPublisher := &mocks.IterativeIPLDPublisher{ mockPublisher := &mocks.IterativeIPLDPublisher{
ReturnCIDPayload: []*eth.CIDPayload{mocks.MockCIDPayload}, ReturnCIDPayload: []*eth.CIDPayload{mocks.MockCIDPayload},
ReturnErr: nil, ReturnErr: nil,
@ -115,7 +105,6 @@ var _ = Describe("BackFiller", func() {
} }
quitChan := make(chan bool, 1) quitChan := make(chan bool, 1)
backfiller := &historical.BackFillService{ backfiller := &historical.BackFillService{
Indexer: mockCidRepo,
Publisher: mockPublisher, Publisher: mockPublisher,
Converter: mockConverter, Converter: mockConverter,
Fetcher: mockFetcher, Fetcher: mockFetcher,
@ -129,8 +118,6 @@ var _ = Describe("BackFiller", func() {
backfiller.BackFill(wg) backfiller.BackFill(wg)
time.Sleep(time.Second * 3) time.Sleep(time.Second * 3)
quitChan <- true quitChan <- true
Expect(len(mockCidRepo.PassedCIDPayload)).To(Equal(1))
Expect(mockCidRepo.PassedCIDPayload[0]).To(Equal(mocks.MockCIDPayload))
Expect(len(mockPublisher.PassedIPLDPayload)).To(Equal(1)) Expect(len(mockPublisher.PassedIPLDPayload)).To(Equal(1))
Expect(mockPublisher.PassedIPLDPayload[0]).To(Equal(mocks.MockConvertedPayload)) Expect(mockPublisher.PassedIPLDPayload[0]).To(Equal(mocks.MockConvertedPayload))
Expect(len(mockConverter.PassedStatediffPayload)).To(Equal(1)) Expect(len(mockConverter.PassedStatediffPayload)).To(Equal(1))
@ -141,9 +128,6 @@ var _ = Describe("BackFiller", func() {
}) })
It("Finds beginning gap", func() { It("Finds beginning gap", func() {
mockCidRepo := &mocks.CIDIndexer{
ReturnErr: nil,
}
mockPublisher := &mocks.IterativeIPLDPublisher{ mockPublisher := &mocks.IterativeIPLDPublisher{
ReturnCIDPayload: []*eth.CIDPayload{mocks.MockCIDPayload, mocks.MockCIDPayload}, ReturnCIDPayload: []*eth.CIDPayload{mocks.MockCIDPayload, mocks.MockCIDPayload},
ReturnErr: nil, ReturnErr: nil,
@ -169,7 +153,6 @@ var _ = Describe("BackFiller", func() {
} }
quitChan := make(chan bool, 1) quitChan := make(chan bool, 1)
backfiller := &historical.BackFillService{ backfiller := &historical.BackFillService{
Indexer: mockCidRepo,
Publisher: mockPublisher, Publisher: mockPublisher,
Converter: mockConverter, Converter: mockConverter,
Fetcher: mockFetcher, Fetcher: mockFetcher,
@ -183,9 +166,6 @@ var _ = Describe("BackFiller", func() {
backfiller.BackFill(wg) backfiller.BackFill(wg)
time.Sleep(time.Second * 3) time.Sleep(time.Second * 3)
quitChan <- true quitChan <- true
Expect(len(mockCidRepo.PassedCIDPayload)).To(Equal(2))
Expect(mockCidRepo.PassedCIDPayload[0]).To(Equal(mocks.MockCIDPayload))
Expect(mockCidRepo.PassedCIDPayload[1]).To(Equal(mocks.MockCIDPayload))
Expect(len(mockPublisher.PassedIPLDPayload)).To(Equal(2)) Expect(len(mockPublisher.PassedIPLDPayload)).To(Equal(2))
Expect(mockPublisher.PassedIPLDPayload[0]).To(Equal(mocks.MockConvertedPayload)) Expect(mockPublisher.PassedIPLDPayload[0]).To(Equal(mocks.MockConvertedPayload))
Expect(mockPublisher.PassedIPLDPayload[1]).To(Equal(mocks.MockConvertedPayload)) Expect(mockPublisher.PassedIPLDPayload[1]).To(Equal(mocks.MockConvertedPayload))

90
pkg/ipfs/builders.go
View File

@ -1,90 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipfs
import (
"context"
"github.com/sirupsen/logrus"
"github.com/ipfs/go-blockservice"
"github.com/ipfs/go-ipfs/core"
"github.com/ipfs/go-ipfs/plugin/loader"
"github.com/ipfs/go-ipfs/repo/fsrepo"
ipld "github.com/ipfs/go-ipld-format"
)
// InitIPFSPlugins is used to initialized IPFS plugins before creating a new IPFS node
// This should only be called once
func InitIPFSPlugins() error {
logrus.Debug("initializing IPFS plugins")
l, err := loader.NewPluginLoader("")
if err != nil {
return err
}
err = l.Initialize()
if err != nil {
return err
}
return l.Inject()
}
// InitIPFSBlockService is used to configure and return a BlockService using an ipfs repo path (e.g. ~/.ipfs)
func InitIPFSBlockService(ipfsPath string) (blockservice.BlockService, error) {
logrus.Debug("initializing IPFS block service interface")
r, openErr := fsrepo.Open(ipfsPath)
if openErr != nil {
return nil, openErr
}
ctx := context.Background()
cfg := &core.BuildCfg{
Online: false,
Repo: r,
}
ipfsNode, newNodeErr := core.NewNode(ctx, cfg)
if newNodeErr != nil {
return nil, newNodeErr
}
return ipfsNode.Blocks, nil
}
type IPFS struct {
n *core.IpfsNode
ctx context.Context
}
func (ipfs IPFS) Add(node ipld.Node) error {
return ipfs.n.DAG.Add(ipfs.n.Context(), node)
}
func InitIPFSNode(repoPath string) (*IPFS, error) {
logrus.Debug("initializing IPFS node interface")
r, err := fsrepo.Open(repoPath)
if err != nil {
return nil, err
}
ctx := context.Background()
cfg := &core.BuildCfg{
Online: false,
Repo: r,
}
ipfsNode, err := core.NewNode(ctx, cfg)
if err != nil {
return nil, err
}
return &IPFS{n: ipfsNode, ctx: ctx}, nil
}

50
pkg/ipfs/dag_putters/btc_header.go
View File

@ -1,50 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package dag_putters
import (
"fmt"
"strings"
node "github.com/ipfs/go-ipld-format"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs/ipld"
)
var (
duplicateKeyErrorString = "pq: duplicate key value violates unique constraint"
)
type BtcHeaderDagPutter struct {
adder *ipfs.IPFS
}
func NewBtcHeaderDagPutter(adder *ipfs.IPFS) *BtcHeaderDagPutter {
return &BtcHeaderDagPutter{adder: adder}
}
func (bhdp *BtcHeaderDagPutter) DagPut(n node.Node) (string, error) {
header, ok := n.(*ipld.BtcHeader)
if !ok {
return "", fmt.Errorf("BtcHeaderDagPutter expected input type %T got %T", &ipld.BtcHeader{}, n)
}
if err := bhdp.adder.Add(header); err != nil && !strings.Contains(err.Error(), duplicateKeyErrorString) {
return "", err
}
return header.Cid().String(), nil
}

46
pkg/ipfs/dag_putters/btc_tx.go
View File

@ -1,46 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package dag_putters
import (
"fmt"
"strings"
node "github.com/ipfs/go-ipld-format"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs/ipld"
)
type BtcTxDagPutter struct {
adder *ipfs.IPFS
}
func NewBtcTxDagPutter(adder *ipfs.IPFS) *BtcTxDagPutter {
return &BtcTxDagPutter{adder: adder}
}
func (etdp *BtcTxDagPutter) DagPut(n node.Node) (string, error) {
transaction, ok := n.(*ipld.BtcTx)
if !ok {
return "", fmt.Errorf("BtcTxDagPutter expected input type %T got %T", &ipld.BtcTx{}, n)
}
if err := etdp.adder.Add(transaction); err != nil && !strings.Contains(err.Error(), duplicateKeyErrorString) {
return "", err
}
return transaction.Cid().String(), nil
}

46
pkg/ipfs/dag_putters/btc_tx_trie.go
View File

@ -1,46 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package dag_putters
import (
"fmt"
"strings"
node "github.com/ipfs/go-ipld-format"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs/ipld"
)
type BtcTxTrieDagPutter struct {
adder *ipfs.IPFS
}
func NewBtcTxTrieDagPutter(adder *ipfs.IPFS) *BtcTxTrieDagPutter {
return &BtcTxTrieDagPutter{adder: adder}
}
func (etdp *BtcTxTrieDagPutter) DagPut(n node.Node) (string, error) {
txTrieNode, ok := n.(*ipld.BtcTxTrie)
if !ok {
return "", fmt.Errorf("BtcTxTrieDagPutter expected input type %T got %T", &ipld.BtcTxTrie{}, n)
}
if err := etdp.adder.Add(txTrieNode); err != nil && !strings.Contains(err.Error(), duplicateKeyErrorString) {
return "", err
}
return txTrieNode.Cid().String(), nil
}

46
pkg/ipfs/dag_putters/eth_header.go
View File

@ -1,46 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package dag_putters
import (
"fmt"
"strings"
node "github.com/ipfs/go-ipld-format"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs/ipld"
)
type EthHeaderDagPutter struct {
adder *ipfs.IPFS
}
func NewEthBlockHeaderDagPutter(adder *ipfs.IPFS) *EthHeaderDagPutter {
return &EthHeaderDagPutter{adder: adder}
}
func (bhdp *EthHeaderDagPutter) DagPut(n node.Node) (string, error) {
header, ok := n.(*ipld.EthHeader)
if !ok {
return "", fmt.Errorf("EthHeaderDagPutter expected input type %T got %T", &ipld.EthHeader{}, n)
}
if err := bhdp.adder.Add(header); err != nil && !strings.Contains(err.Error(), duplicateKeyErrorString) {
return "", err
}
return header.Cid().String(), nil
}

46
pkg/ipfs/dag_putters/eth_receipt.go
View File

@ -1,46 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package dag_putters
import (
"fmt"
"strings"
node "github.com/ipfs/go-ipld-format"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs/ipld"
)
type EthReceiptDagPutter struct {
adder *ipfs.IPFS
}
func NewEthReceiptDagPutter(adder *ipfs.IPFS) *EthReceiptDagPutter {
return &EthReceiptDagPutter{adder: adder}
}
func (erdp *EthReceiptDagPutter) DagPut(n node.Node) (string, error) {
receipt, ok := n.(*ipld.EthReceipt)
if !ok {
return "", fmt.Errorf("EthReceiptDagPutter expected input type %T got type %T", &ipld.EthReceipt{}, n)
}
if err := erdp.adder.Add(receipt); err != nil && !strings.Contains(err.Error(), duplicateKeyErrorString) {
return "", err
}
return receipt.Cid().String(), nil
}

46
pkg/ipfs/dag_putters/eth_receipt_trie.go
View File

@ -1,46 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package dag_putters
import (
"fmt"
"strings"
node "github.com/ipfs/go-ipld-format"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs/ipld"
)
type EthRctTrieDagPutter struct {
adder *ipfs.IPFS
}
func NewEthRctTrieDagPutter(adder *ipfs.IPFS) *EthRctTrieDagPutter {
return &EthRctTrieDagPutter{adder: adder}
}
func (etdp *EthRctTrieDagPutter) DagPut(n node.Node) (string, error) {
rctTrieNode, ok := n.(*ipld.EthRctTrie)
if !ok {
return "", fmt.Errorf("EthRctTrieDagPutter expected input type %T got %T", &ipld.EthRctTrie{}, n)
}
if err := etdp.adder.Add(rctTrieNode); err != nil && !strings.Contains(err.Error(), duplicateKeyErrorString) {
return "", err
}
return rctTrieNode.Cid().String(), nil
}

46
pkg/ipfs/dag_putters/eth_state.go
View File

@ -1,46 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package dag_putters
import (
"fmt"
"strings"
node "github.com/ipfs/go-ipld-format"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs/ipld"
)
type EthStateDagPutter struct {
adder *ipfs.IPFS
}
func NewEthStateDagPutter(adder *ipfs.IPFS) *EthStateDagPutter {
return &EthStateDagPutter{adder: adder}
}
func (erdp *EthStateDagPutter) DagPut(n node.Node) (string, error) {
stateNode, ok := n.(*ipld.EthStateTrie)
if !ok {
return "", fmt.Errorf("EthStateDagPutter expected input type %T got %T", &ipld.EthStateTrie{}, n)
}
if err := erdp.adder.Add(stateNode); err != nil && !strings.Contains(err.Error(), duplicateKeyErrorString) {
return "", err
}
return stateNode.Cid().String(), nil
}

46
pkg/ipfs/dag_putters/eth_storage.go
View File

@ -1,46 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package dag_putters
import (
"fmt"
"strings"
node "github.com/ipfs/go-ipld-format"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs/ipld"
)
type EthStorageDagPutter struct {
adder *ipfs.IPFS
}
func NewEthStorageDagPutter(adder *ipfs.IPFS) *EthStorageDagPutter {
return &EthStorageDagPutter{adder: adder}
}
func (erdp *EthStorageDagPutter) DagPut(n node.Node) (string, error) {
storageNode, ok := n.(*ipld.EthStorageTrie)
if !ok {
return "", fmt.Errorf("EthStorageDagPutter expected input type %T got %T", &ipld.EthStorageTrie{}, n)
}
if err := erdp.adder.Add(storageNode); err != nil && !strings.Contains(err.Error(), duplicateKeyErrorString) {
return "", err
}
return storageNode.Cid().String(), nil
}

46
pkg/ipfs/dag_putters/eth_tx.go
View File

@ -1,46 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package dag_putters
import (
"fmt"
"strings"
node "github.com/ipfs/go-ipld-format"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs/ipld"
)
type EthTxsDagPutter struct {
adder *ipfs.IPFS
}
func NewEthTxsDagPutter(adder *ipfs.IPFS) *EthTxsDagPutter {
return &EthTxsDagPutter{adder: adder}
}
func (etdp *EthTxsDagPutter) DagPut(n node.Node) (string, error) {
transaction, ok := n.(*ipld.EthTx)
if !ok {
return "", fmt.Errorf("EthTxsDagPutter expected input type %T got %T", &ipld.EthTx{}, n)
}
if err := etdp.adder.Add(transaction); err != nil && !strings.Contains(err.Error(), duplicateKeyErrorString) {
return "", err
}
return transaction.Cid().String(), nil
}

46
pkg/ipfs/dag_putters/eth_tx_trie.go
View File

@ -1,46 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package dag_putters
import (
"fmt"
"strings"
node "github.com/ipfs/go-ipld-format"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/ipfs/ipld"
)
type EthTxTrieDagPutter struct {
adder *ipfs.IPFS
}
func NewEthTxTrieDagPutter(adder *ipfs.IPFS) *EthTxTrieDagPutter {
return &EthTxTrieDagPutter{adder: adder}
}
func (etdp *EthTxTrieDagPutter) DagPut(n node.Node) (string, error) {
txTrieNode, ok := n.(*ipld.EthTxTrie)
if !ok {
return "", fmt.Errorf("EthTxTrieDagPutter expected input type %T got %T", &ipld.EthTxTrie{}, n)
}
if err := etdp.adder.Add(txTrieNode); err != nil && !strings.Contains(err.Error(), duplicateKeyErrorString) {
return "", err
}
return txTrieNode.Cid().String(), nil
}

26
pkg/ipfs/interfaces.go
View File

@ -1,26 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipfs
import (
ipld "github.com/ipfs/go-ipld-format"
)
// DagPutter is a general interface for a dag putter
type DagPutter interface {
DagPut(n ipld.Node) (string, error)
}

86
pkg/ipfs/mocks/blockservice.go
View File

@ -1,86 +0,0 @@
package mocks
import (
"context"
"errors"
"github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-ipfs-blockstore"
"github.com/ipfs/go-ipfs-exchange-interface"
)
// MockIPFSBlockService is a mock for testing the ipfs fetcher
type MockIPFSBlockService struct {
Blocks map[cid.Cid]blocks.Block
}
// GetBlock is used to retrieve a block from the mock BlockService
func (bs *MockIPFSBlockService) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) {
if bs.Blocks == nil {
return nil, errors.New("BlockService has not been initialized")
}
blk, ok := bs.Blocks[c]
if ok {
return blk, nil
}
return nil, nil
}
// GetBlocks is used to retrieve a set of blocks from the mock BlockService
func (bs *MockIPFSBlockService) GetBlocks(ctx context.Context, cs []cid.Cid) <-chan blocks.Block {
if bs.Blocks == nil {
panic("BlockService has not been initialized")
}
blkChan := make(chan blocks.Block)
go func() {
for _, c := range cs {
blk, ok := bs.Blocks[c]
if ok {
blkChan <- blk
}
}
close(blkChan)
}()
return blkChan
}
// AddBlock adds a block to the mock BlockService
func (bs *MockIPFSBlockService) AddBlock(blk blocks.Block) error {
if bs.Blocks == nil {
bs.Blocks = make(map[cid.Cid]blocks.Block)
}
bs.Blocks[blk.Cid()] = blk
return nil
}
// AddBlocks adds a set of blocks to the mock BlockService
func (bs *MockIPFSBlockService) AddBlocks(blks []blocks.Block) error {
if bs.Blocks == nil {
bs.Blocks = make(map[cid.Cid]blocks.Block)
}
for _, block := range blks {
bs.Blocks[block.Cid()] = block
}
return nil
}
// Close is here to satisfy the interface
func (*MockIPFSBlockService) Close() error {
panic("implement me")
}
// Blockstore is here to satisfy the interface
func (*MockIPFSBlockService) Blockstore() blockstore.Blockstore {
panic("implement me")
}
// DeleteBlock is here to satisfy the interface
func (*MockIPFSBlockService) DeleteBlock(c cid.Cid) error {
panic("implement me")
}
// Exchange is here to satisfy the interface
func (*MockIPFSBlockService) Exchange() exchange.Interface {
panic("implement me")
}

53
pkg/ipfs/mocks/dag_putters.go
View File

@ -1,53 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package mocks
import (
"errors"
node "github.com/ipfs/go-ipld-format"
"github.com/ethereum/go-ethereum/common"
)
// DagPutter is a mock for testing the ipfs publisher
type DagPutter struct {
PassedNode node.Node
ErrToReturn error
}
// DagPut returns the pre-loaded CIDs or error
func (dp *DagPutter) DagPut(n node.Node) (string, error) {
dp.PassedNode = n
return n.Cid().String(), dp.ErrToReturn
}
// MappedDagPutter is a mock for testing the ipfs publisher
type MappedDagPutter struct {
CIDsToReturn map[common.Hash]string
PassedNode node.Node
ErrToReturn error
}
// DagPut returns the pre-loaded CIDs or error
func (mdp *MappedDagPutter) DagPut(n node.Node) (string, error) {
if mdp.CIDsToReturn == nil {
return "", errors.New("mapped dag putter needs to be initialized with a map of cids to return")
}
hash := common.BytesToHash(n.RawData())
return mdp.CIDsToReturn[hash], nil
}

1
pkg/node/node.go
View File

@ -19,6 +19,7 @@ package node
type Node struct { type Node struct {
GenesisBlock string GenesisBlock string
NetworkID string NetworkID string
ChainID uint64
ID string ID string
ClientName string ClientName string
} }

12
pkg/resync/config.go
View File

@ -51,8 +51,6 @@ type Config struct {
// DB info // DB info
DB *postgres.DB DB *postgres.DB
DBConfig config.Database DBConfig config.Database
IPFSPath string
IPFSMode shared.IPFSMode
HTTPClient interface{} // Note this client is expected to support the retrieval of the specified data type(s) HTTPClient interface{} // Note this client is expected to support the retrieval of the specified data type(s)
NodeInfo node.Node // Info for the associated node NodeInfo node.Node // Info for the associated node
@ -91,16 +89,6 @@ func NewConfig() (*Config, error) {
c.ClearOldCache = viper.GetBool("resync.clearOldCache") c.ClearOldCache = viper.GetBool("resync.clearOldCache")
c.ResetValidation = viper.GetBool("resync.resetValidation") c.ResetValidation = viper.GetBool("resync.resetValidation")
c.IPFSMode, err = shared.GetIPFSMode()
if err != nil {
return nil, err
}
if c.IPFSMode == shared.LocalInterface || c.IPFSMode == shared.RemoteClient {
c.IPFSPath, err = shared.GetIPFSPath()
if err != nil {
return nil, err
}
}
resyncType := viper.GetString("resync.type") resyncType := viper.GetString("resync.type")
c.ResyncType, err = shared.GenerateDataTypeFromString(resyncType) c.ResyncType, err = shared.GenerateDataTypeFromString(resyncType)
if err != nil { if err != nil {

17
pkg/resync/service.go
View File

@ -35,8 +35,6 @@ type Service struct {
Converter shared.PayloadConverter Converter shared.PayloadConverter
// Interface for publishing the IPLD payloads to IPFS // Interface for publishing the IPLD payloads to IPFS
Publisher shared.IPLDPublisher Publisher shared.IPLDPublisher
// Interface for indexing the CIDs of the published IPLDs in Postgres
Indexer shared.CIDIndexer
// Interface for searching and retrieving CIDs from Postgres index // Interface for searching and retrieving CIDs from Postgres index
Retriever shared.CIDRetriever Retriever shared.CIDRetriever
// Interface for fetching payloads over at historical blocks; over http // Interface for fetching payloads over at historical blocks; over http
@ -63,15 +61,11 @@ type Service struct {
// NewResyncService creates and returns a resync service from the provided settings // NewResyncService creates and returns a resync service from the provided settings
func NewResyncService(settings *Config) (Resync, error) { func NewResyncService(settings *Config) (Resync, error) {
publisher, err := builders.NewIPLDPublisher(settings.Chain, settings.IPFSPath, settings.DB, settings.IPFSMode) publisher, err := builders.NewIPLDPublisher(settings.Chain, settings.DB)
if err != nil { if err != nil {
return nil, err return nil, err
} }
indexer, err := builders.NewCIDIndexer(settings.Chain, settings.DB, settings.IPFSMode) converter, err := builders.NewPayloadConverter(settings.Chain, settings.NodeInfo.ChainID)
if err != nil {
return nil, err
}
converter, err := builders.NewPayloadConverter(settings.Chain)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -96,7 +90,6 @@ func NewResyncService(settings *Config) (Resync, error) {
batchNumber = shared.DefaultMaxBatchNumber batchNumber = shared.DefaultMaxBatchNumber
} }
return &Service{ return &Service{
Indexer: indexer,
Converter: converter, Converter: converter,
Publisher: publisher, Publisher: publisher,
Retriever: retriever, Retriever: retriever,
@ -168,13 +161,9 @@ func (rs *Service) resync(id int, heightChan chan []uint64) {
if err != nil { if err != nil {
logrus.Errorf("%s resync worker %d converter error: %s", rs.chain.String(), id, err.Error()) logrus.Errorf("%s resync worker %d converter error: %s", rs.chain.String(), id, err.Error())
} }
cidPayload, err := rs.Publisher.Publish(ipldPayload) if err := rs.Publisher.Publish(ipldPayload); err != nil {
if err != nil {
logrus.Errorf("%s resync worker %d publisher error: %s", rs.chain.String(), id, err.Error()) logrus.Errorf("%s resync worker %d publisher error: %s", rs.chain.String(), id, err.Error())
} }
if err := rs.Indexer.Index(cidPayload); err != nil {
logrus.Errorf("%s resync worker %d indexer error: %s", rs.chain.String(), id, err.Error())
}
} }
logrus.Infof("%s resync worker %d finished section from %d to %d", rs.chain.String(), id, heights[0], heights[len(heights)-1]) logrus.Infof("%s resync worker %d finished section from %d to %d", rs.chain.String(), id, heights[0], heights[len(heights)-1])
case <-rs.quitChan: case <-rs.quitChan:

7
pkg/shared/chain_type.go
View File

@ -29,6 +29,7 @@ const (
Ethereum Ethereum
Bitcoin Bitcoin
Omni Omni
EthereumClassic
) )
func (c ChainType) String() string { func (c ChainType) String() string {
@ -39,6 +40,8 @@ func (c ChainType) String() string {
return "Bitcoin" return "Bitcoin"
case Omni: case Omni:
return "Omni" return "Omni"
case EthereumClassic:
return "EthereumClassic"
default: default:
return "" return ""
} }
@ -52,6 +55,8 @@ func (c ChainType) API() string {
return "btc" return "btc"
case Omni: case Omni:
return "omni" return "omni"
case EthereumClassic:
return "etc"
default: default:
return "" return ""
} }
@ -65,6 +70,8 @@ func NewChainType(name string) (ChainType, error) {
return Bitcoin, nil return Bitcoin, nil
case "omni": case "omni":
return Omni, nil return Omni, nil
case "classic", "etc":
return EthereumClassic, nil
default: default:
return UnknownChain, errors.New("invalid name for chain") return UnknownChain, errors.New("invalid name for chain")
} }

35
pkg/shared/env.go
View File

@ -17,9 +17,6 @@
package shared package shared
import ( import (
"os"
"path/filepath"
"github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/rpc"
"github.com/btcsuite/btcd/rpcclient" "github.com/btcsuite/btcd/rpcclient"
@ -29,8 +26,6 @@ import (
// Env variables // Env variables
const ( const (
IPFS_PATH = "IPFS_PATH"
IPFS_MODE = "IPFS_MODE"
HTTP_TIMEOUT = "HTTP_TIMEOUT" HTTP_TIMEOUT = "HTTP_TIMEOUT"
ETH_WS_PATH = "ETH_WS_PATH" ETH_WS_PATH = "ETH_WS_PATH"
@ -39,6 +34,7 @@ const (
ETH_CLIENT_NAME = "ETH_CLIENT_NAME" ETH_CLIENT_NAME = "ETH_CLIENT_NAME"
ETH_GENESIS_BLOCK = "ETH_GENESIS_BLOCK" ETH_GENESIS_BLOCK = "ETH_GENESIS_BLOCK"
ETH_NETWORK_ID = "ETH_NETWORK_ID" ETH_NETWORK_ID = "ETH_NETWORK_ID"
ETH_CHAIN_ID = "ETH_CHAIN_ID"
BTC_WS_PATH = "BTC_WS_PATH" BTC_WS_PATH = "BTC_WS_PATH"
BTC_HTTP_PATH = "BTC_HTTP_PATH" BTC_HTTP_PATH = "BTC_HTTP_PATH"
@ -48,6 +44,7 @@ const (
BTC_CLIENT_NAME = "BTC_CLIENT_NAME" BTC_CLIENT_NAME = "BTC_CLIENT_NAME"
BTC_GENESIS_BLOCK = "BTC_GENESIS_BLOCK" BTC_GENESIS_BLOCK = "BTC_GENESIS_BLOCK"
BTC_NETWORK_ID = "BTC_NETWORK_ID" BTC_NETWORK_ID = "BTC_NETWORK_ID"
BTC_CHAIN_ID = "BTC_CHAIN_ID"
) )
// GetEthNodeAndClient returns eth node info and client from path url // GetEthNodeAndClient returns eth node info and client from path url
@ -56,6 +53,7 @@ func GetEthNodeAndClient(path string) (node.Node, *rpc.Client, error) {
viper.BindEnv("ethereum.clientName", ETH_CLIENT_NAME) viper.BindEnv("ethereum.clientName", ETH_CLIENT_NAME)
viper.BindEnv("ethereum.genesisBlock", ETH_GENESIS_BLOCK) viper.BindEnv("ethereum.genesisBlock", ETH_GENESIS_BLOCK)
viper.BindEnv("ethereum.networkID", ETH_NETWORK_ID) viper.BindEnv("ethereum.networkID", ETH_NETWORK_ID)
viper.BindEnv("ethereum.chainID", ETH_CHAIN_ID)
rpcClient, err := rpc.Dial(path) rpcClient, err := rpc.Dial(path)
if err != nil { if err != nil {
@ -66,33 +64,10 @@ func GetEthNodeAndClient(path string) (node.Node, *rpc.Client, error) {
ClientName: viper.GetString("ethereum.clientName"), ClientName: viper.GetString("ethereum.clientName"),
GenesisBlock: viper.GetString("ethereum.genesisBlock"), GenesisBlock: viper.GetString("ethereum.genesisBlock"),
NetworkID: viper.GetString("ethereum.networkID"), NetworkID: viper.GetString("ethereum.networkID"),
ChainID: viper.GetUint64("ethereum.chainID"),
}, rpcClient, nil }, rpcClient, nil
} }
// GetIPFSPath returns the ipfs path from the config or env variable
func GetIPFSPath() (string, error) {
viper.BindEnv("ipfs.path", IPFS_PATH)
ipfsPath := viper.GetString("ipfs.path")
if ipfsPath == "" {
home, err := os.UserHomeDir()
if err != nil {
return "", err
}
ipfsPath = filepath.Join(home, ".ipfs")
}
return ipfsPath, nil
}
// GetIPFSMode returns the ipfs mode of operation from the config or env variable
func GetIPFSMode() (IPFSMode, error) {
viper.BindEnv("ipfs.mode", IPFS_MODE)
ipfsMode := viper.GetString("ipfs.mode")
if ipfsMode == "" {
return DirectPostgres, nil
}
return NewIPFSMode(ipfsMode)
}
// GetBtcNodeAndClient returns btc node info from path url // GetBtcNodeAndClient returns btc node info from path url
func GetBtcNodeAndClient(path string) (node.Node, *rpcclient.ConnConfig) { func GetBtcNodeAndClient(path string) (node.Node, *rpcclient.ConnConfig) {
viper.BindEnv("bitcoin.nodeID", BTC_NODE_ID) viper.BindEnv("bitcoin.nodeID", BTC_NODE_ID)
@ -101,6 +76,7 @@ func GetBtcNodeAndClient(path string) (node.Node, *rpcclient.ConnConfig) {
viper.BindEnv("bitcoin.networkID", BTC_NETWORK_ID) viper.BindEnv("bitcoin.networkID", BTC_NETWORK_ID)
viper.BindEnv("bitcoin.pass", BTC_NODE_PASSWORD) viper.BindEnv("bitcoin.pass", BTC_NODE_PASSWORD)
viper.BindEnv("bitcoin.user", BTC_NODE_USER) viper.BindEnv("bitcoin.user", BTC_NODE_USER)
viper.BindEnv("bitcoin.chainID", BTC_CHAIN_ID)
// For bitcoin we load in node info from the config because there is no RPC endpoint to retrieve this from the node // For bitcoin we load in node info from the config because there is no RPC endpoint to retrieve this from the node
return node.Node{ return node.Node{
@ -108,6 +84,7 @@ func GetBtcNodeAndClient(path string) (node.Node, *rpcclient.ConnConfig) {
ClientName: viper.GetString("bitcoin.clientName"), ClientName: viper.GetString("bitcoin.clientName"),
GenesisBlock: viper.GetString("bitcoin.genesisBlock"), GenesisBlock: viper.GetString("bitcoin.genesisBlock"),
NetworkID: viper.GetString("bitcoin.networkID"), NetworkID: viper.GetString("bitcoin.networkID"),
ChainID: viper.GetUint64("bitcoin.chainID"),
}, &rpcclient.ConnConfig{ }, &rpcclient.ConnConfig{
Host: path, Host: path,
HTTPPostMode: true, // Bitcoin core only supports HTTP POST mode HTTPPostMode: true, // Bitcoin core only supports HTTP POST mode

7
pkg/shared/intefaces.go
View File

@ -37,12 +37,7 @@ type PayloadConverter interface {
// IPLDPublisher publishes IPLD payloads and returns a CID payload for indexing // IPLDPublisher publishes IPLD payloads and returns a CID payload for indexing
type IPLDPublisher interface { type IPLDPublisher interface {
Publish(payload ConvertedData) (CIDsForIndexing, error) Publish(payload ConvertedData) error
}
// CIDIndexer indexes a CID payload in Postgres
type CIDIndexer interface {
Index(cids CIDsForIndexing) error
} }
// ResponseFilterer applies a filter to an IPLD payload to return a subscription response packet // ResponseFilterer applies a filter to an IPLD payload to return a subscription response packet

58
pkg/shared/ipfs_mode.go
View File

@ -1,58 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package shared
import (
"errors"
"strings"
)
// IPFSMode enum for specifying how we want to interface and publish objects to IPFS
type IPFSMode int
const (
Unknown IPFSMode = iota
LocalInterface
RemoteClient
DirectPostgres
)
func (c IPFSMode) String() string {
switch c {
case LocalInterface:
return "Local"
case RemoteClient:
return "Remote"
case DirectPostgres:
return "Postgres"
default:
return ""
}
}
func NewIPFSMode(name string) (IPFSMode, error) {
switch strings.ToLower(name) {
case "local", "interface":
return LocalInterface, nil
case "remote", "client":
return RemoteClient, errors.New("remote IPFS client mode is not currently supported")
case "postgres", "direct":
return DirectPostgres, nil
default:
return Unknown, errors.New("unrecognized name for ipfs mode")
}
}

3
pkg/shared/types.go
View File

@ -16,6 +16,9 @@
package shared package shared
// Very loose interface types for generic processing of different blockchains
// TODO: split different blockchain support into separate repos
// These types serve as very loose wrappers around a generic underlying interface{} // These types serve as very loose wrappers around a generic underlying interface{}
type RawChainData interface{} type RawChainData interface{}

1
pkg/validate/service.go
View File

@ -1 +0,0 @@
package validate

14
pkg/watch/config.go
View File

@ -53,8 +53,6 @@ const (
// Config struct // Config struct
type Config struct { type Config struct {
Chain shared.ChainType Chain shared.ChainType
IPFSPath string
IPFSMode shared.IPFSMode
DBConfig config.Database DBConfig config.Database
// Server fields // Server fields
Serve bool Serve bool
@ -96,19 +94,7 @@ func NewConfig() (*Config, error) {
return nil, err return nil, err
} }
c.IPFSMode, err = shared.GetIPFSMode()
if err != nil {
return nil, err
}
if c.IPFSMode == shared.LocalInterface || c.IPFSMode == shared.RemoteClient {
c.IPFSPath, err = shared.GetIPFSPath()
if err != nil {
return nil, err
}
}
c.DBConfig.Init() c.DBConfig.Init()
c.Sync = viper.GetBool("watcher.sync") c.Sync = viper.GetBool("watcher.sync")
if c.Sync { if c.Sync {
workers := viper.GetInt("watcher.workers") workers := viper.GetInt("watcher.workers")

66
pkg/watch/service.go
View File

@ -66,10 +66,8 @@ type Service struct {
Streamer shared.PayloadStreamer Streamer shared.PayloadStreamer
// Interface for converting raw payloads into IPLD object payloads // Interface for converting raw payloads into IPLD object payloads
Converter shared.PayloadConverter Converter shared.PayloadConverter
// Interface for publishing the IPLD payloads to IPFS // Interface for publishing and indexing the PG-IPLD payloads
Publisher shared.IPLDPublisher Publisher shared.IPLDPublisher
// Interface for indexing the CIDs of the published IPLDs in Postgres
Indexer shared.CIDIndexer
// Interface for filtering and serving data according to subscribed clients according to their specification // Interface for filtering and serving data according to subscribed clients according to their specification
Filterer shared.ResponseFilterer Filterer shared.ResponseFilterer
// Interface for fetching IPLD objects from IPFS // Interface for fetching IPLD objects from IPFS
@ -86,12 +84,10 @@ type Service struct {
SubscriptionTypes map[common.Hash]shared.SubscriptionSettings SubscriptionTypes map[common.Hash]shared.SubscriptionSettings
// Info for the Geth node that this watcher is working with // Info for the Geth node that this watcher is working with
NodeInfo *node.Node NodeInfo *node.Node
// Number of publishAndIndex workers // Number of publish workers
WorkerPoolSize int WorkerPoolSize int
// chain type for this service // chain type for this service
chain shared.ChainType chain shared.ChainType
// Path to ipfs data dir
ipfsPath string
// Underlying db // Underlying db
db *postgres.DB db *postgres.DB
// wg for syncing serve processes // wg for syncing serve processes
@ -108,15 +104,11 @@ func NewWatcher(settings *Config) (Watcher, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
sn.Converter, err = builders.NewPayloadConverter(settings.Chain) sn.Converter, err = builders.NewPayloadConverter(settings.Chain, settings.NodeInfo.ChainID)
if err != nil { if err != nil {
return nil, err return nil, err
} }
sn.Publisher, err = builders.NewIPLDPublisher(settings.Chain, settings.IPFSPath, settings.SyncDBConn, settings.IPFSMode) sn.Publisher, err = builders.NewIPLDPublisher(settings.Chain, settings.SyncDBConn)
if err != nil {
return nil, err
}
sn.Indexer, err = builders.NewCIDIndexer(settings.Chain, settings.SyncDBConn, settings.IPFSMode)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -131,7 +123,7 @@ func NewWatcher(settings *Config) (Watcher, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
sn.IPLDFetcher, err = builders.NewIPLDFetcher(settings.Chain, settings.IPFSPath, settings.ServeDBConn, settings.IPFSMode) sn.IPLDFetcher, err = builders.NewIPLDFetcher(settings.Chain, settings.ServeDBConn)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -142,7 +134,6 @@ func NewWatcher(settings *Config) (Watcher, error) {
sn.SubscriptionTypes = make(map[common.Hash]shared.SubscriptionSettings) sn.SubscriptionTypes = make(map[common.Hash]shared.SubscriptionSettings)
sn.WorkerPoolSize = settings.Workers sn.WorkerPoolSize = settings.Workers
sn.NodeInfo = &settings.NodeInfo sn.NodeInfo = &settings.NodeInfo
sn.ipfsPath = settings.IPFSPath
sn.chain = settings.Chain sn.chain = settings.Chain
return sn, nil return sn, nil
} }
@ -154,7 +145,7 @@ func (sap *Service) Protocols() []p2p.Protocol {
// APIs returns the RPC descriptors the watcher service offers // APIs returns the RPC descriptors the watcher service offers
func (sap *Service) APIs() []rpc.API { func (sap *Service) APIs() []rpc.API {
ifnoAPI := NewInfoAPI() infoAPI := NewInfoAPI()
apis := []rpc.API{ apis := []rpc.API{
{ {
Namespace: APIName, Namespace: APIName,
@ -165,23 +156,23 @@ func (sap *Service) APIs() []rpc.API {
{ {
Namespace: "rpc", Namespace: "rpc",
Version: APIVersion, Version: APIVersion,
Service: ifnoAPI, Service: infoAPI,
Public: true, Public: true,
}, },
{ {
Namespace: "net", Namespace: "net",
Version: APIVersion, Version: APIVersion,
Service: ifnoAPI, Service: infoAPI,
Public: true, Public: true,
}, },
{ {
Namespace: "admin", Namespace: "admin",
Version: APIVersion, Version: APIVersion,
Service: ifnoAPI, Service: infoAPI,
Public: true, Public: true,
}, },
} }
chainAPI, err := builders.NewPublicAPI(sap.chain, sap.db, sap.ipfsPath) chainAPI, err := builders.NewPublicAPI(sap.chain, sap.db)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
return apis return apis
@ -190,7 +181,7 @@ func (sap *Service) APIs() []rpc.API {
} }
// Sync streams incoming raw chain data and converts it for further processing // Sync streams incoming raw chain data and converts it for further processing
// It forwards the converted data to the publishAndIndex process(es) it spins up // It forwards the converted data to the publish process(es) it spins up
// If forwards the converted data to a ScreenAndServe process if it there is one listening on the passed screenAndServePayload channel // If forwards the converted data to a ScreenAndServe process if it there is one listening on the passed screenAndServePayload channel
// This continues on no matter if or how many subscribers there are // This continues on no matter if or how many subscribers there are
func (sap *Service) Sync(wg *sync.WaitGroup, screenAndServePayload chan<- shared.ConvertedData) error { func (sap *Service) Sync(wg *sync.WaitGroup, screenAndServePayload chan<- shared.ConvertedData) error {
@ -198,11 +189,11 @@ func (sap *Service) Sync(wg *sync.WaitGroup, screenAndServePayload chan<- shared
if err != nil { if err != nil {
return err return err
} }
// spin up publishAndIndex worker goroutines // spin up publish worker goroutines
publishAndIndexPayload := make(chan shared.ConvertedData, PayloadChanBufferSize) publishPayload := make(chan shared.ConvertedData, PayloadChanBufferSize)
for i := 1; i <= sap.WorkerPoolSize; i++ { for i := 1; i <= sap.WorkerPoolSize; i++ {
go sap.publishAndIndex(wg, i, publishAndIndexPayload) go sap.publish(wg, i, publishPayload)
log.Debugf("%s publishAndIndex worker %d successfully spun up", sap.chain.String(), i) log.Debugf("%s publish worker %d successfully spun up", sap.chain.String(), i)
} }
go func() { go func() {
wg.Add(1) wg.Add(1)
@ -221,13 +212,13 @@ func (sap *Service) Sync(wg *sync.WaitGroup, screenAndServePayload chan<- shared
case screenAndServePayload <- ipldPayload: case screenAndServePayload <- ipldPayload:
default: default:
} }
// Forward the payload to the publishAndIndex workers // Forward the payload to the publish workers
// this channel acts as a ring buffer // this channel acts as a ring buffer
select { select {
case publishAndIndexPayload <- ipldPayload: case publishPayload <- ipldPayload:
default: default:
<-publishAndIndexPayload <-publishPayload
publishAndIndexPayload <- ipldPayload publishPayload <- ipldPayload
} }
case err := <-sub.Err(): case err := <-sub.Err():
log.Errorf("watcher subscription error for chain %s: %v", sap.chain.String(), err) log.Errorf("watcher subscription error for chain %s: %v", sap.chain.String(), err)
@ -241,26 +232,21 @@ func (sap *Service) Sync(wg *sync.WaitGroup, screenAndServePayload chan<- shared
return nil return nil
} }
// publishAndIndex is spun up by SyncAndConvert and receives converted chain data from that process // publish is spun up by SyncAndConvert and receives converted chain data from that process
// it publishes this data to IPFS and indexes their CIDs with useful metadata in Postgres // it publishes this data to IPFS and indexes their CIDs with useful metadata in Postgres
func (sap *Service) publishAndIndex(wg *sync.WaitGroup, id int, publishAndIndexPayload <-chan shared.ConvertedData) { func (sap *Service) publish(wg *sync.WaitGroup, id int, publishPayload <-chan shared.ConvertedData) {
wg.Add(1) wg.Add(1)
defer wg.Done() defer wg.Done()
for { for {
select { select {
case payload := <-publishAndIndexPayload: case payload := <-publishPayload:
log.Debugf("%s watcher publishAndIndex worker %d publishing data streamed at head height %d", sap.chain.String(), id, payload.Height()) log.Debugf("%s watcher sync worker %d publishing and indexing data streamed at head height %d", sap.chain.String(), id, payload.Height())
cidPayload, err := sap.Publisher.Publish(payload) if err := sap.Publisher.Publish(payload); err != nil {
if err != nil { log.Errorf("%s watcher publish worker %d publishing error: %v", sap.chain.String(), id, err)
log.Errorf("%s watcher publishAndIndex worker %d publishing error: %v", sap.chain.String(), id, err)
continue continue
} }
log.Debugf("%s watcher publishAndIndex worker %d indexing data streamed at head height %d", sap.chain.String(), id, payload.Height())
if err := sap.Indexer.Index(cidPayload); err != nil {
log.Errorf("%s watcher publishAndIndex worker %d indexing error: %v", sap.chain.String(), id, err)
}
case <-sap.QuitChan: case <-sap.QuitChan:
log.Infof("%s watcher publishAndIndex worker %d shutting down", sap.chain.String(), id) log.Infof("%s watcher publish worker %d shutting down", sap.chain.String(), id)
return return
} }
} }

6
pkg/watch/service_test.go
View File

@ -36,9 +36,6 @@ var _ = Describe("Service", func() {
wg := new(sync.WaitGroup) wg := new(sync.WaitGroup)
payloadChan := make(chan shared.RawChainData, 1) payloadChan := make(chan shared.RawChainData, 1)
quitChan := make(chan bool, 1) quitChan := make(chan bool, 1)
mockCidIndexer := &mocks.CIDIndexer{
ReturnErr: nil,
}
mockPublisher := &mocks.IPLDPublisher{ mockPublisher := &mocks.IPLDPublisher{
ReturnCIDPayload: mocks.MockCIDPayload, ReturnCIDPayload: mocks.MockCIDPayload,
ReturnErr: nil, ReturnErr: nil,
@ -55,7 +52,6 @@ var _ = Describe("Service", func() {
ReturnErr: nil, ReturnErr: nil,
} }
processor := &watch.Service{ processor := &watch.Service{
Indexer: mockCidIndexer,
Publisher: mockPublisher, Publisher: mockPublisher,
Streamer: mockStreamer, Streamer: mockStreamer,
Converter: mockConverter, Converter: mockConverter,
@ -69,8 +65,6 @@ var _ = Describe("Service", func() {
close(quitChan) close(quitChan)
wg.Wait() wg.Wait()
Expect(mockConverter.PassedStatediffPayload).To(Equal(mocks.MockStateDiffPayload)) Expect(mockConverter.PassedStatediffPayload).To(Equal(mocks.MockStateDiffPayload))
Expect(len(mockCidIndexer.PassedCIDPayload)).To(Equal(1))
Expect(mockCidIndexer.PassedCIDPayload[0]).To(Equal(mocks.MockCIDPayload))
Expect(mockPublisher.PassedIPLDPayload).To(Equal(mocks.MockConvertedPayload)) Expect(mockPublisher.PassedIPLDPayload).To(Equal(mocks.MockConvertedPayload))
Expect(mockStreamer.PassedPayloadChan).To(Equal(payloadChan)) Expect(mockStreamer.PassedPayloadChan).To(Equal(payloadChan))
}) })