combined/directp publish and indexing for btc; tests
This commit is contained in:
parent
65da8cafe7
commit
059664ffc9
1
go.mod
1
go.mod
@ -32,6 +32,7 @@ require (
|
||||
github.com/ipfs/go-ipfs-blockstore v0.0.1
|
||||
github.com/ipfs/go-ipfs-cmds v0.1.1 // indirect
|
||||
github.com/ipfs/go-ipfs-config v0.0.3 // indirect
|
||||
github.com/ipfs/go-ipfs-ds-help v0.0.1
|
||||
github.com/ipfs/go-ipfs-exchange-interface v0.0.1
|
||||
github.com/ipfs/go-ipld-cbor v0.0.3 // indirect
|
||||
github.com/ipfs/go-ipld-format v0.0.2
|
||||
|
@ -69,7 +69,7 @@ type BackFillService struct {
|
||||
|
||||
// NewBackFillService returns a new BackFillInterface
|
||||
func NewBackFillService(settings *Config, screenAndServeChan chan shared.ConvertedData) (BackFillInterface, error) {
|
||||
publisher, err := NewIPLDPublisher(settings.Chain, settings.IPFSPath, settings.IPFSMode)
|
||||
publisher, err := NewIPLDPublisher(settings.Chain, settings.IPFSPath, settings.DB, settings.IPFSMode)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
@ -47,7 +47,7 @@ func (in *CIDIndexer) Index(cids shared.CIDsForIndexing) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
headerID, err := in.indexHeaderCID(tx, cidWrapper.HeaderCID, in.db.NodeID)
|
||||
headerID, err := in.indexHeaderCID(tx, cidWrapper.HeaderCID)
|
||||
if err != nil {
|
||||
logrus.Error("btc indexer error when indexing header")
|
||||
return err
|
||||
@ -59,13 +59,13 @@ func (in *CIDIndexer) Index(cids shared.CIDsForIndexing) error {
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func (in *CIDIndexer) indexHeaderCID(tx *sqlx.Tx, header HeaderModel, nodeID int64) (int64, error) {
|
||||
func (in *CIDIndexer) indexHeaderCID(tx *sqlx.Tx, header HeaderModel) (int64, error) {
|
||||
var headerID int64
|
||||
err := tx.QueryRowx(`INSERT INTO btc.header_cids (block_number, block_hash, parent_hash, cid, timestamp, bits, node_id, times_validated)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
||||
ON CONFLICT (block_number, block_hash) DO UPDATE SET (parent_hash, cid, timestamp, bits, node_id, times_validated) = ($3, $4, $5, $6, $7, btc.header_cids.times_validated + 1)
|
||||
RETURNING id`,
|
||||
header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, header.Timestamp, header.Bits, nodeID, 1).Scan(&headerID)
|
||||
header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, header.Timestamp, header.Bits, in.db.NodeID, 1).Scan(&headerID)
|
||||
return headerID, err
|
||||
}
|
||||
|
||||
|
121
pkg/super_node/btc/publishAndIndexer.go
Normal file
121
pkg/super_node/btc/publishAndIndexer.go
Normal file
@ -0,0 +1,121 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2019 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package btc
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/postgres"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
|
||||
)
|
||||
|
||||
// IPLDPublisherAndIndexer satisfies the IPLDPublisher interface for bitcoin
|
||||
// It interfaces directly with the public.blocks table of PG-IPFS rather than going through an ipfs intermediary
|
||||
// It publishes and indexes IPLDs together in a single sqlx.Tx
|
||||
type IPLDPublisherAndIndexer struct {
|
||||
indexer *CIDIndexer
|
||||
}
|
||||
|
||||
// NewIPLDPublisherAndIndexer creates a pointer to a new IPLDPublisherAndIndexer which satisfies the IPLDPublisher interface
|
||||
func NewIPLDPublisherAndIndexer(db *postgres.DB) *IPLDPublisherAndIndexer {
|
||||
return &IPLDPublisherAndIndexer{
|
||||
indexer: NewCIDIndexer(db),
|
||||
}
|
||||
}
|
||||
|
||||
// Publish publishes an IPLDPayload to IPFS and returns the corresponding CIDPayload
|
||||
func (pub *IPLDPublisherAndIndexer) Publish(payload shared.ConvertedData) (shared.CIDsForIndexing, error) {
|
||||
ipldPayload, ok := payload.(ConvertedPayload)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("btc publisher expected payload type %T got %T", ConvertedPayload{}, payload)
|
||||
}
|
||||
// Generate the iplds
|
||||
headerNode, txNodes, txTrieNodes, err := ipld.FromHeaderAndTxs(ipldPayload.Header, ipldPayload.Txs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Begin new db tx
|
||||
tx, err := pub.indexer.db.Beginx()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Publish trie nodes
|
||||
for _, node := range txTrieNodes {
|
||||
if err := shared.PublishIPLD(tx, node); err != nil {
|
||||
shared.Rollback(tx)
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// Publish and index header
|
||||
if err := shared.PublishIPLD(tx, headerNode); err != nil {
|
||||
shared.Rollback(tx)
|
||||
return nil, err
|
||||
}
|
||||
header := HeaderModel{
|
||||
CID: headerNode.Cid().String(),
|
||||
ParentHash: ipldPayload.Header.PrevBlock.String(),
|
||||
BlockNumber: strconv.Itoa(int(ipldPayload.BlockPayload.BlockHeight)),
|
||||
BlockHash: ipldPayload.Header.BlockHash().String(),
|
||||
Timestamp: ipldPayload.Header.Timestamp.UnixNano(),
|
||||
Bits: ipldPayload.Header.Bits,
|
||||
}
|
||||
headerID, err := pub.indexer.indexHeaderCID(tx, header)
|
||||
if err != nil {
|
||||
shared.Rollback(tx)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Publish and index txs
|
||||
for i, txNode := range txNodes {
|
||||
if err := shared.PublishIPLD(tx, txNode); err != nil {
|
||||
shared.Rollback(tx)
|
||||
return nil, err
|
||||
}
|
||||
txModel := ipldPayload.TxMetaData[i]
|
||||
txModel.CID = txNode.Cid().String()
|
||||
txID, err := pub.indexer.indexTransactionCID(tx, txModel, headerID)
|
||||
if err != nil {
|
||||
shared.Rollback(tx)
|
||||
return nil, err
|
||||
}
|
||||
for _, input := range txModel.TxInputs {
|
||||
if err := pub.indexer.indexTxInput(tx, input, txID); err != nil {
|
||||
shared.Rollback(tx)
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
for _, output := range txModel.TxOutputs {
|
||||
if err := pub.indexer.indexTxOutput(tx, output, txID); err != nil {
|
||||
shared.Rollback(tx)
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// This IPLDPublisher does both publishing and indexing, we do not need to pass anything forward to the indexer
|
||||
return nil, tx.Commit()
|
||||
}
|
||||
|
||||
// Index satisfies the shared.CIDIndexer interface
|
||||
func (pub *IPLDPublisherAndIndexer) Index(cids shared.CIDsForIndexing) error {
|
||||
return nil
|
||||
}
|
121
pkg/super_node/btc/publishAndIndexer_test.go
Normal file
121
pkg/super_node/btc/publishAndIndexer_test.go
Normal file
@ -0,0 +1,121 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2019 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package btc_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
|
||||
"github.com/ipfs/go-cid"
|
||||
"github.com/ipfs/go-ipfs-blockstore"
|
||||
"github.com/ipfs/go-ipfs-ds-help"
|
||||
"github.com/multiformats/go-multihash"
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/pkg/postgres"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/super_node/btc"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/super_node/btc/mocks"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
|
||||
)
|
||||
|
||||
var _ = Describe("PublishAndIndexer", func() {
|
||||
var (
|
||||
db *postgres.DB
|
||||
err error
|
||||
repo *btc.IPLDPublisherAndIndexer
|
||||
ipfsPgGet = `SELECT data FROM public.blocks
|
||||
WHERE key = $1`
|
||||
)
|
||||
BeforeEach(func() {
|
||||
db, err = shared.SetupDB()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
repo = btc.NewIPLDPublisherAndIndexer(db)
|
||||
})
|
||||
AfterEach(func() {
|
||||
btc.TearDownDB(db)
|
||||
})
|
||||
|
||||
Describe("Publish", func() {
|
||||
It("Published and indexes header and transaction IPLDs in a single tx", func() {
|
||||
emptyReturn, err := repo.Publish(mocks.MockConvertedPayload)
|
||||
Expect(emptyReturn).To(BeNil())
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
pgStr := `SELECT * FROM btc.header_cids
|
||||
WHERE block_number = $1`
|
||||
// check header was properly indexed
|
||||
buf := bytes.NewBuffer(make([]byte, 0, 80))
|
||||
err = mocks.MockBlock.Header.Serialize(buf)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
headerBytes := buf.Bytes()
|
||||
c, _ := ipld.RawdataToCid(ipld.MBitcoinHeader, headerBytes, multihash.DBL_SHA2_256)
|
||||
header := new(btc.HeaderModel)
|
||||
err = db.Get(header, pgStr, mocks.MockHeaderMetaData.BlockNumber)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(header.CID).To(Equal(c.String()))
|
||||
Expect(header.BlockNumber).To(Equal(mocks.MockHeaderMetaData.BlockNumber))
|
||||
Expect(header.Bits).To(Equal(mocks.MockHeaderMetaData.Bits))
|
||||
Expect(header.Timestamp).To(Equal(mocks.MockHeaderMetaData.Timestamp))
|
||||
Expect(header.BlockHash).To(Equal(mocks.MockHeaderMetaData.BlockHash))
|
||||
Expect(header.ParentHash).To(Equal(mocks.MockHeaderMetaData.ParentHash))
|
||||
dc, err := cid.Decode(header.CID)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
mhKey := dshelp.CidToDsKey(dc)
|
||||
prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
|
||||
var data []byte
|
||||
err = db.Get(&data, ipfsPgGet, prefixedKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(data).To(Equal(headerBytes))
|
||||
|
||||
// check that txs were properly indexed
|
||||
trxs := make([]btc.TxModel, 0)
|
||||
pgStr = `SELECT transaction_cids.id, transaction_cids.header_id, transaction_cids.index,
|
||||
transaction_cids.tx_hash, transaction_cids.cid, transaction_cids.segwit, transaction_cids.witness_hash
|
||||
FROM btc.transaction_cids INNER JOIN btc.header_cids ON (transaction_cids.header_id = header_cids.id)
|
||||
WHERE header_cids.block_number = $1`
|
||||
err = db.Select(&trxs, pgStr, mocks.MockHeaderMetaData.BlockNumber)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(len(trxs)).To(Equal(3))
|
||||
txData := make([][]byte, len(mocks.MockTransactions))
|
||||
txCIDs := make([]string, len(mocks.MockTransactions))
|
||||
for i, m := range mocks.MockTransactions {
|
||||
buf := bytes.NewBuffer(make([]byte, 0))
|
||||
err = m.MsgTx().Serialize(buf)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
tx := buf.Bytes()
|
||||
txData[i] = tx
|
||||
c, _ := ipld.RawdataToCid(ipld.MBitcoinTx, tx, multihash.DBL_SHA2_256)
|
||||
txCIDs[i] = c.String()
|
||||
}
|
||||
for _, tx := range trxs {
|
||||
Expect(tx.SegWit).To(Equal(false))
|
||||
Expect(tx.HeaderID).To(Equal(header.ID))
|
||||
Expect(tx.WitnessHash).To(Equal(""))
|
||||
Expect(tx.CID).To(Equal(txCIDs[tx.Index]))
|
||||
Expect(tx.TxHash).To(Equal(mocks.MockBlock.Transactions[tx.Index].TxHash().String()))
|
||||
dc, err := cid.Decode(tx.CID)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
mhKey := dshelp.CidToDsKey(dc)
|
||||
prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
|
||||
var data []byte
|
||||
err = db.Get(&data, ipfsPgGet, prefixedKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(data).To(Equal(txData[tx.Index]))
|
||||
}
|
||||
})
|
||||
})
|
||||
})
|
@ -149,23 +149,13 @@ func NewIPLDFetcher(chain shared.ChainType, ipfsPath string) (shared.IPLDFetcher
|
||||
}
|
||||
|
||||
// NewIPLDPublisher constructs an IPLDPublisher for the provided chain type
|
||||
func NewIPLDPublisher(chain shared.ChainType, arg interface{}, ipfsMode shared.IPFSMode) (shared.IPLDPublisher, error) {
|
||||
func NewIPLDPublisher(chain shared.ChainType, ipfsPath string, db *postgres.DB, ipfsMode shared.IPFSMode) (shared.IPLDPublisher, error) {
|
||||
switch chain {
|
||||
case shared.Ethereum:
|
||||
switch ipfsMode {
|
||||
case shared.LocalInterface, shared.RemoteClient:
|
||||
ipfsPath, ok := arg.(string)
|
||||
if !ok {
|
||||
var s string
|
||||
return nil, fmt.Errorf("ethereum IPLDPublisher expected argument type %T got %T", s, arg)
|
||||
}
|
||||
return eth.NewIPLDPublisher(ipfsPath)
|
||||
case shared.DirectPostgres:
|
||||
db, ok := arg.(*postgres.DB)
|
||||
if !ok {
|
||||
var pgdb *postgres.DB
|
||||
return nil, fmt.Errorf("ethereum IPLDPublisher expected argument type %T got %T", pgdb, arg)
|
||||
}
|
||||
return eth.NewIPLDPublisherAndIndexer(db), nil
|
||||
default:
|
||||
return nil, fmt.Errorf("ethereum IPLDPublisher unexpected ipfs mode %s", ipfsMode.String())
|
||||
@ -173,11 +163,6 @@ func NewIPLDPublisher(chain shared.ChainType, arg interface{}, ipfsMode shared.I
|
||||
case shared.Bitcoin:
|
||||
switch ipfsMode {
|
||||
case shared.LocalInterface, shared.RemoteClient:
|
||||
ipfsPath, ok := arg.(string)
|
||||
if !ok {
|
||||
var s string
|
||||
return nil, fmt.Errorf("bitcoin IPLDPublisher expected argument type %T got %T", s, arg)
|
||||
}
|
||||
return btc.NewIPLDPublisher(ipfsPath)
|
||||
case shared.DirectPostgres:
|
||||
// TODO
|
||||
|
@ -54,7 +54,7 @@ func (in *CIDIndexer) Index(cids shared.CIDsForIndexing) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
headerID, err := in.indexHeaderCID(tx, cidPayload.HeaderCID, in.db.NodeID)
|
||||
headerID, err := in.indexHeaderCID(tx, cidPayload.HeaderCID)
|
||||
if err != nil {
|
||||
if err := tx.Rollback(); err != nil {
|
||||
log.Error(err)
|
||||
@ -88,13 +88,13 @@ func (in *CIDIndexer) Index(cids shared.CIDsForIndexing) error {
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func (in *CIDIndexer) indexHeaderCID(tx *sqlx.Tx, header HeaderModel, nodeID int64) (int64, error) {
|
||||
func (in *CIDIndexer) indexHeaderCID(tx *sqlx.Tx, header HeaderModel) (int64, error) {
|
||||
var headerID int64
|
||||
err := tx.QueryRowx(`INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, times_validated)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)
|
||||
ON CONFLICT (block_number, block_hash) DO UPDATE SET (parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, times_validated) = ($3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, eth.header_cids.times_validated + 1)
|
||||
RETURNING id`,
|
||||
header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, header.TotalDifficulty, nodeID, header.Reward, header.StateRoot, header.TxRoot,
|
||||
header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, header.TotalDifficulty, in.db.NodeID, header.Reward, header.StateRoot, header.TxRoot,
|
||||
header.RctRoot, header.UncleRoot, header.Bloom, header.Timestamp, 1).Scan(&headerID)
|
||||
return headerID, err
|
||||
}
|
||||
@ -126,6 +126,15 @@ func (in *CIDIndexer) indexTransactionAndReceiptCIDs(tx *sqlx.Tx, payload *CIDPa
|
||||
return nil
|
||||
}
|
||||
|
||||
func (in *CIDIndexer) indexTransactionCID(tx *sqlx.Tx, transaction TxModel, headerID int64) (int64, error) {
|
||||
var txID int64
|
||||
err := tx.QueryRowx(`INSERT INTO eth.transaction_cids (header_id, tx_hash, cid, dst, src, index) VALUES ($1, $2, $3, $4, $5, $6)
|
||||
ON CONFLICT (header_id, tx_hash) DO UPDATE SET (cid, dst, src, index) = ($3, $4, $5, $6)
|
||||
RETURNING id`,
|
||||
headerID, transaction.TxHash, transaction.CID, transaction.Dst, transaction.Src, transaction.Index).Scan(&txID)
|
||||
return txID, err
|
||||
}
|
||||
|
||||
func (in *CIDIndexer) indexReceiptCID(tx *sqlx.Tx, cidMeta ReceiptModel, txID int64) error {
|
||||
_, err := tx.Exec(`INSERT INTO eth.receipt_cids (tx_id, cid, contract, contract_hash, topic0s, topic1s, topic2s, topic3s, log_contracts) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
|
||||
ON CONFLICT (tx_id) DO UPDATE SET (cid, contract, contract_hash, topic0s, topic1s, topic2s, topic3s, log_contracts) = ($2, $3, $4, $5, $6, $7, $8, $9)`,
|
||||
@ -165,6 +174,19 @@ func (in *CIDIndexer) indexStateAndStorageCIDs(tx *sqlx.Tx, payload *CIDPayload,
|
||||
return nil
|
||||
}
|
||||
|
||||
func (in *CIDIndexer) indexStateCID(tx *sqlx.Tx, stateNode StateNodeModel, headerID int64) (int64, error) {
|
||||
var stateID int64
|
||||
var stateKey string
|
||||
if stateNode.StateKey != nullHash.String() {
|
||||
stateKey = stateNode.StateKey
|
||||
}
|
||||
err := tx.QueryRowx(`INSERT INTO eth.state_cids (header_id, state_leaf_key, cid, state_path, node_type) VALUES ($1, $2, $3, $4, $5)
|
||||
ON CONFLICT (header_id, state_path) DO UPDATE SET (state_leaf_key, cid, node_type) = ($2, $3, $5)
|
||||
RETURNING id`,
|
||||
headerID, stateKey, stateNode.CID, stateNode.Path, stateNode.NodeType).Scan(&stateID)
|
||||
return stateID, err
|
||||
}
|
||||
|
||||
func (in *CIDIndexer) indexStateAccount(tx *sqlx.Tx, stateAccount StateAccountModel, stateID int64) error {
|
||||
_, err := tx.Exec(`INSERT INTO eth.state_accounts (state_id, balance, nonce, code_hash, storage_root) VALUES ($1, $2, $3, $4, $5)
|
||||
ON CONFLICT (state_id) DO UPDATE SET (balance, nonce, code_hash, storage_root) = ($2, $3, $4, $5)`,
|
||||
|
@ -19,15 +19,10 @@ package eth
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/ipfs/go-ipfs-blockstore"
|
||||
|
||||
"github.com/ethereum/go-ethereum/core/state"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/crypto"
|
||||
"github.com/ethereum/go-ethereum/rlp"
|
||||
"github.com/ethereum/go-ethereum/statediff"
|
||||
"github.com/ipfs/go-cid"
|
||||
"github.com/ipfs/go-ipfs-ds-help"
|
||||
"github.com/jmoiron/sqlx"
|
||||
|
||||
common2 "github.com/vulcanize/vulcanizedb/pkg/eth/converters/common"
|
||||
@ -40,13 +35,13 @@ import (
|
||||
// It interfaces directly with the public.blocks table of PG-IPFS rather than going through an ipfs intermediary
|
||||
// It publishes and indexes IPLDs together in a single sqlx.Tx
|
||||
type IPLDPublisherAndIndexer struct {
|
||||
db *postgres.DB
|
||||
indexer *CIDIndexer
|
||||
}
|
||||
|
||||
// NewIPLDPublisherAndIndexer creates a pointer to a new IPLDPublisherAndIndexer which satisfies the IPLDPublisher interface
|
||||
func NewIPLDPublisherAndIndexer(db *postgres.DB) *IPLDPublisherAndIndexer {
|
||||
return &IPLDPublisherAndIndexer{
|
||||
db: db,
|
||||
indexer: NewCIDIndexer(db),
|
||||
}
|
||||
}
|
||||
|
||||
@ -63,27 +58,27 @@ func (pub *IPLDPublisherAndIndexer) Publish(payload shared.ConvertedData) (share
|
||||
}
|
||||
|
||||
// Begin new db tx
|
||||
tx, err := pub.db.Beginx()
|
||||
tx, err := pub.indexer.db.Beginx()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Publish trie nodes
|
||||
for _, node := range txTrieNodes {
|
||||
if err := pub.publishIPLD(tx, node); err != nil {
|
||||
if err := shared.PublishIPLD(tx, node); err != nil {
|
||||
shared.Rollback(tx)
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
for _, node := range rctTrieNodes {
|
||||
if err := pub.publishIPLD(tx, node); err != nil {
|
||||
if err := shared.PublishIPLD(tx, node); err != nil {
|
||||
shared.Rollback(tx)
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// Publish and index header
|
||||
if err := pub.publishIPLD(tx, headerNode); err != nil {
|
||||
if err := shared.PublishIPLD(tx, headerNode); err != nil {
|
||||
shared.Rollback(tx)
|
||||
return nil, err
|
||||
}
|
||||
@ -102,7 +97,7 @@ func (pub *IPLDPublisherAndIndexer) Publish(payload shared.ConvertedData) (share
|
||||
UncleRoot: ipldPayload.Block.UncleHash().String(),
|
||||
Timestamp: ipldPayload.Block.Time(),
|
||||
}
|
||||
headerID, err := pub.indexHeader(tx, header)
|
||||
headerID, err := pub.indexer.indexHeaderCID(tx, header)
|
||||
if err != nil {
|
||||
shared.Rollback(tx)
|
||||
return nil, err
|
||||
@ -110,7 +105,7 @@ func (pub *IPLDPublisherAndIndexer) Publish(payload shared.ConvertedData) (share
|
||||
|
||||
// Publish and index uncles
|
||||
for _, uncleNode := range uncleNodes {
|
||||
if err := pub.publishIPLD(tx, uncleNode); err != nil {
|
||||
if err := shared.PublishIPLD(tx, uncleNode); err != nil {
|
||||
shared.Rollback(tx)
|
||||
return nil, err
|
||||
}
|
||||
@ -121,7 +116,7 @@ func (pub *IPLDPublisherAndIndexer) Publish(payload shared.ConvertedData) (share
|
||||
BlockHash: uncleNode.Hash().String(),
|
||||
Reward: uncleReward.String(),
|
||||
}
|
||||
if err := pub.indexUncle(tx, uncle, headerID); err != nil {
|
||||
if err := pub.indexer.indexUncleCID(tx, uncle, headerID); err != nil {
|
||||
shared.Rollback(tx)
|
||||
return nil, err
|
||||
}
|
||||
@ -129,25 +124,25 @@ func (pub *IPLDPublisherAndIndexer) Publish(payload shared.ConvertedData) (share
|
||||
|
||||
// Publish and index txs and receipts
|
||||
for i, txNode := range txNodes {
|
||||
if err := pub.publishIPLD(tx, txNode); err != nil {
|
||||
if err := shared.PublishIPLD(tx, txNode); err != nil {
|
||||
shared.Rollback(tx)
|
||||
return nil, err
|
||||
}
|
||||
rctNode := rctNodes[i]
|
||||
if err := pub.publishIPLD(tx, rctNode); err != nil {
|
||||
if err := shared.PublishIPLD(tx, rctNode); err != nil {
|
||||
shared.Rollback(tx)
|
||||
return nil, err
|
||||
}
|
||||
txModel := ipldPayload.TxMetaData[i]
|
||||
txModel.CID = txNode.Cid().String()
|
||||
txID, err := pub.indexTx(tx, txModel, headerID)
|
||||
txID, err := pub.indexer.indexTransactionCID(tx, txModel, headerID)
|
||||
if err != nil {
|
||||
shared.Rollback(tx)
|
||||
return nil, err
|
||||
}
|
||||
rctModel := ipldPayload.ReceiptMetaData[i]
|
||||
rctModel.CID = rctNode.Cid().String()
|
||||
if err := pub.indexRct(tx, rctModel, txID); err != nil {
|
||||
if err := pub.indexer.indexReceiptCID(tx, rctModel, txID); err != nil {
|
||||
shared.Rollback(tx)
|
||||
return nil, err
|
||||
}
|
||||
@ -170,7 +165,7 @@ func (pub *IPLDPublisherAndIndexer) publishAndIndexStateAndStorage(tx *sqlx.Tx,
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := pub.publishIPLD(tx, stateIPLD); err != nil {
|
||||
if err := shared.PublishIPLD(tx, stateIPLD); err != nil {
|
||||
shared.Rollback(tx)
|
||||
return err
|
||||
}
|
||||
@ -180,7 +175,7 @@ func (pub *IPLDPublisherAndIndexer) publishAndIndexStateAndStorage(tx *sqlx.Tx,
|
||||
CID: stateIPLD.Cid().String(),
|
||||
NodeType: ResolveFromNodeType(stateNode.Type),
|
||||
}
|
||||
stateID, err := pub.indexState(tx, stateModel, headerID)
|
||||
stateID, err := pub.indexer.indexStateCID(tx, stateModel, headerID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -203,7 +198,7 @@ func (pub *IPLDPublisherAndIndexer) publishAndIndexStateAndStorage(tx *sqlx.Tx,
|
||||
CodeHash: account.CodeHash,
|
||||
StorageRoot: account.Root.String(),
|
||||
}
|
||||
if err := pub.indexAccount(tx, accountModel, stateID); err != nil {
|
||||
if err := pub.indexer.indexStateAccount(tx, accountModel, stateID); err != nil {
|
||||
return err
|
||||
}
|
||||
statePathHash := crypto.Keccak256Hash(stateNode.Path)
|
||||
@ -212,7 +207,7 @@ func (pub *IPLDPublisherAndIndexer) publishAndIndexStateAndStorage(tx *sqlx.Tx,
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := pub.publishIPLD(tx, storageIPLD); err != nil {
|
||||
if err := shared.PublishIPLD(tx, storageIPLD); err != nil {
|
||||
return err
|
||||
}
|
||||
storageModel := StorageNodeModel{
|
||||
@ -221,7 +216,7 @@ func (pub *IPLDPublisherAndIndexer) publishAndIndexStateAndStorage(tx *sqlx.Tx,
|
||||
CID: storageIPLD.Cid().String(),
|
||||
NodeType: ResolveFromNodeType(storageNode.Type),
|
||||
}
|
||||
if err := pub.indexStorage(tx, storageModel, stateID); err != nil {
|
||||
if err := pub.indexer.indexStorageCID(tx, storageModel, stateID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@ -234,86 +229,3 @@ func (pub *IPLDPublisherAndIndexer) publishAndIndexStateAndStorage(tx *sqlx.Tx,
|
||||
func (pub *IPLDPublisherAndIndexer) Index(cids shared.CIDsForIndexing) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
type ipldBase interface {
|
||||
Cid() cid.Cid
|
||||
RawData() []byte
|
||||
}
|
||||
|
||||
func (pub *IPLDPublisherAndIndexer) publishIPLD(tx *sqlx.Tx, i ipldBase) error {
|
||||
dbKey := dshelp.CidToDsKey(i.Cid())
|
||||
prefixedKey := blockstore.BlockPrefix.String() + dbKey.String()
|
||||
raw := i.RawData()
|
||||
_, err := tx.Exec(`INSERT INTO public.blocks (key, data) VALUES ($1, $2) ON CONFLICT (key) DO NOTHING`, prefixedKey, raw)
|
||||
return err
|
||||
}
|
||||
|
||||
func (pub *IPLDPublisherAndIndexer) generateAndPublishBlockIPLDs(tx *sqlx.Tx, body *types.Block, receipts types.Receipts) (*ipld.EthHeader,
|
||||
[]*ipld.EthHeader, []*ipld.EthTx, []*ipld.EthTxTrie, []*ipld.EthReceipt, []*ipld.EthRctTrie, error) {
|
||||
return ipld.FromBlockAndReceipts(body, receipts)
|
||||
}
|
||||
|
||||
func (pub *IPLDPublisherAndIndexer) indexHeader(tx *sqlx.Tx, header HeaderModel) (int64, error) {
|
||||
var headerID int64
|
||||
err := tx.QueryRowx(`INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, times_validated)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)
|
||||
ON CONFLICT (block_number, block_hash) DO UPDATE SET (parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, times_validated) = ($3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, eth.header_cids.times_validated + 1)
|
||||
RETURNING id`,
|
||||
header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, header.TotalDifficulty, pub.db.NodeID, header.Reward, header.StateRoot, header.TxRoot,
|
||||
header.RctRoot, header.UncleRoot, header.Bloom, header.Timestamp, 1).Scan(&headerID)
|
||||
return headerID, err
|
||||
}
|
||||
|
||||
func (pub *IPLDPublisherAndIndexer) indexUncle(tx *sqlx.Tx, uncle UncleModel, headerID int64) error {
|
||||
_, err := tx.Exec(`INSERT INTO eth.uncle_cids (block_hash, header_id, parent_hash, cid, reward) VALUES ($1, $2, $3, $4, $5)
|
||||
ON CONFLICT (header_id, block_hash) DO UPDATE SET (parent_hash, cid, reward) = ($3, $4, $5)`,
|
||||
uncle.BlockHash, headerID, uncle.ParentHash, uncle.CID, uncle.Reward)
|
||||
return err
|
||||
}
|
||||
|
||||
func (pub *IPLDPublisherAndIndexer) indexTx(tx *sqlx.Tx, transaction TxModel, headerID int64) (int64, error) {
|
||||
var txID int64
|
||||
err := tx.QueryRowx(`INSERT INTO eth.transaction_cids (header_id, tx_hash, cid, dst, src, index) VALUES ($1, $2, $3, $4, $5, $6)
|
||||
ON CONFLICT (header_id, tx_hash) DO UPDATE SET (cid, dst, src, index) = ($3, $4, $5, $6)
|
||||
RETURNING id`,
|
||||
headerID, transaction.TxHash, transaction.CID, transaction.Dst, transaction.Src, transaction.Index).Scan(&txID)
|
||||
return txID, err
|
||||
}
|
||||
|
||||
func (pub *IPLDPublisherAndIndexer) indexRct(tx *sqlx.Tx, receipt ReceiptModel, txID int64) error {
|
||||
_, err := tx.Exec(`INSERT INTO eth.receipt_cids (tx_id, cid, contract, contract_hash, topic0s, topic1s, topic2s, topic3s, log_contracts) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
|
||||
ON CONFLICT (tx_id) DO UPDATE SET (cid, contract, contract_hash, topic0s, topic1s, topic2s, topic3s, log_contracts) = ($2, $3, $4, $5, $6, $7, $8, $9)`,
|
||||
txID, receipt.CID, receipt.Contract, receipt.ContractHash, receipt.Topic0s, receipt.Topic1s, receipt.Topic2s, receipt.Topic3s, receipt.LogContracts)
|
||||
return err
|
||||
}
|
||||
|
||||
func (pub *IPLDPublisherAndIndexer) indexState(tx *sqlx.Tx, stateNode StateNodeModel, headerID int64) (int64, error) {
|
||||
var stateID int64
|
||||
var stateKey string
|
||||
if stateNode.StateKey != nullHash.String() {
|
||||
stateKey = stateNode.StateKey
|
||||
}
|
||||
err := tx.QueryRowx(`INSERT INTO eth.state_cids (header_id, state_leaf_key, cid, state_path, node_type) VALUES ($1, $2, $3, $4, $5)
|
||||
ON CONFLICT (header_id, state_path) DO UPDATE SET (state_leaf_key, cid, node_type) = ($2, $3, $5)
|
||||
RETURNING id`,
|
||||
headerID, stateKey, stateNode.CID, stateNode.Path, stateNode.NodeType).Scan(&stateID)
|
||||
return stateID, err
|
||||
}
|
||||
|
||||
func (pub *IPLDPublisherAndIndexer) indexStorage(tx *sqlx.Tx, storageNode StorageNodeModel, stateID int64) error {
|
||||
var storageKey string
|
||||
if storageNode.StorageKey != nullHash.String() {
|
||||
storageKey = storageNode.StorageKey
|
||||
}
|
||||
_, err := tx.Exec(`INSERT INTO eth.storage_cids (state_id, storage_leaf_key, cid, storage_path, node_type) VALUES ($1, $2, $3, $4, $5)
|
||||
ON CONFLICT (state_id, storage_path) DO UPDATE SET (storage_leaf_key, cid, node_type) = ($2, $3, $5)`,
|
||||
stateID, storageKey, storageNode.CID, storageNode.Path, storageNode.NodeType)
|
||||
return err
|
||||
}
|
||||
|
||||
func (pub *IPLDPublisherAndIndexer) indexAccount(tx *sqlx.Tx, account StateAccountModel, stateID int64) error {
|
||||
_, err := tx.Exec(`INSERT INTO eth.state_accounts (state_id, balance, nonce, code_hash, storage_root) VALUES ($1, $2, $3, $4, $5)
|
||||
ON CONFLICT (state_id) DO UPDATE SET (balance, nonce, code_hash, storage_root) = ($2, $3, $4, $5)`,
|
||||
stateID, account.Balance, account.Nonce, account.CodeHash, account.StorageRoot)
|
||||
return err
|
||||
}
|
||||
|
@ -64,7 +64,7 @@ type Service struct {
|
||||
|
||||
// NewResyncService creates and returns a resync service from the provided settings
|
||||
func NewResyncService(settings *Config) (Resync, error) {
|
||||
publisher, err := super_node.NewIPLDPublisher(settings.Chain, settings.IPFSPath, settings.IPFSMode)
|
||||
publisher, err := super_node.NewIPLDPublisher(settings.Chain, settings.IPFSPath, settings.DB, settings.IPFSMode)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
@ -109,7 +109,7 @@ func NewSuperNode(settings *Config) (SuperNode, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sn.Publisher, err = NewIPLDPublisher(settings.Chain, settings.IPFSPath, settings.IPFSMode)
|
||||
sn.Publisher, err = NewIPLDPublisher(settings.Chain, settings.IPFSPath, settings.DB, settings.IPFSMode)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
@ -19,11 +19,13 @@ package shared
|
||||
import (
|
||||
"bytes"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ipfs/go-ipfs-blockstore"
|
||||
"github.com/ipfs/go-ipfs-ds-help"
|
||||
node "github.com/ipfs/go-ipld-format"
|
||||
"github.com/jmoiron/sqlx"
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
|
||||
)
|
||||
|
||||
@ -79,3 +81,12 @@ func Rollback(tx *sqlx.Tx) {
|
||||
logrus.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
// PublishIPLD is used to insert an ipld into Postgres blockstore with the provided tx
|
||||
func PublishIPLD(tx *sqlx.Tx, i node.Node) error {
|
||||
dbKey := dshelp.CidToDsKey(i.Cid())
|
||||
prefixedKey := blockstore.BlockPrefix.String() + dbKey.String()
|
||||
raw := i.RawData()
|
||||
_, err := tx.Exec(`INSERT INTO public.blocks (key, data) VALUES ($1, $2) ON CONFLICT (key) DO NOTHING`, prefixedKey, raw)
|
||||
return err
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user