cerc-io/ipld-eth-server
18
0
Fork 1
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases 37 Wiki Activity

WIP: Refactor and update to use new schema. #118

Closed
arijitAD wants to merge 4 commits from postgres_refactor into master
pull from: postgres_refactor
merge into: cerc-io:master
Conversation 0 Commits 4 Files Changed 28 +32 -34
6 changed files with 32 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit c44598971b - Show all commits

4
go.mod
View File

@ -35,6 +35,6 @@ require (
replace (
github.com/ethereum/go-ethereum v1.10.11 => /Users/arijitdas/go/src/github.com/ethereum/go-ethereum
github.com/vulcanize/eth-ipfs-state-validator v0.0.2 => /Users/arijitdas/go/src/github.com/Vulcanize/eth-ipfs-state-validator
github.com/vulcanize/ipfs-ethdb v0.0.5 => /Users/arijitdas/go/src/github.com/Vulcanize/ipfs-ethdb
github.com/vulcanize/eth-ipfs-state-validator v0.0.2 => github.com/Vulcanize/eth-ipfs-state-validator v0.0.0-20211125063620-bd5bbc60275f
github.com/vulcanize/ipfs-ethdb v0.0.5 => github.com/Vulcanize/ipfs-ethdb v0.0.0-20211125071127-730512e6953c
)

5
go.sum
View File

@ -84,6 +84,10 @@ github.com/VictoriaMetrics/fastcache v1.5.3/go.mod h1:+jv9Ckb+za/P1ZRg/sulP5Ni1v
github.com/VictoriaMetrics/fastcache v1.6.0 h1:C/3Oi3EiBCqufydp1neRZkqcwmEiuRT9c3fqvvgKm5o=
github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw=
github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g=
github.com/Vulcanize/eth-ipfs-state-validator v0.0.0-20211125063620-bd5bbc60275f h1:Ola2m1pH6ax3LZPDF9S6aeC6aGMidDEgonKPRS8eEAM=
github.com/Vulcanize/eth-ipfs-state-validator v0.0.0-20211125063620-bd5bbc60275f/go.mod h1:cyBC11l/fNcAfEQgx6yjNtzlgfQZqG821k0/qll2lDE=
github.com/Vulcanize/ipfs-ethdb v0.0.0-20211125071127-730512e6953c h1:ooFjd7mPek6aB4pyxa9PolqrjG67ldSiXaZ4755OAZg=
github.com/Vulcanize/ipfs-ethdb v0.0.0-20211125071127-730512e6953c/go.mod h1:Emd7r9hHiZkW/yyOMNjDo89GwYIVQeMU8W9ywwTquBI=
github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c=
github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
@ -748,7 +752,6 @@ github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9
github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8=
github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
github.com/jackc/fake v0.0.0-20150926172116-812a484cc733/go.mod h1:WrMFNQdiFJ80sQsxDoMokWK1W5TQtxBFNpzWTD84ibQ=
github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA=
github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE=
github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s=

23
pkg/eth/backend.go
View File

@ -41,7 +41,6 @@ import (
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
"github.com/ethereum/go-ethereum/statediff/indexer/models"
"github.com/ethereum/go-ethereum/trie"
log "github.com/sirupsen/logrus"
validator "github.com/vulcanize/eth-ipfs-state-validator/pkg"
@ -70,7 +69,7 @@ const (
RetrieveCanonicalHeaderByNumber = `SELECT cid, data FROM eth.header_cids
INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key)
WHERE block_hash = (SELECT canonical_header_id($1))`
RetrieveTD = `SELECT td FROM eth.header_cids
RetrieveTD = `SELECT CAST(td as Text) FROM eth.header_cids
WHERE header_cids.block_hash = $1`
RetrieveRPCTransaction = `SELECT blocks.data, block_hash, block_number, index FROM public.blocks, eth.transaction_cids, eth.header_cids
WHERE blocks.key = transaction_cids.mh_key
@ -302,9 +301,6 @@ func (b *Backend) BlockByNumber(ctx context.Context, blockNumber rpc.BlockNumber
// TODO: optimize this by retrieving iplds directly rather than the cids first (this is remanent from when we fetched iplds through ipfs blockservice interface)
headerCID, uncleCIDs, txCIDs, rctCIDs, err := b.Retriever.RetrieveBlockByHash(ctx, canonicalHash)
if err != nil {
if err == dbsql.ErrNoRows {
return nil, nil
}
return nil, err
}
@ -401,7 +397,6 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo
if err == dbsql.ErrNoRows {
return nil, nil
}
return nil, err
}
// Begin tx
@ -617,7 +612,12 @@ type rowResult struct {
// GetCanonicalHeader gets the canonical header for the provided number, if there is one
func (b *Backend) GetCanonicalHeader(ctx context.Context, number uint64) (string, []byte, error) {
headerResult := new(rowResult)
return headerResult.CID, headerResult.Data, b.DB.QueryRow(ctx, RetrieveCanonicalHeaderByNumber, number).Scan(&headerResult.CID, &headerResult.Data)
err := b.DB.QueryRow(ctx, RetrieveCanonicalHeaderByNumber, number).Scan(&headerResult.CID, &headerResult.Data)
if err != nil {
return "", nil, err
}
return headerResult.CID, headerResult.Data, nil
}
// GetEVM constructs and returns a vm.EVM
@ -745,7 +745,8 @@ func (b *Backend) GetCodeByHash(ctx context.Context, address common.Address, has
err = tx.Commit(ctx)
}
}()
err = tx.Get(ctx, &codeHash, RetrieveCodeHashByLeafKeyAndBlockHash, leafKey.Hex(), hash.Hex())
err = tx.QueryRow(ctx, RetrieveCodeHashByLeafKeyAndBlockHash, leafKey.Hex(), hash.Hex()).Scan(&codeHash)
if err != nil {
return nil, err
}
@ -757,7 +758,11 @@ func (b *Backend) GetCodeByHash(ctx context.Context, address common.Address, has
}
code := make([]byte, 0)
err = tx.Get(ctx, &code, RetrieveCodeByMhKey, mhKey)
err = tx.QueryRow(ctx, RetrieveCodeByMhKey, mhKey).Scan(&code)
if err != nil {
return nil, err
}
return code, err
}

17
pkg/eth/cid_retriever.go
View File

@ -169,7 +169,9 @@ func (ecr *CIDRetriever) Retrieve(ctx context.Context, filter SubscriptionSettin
func (ecr *CIDRetriever) RetrieveHeaderCIDs(ctx context.Context, tx sql.Tx, blockNumber int64) ([]models.HeaderModel, error) {
log.Debug("retrieving header cids for block ", blockNumber)
headers := make([]models.HeaderModel, 0)
pgStr := `SELECT * FROM eth.header_cids
pgStr := `SELECT CAST(block_number as Text), block_hash,parent_hash,cid,mh_key,CAST(td as Text),node_id,
CAST(reward as Text), state_root,uncle_root,tx_root,receipt_root,bloom,timestamp,times_validated,
coinbase FROM eth.header_cids
WHERE block_number = $1`
return headers, tx.Select(ctx, &headers, pgStr, blockNumber)
@ -179,7 +181,7 @@ func (ecr *CIDRetriever) RetrieveHeaderCIDs(ctx context.Context, tx sql.Tx, bloc
func (ecr *CIDRetriever) RetrieveUncleCIDsByHeaderID(ctx context.Context, tx sql.Tx, headerID string) ([]models.UncleModel, error) {
log.Debug("retrieving uncle cids for block id ", headerID)
headers := make([]models.UncleModel, 0)
pgStr := `SELECT * FROM eth.uncle_cids
pgStr := `SELECT header_id,block_hash,parent_hash,cid,mh_key, CAST(reward as text) FROM eth.uncle_cids
WHERE header_id = $1`
return headers, tx.Select(ctx, &headers, pgStr, headerID)
@ -192,8 +194,7 @@ func (ecr *CIDRetriever) RetrieveTxCIDs(ctx context.Context, tx sql.Tx, txFilter
args := make([]interface{}, 0, 3)
results := make([]models.TxModel, 0)
id := 1
pgStr := fmt.Sprintf(`SELECT transaction_cids.tx_hash, transaction_cids.header_id,
transaction_cids.tx_hash, transaction_cids.cid, transaction_cids.mh_key,
pgStr := fmt.Sprintf(`SELECT transaction_cids.tx_hash, transaction_cids.header_id,transaction_cids.cid, transaction_cids.mh_key,
transaction_cids.dst, transaction_cids.src, transaction_cids.index, transaction_cids.tx_data
FROM eth.transaction_cids INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.block_hash)
WHERE header_cids.block_hash = $%d`, id)
@ -295,7 +296,7 @@ func receiptFilterConditions(id *int, pgStr string, args []interface{}, rctFilte
func (ecr *CIDRetriever) RetrieveRctCIDsByHeaderID(ctx context.Context, tx sql.Tx, rctFilter ReceiptFilter, headerID string, trxHashes []string) ([]models.ReceiptModel, error) {
log.Debug("retrieving receipt cids for header id ", headerID)
args := make([]interface{}, 0, 4)
pgStr := `SELECT receipt_cids.tx_id, receipt_cids.tx_id, receipt_cids.leaf_cid, receipt_cids.leaf_mh_key,
pgStr := `SELECT receipt_cids.tx_id, receipt_cids.leaf_cid, receipt_cids.leaf_mh_key,
receipt_cids.contract, receipt_cids.contract_hash
FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
WHERE receipt_cids.tx_id = transaction_cids.tx_hash
@ -344,7 +345,7 @@ func (ecr *CIDRetriever) RetrieveFilteredLog(ctx context.Context, tx sql.Tx, rct
pgStr := `SELECT eth.log_cids.leaf_cid, eth.log_cids.index, eth.log_cids.rct_id,
eth.log_cids.address, eth.log_cids.topic0, eth.log_cids.topic1, eth.log_cids.topic2, eth.log_cids.topic3,
eth.log_cids.log_data, eth.transaction_cids.tx_hash, eth.transaction_cids.index as txn_index,
header_cids.block_hash, header_cids.block_number
header_cids.block_hash, CAST(header_cids.block_number as Text)
FROM eth.log_cids, eth.receipt_cids, eth.transaction_cids, eth.header_cids
WHERE eth.log_cids.rct_id = receipt_cids.tx_id
AND receipt_cids.tx_id = transaction_cids.tx_hash
@ -569,10 +570,10 @@ func (ecr *CIDRetriever) RetrieveBlockByNumber(ctx context.Context, blockNumber
// RetrieveHeaderCIDByHash returns the header for the given block hash
func (ecr *CIDRetriever) RetrieveHeaderCIDByHash(ctx context.Context, tx sql.Tx, blockHash common.Hash) (models.HeaderModel, error) {
log.Debug("retrieving header cids for block hash ", blockHash.String())
pgStr := `SELECT * FROM eth.header_cids
pgStr := `SELECT block_hash,cid,mh_key FROM eth.header_cids
WHERE block_hash = $1`
var headerCID models.HeaderModel
return headerCID, tx.Get(ctx, &headerCID, pgStr, blockHash.String())
return headerCID, tx.QueryRow(ctx, pgStr, blockHash.String()).Scan(&headerCID.BlockHash, &headerCID.CID, &headerCID.MhKey)
}
// RetrieveTxCIDsByHeaderID retrieves all tx CIDs for the given header id

6
pkg/eth/test_helpers/test_data.go
View File

@ -437,12 +437,11 @@ var (
StateNodes: MockStateNodes,
}
Reward = shared.CalcEthBlockReward(MockBlock.Header(), MockBlock.Uncles(), MockBlock.Transactions(), MockReceipts)
Reward = shared.CalcEthBlockReward(MockBlock.Header(), MockBlock.Uncles(), MockBlock.Transactions(), MockReceipts)
MockCIDWrapper = &eth.CIDWrapper{
BlockNumber: new(big.Int).Set(BlockNumber),
Header: models.HeaderModel{
BlockNumber: "1",
BlockNumber: BlockNumber.String(),
BlockHash: MockBlock.Hash().String(),
ParentHash: "0x0000000000000000000000000000000000000000000000000000000000000000",
CID: HeaderCID.String(),
@ -456,6 +455,7 @@ var (
Bloom: MockBlock.Bloom().Bytes(),
Timestamp: MockBlock.Time(),
TimesValidated: 1,
Coinbase: "0x0000000000000000000000000000000000000000",
},
Transactions: MockTrxMetaPostPublsh,
Receipts: MockRctMetaPostPublish,

11
pkg/shared/functions.go
View File

@ -62,17 +62,6 @@ func PublishIPLD(ctx context.Context, tx sql.Tx, i node.Node) error {
return err
}
// FetchIPLD is used to retrieve an ipld from Postgres blockstore with the provided tx and cid string
func FetchIPLD(tx *sqlx.Tx, cid string) ([]byte, error) {
mhKey, err := MultihashKeyFromCIDString(cid)
if err != nil {
return nil, err
}
pgStr := `SELECT data FROM public.blocks WHERE key = $1`
var block []byte
return block, tx.Get(&block, pgStr, mhKey)
}
// FetchIPLDByMhKey is used to retrieve an ipld from Postgres blockstore with the provided tx and mhkey string
func FetchIPLDByMhKey(ctx context.Context, tx sql.Tx, mhKey string) ([]byte, error) {
pgStr := `SELECT data FROM public.blocks WHERE key = $1`