forked from cerc-io/ipld-eth-server
expand retriever, fetcher, and resolver interfaces in prep for eth api mimick
This commit is contained in:
Ian Norden
+99
-90
@@ -18,6 +18,7 @@ package ipfs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ipfs/go-block-format"
|
||||
@@ -26,9 +27,19 @@ import (
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
var (
|
||||
errUnexpectedNumberOfIPLDs = errors.New("ipfs batch fetch returned unexpected number of IPLDs")
|
||||
)
|
||||
|
||||
// IPLDFetcher is an interface for fetching IPLDs
|
||||
type IPLDFetcher interface {
|
||||
FetchIPLDs(cids CIDWrapper) (*IPLDWrapper, error)
|
||||
FetchHeaders(cids []string) ([]blocks.Block, error)
|
||||
FetchUncles(cids []string) ([]blocks.Block, error)
|
||||
FetchTrxs(cids []string) ([]blocks.Block, error)
|
||||
FetchRcts(cids []string) ([]blocks.Block, error)
|
||||
FetchState(cids []StateNodeCID) (map[common.Hash]blocks.Block, error)
|
||||
FetchStorage(cids []StorageNodeCID) (map[common.Hash]map[common.Hash]blocks.Block, error)
|
||||
}
|
||||
|
||||
// EthIPLDFetcher is used to fetch ETH IPLD objects from IPFS
|
||||
@@ -51,165 +62,163 @@ func NewIPLDFetcher(ipfsPath string) (*EthIPLDFetcher, error) {
|
||||
func (f *EthIPLDFetcher) FetchIPLDs(cids CIDWrapper) (*IPLDWrapper, error) {
|
||||
|
||||
log.Debug("fetching iplds")
|
||||
blocks := &IPLDWrapper{
|
||||
BlockNumber: cids.BlockNumber,
|
||||
Headers: make([]blocks.Block, 0),
|
||||
Uncles: make([]blocks.Block, 0),
|
||||
Transactions: make([]blocks.Block, 0),
|
||||
Receipts: make([]blocks.Block, 0),
|
||||
StateNodes: make(map[common.Hash]blocks.Block),
|
||||
StorageNodes: make(map[common.Hash]map[common.Hash]blocks.Block),
|
||||
iplds := new(IPLDWrapper)
|
||||
iplds.BlockNumber = cids.BlockNumber
|
||||
var err error
|
||||
iplds.Headers, err = f.FetchHeaders(cids.Headers)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
headersErr := f.fetchHeaders(cids, blocks)
|
||||
if headersErr != nil {
|
||||
return nil, headersErr
|
||||
iplds.Uncles, err = f.FetchUncles(cids.Uncles)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
unclesErr := f.fetchUncles(cids, blocks)
|
||||
if unclesErr != nil {
|
||||
return nil, unclesErr
|
||||
iplds.Transactions, err = f.FetchTrxs(cids.Transactions)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
trxsErr := f.fetchTrxs(cids, blocks)
|
||||
if trxsErr != nil {
|
||||
return nil, trxsErr
|
||||
iplds.Receipts, err = f.FetchRcts(cids.Receipts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rctsErr := f.fetchRcts(cids, blocks)
|
||||
if rctsErr != nil {
|
||||
return nil, rctsErr
|
||||
iplds.StateNodes, err = f.FetchState(cids.StateNodes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
storageErr := f.fetchStorage(cids, blocks)
|
||||
if storageErr != nil {
|
||||
return nil, storageErr
|
||||
iplds.StorageNodes, err = f.FetchStorage(cids.StorageNodes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stateErr := f.fetchState(cids, blocks)
|
||||
if stateErr != nil {
|
||||
return nil, stateErr
|
||||
}
|
||||
|
||||
return blocks, nil
|
||||
return iplds, nil
|
||||
}
|
||||
|
||||
// fetchHeaders fetches headers
|
||||
// FetchHeaders fetches headers
|
||||
// It uses the f.fetchBatch method
|
||||
func (f *EthIPLDFetcher) fetchHeaders(cids CIDWrapper, blocks *IPLDWrapper) error {
|
||||
func (f *EthIPLDFetcher) FetchHeaders(cids []string) ([]blocks.Block, error) {
|
||||
log.Debug("fetching header iplds")
|
||||
headerCids := make([]cid.Cid, 0, len(cids.Headers))
|
||||
for _, c := range cids.Headers {
|
||||
headerCids := make([]cid.Cid, 0, len(cids))
|
||||
for _, c := range cids {
|
||||
dc, err := cid.Decode(c)
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
headerCids = append(headerCids, dc)
|
||||
}
|
||||
blocks.Headers = f.fetchBatch(headerCids)
|
||||
if len(blocks.Headers) != len(headerCids) {
|
||||
log.Errorf("ipfs fetcher: number of header blocks returned (%d) does not match number expected (%d)", len(blocks.Headers), len(headerCids))
|
||||
headers := f.fetchBatch(headerCids)
|
||||
if len(headers) != len(headerCids) {
|
||||
log.Errorf("ipfs fetcher: number of header blocks returned (%d) does not match number expected (%d)", len(headers), len(headerCids))
|
||||
return headers, errUnexpectedNumberOfIPLDs
|
||||
}
|
||||
return nil
|
||||
return headers, nil
|
||||
}
|
||||
|
||||
// fetchUncles fetches uncles
|
||||
// FetchUncles fetches uncles
|
||||
// It uses the f.fetchBatch method
|
||||
func (f *EthIPLDFetcher) fetchUncles(cids CIDWrapper, blocks *IPLDWrapper) error {
|
||||
func (f *EthIPLDFetcher) FetchUncles(cids []string) ([]blocks.Block, error) {
|
||||
log.Debug("fetching uncle iplds")
|
||||
uncleCids := make([]cid.Cid, 0, len(cids.Uncles))
|
||||
for _, c := range cids.Uncles {
|
||||
uncleCids := make([]cid.Cid, 0, len(cids))
|
||||
for _, c := range cids {
|
||||
dc, err := cid.Decode(c)
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
uncleCids = append(uncleCids, dc)
|
||||
}
|
||||
blocks.Uncles = f.fetchBatch(uncleCids)
|
||||
if len(blocks.Uncles) != len(uncleCids) {
|
||||
log.Errorf("ipfs fetcher: number of uncle blocks returned (%d) does not match number expected (%d)", len(blocks.Uncles), len(uncleCids))
|
||||
uncles := f.fetchBatch(uncleCids)
|
||||
if len(uncles) != len(uncleCids) {
|
||||
log.Errorf("ipfs fetcher: number of uncle blocks returned (%d) does not match number expected (%d)", len(uncles), len(uncleCids))
|
||||
return uncles, errUnexpectedNumberOfIPLDs
|
||||
}
|
||||
return nil
|
||||
return uncles, nil
|
||||
}
|
||||
|
||||
// fetchTrxs fetches transactions
|
||||
// FetchTrxs fetches transactions
|
||||
// It uses the f.fetchBatch method
|
||||
func (f *EthIPLDFetcher) fetchTrxs(cids CIDWrapper, blocks *IPLDWrapper) error {
|
||||
func (f *EthIPLDFetcher) FetchTrxs(cids []string) ([]blocks.Block, error) {
|
||||
log.Debug("fetching transaction iplds")
|
||||
trxCids := make([]cid.Cid, 0, len(cids.Transactions))
|
||||
for _, c := range cids.Transactions {
|
||||
trxCids := make([]cid.Cid, 0, len(cids))
|
||||
for _, c := range cids {
|
||||
dc, err := cid.Decode(c)
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
trxCids = append(trxCids, dc)
|
||||
}
|
||||
blocks.Transactions = f.fetchBatch(trxCids)
|
||||
if len(blocks.Transactions) != len(trxCids) {
|
||||
log.Errorf("ipfs fetcher: number of transaction blocks returned (%d) does not match number expected (%d)", len(blocks.Transactions), len(trxCids))
|
||||
trxs := f.fetchBatch(trxCids)
|
||||
if len(trxs) != len(trxCids) {
|
||||
log.Errorf("ipfs fetcher: number of transaction blocks returned (%d) does not match number expected (%d)", len(trxs), len(trxCids))
|
||||
return trxs, errUnexpectedNumberOfIPLDs
|
||||
}
|
||||
return nil
|
||||
return trxs, nil
|
||||
}
|
||||
|
||||
// fetchRcts fetches receipts
|
||||
// FetchRcts fetches receipts
|
||||
// It uses the f.fetchBatch method
|
||||
func (f *EthIPLDFetcher) fetchRcts(cids CIDWrapper, blocks *IPLDWrapper) error {
|
||||
func (f *EthIPLDFetcher) FetchRcts(cids []string) ([]blocks.Block, error) {
|
||||
log.Debug("fetching receipt iplds")
|
||||
rctCids := make([]cid.Cid, 0, len(cids.Receipts))
|
||||
for _, c := range cids.Receipts {
|
||||
rctCids := make([]cid.Cid, 0, len(cids))
|
||||
for _, c := range cids {
|
||||
dc, err := cid.Decode(c)
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
rctCids = append(rctCids, dc)
|
||||
}
|
||||
blocks.Receipts = f.fetchBatch(rctCids)
|
||||
if len(blocks.Receipts) != len(rctCids) {
|
||||
log.Errorf("ipfs fetcher: number of receipt blocks returned (%d) does not match number expected (%d)", len(blocks.Receipts), len(rctCids))
|
||||
rcts := f.fetchBatch(rctCids)
|
||||
if len(rcts) != len(rctCids) {
|
||||
log.Errorf("ipfs fetcher: number of receipt blocks returned (%d) does not match number expected (%d)", len(rcts), len(rctCids))
|
||||
return rcts, errUnexpectedNumberOfIPLDs
|
||||
}
|
||||
return nil
|
||||
return rcts, nil
|
||||
}
|
||||
|
||||
// fetchState fetches state nodes
|
||||
// FetchState fetches state nodes
|
||||
// It uses the single f.fetch method instead of the batch fetch, because it
|
||||
// needs to maintain the data's relation to state keys
|
||||
func (f *EthIPLDFetcher) fetchState(cids CIDWrapper, blocks *IPLDWrapper) error {
|
||||
func (f *EthIPLDFetcher) FetchState(cids []StateNodeCID) (map[common.Hash]blocks.Block, error) {
|
||||
log.Debug("fetching state iplds")
|
||||
for _, stateNode := range cids.StateNodes {
|
||||
stateNodes := make(map[common.Hash]blocks.Block)
|
||||
for _, stateNode := range cids {
|
||||
if stateNode.CID == "" || stateNode.Key == "" {
|
||||
continue
|
||||
}
|
||||
dc, decodeErr := cid.Decode(stateNode.CID)
|
||||
if decodeErr != nil {
|
||||
return decodeErr
|
||||
dc, err := cid.Decode(stateNode.CID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
block, fetchErr := f.fetch(dc)
|
||||
if fetchErr != nil {
|
||||
return fetchErr
|
||||
state, err := f.fetch(dc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
blocks.StateNodes[common.HexToHash(stateNode.Key)] = block
|
||||
stateNodes[common.HexToHash(stateNode.Key)] = state
|
||||
}
|
||||
return nil
|
||||
return stateNodes, nil
|
||||
}
|
||||
|
||||
// fetchStorage fetches storage nodes
|
||||
// FetchStorage fetches storage nodes
|
||||
// It uses the single f.fetch method instead of the batch fetch, because it
|
||||
// needs to maintain the data's relation to state and storage keys
|
||||
func (f *EthIPLDFetcher) fetchStorage(cids CIDWrapper, blks *IPLDWrapper) error {
|
||||
func (f *EthIPLDFetcher) FetchStorage(cids []StorageNodeCID) (map[common.Hash]map[common.Hash]blocks.Block, error) {
|
||||
log.Debug("fetching storage iplds")
|
||||
for _, storageNode := range cids.StorageNodes {
|
||||
storageNodes := make(map[common.Hash]map[common.Hash]blocks.Block)
|
||||
for _, storageNode := range cids {
|
||||
if storageNode.CID == "" || storageNode.Key == "" || storageNode.StateKey == "" {
|
||||
continue
|
||||
}
|
||||
dc, decodeErr := cid.Decode(storageNode.CID)
|
||||
if decodeErr != nil {
|
||||
return decodeErr
|
||||
dc, err := cid.Decode(storageNode.CID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
blk, fetchErr := f.fetch(dc)
|
||||
if fetchErr != nil {
|
||||
return fetchErr
|
||||
storage, err := f.fetch(dc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if blks.StorageNodes[common.HexToHash(storageNode.StateKey)] == nil {
|
||||
blks.StorageNodes[common.HexToHash(storageNode.StateKey)] = make(map[common.Hash]blocks.Block)
|
||||
if storageNodes[common.HexToHash(storageNode.StateKey)] == nil {
|
||||
storageNodes[common.HexToHash(storageNode.StateKey)] = make(map[common.Hash]blocks.Block)
|
||||
}
|
||||
blks.StorageNodes[common.HexToHash(storageNode.StateKey)][common.HexToHash(storageNode.Key)] = blk
|
||||
storageNodes[common.HexToHash(storageNode.StateKey)][common.HexToHash(storageNode.Key)] = storage
|
||||
}
|
||||
return nil
|
||||
return storageNodes, nil
|
||||
}
|
||||
|
||||
// fetch is used to fetch a single cid
|
||||
|
||||
@@ -312,6 +312,7 @@ var (
|
||||
MockSeeNodePayload = streamer.SuperNodePayload{
|
||||
BlockNumber: big.NewInt(1),
|
||||
HeadersRlp: [][]byte{MockHeaderRlp},
|
||||
UnclesRlp: [][]byte{},
|
||||
TransactionsRlp: [][]byte{MockTransactions.GetRlp(0), MockTransactions.GetRlp(1)},
|
||||
ReceiptsRlp: [][]byte{MockTransactions.GetRlp(0), MockTransactions.GetRlp(1)},
|
||||
StateNodesRlp: map[common.Hash][]byte{
|
||||
|
||||
+48
-39
@@ -25,6 +25,12 @@ import (
|
||||
// IPLDResolver is the interface to resolving IPLDs
|
||||
type IPLDResolver interface {
|
||||
ResolveIPLDs(ipfsBlocks IPLDWrapper) streamer.SuperNodePayload
|
||||
ResolveHeaders(iplds []blocks.Block) [][]byte
|
||||
ResolveUncles(iplds []blocks.Block) [][]byte
|
||||
ResolveTransactions(iplds []blocks.Block) [][]byte
|
||||
ResolveReceipts(blocks []blocks.Block) [][]byte
|
||||
ResolveState(iplds map[common.Hash]blocks.Block) map[common.Hash][]byte
|
||||
ResolveStorage(iplds map[common.Hash]map[common.Hash]blocks.Block) map[common.Hash]map[common.Hash][]byte
|
||||
}
|
||||
|
||||
// EthIPLDResolver is the underlying struct to support the IPLDResolver interface
|
||||
@@ -37,61 +43,64 @@ func NewIPLDResolver() *EthIPLDResolver {
|
||||
|
||||
// ResolveIPLDs is the exported method for resolving all of the ETH IPLDs packaged in an IpfsBlockWrapper
|
||||
func (eir *EthIPLDResolver) ResolveIPLDs(ipfsBlocks IPLDWrapper) streamer.SuperNodePayload {
|
||||
response := &streamer.SuperNodePayload{
|
||||
return streamer.SuperNodePayload{
|
||||
BlockNumber: ipfsBlocks.BlockNumber,
|
||||
StateNodesRlp: make(map[common.Hash][]byte),
|
||||
StorageNodesRlp: make(map[common.Hash]map[common.Hash][]byte),
|
||||
}
|
||||
eir.resolveHeaders(ipfsBlocks.Headers, response)
|
||||
eir.resolveUncles(ipfsBlocks.Uncles, response)
|
||||
eir.resolveTransactions(ipfsBlocks.Transactions, response)
|
||||
eir.resolveReceipts(ipfsBlocks.Receipts, response)
|
||||
eir.resolveState(ipfsBlocks.StateNodes, response)
|
||||
eir.resolveStorage(ipfsBlocks.StorageNodes, response)
|
||||
return *response
|
||||
}
|
||||
|
||||
func (eir *EthIPLDResolver) resolveHeaders(blocks []blocks.Block, response *streamer.SuperNodePayload) {
|
||||
for _, block := range blocks {
|
||||
raw := block.RawData()
|
||||
response.HeadersRlp = append(response.HeadersRlp, raw)
|
||||
HeadersRlp: eir.ResolveHeaders(ipfsBlocks.Headers),
|
||||
UnclesRlp: eir.ResolveUncles(ipfsBlocks.Uncles),
|
||||
TransactionsRlp: eir.ResolveTransactions(ipfsBlocks.Transactions),
|
||||
ReceiptsRlp: eir.ResolveReceipts(ipfsBlocks.Receipts),
|
||||
StateNodesRlp: eir.ResolveState(ipfsBlocks.StateNodes),
|
||||
StorageNodesRlp: eir.ResolveStorage(ipfsBlocks.StorageNodes),
|
||||
}
|
||||
}
|
||||
|
||||
func (eir *EthIPLDResolver) resolveUncles(blocks []blocks.Block, response *streamer.SuperNodePayload) {
|
||||
for _, block := range blocks {
|
||||
raw := block.RawData()
|
||||
response.UnclesRlp = append(response.UnclesRlp, raw)
|
||||
func (eir *EthIPLDResolver) ResolveHeaders(iplds []blocks.Block) [][]byte {
|
||||
headerRlps := make([][]byte, 0, len(iplds))
|
||||
for _, ipld := range iplds {
|
||||
headerRlps = append(headerRlps, ipld.RawData())
|
||||
}
|
||||
return headerRlps
|
||||
}
|
||||
|
||||
func (eir *EthIPLDResolver) resolveTransactions(blocks []blocks.Block, response *streamer.SuperNodePayload) {
|
||||
for _, block := range blocks {
|
||||
raw := block.RawData()
|
||||
response.TransactionsRlp = append(response.TransactionsRlp, raw)
|
||||
func (eir *EthIPLDResolver) ResolveUncles(iplds []blocks.Block) [][]byte {
|
||||
uncleRlps := make([][]byte, 0, len(iplds))
|
||||
for _, ipld := range iplds {
|
||||
uncleRlps = append(uncleRlps, ipld.RawData())
|
||||
}
|
||||
return uncleRlps
|
||||
}
|
||||
|
||||
func (eir *EthIPLDResolver) resolveReceipts(blocks []blocks.Block, response *streamer.SuperNodePayload) {
|
||||
for _, block := range blocks {
|
||||
raw := block.RawData()
|
||||
response.ReceiptsRlp = append(response.ReceiptsRlp, raw)
|
||||
func (eir *EthIPLDResolver) ResolveTransactions(iplds []blocks.Block) [][]byte {
|
||||
trxs := make([][]byte, 0, len(iplds))
|
||||
for _, ipld := range iplds {
|
||||
trxs = append(trxs, ipld.RawData())
|
||||
}
|
||||
return trxs
|
||||
}
|
||||
|
||||
func (eir *EthIPLDResolver) resolveState(blocks map[common.Hash]blocks.Block, response *streamer.SuperNodePayload) {
|
||||
for key, block := range blocks {
|
||||
raw := block.RawData()
|
||||
response.StateNodesRlp[key] = raw
|
||||
func (eir *EthIPLDResolver) ResolveReceipts(iplds []blocks.Block) [][]byte {
|
||||
rcts := make([][]byte, 0, len(iplds))
|
||||
for _, ipld := range iplds {
|
||||
rcts = append(rcts, ipld.RawData())
|
||||
}
|
||||
return rcts
|
||||
}
|
||||
|
||||
func (eir *EthIPLDResolver) resolveStorage(blocks map[common.Hash]map[common.Hash]blocks.Block, response *streamer.SuperNodePayload) {
|
||||
for stateKey, storageBlocks := range blocks {
|
||||
response.StorageNodesRlp[stateKey] = make(map[common.Hash][]byte)
|
||||
for storageKey, storageVal := range storageBlocks {
|
||||
raw := storageVal.RawData()
|
||||
response.StorageNodesRlp[stateKey][storageKey] = raw
|
||||
func (eir *EthIPLDResolver) ResolveState(iplds map[common.Hash]blocks.Block) map[common.Hash][]byte {
|
||||
stateNodes := make(map[common.Hash][]byte, len(iplds))
|
||||
for key, ipld := range iplds {
|
||||
stateNodes[key] = ipld.RawData()
|
||||
}
|
||||
return stateNodes
|
||||
}
|
||||
|
||||
func (eir *EthIPLDResolver) ResolveStorage(iplds map[common.Hash]map[common.Hash]blocks.Block) map[common.Hash]map[common.Hash][]byte {
|
||||
storageNodes := make(map[common.Hash]map[common.Hash][]byte)
|
||||
for stateKey, storageIPLDs := range iplds {
|
||||
storageNodes[stateKey] = make(map[common.Hash][]byte)
|
||||
for storageKey, storageVal := range storageIPLDs {
|
||||
storageNodes[stateKey][storageKey] = storageVal.RawData()
|
||||
}
|
||||
}
|
||||
return storageNodes
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user