cerc-io/ipld-eth-server
12
0
Fork 1
Code Issues 10 Pull Requests 2 Actions Packages Projects Releases 37 Wiki Activity

review fixes

Browse Source
This commit is contained in:
Ian Norden
2019-12-02 13:24:58 -06:00
parent 2244d1869f
commit 0bbb7a30d1
34 changed files with 253 additions and 469 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/ipfs/api.go
-83
View File
@@ -1,83 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipfs
import (
"context"
"github.com/ethereum/go-ethereum/rpc"
log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/config"
)
// APIName is the namespace used for the state diffing service API
const APIName = "vulcanizedb"
// APIVersion is the version of the state diffing service API
const APIVersion = "0.0.1"
// PublicSeedNodeAPI is the public api for the seed node
type PublicSeedNodeAPI struct {
snp SyncPublishScreenAndServe
}
// NewPublicSeedNodeAPI creates a new PublicSeedNodeAPI with the provided underlying SyncPublishScreenAndServe process
func NewPublicSeedNodeAPI(snp SyncPublishScreenAndServe) *PublicSeedNodeAPI {
return &PublicSeedNodeAPI{
snp: snp,
}
}
// Stream is the public method to setup a subscription that fires off SyncPublishScreenAndServe payloads as they are created
func (api *PublicSeedNodeAPI) Stream(ctx context.Context, streamFilters config.Subscription) (*rpc.Subscription, error) {
// ensure that the RPC connection supports subscriptions
notifier, supported := rpc.NotifierFromContext(ctx)
if !supported {
return nil, rpc.ErrNotificationsUnsupported
}
// create subscription and start waiting for statediff events
rpcSub := notifier.CreateSubscription()
go func() {
// subscribe to events from the SyncPublishScreenAndServe service
payloadChannel := make(chan ResponsePayload, payloadChanBufferSize)
quitChan := make(chan bool, 1)
go api.snp.Subscribe(rpcSub.ID, payloadChannel, quitChan, streamFilters)
// loop and await state diff payloads and relay them to the subscriber with then notifier
for {
select {
case packet := <-payloadChannel:
if notifyErr := notifier.Notify(rpcSub.ID, packet); notifyErr != nil {
log.Error("Failed to send state diff packet", "err", notifyErr)
api.snp.Unsubscribe(rpcSub.ID)
return
}
case <-rpcSub.Err():
api.snp.Unsubscribe(rpcSub.ID)
return
case <-quitChan:
// don't need to unsubscribe, SyncPublishScreenAndServe service does so before sending the quit signal
return
}
}
}()
return rpcSub, nil
}
pkg/ipfs/converter.go
+4 -1
View File
@@ -49,6 +49,9 @@ func (pc *Converter) Convert(payload statediff.Payload) (*IPLDPayload, error) {
// Unpack block rlp to access fields
block := new(types.Block)
err := rlp.DecodeBytes(payload.BlockRlp, block)
if err != nil {
return nil, err
}
header := block.Header()
headerRlp, err := rlp.EncodeToBytes(header)
if err != nil {
@@ -74,7 +77,7 @@ func (pc *Converter) Convert(payload statediff.Payload) (*IPLDPayload, error) {
}
txMeta := &TrxMetaData{
Dst: handleNullAddr(trx.To()),
Src: from.Hex(),
Src: handleNullAddr(&from),
}
// txMeta will have same index as its corresponding trx in the convertedPayload.BlockBody
convertedPayload.TrxMetaData = append(convertedPayload.TrxMetaData, txMeta)
pkg/ipfs/converter_test.go
-37
View File
@@ -1,37 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipfs_test
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks"
)
var _ = Describe("Converter", func() {
Describe("Convert", func() {
It("Converts StatediffPayloads into IPLDPayloads", func() {
mockConverter := mocks.PayloadConverter{}
mockConverter.ReturnIPLDPayload = &mocks.MockIPLDPayload
ipldPayload, err := mockConverter.Convert(mocks.MockStatediffPayload)
Expect(err).ToNot(HaveOccurred())
Expect(ipldPayload).To(Equal(&mocks.MockIPLDPayload))
Expect(mockConverter.PassedStatediffPayload).To(Equal(mocks.MockStatediffPayload))
})
})
})
pkg/ipfs/fetcher.go
+10 -10
View File
@@ -28,7 +28,7 @@ import (
// IPLDFetcher is an interface for fetching IPLDs
type IPLDFetcher interface {
FetchCIDs(cids CidWrapper) (*IpldWrapper, error)
FetchCIDs(cids CIDWrapper) (*IPLDWrapper, error)
}
// EthIPLDFetcher is used to fetch ETH IPLD objects from IPFS
@@ -47,11 +47,11 @@ func NewIPLDFetcher(ipfsPath string) (*EthIPLDFetcher, error) {
}, nil
}
// FetchCIDs is the exported method for fetching and returning all the cids passed in a CidWrapper
func (f *EthIPLDFetcher) FetchCIDs(cids CidWrapper) (*IpldWrapper, error) {
// FetchCIDs is the exported method for fetching and returning all the cids passed in a CIDWrapper
func (f *EthIPLDFetcher) FetchCIDs(cids CIDWrapper) (*IPLDWrapper, error) {
log.Debug("fetching iplds")
blocks := &IpldWrapper{
blocks := &IPLDWrapper{
BlockNumber: cids.BlockNumber,
Headers: make([]blocks.Block, 0),
Uncles: make([]blocks.Block, 0),
@@ -91,7 +91,7 @@ func (f *EthIPLDFetcher) FetchCIDs(cids CidWrapper) (*IpldWrapper, error) {
// fetchHeaders fetches headers
// It uses the f.fetchBatch method
func (f *EthIPLDFetcher) fetchHeaders(cids CidWrapper, blocks *IpldWrapper) error {
func (f *EthIPLDFetcher) fetchHeaders(cids CIDWrapper, blocks *IPLDWrapper) error {
log.Debug("fetching header iplds")
headerCids := make([]cid.Cid, 0, len(cids.Headers))
for _, c := range cids.Headers {
@@ -110,7 +110,7 @@ func (f *EthIPLDFetcher) fetchHeaders(cids CidWrapper, blocks *IpldWrapper) erro
// fetchUncles fetches uncles
// It uses the f.fetchBatch method
func (f *EthIPLDFetcher) fetchUncles(cids CidWrapper, blocks *IpldWrapper) error {
func (f *EthIPLDFetcher) fetchUncles(cids CIDWrapper, blocks *IPLDWrapper) error {
log.Debug("fetching uncle iplds")
uncleCids := make([]cid.Cid, 0, len(cids.Uncles))
for _, c := range cids.Uncles {
@@ -129,7 +129,7 @@ func (f *EthIPLDFetcher) fetchUncles(cids CidWrapper, blocks *IpldWrapper) error
// fetchTrxs fetches transactions
// It uses the f.fetchBatch method
func (f *EthIPLDFetcher) fetchTrxs(cids CidWrapper, blocks *IpldWrapper) error {
func (f *EthIPLDFetcher) fetchTrxs(cids CIDWrapper, blocks *IPLDWrapper) error {
log.Debug("fetching transaction iplds")
trxCids := make([]cid.Cid, 0, len(cids.Transactions))
for _, c := range cids.Transactions {
@@ -148,7 +148,7 @@ func (f *EthIPLDFetcher) fetchTrxs(cids CidWrapper, blocks *IpldWrapper) error {
// fetchRcts fetches receipts
// It uses the f.fetchBatch method
func (f *EthIPLDFetcher) fetchRcts(cids CidWrapper, blocks *IpldWrapper) error {
func (f *EthIPLDFetcher) fetchRcts(cids CIDWrapper, blocks *IPLDWrapper) error {
log.Debug("fetching receipt iplds")
rctCids := make([]cid.Cid, 0, len(cids.Receipts))
for _, c := range cids.Receipts {
@@ -168,7 +168,7 @@ func (f *EthIPLDFetcher) fetchRcts(cids CidWrapper, blocks *IpldWrapper) error {
// fetchState fetches state nodes
// It uses the single f.fetch method instead of the batch fetch, because it
// needs to maintain the data's relation to state keys
func (f *EthIPLDFetcher) fetchState(cids CidWrapper, blocks *IpldWrapper) error {
func (f *EthIPLDFetcher) fetchState(cids CIDWrapper, blocks *IPLDWrapper) error {
log.Debug("fetching state iplds")
for _, stateNode := range cids.StateNodes {
if stateNode.CID == "" || stateNode.Key == "" {
@@ -190,7 +190,7 @@ func (f *EthIPLDFetcher) fetchState(cids CidWrapper, blocks *IpldWrapper) error
// fetchStorage fetches storage nodes
// It uses the single f.fetch method instead of the batch fetch, because it
// needs to maintain the data's relation to state and storage keys
func (f *EthIPLDFetcher) fetchStorage(cids CidWrapper, blocks *IpldWrapper) error {
func (f *EthIPLDFetcher) fetchStorage(cids CIDWrapper, blocks *IPLDWrapper) error {
log.Debug("fetching storage iplds")
for _, storageNode := range cids.StorageNodes {
if storageNode.CID == "" || storageNode.Key == "" || storageNode.StateKey == "" {
pkg/ipfs/fetcher_test.go
-17
View File
@@ -1,17 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipfs
pkg/ipfs/helpers.go
+3 -2
View File
@@ -55,8 +55,9 @@ func HexToKey(hex string) common.Hash {
return crypto.Keccak256Hash(addr[:])
}
func emptyCidWrapper(cids CidWrapper) bool {
if len(cids.Transactions) > 0 || len(cids.Headers) > 0 || len(cids.Uncles) > 0 || len(cids.Receipts) > 0 || len(cids.StateNodes) > 0 || len(cids.StorageNodes) > 0 || cids.BlockNumber == nil {
// EmptyCIDWrapper returns whether or not the provided CIDWrapper has any Cids we need to process
func EmptyCIDWrapper(cids CIDWrapper) bool {
if len(cids.Transactions) > 0 || len(cids.Headers) > 0 || len(cids.Uncles) > 0 || len(cids.Receipts) > 0 || len(cids.StateNodes) > 0 || len(cids.StorageNodes) > 0 {
return false
}
return true
pkg/ipfs/ipfs_suite_test.go
-36
View File
@@ -1,36 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipfs_test
import (
"io/ioutil"
"testing"
"github.com/sirupsen/logrus"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
func TestIPFS(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "IPFS Suite Test")
}
var _ = BeforeSuite(func() {
logrus.SetOutput(ioutil.Discard)
})
pkg/ipfs/mocks/api.go
-1
View File
@@ -1 +0,0 @@
package mocks
pkg/ipfs/mocks/repository.go
-31
View File
@@ -1,31 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package mocks
import "github.com/vulcanize/vulcanizedb/pkg/ipfs"
// CIDRepository is the underlying struct for the Repository interface
type CIDRepository struct {
PassedCIDPayload *ipfs.CIDPayload
ReturnErr error
}
// Index indexes a cidPayload in Postgres
func (repo *CIDRepository) Index(cidPayload *ipfs.CIDPayload) error {
repo.PassedCIDPayload = cidPayload
return repo.ReturnErr
}
pkg/ipfs/mocks/screener.go
-1
View File
@@ -1 +0,0 @@
package mocks
pkg/ipfs/mocks/streamer.go
-43
View File
@@ -1,43 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package mocks
import (
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/statediff"
)
// StateDiffStreamer is the underlying struct for the Streamer interface
type StateDiffStreamer struct {
PassedPayloadChan chan statediff.Payload
ReturnSub *rpc.ClientSubscription
ReturnErr error
StreamPayloads []statediff.Payload
}
// Stream is the main loop for subscribing to data from the Geth state diff process
func (sds *StateDiffStreamer) Stream(payloadChan chan statediff.Payload) (*rpc.ClientSubscription, error) {
sds.PassedPayloadChan = payloadChan
go func() {
for _, payload := range sds.StreamPayloads {
sds.PassedPayloadChan <- payload
}
}()
return sds.ReturnSub, sds.ReturnErr
}
pkg/ipfs/publisher.go
+4 -4
View File
@@ -21,7 +21,7 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
rlp2 "github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ipfs/go-ipfs/plugin/loader"
"github.com/vulcanize/eth-block-extractor/pkg/ipfs"
@@ -30,7 +30,7 @@ import (
"github.com/vulcanize/eth-block-extractor/pkg/ipfs/eth_block_transactions"
"github.com/vulcanize/eth-block-extractor/pkg/ipfs/eth_state_trie"
"github.com/vulcanize/eth-block-extractor/pkg/ipfs/eth_storage_trie"
"github.com/vulcanize/eth-block-extractor/pkg/wrappers/rlp"
rlp2 "github.com/vulcanize/eth-block-extractor/pkg/wrappers/rlp"
)
// IPLDPublisher is the interface for publishing an IPLD payload
@@ -66,7 +66,7 @@ func NewIPLDPublisher(ipfsPath string) (*Publisher, error) {
return nil, err
}
return &Publisher{
HeaderPutter: eth_block_header.NewBlockHeaderDagPutter(node, rlp.RlpDecoder{}),
HeaderPutter: eth_block_header.NewBlockHeaderDagPutter(node, rlp2.RlpDecoder{}),
TransactionPutter: eth_block_transactions.NewBlockTransactionsDagPutter(node),
ReceiptPutter: eth_block_receipts.NewEthBlockReceiptDagPutter(node),
StatePutter: eth_state_trie.NewStateTrieDagPutter(node),
@@ -85,7 +85,7 @@ func (pub *Publisher) Publish(payload *IPLDPayload) (*CIDPayload, error) {
// Process and publish uncles
uncleCids := make(map[common.Hash]string)
for _, uncle := range payload.BlockBody.Uncles {
uncleRlp, err := rlp2.EncodeToBytes(uncle)
uncleRlp, err := rlp.EncodeToBytes(uncle)
if err != nil {
return nil, err
}
pkg/ipfs/publisher_test.go
-37
View File
@@ -1,37 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipfs_test
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks"
)
var _ = Describe("Publisher", func() {
Describe("Publish", func() {
It("Publishes IPLDPayload to IPFS", func() {
mockPublisher := mocks.IPLDPublisher{}
mockPublisher.ReturnCIDPayload = &mocks.MockCIDPayload
cidPayload, err := mockPublisher.Publish(&mocks.MockIPLDPayload)
Expect(err).ToNot(HaveOccurred())
Expect(cidPayload).To(Equal(&mocks.MockCIDPayload))
Expect(mockPublisher.PassedIPLDPayload).To(Equal(&mocks.MockIPLDPayload))
})
})
})
pkg/ipfs/repository.go
-142
View File
@@ -1,142 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipfs
import (
"github.com/jmoiron/sqlx"
"github.com/lib/pq"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
// CIDRepository is an interface for indexing CIDPayloads
type CIDRepository interface {
Index(cidPayload *CIDPayload) error
}
// Repository is the underlying struct for the CIDRepository interface
type Repository struct {
db *postgres.DB
}
// NewCIDRepository creates a new pointer to a Repository which satisfies the CIDRepository interface
func NewCIDRepository(db *postgres.DB) *Repository {
return &Repository{
db: db,
}
}
// Index indexes a cidPayload in Postgres
func (repo *Repository) Index(cidPayload *CIDPayload) error {
tx, err := repo.db.Beginx()
if err != nil {
return err
}
headerID, err := repo.indexHeaderCID(tx, cidPayload.HeaderCID, cidPayload.BlockNumber, cidPayload.BlockHash.Hex())
if err != nil {
tx.Rollback()
return err
}
for uncleHash, cid := range cidPayload.UncleCIDS {
err = repo.indexUncleCID(tx, cid, cidPayload.BlockNumber, uncleHash.Hex())
if err != nil {
tx.Rollback()
return err
}
}
err = repo.indexTransactionAndReceiptCIDs(tx, cidPayload, headerID)
if err != nil {
tx.Rollback()
return err
}
err = repo.indexStateAndStorageCIDs(tx, cidPayload, headerID)
if err != nil {
tx.Rollback()
return err
}
return tx.Commit()
}
func (repo *Repository) indexHeaderCID(tx *sqlx.Tx, cid, blockNumber, hash string) (int64, error) {
var headerID int64
err := tx.QueryRowx(`INSERT INTO public.header_cids (block_number, block_hash, cid, final) VALUES ($1, $2, $3, $4)
ON CONFLICT (block_number, block_hash) DO UPDATE SET (cid, final) = ($3, $4)
RETURNING id`,
blockNumber, hash, cid, true).Scan(&headerID)
return headerID, err
}
func (repo *Repository) indexUncleCID(tx *sqlx.Tx, cid, blockNumber, hash string) error {
_, err := tx.Exec(`INSERT INTO public.header_cids (block_number, block_hash, cid, final) VALUES ($1, $2, $3, $4)
ON CONFLICT (block_number, block_hash) DO UPDATE SET (cid, final) = ($3, $4)`,
blockNumber, hash, cid, false)
return err
}
func (repo *Repository) indexTransactionAndReceiptCIDs(tx *sqlx.Tx, payload *CIDPayload, headerID int64) error {
for hash, trxCidMeta := range payload.TransactionCIDs {
var txID int64
err := tx.QueryRowx(`INSERT INTO public.transaction_cids (header_id, tx_hash, cid, dst, src) VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (header_id, tx_hash) DO UPDATE SET (cid, dst, src) = ($3, $4, $5)
RETURNING id`,
headerID, hash.Hex(), trxCidMeta.CID, trxCidMeta.Dst, trxCidMeta.Src).Scan(&txID)
if err != nil {
return err
}
receiptCidMeta, ok := payload.ReceiptCIDs[hash]
if ok {
err = repo.indexReceiptCID(tx, receiptCidMeta, txID)
if err != nil {
return err
}
}
}
return nil
}
func (repo *Repository) indexReceiptCID(tx *sqlx.Tx, cidMeta *ReceiptMetaData, txID int64) error {
_, err := tx.Exec(`INSERT INTO public.receipt_cids (tx_id, cid, contract, topic0s) VALUES ($1, $2, $3, $4)`,
txID, cidMeta.CID, cidMeta.ContractAddress, pq.Array(cidMeta.Topic0s))
return err
}
func (repo *Repository) indexStateAndStorageCIDs(tx *sqlx.Tx, payload *CIDPayload, headerID int64) error {
for accountKey, stateCID := range payload.StateNodeCIDs {
var stateID int64
err := tx.QueryRowx(`INSERT INTO public.state_cids (header_id, state_key, cid, leaf) VALUES ($1, $2, $3, $4)
ON CONFLICT (header_id, state_key) DO UPDATE SET (cid, leaf) = ($3, $4)
RETURNING id`,
headerID, accountKey.Hex(), stateCID.CID, stateCID.Leaf).Scan(&stateID)
if err != nil {
return err
}
for _, storageCID := range payload.StorageNodeCIDs[accountKey] {
err = repo.indexStorageCID(tx, storageCID, stateID)
if err != nil {
return err
}
}
}
return nil
}
func (repo *Repository) indexStorageCID(tx *sqlx.Tx, storageCID StorageNodeCID, stateID int64) error {
_, err := tx.Exec(`INSERT INTO public.storage_cids (state_id, storage_key, cid, leaf) VALUES ($1, $2, $3, $4)
ON CONFLICT (state_id, storage_key) DO UPDATE SET (cid, leaf) = ($3, $4)`,
stateID, storageCID.Key, storageCID.CID, storageCID.Leaf)
return err
}
pkg/ipfs/repository_test.go
-35
View File
@@ -1,35 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipfs_test
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks"
)
var _ = Describe("Repository", func() {
Describe("Index", func() {
It("Indexes CIDs against their metadata", func() {
mockRepo := mocks.CIDRepository{}
err := mockRepo.Index(&mocks.MockCIDPayload)
Expect(err).ToNot(HaveOccurred())
Expect(mockRepo.PassedCIDPayload).To(Equal(&mocks.MockCIDPayload))
})
})
})
pkg/ipfs/resolver.go
+10 -9
View File
@@ -19,11 +19,12 @@ package ipfs
import (
"github.com/ethereum/go-ethereum/common"
"github.com/ipfs/go-block-format"
"github.com/vulcanize/vulcanizedb/libraries/shared/streamer"
)
// IPLDResolver is the interface to resolving IPLDs
type IPLDResolver interface {
ResolveIPLDs(ipfsBlocks IpldWrapper) (*ResponsePayload, error)
ResolveIPLDs(ipfsBlocks IPLDWrapper) (*streamer.SeedNodePayload, error)
}
// EthIPLDResolver is the underlying struct to support the IPLDResolver interface
@@ -35,8 +36,8 @@ func NewIPLDResolver() *EthIPLDResolver {
}
// ResolveIPLDs is the exported method for resolving all of the ETH IPLDs packaged in an IpfsBlockWrapper
func (eir *EthIPLDResolver) ResolveIPLDs(ipfsBlocks IpldWrapper) (*ResponsePayload, error) {
response := new(ResponsePayload)
func (eir *EthIPLDResolver) ResolveIPLDs(ipfsBlocks IPLDWrapper) (*streamer.SeedNodePayload, error) {
response := new(streamer.SeedNodePayload)
response.BlockNumber = ipfsBlocks.BlockNumber
eir.resolveHeaders(ipfsBlocks.Headers, response)
eir.resolveUncles(ipfsBlocks.Uncles, response)
@@ -47,35 +48,35 @@ func (eir *EthIPLDResolver) ResolveIPLDs(ipfsBlocks IpldWrapper) (*ResponsePaylo
return response, nil
}
func (eir *EthIPLDResolver) resolveHeaders(blocks []blocks.Block, response *ResponsePayload) {
func (eir *EthIPLDResolver) resolveHeaders(blocks []blocks.Block, response *streamer.SeedNodePayload) {
for _, block := range blocks {
raw := block.RawData()
response.HeadersRlp = append(response.HeadersRlp, raw)
}
}
func (eir *EthIPLDResolver) resolveUncles(blocks []blocks.Block, response *ResponsePayload) {
func (eir *EthIPLDResolver) resolveUncles(blocks []blocks.Block, response *streamer.SeedNodePayload) {
for _, block := range blocks {
raw := block.RawData()
response.UnclesRlp = append(response.UnclesRlp, raw)
}
}
func (eir *EthIPLDResolver) resolveTransactions(blocks []blocks.Block, response *ResponsePayload) {
func (eir *EthIPLDResolver) resolveTransactions(blocks []blocks.Block, response *streamer.SeedNodePayload) {
for _, block := range blocks {
raw := block.RawData()
response.TransactionsRlp = append(response.TransactionsRlp, raw)
}
}
func (eir *EthIPLDResolver) resolveReceipts(blocks []blocks.Block, response *ResponsePayload) {
func (eir *EthIPLDResolver) resolveReceipts(blocks []blocks.Block, response *streamer.SeedNodePayload) {
for _, block := range blocks {
raw := block.RawData()
response.ReceiptsRlp = append(response.ReceiptsRlp, raw)
}
}
func (eir *EthIPLDResolver) resolveState(blocks map[common.Hash]blocks.Block, response *ResponsePayload) {
func (eir *EthIPLDResolver) resolveState(blocks map[common.Hash]blocks.Block, response *streamer.SeedNodePayload) {
if response.StateNodesRlp == nil {
response.StateNodesRlp = make(map[common.Hash][]byte)
}
@@ -85,7 +86,7 @@ func (eir *EthIPLDResolver) resolveState(blocks map[common.Hash]blocks.Block, re
}
}
func (eir *EthIPLDResolver) resolveStorage(blocks map[common.Hash]map[common.Hash]blocks.Block, response *ResponsePayload) {
func (eir *EthIPLDResolver) resolveStorage(blocks map[common.Hash]map[common.Hash]blocks.Block, response *streamer.SeedNodePayload) {
if response.StateNodesRlp == nil {
response.StorageNodesRlp = make(map[common.Hash]map[common.Hash][]byte)
}
pkg/ipfs/retreiver.go
-265
View File
@@ -1,265 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipfs
import (
"math/big"
"github.com/jmoiron/sqlx"
"github.com/lib/pq"
log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
// CIDRetriever is the interface for retrieving CIDs from the Postgres cache
type CIDRetriever interface {
RetrieveCIDs(streamFilters config.Subscription, blockNumber int64) (*CidWrapper, error)
RetrieveLastBlockNumber() (int64, error)
RetrieveFirstBlockNumber() (int64, error)
}
// EthCIDRetriever is the underlying struct supporting the CIDRetriever interface
type EthCIDRetriever struct {
db *postgres.DB
}
// NewCIDRetriever returns a pointer to a new EthCIDRetriever which supports the CIDRetriever interface
func NewCIDRetriever(db *postgres.DB) *EthCIDRetriever {
return &EthCIDRetriever{
db: db,
}
}
func (ecr *EthCIDRetriever) RetrieveFirstBlockNumber() (int64, error) {
var blockNumber int64
err := ecr.db.Get(&blockNumber, "SELECT block_number FROM header_cids ORDER BY block_number ASC LIMIT 1")
return blockNumber, err
}
// GetLastBlockNumber is used to retrieve the latest block number in the cache
func (ecr *EthCIDRetriever) RetrieveLastBlockNumber() (int64, error) {
var blockNumber int64
err := ecr.db.Get(&blockNumber, "SELECT block_number FROM header_cids ORDER BY block_number DESC LIMIT 1 ")
return blockNumber, err
}
// RetrieveCIDs is used to retrieve all of the CIDs which conform to the passed StreamFilters
func (ecr *EthCIDRetriever) RetrieveCIDs(streamFilters config.Subscription, blockNumber int64) (*CidWrapper, error) {
log.Debug("retrieving cids")
var err error
tx, err := ecr.db.Beginx()
if err != nil {
return nil, err
}
// THIS IS SUPER EXPENSIVE HAVING TO CYCLE THROUGH EACH BLOCK, NEED BETTER WAY TO FETCH CIDS
// WHILE STILL MAINTAINING RELATION INFO ABOUT WHAT BLOCK THE CIDS BELONG TO
cw := new(CidWrapper)
cw.BlockNumber = big.NewInt(blockNumber)
// Retrieve cached header CIDs
if !streamFilters.HeaderFilter.Off {
cw.Headers, err = ecr.retrieveHeaderCIDs(tx, streamFilters, blockNumber)
if err != nil {
tx.Rollback()
log.Error("header cid retrieval error")
return nil, err
}
if !streamFilters.HeaderFilter.FinalOnly {
cw.Uncles, err = ecr.retrieveUncleCIDs(tx, streamFilters, blockNumber)
if err != nil {
tx.Rollback()
log.Error("header cid retrieval error")
return nil, err
}
}
}
// Retrieve cached trx CIDs
var trxIds []int64
if !streamFilters.TrxFilter.Off {
cw.Transactions, trxIds, err = ecr.retrieveTrxCIDs(tx, streamFilters, blockNumber)
if err != nil {
tx.Rollback()
log.Error("transaction cid retrieval error")
return nil, err
}
}
// Retrieve cached receipt CIDs
if !streamFilters.ReceiptFilter.Off {
cw.Receipts, err = ecr.retrieveRctCIDs(tx, streamFilters, blockNumber, trxIds)
if err != nil {
tx.Rollback()
log.Error("receipt cid retrieval error")
return nil, err
}
}
// Retrieve cached state CIDs
if !streamFilters.StateFilter.Off {
cw.StateNodes, err = ecr.retrieveStateCIDs(tx, streamFilters, blockNumber)
if err != nil {
tx.Rollback()
log.Error("state cid retrieval error")
return nil, err
}
}
// Retrieve cached storage CIDs
if !streamFilters.StorageFilter.Off {
cw.StorageNodes, err = ecr.retrieveStorageCIDs(tx, streamFilters, blockNumber)
if err != nil {
tx.Rollback()
log.Error("storage cid retrieval error")
return nil, err
}
}
return cw, tx.Commit()
}
func (ecr *EthCIDRetriever) retrieveHeaderCIDs(tx *sqlx.Tx, streamFilters config.Subscription, blockNumber int64) ([]string, error) {
log.Debug("retrieving header cids for block ", blockNumber)
headers := make([]string, 0)
pgStr := `SELECT cid FROM header_cids
WHERE block_number = $1 AND final IS TRUE`
err := tx.Select(&headers, pgStr, blockNumber)
return headers, err
}
func (ecr *EthCIDRetriever) retrieveUncleCIDs(tx *sqlx.Tx, streamFilters config.Subscription, blockNumber int64) ([]string, error) {
log.Debug("retrieving header cids for block ", blockNumber)
headers := make([]string, 0)
pgStr := `SELECT cid FROM header_cids
WHERE block_number = $1 AND final IS FALSE`
err := tx.Select(&headers, pgStr, blockNumber)
return headers, err
}
func (ecr *EthCIDRetriever) retrieveTrxCIDs(tx *sqlx.Tx, streamFilters config.Subscription, blockNumber int64) ([]string, []int64, error) {
log.Debug("retrieving transaction cids for block ", blockNumber)
args := make([]interface{}, 0, 3)
type result struct {
ID int64 `db:"id"`
Cid string `db:"cid"`
}
results := make([]result, 0)
pgStr := `SELECT transaction_cids.id, transaction_cids.cid FROM transaction_cids INNER JOIN header_cids ON (transaction_cids.header_id = header_cids.id)
WHERE header_cids.block_number = $1`
args = append(args, blockNumber)
if len(streamFilters.TrxFilter.Dst) > 0 {
pgStr += ` AND transaction_cids.dst = ANY($2::VARCHAR(66)[])`
args = append(args, pq.Array(streamFilters.TrxFilter.Dst))
}
if len(streamFilters.TrxFilter.Src) > 0 {
pgStr += ` AND transaction_cids.src = ANY($3::VARCHAR(66)[])`
args = append(args, pq.Array(streamFilters.TrxFilter.Src))
}
err := tx.Select(&results, pgStr, args...)
if err != nil {
return nil, nil, err
}
ids := make([]int64, 0, len(results))
cids := make([]string, 0, len(results))
for _, res := range results {
cids = append(cids, res.Cid)
ids = append(ids, res.ID)
}
return cids, ids, nil
}
func (ecr *EthCIDRetriever) retrieveRctCIDs(tx *sqlx.Tx, streamFilters config.Subscription, blockNumber int64, trxIds []int64) ([]string, error) {
log.Debug("retrieving receipt cids for block ", blockNumber)
args := make([]interface{}, 0, 2)
pgStr := `SELECT receipt_cids.cid FROM receipt_cids, transaction_cids, header_cids
WHERE receipt_cids.tx_id = transaction_cids.id
AND transaction_cids.header_id = header_cids.id
AND header_cids.block_number = $1`
args = append(args, blockNumber)
if len(streamFilters.ReceiptFilter.Topic0s) > 0 {
pgStr += ` AND ((receipt_cids.topic0s && $2::VARCHAR(66)[]`
args = append(args, pq.Array(streamFilters.ReceiptFilter.Topic0s))
}
if len(streamFilters.ReceiptFilter.Contracts) > 0 {
pgStr += ` AND receipt_cids.contract = ANY($3::VARCHAR(66)[])`
} else {
pgStr += `)`
}
if len(trxIds) > 0 {
pgStr += ` OR receipt_cids.tx_id = ANY($4::INTEGER[]))`
args = append(args, pq.Array(trxIds))
} else {
pgStr += `)`
}
receiptCids := make([]string, 0)
err := tx.Select(&receiptCids, pgStr, args...)
return receiptCids, err
}
func (ecr *EthCIDRetriever) retrieveStateCIDs(tx *sqlx.Tx, streamFilters config.Subscription, blockNumber int64) ([]StateNodeCID, error) {
log.Debug("retrieving state cids for block ", blockNumber)
args := make([]interface{}, 0, 2)
pgStr := `SELECT state_cids.cid, state_cids.state_key FROM state_cids INNER JOIN header_cids ON (state_cids.header_id = header_cids.id)
WHERE header_cids.block_number = $1`
args = append(args, blockNumber)
addrLen := len(streamFilters.StateFilter.Addresses)
if addrLen > 0 {
keys := make([]string, 0, addrLen)
for _, addr := range streamFilters.StateFilter.Addresses {
keys = append(keys, HexToKey(addr).Hex())
}
pgStr += ` AND state_cids.state_key = ANY($2::VARCHAR(66)[])`
args = append(args, pq.Array(keys))
}
if !streamFilters.StorageFilter.IntermediateNodes {
pgStr += ` AND state_cids.leaf = TRUE`
}
stateNodeCIDs := make([]StateNodeCID, 0)
err := tx.Select(&stateNodeCIDs, pgStr, args...)
return stateNodeCIDs, err
}
func (ecr *EthCIDRetriever) retrieveStorageCIDs(tx *sqlx.Tx, streamFilters config.Subscription, blockNumber int64) ([]StorageNodeCID, error) {
log.Debug("retrieving storage cids for block ", blockNumber)
args := make([]interface{}, 0, 3)
pgStr := `SELECT storage_cids.cid, state_cids.state_key, storage_cids.storage_key FROM storage_cids, state_cids, header_cids
WHERE storage_cids.state_id = state_cids.id
AND state_cids.header_id = header_cids.id
AND header_cids.block_number = $1`
args = append(args, blockNumber)
addrLen := len(streamFilters.StorageFilter.Addresses)
if addrLen > 0 {
keys := make([]string, 0, addrLen)
for _, addr := range streamFilters.StorageFilter.Addresses {
keys = append(keys, HexToKey(addr).Hex())
}
pgStr += ` AND state_cids.state_key = ANY($2::VARCHAR(66)[])`
args = append(args, pq.Array(keys))
}
if len(streamFilters.StorageFilter.StorageKeys) > 0 {
pgStr += ` AND storage_cids.storage_key = ANY($3::VARCHAR(66)[])`
args = append(args, pq.Array(streamFilters.StorageFilter.StorageKeys))
}
if !streamFilters.StorageFilter.IntermediateNodes {
pgStr += ` AND storage_cids.leaf = TRUE`
}
storageNodeCIDs := make([]StorageNodeCID, 0)
err := tx.Select(&storageNodeCIDs, pgStr, args...)
return storageNodeCIDs, err
}
pkg/ipfs/retreiver_test.go
-1
View File
@@ -1 +0,0 @@
package ipfs
pkg/ipfs/screener.go
-241
View File
@@ -1,241 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipfs
import (
"bytes"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/rlp"
"github.com/vulcanize/vulcanizedb/pkg/config"
)
// ResponseScreener is the inteface used to screen eth data and package appropriate data into a response payload
type ResponseScreener interface {
ScreenResponse(streamFilters config.Subscription, payload IPLDPayload) (*ResponsePayload, error)
}
// Screener is the underlying struct for the ReponseScreener interface
type Screener struct{}
// NewResponseScreener creates a new Screener satisfyign the ReponseScreener interface
func NewResponseScreener() *Screener {
return &Screener{}
}
// ScreenResponse is used to filter through eth data to extract and package requested data into a ResponsePayload
func (s *Screener) ScreenResponse(streamFilters config.Subscription, payload IPLDPayload) (*ResponsePayload, error) {
response := new(ResponsePayload)
err := s.filterHeaders(streamFilters, response, payload)
if err != nil {
return nil, err
}
txHashes, err := s.filterTransactions(streamFilters, response, payload)
if err != nil {
return nil, err
}
err = s.filerReceipts(streamFilters, response, payload, txHashes)
if err != nil {
return nil, err
}
err = s.filterState(streamFilters, response, payload)
if err != nil {
return nil, err
}
err = s.filterStorage(streamFilters, response, payload)
if err != nil {
return nil, err
}
response.BlockNumber = payload.BlockNumber
return response, nil
}
func (s *Screener) filterHeaders(streamFilters config.Subscription, response *ResponsePayload, payload IPLDPayload) error {
if !streamFilters.HeaderFilter.Off && checkRange(streamFilters.StartingBlock.Int64(), streamFilters.EndingBlock.Int64(), payload.BlockNumber.Int64()) {
response.HeadersRlp = append(response.HeadersRlp, payload.HeaderRLP)
if !streamFilters.HeaderFilter.FinalOnly {
for _, uncle := range payload.BlockBody.Uncles {
uncleRlp, err := rlp.EncodeToBytes(uncle)
if err != nil {
return err
}
response.UnclesRlp = append(response.UnclesRlp, uncleRlp)
}
}
}
return nil
}
func checkRange(start, end, actual int64) bool {
if (end <= 0 || end >= actual) && start <= actual {
return true
}
return false
}
func (s *Screener) filterTransactions(streamFilters config.Subscription, response *ResponsePayload, payload IPLDPayload) ([]common.Hash, error) {
trxHashes := make([]common.Hash, 0, len(payload.BlockBody.Transactions))
if !streamFilters.TrxFilter.Off && checkRange(streamFilters.StartingBlock.Int64(), streamFilters.EndingBlock.Int64(), payload.BlockNumber.Int64()) {
for i, trx := range payload.BlockBody.Transactions {
if checkTransactions(streamFilters.TrxFilter.Src, streamFilters.TrxFilter.Dst, payload.TrxMetaData[i].Src, payload.TrxMetaData[i].Dst) {
trxBuffer := new(bytes.Buffer)
err := trx.EncodeRLP(trxBuffer)
if err != nil {
return nil, err
}
trxHashes = append(trxHashes, trx.Hash())
response.TransactionsRlp = append(response.TransactionsRlp, trxBuffer.Bytes())
}
}
}
return trxHashes, nil
}
func checkTransactions(wantedSrc, wantedDst []string, actualSrc, actualDst string) bool {
// If we aren't filtering for any addresses, every transaction is a go
if len(wantedDst) == 0 && len(wantedSrc) == 0 {
return true
}
for _, src := range wantedSrc {
if src == actualSrc {
return true
}
}
for _, dst := range wantedDst {
if dst == actualDst {
return true
}
}
return false
}
func (s *Screener) filerReceipts(streamFilters config.Subscription, response *ResponsePayload, payload IPLDPayload, trxHashes []common.Hash) error {
if !streamFilters.ReceiptFilter.Off && checkRange(streamFilters.StartingBlock.Int64(), streamFilters.EndingBlock.Int64(), payload.BlockNumber.Int64()) {
for i, receipt := range payload.Receipts {
if checkReceipts(receipt, streamFilters.ReceiptFilter.Topic0s, payload.ReceiptMetaData[i].Topic0s, streamFilters.ReceiptFilter.Contracts, payload.ReceiptMetaData[i].ContractAddress, trxHashes) {
receiptForStorage := (*types.ReceiptForStorage)(receipt)
receiptBuffer := new(bytes.Buffer)
err := receiptForStorage.EncodeRLP(receiptBuffer)
if err != nil {
return err
}
response.ReceiptsRlp = append(response.ReceiptsRlp, receiptBuffer.Bytes())
}
}
}
return nil
}
func checkReceipts(rct *types.Receipt, wantedTopics, actualTopics, wantedContracts []string, actualContract string, wantedTrxHashes []common.Hash) bool {
// If we aren't filtering for any topics or contracts, all topics are a go
if len(wantedTopics) == 0 && len(wantedContracts) == 0 {
return true
}
// No matter what filters we have, we keep receipts for the trxs we are interested in
for _, wantedTrxHash := range wantedTrxHashes {
if bytes.Equal(wantedTrxHash.Bytes(), rct.TxHash.Bytes()) {
return true
}
}
if len(wantedContracts) == 0 {
// We keep all receipts that have logs we are interested in
for _, wantedTopic := range wantedTopics {
for _, actualTopic := range actualTopics {
if wantedTopic == actualTopic {
return true
}
}
}
} else {
// We only keep receipts with logs of interest if we are interested in that contract
for _, wantedTopic := range wantedTopics {
for _, actualTopic := range actualTopics {
if wantedTopic == actualTopic {
for _, wantedContract := range wantedContracts {
if wantedContract == actualContract {
return true
}
}
}
}
}
}
return false
}
func (s *Screener) filterState(streamFilters config.Subscription, response *ResponsePayload, payload IPLDPayload) error {
if !streamFilters.StateFilter.Off && checkRange(streamFilters.StartingBlock.Int64(), streamFilters.EndingBlock.Int64(), payload.BlockNumber.Int64()) {
response.StateNodesRlp = make(map[common.Hash][]byte)
keyFilters := make([]common.Hash, 0, len(streamFilters.StateFilter.Addresses))
for _, addr := range streamFilters.StateFilter.Addresses {
keyFilter := AddressToKey(common.HexToAddress(addr))
keyFilters = append(keyFilters, keyFilter)
}
for key, stateNode := range payload.StateNodes {
if checkNodeKeys(keyFilters, key) {
if stateNode.Leaf || streamFilters.StateFilter.IntermediateNodes {
response.StateNodesRlp[key] = stateNode.Value
}
}
}
}
return nil
}
func checkNodeKeys(wantedKeys []common.Hash, actualKey common.Hash) bool {
// If we aren't filtering for any specific keys, all nodes are a go
if len(wantedKeys) == 0 {
return true
}
for _, key := range wantedKeys {
if bytes.Equal(key.Bytes(), actualKey.Bytes()) {
return true
}
}
return false
}
func (s *Screener) filterStorage(streamFilters config.Subscription, response *ResponsePayload, payload IPLDPayload) error {
if !streamFilters.StorageFilter.Off && checkRange(streamFilters.StartingBlock.Int64(), streamFilters.EndingBlock.Int64(), payload.BlockNumber.Int64()) {
response.StorageNodesRlp = make(map[common.Hash]map[common.Hash][]byte)
stateKeyFilters := make([]common.Hash, 0, len(streamFilters.StorageFilter.Addresses))
for _, addr := range streamFilters.StorageFilter.Addresses {
keyFilter := AddressToKey(common.HexToAddress(addr))
stateKeyFilters = append(stateKeyFilters, keyFilter)
}
storageKeyFilters := make([]common.Hash, 0, len(streamFilters.StorageFilter.StorageKeys))
for _, store := range streamFilters.StorageFilter.StorageKeys {
keyFilter := HexToKey(store)
storageKeyFilters = append(storageKeyFilters, keyFilter)
}
for stateKey, storageNodes := range payload.StorageNodes {
if checkNodeKeys(stateKeyFilters, stateKey) {
response.StorageNodesRlp[stateKey] = make(map[common.Hash][]byte)
for _, storageNode := range storageNodes {
if checkNodeKeys(storageKeyFilters, storageNode.Key) {
response.StorageNodesRlp[stateKey][storageNode.Key] = storageNode.Value
}
}
}
}
}
return nil
}
pkg/ipfs/screener_test.go
-1
View File
@@ -1 +0,0 @@
package ipfs
pkg/ipfs/service.go
-411
View File
@@ -1,411 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipfs
import (
"sync"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/node"
"github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/statediff"
log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
const payloadChanBufferSize = 20000 // the max eth sub buffer size
const workerPoolSize = 1
// SyncPublishScreenAndServe is the top level interface for streaming, converting to IPLDs, publishing,
// and indexing all Ethereum data; screening this data; and serving it up to subscribed clients
// This service is compatible with the Ethereum service interface (node.Service)
type SyncPublishScreenAndServe interface {
// APIs(), Protocols(), Start() and Stop()
node.Service
// Main event loop for syncAndPublish processes
SyncAndPublish(wg *sync.WaitGroup, forwardPayloadChan chan<- IPLDPayload, forwardQuitchan chan<- bool) error
// Main event loop for handling client pub-sub
ScreenAndServe(screenAndServePayload <-chan IPLDPayload, screenAndServeQuit <-chan bool)
// Method to subscribe to receive state diff processing output
Subscribe(id rpc.ID, sub chan<- ResponsePayload, quitChan chan<- bool, streamFilters config.Subscription)
// Method to unsubscribe from state diff processing
Unsubscribe(id rpc.ID)
}
// Service is the underlying struct for the SyncAndPublish interface
type Service struct {
// Used to sync access to the Subscriptions
sync.Mutex
// Interface for streaming statediff payloads over a geth rpc subscription
Streamer StateDiffStreamer
// Interface for converting statediff payloads into ETH-IPLD object payloads
Converter PayloadConverter
// Interface for publishing the ETH-IPLD payloads to IPFS
Publisher IPLDPublisher
// Interface for indexing the CIDs of the published ETH-IPLDs in Postgres
Repository CIDRepository
// Interface for filtering and serving data according to subscribed clients according to their specification
Screener ResponseScreener
// Interface for fetching ETH-IPLD objects from IPFS
Fetcher IPLDFetcher
// Interface for searching and retrieving CIDs from Postgres index
Retriever CIDRetriever
// Interface for resolving ipfs blocks to their data types
Resolver IPLDResolver
// Chan the processor uses to subscribe to state diff payloads from the Streamer
PayloadChan chan statediff.Payload
// Used to signal shutdown of the service
QuitChan chan bool
// A mapping of rpc.IDs to their subscription channels, mapped to their subscription type (hash of the StreamFilters)
Subscriptions map[common.Hash]map[rpc.ID]Subscription
// A mapping of subscription hash type to the corresponding StreamFilters
SubscriptionTypes map[common.Hash]config.Subscription
}
// NewIPFSProcessor creates a new Processor interface using an underlying Processor struct
func NewIPFSProcessor(ipfsPath string, db *postgres.DB, ethClient core.EthClient, rpcClient core.RpcClient, qc chan bool) (SyncPublishScreenAndServe, error) {
publisher, err := NewIPLDPublisher(ipfsPath)
if err != nil {
return nil, err
}
fetcher, err := NewIPLDFetcher(ipfsPath)
if err != nil {
return nil, err
}
return &Service{
Streamer: NewStateDiffStreamer(rpcClient),
Repository: NewCIDRepository(db),
Converter: NewPayloadConverter(ethClient),
Publisher: publisher,
Screener: NewResponseScreener(),
Fetcher: fetcher,
Retriever: NewCIDRetriever(db),
Resolver: NewIPLDResolver(),
PayloadChan: make(chan statediff.Payload, payloadChanBufferSize),
QuitChan: qc,
Subscriptions: make(map[common.Hash]map[rpc.ID]Subscription),
SubscriptionTypes: make(map[common.Hash]config.Subscription),
}, nil
}
// Protocols exports the services p2p protocols, this service has none
func (sap *Service) Protocols() []p2p.Protocol {
return []p2p.Protocol{}
}
// APIs returns the RPC descriptors the StateDiffingService offers
func (sap *Service) APIs() []rpc.API {
return []rpc.API{
{
Namespace: APIName,
Version: APIVersion,
Service: NewPublicSeedNodeAPI(sap),
Public: true,
},
}
}
// SyncAndPublish is the backend processing loop which streams data from geth, converts it to iplds, publishes them to ipfs, and indexes their cids
// This continues on no matter if or how many subscribers there are, it then forwards the data to the ScreenAndServe() loop
// which filters and sends relevant data to client subscriptions, if there are any
func (sap *Service) SyncAndPublish(wg *sync.WaitGroup, screenAndServePayload chan<- IPLDPayload, screenAndServeQuit chan<- bool) error {
sub, err := sap.Streamer.Stream(sap.PayloadChan)
if err != nil {
return err
}
wg.Add(1)
// Channels for forwarding data to the publishAndIndex workers
publishAndIndexPayload := make(chan IPLDPayload, payloadChanBufferSize)
publishAndIndexQuit := make(chan bool, workerPoolSize)
// publishAndIndex worker pool to handle publishing and indexing concurrently, while
// limiting the number of Postgres connections we can possibly open so as to prevent error
for i := 0; i < workerPoolSize; i++ {
sap.publishAndIndex(i, publishAndIndexPayload, publishAndIndexQuit)
}
go func() {
for {
select {
case payload := <-sap.PayloadChan:
if payload.Err != nil {
log.Error(err)
continue
}
ipldPayload, err := sap.Converter.Convert(payload)
if err != nil {
log.Error(err)
continue
}
// If we have a ScreenAndServe process running, forward the payload to it
select {
case screenAndServePayload <- *ipldPayload:
default:
}
// Forward the payload to the publishAndIndex workers
select {
case publishAndIndexPayload <- *ipldPayload:
default:
}
case err = <-sub.Err():
log.Error(err)
case <-sap.QuitChan:
// If we have a ScreenAndServe process running, forward the quit signal to it
select {
case screenAndServeQuit <- true:
default:
}
// Also forward a quit signal for each of the workers
for i := 0; i < workerPoolSize; i++ {
select {
case publishAndIndexQuit <- true:
default:
}
}
log.Info("quiting SyncAndPublish process")
wg.Done()
return
}
}
}()
return nil
}
func (sap *Service) publishAndIndex(id int, publishAndIndexPayload <-chan IPLDPayload, publishAndIndexQuit <-chan bool) {
go func() {
for {
select {
case payload := <-publishAndIndexPayload:
cidPayload, err := sap.Publisher.Publish(&payload)
if err != nil {
log.Errorf("worker %d error: %v", id, err)
continue
}
err = sap.Repository.Index(cidPayload)
if err != nil {
log.Errorf("worker %d error: %v", id, err)
}
case <-publishAndIndexQuit:
log.Infof("quiting publishAndIndex worked %d", id)
return
}
}
}()
}
// ScreenAndServe is the loop used to screen data streamed from the state diffing eth node
// and send the appropriate portions of it to a requesting client subscription, according to their subscription configuration
func (sap *Service) ScreenAndServe(screenAndServePayload <-chan IPLDPayload, screenAndServeQuit <-chan bool) {
go func() {
for {
select {
case payload := <-screenAndServePayload:
err := sap.sendResponse(payload)
if err != nil {
log.Error(err)
}
case <-screenAndServeQuit:
log.Info("quiting ScreenAndServe process")
return
}
}
}()
}
func (sap *Service) sendResponse(payload IPLDPayload) error {
sap.Lock()
for ty, subs := range sap.Subscriptions {
// Retrieve the subscription parameters for this subscription type
subConfig, ok := sap.SubscriptionTypes[ty]
if !ok {
log.Errorf("subscription configuration for subscription type %s not available", ty.Hex())
continue
}
response, err := sap.Screener.ScreenResponse(subConfig, payload)
if err != nil {
log.Error(err)
continue
}
for id, sub := range subs {
select {
case sub.PayloadChan <- *response:
log.Infof("sending seed node payload to subscription %s", id)
default:
log.Infof("unable to send payload to subscription %s; channel has no receiver", id)
}
}
}
sap.Unlock()
return nil
}
// Subscribe is used by the API to subscribe to the service loop
func (sap *Service) Subscribe(id rpc.ID, sub chan<- ResponsePayload, quitChan chan<- bool, streamFilters config.Subscription) {
log.Info("Subscribing to the seed node service")
// Subscription type is defined as the hash of its content
// Group subscriptions by type and screen payloads once for subs of the same type
by, err := rlp.EncodeToBytes(streamFilters)
if err != nil {
log.Error(err)
}
subscriptionHash := crypto.Keccak256(by)
subscriptionType := common.BytesToHash(subscriptionHash)
subscription := Subscription{
PayloadChan: sub,
QuitChan: quitChan,
}
// If the subscription requests a backfill, use the Postgres index to lookup and retrieve historical data
// Otherwise we only filter new data as it is streamed in from the state diffing geth node
if streamFilters.BackFill || streamFilters.BackFillOnly {
sap.backFill(subscription, id, streamFilters)
}
if !streamFilters.BackFillOnly {
sap.Lock()
if sap.Subscriptions[subscriptionType] == nil {
sap.Subscriptions[subscriptionType] = make(map[rpc.ID]Subscription)
}
sap.Subscriptions[subscriptionType][id] = subscription
sap.SubscriptionTypes[subscriptionType] = streamFilters
sap.Unlock()
}
}
func (sap *Service) backFill(sub Subscription, id rpc.ID, con config.Subscription) {
log.Debug("back-filling data for id", id)
// Retrieve cached CIDs relevant to this subscriber
var endingBlock int64
var startingBlock int64
var err error
startingBlock, err = sap.Retriever.RetrieveFirstBlockNumber()
if err != nil {
sub.PayloadChan <- ResponsePayload{
ErrMsg: "unable to set block range; error: " + err.Error(),
}
}
if startingBlock < con.StartingBlock.Int64() {
startingBlock = con.StartingBlock.Int64()
}
if con.EndingBlock.Int64() <= 0 || con.EndingBlock.Int64() <= startingBlock {
endingBlock, err = sap.Retriever.RetrieveLastBlockNumber()
if err != nil {
sub.PayloadChan <- ResponsePayload{
ErrMsg: "unable to set block range; error: " + err.Error(),
}
}
}
log.Debug("backfill starting block:", con.StartingBlock)
log.Debug("backfill ending block:", endingBlock)
// Backfilled payloads are sent concurrently to the streamed payloads, so the receiver needs to pay attention to
// the blocknumbers in the payloads they receive to keep things in order
// TODO: separate backfill into a different rpc subscription method altogether?
go func() {
for i := con.StartingBlock.Int64(); i <= endingBlock; i++ {
cidWrapper, err := sap.Retriever.RetrieveCIDs(con, i)
if err != nil {
sub.PayloadChan <- ResponsePayload{
ErrMsg: "CID retrieval error: " + err.Error(),
}
continue
}
if emptyCidWrapper(*cidWrapper) {
continue
}
blocksWrapper, err := sap.Fetcher.FetchCIDs(*cidWrapper)
if err != nil {
log.Error(err)
sub.PayloadChan <- ResponsePayload{
ErrMsg: "IPLD fetching error: " + err.Error(),
}
continue
}
backFillIplds, err := sap.Resolver.ResolveIPLDs(*blocksWrapper)
if err != nil {
log.Error(err)
sub.PayloadChan <- ResponsePayload{
ErrMsg: "IPLD resolving error: " + err.Error(),
}
continue
}
select {
case sub.PayloadChan <- *backFillIplds:
log.Infof("sending seed node back-fill payload to subscription %s", id)
default:
log.Infof("unable to send back-fill payload to subscription %s; channel has no receiver", id)
}
}
}()
}
// Unsubscribe is used to unsubscribe to the StateDiffingService loop
func (sap *Service) Unsubscribe(id rpc.ID) {
log.Info("Unsubscribing from the seed node service")
sap.Lock()
for ty := range sap.Subscriptions {
delete(sap.Subscriptions[ty], id)
if len(sap.Subscriptions[ty]) == 0 {
// If we removed the last subscription of this type, remove the subscription type outright
delete(sap.Subscriptions, ty)
delete(sap.SubscriptionTypes, ty)
}
}
sap.Unlock()
}
// Start is used to begin the service
func (sap *Service) Start(*p2p.Server) error {
log.Info("Starting seed node service")
wg := new(sync.WaitGroup)
payloadChan := make(chan IPLDPayload, payloadChanBufferSize)
quitChan := make(chan bool, 1)
if err := sap.SyncAndPublish(wg, payloadChan, quitChan); err != nil {
return err
}
sap.ScreenAndServe(payloadChan, quitChan)
return nil
}
// Stop is used to close down the service
func (sap *Service) Stop() error {
log.Info("Stopping seed node service")
close(sap.QuitChan)
return nil
}
// close is used to close all listening subscriptions
func (sap *Service) close() {
sap.Lock()
for ty, subs := range sap.Subscriptions {
for id, sub := range subs {
select {
case sub.QuitChan <- true:
log.Infof("closing subscription %s", id)
default:
log.Infof("unable to close subscription %s; channel has no receiver", id)
}
}
delete(sap.Subscriptions, ty)
delete(sap.SubscriptionTypes, ty)
}
sap.Unlock()
}
pkg/ipfs/service_test.go
-76
View File
@@ -1,76 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipfs_test
import (
"sync"
"time"
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/statediff"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks"
)
var _ = Describe("Service", func() {
Describe("Loop", func() {
It("Streams StatediffPayloads, converts them to IPLDPayloads, publishes IPLDPayloads, and indexes CIDPayloads", func() {
wg := new(sync.WaitGroup)
payloadChan := make(chan statediff.Payload, 1)
quitChan := make(chan bool, 1)
mockCidRepo := &mocks.CIDRepository{
ReturnErr: nil,
}
mockPublisher := &mocks.IPLDPublisher{
ReturnCIDPayload: &mocks.MockCIDPayload,
ReturnErr: nil,
}
mockStreamer := &mocks.StateDiffStreamer{
ReturnSub: &rpc.ClientSubscription{},
StreamPayloads: []statediff.Payload{
mocks.MockStatediffPayload,
},
ReturnErr: nil,
}
mockConverter := &mocks.PayloadConverter{
ReturnIPLDPayload: &mocks.MockIPLDPayload,
ReturnErr: nil,
}
processor := &ipfs.Service{
Repository: mockCidRepo,
Publisher: mockPublisher,
Streamer: mockStreamer,
Converter: mockConverter,
PayloadChan: payloadChan,
QuitChan: quitChan,
}
err := processor.SyncAndPublish(wg, nil, nil)
Expect(err).ToNot(HaveOccurred())
time.Sleep(2 * time.Second)
quitChan <- true
wg.Wait()
Expect(mockConverter.PassedStatediffPayload).To(Equal(mocks.MockStatediffPayload))
Expect(mockCidRepo.PassedCIDPayload).To(Equal(&mocks.MockCIDPayload))
Expect(mockPublisher.PassedIPLDPayload).To(Equal(&mocks.MockIPLDPayload))
Expect(mockStreamer.PassedPayloadChan).To(Equal(payloadChan))
})
})
})
pkg/ipfs/streamer.go
-46
View File
@@ -1,46 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipfs
import (
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/statediff"
"github.com/vulcanize/vulcanizedb/pkg/core"
)
// StateDiffStreamer is the interface for streaming a statediff subscription
type StateDiffStreamer interface {
Stream(payloadChan chan statediff.Payload) (*rpc.ClientSubscription, error)
}
// Streamer is the underlying struct for the StateDiffStreamer interface
type Streamer struct {
Client core.RpcClient
}
// NewStateDiffStreamer creates a pointer to a new Streamer which satisfies the StateDiffStreamer interface
func NewStateDiffStreamer(client core.RpcClient) *Streamer {
return &Streamer{
Client: client,
}
}
// Stream is the main loop for subscribing to data from the Geth state diff process
func (sds *Streamer) Stream(payloadChan chan statediff.Payload) (*rpc.ClientSubscription, error) {
return sds.Client.Subscribe("statediff", payloadChan, "stream")
}
pkg/ipfs/streamer_test.go
-45
View File
@@ -1,45 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipfs_test
import (
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/statediff"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks"
)
var _ = Describe("Streamer", func() {
Describe("Stream", func() {
It("Streams StatediffPayloads from a Geth RPC subscription", func() {
mockStreamer := mocks.StateDiffStreamer{}
mockStreamer.ReturnSub = &rpc.ClientSubscription{}
mockStreamer.StreamPayloads = []statediff.Payload{
mocks.MockStatediffPayload,
}
payloadChan := make(chan statediff.Payload, 1)
sub, err := mockStreamer.Stream(payloadChan)
Expect(err).ToNot(HaveOccurred())
Expect(sub).To(Equal(&rpc.ClientSubscription{}))
Expect(mockStreamer.PassedPayloadChan).To(Equal(payloadChan))
streamedPayload := <-payloadChan
Expect(streamedPayload).To(Equal(mocks.MockStatediffPayload))
})
})
})
pkg/ipfs/types.go
+4 -44
View File
@@ -17,7 +17,6 @@
package ipfs
import (
"encoding/json"
"math/big"
"github.com/ethereum/go-ethereum/common"
@@ -25,47 +24,8 @@ import (
"github.com/ipfs/go-block-format"
)
// Subscription holds the information for an individual client subscription
type Subscription struct {
PayloadChan chan<- ResponsePayload
QuitChan chan<- bool
}
// ResponsePayload holds the data returned from the seed node to the requesting client
type ResponsePayload struct {
BlockNumber *big.Int `json:"blockNumber"`
HeadersRlp [][]byte `json:"headersRlp"`
UnclesRlp [][]byte `json:"unclesRlp"`
TransactionsRlp [][]byte `json:"transactionsRlp"`
ReceiptsRlp [][]byte `json:"receiptsRlp"`
StateNodesRlp map[common.Hash][]byte `json:"stateNodesRlp"`
StorageNodesRlp map[common.Hash]map[common.Hash][]byte `json:"storageNodesRlp"`
ErrMsg string `json:"errMsg"`
encoded []byte
err error
}
func (sd *ResponsePayload) ensureEncoded() {
if sd.encoded == nil && sd.err == nil {
sd.encoded, sd.err = json.Marshal(sd)
}
}
// Length to implement Encoder interface for StateDiff
func (sd *ResponsePayload) Length() int {
sd.ensureEncoded()
return len(sd.encoded)
}
// Encode to implement Encoder interface for StateDiff
func (sd *ResponsePayload) Encode() ([]byte, error) {
sd.ensureEncoded()
return sd.encoded, sd.err
}
// CidWrapper is used to package CIDs retrieved from the local Postgres cache
type CidWrapper struct {
// CIDWrapper is used to package CIDs retrieved from the local Postgres cache
type CIDWrapper struct {
BlockNumber *big.Int
Headers []string
Uncles []string
@@ -75,8 +35,8 @@ type CidWrapper struct {
StorageNodes []StorageNodeCID
}
// IpldWrapper is used to package raw IPLD block data for resolution
type IpldWrapper struct {
// IPLDWrapper is used to package raw IPLD block data for resolution
type IPLDWrapper struct {
BlockNumber *big.Int
Headers []blocks.Block
Uncles []blocks.Block