cerc-io/ipld-eth-server
18
0
Fork 1
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases 37 Wiki Activity

review fixes

Browse Source
This commit is contained in:
Ian Norden 2019-08-05 12:56:15 -05:00
parent 2244d1869f
commit 0bbb7a30d1
34 changed files with 253 additions and 469 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/root.go
View File

@ -40,7 +40,7 @@ var (
cfgFile string cfgFile string
databaseConfig config.Database databaseConfig config.Database
genConfig config.Plugin genConfig config.Plugin
subConfig config.Subscription subscriptionConfig config.Subscription
ipc string ipc string
levelDbPath string levelDbPath string
queueRecheckInterval time.Duration queueRecheckInterval time.Duration

16
cmd/streamSubscribe.go
View File

@ -33,14 +33,14 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/config" "github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/geth/client" "github.com/vulcanize/vulcanizedb/pkg/geth/client"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
) )
// streamSubscribeCmd represents the streamSubscribe command // streamSubscribeCmd represents the streamSubscribe command
var streamSubscribeCmd = &cobra.Command{ var streamSubscribeCmd = &cobra.Command{
Use: "streamSubscribe", Use: "streamSubscribe",
Short: "This command is used to subscribe to the seed node stream with the provided filters", Short: "This command is used to subscribe to the seed node stream with the provided filters",
Long: ``, Long: `This command is for demo and testing purposes and is used to subscribe to the seed node with the provided subscription configuration parameters.
It does not do anything with the data streamed from the seed node other than unpack it and print it out for demonstration purposes.`,
Run: func(cmd *cobra.Command, args []string) { Run: func(cmd *cobra.Command, args []string) {
streamSubscribe() streamSubscribe()
}, },
@ -52,17 +52,17 @@ func init() {
func streamSubscribe() { func streamSubscribe() {
// Prep the subscription config/filters to be sent to the server // Prep the subscription config/filters to be sent to the server
subscriptionConfig() configureSubscription()
// Create a new rpc client and a subscription streamer with that client // Create a new rpc client and a subscription streamer with that client
rpcClient := getRpcClient() rpcClient := getRpcClient()
str := streamer.NewSeedStreamer(rpcClient) str := streamer.NewSeedNodeStreamer(rpcClient)
// Buffered channel for reading subscription payloads // Buffered channel for reading subscription payloads
payloadChan := make(chan ipfs.ResponsePayload, 20000) payloadChan := make(chan streamer.SeedNodePayload, 20000)
// Subscribe to the seed node service with the given config/filter parameters // Subscribe to the seed node service with the given config/filter parameters
sub, err := str.Stream(payloadChan, subConfig) sub, err := str.Stream(payloadChan, subscriptionConfig)
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
@ -161,9 +161,9 @@ func streamSubscribe() {
} }
} }
func subscriptionConfig() { func configureSubscription() {
log.Info("loading subscription config") log.Info("loading subscription config")
subConfig = config.Subscription{ subscriptionConfig = config.Subscription{
// Below default to false, which means we do not backfill by default // Below default to false, which means we do not backfill by default
BackFill: viper.GetBool("subscription.backfill"), BackFill: viper.GetBool("subscription.backfill"),
BackFillOnly: viper.GetBool("subscription.backfillOnly"), BackFillOnly: viper.GetBool("subscription.backfillOnly"),

5
cmd/syncAndPublish.go
View File

@ -20,6 +20,8 @@ import (
"path/filepath" "path/filepath"
syn "sync" syn "sync"
"github.com/vulcanize/vulcanizedb/pkg/seed_node"
"github.com/spf13/viper" "github.com/spf13/viper"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
@ -33,7 +35,6 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/geth/client" "github.com/vulcanize/vulcanizedb/pkg/geth/client"
vRpc "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc" vRpc "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
"github.com/vulcanize/vulcanizedb/pkg/geth/node" "github.com/vulcanize/vulcanizedb/pkg/geth/node"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
"github.com/vulcanize/vulcanizedb/utils" "github.com/vulcanize/vulcanizedb/utils"
) )
@ -68,7 +69,7 @@ func syncAndPublish() {
} }
ipfsPath = filepath.Join(home, ".ipfs") ipfsPath = filepath.Join(home, ".ipfs")
} }
processor, err := ipfs.NewIPFSProcessor(ipfsPath, &db, ethClient, rpcClient, quitChan) processor, err := seed_node.NewProcessor(ipfsPath, &db, ethClient, rpcClient, quitChan)
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }

8
cmd/syncPublishScreenAndServe.go
View File

@ -20,6 +20,8 @@ import (
"path/filepath" "path/filepath"
syn "sync" syn "sync"
"github.com/vulcanize/vulcanizedb/pkg/seed_node"
"github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/rpc"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/spf13/cobra" "github.com/spf13/cobra"
@ -61,7 +63,7 @@ func syncPublishScreenAndServe() {
} }
ipfsPath = filepath.Join(home, ".ipfs") ipfsPath = filepath.Join(home, ".ipfs")
} }
processor, err := ipfs.NewIPFSProcessor(ipfsPath, &db, ethClient, rpcClient, quitChan) processor, err := seed_node.NewProcessor(ipfsPath, &db, ethClient, rpcClient, quitChan)
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
@ -94,7 +96,9 @@ func syncPublishScreenAndServe() {
if wsEndpoint == "" { if wsEndpoint == "" {
wsEndpoint = "127.0.0.1:80" wsEndpoint = "127.0.0.1:80"
} }
_, _, err = rpc.StartWSEndpoint(wsEndpoint, processor.APIs(), []string{"vulcanizedb"}, nil, true) var exposeAll = true
var wsOrigins []string = nil
_, _, err = rpc.StartWSEndpoint(wsEndpoint, processor.APIs(), []string{"vulcanizedb"}, wsOrigins, exposeAll)
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }

2
db/migrations/00033_create_ipfs_blocks_table.sql
View File

@ -1,5 +1,5 @@
-- +goose Up -- +goose Up
CREATE TABLE public.blocks ( CREATE TABLE IF NOT EXISTS public.blocks (
key TEXT UNIQUE NOT NULL, key TEXT UNIQUE NOT NULL,
data BYTEA NOT NULL data BYTEA NOT NULL
); );

2
documentation/contributing.md
View File

@ -14,7 +14,7 @@ conform to the
[standard-readme specification](https://github.com/RichardLitt/standard-readme). [standard-readme specification](https://github.com/RichardLitt/standard-readme).
- Once a Pull Request has received two approvals it can be merged in by a core developer. - Once a Pull Request has received two approvals it can be merged in by a core developer.
Pull requests should be opened against the `master` branch. Periodically, updates on `master` will be ported over to `staging` for tagged release. Pull requests should be opened against the `staging` branch. Periodically, updates on `staging` will be ported over to `master` for tagged release.
## Creating a new migration file ## Creating a new migration file
1. `make new_migration NAME=add_columnA_to_table1` 1. `make new_migration NAME=add_columnA_to_table1`

7
documentation/seed-node.md
View File

@ -131,8 +131,8 @@ when the geth sync was started), and `client.ipfsPath` which is the path the ipf
#### syncPublishScreenAndServe #### syncPublishScreenAndServe
`syncPublishScreenAndServe` does everythin th at `syncAndPublish` does, plut it opens up an RPC server which exposes `syncPublishScreenAndServe` does everything that `syncAndPublish` does, plus it opens up an RPC server which exposes
an endpoint to allow transformers to subscribe to subsets of the sync-and-published data that are relevant to thier transformations an endpoint to allow transformers to subscribe to subsets of the sync-and-published data that are relevant to their transformations
Usage: Usage:
@ -152,11 +152,12 @@ The config file for the `syncPublishScreenAndServe` command has two additional f
[server] [server]
ipcPath = "/Users/user/.vulcanize/vulcanize.ipc" ipcPath = "/Users/user/.vulcanize/vulcanize.ipc"
wsEndpoint = "127.0.0.1:2019" wsEndpoint = "127.0.0.1:80"
``` ```
The additional `server.ipcPath` and `server.wsEndpoint` fields are used to set what ipc endpoint and ws url The additional `server.ipcPath` and `server.wsEndpoint` fields are used to set what ipc endpoint and ws url
the `syncPublishScreenAndServe` rpc server will expose itself to subscribing transformers over, respectively. the `syncPublishScreenAndServe` rpc server will expose itself to subscribing transformers over, respectively.
Any valid and available path and endpoint is acceptable, but keep in mind that this path and endpoint need to be known by transformers for them to subscribe to the seed node.
#### Subscribing #### Subscribing

84
libraries/shared/streamer/seed_node_streamer.go Normal file
View File

@ -0,0 +1,84 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
// Streamer is used by watchers to stream eth data from a vulcanizedb seed node
package streamer
import (
"encoding/json"
"math/big"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/rpc"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/core"
)
// ISeedNodeStreamer is the interface for streaming data from a vulcanizeDB seed node
type ISeedNodeStreamer interface {
Stream(payloadChan chan SeedNodePayload, streamFilters config.Subscription) (*rpc.ClientSubscription, error)
}
// SeedNodeStreamer is the underlying struct for the ISeedNodeStreamer interface
type SeedNodeStreamer struct {
Client core.RpcClient
}
// NewSeedNodeStreamer creates a pointer to a new SeedNodeStreamer which satisfies the ISeedNodeStreamer interface
func NewSeedNodeStreamer(client core.RpcClient) *SeedNodeStreamer {
return &SeedNodeStreamer{
Client: client,
}
}
// Stream is the main loop for subscribing to data from a vulcanizedb seed node
func (sds *SeedNodeStreamer) Stream(payloadChan chan SeedNodePayload, streamFilters config.Subscription) (*rpc.ClientSubscription, error) {
return sds.Client.Subscribe("vulcanizedb", payloadChan, "stream", streamFilters)
}
// Payload holds the data returned from the seed node to the requesting client
type SeedNodePayload struct {
BlockNumber *big.Int `json:"blockNumber"`
HeadersRlp [][]byte `json:"headersRlp"`
UnclesRlp [][]byte `json:"unclesRlp"`
TransactionsRlp [][]byte `json:"transactionsRlp"`
ReceiptsRlp [][]byte `json:"receiptsRlp"`
StateNodesRlp map[common.Hash][]byte `json:"stateNodesRlp"`
StorageNodesRlp map[common.Hash]map[common.Hash][]byte `json:"storageNodesRlp"`
ErrMsg string `json:"errMsg"`
encoded []byte
err error
}
func (sd *SeedNodePayload) ensureEncoded() {
if sd.encoded == nil && sd.err == nil {
sd.encoded, sd.err = json.Marshal(sd)
}
}
// Length to implement Encoder interface for StateDiff
func (sd *SeedNodePayload) Length() int {
sd.ensureEncoded()
return len(sd.encoded)
}
// Encode to implement Encoder interface for StateDiff
func (sd *SeedNodePayload) Encode() ([]byte, error) {
sd.ensureEncoded()
return sd.encoded, sd.err
}

48
libraries/shared/streamer/streamer.go
View File

@ -1,48 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
// Streamer is used by watchers to stream eth data from a vulcanizedb seed node
package streamer
import (
"github.com/ethereum/go-ethereum/rpc"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
)
// IStreamer is the interface for streaming data from a vulcanizeDB seed node
type IStreamer interface {
Stream(payloadChan chan ipfs.ResponsePayload, streamFilters config.Subscription) (*rpc.ClientSubscription, error)
}
// Streamer is the underlying struct for the IStreamer interface
type Streamer struct {
Client core.RpcClient
}
// NewSeedStreamer creates a pointer to a new Streamer which satisfies the IStreamer interface
func NewSeedStreamer(client core.RpcClient) *Streamer {
return &Streamer{
Client: client,
}
}
// Stream is the main loop for subscribing to data from a vulcanizedb seed node
func (sds *Streamer) Stream(payloadChan chan ipfs.ResponsePayload, streamFilters config.Subscription) (*rpc.ClientSubscription, error) {
return sds.Client.Subscribe("vulcanizedb", payloadChan, "stream", streamFilters)
}

5
pkg/ipfs/converter.go
View File

@ -49,6 +49,9 @@ func (pc *Converter) Convert(payload statediff.Payload) (*IPLDPayload, error) {
// Unpack block rlp to access fields // Unpack block rlp to access fields
block := new(types.Block) block := new(types.Block)
err := rlp.DecodeBytes(payload.BlockRlp, block) err := rlp.DecodeBytes(payload.BlockRlp, block)
if err != nil {
return nil, err
}
header := block.Header() header := block.Header()
headerRlp, err := rlp.EncodeToBytes(header) headerRlp, err := rlp.EncodeToBytes(header)
if err != nil { if err != nil {
@ -74,7 +77,7 @@ func (pc *Converter) Convert(payload statediff.Payload) (*IPLDPayload, error) {
} }
txMeta := &TrxMetaData{ txMeta := &TrxMetaData{
Dst: handleNullAddr(trx.To()), Dst: handleNullAddr(trx.To()),
Src: from.Hex(), Src: handleNullAddr(&from),
} }
// txMeta will have same index as its corresponding trx in the convertedPayload.BlockBody // txMeta will have same index as its corresponding trx in the convertedPayload.BlockBody
convertedPayload.TrxMetaData = append(convertedPayload.TrxMetaData, txMeta) convertedPayload.TrxMetaData = append(convertedPayload.TrxMetaData, txMeta)

37
pkg/ipfs/converter_test.go
View File

@ -1,37 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipfs_test
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks"
)
var _ = Describe("Converter", func() {
Describe("Convert", func() {
It("Converts StatediffPayloads into IPLDPayloads", func() {
mockConverter := mocks.PayloadConverter{}
mockConverter.ReturnIPLDPayload = &mocks.MockIPLDPayload
ipldPayload, err := mockConverter.Convert(mocks.MockStatediffPayload)
Expect(err).ToNot(HaveOccurred())
Expect(ipldPayload).To(Equal(&mocks.MockIPLDPayload))
Expect(mockConverter.PassedStatediffPayload).To(Equal(mocks.MockStatediffPayload))
})
})
})

20
pkg/ipfs/fetcher.go
View File

@ -28,7 +28,7 @@ import (
// IPLDFetcher is an interface for fetching IPLDs // IPLDFetcher is an interface for fetching IPLDs
type IPLDFetcher interface { type IPLDFetcher interface {
FetchCIDs(cids CidWrapper) (*IpldWrapper, error) FetchCIDs(cids CIDWrapper) (*IPLDWrapper, error)
} }
// EthIPLDFetcher is used to fetch ETH IPLD objects from IPFS // EthIPLDFetcher is used to fetch ETH IPLD objects from IPFS
@ -47,11 +47,11 @@ func NewIPLDFetcher(ipfsPath string) (*EthIPLDFetcher, error) {
}, nil }, nil
} }
// FetchCIDs is the exported method for fetching and returning all the cids passed in a CidWrapper // FetchCIDs is the exported method for fetching and returning all the cids passed in a CIDWrapper
func (f *EthIPLDFetcher) FetchCIDs(cids CidWrapper) (*IpldWrapper, error) { func (f *EthIPLDFetcher) FetchCIDs(cids CIDWrapper) (*IPLDWrapper, error) {
log.Debug("fetching iplds") log.Debug("fetching iplds")
blocks := &IpldWrapper{ blocks := &IPLDWrapper{
BlockNumber: cids.BlockNumber, BlockNumber: cids.BlockNumber,
Headers: make([]blocks.Block, 0), Headers: make([]blocks.Block, 0),
Uncles: make([]blocks.Block, 0), Uncles: make([]blocks.Block, 0),
@ -91,7 +91,7 @@ func (f *EthIPLDFetcher) FetchCIDs(cids CidWrapper) (*IpldWrapper, error) {
// fetchHeaders fetches headers // fetchHeaders fetches headers
// It uses the f.fetchBatch method // It uses the f.fetchBatch method
func (f *EthIPLDFetcher) fetchHeaders(cids CidWrapper, blocks *IpldWrapper) error { func (f *EthIPLDFetcher) fetchHeaders(cids CIDWrapper, blocks *IPLDWrapper) error {
log.Debug("fetching header iplds") log.Debug("fetching header iplds")
headerCids := make([]cid.Cid, 0, len(cids.Headers)) headerCids := make([]cid.Cid, 0, len(cids.Headers))
for _, c := range cids.Headers { for _, c := range cids.Headers {
@ -110,7 +110,7 @@ func (f *EthIPLDFetcher) fetchHeaders(cids CidWrapper, blocks *IpldWrapper) erro
// fetchUncles fetches uncles // fetchUncles fetches uncles
// It uses the f.fetchBatch method // It uses the f.fetchBatch method
func (f *EthIPLDFetcher) fetchUncles(cids CidWrapper, blocks *IpldWrapper) error { func (f *EthIPLDFetcher) fetchUncles(cids CIDWrapper, blocks *IPLDWrapper) error {
log.Debug("fetching uncle iplds") log.Debug("fetching uncle iplds")
uncleCids := make([]cid.Cid, 0, len(cids.Uncles)) uncleCids := make([]cid.Cid, 0, len(cids.Uncles))
for _, c := range cids.Uncles { for _, c := range cids.Uncles {
@ -129,7 +129,7 @@ func (f *EthIPLDFetcher) fetchUncles(cids CidWrapper, blocks *IpldWrapper) error
// fetchTrxs fetches transactions // fetchTrxs fetches transactions
// It uses the f.fetchBatch method // It uses the f.fetchBatch method
func (f *EthIPLDFetcher) fetchTrxs(cids CidWrapper, blocks *IpldWrapper) error { func (f *EthIPLDFetcher) fetchTrxs(cids CIDWrapper, blocks *IPLDWrapper) error {
log.Debug("fetching transaction iplds") log.Debug("fetching transaction iplds")
trxCids := make([]cid.Cid, 0, len(cids.Transactions)) trxCids := make([]cid.Cid, 0, len(cids.Transactions))
for _, c := range cids.Transactions { for _, c := range cids.Transactions {
@ -148,7 +148,7 @@ func (f *EthIPLDFetcher) fetchTrxs(cids CidWrapper, blocks *IpldWrapper) error {
// fetchRcts fetches receipts // fetchRcts fetches receipts
// It uses the f.fetchBatch method // It uses the f.fetchBatch method
func (f *EthIPLDFetcher) fetchRcts(cids CidWrapper, blocks *IpldWrapper) error { func (f *EthIPLDFetcher) fetchRcts(cids CIDWrapper, blocks *IPLDWrapper) error {
log.Debug("fetching receipt iplds") log.Debug("fetching receipt iplds")
rctCids := make([]cid.Cid, 0, len(cids.Receipts)) rctCids := make([]cid.Cid, 0, len(cids.Receipts))
for _, c := range cids.Receipts { for _, c := range cids.Receipts {
@ -168,7 +168,7 @@ func (f *EthIPLDFetcher) fetchRcts(cids CidWrapper, blocks *IpldWrapper) error {
// fetchState fetches state nodes // fetchState fetches state nodes
// It uses the single f.fetch method instead of the batch fetch, because it // It uses the single f.fetch method instead of the batch fetch, because it
// needs to maintain the data's relation to state keys // needs to maintain the data's relation to state keys
func (f *EthIPLDFetcher) fetchState(cids CidWrapper, blocks *IpldWrapper) error { func (f *EthIPLDFetcher) fetchState(cids CIDWrapper, blocks *IPLDWrapper) error {
log.Debug("fetching state iplds") log.Debug("fetching state iplds")
for _, stateNode := range cids.StateNodes { for _, stateNode := range cids.StateNodes {
if stateNode.CID == "" || stateNode.Key == "" { if stateNode.CID == "" || stateNode.Key == "" {
@ -190,7 +190,7 @@ func (f *EthIPLDFetcher) fetchState(cids CidWrapper, blocks *IpldWrapper) error
// fetchStorage fetches storage nodes // fetchStorage fetches storage nodes
// It uses the single f.fetch method instead of the batch fetch, because it // It uses the single f.fetch method instead of the batch fetch, because it
// needs to maintain the data's relation to state and storage keys // needs to maintain the data's relation to state and storage keys
func (f *EthIPLDFetcher) fetchStorage(cids CidWrapper, blocks *IpldWrapper) error { func (f *EthIPLDFetcher) fetchStorage(cids CIDWrapper, blocks *IPLDWrapper) error {
log.Debug("fetching storage iplds") log.Debug("fetching storage iplds")
for _, storageNode := range cids.StorageNodes { for _, storageNode := range cids.StorageNodes {
if storageNode.CID == "" || storageNode.Key == "" || storageNode.StateKey == "" { if storageNode.CID == "" || storageNode.Key == "" || storageNode.StateKey == "" {

5
pkg/ipfs/helpers.go
View File

@ -55,8 +55,9 @@ func HexToKey(hex string) common.Hash {
return crypto.Keccak256Hash(addr[:]) return crypto.Keccak256Hash(addr[:])
} }
func emptyCidWrapper(cids CidWrapper) bool { // EmptyCIDWrapper returns whether or not the provided CIDWrapper has any Cids we need to process
if len(cids.Transactions) > 0 || len(cids.Headers) > 0 || len(cids.Uncles) > 0 || len(cids.Receipts) > 0 || len(cids.StateNodes) > 0 || len(cids.StorageNodes) > 0 || cids.BlockNumber == nil { func EmptyCIDWrapper(cids CIDWrapper) bool {
if len(cids.Transactions) > 0 || len(cids.Headers) > 0 || len(cids.Uncles) > 0 || len(cids.Receipts) > 0 || len(cids.StateNodes) > 0 || len(cids.StorageNodes) > 0 {
return false return false
} }
return true return true

1
pkg/ipfs/mocks/api.go
View File

@ -1 +0,0 @@
package mocks

1
pkg/ipfs/mocks/screener.go
View File

@ -1 +0,0 @@
package mocks

43
pkg/ipfs/mocks/streamer.go
View File

@ -1,43 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package mocks
import (
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/statediff"
)
// StateDiffStreamer is the underlying struct for the Streamer interface
type StateDiffStreamer struct {
PassedPayloadChan chan statediff.Payload
ReturnSub *rpc.ClientSubscription
ReturnErr error
StreamPayloads []statediff.Payload
}
// Stream is the main loop for subscribing to data from the Geth state diff process
func (sds *StateDiffStreamer) Stream(payloadChan chan statediff.Payload) (*rpc.ClientSubscription, error) {
sds.PassedPayloadChan = payloadChan
go func() {
for _, payload := range sds.StreamPayloads {
sds.PassedPayloadChan <- payload
}
}()
return sds.ReturnSub, sds.ReturnErr
}

8
pkg/ipfs/publisher.go
View File

@ -21,7 +21,7 @@ import (
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/types"
rlp2 "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rlp"
"github.com/ipfs/go-ipfs/plugin/loader" "github.com/ipfs/go-ipfs/plugin/loader"
"github.com/vulcanize/eth-block-extractor/pkg/ipfs" "github.com/vulcanize/eth-block-extractor/pkg/ipfs"
@ -30,7 +30,7 @@ import (
"github.com/vulcanize/eth-block-extractor/pkg/ipfs/eth_block_transactions" "github.com/vulcanize/eth-block-extractor/pkg/ipfs/eth_block_transactions"
"github.com/vulcanize/eth-block-extractor/pkg/ipfs/eth_state_trie" "github.com/vulcanize/eth-block-extractor/pkg/ipfs/eth_state_trie"
"github.com/vulcanize/eth-block-extractor/pkg/ipfs/eth_storage_trie" "github.com/vulcanize/eth-block-extractor/pkg/ipfs/eth_storage_trie"
"github.com/vulcanize/eth-block-extractor/pkg/wrappers/rlp" rlp2 "github.com/vulcanize/eth-block-extractor/pkg/wrappers/rlp"
) )
// IPLDPublisher is the interface for publishing an IPLD payload // IPLDPublisher is the interface for publishing an IPLD payload
@ -66,7 +66,7 @@ func NewIPLDPublisher(ipfsPath string) (*Publisher, error) {
return nil, err return nil, err
} }
return &Publisher{ return &Publisher{
HeaderPutter: eth_block_header.NewBlockHeaderDagPutter(node, rlp.RlpDecoder{}), HeaderPutter: eth_block_header.NewBlockHeaderDagPutter(node, rlp2.RlpDecoder{}),
TransactionPutter: eth_block_transactions.NewBlockTransactionsDagPutter(node), TransactionPutter: eth_block_transactions.NewBlockTransactionsDagPutter(node),
ReceiptPutter: eth_block_receipts.NewEthBlockReceiptDagPutter(node), ReceiptPutter: eth_block_receipts.NewEthBlockReceiptDagPutter(node),
StatePutter: eth_state_trie.NewStateTrieDagPutter(node), StatePutter: eth_state_trie.NewStateTrieDagPutter(node),
@ -85,7 +85,7 @@ func (pub *Publisher) Publish(payload *IPLDPayload) (*CIDPayload, error) {
// Process and publish uncles // Process and publish uncles
uncleCids := make(map[common.Hash]string) uncleCids := make(map[common.Hash]string)
for _, uncle := range payload.BlockBody.Uncles { for _, uncle := range payload.BlockBody.Uncles {
uncleRlp, err := rlp2.EncodeToBytes(uncle) uncleRlp, err := rlp.EncodeToBytes(uncle)
if err != nil { if err != nil {
return nil, err return nil, err
} }

37
pkg/ipfs/publisher_test.go
View File

@ -1,37 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipfs_test
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks"
)
var _ = Describe("Publisher", func() {
Describe("Publish", func() {
It("Publishes IPLDPayload to IPFS", func() {
mockPublisher := mocks.IPLDPublisher{}
mockPublisher.ReturnCIDPayload = &mocks.MockCIDPayload
cidPayload, err := mockPublisher.Publish(&mocks.MockIPLDPayload)
Expect(err).ToNot(HaveOccurred())
Expect(cidPayload).To(Equal(&mocks.MockCIDPayload))
Expect(mockPublisher.PassedIPLDPayload).To(Equal(&mocks.MockIPLDPayload))
})
})
})

35
pkg/ipfs/repository_test.go
View File

@ -1,35 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipfs_test
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks"
)
var _ = Describe("Repository", func() {
Describe("Index", func() {
It("Indexes CIDs against their metadata", func() {
mockRepo := mocks.CIDRepository{}
err := mockRepo.Index(&mocks.MockCIDPayload)
Expect(err).ToNot(HaveOccurred())
Expect(mockRepo.PassedCIDPayload).To(Equal(&mocks.MockCIDPayload))
})
})
})

19
pkg/ipfs/resolver.go
View File

@ -19,11 +19,12 @@ package ipfs
import ( import (
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ipfs/go-block-format" "github.com/ipfs/go-block-format"
"github.com/vulcanize/vulcanizedb/libraries/shared/streamer"
) )
// IPLDResolver is the interface to resolving IPLDs // IPLDResolver is the interface to resolving IPLDs
type IPLDResolver interface { type IPLDResolver interface {
ResolveIPLDs(ipfsBlocks IpldWrapper) (*ResponsePayload, error) ResolveIPLDs(ipfsBlocks IPLDWrapper) (*streamer.SeedNodePayload, error)
} }
// EthIPLDResolver is the underlying struct to support the IPLDResolver interface // EthIPLDResolver is the underlying struct to support the IPLDResolver interface
@ -35,8 +36,8 @@ func NewIPLDResolver() *EthIPLDResolver {
} }
// ResolveIPLDs is the exported method for resolving all of the ETH IPLDs packaged in an IpfsBlockWrapper // ResolveIPLDs is the exported method for resolving all of the ETH IPLDs packaged in an IpfsBlockWrapper
func (eir *EthIPLDResolver) ResolveIPLDs(ipfsBlocks IpldWrapper) (*ResponsePayload, error) { func (eir *EthIPLDResolver) ResolveIPLDs(ipfsBlocks IPLDWrapper) (*streamer.SeedNodePayload, error) {
response := new(ResponsePayload) response := new(streamer.SeedNodePayload)
response.BlockNumber = ipfsBlocks.BlockNumber response.BlockNumber = ipfsBlocks.BlockNumber
eir.resolveHeaders(ipfsBlocks.Headers, response) eir.resolveHeaders(ipfsBlocks.Headers, response)
eir.resolveUncles(ipfsBlocks.Uncles, response) eir.resolveUncles(ipfsBlocks.Uncles, response)
@ -47,35 +48,35 @@ func (eir *EthIPLDResolver) ResolveIPLDs(ipfsBlocks IpldWrapper) (*ResponsePaylo
return response, nil return response, nil
} }
func (eir *EthIPLDResolver) resolveHeaders(blocks []blocks.Block, response *ResponsePayload) { func (eir *EthIPLDResolver) resolveHeaders(blocks []blocks.Block, response *streamer.SeedNodePayload) {
for _, block := range blocks { for _, block := range blocks {
raw := block.RawData() raw := block.RawData()
response.HeadersRlp = append(response.HeadersRlp, raw) response.HeadersRlp = append(response.HeadersRlp, raw)
} }
} }
func (eir *EthIPLDResolver) resolveUncles(blocks []blocks.Block, response *ResponsePayload) { func (eir *EthIPLDResolver) resolveUncles(blocks []blocks.Block, response *streamer.SeedNodePayload) {
for _, block := range blocks { for _, block := range blocks {
raw := block.RawData() raw := block.RawData()
response.UnclesRlp = append(response.UnclesRlp, raw) response.UnclesRlp = append(response.UnclesRlp, raw)
} }
} }
func (eir *EthIPLDResolver) resolveTransactions(blocks []blocks.Block, response *ResponsePayload) { func (eir *EthIPLDResolver) resolveTransactions(blocks []blocks.Block, response *streamer.SeedNodePayload) {
for _, block := range blocks { for _, block := range blocks {
raw := block.RawData() raw := block.RawData()
response.TransactionsRlp = append(response.TransactionsRlp, raw) response.TransactionsRlp = append(response.TransactionsRlp, raw)
} }
} }
func (eir *EthIPLDResolver) resolveReceipts(blocks []blocks.Block, response *ResponsePayload) { func (eir *EthIPLDResolver) resolveReceipts(blocks []blocks.Block, response *streamer.SeedNodePayload) {
for _, block := range blocks { for _, block := range blocks {
raw := block.RawData() raw := block.RawData()
response.ReceiptsRlp = append(response.ReceiptsRlp, raw) response.ReceiptsRlp = append(response.ReceiptsRlp, raw)
} }
} }
func (eir *EthIPLDResolver) resolveState(blocks map[common.Hash]blocks.Block, response *ResponsePayload) { func (eir *EthIPLDResolver) resolveState(blocks map[common.Hash]blocks.Block, response *streamer.SeedNodePayload) {
if response.StateNodesRlp == nil { if response.StateNodesRlp == nil {
response.StateNodesRlp = make(map[common.Hash][]byte) response.StateNodesRlp = make(map[common.Hash][]byte)
} }
@ -85,7 +86,7 @@ func (eir *EthIPLDResolver) resolveState(blocks map[common.Hash]blocks.Block, re
} }
} }
func (eir *EthIPLDResolver) resolveStorage(blocks map[common.Hash]map[common.Hash]blocks.Block, response *ResponsePayload) { func (eir *EthIPLDResolver) resolveStorage(blocks map[common.Hash]map[common.Hash]blocks.Block, response *streamer.SeedNodePayload) {
if response.StateNodesRlp == nil { if response.StateNodesRlp == nil {
response.StorageNodesRlp = make(map[common.Hash]map[common.Hash][]byte) response.StorageNodesRlp = make(map[common.Hash]map[common.Hash][]byte)
} }

1
pkg/ipfs/retreiver_test.go
View File

@ -1 +0,0 @@
package ipfs

1
pkg/ipfs/screener_test.go
View File

@ -1 +0,0 @@
package ipfs

46
pkg/ipfs/streamer.go
View File

@ -1,46 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipfs
import (
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/statediff"
"github.com/vulcanize/vulcanizedb/pkg/core"
)
// StateDiffStreamer is the interface for streaming a statediff subscription
type StateDiffStreamer interface {
Stream(payloadChan chan statediff.Payload) (*rpc.ClientSubscription, error)
}
// Streamer is the underlying struct for the StateDiffStreamer interface
type Streamer struct {
Client core.RpcClient
}
// NewStateDiffStreamer creates a pointer to a new Streamer which satisfies the StateDiffStreamer interface
func NewStateDiffStreamer(client core.RpcClient) *Streamer {
return &Streamer{
Client: client,
}
}
// Stream is the main loop for subscribing to data from the Geth state diff process
func (sds *Streamer) Stream(payloadChan chan statediff.Payload) (*rpc.ClientSubscription, error) {
return sds.Client.Subscribe("statediff", payloadChan, "stream")
}

45
pkg/ipfs/streamer_test.go
View File

@ -1,45 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipfs_test
import (
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/statediff"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks"
)
var _ = Describe("Streamer", func() {
Describe("Stream", func() {
It("Streams StatediffPayloads from a Geth RPC subscription", func() {
mockStreamer := mocks.StateDiffStreamer{}
mockStreamer.ReturnSub = &rpc.ClientSubscription{}
mockStreamer.StreamPayloads = []statediff.Payload{
mocks.MockStatediffPayload,
}
payloadChan := make(chan statediff.Payload, 1)
sub, err := mockStreamer.Stream(payloadChan)
Expect(err).ToNot(HaveOccurred())
Expect(sub).To(Equal(&rpc.ClientSubscription{}))
Expect(mockStreamer.PassedPayloadChan).To(Equal(payloadChan))
streamedPayload := <-payloadChan
Expect(streamedPayload).To(Equal(mocks.MockStatediffPayload))
})
})
})

48
pkg/ipfs/types.go
View File

@ -17,7 +17,6 @@
package ipfs package ipfs
import ( import (
"encoding/json"
"math/big" "math/big"
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
@ -25,47 +24,8 @@ import (
"github.com/ipfs/go-block-format" "github.com/ipfs/go-block-format"
) )
// Subscription holds the information for an individual client subscription // CIDWrapper is used to package CIDs retrieved from the local Postgres cache
type Subscription struct { type CIDWrapper struct {
PayloadChan chan<- ResponsePayload
QuitChan chan<- bool
}
// ResponsePayload holds the data returned from the seed node to the requesting client
type ResponsePayload struct {
BlockNumber *big.Int `json:"blockNumber"`
HeadersRlp [][]byte `json:"headersRlp"`
UnclesRlp [][]byte `json:"unclesRlp"`
TransactionsRlp [][]byte `json:"transactionsRlp"`
ReceiptsRlp [][]byte `json:"receiptsRlp"`
StateNodesRlp map[common.Hash][]byte `json:"stateNodesRlp"`
StorageNodesRlp map[common.Hash]map[common.Hash][]byte `json:"storageNodesRlp"`
ErrMsg string `json:"errMsg"`
encoded []byte
err error
}
func (sd *ResponsePayload) ensureEncoded() {
if sd.encoded == nil && sd.err == nil {
sd.encoded, sd.err = json.Marshal(sd)
}
}
// Length to implement Encoder interface for StateDiff
func (sd *ResponsePayload) Length() int {
sd.ensureEncoded()
return len(sd.encoded)
}
// Encode to implement Encoder interface for StateDiff
func (sd *ResponsePayload) Encode() ([]byte, error) {
sd.ensureEncoded()
return sd.encoded, sd.err
}
// CidWrapper is used to package CIDs retrieved from the local Postgres cache
type CidWrapper struct {
BlockNumber *big.Int BlockNumber *big.Int
Headers []string Headers []string
Uncles []string Uncles []string
@ -75,8 +35,8 @@ type CidWrapper struct {
StorageNodes []StorageNodeCID StorageNodes []StorageNodeCID
} }
// IpldWrapper is used to package raw IPLD block data for resolution // IPLDWrapper is used to package raw IPLD block data for resolution
type IpldWrapper struct { type IPLDWrapper struct {
BlockNumber *big.Int BlockNumber *big.Int
Headers []blocks.Block Headers []blocks.Block
Uncles []blocks.Block Uncles []blocks.Block

12
pkg/ipfs/api.go → pkg/seed_node/api.go
View File

@ -14,11 +14,13 @@
// You should have received a copy of the GNU Affero General Public License // You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>. // along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipfs package seed_node
import ( import (
"context" "context"
"github.com/vulcanize/vulcanizedb/libraries/shared/streamer"
"github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/rpc"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
@ -33,13 +35,13 @@ const APIVersion = "0.0.1"
// PublicSeedNodeAPI is the public api for the seed node // PublicSeedNodeAPI is the public api for the seed node
type PublicSeedNodeAPI struct { type PublicSeedNodeAPI struct {
snp SyncPublishScreenAndServe snp Processor
} }
// NewPublicSeedNodeAPI creates a new PublicSeedNodeAPI with the provided underlying SyncPublishScreenAndServe process // NewPublicSeedNodeAPI creates a new PublicSeedNodeAPI with the provided underlying SyncPublishScreenAndServe process
func NewPublicSeedNodeAPI(snp SyncPublishScreenAndServe) *PublicSeedNodeAPI { func NewPublicSeedNodeAPI(seedNodeProcessor Processor) *PublicSeedNodeAPI {
return &PublicSeedNodeAPI{ return &PublicSeedNodeAPI{
snp: snp, snp: seedNodeProcessor,
} }
} }
@ -56,7 +58,7 @@ func (api *PublicSeedNodeAPI) Stream(ctx context.Context, streamFilters config.S
go func() { go func() {
// subscribe to events from the SyncPublishScreenAndServe service // subscribe to events from the SyncPublishScreenAndServe service
payloadChannel := make(chan ResponsePayload, payloadChanBufferSize) payloadChannel := make(chan streamer.SeedNodePayload, payloadChanBufferSize)
quitChan := make(chan bool, 1) quitChan := make(chan bool, 1)
go api.snp.Subscribe(rpcSub.ID, payloadChannel, quitChan, streamFilters) go api.snp.Subscribe(rpcSub.ID, payloadChannel, quitChan, streamFilters)

43
pkg/ipfs/screener.go → pkg/seed_node/filterer.go
View File

@ -14,11 +14,14 @@
// You should have received a copy of the GNU Affero General Public License // You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>. // along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipfs package seed_node
import ( import (
"bytes" "bytes"
"github.com/vulcanize/vulcanizedb/libraries/shared/streamer"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rlp"
@ -26,22 +29,22 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/config" "github.com/vulcanize/vulcanizedb/pkg/config"
) )
// ResponseScreener is the inteface used to screen eth data and package appropriate data into a response payload // ResponseFilterer is the inteface used to screen eth data and package appropriate data into a response payload
type ResponseScreener interface { type ResponseFilterer interface {
ScreenResponse(streamFilters config.Subscription, payload IPLDPayload) (*ResponsePayload, error) FilterResponse(streamFilters config.Subscription, payload ipfs.IPLDPayload) (*streamer.SeedNodePayload, error)
} }
// Screener is the underlying struct for the ReponseScreener interface // Filterer is the underlying struct for the ReponseFilterer interface
type Screener struct{} type Filterer struct{}
// NewResponseScreener creates a new Screener satisfyign the ReponseScreener interface // NewResponseFilterer creates a new Filterer satisfyign the ReponseFilterer interface
func NewResponseScreener() *Screener { func NewResponseFilterer() *Filterer {
return &Screener{} return &Filterer{}
} }
// ScreenResponse is used to filter through eth data to extract and package requested data into a ResponsePayload // FilterResponse is used to filter through eth data to extract and package requested data into a Payload
func (s *Screener) ScreenResponse(streamFilters config.Subscription, payload IPLDPayload) (*ResponsePayload, error) { func (s *Filterer) FilterResponse(streamFilters config.Subscription, payload ipfs.IPLDPayload) (*streamer.SeedNodePayload, error) {
response := new(ResponsePayload) response := new(streamer.SeedNodePayload)
err := s.filterHeaders(streamFilters, response, payload) err := s.filterHeaders(streamFilters, response, payload)
if err != nil { if err != nil {
return nil, err return nil, err
@ -66,7 +69,7 @@ func (s *Screener) ScreenResponse(streamFilters config.Subscription, payload IPL
return response, nil return response, nil
} }
func (s *Screener) filterHeaders(streamFilters config.Subscription, response *ResponsePayload, payload IPLDPayload) error { func (s *Filterer) filterHeaders(streamFilters config.Subscription, response *streamer.SeedNodePayload, payload ipfs.IPLDPayload) error {
if !streamFilters.HeaderFilter.Off && checkRange(streamFilters.StartingBlock.Int64(), streamFilters.EndingBlock.Int64(), payload.BlockNumber.Int64()) { if !streamFilters.HeaderFilter.Off && checkRange(streamFilters.StartingBlock.Int64(), streamFilters.EndingBlock.Int64(), payload.BlockNumber.Int64()) {
response.HeadersRlp = append(response.HeadersRlp, payload.HeaderRLP) response.HeadersRlp = append(response.HeadersRlp, payload.HeaderRLP)
if !streamFilters.HeaderFilter.FinalOnly { if !streamFilters.HeaderFilter.FinalOnly {
@ -89,7 +92,7 @@ func checkRange(start, end, actual int64) bool {
return false return false
} }
func (s *Screener) filterTransactions(streamFilters config.Subscription, response *ResponsePayload, payload IPLDPayload) ([]common.Hash, error) { func (s *Filterer) filterTransactions(streamFilters config.Subscription, response *streamer.SeedNodePayload, payload ipfs.IPLDPayload) ([]common.Hash, error) {
trxHashes := make([]common.Hash, 0, len(payload.BlockBody.Transactions)) trxHashes := make([]common.Hash, 0, len(payload.BlockBody.Transactions))
if !streamFilters.TrxFilter.Off && checkRange(streamFilters.StartingBlock.Int64(), streamFilters.EndingBlock.Int64(), payload.BlockNumber.Int64()) { if !streamFilters.TrxFilter.Off && checkRange(streamFilters.StartingBlock.Int64(), streamFilters.EndingBlock.Int64(), payload.BlockNumber.Int64()) {
for i, trx := range payload.BlockBody.Transactions { for i, trx := range payload.BlockBody.Transactions {
@ -125,7 +128,7 @@ func checkTransactions(wantedSrc, wantedDst []string, actualSrc, actualDst strin
return false return false
} }
func (s *Screener) filerReceipts(streamFilters config.Subscription, response *ResponsePayload, payload IPLDPayload, trxHashes []common.Hash) error { func (s *Filterer) filerReceipts(streamFilters config.Subscription, response *streamer.SeedNodePayload, payload ipfs.IPLDPayload, trxHashes []common.Hash) error {
if !streamFilters.ReceiptFilter.Off && checkRange(streamFilters.StartingBlock.Int64(), streamFilters.EndingBlock.Int64(), payload.BlockNumber.Int64()) { if !streamFilters.ReceiptFilter.Off && checkRange(streamFilters.StartingBlock.Int64(), streamFilters.EndingBlock.Int64(), payload.BlockNumber.Int64()) {
for i, receipt := range payload.Receipts { for i, receipt := range payload.Receipts {
if checkReceipts(receipt, streamFilters.ReceiptFilter.Topic0s, payload.ReceiptMetaData[i].Topic0s, streamFilters.ReceiptFilter.Contracts, payload.ReceiptMetaData[i].ContractAddress, trxHashes) { if checkReceipts(receipt, streamFilters.ReceiptFilter.Topic0s, payload.ReceiptMetaData[i].Topic0s, streamFilters.ReceiptFilter.Contracts, payload.ReceiptMetaData[i].ContractAddress, trxHashes) {
@ -181,12 +184,12 @@ func checkReceipts(rct *types.Receipt, wantedTopics, actualTopics, wantedContrac
return false return false
} }
func (s *Screener) filterState(streamFilters config.Subscription, response *ResponsePayload, payload IPLDPayload) error { func (s *Filterer) filterState(streamFilters config.Subscription, response *streamer.SeedNodePayload, payload ipfs.IPLDPayload) error {
if !streamFilters.StateFilter.Off && checkRange(streamFilters.StartingBlock.Int64(), streamFilters.EndingBlock.Int64(), payload.BlockNumber.Int64()) { if !streamFilters.StateFilter.Off && checkRange(streamFilters.StartingBlock.Int64(), streamFilters.EndingBlock.Int64(), payload.BlockNumber.Int64()) {
response.StateNodesRlp = make(map[common.Hash][]byte) response.StateNodesRlp = make(map[common.Hash][]byte)
keyFilters := make([]common.Hash, 0, len(streamFilters.StateFilter.Addresses)) keyFilters := make([]common.Hash, 0, len(streamFilters.StateFilter.Addresses))
for _, addr := range streamFilters.StateFilter.Addresses { for _, addr := range streamFilters.StateFilter.Addresses {
keyFilter := AddressToKey(common.HexToAddress(addr)) keyFilter := ipfs.AddressToKey(common.HexToAddress(addr))
keyFilters = append(keyFilters, keyFilter) keyFilters = append(keyFilters, keyFilter)
} }
for key, stateNode := range payload.StateNodes { for key, stateNode := range payload.StateNodes {
@ -213,17 +216,17 @@ func checkNodeKeys(wantedKeys []common.Hash, actualKey common.Hash) bool {
return false return false
} }
func (s *Screener) filterStorage(streamFilters config.Subscription, response *ResponsePayload, payload IPLDPayload) error { func (s *Filterer) filterStorage(streamFilters config.Subscription, response *streamer.SeedNodePayload, payload ipfs.IPLDPayload) error {
if !streamFilters.StorageFilter.Off && checkRange(streamFilters.StartingBlock.Int64(), streamFilters.EndingBlock.Int64(), payload.BlockNumber.Int64()) { if !streamFilters.StorageFilter.Off && checkRange(streamFilters.StartingBlock.Int64(), streamFilters.EndingBlock.Int64(), payload.BlockNumber.Int64()) {
response.StorageNodesRlp = make(map[common.Hash]map[common.Hash][]byte) response.StorageNodesRlp = make(map[common.Hash]map[common.Hash][]byte)
stateKeyFilters := make([]common.Hash, 0, len(streamFilters.StorageFilter.Addresses)) stateKeyFilters := make([]common.Hash, 0, len(streamFilters.StorageFilter.Addresses))
for _, addr := range streamFilters.StorageFilter.Addresses { for _, addr := range streamFilters.StorageFilter.Addresses {
keyFilter := AddressToKey(common.HexToAddress(addr)) keyFilter := ipfs.AddressToKey(common.HexToAddress(addr))
stateKeyFilters = append(stateKeyFilters, keyFilter) stateKeyFilters = append(stateKeyFilters, keyFilter)
} }
storageKeyFilters := make([]common.Hash, 0, len(streamFilters.StorageFilter.StorageKeys)) storageKeyFilters := make([]common.Hash, 0, len(streamFilters.StorageFilter.StorageKeys))
for _, store := range streamFilters.StorageFilter.StorageKeys { for _, store := range streamFilters.StorageFilter.StorageKeys {
keyFilter := HexToKey(store) keyFilter := ipfs.HexToKey(store)
storageKeyFilters = append(storageKeyFilters, keyFilter) storageKeyFilters = append(storageKeyFilters, keyFilter)
} }
for stateKey, storageNodes := range payload.StorageNodes { for stateKey, storageNodes := range payload.StorageNodes {

0
pkg/ipfs/mocks/repository.go → pkg/seed_node/mocks/repository.go
View File

17
pkg/ipfs/repository.go → pkg/seed_node/repository.go
View File

@ -14,18 +14,19 @@
// You should have received a copy of the GNU Affero General Public License // You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>. // along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipfs package seed_node
import ( import (
"github.com/jmoiron/sqlx" "github.com/jmoiron/sqlx"
"github.com/lib/pq" "github.com/lib/pq"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
) )
// CIDRepository is an interface for indexing CIDPayloads // CIDRepository is an interface for indexing ipfs.CIDPayloads
type CIDRepository interface { type CIDRepository interface {
Index(cidPayload *CIDPayload) error Index(cidPayload *ipfs.CIDPayload) error
} }
// Repository is the underlying struct for the CIDRepository interface // Repository is the underlying struct for the CIDRepository interface
@ -41,7 +42,7 @@ func NewCIDRepository(db *postgres.DB) *Repository {
} }
// Index indexes a cidPayload in Postgres // Index indexes a cidPayload in Postgres
func (repo *Repository) Index(cidPayload *CIDPayload) error { func (repo *Repository) Index(cidPayload *ipfs.CIDPayload) error {
tx, err := repo.db.Beginx() tx, err := repo.db.Beginx()
if err != nil { if err != nil {
return err return err
@ -87,7 +88,7 @@ func (repo *Repository) indexUncleCID(tx *sqlx.Tx, cid, blockNumber, hash string
return err return err
} }
func (repo *Repository) indexTransactionAndReceiptCIDs(tx *sqlx.Tx, payload *CIDPayload, headerID int64) error { func (repo *Repository) indexTransactionAndReceiptCIDs(tx *sqlx.Tx, payload *ipfs.CIDPayload, headerID int64) error {
for hash, trxCidMeta := range payload.TransactionCIDs { for hash, trxCidMeta := range payload.TransactionCIDs {
var txID int64 var txID int64
err := tx.QueryRowx(`INSERT INTO public.transaction_cids (header_id, tx_hash, cid, dst, src) VALUES ($1, $2, $3, $4, $5) err := tx.QueryRowx(`INSERT INTO public.transaction_cids (header_id, tx_hash, cid, dst, src) VALUES ($1, $2, $3, $4, $5)
@ -108,13 +109,13 @@ func (repo *Repository) indexTransactionAndReceiptCIDs(tx *sqlx.Tx, payload *CID
return nil return nil
} }
func (repo *Repository) indexReceiptCID(tx *sqlx.Tx, cidMeta *ReceiptMetaData, txID int64) error { func (repo *Repository) indexReceiptCID(tx *sqlx.Tx, cidMeta *ipfs.ReceiptMetaData, txID int64) error {
_, err := tx.Exec(`INSERT INTO public.receipt_cids (tx_id, cid, contract, topic0s) VALUES ($1, $2, $3, $4)`, _, err := tx.Exec(`INSERT INTO public.receipt_cids (tx_id, cid, contract, topic0s) VALUES ($1, $2, $3, $4)`,
txID, cidMeta.CID, cidMeta.ContractAddress, pq.Array(cidMeta.Topic0s)) txID, cidMeta.CID, cidMeta.ContractAddress, pq.Array(cidMeta.Topic0s))
return err return err
} }
func (repo *Repository) indexStateAndStorageCIDs(tx *sqlx.Tx, payload *CIDPayload, headerID int64) error { func (repo *Repository) indexStateAndStorageCIDs(tx *sqlx.Tx, payload *ipfs.CIDPayload, headerID int64) error {
for accountKey, stateCID := range payload.StateNodeCIDs { for accountKey, stateCID := range payload.StateNodeCIDs {
var stateID int64 var stateID int64
err := tx.QueryRowx(`INSERT INTO public.state_cids (header_id, state_key, cid, leaf) VALUES ($1, $2, $3, $4) err := tx.QueryRowx(`INSERT INTO public.state_cids (header_id, state_key, cid, leaf) VALUES ($1, $2, $3, $4)
@ -134,7 +135,7 @@ func (repo *Repository) indexStateAndStorageCIDs(tx *sqlx.Tx, payload *CIDPayloa
return nil return nil
} }
func (repo *Repository) indexStorageCID(tx *sqlx.Tx, storageCID StorageNodeCID, stateID int64) error { func (repo *Repository) indexStorageCID(tx *sqlx.Tx, storageCID ipfs.StorageNodeCID, stateID int64) error {
_, err := tx.Exec(`INSERT INTO public.storage_cids (state_id, storage_key, cid, leaf) VALUES ($1, $2, $3, $4) _, err := tx.Exec(`INSERT INTO public.storage_cids (state_id, storage_key, cid, leaf) VALUES ($1, $2, $3, $4)
ON CONFLICT (state_id, storage_key) DO UPDATE SET (cid, leaf) = ($3, $4)`, ON CONFLICT (state_id, storage_key) DO UPDATE SET (cid, leaf) = ($3, $4)`,
stateID, storageCID.Key, storageCID.CID, storageCID.Leaf) stateID, storageCID.Key, storageCID.CID, storageCID.Leaf)

22
pkg/ipfs/retreiver.go → pkg/seed_node/retreiver.go
View File

@ -14,11 +14,13 @@
// You should have received a copy of the GNU Affero General Public License // You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>. // along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipfs package seed_node
import ( import (
"math/big" "math/big"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
"github.com/jmoiron/sqlx" "github.com/jmoiron/sqlx"
"github.com/lib/pq" "github.com/lib/pq"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
@ -29,7 +31,7 @@ import (
// CIDRetriever is the interface for retrieving CIDs from the Postgres cache // CIDRetriever is the interface for retrieving CIDs from the Postgres cache
type CIDRetriever interface { type CIDRetriever interface {
RetrieveCIDs(streamFilters config.Subscription, blockNumber int64) (*CidWrapper, error) RetrieveCIDs(streamFilters config.Subscription, blockNumber int64) (*ipfs.CIDWrapper, error)
RetrieveLastBlockNumber() (int64, error) RetrieveLastBlockNumber() (int64, error)
RetrieveFirstBlockNumber() (int64, error) RetrieveFirstBlockNumber() (int64, error)
} }
@ -60,7 +62,7 @@ func (ecr *EthCIDRetriever) RetrieveLastBlockNumber() (int64, error) {
} }
// RetrieveCIDs is used to retrieve all of the CIDs which conform to the passed StreamFilters // RetrieveCIDs is used to retrieve all of the CIDs which conform to the passed StreamFilters
func (ecr *EthCIDRetriever) RetrieveCIDs(streamFilters config.Subscription, blockNumber int64) (*CidWrapper, error) { func (ecr *EthCIDRetriever) RetrieveCIDs(streamFilters config.Subscription, blockNumber int64) (*ipfs.CIDWrapper, error) {
log.Debug("retrieving cids") log.Debug("retrieving cids")
var err error var err error
tx, err := ecr.db.Beginx() tx, err := ecr.db.Beginx()
@ -69,7 +71,7 @@ func (ecr *EthCIDRetriever) RetrieveCIDs(streamFilters config.Subscription, bloc
} }
// THIS IS SUPER EXPENSIVE HAVING TO CYCLE THROUGH EACH BLOCK, NEED BETTER WAY TO FETCH CIDS // THIS IS SUPER EXPENSIVE HAVING TO CYCLE THROUGH EACH BLOCK, NEED BETTER WAY TO FETCH CIDS
// WHILE STILL MAINTAINING RELATION INFO ABOUT WHAT BLOCK THE CIDS BELONG TO // WHILE STILL MAINTAINING RELATION INFO ABOUT WHAT BLOCK THE CIDS BELONG TO
cw := new(CidWrapper) cw := new(ipfs.CIDWrapper)
cw.BlockNumber = big.NewInt(blockNumber) cw.BlockNumber = big.NewInt(blockNumber)
// Retrieve cached header CIDs // Retrieve cached header CIDs
@ -212,7 +214,7 @@ func (ecr *EthCIDRetriever) retrieveRctCIDs(tx *sqlx.Tx, streamFilters config.Su
return receiptCids, err return receiptCids, err
} }
func (ecr *EthCIDRetriever) retrieveStateCIDs(tx *sqlx.Tx, streamFilters config.Subscription, blockNumber int64) ([]StateNodeCID, error) { func (ecr *EthCIDRetriever) retrieveStateCIDs(tx *sqlx.Tx, streamFilters config.Subscription, blockNumber int64) ([]ipfs.StateNodeCID, error) {
log.Debug("retrieving state cids for block ", blockNumber) log.Debug("retrieving state cids for block ", blockNumber)
args := make([]interface{}, 0, 2) args := make([]interface{}, 0, 2)
pgStr := `SELECT state_cids.cid, state_cids.state_key FROM state_cids INNER JOIN header_cids ON (state_cids.header_id = header_cids.id) pgStr := `SELECT state_cids.cid, state_cids.state_key FROM state_cids INNER JOIN header_cids ON (state_cids.header_id = header_cids.id)
@ -222,7 +224,7 @@ func (ecr *EthCIDRetriever) retrieveStateCIDs(tx *sqlx.Tx, streamFilters config.
if addrLen > 0 { if addrLen > 0 {
keys := make([]string, 0, addrLen) keys := make([]string, 0, addrLen)
for _, addr := range streamFilters.StateFilter.Addresses { for _, addr := range streamFilters.StateFilter.Addresses {
keys = append(keys, HexToKey(addr).Hex()) keys = append(keys, ipfs.HexToKey(addr).Hex())
} }
pgStr += ` AND state_cids.state_key = ANY($2::VARCHAR(66)[])` pgStr += ` AND state_cids.state_key = ANY($2::VARCHAR(66)[])`
args = append(args, pq.Array(keys)) args = append(args, pq.Array(keys))
@ -230,12 +232,12 @@ func (ecr *EthCIDRetriever) retrieveStateCIDs(tx *sqlx.Tx, streamFilters config.
if !streamFilters.StorageFilter.IntermediateNodes { if !streamFilters.StorageFilter.IntermediateNodes {
pgStr += ` AND state_cids.leaf = TRUE` pgStr += ` AND state_cids.leaf = TRUE`
} }
stateNodeCIDs := make([]StateNodeCID, 0) stateNodeCIDs := make([]ipfs.StateNodeCID, 0)
err := tx.Select(&stateNodeCIDs, pgStr, args...) err := tx.Select(&stateNodeCIDs, pgStr, args...)
return stateNodeCIDs, err return stateNodeCIDs, err
} }
func (ecr *EthCIDRetriever) retrieveStorageCIDs(tx *sqlx.Tx, streamFilters config.Subscription, blockNumber int64) ([]StorageNodeCID, error) { func (ecr *EthCIDRetriever) retrieveStorageCIDs(tx *sqlx.Tx, streamFilters config.Subscription, blockNumber int64) ([]ipfs.StorageNodeCID, error) {
log.Debug("retrieving storage cids for block ", blockNumber) log.Debug("retrieving storage cids for block ", blockNumber)
args := make([]interface{}, 0, 3) args := make([]interface{}, 0, 3)
pgStr := `SELECT storage_cids.cid, state_cids.state_key, storage_cids.storage_key FROM storage_cids, state_cids, header_cids pgStr := `SELECT storage_cids.cid, state_cids.state_key, storage_cids.storage_key FROM storage_cids, state_cids, header_cids
@ -247,7 +249,7 @@ func (ecr *EthCIDRetriever) retrieveStorageCIDs(tx *sqlx.Tx, streamFilters confi
if addrLen > 0 { if addrLen > 0 {
keys := make([]string, 0, addrLen) keys := make([]string, 0, addrLen)
for _, addr := range streamFilters.StorageFilter.Addresses { for _, addr := range streamFilters.StorageFilter.Addresses {
keys = append(keys, HexToKey(addr).Hex()) keys = append(keys, ipfs.HexToKey(addr).Hex())
} }
pgStr += ` AND state_cids.state_key = ANY($2::VARCHAR(66)[])` pgStr += ` AND state_cids.state_key = ANY($2::VARCHAR(66)[])`
args = append(args, pq.Array(keys)) args = append(args, pq.Array(keys))
@ -259,7 +261,7 @@ func (ecr *EthCIDRetriever) retrieveStorageCIDs(tx *sqlx.Tx, streamFilters confi
if !streamFilters.StorageFilter.IntermediateNodes { if !streamFilters.StorageFilter.IntermediateNodes {
pgStr += ` AND storage_cids.leaf = TRUE` pgStr += ` AND storage_cids.leaf = TRUE`
} }
storageNodeCIDs := make([]StorageNodeCID, 0) storageNodeCIDs := make([]ipfs.StorageNodeCID, 0)
err := tx.Select(&storageNodeCIDs, pgStr, args...) err := tx.Select(&storageNodeCIDs, pgStr, args...)
return storageNodeCIDs, err return storageNodeCIDs, err
} }

6
pkg/ipfs/ipfs_suite_test.go → pkg/seed_node/seed_node_suite_test.go
View File

@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License // You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>. // along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipfs_test package seed_node_test
import ( import (
"io/ioutil" "io/ioutil"
@ -26,9 +26,9 @@ import (
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
) )
func TestIPFS(t *testing.T) { func TestSeedNode(t *testing.T) {
RegisterFailHandler(Fail) RegisterFailHandler(Fail)
RunSpecs(t, "IPFS Suite Test") RunSpecs(t, "Seed Node Suite Test")
} }
var _ = BeforeSuite(func() { var _ = BeforeSuite(func() {

71
pkg/ipfs/service.go → pkg/seed_node/service.go
View File

@ -14,11 +14,14 @@
// You should have received a copy of the GNU Affero General Public License // You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>. // along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipfs package seed_node
import ( import (
"sync" "sync"
"github.com/vulcanize/vulcanizedb/libraries/shared/streamer"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/node"
@ -36,18 +39,18 @@ import (
const payloadChanBufferSize = 20000 // the max eth sub buffer size const payloadChanBufferSize = 20000 // the max eth sub buffer size
const workerPoolSize = 1 const workerPoolSize = 1
// SyncPublishScreenAndServe is the top level interface for streaming, converting to IPLDs, publishing, // Processor is the top level interface for streaming, converting to IPLDs, publishing,
// and indexing all Ethereum data; screening this data; and serving it up to subscribed clients // and indexing all Ethereum data; screening this data; and serving it up to subscribed clients
// This service is compatible with the Ethereum service interface (node.Service) // This service is compatible with the Ethereum service interface (node.Service)
type SyncPublishScreenAndServe interface { type Processor interface {
// APIs(), Protocols(), Start() and Stop() // APIs(), Protocols(), Start() and Stop()
node.Service node.Service
// Main event loop for syncAndPublish processes // Main event loop for syncAndPublish processes
SyncAndPublish(wg *sync.WaitGroup, forwardPayloadChan chan<- IPLDPayload, forwardQuitchan chan<- bool) error SyncAndPublish(wg *sync.WaitGroup, forwardPayloadChan chan<- ipfs.IPLDPayload, forwardQuitchan chan<- bool) error
// Main event loop for handling client pub-sub // Main event loop for handling client pub-sub
ScreenAndServe(screenAndServePayload <-chan IPLDPayload, screenAndServeQuit <-chan bool) ScreenAndServe(screenAndServePayload <-chan ipfs.IPLDPayload, screenAndServeQuit <-chan bool)
// Method to subscribe to receive state diff processing output // Method to subscribe to receive state diff processing output
Subscribe(id rpc.ID, sub chan<- ResponsePayload, quitChan chan<- bool, streamFilters config.Subscription) Subscribe(id rpc.ID, sub chan<- streamer.SeedNodePayload, quitChan chan<- bool, streamFilters config.Subscription)
// Method to unsubscribe from state diff processing // Method to unsubscribe from state diff processing
Unsubscribe(id rpc.ID) Unsubscribe(id rpc.ID)
} }
@ -57,21 +60,21 @@ type Service struct {
// Used to sync access to the Subscriptions // Used to sync access to the Subscriptions
sync.Mutex sync.Mutex
// Interface for streaming statediff payloads over a geth rpc subscription // Interface for streaming statediff payloads over a geth rpc subscription
Streamer StateDiffStreamer Streamer streamer.IStateDiffStreamer
// Interface for converting statediff payloads into ETH-IPLD object payloads // Interface for converting statediff payloads into ETH-IPLD object payloads
Converter PayloadConverter Converter ipfs.PayloadConverter
// Interface for publishing the ETH-IPLD payloads to IPFS // Interface for publishing the ETH-IPLD payloads to IPFS
Publisher IPLDPublisher Publisher ipfs.IPLDPublisher
// Interface for indexing the CIDs of the published ETH-IPLDs in Postgres // Interface for indexing the CIDs of the published ETH-IPLDs in Postgres
Repository CIDRepository Repository CIDRepository
// Interface for filtering and serving data according to subscribed clients according to their specification // Interface for filtering and serving data according to subscribed clients according to their specification
Screener ResponseScreener Filterer ResponseFilterer
// Interface for fetching ETH-IPLD objects from IPFS // Interface for fetching ETH-IPLD objects from IPFS
Fetcher IPLDFetcher Fetcher ipfs.IPLDFetcher
// Interface for searching and retrieving CIDs from Postgres index // Interface for searching and retrieving CIDs from Postgres index
Retriever CIDRetriever Retriever CIDRetriever
// Interface for resolving ipfs blocks to their data types // Interface for resolving ipfs blocks to their data types
Resolver IPLDResolver Resolver ipfs.IPLDResolver
// Chan the processor uses to subscribe to state diff payloads from the Streamer // Chan the processor uses to subscribe to state diff payloads from the Streamer
PayloadChan chan statediff.Payload PayloadChan chan statediff.Payload
// Used to signal shutdown of the service // Used to signal shutdown of the service
@ -82,25 +85,25 @@ type Service struct {
SubscriptionTypes map[common.Hash]config.Subscription SubscriptionTypes map[common.Hash]config.Subscription
} }
// NewIPFSProcessor creates a new Processor interface using an underlying Processor struct // NewProcessor creates a new Processor interface using an underlying Service struct
func NewIPFSProcessor(ipfsPath string, db *postgres.DB, ethClient core.EthClient, rpcClient core.RpcClient, qc chan bool) (SyncPublishScreenAndServe, error) { func NewProcessor(ipfsPath string, db *postgres.DB, ethClient core.EthClient, rpcClient core.RpcClient, qc chan bool) (Processor, error) {
publisher, err := NewIPLDPublisher(ipfsPath) publisher, err := ipfs.NewIPLDPublisher(ipfsPath)
if err != nil { if err != nil {
return nil, err return nil, err
} }
fetcher, err := NewIPLDFetcher(ipfsPath) fetcher, err := ipfs.NewIPLDFetcher(ipfsPath)
if err != nil { if err != nil {
return nil, err return nil, err
} }
return &Service{ return &Service{
Streamer: NewStateDiffStreamer(rpcClient), Streamer: streamer.NewStateDiffStreamer(rpcClient),
Repository: NewCIDRepository(db), Repository: NewCIDRepository(db),
Converter: NewPayloadConverter(ethClient), Converter: ipfs.NewPayloadConverter(ethClient),
Publisher: publisher, Publisher: publisher,
Screener: NewResponseScreener(), Filterer: NewResponseFilterer(),
Fetcher: fetcher, Fetcher: fetcher,
Retriever: NewCIDRetriever(db), Retriever: NewCIDRetriever(db),
Resolver: NewIPLDResolver(), Resolver: ipfs.NewIPLDResolver(),
PayloadChan: make(chan statediff.Payload, payloadChanBufferSize), PayloadChan: make(chan statediff.Payload, payloadChanBufferSize),
QuitChan: qc, QuitChan: qc,
Subscriptions: make(map[common.Hash]map[rpc.ID]Subscription), Subscriptions: make(map[common.Hash]map[rpc.ID]Subscription),
@ -128,7 +131,7 @@ func (sap *Service) APIs() []rpc.API {
// SyncAndPublish is the backend processing loop which streams data from geth, converts it to iplds, publishes them to ipfs, and indexes their cids // SyncAndPublish is the backend processing loop which streams data from geth, converts it to iplds, publishes them to ipfs, and indexes their cids
// This continues on no matter if or how many subscribers there are, it then forwards the data to the ScreenAndServe() loop // This continues on no matter if or how many subscribers there are, it then forwards the data to the ScreenAndServe() loop
// which filters and sends relevant data to client subscriptions, if there are any // which filters and sends relevant data to client subscriptions, if there are any
func (sap *Service) SyncAndPublish(wg *sync.WaitGroup, screenAndServePayload chan<- IPLDPayload, screenAndServeQuit chan<- bool) error { func (sap *Service) SyncAndPublish(wg *sync.WaitGroup, screenAndServePayload chan<- ipfs.IPLDPayload, screenAndServeQuit chan<- bool) error {
sub, err := sap.Streamer.Stream(sap.PayloadChan) sub, err := sap.Streamer.Stream(sap.PayloadChan)
if err != nil { if err != nil {
return err return err
@ -136,7 +139,7 @@ func (sap *Service) SyncAndPublish(wg *sync.WaitGroup, screenAndServePayload cha
wg.Add(1) wg.Add(1)
// Channels for forwarding data to the publishAndIndex workers // Channels for forwarding data to the publishAndIndex workers
publishAndIndexPayload := make(chan IPLDPayload, payloadChanBufferSize) publishAndIndexPayload := make(chan ipfs.IPLDPayload, payloadChanBufferSize)
publishAndIndexQuit := make(chan bool, workerPoolSize) publishAndIndexQuit := make(chan bool, workerPoolSize)
// publishAndIndex worker pool to handle publishing and indexing concurrently, while // publishAndIndex worker pool to handle publishing and indexing concurrently, while
// limiting the number of Postgres connections we can possibly open so as to prevent error // limiting the number of Postgres connections we can possibly open so as to prevent error
@ -192,7 +195,7 @@ func (sap *Service) SyncAndPublish(wg *sync.WaitGroup, screenAndServePayload cha
return nil return nil
} }
func (sap *Service) publishAndIndex(id int, publishAndIndexPayload <-chan IPLDPayload, publishAndIndexQuit <-chan bool) { func (sap *Service) publishAndIndex(id int, publishAndIndexPayload <-chan ipfs.IPLDPayload, publishAndIndexQuit <-chan bool) {
go func() { go func() {
for { for {
select { select {
@ -216,7 +219,7 @@ func (sap *Service) publishAndIndex(id int, publishAndIndexPayload <-chan IPLDPa
// ScreenAndServe is the loop used to screen data streamed from the state diffing eth node // ScreenAndServe is the loop used to screen data streamed from the state diffing eth node
// and send the appropriate portions of it to a requesting client subscription, according to their subscription configuration // and send the appropriate portions of it to a requesting client subscription, according to their subscription configuration
func (sap *Service) ScreenAndServe(screenAndServePayload <-chan IPLDPayload, screenAndServeQuit <-chan bool) { func (sap *Service) ScreenAndServe(screenAndServePayload <-chan ipfs.IPLDPayload, screenAndServeQuit <-chan bool) {
go func() { go func() {
for { for {
select { select {
@ -233,7 +236,7 @@ func (sap *Service) ScreenAndServe(screenAndServePayload <-chan IPLDPayload, scr
}() }()
} }
func (sap *Service) sendResponse(payload IPLDPayload) error { func (sap *Service) sendResponse(payload ipfs.IPLDPayload) error {
sap.Lock() sap.Lock()
for ty, subs := range sap.Subscriptions { for ty, subs := range sap.Subscriptions {
// Retrieve the subscription parameters for this subscription type // Retrieve the subscription parameters for this subscription type
@ -242,7 +245,7 @@ func (sap *Service) sendResponse(payload IPLDPayload) error {
log.Errorf("subscription configuration for subscription type %s not available", ty.Hex()) log.Errorf("subscription configuration for subscription type %s not available", ty.Hex())
continue continue
} }
response, err := sap.Screener.ScreenResponse(subConfig, payload) response, err := sap.Filterer.FilterResponse(subConfig, payload)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
continue continue
@ -261,7 +264,7 @@ func (sap *Service) sendResponse(payload IPLDPayload) error {
} }
// Subscribe is used by the API to subscribe to the service loop // Subscribe is used by the API to subscribe to the service loop
func (sap *Service) Subscribe(id rpc.ID, sub chan<- ResponsePayload, quitChan chan<- bool, streamFilters config.Subscription) { func (sap *Service) Subscribe(id rpc.ID, sub chan<- streamer.SeedNodePayload, quitChan chan<- bool, streamFilters config.Subscription) {
log.Info("Subscribing to the seed node service") log.Info("Subscribing to the seed node service")
// Subscription type is defined as the hash of its content // Subscription type is defined as the hash of its content
// Group subscriptions by type and screen payloads once for subs of the same type // Group subscriptions by type and screen payloads once for subs of the same type
@ -299,7 +302,7 @@ func (sap *Service) backFill(sub Subscription, id rpc.ID, con config.Subscriptio
var err error var err error
startingBlock, err = sap.Retriever.RetrieveFirstBlockNumber() startingBlock, err = sap.Retriever.RetrieveFirstBlockNumber()
if err != nil { if err != nil {
sub.PayloadChan <- ResponsePayload{ sub.PayloadChan <- streamer.SeedNodePayload{
ErrMsg: "unable to set block range; error: " + err.Error(), ErrMsg: "unable to set block range; error: " + err.Error(),
} }
} }
@ -309,7 +312,7 @@ func (sap *Service) backFill(sub Subscription, id rpc.ID, con config.Subscriptio
if con.EndingBlock.Int64() <= 0 || con.EndingBlock.Int64() <= startingBlock { if con.EndingBlock.Int64() <= 0 || con.EndingBlock.Int64() <= startingBlock {
endingBlock, err = sap.Retriever.RetrieveLastBlockNumber() endingBlock, err = sap.Retriever.RetrieveLastBlockNumber()
if err != nil { if err != nil {
sub.PayloadChan <- ResponsePayload{ sub.PayloadChan <- streamer.SeedNodePayload{
ErrMsg: "unable to set block range; error: " + err.Error(), ErrMsg: "unable to set block range; error: " + err.Error(),
} }
} }
@ -323,18 +326,18 @@ func (sap *Service) backFill(sub Subscription, id rpc.ID, con config.Subscriptio
for i := con.StartingBlock.Int64(); i <= endingBlock; i++ { for i := con.StartingBlock.Int64(); i <= endingBlock; i++ {
cidWrapper, err := sap.Retriever.RetrieveCIDs(con, i) cidWrapper, err := sap.Retriever.RetrieveCIDs(con, i)
if err != nil { if err != nil {
sub.PayloadChan <- ResponsePayload{ sub.PayloadChan <- streamer.SeedNodePayload{
ErrMsg: "CID retrieval error: " + err.Error(), ErrMsg: "CID retrieval error: " + err.Error(),
} }
continue continue
} }
if emptyCidWrapper(*cidWrapper) { if ipfs.EmptyCIDWrapper(*cidWrapper) {
continue continue
} }
blocksWrapper, err := sap.Fetcher.FetchCIDs(*cidWrapper) blocksWrapper, err := sap.Fetcher.FetchCIDs(*cidWrapper)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
sub.PayloadChan <- ResponsePayload{ sub.PayloadChan <- streamer.SeedNodePayload{
ErrMsg: "IPLD fetching error: " + err.Error(), ErrMsg: "IPLD fetching error: " + err.Error(),
} }
continue continue
@ -342,7 +345,7 @@ func (sap *Service) backFill(sub Subscription, id rpc.ID, con config.Subscriptio
backFillIplds, err := sap.Resolver.ResolveIPLDs(*blocksWrapper) backFillIplds, err := sap.Resolver.ResolveIPLDs(*blocksWrapper)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
sub.PayloadChan <- ResponsePayload{ sub.PayloadChan <- streamer.SeedNodePayload{
ErrMsg: "IPLD resolving error: " + err.Error(), ErrMsg: "IPLD resolving error: " + err.Error(),
} }
continue continue
@ -376,7 +379,7 @@ func (sap *Service) Unsubscribe(id rpc.ID) {
func (sap *Service) Start(*p2p.Server) error { func (sap *Service) Start(*p2p.Server) error {
log.Info("Starting seed node service") log.Info("Starting seed node service")
wg := new(sync.WaitGroup) wg := new(sync.WaitGroup)
payloadChan := make(chan IPLDPayload, payloadChanBufferSize) payloadChan := make(chan ipfs.IPLDPayload, payloadChanBufferSize)
quitChan := make(chan bool, 1) quitChan := make(chan bool, 1)
if err := sap.SyncAndPublish(wg, payloadChan, quitChan); err != nil { if err := sap.SyncAndPublish(wg, payloadChan, quitChan); err != nil {
return err return err

13
pkg/ipfs/service_test.go → pkg/seed_node/service_test.go
View File

@ -14,19 +14,22 @@
// You should have received a copy of the GNU Affero General Public License // You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>. // along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipfs_test package seed_node_test
import ( import (
"sync" "sync"
"time" "time"
"github.com/vulcanize/vulcanizedb/pkg/seed_node"
"github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/statediff" "github.com/ethereum/go-ethereum/statediff"
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/ipfs" mocks2 "github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks" "github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks"
mocks3 "github.com/vulcanize/vulcanizedb/pkg/seed_node/mocks"
) )
var _ = Describe("Service", func() { var _ = Describe("Service", func() {
@ -36,14 +39,14 @@ var _ = Describe("Service", func() {
wg := new(sync.WaitGroup) wg := new(sync.WaitGroup)
payloadChan := make(chan statediff.Payload, 1) payloadChan := make(chan statediff.Payload, 1)
quitChan := make(chan bool, 1) quitChan := make(chan bool, 1)
mockCidRepo := &mocks.CIDRepository{ mockCidRepo := &mocks3.CIDRepository{
ReturnErr: nil, ReturnErr: nil,
} }
mockPublisher := &mocks.IPLDPublisher{ mockPublisher := &mocks.IPLDPublisher{
ReturnCIDPayload: &mocks.MockCIDPayload, ReturnCIDPayload: &mocks.MockCIDPayload,
ReturnErr: nil, ReturnErr: nil,
} }
mockStreamer := &mocks.StateDiffStreamer{ mockStreamer := &mocks2.StateDiffStreamer{
ReturnSub: &rpc.ClientSubscription{}, ReturnSub: &rpc.ClientSubscription{},
StreamPayloads: []statediff.Payload{ StreamPayloads: []statediff.Payload{
mocks.MockStatediffPayload, mocks.MockStatediffPayload,
@ -54,7 +57,7 @@ var _ = Describe("Service", func() {
ReturnIPLDPayload: &mocks.MockIPLDPayload, ReturnIPLDPayload: &mocks.MockIPLDPayload,
ReturnErr: nil, ReturnErr: nil,
} }
processor := &ipfs.Service{ processor := &seed_node.Service{
Repository: mockCidRepo, Repository: mockCidRepo,
Publisher: mockPublisher, Publisher: mockPublisher,
Streamer: mockStreamer, Streamer: mockStreamer,

12
pkg/ipfs/fetcher_test.go → pkg/seed_node/subscription.go
View File

@ -14,4 +14,14 @@
// You should have received a copy of the GNU Affero General Public License // You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>. // along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipfs package seed_node
import (
"github.com/vulcanize/vulcanizedb/libraries/shared/streamer"
)
// Subscription holds the information for an individual client subscription to the seed node
type Subscription struct {
PayloadChan chan<- streamer.SeedNodePayload
QuitChan chan<- bool
}