cerc-io/ipld-eth-server
12
0
Fork 1
Code Issues 10 Pull Requests 2 Actions Packages Projects Releases 37 Wiki Activity

goimports + streamSubscribe command for raw access to the seed node data

Browse Source
This commit is contained in:
Ian Norden
2019-12-02 13:24:51 -06:00
parent 8ccdfd4835
commit b1bb646ad5
26 changed files with 362 additions and 234 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/ipfs/api.go
+3 -1
View File
@@ -19,6 +19,8 @@ package ipfs
import (
"context"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/rpc"
)
@@ -42,7 +44,7 @@ func NewPublicSeedNodeAPI(snp SyncPublishScreenAndServe) *PublicSeedNodeAPI {
}
// Stream is the public method to setup a subscription that fires off SyncPublishScreenAndServe payloads as they are created
func (api *PublicSeedNodeAPI) Stream(ctx context.Context, streamFilters StreamFilters) (*rpc.Subscription, error) {
func (api *PublicSeedNodeAPI) Stream(ctx context.Context, streamFilters config.Subscription) (*rpc.Subscription, error) {
// ensure that the RPC connection supports subscriptions
notifier, supported := rpc.NotifierFromContext(ctx)
if !supported {
pkg/ipfs/converter_test.go
+6 -6
View File
@@ -20,19 +20,19 @@ import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/test_helpers"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/test_helpers/mocks"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/helpers"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/helpers/mocks"
)
var _ = Describe("Converter", func() {
Describe("Convert", func() {
It("Converts StatediffPayloads into IPLDPayloads", func() {
mockConverter := mocks.PayloadConverter{}
mockConverter.ReturnIPLDPayload = &test_helpers.MockIPLDPayload
ipldPayload, err := mockConverter.Convert(test_helpers.MockStatediffPayload)
mockConverter.ReturnIPLDPayload = &helpers.MockIPLDPayload
ipldPayload, err := mockConverter.Convert(helpers.MockStatediffPayload)
Expect(err).ToNot(HaveOccurred())
Expect(ipldPayload).To(Equal(&test_helpers.MockIPLDPayload))
Expect(mockConverter.PassedStatediffPayload).To(Equal(test_helpers.MockStatediffPayload))
Expect(ipldPayload).To(Equal(&helpers.MockIPLDPayload))
Expect(mockConverter.PassedStatediffPayload).To(Equal(helpers.MockStatediffPayload))
})
})
})
pkg/ipfs/test_helpers/mocks/api.go → pkg/ipfs/helpers/mocks/api.go
View File
pkg/ipfs/test_helpers/mocks/converter.go → pkg/ipfs/helpers/mocks/converter.go
View File
pkg/ipfs/test_helpers/mocks/publisher.go → pkg/ipfs/helpers/mocks/publisher.go
View File
pkg/ipfs/helpers/mocks/repository.go
+31
View File
@@ -0,0 +1,31 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package mocks
import "github.com/vulcanize/vulcanizedb/pkg/ipfs"
// CIDRepository is the underlying struct for the Repository interface
type CIDRepository struct {
PassedCIDPayload *ipfs.CIDPayload
ReturnErr error
}
// Index indexes a cidPayload in Postgres
func (repo *CIDRepository) Index(cidPayload *ipfs.CIDPayload) error {
repo.PassedCIDPayload = cidPayload
return repo.ReturnErr
}
pkg/ipfs/test_helpers/mocks/screener.go → pkg/ipfs/helpers/mocks/screener.go
View File
pkg/ipfs/test_helpers/mocks/streamer.go → pkg/ipfs/helpers/mocks/streamer.go
View File
pkg/ipfs/test_helpers/test_data.go → pkg/ipfs/helpers/test_data.go
+1 -1
View File
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package test_helpers
package helpers
import (
"errors"
pkg/ipfs/publisher_test.go
+6 -6
View File
@@ -20,19 +20,19 @@ import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/test_helpers"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/test_helpers/mocks"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/helpers"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/helpers/mocks"
)
var _ = Describe("Publisher", func() {
Describe("Publish", func() {
It("Publishes IPLDPayload to IPFS", func() {
mockPublisher := mocks.IPLDPublisher{}
mockPublisher.ReturnCIDPayload = &test_helpers.MockCIDPayload
cidPayload, err := mockPublisher.Publish(&test_helpers.MockIPLDPayload)
mockPublisher.ReturnCIDPayload = &helpers.MockCIDPayload
cidPayload, err := mockPublisher.Publish(&helpers.MockIPLDPayload)
Expect(err).ToNot(HaveOccurred())
Expect(cidPayload).To(Equal(&test_helpers.MockCIDPayload))
Expect(mockPublisher.PassedIPLDPayload).To(Equal(&test_helpers.MockIPLDPayload))
Expect(cidPayload).To(Equal(&helpers.MockCIDPayload))
Expect(mockPublisher.PassedIPLDPayload).To(Equal(&helpers.MockIPLDPayload))
})
})
})
pkg/ipfs/repository.go
+5
View File
@@ -17,6 +17,7 @@
package ipfs
import (
"github.com/i-norden/go-ethereum/core"
"github.com/jmoiron/sqlx"
"github.com/lib/pq"
@@ -144,3 +145,7 @@ func (repo *Repository) indexStorageCID(tx *sqlx.Tx, storageCID StorageNodeCID,
stateID, storageCID.Key, storageCID.CID, storageCID.Leaf)
return err
}
type RepositoryError struct {
core.Message
}
pkg/ipfs/repository_test.go
+4 -4
View File
@@ -20,17 +20,17 @@ import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/test_helpers"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/test_helpers/mocks"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/helpers"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/helpers/mocks"
)
var _ = Describe("Repository", func() {
Describe("Index", func() {
It("Indexes CIDs against their metadata", func() {
mockRepo := mocks.CIDRepository{}
err := mockRepo.Index(&test_helpers.MockCIDPayload)
err := mockRepo.Index(&helpers.MockCIDPayload)
Expect(err).ToNot(HaveOccurred())
Expect(mockRepo.PassedCIDPayload).To(Equal(&test_helpers.MockCIDPayload))
Expect(mockRepo.PassedCIDPayload).To(Equal(&helpers.MockCIDPayload))
})
})
})
pkg/ipfs/retreiver.go
+8 -7
View File
@@ -19,12 +19,13 @@ package ipfs
import (
"github.com/jmoiron/sqlx"
"github.com/lib/pq"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
// CIDRetriever is the interface for retrieving CIDs from the Postgres cache
type CIDRetriever interface {
RetrieveCIDs(streamFilters StreamFilters) ([]CidWrapper, error)
RetrieveCIDs(streamFilters config.Subscription) ([]CidWrapper, error)
}
// EthCIDRetriever is the underlying struct supporting the CIDRetriever interface
@@ -47,7 +48,7 @@ func (ecr *EthCIDRetriever) GetLastBlockNumber() (int64, error) {
}
// RetrieveCIDs is used to retrieve all of the CIDs which conform to the passed StreamFilters
func (ecr *EthCIDRetriever) RetrieveCIDs(streamFilters StreamFilters) ([]CidWrapper, error) {
func (ecr *EthCIDRetriever) RetrieveCIDs(streamFilters config.Subscription) ([]CidWrapper, error) {
var endingBlock int64
var err error
if streamFilters.EndingBlock <= 0 || streamFilters.EndingBlock <= streamFilters.StartingBlock {
@@ -112,7 +113,7 @@ func (ecr *EthCIDRetriever) RetrieveCIDs(streamFilters StreamFilters) ([]CidWrap
return cids, err
}
func (ecr *EthCIDRetriever) retrieveHeaderCIDs(tx *sqlx.Tx, streamFilters StreamFilters, cids *CidWrapper, blockNumber int64) error {
func (ecr *EthCIDRetriever) retrieveHeaderCIDs(tx *sqlx.Tx, streamFilters config.Subscription, cids *CidWrapper, blockNumber int64) error {
var pgStr string
if streamFilters.HeaderFilter.FinalOnly {
pgStr = `SELECT cid FROM header_cids
@@ -125,7 +126,7 @@ func (ecr *EthCIDRetriever) retrieveHeaderCIDs(tx *sqlx.Tx, streamFilters Stream
return tx.Select(cids.Headers, pgStr, blockNumber)
}
func (ecr *EthCIDRetriever) retrieveTrxCIDs(tx *sqlx.Tx, streamFilters StreamFilters, cids *CidWrapper, blockNumber int64) ([]int64, error) {
func (ecr *EthCIDRetriever) retrieveTrxCIDs(tx *sqlx.Tx, streamFilters config.Subscription, cids *CidWrapper, blockNumber int64) ([]int64, error) {
args := make([]interface{}, 0, 3)
type result struct {
ID int64 `db:"id"`
@@ -155,7 +156,7 @@ func (ecr *EthCIDRetriever) retrieveTrxCIDs(tx *sqlx.Tx, streamFilters StreamFil
return ids, nil
}
func (ecr *EthCIDRetriever) retrieveRctCIDs(tx *sqlx.Tx, streamFilters StreamFilters, cids *CidWrapper, blockNumber int64, trxIds []int64) error {
func (ecr *EthCIDRetriever) retrieveRctCIDs(tx *sqlx.Tx, streamFilters config.Subscription, cids *CidWrapper, blockNumber int64, trxIds []int64) error {
args := make([]interface{}, 0, 2)
pgStr := `SELECT receipt_cids.cid FROM receipt_cids, transaction_cids, header_cids
WHERE receipt_cids.tx_id = transaction_cids.id
@@ -175,7 +176,7 @@ func (ecr *EthCIDRetriever) retrieveRctCIDs(tx *sqlx.Tx, streamFilters StreamFil
return tx.Select(cids.Receipts, pgStr, args...)
}
func (ecr *EthCIDRetriever) retrieveStateCIDs(tx *sqlx.Tx, streamFilters StreamFilters, cids *CidWrapper, blockNumber int64) error {
func (ecr *EthCIDRetriever) retrieveStateCIDs(tx *sqlx.Tx, streamFilters config.Subscription, cids *CidWrapper, blockNumber int64) error {
args := make([]interface{}, 0, 2)
pgStr := `SELECT state_cids.cid, state_cids.state_key FROM state_cids INNER JOIN header_cids ON (state_cids.header_id = header_cids.id)
WHERE header_cids.block_number = $1`
@@ -192,7 +193,7 @@ func (ecr *EthCIDRetriever) retrieveStateCIDs(tx *sqlx.Tx, streamFilters StreamF
return tx.Select(cids.StateNodes, pgStr, args...)
}
func (ecr *EthCIDRetriever) retrieveStorageCIDs(tx *sqlx.Tx, streamFilters StreamFilters, cids *CidWrapper, blockNumber int64) error {
func (ecr *EthCIDRetriever) retrieveStorageCIDs(tx *sqlx.Tx, streamFilters config.Subscription, cids *CidWrapper, blockNumber int64) error {
args := make([]interface{}, 0, 3)
pgStr := `SELECT storage_cids.cid, state_cids.state_key, storage_cids.storage_key FROM storage_cids, state_cids, header_cids
WHERE storage_cids.state_id = state_cids.id
pkg/ipfs/screener.go
+9 -7
View File
@@ -19,6 +19,8 @@ package ipfs
import (
"bytes"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/common"
@@ -27,7 +29,7 @@ import (
// ResponseScreener is the inteface used to screen eth data and package appropriate data into a response payload
type ResponseScreener interface {
ScreenResponse(streamFilters *StreamFilters, payload IPLDPayload) (*ResponsePayload, error)
ScreenResponse(streamFilters *config.Subscription, payload IPLDPayload) (*ResponsePayload, error)
}
// Screener is the underlying struct for the ReponseScreener interface
@@ -39,7 +41,7 @@ func NewResponseScreener() *Screener {
}
// ScreenResponse is used to filter through eth data to extract and package requested data into a ResponsePayload
func (s *Screener) ScreenResponse(streamFilters *StreamFilters, payload IPLDPayload) (*ResponsePayload, error) {
func (s *Screener) ScreenResponse(streamFilters *config.Subscription, payload IPLDPayload) (*ResponsePayload, error) {
response := new(ResponsePayload)
err := s.filterHeaders(streamFilters, response, payload)
if err != nil {
@@ -64,7 +66,7 @@ func (s *Screener) ScreenResponse(streamFilters *StreamFilters, payload IPLDPayl
return response, nil
}
func (s *Screener) filterHeaders(streamFilters *StreamFilters, response *ResponsePayload, payload IPLDPayload) error {
func (s *Screener) filterHeaders(streamFilters *config.Subscription, response *ResponsePayload, payload IPLDPayload) error {
if !streamFilters.HeaderFilter.Off && checkRange(streamFilters.StartingBlock, streamFilters.EndingBlock, payload.BlockNumber.Int64()) {
response.HeadersRlp = append(response.HeadersRlp, payload.HeaderRLP)
if !streamFilters.HeaderFilter.FinalOnly {
@@ -87,7 +89,7 @@ func checkRange(start, end, actual int64) bool {
return false
}
func (s *Screener) filterTransactions(streamFilters *StreamFilters, response *ResponsePayload, payload IPLDPayload) ([]common.Hash, error) {
func (s *Screener) filterTransactions(streamFilters *config.Subscription, response *ResponsePayload, payload IPLDPayload) ([]common.Hash, error) {
trxHashes := make([]common.Hash, 0, len(payload.BlockBody.Transactions))
if !streamFilters.TrxFilter.Off && checkRange(streamFilters.StartingBlock, streamFilters.EndingBlock, payload.BlockNumber.Int64()) {
for i, trx := range payload.BlockBody.Transactions {
@@ -123,7 +125,7 @@ func checkTransactions(wantedSrc, wantedDst []string, actualSrc, actualDst strin
return false
}
func (s *Screener) filerReceipts(streamFilters *StreamFilters, response *ResponsePayload, payload IPLDPayload, trxHashes []common.Hash) error {
func (s *Screener) filerReceipts(streamFilters *config.Subscription, response *ResponsePayload, payload IPLDPayload, trxHashes []common.Hash) error {
if !streamFilters.ReceiptFilter.Off && checkRange(streamFilters.StartingBlock, streamFilters.EndingBlock, payload.BlockNumber.Int64()) {
for i, receipt := range payload.Receipts {
if checkReceipts(receipt, streamFilters.ReceiptFilter.Topic0s, payload.ReceiptMetaData[i].Topic0s, trxHashes) {
@@ -159,7 +161,7 @@ func checkReceipts(rct *types.Receipt, wantedTopics, actualTopics []string, want
return false
}
func (s *Screener) filterState(streamFilters *StreamFilters, response *ResponsePayload, payload IPLDPayload) error {
func (s *Screener) filterState(streamFilters *config.Subscription, response *ResponsePayload, payload IPLDPayload) error {
response.StateNodesRlp = make(map[common.Hash][]byte)
if !streamFilters.StateFilter.Off && checkRange(streamFilters.StartingBlock, streamFilters.EndingBlock, payload.BlockNumber.Int64()) {
keyFilters := make([]common.Hash, 0, len(streamFilters.StateFilter.Addresses))
@@ -191,7 +193,7 @@ func checkNodeKeys(wantedKeys []common.Hash, actualKey common.Hash) bool {
return false
}
func (s *Screener) filterStorage(streamFilters *StreamFilters, response *ResponsePayload, payload IPLDPayload) error {
func (s *Screener) filterStorage(streamFilters *config.Subscription, response *ResponsePayload, payload IPLDPayload) error {
if !streamFilters.StorageFilter.Off && checkRange(streamFilters.StartingBlock, streamFilters.EndingBlock, payload.BlockNumber.Int64()) {
stateKeyFilters := make([]common.Hash, 0, len(streamFilters.StorageFilter.Addresses))
for _, addr := range streamFilters.StorageFilter.Addresses {
pkg/ipfs/service.go
+4 -2
View File
@@ -20,6 +20,8 @@ import (
"fmt"
"sync"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/ethereum/go-ethereum/node"
"github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/rpc"
@@ -42,7 +44,7 @@ type SyncPublishScreenAndServe interface {
// Main event loop for handling client pub-sub
ScreenAndServe(wg *sync.WaitGroup, receivePayloadChan <-chan IPLDPayload, receiveQuitchan <-chan bool)
// Method to subscribe to receive state diff processing output
Subscribe(id rpc.ID, sub chan<- ResponsePayload, quitChan chan<- bool, streamFilters *StreamFilters)
Subscribe(id rpc.ID, sub chan<- ResponsePayload, quitChan chan<- bool, streamFilters *config.Subscription)
// Method to unsubscribe from state diff processing
Unsubscribe(id rpc.ID) error
}
@@ -202,7 +204,7 @@ func (sap *Service) processResponse(payload IPLDPayload) error {
}
// Subscribe is used by the API to subscribe to the service loop
func (sap *Service) Subscribe(id rpc.ID, sub chan<- ResponsePayload, quitChan chan<- bool, streamFilters *StreamFilters) {
func (sap *Service) Subscribe(id rpc.ID, sub chan<- ResponsePayload, quitChan chan<- bool, streamFilters *config.Subscription) {
log.Info("Subscribing to the statediff service")
sap.Lock()
sap.Subscriptions[id] = Subscription{
pkg/ipfs/service_test.go
+8 -8
View File
@@ -26,8 +26,8 @@ import (
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/test_helpers"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/test_helpers/mocks"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/helpers"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/helpers/mocks"
)
var _ = Describe("Service", func() {
@@ -41,18 +41,18 @@ var _ = Describe("Service", func() {
ReturnErr: nil,
}
mockPublisher := &mocks.IPLDPublisher{
ReturnCIDPayload: &test_helpers.MockCIDPayload,
ReturnCIDPayload: &helpers.MockCIDPayload,
ReturnErr: nil,
}
mockStreamer := &mocks.StateDiffStreamer{
ReturnSub: &rpc.ClientSubscription{},
StreamPayloads: []statediff.Payload{
test_helpers.MockStatediffPayload,
helpers.MockStatediffPayload,
},
ReturnErr: nil,
}
mockConverter := &mocks.PayloadConverter{
ReturnIPLDPayload: &test_helpers.MockIPLDPayload,
ReturnIPLDPayload: &helpers.MockIPLDPayload,
ReturnErr: nil,
}
processor := &ipfs.Service{
@@ -68,9 +68,9 @@ var _ = Describe("Service", func() {
time.Sleep(2 * time.Second)
quitChan <- true
wg.Wait()
Expect(mockConverter.PassedStatediffPayload).To(Equal(test_helpers.MockStatediffPayload))
Expect(mockCidRepo.PassedCIDPayload).To(Equal(&test_helpers.MockCIDPayload))
Expect(mockPublisher.PassedIPLDPayload).To(Equal(&test_helpers.MockIPLDPayload))
Expect(mockConverter.PassedStatediffPayload).To(Equal(helpers.MockStatediffPayload))
Expect(mockCidRepo.PassedCIDPayload).To(Equal(&helpers.MockCIDPayload))
Expect(mockPublisher.PassedIPLDPayload).To(Equal(&helpers.MockIPLDPayload))
Expect(mockStreamer.PassedPayloadChan).To(Equal(payloadChan))
})
})
pkg/ipfs/streamer_test.go
+4 -4
View File
@@ -22,8 +22,8 @@ import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/test_helpers"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/test_helpers/mocks"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/helpers"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/helpers/mocks"
)
var _ = Describe("Streamer", func() {
@@ -32,7 +32,7 @@ var _ = Describe("Streamer", func() {
mockStreamer := mocks.StateDiffStreamer{}
mockStreamer.ReturnSub = &rpc.ClientSubscription{}
mockStreamer.StreamPayloads = []statediff.Payload{
test_helpers.MockStatediffPayload,
helpers.MockStatediffPayload,
}
payloadChan := make(chan statediff.Payload, 1)
sub, err := mockStreamer.Stream(payloadChan)
@@ -40,7 +40,7 @@ var _ = Describe("Streamer", func() {
Expect(sub).To(Equal(&rpc.ClientSubscription{}))
Expect(mockStreamer.PassedPayloadChan).To(Equal(payloadChan))
streamedPayload := <-payloadChan
Expect(streamedPayload).To(Equal(test_helpers.MockStatediffPayload))
Expect(streamedPayload).To(Equal(helpers.MockStatediffPayload))
})
})
})
pkg/ipfs/types.go
+3 -33
View File
@@ -20,6 +20,8 @@ import (
"encoding/json"
"math/big"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/ipfs/go-block-format"
"github.com/ethereum/go-ethereum/common"
@@ -30,7 +32,7 @@ import (
type Subscription struct {
PayloadChan chan<- ResponsePayload
QuitChan chan<- bool
StreamFilters *StreamFilters
StreamFilters *config.Subscription
}
// ResponsePayload holds the data returned from the seed node to the requesting client
@@ -149,35 +151,3 @@ type TrxMetaData struct {
Src string
Dst string
}
// StreamFilters are defined by the client to specifiy which data to receive from the seed node
type StreamFilters struct {
BackFill bool
BackFillOnly bool
StartingBlock int64
EndingBlock int64 // set to 0 or a negative value to have no ending block
HeaderFilter struct {
Off bool
FinalOnly bool
}
TrxFilter struct {
Off bool
Src []string
Dst []string
}
ReceiptFilter struct {
Off bool
Topic0s []string
}
StateFilter struct {
Off bool
Addresses []string // is converted to state key by taking its keccak256 hash
IntermediateNodes bool
}
StorageFilter struct {
Off bool
Addresses []string
StorageKeys []string
IntermediateNodes bool
}
}