cerc-io/ipld-eth-server
12
0
Fork 1
Code Issues 10 Pull Requests 2 Actions Packages Projects Releases 37 Wiki Activity

seed => super; port 80 => port 8080; backfill process for the super_node

Browse Source
This commit is contained in:
Ian Norden
2019-12-02 13:24:58 -06:00
parent 83fd76bc8a
commit 40c3aff597
33 changed files with 843 additions and 373 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/streamSubscribe.go
+7 -7
View File
@@ -38,9 +38,9 @@ import (
// streamSubscribeCmd represents the streamSubscribe command
var streamSubscribeCmd = &cobra.Command{
Use: "streamSubscribe",
Short: "This command is used to subscribe to the seed node stream with the provided filters",
Long: `This command is for demo and testing purposes and is used to subscribe to the seed node with the provided subscription configuration parameters.
It does not do anything with the data streamed from the seed node other than unpack it and print it out for demonstration purposes.`,
Short: "This command is used to subscribe to the super node stream with the provided filters",
Long: `This command is for demo and testing purposes and is used to subscribe to the super node with the provided subscription configuration parameters.
It does not do anything with the data streamed from the super node other than unpack it and print it out for demonstration purposes.`,
Run: func(cmd *cobra.Command, args []string) {
streamSubscribe()
},
@@ -56,12 +56,12 @@ func streamSubscribe() {
// Create a new rpc client and a subscription streamer with that client
rpcClient := getRpcClient()
str := streamer.NewSeedNodeStreamer(rpcClient)
str := streamer.NewSuperNodeStreamer(rpcClient)
// Buffered channel for reading subscription payloads
payloadChan := make(chan streamer.SeedNodePayload, 20000)
payloadChan := make(chan streamer.SuperNodePayload, 20000)
// Subscribe to the seed node service with the given config/filter parameters
// Subscribe to the super node service with the given config/filter parameters
sub, err := str.Stream(payloadChan, subscriptionConfig)
if err != nil {
log.Fatal(err)
@@ -217,7 +217,7 @@ func configureSubscription() {
func getRpcClient() core.RpcClient {
vulcPath := viper.GetString("subscription.path")
if vulcPath == "" {
vulcPath = "ws://127.0.0.1:80" // default to and try the default ws url if no path is provided
vulcPath = "ws://127.0.0.1:8080" // default to and try the default ws url if no path is provided
}
rawRpcClient, err := rpc.Dial(vulcPath)
if err != nil {
cmd/syncAndPublish.go
+51 -27
View File
@@ -16,11 +16,13 @@
package cmd
import (
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
"os"
"path/filepath"
syn "sync"
"time"
"github.com/vulcanize/vulcanizedb/pkg/seed_node"
"github.com/vulcanize/vulcanizedb/pkg/super_node"
"github.com/spf13/viper"
@@ -51,44 +53,34 @@ it maintains a local index of the IPLD objects' CIDs in Postgres.`,
},
}
var ipfsPath string
func init() {
rootCmd.AddCommand(syncAndPublishCmd)
}
func syncAndPublish() {
blockChain, rpcClient := getBlockChainAndClient()
db := utils.LoadPostgres(databaseConfig, blockChain.Node())
quitChan := make(chan bool)
ipfsPath := viper.GetString("client.ipfsPath")
if ipfsPath == "" {
home, err := os.UserHomeDir()
superNode, err := newSuperNode()
if err != nil {
log.Fatal(err)
}
wg := &syn.WaitGroup{}
err = superNode.SyncAndPublish(wg, nil, nil)
if err != nil {
log.Fatal(err)
}
if viper.GetBool("backfill.on") && viper.GetString("backfill.ipcPath") != "" {
backfiller := newBackFiller(superNode.GetPublisher())
if err != nil {
log.Fatal(err)
}
ipfsPath = filepath.Join(home, ".ipfs")
}
workers := viper.GetInt("client.workers")
if workers < 1 {
workers = 1
}
processor, err := seed_node.NewSeedNode(ipfsPath, &db, rpcClient, quitChan, workers, blockChain.Node())
if err != nil {
log.Fatal(err)
}
wg := &syn.WaitGroup{}
err = processor.SyncAndPublish(wg, nil, nil)
if err != nil {
log.Fatal(err)
backfiller.FillGaps(wg, nil)
}
wg.Wait() // If an error was thrown, wg.Add was never called and this will fall through
}
func getBlockChainAndClient() (*geth.BlockChain, core.RpcClient) {
rawRpcClient, err := rpc.Dial(ipc)
func getBlockChainAndClient(path string) (*geth.BlockChain, core.RpcClient) {
rawRpcClient, err := rpc.Dial(path)
if err != nil {
log.Fatal(err)
}
@@ -100,3 +92,35 @@ func getBlockChainAndClient() (*geth.BlockChain, core.RpcClient) {
blockChain := geth.NewBlockChain(vdbEthClient, rpcClient, vdbNode, transactionConverter)
return blockChain, rpcClient
}
func newSuperNode() (super_node.NodeInterface, error) {
blockChain, rpcClient := getBlockChainAndClient(ipc)
db := utils.LoadPostgres(databaseConfig, blockChain.Node())
quitChan := make(chan bool)
ipfsPath = viper.GetString("client.ipfsPath")
if ipfsPath == "" {
home, err := os.UserHomeDir()
if err != nil {
log.Fatal(err)
}
ipfsPath = filepath.Join(home, ".ipfs")
}
workers := viper.GetInt("client.workers")
if workers < 1 {
workers = 1
}
return super_node.NewSuperNode(ipfsPath, &db, rpcClient, quitChan, workers, blockChain.Node())
}
func newBackFiller(ipfsPublisher ipfs.IPLDPublisher) super_node.BackFillInterface {
blockChain, archivalRpcClient := getBlockChainAndClient(viper.GetString("backfill.ipcPath"))
db := utils.LoadPostgres(databaseConfig, blockChain.Node())
freq := viper.GetInt("backfill.frequency")
var frequency time.Duration
if freq <= 0 {
frequency = time.Minute * 5
} else {
frequency = time.Duration(freq)
}
return super_node.NewBackFillService(ipfsPublisher, &db, archivalRpcClient, time.Minute*frequency)
}
cmd/syncPublishScreenAndServe.go
+13 -26
View File
@@ -20,15 +20,12 @@ import (
"path/filepath"
syn "sync"
"github.com/vulcanize/vulcanizedb/pkg/seed_node"
"github.com/ethereum/go-ethereum/rpc"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
"github.com/vulcanize/vulcanizedb/utils"
)
// syncPublishScreenAndServeCmd represents the syncPublishScreenAndServe command
@@ -50,24 +47,7 @@ func init() {
}
func syncPublishScreenAndServe() {
blockChain, rpcClient := getBlockChainAndClient()
db := utils.LoadPostgres(databaseConfig, blockChain.Node())
quitChan := make(chan bool, 1)
ipfsPath := viper.GetString("client.ipfsPath")
if ipfsPath == "" {
home, err := os.UserHomeDir()
if err != nil {
log.Fatal(err)
}
ipfsPath = filepath.Join(home, ".ipfs")
}
workers := viper.GetInt("client.workers")
if workers < 1 {
workers = 1
}
processor, err := seed_node.NewSeedNode(ipfsPath, &db, rpcClient, quitChan, workers, blockChain.Node())
superNode, err := newSuperNode()
if err != nil {
log.Fatal(err)
}
@@ -75,11 +55,18 @@ func syncPublishScreenAndServe() {
wg := &syn.WaitGroup{}
forwardPayloadChan := make(chan ipfs.IPLDPayload, 20000)
forwardQuitChan := make(chan bool, 1)
err = processor.SyncAndPublish(wg, forwardPayloadChan, forwardQuitChan)
err = superNode.SyncAndPublish(wg, forwardPayloadChan, forwardQuitChan)
if err != nil {
log.Fatal(err)
}
processor.ScreenAndServe(forwardPayloadChan, forwardQuitChan)
superNode.ScreenAndServe(forwardPayloadChan, forwardQuitChan)
if viper.GetBool("backfill.on") && viper.GetString("backfill.ipcPath") != "" {
backfiller := newBackFiller(superNode.GetPublisher())
if err != nil {
log.Fatal(err)
}
backfiller.FillGaps(wg, nil)
}
var ipcPath string
ipcPath = viper.GetString("server.ipcPath")
@@ -90,7 +77,7 @@ func syncPublishScreenAndServe() {
}
ipcPath = filepath.Join(home, ".vulcanize/vulcanize.ipc")
}
_, _, err = rpc.StartIPCEndpoint(ipcPath, processor.APIs())
_, _, err = rpc.StartIPCEndpoint(ipcPath, superNode.APIs())
if err != nil {
log.Fatal(err)
}
@@ -98,11 +85,11 @@ func syncPublishScreenAndServe() {
var wsEndpoint string
wsEndpoint = viper.GetString("server.wsEndpoint")
if wsEndpoint == "" {
wsEndpoint = "127.0.0.1:80"
wsEndpoint = "127.0.0.1:8080"
}
var exposeAll = true
var wsOrigins []string = nil
_, _, err = rpc.StartWSEndpoint(wsEndpoint, processor.APIs(), []string{"vulcanizedb"}, wsOrigins, exposeAll)
_, _, err = rpc.StartWSEndpoint(wsEndpoint, superNode.APIs(), []string{"vulcanizedb"}, wsOrigins, exposeAll)
if err != nil {
log.Fatal(err)
}