seed => super; port 80 => port 8080; backfill process for the super_node

cmd/streamSubscribe.go

@@ -38,9 +38,9 @@ import (
 // streamSubscribeCmd represents the streamSubscribe command
 var streamSubscribeCmd = &cobra.Command{
 	Use:   "streamSubscribe",
-	Short: "This command is used to subscribe to the seed node stream with the provided filters",
-	Long: `This command is for demo and testing purposes and is used to subscribe to the seed node with the provided subscription configuration parameters.
-It does not do anything with the data streamed from the seed node other than unpack it and print it out for demonstration purposes.`,
+	Short: "This command is used to subscribe to the super node stream with the provided filters",
+	Long: `This command is for demo and testing purposes and is used to subscribe to the super node with the provided subscription configuration parameters.
+It does not do anything with the data streamed from the super node other than unpack it and print it out for demonstration purposes.`,
 	Run: func(cmd *cobra.Command, args []string) {
 		streamSubscribe()
 	},
@@ -56,12 +56,12 @@ func streamSubscribe() {
 
 	// Create a new rpc client and a subscription streamer with that client
 	rpcClient := getRpcClient()
-	str := streamer.NewSeedNodeStreamer(rpcClient)
+	str := streamer.NewSuperNodeStreamer(rpcClient)
 
 	// Buffered channel for reading subscription payloads
-	payloadChan := make(chan streamer.SeedNodePayload, 20000)
+	payloadChan := make(chan streamer.SuperNodePayload, 20000)
 
-	// Subscribe to the seed node service with the given config/filter parameters
+	// Subscribe to the super node service with the given config/filter parameters
 	sub, err := str.Stream(payloadChan, subscriptionConfig)
 	if err != nil {
 		log.Fatal(err)
@@ -217,7 +217,7 @@ func configureSubscription() {
 func getRpcClient() core.RpcClient {
 	vulcPath := viper.GetString("subscription.path")
 	if vulcPath == "" {
-		vulcPath = "ws://127.0.0.1:80" // default to and try the default ws url if no path is provided
+		vulcPath = "ws://127.0.0.1:8080" // default to and try the default ws url if no path is provided
 	}
 	rawRpcClient, err := rpc.Dial(vulcPath)
 	if err != nil {

cmd/syncAndPublish.go

@@ -16,11 +16,13 @@
 package cmd
 
 import (
+	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
 	"os"
 	"path/filepath"
 	syn "sync"
+	"time"
 
-	"github.com/vulcanize/vulcanizedb/pkg/seed_node"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node"
 
 	"github.com/spf13/viper"
 
@@ -51,44 +53,34 @@ it maintains a local index of the IPLD objects' CIDs in Postgres.`,
 	},
 }
 
+var ipfsPath string
+
 func init() {
 	rootCmd.AddCommand(syncAndPublishCmd)
 }
 
 func syncAndPublish() {
-	blockChain, rpcClient := getBlockChainAndClient()
-
-	db := utils.LoadPostgres(databaseConfig, blockChain.Node())
-	quitChan := make(chan bool)
-
-	ipfsPath := viper.GetString("client.ipfsPath")
-	if ipfsPath == "" {
-		home, err := os.UserHomeDir()
+	superNode, err := newSuperNode()
+	if err != nil {
+		log.Fatal(err)
+	}
+	wg := &syn.WaitGroup{}
+	err = superNode.SyncAndPublish(wg, nil, nil)
+	if err != nil {
+		log.Fatal(err)
+	}
+	if viper.GetBool("backfill.on") && viper.GetString("backfill.ipcPath") != "" {
+		backfiller := newBackFiller(superNode.GetPublisher())
 		if err != nil {
 			log.Fatal(err)
 		}
-		ipfsPath = filepath.Join(home, ".ipfs")
-	}
-	workers := viper.GetInt("client.workers")
-	if workers < 1 {
-		workers = 1
-	}
-	processor, err := seed_node.NewSeedNode(ipfsPath, &db, rpcClient, quitChan, workers, blockChain.Node())
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	wg := &syn.WaitGroup{}
-	err = processor.SyncAndPublish(wg, nil, nil)
-	if err != nil {
-		log.Fatal(err)
+		backfiller.FillGaps(wg, nil)
 	}
 	wg.Wait() // If an error was thrown, wg.Add was never called and this will fall through
 }
 
-func getBlockChainAndClient() (*geth.BlockChain, core.RpcClient) {
-	rawRpcClient, err := rpc.Dial(ipc)
-
+func getBlockChainAndClient(path string) (*geth.BlockChain, core.RpcClient) {
+	rawRpcClient, err := rpc.Dial(path)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -100,3 +92,35 @@ func getBlockChainAndClient() (*geth.BlockChain, core.RpcClient) {
 	blockChain := geth.NewBlockChain(vdbEthClient, rpcClient, vdbNode, transactionConverter)
 	return blockChain, rpcClient
 }
+
+func newSuperNode() (super_node.NodeInterface, error) {
+	blockChain, rpcClient := getBlockChainAndClient(ipc)
+	db := utils.LoadPostgres(databaseConfig, blockChain.Node())
+	quitChan := make(chan bool)
+	ipfsPath = viper.GetString("client.ipfsPath")
+	if ipfsPath == "" {
+		home, err := os.UserHomeDir()
+		if err != nil {
+			log.Fatal(err)
+		}
+		ipfsPath = filepath.Join(home, ".ipfs")
+	}
+	workers := viper.GetInt("client.workers")
+	if workers < 1 {
+		workers = 1
+	}
+	return super_node.NewSuperNode(ipfsPath, &db, rpcClient, quitChan, workers, blockChain.Node())
+}
+
+func newBackFiller(ipfsPublisher ipfs.IPLDPublisher) super_node.BackFillInterface {
+	blockChain, archivalRpcClient := getBlockChainAndClient(viper.GetString("backfill.ipcPath"))
+	db := utils.LoadPostgres(databaseConfig, blockChain.Node())
+	freq := viper.GetInt("backfill.frequency")
+	var frequency time.Duration
+	if freq <= 0 {
+		frequency = time.Minute * 5
+	} else {
+		frequency = time.Duration(freq)
+	}
+	return super_node.NewBackFillService(ipfsPublisher, &db, archivalRpcClient, time.Minute*frequency)
+}

cmd/syncPublishScreenAndServe.go

@@ -20,15 +20,12 @@ import (
 	"path/filepath"
 	syn "sync"
 
-	"github.com/vulcanize/vulcanizedb/pkg/seed_node"
-
 	"github.com/ethereum/go-ethereum/rpc"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
 	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
-	"github.com/vulcanize/vulcanizedb/utils"
 )
 
 // syncPublishScreenAndServeCmd represents the syncPublishScreenAndServe command
@@ -50,24 +47,7 @@ func init() {
 }
 
 func syncPublishScreenAndServe() {
-	blockChain, rpcClient := getBlockChainAndClient()
-
-	db := utils.LoadPostgres(databaseConfig, blockChain.Node())
-	quitChan := make(chan bool, 1)
-
-	ipfsPath := viper.GetString("client.ipfsPath")
-	if ipfsPath == "" {
-		home, err := os.UserHomeDir()
-		if err != nil {
-			log.Fatal(err)
-		}
-		ipfsPath = filepath.Join(home, ".ipfs")
-	}
-	workers := viper.GetInt("client.workers")
-	if workers < 1 {
-		workers = 1
-	}
-	processor, err := seed_node.NewSeedNode(ipfsPath, &db, rpcClient, quitChan, workers, blockChain.Node())
+	superNode, err := newSuperNode()
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -75,11 +55,18 @@ func syncPublishScreenAndServe() {
 	wg := &syn.WaitGroup{}
 	forwardPayloadChan := make(chan ipfs.IPLDPayload, 20000)
 	forwardQuitChan := make(chan bool, 1)
-	err = processor.SyncAndPublish(wg, forwardPayloadChan, forwardQuitChan)
+	err = superNode.SyncAndPublish(wg, forwardPayloadChan, forwardQuitChan)
 	if err != nil {
 		log.Fatal(err)
 	}
-	processor.ScreenAndServe(forwardPayloadChan, forwardQuitChan)
+	superNode.ScreenAndServe(forwardPayloadChan, forwardQuitChan)
+	if viper.GetBool("backfill.on") && viper.GetString("backfill.ipcPath") != "" {
+		backfiller := newBackFiller(superNode.GetPublisher())
+		if err != nil {
+			log.Fatal(err)
+		}
+		backfiller.FillGaps(wg, nil)
+	}
 
 	var ipcPath string
 	ipcPath = viper.GetString("server.ipcPath")
@@ -90,7 +77,7 @@ func syncPublishScreenAndServe() {
 		}
 		ipcPath = filepath.Join(home, ".vulcanize/vulcanize.ipc")
 	}
-	_, _, err = rpc.StartIPCEndpoint(ipcPath, processor.APIs())
+	_, _, err = rpc.StartIPCEndpoint(ipcPath, superNode.APIs())
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -98,11 +85,11 @@ func syncPublishScreenAndServe() {
 	var wsEndpoint string
 	wsEndpoint = viper.GetString("server.wsEndpoint")
 	if wsEndpoint == "" {
-		wsEndpoint = "127.0.0.1:80"
+		wsEndpoint = "127.0.0.1:8080"
 	}
 	var exposeAll = true
 	var wsOrigins []string = nil
-	_, _, err = rpc.StartWSEndpoint(wsEndpoint, processor.APIs(), []string{"vulcanizedb"}, wsOrigins, exposeAll)
+	_, _, err = rpc.StartWSEndpoint(wsEndpoint, superNode.APIs(), []string{"vulcanizedb"}, wsOrigins, exposeAll)
 	if err != nil {
 		log.Fatal(err)
 	}