deep-stack/ipld-eth-server
4
0
Fork 0
forked from cerc-io/ipld-eth-server
Code Pull Requests Activity

logWithCommand; rebase fixes; config for testing super node subscription

Browse Source
This commit is contained in:
Ian Norden
2019-12-02 13:24:58 -06:00
parent e912fa6c87
commit c16ac026db
30 changed files with 487 additions and 234 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/coldImport.go
+7 -7
View File
@@ -39,8 +39,8 @@ var coldImportCmd = &cobra.Command{
Geth must be synced over all of the desired blocks and must not be running in order to execute this command.`,
Run: func(cmd *cobra.Command, args []string) {
SubCommand = cmd.CalledAs()
LogWithCommand = *log.WithField("SubCommand", SubCommand)
subCommand = cmd.CalledAs()
logWithCommand = *log.WithField("SubCommand", subCommand)
coldImport()
},
}
@@ -57,7 +57,7 @@ func coldImport() {
ethDBConfig := ethereum.CreateDatabaseConfig(ethereum.Level, levelDbPath)
ethDB, err := ethereum.CreateDatabase(ethDBConfig)
if err != nil {
LogWithCommand.Fatal("Error connecting to ethereum db: ", err)
logWithCommand.Fatal("Error connecting to ethereum db: ", err)
}
mostRecentBlockNumberInDb := ethDB.GetHeadBlockNumber()
if syncAll {
@@ -65,10 +65,10 @@ func coldImport() {
endingBlockNumber = mostRecentBlockNumberInDb
}
if endingBlockNumber < startingBlockNumber {
LogWithCommand.Fatal("Ending block number must be greater than starting block number for cold import.")
logWithCommand.Fatal("Ending block number must be greater than starting block number for cold import.")
}
if endingBlockNumber > mostRecentBlockNumberInDb {
LogWithCommand.Fatal("Ending block number is greater than most recent block in db: ", mostRecentBlockNumberInDb)
logWithCommand.Fatal("Ending block number is greater than most recent block in db: ", mostRecentBlockNumberInDb)
}
// init pg db
@@ -78,7 +78,7 @@ func coldImport() {
nodeBuilder := cold_import.NewColdImportNodeBuilder(reader, parser)
coldNode, err := nodeBuilder.GetNode(genesisBlock, levelDbPath)
if err != nil {
LogWithCommand.Fatal("Error getting node: ", err)
logWithCommand.Fatal("Error getting node: ", err)
}
pgDB := utils.LoadPostgres(databaseConfig, coldNode)
@@ -92,6 +92,6 @@ func coldImport() {
coldImporter := cold_import.NewColdImporter(ethDB, blockRepository, receiptRepository, blockConverter)
err = coldImporter.Execute(startingBlockNumber, endingBlockNumber, coldNode.ID)
if err != nil {
LogWithCommand.Fatal("Error executing cold import: ", err)
logWithCommand.Fatal("Error executing cold import: ", err)
}
}
cmd/compose.go
+18 -18
View File
@@ -102,8 +102,8 @@ single config file or in separate command instances using different config files
Specify config location when executing the command:
./vulcanizedb compose --config=./environments/config_name.toml`,
Run: func(cmd *cobra.Command, args []string) {
SubCommand = cmd.CalledAs()
LogWithCommand = *log.WithField("SubCommand", SubCommand)
subCommand = cmd.CalledAs()
logWithCommand = *log.WithField("SubCommand", subCommand)
compose()
},
}
@@ -113,25 +113,25 @@ func compose() {
prepConfig()
// Generate code to build the plugin according to the config file
LogWithCommand.Info("generating plugin")
logWithCommand.Info("generating plugin")
generator, err := p2.NewGenerator(genConfig, databaseConfig)
if err != nil {
LogWithCommand.Debug("initializing plugin generator failed")
LogWithCommand.Fatal(err)
logWithCommand.Debug("initializing plugin generator failed")
logWithCommand.Fatal(err)
}
err = generator.GenerateExporterPlugin()
if err != nil {
LogWithCommand.Debug("generating plugin failed")
LogWithCommand.Fatal(err)
logWithCommand.Debug("generating plugin failed")
logWithCommand.Fatal(err)
}
// TODO: Embed versioning info in the .so files so we know which version of vulcanizedb to run them with
_, pluginPath, err := genConfig.GetPluginPaths()
if err != nil {
LogWithCommand.Debug("getting plugin path failed")
LogWithCommand.Fatal(err)
logWithCommand.Debug("getting plugin path failed")
logWithCommand.Fatal(err)
}
fmt.Printf("Composed plugin %s", pluginPath)
LogWithCommand.Info("plugin .so file output to ", pluginPath)
logWithCommand.Info("plugin .so file output to ", pluginPath)
}
func init() {
@@ -139,38 +139,38 @@ func init() {
}
func prepConfig() {
LogWithCommand.Info("configuring plugin")
logWithCommand.Info("configuring plugin")
names := viper.GetStringSlice("exporter.transformerNames")
transformers := make(map[string]config.Transformer)
for _, name := range names {
transformer := viper.GetStringMapString("exporter." + name)
p, pOK := transformer["path"]
if !pOK || p == "" {
LogWithCommand.Fatal(name, " transformer config is missing `path` value")
logWithCommand.Fatal(name, " transformer config is missing `path` value")
}
r, rOK := transformer["repository"]
if !rOK || r == "" {
LogWithCommand.Fatal(name, " transformer config is missing `repository` value")
logWithCommand.Fatal(name, " transformer config is missing `repository` value")
}
m, mOK := transformer["migrations"]
if !mOK || m == "" {
LogWithCommand.Fatal(name, " transformer config is missing `migrations` value")
logWithCommand.Fatal(name, " transformer config is missing `migrations` value")
}
mr, mrOK := transformer["rank"]
if !mrOK || mr == "" {
LogWithCommand.Fatal(name, " transformer config is missing `rank` value")
logWithCommand.Fatal(name, " transformer config is missing `rank` value")
}
rank, err := strconv.ParseUint(mr, 10, 64)
if err != nil {
LogWithCommand.Fatal(name, " migration `rank` can't be converted to an unsigned integer")
logWithCommand.Fatal(name, " migration `rank` can't be converted to an unsigned integer")
}
t, tOK := transformer["type"]
if !tOK {
LogWithCommand.Fatal(name, " transformer config is missing `type` value")
logWithCommand.Fatal(name, " transformer config is missing `type` value")
}
transformerType := config.GetTransformerType(t)
if transformerType == config.UnknownTransformerType {
LogWithCommand.Fatal(errors.New(`unknown transformer type in exporter config accepted types are "eth_event", "eth_storage"`))
logWithCommand.Fatal(errors.New(`unknown transformer type in exporter config accepted types are "eth_event", "eth_storage"`))
}
transformers[name] = config.Transformer{
cmd/composeAndExecute.go
+15 -15
View File
@@ -107,8 +107,8 @@ single config file or in separate command instances using different config files
Specify config location when executing the command:
./vulcanizedb composeAndExecute --config=./environments/config_name.toml`,
Run: func(cmd *cobra.Command, args []string) {
SubCommand = cmd.CalledAs()
LogWithCommand = *log.WithField("SubCommand", SubCommand)
subCommand = cmd.CalledAs()
logWithCommand = *log.WithField("SubCommand", subCommand)
composeAndExecute()
},
}
@@ -118,44 +118,44 @@ func composeAndExecute() {
prepConfig()
// Generate code to build the plugin according to the config file
LogWithCommand.Info("generating plugin")
logWithCommand.Info("generating plugin")
generator, err := p2.NewGenerator(genConfig, databaseConfig)
if err != nil {
LogWithCommand.Fatal(err)
logWithCommand.Fatal(err)
}
err = generator.GenerateExporterPlugin()
if err != nil {
LogWithCommand.Debug("generating plugin failed")
LogWithCommand.Fatal(err)
logWithCommand.Debug("generating plugin failed")
logWithCommand.Fatal(err)
}
// Get the plugin path and load the plugin
_, pluginPath, err := genConfig.GetPluginPaths()
if err != nil {
LogWithCommand.Fatal(err)
logWithCommand.Fatal(err)
}
if !genConfig.Save {
defer helpers.ClearFiles(pluginPath)
}
LogWithCommand.Info("linking plugin ", pluginPath)
logWithCommand.Info("linking plugin ", pluginPath)
plug, err := plugin.Open(pluginPath)
if err != nil {
LogWithCommand.Debug("linking plugin failed")
LogWithCommand.Fatal(err)
logWithCommand.Debug("linking plugin failed")
logWithCommand.Fatal(err)
}
// Load the `Exporter` symbol from the plugin
LogWithCommand.Info("loading transformers from plugin")
logWithCommand.Info("loading transformers from plugin")
symExporter, err := plug.Lookup("Exporter")
if err != nil {
LogWithCommand.Debug("loading Exporter symbol failed")
LogWithCommand.Fatal(err)
logWithCommand.Debug("loading Exporter symbol failed")
logWithCommand.Fatal(err)
}
// Assert that the symbol is of type Exporter
exporter, ok := symExporter.(Exporter)
if !ok {
LogWithCommand.Debug("plugged-in symbol not of type Exporter")
logWithCommand.Debug("plugged-in symbol not of type Exporter")
os.Exit(1)
}
@@ -173,7 +173,7 @@ func composeAndExecute() {
ew := watcher.NewEventWatcher(&db, blockChain)
err := ew.AddTransformers(ethEventInitializers)
if err != nil {
LogWithCommand.Fatalf("failed to add event transformer initializers to watcher: %s", err.Error())
logWithCommand.Fatalf("failed to add event transformer initializers to watcher: %s", err.Error())
}
wg.Add(1)
go watchEthEvents(&ew, &wg)
cmd/contractWatcher.go
+5 -5
View File
@@ -79,8 +79,8 @@ Requires a .toml config file:
piping = true
`,
Run: func(cmd *cobra.Command, args []string) {
SubCommand = cmd.CalledAs()
LogWithCommand = *log.WithField("SubCommand", SubCommand)
subCommand = cmd.CalledAs()
logWithCommand = *log.WithField("SubCommand", subCommand)
contractWatcher()
},
}
@@ -105,18 +105,18 @@ func contractWatcher() {
case "full":
t = ft.NewTransformer(con, blockChain, &db)
default:
LogWithCommand.Fatal("Invalid mode")
logWithCommand.Fatal("Invalid mode")
}
err := t.Init()
if err != nil {
LogWithCommand.Fatal(fmt.Sprintf("Failed to initialize transformer, err: %v ", err))
logWithCommand.Fatal(fmt.Sprintf("Failed to initialize transformer, err: %v ", err))
}
for range ticker.C {
err = t.Execute()
if err != nil {
LogWithCommand.Error("Execution error for transformer: ", t.GetConfig().Name, err)
logWithCommand.Error("Execution error for transformer: ", t.GetConfig().Name, err)
}
}
}
cmd/execute.go
+15 -15
View File
@@ -60,8 +60,8 @@ must have been composed by the same version of vulcanizedb or else it will not b
Specify config location when executing the command:
./vulcanizedb execute --config=./environments/config_name.toml`,
Run: func(cmd *cobra.Command, args []string) {
SubCommand = cmd.CalledAs()
LogWithCommand = *log.WithField("SubCommand", SubCommand)
subCommand = cmd.CalledAs()
logWithCommand = *log.WithField("SubCommand", subCommand)
execute()
},
}
@@ -73,29 +73,29 @@ func execute() {
// Get the plugin path and load the plugin
_, pluginPath, err := genConfig.GetPluginPaths()
if err != nil {
LogWithCommand.Fatal(err)
logWithCommand.Fatal(err)
}
fmt.Printf("Executing plugin %s", pluginPath)
LogWithCommand.Info("linking plugin ", pluginPath)
logWithCommand.Info("linking plugin ", pluginPath)
plug, err := plugin.Open(pluginPath)
if err != nil {
LogWithCommand.Warn("linking plugin failed")
LogWithCommand.Fatal(err)
logWithCommand.Warn("linking plugin failed")
logWithCommand.Fatal(err)
}
// Load the `Exporter` symbol from the plugin
LogWithCommand.Info("loading transformers from plugin")
logWithCommand.Info("loading transformers from plugin")
symExporter, err := plug.Lookup("Exporter")
if err != nil {
LogWithCommand.Warn("loading Exporter symbol failed")
LogWithCommand.Fatal(err)
logWithCommand.Warn("loading Exporter symbol failed")
logWithCommand.Fatal(err)
}
// Assert that the symbol is of type Exporter
exporter, ok := symExporter.(Exporter)
if !ok {
LogWithCommand.Fatal("plugged-in symbol not of type Exporter")
logWithCommand.Fatal("plugged-in symbol not of type Exporter")
}
// Use the Exporters export method to load the EventTransformerInitializer, StorageTransformerInitializer, and ContractTransformerInitializer sets
@@ -112,7 +112,7 @@ func execute() {
ew := watcher.NewEventWatcher(&db, blockChain)
err = ew.AddTransformers(ethEventInitializers)
if err != nil {
LogWithCommand.Fatalf("failed to add event transformer initializers to watcher: %s", err.Error())
logWithCommand.Fatalf("failed to add event transformer initializers to watcher: %s", err.Error())
}
wg.Add(1)
go watchEthEvents(&ew, &wg)
@@ -162,7 +162,7 @@ type Exporter interface {
func watchEthEvents(w *watcher.EventWatcher, wg *syn.WaitGroup) {
defer wg.Done()
// Execute over the EventTransformerInitializer set using the watcher
LogWithCommand.Info("executing event transformers")
logWithCommand.Info("executing event transformers")
var recheck constants.TransformerExecution
if recheckHeadersArg {
recheck = constants.HeaderRecheck
@@ -171,14 +171,14 @@ func watchEthEvents(w *watcher.EventWatcher, wg *syn.WaitGroup) {
}
err := w.Execute(recheck)
if err != nil {
LogWithCommand.Fatalf("error executing event watcher: %s", err.Error())
logWithCommand.Fatalf("error executing event watcher: %s", err.Error())
}
}
func watchEthStorage(w watcher.IStorageWatcher, wg *syn.WaitGroup) {
defer wg.Done()
// Execute over the StorageTransformerInitializer set using the storage watcher
LogWithCommand.Info("executing storage transformers")
logWithCommand.Info("executing storage transformers")
on := viper.GetBool("storageBackFill.on")
if on {
backFillStorage(w)
@@ -198,7 +198,7 @@ func backFillStorage(w watcher.IStorageWatcher) {
func watchEthContract(w *watcher.ContractWatcher, wg *syn.WaitGroup) {
defer wg.Done()
// Execute over the ContractTransformerInitializer set using the contract watcher
LogWithCommand.Info("executing contract_watcher transformers")
logWithCommand.Info("executing contract_watcher transformers")
ticker := time.NewTicker(pollingInterval)
defer ticker.Stop()
for range ticker.C {
cmd/fullSync.go
+8 -8
View File
@@ -49,8 +49,8 @@ Expects ethereum node to be running and requires a .toml config:
ipcPath = "/Users/user/Library/Ethereum/geth.ipc"
`,
Run: func(cmd *cobra.Command, args []string) {
SubCommand = cmd.CalledAs()
LogWithCommand = *log.WithField("SubCommand", SubCommand)
subCommand = cmd.CalledAs()
logWithCommand = *log.WithField("SubCommand", subCommand)
fullSync()
},
}
@@ -63,7 +63,7 @@ func init() {
func backFillAllBlocks(blockchain core.BlockChain, blockRepository datastore.BlockRepository, missingBlocksPopulated chan int, startingBlockNumber int64) {
populated, err := history.PopulateMissingBlocks(blockchain, blockRepository, startingBlockNumber)
if err != nil {
LogWithCommand.Error("backfillAllBlocks: error in populateMissingBlocks: ", err)
logWithCommand.Error("backfillAllBlocks: error in populateMissingBlocks: ", err)
}
missingBlocksPopulated <- populated
}
@@ -75,13 +75,13 @@ func fullSync() {
blockChain := getBlockChain()
lastBlock, err := blockChain.LastBlock()
if err != nil {
LogWithCommand.Error("fullSync: Error getting last block: ", err)
logWithCommand.Error("fullSync: Error getting last block: ", err)
}
if lastBlock.Int64() == 0 {
LogWithCommand.Fatal("geth initial: state sync not finished")
logWithCommand.Fatal("geth initial: state sync not finished")
}
if startingBlockNumber > lastBlock.Int64() {
LogWithCommand.Fatal("fullSync: starting block number > current block number")
logWithCommand.Fatal("fullSync: starting block number > current block number")
}
db := utils.LoadPostgres(databaseConfig, blockChain.Node())
@@ -95,9 +95,9 @@ func fullSync() {
case <-ticker.C:
window, err := validator.ValidateBlocks()
if err != nil {
LogWithCommand.Error("fullSync: error in validateBlocks: ", err)
logWithCommand.Error("fullSync: error in validateBlocks: ", err)
}
LogWithCommand.Debug(window.GetString())
logWithCommand.Debug(window.GetString())
case <-missingBlocksPopulated:
go backFillAllBlocks(blockChain, blockRepository, missingBlocksPopulated, startingBlockNumber)
}
cmd/headerSync.go
+8 -8
View File
@@ -50,8 +50,8 @@ Expects ethereum node to be running and requires a .toml config:
ipcPath = "/Users/user/Library/Ethereum/geth.ipc"
`,
Run: func(cmd *cobra.Command, args []string) {
SubCommand = cmd.CalledAs()
LogWithCommand = *log.WithField("SubCommand", SubCommand)
subCommand = cmd.CalledAs()
logWithCommand = *log.WithField("SubCommand", subCommand)
headerSync()
},
}
@@ -66,7 +66,7 @@ func backFillAllHeaders(blockchain core.BlockChain, headerRepository datastore.H
if err != nil {
// TODO Lots of possible errors in the call stack above. If errors occur, we still put
// 0 in the channel, triggering another round
LogWithCommand.Error("backfillAllHeaders: Error populating headers: ", err)
logWithCommand.Error("backfillAllHeaders: Error populating headers: ", err)
}
missingBlocksPopulated <- populated
}
@@ -88,9 +88,9 @@ func headerSync() {
case <-ticker.C:
window, err := validator.ValidateHeaders()
if err != nil {
LogWithCommand.Error("headerSync: ValidateHeaders failed: ", err)
logWithCommand.Error("headerSync: ValidateHeaders failed: ", err)
}
LogWithCommand.Debug(window.GetString())
logWithCommand.Debug(window.GetString())
case n := <-missingBlocksPopulated:
if n == 0 {
time.Sleep(3 * time.Second)
@@ -103,12 +103,12 @@ func headerSync() {
func validateArgs(blockChain *eth.BlockChain) {
lastBlock, err := blockChain.LastBlock()
if err != nil {
LogWithCommand.Error("validateArgs: Error getting last block: ", err)
logWithCommand.Error("validateArgs: Error getting last block: ", err)
}
if lastBlock.Int64() == 0 {
LogWithCommand.Fatal("geth initial: state sync not finished")
logWithCommand.Fatal("geth initial: state sync not finished")
}
if startingBlockNumber > lastBlock.Int64() {
LogWithCommand.Fatal("starting block number > current block number")
logWithCommand.Fatal("starting block number > current block number")
}
}
cmd/root.go
+3 -3
View File
@@ -49,8 +49,8 @@ var (
syncAll bool
endingBlockNumber int64
recheckHeadersArg bool
SubCommand string
LogWithCommand log.Entry
subCommand string
logWithCommand log.Entry
storageDiffsSource string
)
@@ -170,7 +170,7 @@ func getClients() (client.RPCClient, *ethclient.Client) {
rawRPCClient, err := rpc.Dial(ipc)
if err != nil {
LogWithCommand.Fatal(err)
logWithCommand.Fatal(err)
}
rpcClient := client.NewRPCClient(rawRPCClient, ipc)
ethClient := ethclient.NewClient(rawRPCClient)
cmd/screenAndServe.go
+5 -3
View File
@@ -41,6 +41,8 @@ var screenAndServeCmd = &cobra.Command{
relays relevant data to requesting clients. In this mode, the super-node can only relay data which it has
already indexed it does not stream out live data.`,
Run: func(cmd *cobra.Command, args []string) {
subCommand = cmd.CalledAs()
logWithCommand = *log.WithField("SubCommand", subCommand)
screenAndServe()
},
}
@@ -52,7 +54,7 @@ func init() {
func screenAndServe() {
superNode, newNodeErr := newSuperNodeWithoutPairedGethNode()
if newNodeErr != nil {
log.Fatal(newNodeErr)
logWithCommand.Fatal(newNodeErr)
}
wg := &syn.WaitGroup{}
quitChan := make(chan bool, 1)
@@ -61,7 +63,7 @@ func screenAndServe() {
serverErr := startServers(superNode)
if serverErr != nil {
log.Fatal(serverErr)
logWithCommand.Fatal(serverErr)
}
wg.Wait()
}
@@ -87,7 +89,7 @@ func startServers(superNode super_node.NodeInterface) error {
wsEndpoint = "127.0.0.1:8080"
}
var exposeAll = true
var wsOrigins []string = nil
var wsOrigins []string
_, _, wsErr := rpc.StartWSEndpoint(wsEndpoint, superNode.APIs(), []string{"vdb"}, wsOrigins, exposeAll)
if wsErr != nil {
return wsErr
cmd/streamSubscribe.go
+21 -19
View File
@@ -32,7 +32,7 @@ import (
"github.com/vulcanize/vulcanizedb/libraries/shared/streamer"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/geth/client"
"github.com/vulcanize/vulcanizedb/pkg/eth/client"
)
// streamSubscribeCmd represents the streamSubscribe command
@@ -42,6 +42,8 @@ var streamSubscribeCmd = &cobra.Command{
Long: `This command is for demo and testing purposes and is used to subscribe to the super node with the provided subscription configuration parameters.
It does not do anything with the data streamed from the super node other than unpack it and print it out for demonstration purposes.`,
Run: func(cmd *cobra.Command, args []string) {
subCommand = cmd.CalledAs()
logWithCommand = *log.WithField("SubCommand", subCommand)
streamSubscribe()
},
}
@@ -55,7 +57,7 @@ func streamSubscribe() {
configureSubscription()
// Create a new rpc client and a subscription streamer with that client
rpcClient := getRpcClient()
rpcClient := getRPCClient()
str := streamer.NewSuperNodeStreamer(rpcClient)
// Buffered channel for reading subscription payloads
@@ -64,22 +66,22 @@ func streamSubscribe() {
// Subscribe to the super node service with the given config/filter parameters
sub, err := str.Stream(payloadChan, subscriptionConfig)
if err != nil {
log.Fatal(err)
logWithCommand.Fatal(err)
}
log.Info("awaiting payloads")
logWithCommand.Info("awaiting payloads")
// Receive response payloads and print out the results
for {
select {
case payload := <-payloadChan:
if payload.ErrMsg != "" {
log.Error(payload.ErrMsg)
logWithCommand.Error(payload.ErrMsg)
continue
}
for _, headerRlp := range payload.HeadersRlp {
var header types.Header
err = rlp.Decode(bytes.NewBuffer(headerRlp), &header)
if err != nil {
log.Error(err)
logWithCommand.Error(err)
continue
}
fmt.Printf("Header number %d, hash %s\n", header.Number.Int64(), header.Hash().Hex())
@@ -91,7 +93,7 @@ func streamSubscribe() {
stream := rlp.NewStream(buff, 0)
err := trx.DecodeRLP(stream)
if err != nil {
log.Error(err)
logWithCommand.Error(err)
continue
}
fmt.Printf("Transaction with hash %s\n", trx.Hash().Hex())
@@ -103,14 +105,14 @@ func streamSubscribe() {
stream := rlp.NewStream(buff, 0)
err = rct.DecodeRLP(stream)
if err != nil {
log.Error(err)
logWithCommand.Error(err)
continue
}
fmt.Printf("Receipt with block hash %s, trx hash %s\n", rct.BlockHash.Hex(), rct.TxHash.Hex())
fmt.Printf("rct: %v\n", rct)
for _, l := range rct.Logs {
if len(l.Topics) < 1 {
log.Error(fmt.Sprintf("log only has %d topics", len(l.Topics)))
logWithCommand.Error(fmt.Sprintf("log only has %d topics", len(l.Topics)))
continue
}
fmt.Printf("Log for block hash %s, trx hash %s, address %s, and with topic0 %s\n",
@@ -123,7 +125,7 @@ func streamSubscribe() {
var acct state.Account
err = rlp.Decode(bytes.NewBuffer(stateRlp), &acct)
if err != nil {
log.Error(err)
logWithCommand.Error(err)
continue
}
fmt.Printf("Account for key %s, and root %s, with balance %d\n",
@@ -135,9 +137,9 @@ func streamSubscribe() {
for storageKey, storageRlp := range mappedRlp {
fmt.Printf("with storage key %s\n", storageKey.Hex())
var i []interface{}
err := rlp.DecodeBytes(storageRlp, i)
err := rlp.DecodeBytes(storageRlp, &i)
if err != nil {
log.Error(err)
logWithCommand.Error(err)
continue
}
// if a leaf node
@@ -146,7 +148,7 @@ func streamSubscribe() {
if !ok {
continue
}
valueBytes, ok := i[0].([]byte)
valueBytes, ok := i[1].([]byte)
if !ok {
continue
}
@@ -156,13 +158,13 @@ func streamSubscribe() {
}
}
case err = <-sub.Err():
log.Fatal(err)
logWithCommand.Fatal(err)
}
}
}
func configureSubscription() {
log.Info("loading subscription config")
logWithCommand.Info("loading subscription config")
subscriptionConfig = config.Subscription{
// Below default to false, which means we do not backfill by default
BackFill: viper.GetBool("subscription.backfill"),
@@ -214,14 +216,14 @@ func configureSubscription() {
}
}
func getRpcClient() core.RpcClient {
func getRPCClient() core.RpcClient {
vulcPath := viper.GetString("subscription.path")
if vulcPath == "" {
vulcPath = "ws://127.0.0.1:8080" // default to and try the default ws url if no path is provided
}
rawRpcClient, err := rpc.Dial(vulcPath)
rawRPCClient, err := rpc.Dial(vulcPath)
if err != nil {
log.Fatal(err)
logWithCommand.Fatal(err)
}
return client.NewRpcClient(rawRpcClient, vulcPath)
return client.NewRpcClient(rawRPCClient, vulcPath)
}
cmd/syncAndPublish.go
+20 -18
View File
@@ -28,10 +28,10 @@ import (
"github.com/spf13/viper"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/geth"
"github.com/vulcanize/vulcanizedb/pkg/geth/client"
vRpc "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
"github.com/vulcanize/vulcanizedb/pkg/geth/node"
"github.com/vulcanize/vulcanizedb/pkg/eth"
"github.com/vulcanize/vulcanizedb/pkg/eth/client"
vRpc "github.com/vulcanize/vulcanizedb/pkg/eth/converters/rpc"
"github.com/vulcanize/vulcanizedb/pkg/eth/node"
"github.com/vulcanize/vulcanizedb/pkg/super_node"
"github.com/vulcanize/vulcanizedb/utils"
)
@@ -45,6 +45,8 @@ all block and state (diff) data over a websocket subscription. This process
then converts the eth data to IPLD objects and publishes them to IPFS. Additionally,
it maintains a local index of the IPLD objects' CIDs in Postgres.`,
Run: func(cmd *cobra.Command, args []string) {
subCommand = cmd.CalledAs()
logWithCommand = *log.WithField("SubCommand", subCommand)
syncAndPublish()
},
}
@@ -58,34 +60,34 @@ func init() {
func syncAndPublish() {
superNode, newNodeErr := newSuperNode()
if newNodeErr != nil {
log.Fatal(newNodeErr)
logWithCommand.Fatal(newNodeErr)
}
wg := &syn.WaitGroup{}
syncAndPubErr := superNode.SyncAndPublish(wg, nil, nil)
if syncAndPubErr != nil {
log.Fatal(syncAndPubErr)
logWithCommand.Fatal(syncAndPubErr)
}
if viper.GetBool("backfill.on") && viper.GetString("backfill.ipcPath") != "" {
if viper.GetBool("superNodeBackFill.on") && viper.GetString("superNodeBackFill.rpcPath") != "" {
backfiller, newBackFillerErr := newBackFiller()
if newBackFillerErr != nil {
log.Fatal(newBackFillerErr)
logWithCommand.Fatal(newBackFillerErr)
}
backfiller.FillGaps(wg, nil)
}
wg.Wait() // If an error was thrown, wg.Add was never called and this will fall through
}
func getBlockChainAndClient(path string) (*geth.BlockChain, core.RpcClient) {
rawRpcClient, dialErr := rpc.Dial(path)
func getBlockChainAndClient(path string) (*eth.BlockChain, core.RpcClient) {
rawRPCClient, dialErr := rpc.Dial(path)
if dialErr != nil {
log.Fatal(dialErr)
logWithCommand.Fatal(dialErr)
}
rpcClient := client.NewRpcClient(rawRpcClient, ipc)
ethClient := ethclient.NewClient(rawRpcClient)
rpcClient := client.NewRpcClient(rawRPCClient, ipc)
ethClient := ethclient.NewClient(rawRPCClient)
vdbEthClient := client.NewEthClient(ethClient)
vdbNode := node.MakeNode(rpcClient)
transactionConverter := vRpc.NewRpcTransactionConverter(ethClient)
blockChain := geth.NewBlockChain(vdbEthClient, rpcClient, vdbNode, transactionConverter)
blockChain := eth.NewBlockChain(vdbEthClient, rpcClient, vdbNode, transactionConverter)
return blockChain, rpcClient
}
@@ -97,7 +99,7 @@ func newSuperNode() (super_node.NodeInterface, error) {
if ipfsPath == "" {
home, homeDirErr := os.UserHomeDir()
if homeDirErr != nil {
log.Fatal(homeDirErr)
logWithCommand.Fatal(homeDirErr)
}
ipfsPath = filepath.Join(home, ".ipfs")
}
@@ -109,14 +111,14 @@ func newSuperNode() (super_node.NodeInterface, error) {
}
func newBackFiller() (super_node.BackFillInterface, error) {
blockChain, archivalRpcClient := getBlockChainAndClient(viper.GetString("backfill.ipcPath"))
blockChain, archivalRPCClient := getBlockChainAndClient(viper.GetString("superNodeBackFill.rpcPath"))
db := utils.LoadPostgres(databaseConfig, blockChain.Node())
freq := viper.GetInt("backfill.frequency")
freq := viper.GetInt("superNodeBackFill.frequency")
var frequency time.Duration
if freq <= 0 {
frequency = time.Minute * 5
} else {
frequency = time.Duration(freq)
}
return super_node.NewBackFillService(ipfsPath, &db, archivalRpcClient, time.Minute*frequency)
return super_node.NewBackFillService(ipfsPath, &db, archivalRPCClient, time.Minute*frequency)
}
cmd/syncPublishScreenAndServe.go
+7 -5
View File
@@ -35,6 +35,8 @@ then converts the eth data to IPLD objects and publishes them to IPFS. Additiona
it maintains a local index of the IPLD objects' CIDs in Postgres. It then opens up a server which
relays relevant data to requesting clients.`,
Run: func(cmd *cobra.Command, args []string) {
subCommand = cmd.CalledAs()
logWithCommand = *log.WithField("SubCommand", subCommand)
syncPublishScreenAndServe()
},
}
@@ -46,7 +48,7 @@ func init() {
func syncPublishScreenAndServe() {
superNode, newNodeErr := newSuperNode()
if newNodeErr != nil {
log.Fatal(newNodeErr)
logWithCommand.Fatal(newNodeErr)
}
wg := &syn.WaitGroup{}
@@ -54,20 +56,20 @@ func syncPublishScreenAndServe() {
forwardQuitChan := make(chan bool, 1)
syncAndPubErr := superNode.SyncAndPublish(wg, forwardPayloadChan, forwardQuitChan)
if syncAndPubErr != nil {
log.Fatal(syncAndPubErr)
logWithCommand.Fatal(syncAndPubErr)
}
superNode.ScreenAndServe(wg, forwardPayloadChan, forwardQuitChan)
if viper.GetBool("backfill.on") && viper.GetString("backfill.ipcPath") != "" {
if viper.GetBool("superNodeBackFill.on") && viper.GetString("superNodeBackFill.rpcPath") != "" {
backfiller, newBackFillerErr := newBackFiller()
if newBackFillerErr != nil {
log.Fatal(newBackFillerErr)
logWithCommand.Fatal(newBackFillerErr)
}
backfiller.FillGaps(wg, nil)
}
serverErr := startServers(superNode)
if serverErr != nil {
log.Fatal(serverErr)
logWithCommand.Fatal(serverErr)
}
wg.Wait()
}