watcher configuration; super node versioning

This commit is contained in:
Ian Norden
2020-02-25 16:38:46 -06:00
parent 25aa4634e9
commit 330a083749
13 changed files with 381 additions and 77 deletions
+37 -2
View File
@@ -19,8 +19,8 @@ package super_node
import (
"context"
"github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/rpc"
log "github.com/sirupsen/logrus"
@@ -105,6 +105,41 @@ func (api *PublicSuperNodeAPI) Stream(ctx context.Context, rlpParams []byte) (*r
}
// Node is a public rpc method to allow transformers to fetch the node info for the super node
func (api *PublicSuperNodeAPI) Node() core.Node {
// NOTE: this is the node info for the node that the super node is syncing from, not the node info for the super node itself
func (api *PublicSuperNodeAPI) Node() *core.Node {
return api.sn.Node()
}
// Chain returns the chain type that this super node instance supports
func (api *PublicSuperNodeAPI) Chain() shared.ChainType {
return api.sn.Chain()
}
// Struct for holding super node meta data
type InfoAPI struct{}
// NewPublicSuperNodeAPI creates a new PublicSuperNodeAPI with the provided underlying SyncPublishScreenAndServe process
func NewInfoAPI() *InfoAPI {
return &InfoAPI{}
}
// Modules returns modules supported by this api
func (iapi *InfoAPI) Modules() map[string]string {
return map[string]string{
"vdb": "Stream",
}
}
// NodeInfo gathers and returns a collection of metadata for the super node
func (iapi *InfoAPI) NodeInfo() *p2p.NodeInfo {
return &p2p.NodeInfo{
// TODO: formalize this
ID: "vulcanizeDB",
Name: "superNode",
}
}
// Version returns the version of the super node
func (iapi *InfoAPI) Version() string {
return VersionWithMeta
}
+13 -10
View File
@@ -17,7 +17,7 @@
package super_node
import (
"errors"
"fmt"
"sync"
"sync/atomic"
"time"
@@ -59,10 +59,12 @@ type BackFillService struct {
BatchSize uint64
// Channel for receiving quit signal
QuitChan chan bool
// Chain type
chain shared.ChainType
}
// NewBackFillService returns a new BackFillInterface
func NewBackFillService(settings *shared.SuperNodeConfig, screenAndServeChan chan shared.ConvertedData) (BackFillInterface, error) {
func NewBackFillService(settings *Config, screenAndServeChan chan shared.ConvertedData) (BackFillInterface, error) {
publisher, err := NewIPLDPublisher(settings.Chain, settings.IPFSPath)
if err != nil {
return nil, err
@@ -97,6 +99,7 @@ func NewBackFillService(settings *shared.SuperNodeConfig, screenAndServeChan cha
BatchSize: batchSize,
ScreenAndServeChan: screenAndServeChan,
QuitChan: settings.Quit,
chain: settings.Chain,
}, nil
}
@@ -109,18 +112,18 @@ func (bfs *BackFillService) FillGapsInSuperNode(wg *sync.WaitGroup) {
for {
select {
case <-bfs.QuitChan:
log.Info("quiting FillGaps process")
log.Infof("quiting %s FillGaps process", bfs.chain.String())
wg.Done()
return
case <-ticker.C:
log.Info("searching for gaps in the super node database")
log.Infof("searching for gaps in the %s super node database", bfs.chain.String())
startingBlock, err := bfs.Retriever.RetrieveFirstBlockNumber()
if err != nil {
log.Error(err)
continue
}
if startingBlock != 0 {
log.Info("found gap at the beginning of the sync")
log.Infof("found gap at the beginning of the %s sync", bfs.chain.String())
bfs.fillGaps(0, uint64(startingBlock-1))
}
gaps, err := bfs.Retriever.RetrieveGapsInData()
@@ -136,11 +139,11 @@ func (bfs *BackFillService) FillGapsInSuperNode(wg *sync.WaitGroup) {
}
}
}()
log.Info("fillGaps goroutine successfully spun up")
log.Infof("%s fillGaps goroutine successfully spun up", bfs.chain.String())
}
func (bfs *BackFillService) fillGaps(startingBlock, endingBlock uint64) error {
log.Infof("going to fill in gap from %d to %d", startingBlock, endingBlock)
log.Infof("going to fill in %s gap from %d to %d", bfs.chain.String(), startingBlock, endingBlock)
errChan := make(chan error)
done := make(chan bool)
err := bfs.backFill(startingBlock, endingBlock, errChan, done)
@@ -152,7 +155,7 @@ func (bfs *BackFillService) fillGaps(startingBlock, endingBlock uint64) error {
case err := <-errChan:
log.Error(err)
case <-done:
log.Infof("finished filling in gap from %d to %d", startingBlock, endingBlock)
log.Infof("finished filling in %s gap from %d to %d", bfs.chain.String(), startingBlock, endingBlock)
return nil
}
}
@@ -162,7 +165,7 @@ func (bfs *BackFillService) fillGaps(startingBlock, endingBlock uint64) error {
// It splits a large range up into smaller chunks, batch fetching and processing those chunks concurrently
func (bfs *BackFillService) backFill(startingBlock, endingBlock uint64, errChan chan error, done chan bool) error {
if endingBlock < startingBlock {
return errors.New("backfill: ending block number needs to be greater than starting block number")
return fmt.Errorf("%s backfill: ending block number needs to be greater than starting block number", bfs.chain.String())
}
//
// break the range up into bins of smaller ranges
@@ -230,7 +233,7 @@ func (bfs *BackFillService) backFill(startingBlock, endingBlock uint64, errChan
case forwardDone <- true:
default:
}
log.Infof("finished filling in gap sub-bin from %d to %d", doneWithHeights[0], doneWithHeights[1])
log.Infof("finished filling in %s gap sub-bin from %d to %d", bfs.chain.String(), doneWithHeights[0], doneWithHeights[1])
goroutinesFinished++
if goroutinesFinished >= len(blockRangeBins) {
done <- true
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package shared
package super_node
import (
"fmt"
@@ -31,13 +31,14 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/eth/core"
"github.com/vulcanize/vulcanizedb/pkg/eth/node"
"github.com/vulcanize/vulcanizedb/pkg/postgres"
"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
"github.com/vulcanize/vulcanizedb/utils"
)
// SuperNodeConfig struct
type SuperNodeConfig struct {
// Config struct
type Config struct {
// Ubiquitous fields
Chain ChainType
Chain shared.ChainType
IPFSPath string
DB *postgres.DB
DBConfig config.Database
@@ -61,9 +62,9 @@ type SuperNodeConfig struct {
// NewSuperNodeConfigs is used to initialize multiple SuperNode configs from a single config .toml file
// Separate chain supernode instances need to be ran in the same process in order to avoid lock contention on the ipfs repository
func NewSuperNodeConfigs() ([]*SuperNodeConfig, error) {
func NewSuperNodeConfigs() ([]*Config, error) {
chains := viper.GetStringSlice("superNode.chains")
configs := make([]*SuperNodeConfig, len(chains))
configs := make([]*Config, len(chains))
var err error
ipfsPath := viper.GetString("superNode.ipfsPath")
if ipfsPath == "" {
@@ -74,8 +75,8 @@ func NewSuperNodeConfigs() ([]*SuperNodeConfig, error) {
ipfsPath = filepath.Join(home, ".ipfs")
}
for i, chain := range chains {
sn := new(SuperNodeConfig)
sn.Chain, err = NewChainType(chain)
sn := new(Config)
sn.Chain, err = shared.NewChainType(chain)
if err != nil {
return nil, err
}
@@ -96,9 +97,12 @@ func NewSuperNodeConfigs() ([]*SuperNodeConfig, error) {
}
sn.Workers = workers
switch sn.Chain {
case Ethereum:
case shared.Ethereum:
sn.NodeInfo, sn.WSClient, err = getEthNodeAndClient(viper.GetString("superNode.ethereum.sync.wsPath"))
case Bitcoin:
if err != nil {
return nil, err
}
case shared.Bitcoin:
sn.NodeInfo = core.Node{
ID: viper.GetString("superNode.bitcoin.node.nodeID"),
ClientName: viper.GetString("superNode.bitcoin.node.clientName"),
@@ -146,21 +150,21 @@ func NewSuperNodeConfigs() ([]*SuperNodeConfig, error) {
}
configs[i] = sn
}
return configs, err
return configs, nil
}
// BackFillFields is used to fill in the BackFill fields of the config
func (sn *SuperNodeConfig) BackFillFields(chain string) error {
func (sn *Config) BackFillFields(chain string) error {
sn.BackFill = true
var httpClient interface{}
var err error
switch sn.Chain {
case Ethereum:
case shared.Ethereum:
_, httpClient, err = getEthNodeAndClient(viper.GetString("superNode.ethereum.backFill.httpPath"))
if err != nil {
return err
}
case Bitcoin:
case shared.Bitcoin:
httpClient = &rpcclient.ConnConfig{
Host: viper.GetString("superNode.bitcoin.backFill.httpPath"),
HTTPPostMode: true, // Bitcoin core only supports HTTP POST mode
+48 -29
View File
@@ -52,7 +52,7 @@ type SuperNode interface {
// Method to unsubscribe from the service
Unsubscribe(id rpc.ID)
// Method to access the node info for the service
Node() core.Node
Node() *core.Node
// Method to access chain type
Chain() shared.ChainType
}
@@ -84,7 +84,7 @@ type Service struct {
// A mapping of subscription params hash to the corresponding subscription params
SubscriptionTypes map[common.Hash]shared.SubscriptionSettings
// Info for the Geth node that this super node is working with
NodeInfo core.Node
NodeInfo *core.Node
// Number of publishAndIndex workers
WorkerPoolSize int
// chain type for this service
@@ -96,7 +96,7 @@ type Service struct {
}
// NewSuperNode creates a new super_node.Interface using an underlying super_node.Service struct
func NewSuperNode(settings *shared.SuperNodeConfig) (SuperNode, error) {
func NewSuperNode(settings *Config) (SuperNode, error) {
sn := new(Service)
var err error
// If we are syncing, initialize the needed interfaces
@@ -137,7 +137,7 @@ func NewSuperNode(settings *shared.SuperNodeConfig) (SuperNode, error) {
sn.Subscriptions = make(map[common.Hash]map[rpc.ID]Subscription)
sn.SubscriptionTypes = make(map[common.Hash]shared.SubscriptionSettings)
sn.WorkerPoolSize = settings.Workers
sn.NodeInfo = settings.NodeInfo
sn.NodeInfo = &settings.NodeInfo
sn.ipfsPath = settings.IPFSPath
sn.chain = settings.Chain
sn.db = settings.DB
@@ -151,6 +151,7 @@ func (sap *Service) Protocols() []p2p.Protocol {
// APIs returns the RPC descriptors the super node service offers
func (sap *Service) APIs() []rpc.API {
ifnoAPI := NewInfoAPI()
apis := []rpc.API{
{
Namespace: APIName,
@@ -158,6 +159,24 @@ func (sap *Service) APIs() []rpc.API {
Service: NewPublicSuperNodeAPI(sap),
Public: true,
},
{
Namespace: "rpc",
Version: APIVersion,
Service: ifnoAPI,
Public: true,
},
{
Namespace: "net",
Version: APIVersion,
Service: ifnoAPI,
Public: true,
},
{
Namespace: "admin",
Version: APIVersion,
Service: ifnoAPI,
Public: true,
},
}
chainAPI, err := NewPublicAPI(sap.chain, sap.db, sap.ipfsPath)
if err != nil {
@@ -204,13 +223,13 @@ func (sap *Service) ProcessData(wg *sync.WaitGroup, screenAndServePayload chan<-
case err := <-sub.Err():
log.Error(err)
case <-sap.QuitChan:
log.Info("quiting SyncAndPublish process")
log.Infof("quiting %s SyncAndPublish process", sap.chain.String())
wg.Done()
return
}
}
}()
log.Info("ProcessData goroutine successfully spun up")
log.Infof("%s ProcessData goroutine successfully spun up", sap.chain.String())
return nil
}
@@ -232,7 +251,7 @@ func (sap *Service) publishAndIndex(id int, publishAndIndexPayload <-chan shared
}
}
}()
log.Debugf("publishAndIndex goroutine successfully spun up")
log.Debugf("%s publishAndIndex goroutine successfully spun up", sap.chain.String())
}
// FilterAndServe listens for incoming converter data off the screenAndServePayload from the SyncAndConvert process
@@ -247,24 +266,24 @@ func (sap *Service) FilterAndServe(wg *sync.WaitGroup, screenAndServePayload <-c
case payload := <-screenAndServePayload:
sap.filterAndServe(payload)
case <-sap.QuitChan:
log.Info("quiting ScreenAndServe process")
log.Infof("quiting %s ScreenAndServe process", sap.chain.String())
wg.Done()
return
}
}
}()
log.Info("FilterAndServe goroutine successfully spun up")
log.Infof("%s FilterAndServe goroutine successfully spun up", sap.chain.String())
}
// filterAndServe filters the payload according to each subscription type and sends to the subscriptions
func (sap *Service) filterAndServe(payload shared.ConvertedData) {
log.Debugf("Sending payload to subscriptions")
log.Debugf("Sending %s payload to subscriptions", sap.chain.String())
sap.Lock()
for ty, subs := range sap.Subscriptions {
// Retrieve the subscription parameters for this subscription type
subConfig, ok := sap.SubscriptionTypes[ty]
if !ok {
log.Errorf("subscription configuration for subscription type %s not available", ty.Hex())
log.Errorf("%s subscription configuration for subscription type %s not available", sap.chain.String(), ty.Hex())
sap.closeType(ty)
continue
}
@@ -288,9 +307,9 @@ func (sap *Service) filterAndServe(payload shared.ConvertedData) {
for id, sub := range subs {
select {
case sub.PayloadChan <- SubscriptionPayload{Data: responseRLP, Err: "", Flag: EmptyFlag, Height: response.Height()}:
log.Debugf("sending super node payload to subscription %s", id)
log.Debugf("sending super node %s payload to subscription %s", sap.chain.String(), id)
default:
log.Infof("unable to send payload to subscription %s; channel has no receiver", id)
log.Infof("unable to send %s payload to subscription %s; channel has no receiver", sap.chain.String(), id)
}
}
}
@@ -300,7 +319,7 @@ func (sap *Service) filterAndServe(payload shared.ConvertedData) {
// Subscribe is used by the API to remotely subscribe to the service loop
// The params must be rlp serializable and satisfy the SubscriptionSettings() interface
func (sap *Service) Subscribe(id rpc.ID, sub chan<- SubscriptionPayload, quitChan chan<- bool, params shared.SubscriptionSettings) {
log.Infof("New subscription %s", id)
log.Infof("New %s subscription %s", sap.chain.String(), id)
subscription := Subscription{
ID: id,
PayloadChan: sub,
@@ -342,7 +361,7 @@ func (sap *Service) Subscribe(id rpc.ID, sub chan<- SubscriptionPayload, quitCha
// sendHistoricalData sends historical data to the requesting subscription
func (sap *Service) sendHistoricalData(sub Subscription, id rpc.ID, params shared.SubscriptionSettings) error {
log.Info("Sending historical data to subscription", id)
log.Infof("Sending %s historical data to subscription %s", sap.chain.String(), id)
// Retrieve cached CIDs relevant to this subscriber
var endingBlock int64
var startingBlock int64
@@ -361,13 +380,13 @@ func (sap *Service) sendHistoricalData(sub Subscription, id rpc.ID, params share
if endingBlock > params.EndingBlock().Int64() && params.EndingBlock().Int64() > 0 && params.EndingBlock().Int64() > startingBlock {
endingBlock = params.EndingBlock().Int64()
}
log.Debug("historical data starting block:", params.StartingBlock())
log.Debug("historical data ending block:", endingBlock)
log.Debugf("%s historical data starting block: %d", sap.chain.String(), params.StartingBlock().Int64())
log.Debugf("%s historical data ending block: %d", sap.chain.String(), endingBlock)
go func() {
for i := startingBlock; i <= endingBlock; i++ {
cidWrappers, empty, err := sap.Retriever.Retrieve(params, i)
if err != nil {
sendNonBlockingErr(sub, fmt.Errorf("CID Retrieval error at block %d\r%s", i, err.Error()))
sendNonBlockingErr(sub, fmt.Errorf("%s CID Retrieval error at block %d\r%s", sap.chain.String(), i, err.Error()))
continue
}
if empty {
@@ -376,7 +395,7 @@ func (sap *Service) sendHistoricalData(sub Subscription, id rpc.ID, params share
for _, cids := range cidWrappers {
response, err := sap.IPLDFetcher.Fetch(cids)
if err != nil {
sendNonBlockingErr(sub, fmt.Errorf("IPLD Fetching error at block %d\r%s", i, err.Error()))
sendNonBlockingErr(sub, fmt.Errorf("%s IPLD Fetching error at block %d\r%s", sap.chain.String(), i, err.Error()))
continue
}
responseRLP, err := rlp.EncodeToBytes(response)
@@ -386,18 +405,18 @@ func (sap *Service) sendHistoricalData(sub Subscription, id rpc.ID, params share
}
select {
case sub.PayloadChan <- SubscriptionPayload{Data: responseRLP, Err: "", Flag: EmptyFlag, Height: response.Height()}:
log.Debugf("sending super node historical data payload to subscription %s", id)
log.Debugf("sending super node historical data payload to %s subscription %s", sap.chain.String(), id)
default:
log.Infof("unable to send back-fill payload to subscription %s; channel has no receiver", id)
log.Infof("unable to send back-fill payload to %s subscription %s; channel has no receiver", sap.chain.String(), id)
}
}
}
// when we are done backfilling send an empty payload signifying so in the msg
select {
case sub.PayloadChan <- SubscriptionPayload{Data: nil, Err: "", Flag: BackFillCompleteFlag}:
log.Debugf("sending backfill completion notice to subscription %s", id)
log.Debugf("sending backfill completion notice to %s subscription %s", sap.chain.String(), id)
default:
log.Infof("unable to send backfill completion notice to subscription %s", id)
log.Infof("unable to send backfill completion notice to %s subscription %s", sap.chain.String(), id)
}
}()
return nil
@@ -405,7 +424,7 @@ func (sap *Service) sendHistoricalData(sub Subscription, id rpc.ID, params share
// Unsubscribe is used by the API to remotely unsubscribe to the StateDiffingService loop
func (sap *Service) Unsubscribe(id rpc.ID) {
log.Infof("Unsubscribing %s from the super node service", id)
log.Infof("Unsubscribing %s from the %s super node service", id, sap.chain.String())
sap.Lock()
for ty := range sap.Subscriptions {
delete(sap.Subscriptions[ty], id)
@@ -421,7 +440,7 @@ func (sap *Service) Unsubscribe(id rpc.ID) {
// Start is used to begin the service
// This is mostly just to satisfy the node.Service interface
func (sap *Service) Start(*p2p.Server) error {
log.Info("Starting super node service")
log.Infof("Starting %s super node service", sap.chain.String())
wg := new(sync.WaitGroup)
payloadChan := make(chan shared.ConvertedData, PayloadChanBufferSize)
if err := sap.ProcessData(wg, payloadChan); err != nil {
@@ -434,7 +453,7 @@ func (sap *Service) Start(*p2p.Server) error {
// Stop is used to close down the service
// This is mostly just to satisfy the node.Service interface
func (sap *Service) Stop() error {
log.Info("Stopping super node service")
log.Infof("Stopping %s super node service", sap.chain.String())
sap.Lock()
close(sap.QuitChan)
sap.close()
@@ -443,7 +462,7 @@ func (sap *Service) Stop() error {
}
// Node returns the node info for this service
func (sap *Service) Node() core.Node {
func (sap *Service) Node() *core.Node {
return sap.NodeInfo
}
@@ -455,7 +474,7 @@ func (sap *Service) Chain() shared.ChainType {
// close is used to close all listening subscriptions
// close needs to be called with subscription access locked
func (sap *Service) close() {
log.Info("Closing all subscriptions")
log.Infof("Closing all %s subscriptions", sap.chain.String())
for subType, subs := range sap.Subscriptions {
for _, sub := range subs {
sendNonBlockingQuit(sub)
@@ -468,7 +487,7 @@ func (sap *Service) close() {
// closeType is used to close all subscriptions of given type
// closeType needs to be called with subscription access locked
func (sap *Service) closeType(subType common.Hash) {
log.Infof("Closing all subscriptions of type %s", subType.String())
log.Infof("Closing all %s subscriptions of type %s", sap.chain.String(), subType.String())
subs := sap.Subscriptions[subType]
for _, sub := range subs {
sendNonBlockingQuit(sub)
+63
View File
@@ -0,0 +1,63 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package super_node
import "fmt"
const (
VersionMajor = 0 // Major version component of the current release
VersionMinor = 1 // Minor version component of the current release
VersionPatch = 0 // Patch version component of the current release
VersionMeta = "alpha" // Version metadata to append to the version string
)
// Version holds the textual version string.
var Version = func() string {
return fmt.Sprintf("%d.%d.%d", VersionMajor, VersionMinor, VersionPatch)
}()
// VersionWithMeta holds the textual version string including the metadata.
var VersionWithMeta = func() string {
v := Version
if VersionMeta != "" {
v += "-" + VersionMeta
}
return v
}()
// ArchiveVersion holds the textual version string
func ArchiveVersion(gitCommit string) string {
vsn := Version
if VersionMeta != "stable" {
vsn += "-" + VersionMeta
}
if len(gitCommit) >= 8 {
vsn += "-" + gitCommit[:8]
}
return vsn
}
func VersionWithCommit(gitCommit, gitDate string) string {
vsn := VersionWithMeta
if len(gitCommit) >= 8 {
vsn += "-" + gitCommit[:8]
}
if (VersionMeta != "stable") && (gitDate != "") {
vsn += "-" + gitDate
}
return vsn
}