cerc-io/ipld-eth-server
12
0
Fork 1
Code Issues 10 Pull Requests 2 Actions Packages Projects Releases 37 Wiki Activity

work on wasm watchers

Browse Source
This commit is contained in:
Ian Norden
2020-02-24 12:54:10 -06:00
parent e3e8700d34
commit 25aa4634e9
32 changed files with 683 additions and 339 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/watcher/constructors.go
+1 -1
View File
@@ -23,7 +23,7 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/eth/core"
"github.com/vulcanize/vulcanizedb/pkg/postgres"
shared2 "github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
"github.com/vulcanize/vulcanizedb/pkg/super_node/watcher/eth"
"github.com/vulcanize/vulcanizedb/pkg/watcher/eth"
"github.com/vulcanize/vulcanizedb/pkg/watcher/shared"
)
pkg/watcher/eth/repository.go
+41 -4
View File
@@ -19,15 +19,21 @@ package eth
import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/postgres"
"github.com/vulcanize/vulcanizedb/pkg/super_node"
"github.com/vulcanize/vulcanizedb/pkg/watcher/shared"
)
var (
vacuumThreshold int64 = 5000 // dont know how to decided what this should be set to
)
// Repository is the underlying struct for satisfying the shared.Repository interface for eth
type Repository struct {
db *postgres.DB
triggerFunctions [][2]string
deleteCalls int64
}
// NewRepository returns a new eth.Repository that satisfies the shared.Repository interface
@@ -35,6 +41,7 @@ func NewRepository(db *postgres.DB, triggerFunctions [][2]string) shared.Reposit
return &Repository{
db: db,
triggerFunctions: triggerFunctions,
deleteCalls: 0,
}
}
@@ -46,12 +53,42 @@ func (r *Repository) LoadTriggers() error {
// QueueData puts super node payload data into the db queue
func (r *Repository) QueueData(payload super_node.SubscriptionPayload) error {
panic("implement me")
pgStr := `INSERT INTO eth.queued_data (data, height) VALUES ($1, $2)
ON CONFLICT (height) DO UPDATE SET (data) VALUES ($1)`
_, err := r.db.Exec(pgStr, payload.Data, payload.Height)
return err
}
// GetQueueData grabs super node payload data from the db queue
func (r *Repository) GetQueueData(height int64, hash string) (super_node.SubscriptionPayload, error) {
panic("implement me")
// GetQueueData grabs a chunk super node payload data from the queue table so that it can
// be forwarded to the ready table
// this is used to make sure we enter data into the ready table in sequential order
// even if we receive data out-of-order
// it returns the new index
// delete the data it retrieves so as to clear the queue
func (r *Repository) GetQueueData(height int64) (super_node.SubscriptionPayload, int64, error) {
r.deleteCalls++
pgStr := `DELETE FROM eth.queued_data
WHERE height = $1
RETURNING *`
var res shared.QueuedData
if err := r.db.Get(&res, pgStr, height); err != nil {
return super_node.SubscriptionPayload{}, height, err
}
payload := super_node.SubscriptionPayload{
Data: res.Data,
Height: res.Height,
Flag: super_node.EmptyFlag,
}
height++
// Periodically clean up space in the queue table
if r.deleteCalls >= vacuumThreshold {
_, err := r.db.Exec(`VACUUM ANALYZE eth.queued_data`)
if err != nil {
logrus.Error(err)
}
r.deleteCalls = 0
}
return payload, height, nil
}
// ReadyData puts super node payload data in the tables ready for processing by trigger functions
pkg/watcher/service.go
+43 -4
View File
@@ -19,6 +19,9 @@ package watcher
import (
"sync"
"sync/atomic"
"time"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/rpc"
"github.com/sirupsen/logrus"
@@ -84,7 +87,11 @@ func (s *Service) Init() error {
// Watch is the top level loop for watching super node
func (s *Service) Watch(wg *sync.WaitGroup) error {
sub, err := s.SuperNodeStreamer.Stream(s.PayloadChan, s.WatcherConfig.SubscriptionConfig)
rlpConfig, err := rlp.EncodeToBytes(s.WatcherConfig.SubscriptionConfig)
if err != nil {
return err
}
sub, err := s.SuperNodeStreamer.Stream(s.PayloadChan, rlpConfig)
if err != nil {
return err
}
@@ -108,8 +115,13 @@ func (s *Service) Watch(wg *sync.WaitGroup) error {
// combinedQueuing assumes data is not necessarily going to come in linear order
// this is true when we are backfilling and streaming at the head or when we are
// only streaming at the head since reorgs can occur
// NOTE: maybe we should push everything to the wait queue, otherwise the index could be shifted as we retrieve data from it
func (s *Service) combinedQueuing(wg *sync.WaitGroup, sub *rpc.ClientSubscription) {
wg.Add(1)
// this goroutine is responsible for allocating incoming data to the ready or wait queue
// depending on if it is at the current index or not
forwardQuit := make(chan bool)
go func() {
for {
select {
@@ -120,7 +132,8 @@ func (s *Service) combinedQueuing(wg *sync.WaitGroup, sub *rpc.ClientSubscriptio
continue
}
if payload.Height == atomic.LoadInt64(s.payloadIndex) {
// If the data is at our current index it is ready to be processed; add it to the ready data queue
// If the data is at our current index it is ready to be processed
// add it to the ready data queue and increment the index
if err := s.Repository.ReadyData(payload); err != nil {
logrus.Error(err)
}
@@ -134,15 +147,41 @@ func (s *Service) combinedQueuing(wg *sync.WaitGroup, sub *rpc.ClientSubscriptio
logrus.Error(err)
case <-s.QuitChan:
logrus.Info("Watcher shutting down")
forwardQuit <- true
wg.Done()
return
}
}
}()
ticker := time.NewTicker(5 * time.Second)
// this goroutine is responsible for moving data from the wait queue to the ready queue
// preserving the correct order and alignment with the current index
go func() {
for {
select {
case <-ticker.C:
// retrieve queued data, in order, and forward it to the ready queue
index := atomic.LoadInt64(s.payloadIndex)
queueData, newIndex, err := s.Repository.GetQueueData(index)
if err != nil {
logrus.Error(err)
continue
}
atomic.StoreInt64(s.payloadIndex, newIndex)
if err := s.Repository.ReadyData(queueData); err != nil {
logrus.Error(err)
}
case <-forwardQuit:
return
default:
// do nothing, wait til next tick
}
}
}()
}
// backFillOnlyQueuing assumes the data is coming in contiguously and behind the head
// it puts all data on the ready queue
// backFillOnlyQueuing assumes the data is coming in contiguously from behind the head
// it puts all data directly into the ready queue
// it continues until the watcher is told to quit or we receive notification that the backfill is finished
func (s *Service) backFillOnlyQueuing(wg *sync.WaitGroup, sub *rpc.ClientSubscription) {
wg.Add(1)
pkg/watcher/shared/interfaces.go
+2 -3
View File
@@ -20,18 +20,17 @@ import (
"github.com/ethereum/go-ethereum/rpc"
"github.com/vulcanize/vulcanizedb/pkg/super_node"
"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
)
// Repository is the interface for the Postgres database
type Repository interface {
LoadTriggers() error
QueueData(payload super_node.SubscriptionPayload) error
GetQueueData(height int64, hash string) (super_node.SubscriptionPayload, error)
GetQueueData(height int64) (super_node.SubscriptionPayload, int64, error)
ReadyData(payload super_node.SubscriptionPayload) error
}
// SuperNodeStreamer is the interface for streaming data from a vulcanizeDB super node
type SuperNodeStreamer interface {
Stream(payloadChan chan super_node.SubscriptionPayload, params shared.SubscriptionSettings) (*rpc.ClientSubscription, error)
Stream(payloadChan chan super_node.SubscriptionPayload, rlpParams []byte) (*rpc.ClientSubscription, error)
}
pkg/watcher/shared/models.go
+24
View File
@@ -0,0 +1,24 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package shared
// QueuedData is the db model for queued data
type QueuedData struct {
ID int64 `db:"id"`
Data []byte `db:"data"`
Height int64 `db:"height"`
}