2019-01-25 08:59:23 +00:00
// Copyright © 2019 Vulcanize, Inc
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
2019-04-24 20:05:57 +00:00
"os"
"plugin"
syn "sync"
2019-04-29 20:21:32 +00:00
"time"
2019-04-24 20:05:57 +00:00
2019-02-18 10:32:20 +00:00
log "github.com/sirupsen/logrus"
2019-01-25 08:59:23 +00:00
"github.com/spf13/cobra"
2019-04-24 20:05:57 +00:00
"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
2019-01-25 08:59:23 +00:00
"github.com/vulcanize/vulcanizedb/libraries/shared/watcher"
2019-02-07 21:35:25 +00:00
"github.com/vulcanize/vulcanizedb/pkg/fs"
2019-02-02 22:15:09 +00:00
p2 "github.com/vulcanize/vulcanizedb/pkg/plugin"
"github.com/vulcanize/vulcanizedb/pkg/plugin/helpers"
2019-01-25 08:59:23 +00:00
"github.com/vulcanize/vulcanizedb/utils"
)
2019-02-28 17:51:54 +00:00
// composeAndExecuteCmd represents the composeAndExecute command
2019-01-25 08:59:23 +00:00
var composeAndExecuteCmd = & cobra . Command {
Use : "composeAndExecute" ,
Short : "Composes, loads, and executes transformer initializer plugin" ,
Long : ` This command needs a config . toml file of form :
[ database ]
2019-02-25 09:33:05 +00:00
name = "vulcanize_public"
2019-01-25 08:59:23 +00:00
hostname = "localhost"
2019-02-25 09:33:05 +00:00
user = "vulcanize"
2019-01-25 08:59:23 +00:00
password = "vulcanize"
2019-02-25 09:33:05 +00:00
port = 5432
2019-01-25 08:59:23 +00:00
[ client ]
2019-03-14 21:49:27 +00:00
ipcPath = "/Users/user/Library/Ethereum/geth.ipc"
2019-01-25 08:59:23 +00:00
[ exporter ]
2019-02-25 09:33:05 +00:00
home = "github.com/vulcanize/vulcanizedb"
name = "exampleTransformerExporter"
2019-02-13 21:00:09 +00:00
save = false
transformerNames = [
"transformer1" ,
"transformer2" ,
"transformer3" ,
"transformer4" ,
]
[ exporter . transformer1 ]
path = "path/to/transformer1"
type = "eth_event"
repository = "github.com/account/repo"
migrations = "db/migrations"
2019-03-07 17:17:07 +00:00
rank = "0"
2019-02-13 21:00:09 +00:00
[ exporter . transformer2 ]
path = "path/to/transformer2"
2019-03-13 16:14:35 +00:00
type = "eth_contract"
2019-02-13 21:00:09 +00:00
repository = "github.com/account/repo"
migrations = "db/migrations"
2019-03-07 17:17:07 +00:00
rank = "2"
2019-02-25 09:33:05 +00:00
[ exporter . transformer3 ]
2019-02-13 21:00:09 +00:00
path = "path/to/transformer3"
2019-02-25 09:33:05 +00:00
type = "eth_event"
2019-02-13 21:00:09 +00:00
repository = "github.com/account/repo"
migrations = "db/migrations"
2019-03-07 17:17:07 +00:00
rank = "0"
2019-02-25 09:33:05 +00:00
[ exporter . transformer4 ]
2019-02-13 21:00:09 +00:00
path = "path/to/transformer4"
2019-02-25 09:33:05 +00:00
type = "eth_storage"
2019-02-13 21:00:09 +00:00
repository = "github.com/account2/repo2"
migrations = "to/db/migrations"
2019-03-07 17:17:07 +00:00
rank = "1"
2019-02-13 21:00:09 +00:00
2019-01-25 08:59:23 +00:00
2019-02-28 17:51:54 +00:00
Note : If any of the plugin transformer need additional
configuration variables include them in the . toml file as well
2019-01-25 08:59:23 +00:00
2019-02-07 21:35:25 +00:00
This information is used to write and build a go plugin with a transformer
2019-01-25 08:59:23 +00:00
set composed from the transformer imports specified in the config file
2019-02-07 21:35:25 +00:00
This plugin is loaded and the set of transformer initializers is exported
from it and loaded into and executed over by the appropriate watcher .
The type of watcher that the transformer works with is specified using the
2019-02-13 21:00:09 +00:00
type variable for each transformer in the config . Currently there are watchers
2019-02-07 21:35:25 +00:00
of event data from an eth node ( eth_event ) and storage data from an eth node
2019-03-13 16:14:35 +00:00
( eth_storage ) , and a more generic interface for accepting contract_watcher pkg
based transformers which can perform both event watching and public method
polling ( eth_contract ) .
2019-02-07 21:35:25 +00:00
Transformers of different types can be ran together in the same command using a
single config file or in separate command instances using different config files
Specify config location when executing the command :
. / vulcanizedb composeAndExecute -- config = . / environments / config_name . toml ` ,
2019-01-25 08:59:23 +00:00
Run : func ( cmd * cobra . Command , args [ ] string ) {
composeAndExecute ( )
} ,
}
func composeAndExecute ( ) {
2019-02-07 21:35:25 +00:00
// Build plugin generator config
prepConfig ( )
2019-01-31 23:14:42 +00:00
2019-02-07 21:35:25 +00:00
// Generate code to build the plugin according to the config file
2019-02-18 10:32:20 +00:00
log . Info ( "generating plugin" )
2019-02-02 22:15:09 +00:00
generator , err := p2 . NewGenerator ( genConfig , databaseConfig )
2019-01-25 08:59:23 +00:00
if err != nil {
2019-02-02 22:15:09 +00:00
log . Fatal ( err )
}
err = generator . GenerateExporterPlugin ( )
if err != nil {
2019-02-18 10:32:20 +00:00
log . Debug ( "generating plugin failed" )
2019-01-25 08:59:23 +00:00
log . Fatal ( err )
}
2019-01-31 01:38:56 +00:00
// Get the plugin path and load the plugin
2019-02-02 22:15:09 +00:00
_ , pluginPath , err := genConfig . GetPluginPaths ( )
2019-01-25 08:59:23 +00:00
if err != nil {
log . Fatal ( err )
}
2019-02-02 22:15:09 +00:00
if ! genConfig . Save {
defer helpers . ClearFiles ( pluginPath )
2019-01-31 23:14:42 +00:00
}
2019-02-28 17:51:54 +00:00
log . Info ( "linking plugin" , pluginPath )
2019-01-25 08:59:23 +00:00
plug , err := plugin . Open ( pluginPath )
if err != nil {
2019-02-28 17:51:54 +00:00
log . Debug ( "linking plugin failed" )
2019-01-25 08:59:23 +00:00
log . Fatal ( err )
}
2019-01-31 01:38:56 +00:00
// Load the `Exporter` symbol from the plugin
2019-02-18 10:32:20 +00:00
log . Info ( "loading transformers from plugin" )
2019-01-25 08:59:23 +00:00
symExporter , err := plug . Lookup ( "Exporter" )
if err != nil {
2019-02-18 10:32:20 +00:00
log . Debug ( "loading Exporter symbol failed" )
2019-01-25 08:59:23 +00:00
log . Fatal ( err )
}
2019-01-31 01:38:56 +00:00
// Assert that the symbol is of type Exporter
2019-01-25 08:59:23 +00:00
exporter , ok := symExporter . ( Exporter )
if ! ok {
2019-02-18 10:32:20 +00:00
log . Debug ( "plugged-in symbol not of type Exporter" )
2019-01-25 08:59:23 +00:00
os . Exit ( 1 )
}
2019-03-13 16:14:35 +00:00
// Use the Exporters export method to load the EventTransformerInitializer, StorageTransformerInitializer, and ContractTransformerInitializer sets
ethEventInitializers , ethStorageInitializers , ethContractInitializers := exporter . Export ( )
2019-01-31 01:38:56 +00:00
// Setup bc and db objects
blockChain := getBlockChain ( )
db := utils . LoadPostgres ( databaseConfig , blockChain . Node ( ) )
2019-02-07 21:35:25 +00:00
// Execute over transformer sets returned by the exporter
// Use WaitGroup to wait on both goroutines
var wg syn . WaitGroup
if len ( ethEventInitializers ) > 0 {
2019-02-10 22:42:41 +00:00
ew := watcher . NewEventWatcher ( & db , blockChain )
ew . AddTransformers ( ethEventInitializers )
2019-02-07 21:35:25 +00:00
wg . Add ( 1 )
2019-02-10 22:42:41 +00:00
go watchEthEvents ( & ew , & wg )
2019-02-07 21:35:25 +00:00
}
if len ( ethStorageInitializers ) > 0 {
tailer := fs . FileTailer { Path : storageDiffsPath }
2019-04-24 20:05:57 +00:00
storageFetcher := fetcher . NewCsvTailStorageFetcher ( tailer )
sw := watcher . NewStorageWatcher ( storageFetcher , & db )
2019-02-10 22:42:41 +00:00
sw . AddTransformers ( ethStorageInitializers )
2019-02-07 21:35:25 +00:00
wg . Add ( 1 )
2019-02-10 22:42:41 +00:00
go watchEthStorage ( & sw , & wg )
2019-02-07 21:35:25 +00:00
}
2019-03-11 16:43:43 +00:00
2019-03-13 16:14:35 +00:00
if len ( ethContractInitializers ) > 0 {
gw := watcher . NewContractWatcher ( & db , blockChain )
gw . AddTransformers ( ethContractInitializers )
2019-03-11 16:43:43 +00:00
wg . Add ( 1 )
2019-03-14 21:49:27 +00:00
go watchEthContract ( & gw , & wg )
2019-03-11 16:43:43 +00:00
}
2019-02-07 21:35:25 +00:00
wg . Wait ( )
}
func init ( ) {
rootCmd . AddCommand ( composeAndExecuteCmd )
2019-04-29 20:21:32 +00:00
composeAndExecuteCmd . Flags ( ) . BoolVarP ( & recheckHeadersArg , "recheck-headers" , "r" , false , "whether to re-check headers for watched events" )
composeAndExecuteCmd . Flags ( ) . DurationVarP ( & queueRecheckInterval , "queue-recheck-interval" , "q" , 5 * time . Minute , "how often to recheck queued storage diffs" )
2019-02-07 21:35:25 +00:00
}