remove maker migrations and convert back to timestamps and fix bug in
composeAndExecute command
This commit is contained in:
Ian Norden
+42
-32
@@ -54,23 +54,35 @@ var composeAndExecuteCmd = &cobra.Command{
|
||||
ipcPath = "http://kovan0.vulcanize.io:8545"
|
||||
|
||||
[exporter]
|
||||
name = "exporter"
|
||||
[exporter.transformers]
|
||||
transformer1 = "path/to/transformer1"
|
||||
transformer2 = "path/to/transformer2"
|
||||
transformer3 = "path/to/transformer3"
|
||||
transformer4 = "path/to/transformer4"
|
||||
[exporter.types]
|
||||
transformer1 = "eth_event"
|
||||
transformer2 = "eth_event"
|
||||
transformer3 = "eth_event"
|
||||
transformer4 = "eth_storage"
|
||||
[exporter.repositories]
|
||||
transformers = "github.com/account/repo"
|
||||
transformer4 = "github.com/account2/repo2"
|
||||
[exporter.migrations]
|
||||
transformers = "db/migrations"
|
||||
transformer4 = "to/db/migrations"
|
||||
name = "exampleTransformerExporter"
|
||||
save = false
|
||||
transformerNames = [
|
||||
"transformer1",
|
||||
"transformer2",
|
||||
"transformer3",
|
||||
"transformer4",
|
||||
]
|
||||
[exporter.transformer1]
|
||||
path = "path/to/transformer1"
|
||||
type = "eth_event"
|
||||
repository = "github.com/account/repo"
|
||||
migrations = "db/migrations"
|
||||
[exporter.transformer2]
|
||||
path = "path/to/transformer2"
|
||||
type = "eth_event"
|
||||
repository = "github.com/account/repo"
|
||||
migrations = "db/migrations"
|
||||
[exporter.transformer3]
|
||||
path = "path/to/transformer3"
|
||||
type = "eth_storage"
|
||||
repository = "github.com/account/repo"
|
||||
migrations = "db/migrations"
|
||||
[exporter.transformer4]
|
||||
path = "path/to/transformer4"
|
||||
type = "eth_event"
|
||||
repository = "github.com/account2/repo2"
|
||||
migrations = "to/db/migrations"
|
||||
|
||||
|
||||
Note: If any of the imported transformer need additional
|
||||
config variables do not forget to include those as well
|
||||
@@ -81,7 +93,7 @@ This plugin is loaded and the set of transformer initializers is exported
|
||||
from it and loaded into and executed over by the appropriate watcher.
|
||||
|
||||
The type of watcher that the transformer works with is specified using the
|
||||
exporter.types config variable as shown above. Currently there are watchers
|
||||
type variable for each transformer in the config. Currently there are watchers
|
||||
of event data from an eth node (eth_event) and storage data from an eth node
|
||||
(eth_storage). Soon there will be watchers for ipfs (ipfs_event and ipfs_storage).
|
||||
|
||||
@@ -217,34 +229,32 @@ func prepConfig() {
|
||||
transformers := make(map[string]config.Transformer)
|
||||
for _, name := range names {
|
||||
transformer := viper.GetStringMapString("exporter." + name)
|
||||
_, ok := transformer["path"]
|
||||
if !ok {
|
||||
p, ok := transformer["path"]
|
||||
if !ok || p == "" {
|
||||
log.Fatal(fmt.Sprintf("%s transformer config is missing `path` value", name))
|
||||
}
|
||||
_, ok = transformer["repository"]
|
||||
if !ok {
|
||||
r, ok := transformer["repository"]
|
||||
if !ok || r == "" {
|
||||
log.Fatal(fmt.Sprintf("%s transformer config is missing `repository` value", name))
|
||||
}
|
||||
_, ok = transformer["migrations"]
|
||||
if !ok {
|
||||
m, ok := transformer["migrations"]
|
||||
if !ok || m == "" {
|
||||
log.Fatal(fmt.Sprintf("%s transformer config is missing `migrations` value", name))
|
||||
}
|
||||
ty, ok := transformer["type"]
|
||||
t, ok := transformer["type"]
|
||||
if !ok {
|
||||
log.Fatal(fmt.Sprintf("%s transformer config is missing `type` value", name))
|
||||
}
|
||||
|
||||
transformerType := config.GetTransformerType(ty)
|
||||
transformerType := config.GetTransformerType(t)
|
||||
if transformerType == config.UnknownTransformerType {
|
||||
log.Fatal(errors.New(`unknown transformer type in exporter config
|
||||
accepted types are "eth_event", "eth_storage"`))
|
||||
log.Fatal(errors.New(`unknown transformer type in exporter config accepted types are "eth_event", "eth_storage"`))
|
||||
}
|
||||
|
||||
transformers[name] = config.Transformer{
|
||||
Path: transformer["path"],
|
||||
Path: p,
|
||||
Type: transformerType,
|
||||
RepositoryPath: transformer["repository"],
|
||||
MigrationPath: transformer["migrations"],
|
||||
RepositoryPath: r,
|
||||
MigrationPath: m,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,71 +0,0 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2018 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/vulcanize/vulcanizedb/libraries/shared"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/fs"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/storage"
|
||||
"log"
|
||||
)
|
||||
|
||||
// parseStorageDiffsCmd represents the parseStorageDiffs command
|
||||
var parseStorageDiffsCmd = &cobra.Command{
|
||||
Use: "parseStorageDiffs",
|
||||
Short: "Continuously ingest storage diffs from a CSV file",
|
||||
Long: `Read storage diffs out of a CSV file that is constantly receiving
|
||||
new rows from an Ethereum node. For example:
|
||||
|
||||
./vulcanizedb parseStorageDiffs --config=environments/staging.toml
|
||||
|
||||
Note that the path to your storage diffs must be configured in your toml
|
||||
file under storageDiffsPath.`,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
parseStorageDiffs()
|
||||
},
|
||||
}
|
||||
|
||||
func init() {
|
||||
rootCmd.AddCommand(parseStorageDiffsCmd)
|
||||
}
|
||||
|
||||
func parseStorageDiffs() {
|
||||
blockChain := getBlockChain()
|
||||
db, err := postgres.NewDB(databaseConfig, blockChain.Node())
|
||||
if err != nil {
|
||||
log.Fatal("Failed to initialize database: ", err)
|
||||
}
|
||||
|
||||
tailer := fs.FileTailer{Path: storageDiffsPath}
|
||||
|
||||
// TODO: configure transformers
|
||||
watcher := shared.NewStorageWatcher(tailer, db)
|
||||
watcher.AddTransformers([]storage.TransformerInitializer{
|
||||
transformers.GetCatStorageTransformer().NewTransformer,
|
||||
transformers.GetPitStorageTransformer().NewTransformer,
|
||||
transformers.GetVatStorageTransformer().NewTransformer,
|
||||
transformers.GetVowStorageTransformer().NewTransformer,
|
||||
})
|
||||
|
||||
err = watcher.Execute()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user