forked from cerc-io/ipld-eth-server
goimports + streamSubscribe command for raw access to the seed node data
This commit is contained in:
Ian Norden
@@ -40,6 +40,7 @@ var (
|
||||
cfgFile string
|
||||
databaseConfig config.Database
|
||||
genConfig config.Plugin
|
||||
subConfig config.Subscription
|
||||
ipc string
|
||||
levelDbPath string
|
||||
queueRecheckInterval time.Duration
|
||||
|
||||
@@ -0,0 +1,214 @@
|
||||
// Copyright © 2019 Vulcanize, Inc
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
|
||||
"github.com/ethereum/go-ethereum/core/state"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/rlp"
|
||||
"github.com/ethereum/go-ethereum/rpc"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/viper"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/libraries/shared/streamer"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/config"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/geth/client"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
|
||||
)
|
||||
|
||||
// streamSubscribeCmd represents the streamSubscribe command
|
||||
var streamSubscribeCmd = &cobra.Command{
|
||||
Use: "streamSubscribe",
|
||||
Short: "This command is used to subscribe to the seed node stream with the provided filters",
|
||||
Long: ``,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
streamSubscribe()
|
||||
},
|
||||
}
|
||||
|
||||
func init() {
|
||||
rootCmd.AddCommand(streamSubscribeCmd)
|
||||
}
|
||||
|
||||
func streamSubscribe() {
|
||||
// Prep the subscription config/filters to be sent to the server
|
||||
subscriptionConfig()
|
||||
|
||||
// Create a new rpc client and a subscription streamer with that client
|
||||
rpcClient := getRpcClient()
|
||||
str := streamer.NewSeedStreamer(rpcClient)
|
||||
|
||||
// Buffered channel for reading subscription payloads
|
||||
payloadChan := make(chan ipfs.ResponsePayload, 8000)
|
||||
|
||||
// Subscribe to the seed node service with the given config/filter parameters
|
||||
sub, err := str.Stream(payloadChan, subConfig)
|
||||
if err != nil {
|
||||
println(err.Error())
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// Receive response payloads and print out the results
|
||||
for {
|
||||
select {
|
||||
case payload := <-payloadChan:
|
||||
if payload.Err != nil {
|
||||
log.Error(payload.Err)
|
||||
}
|
||||
for _, headerRlp := range payload.HeadersRlp {
|
||||
var header types.Header
|
||||
err = rlp.Decode(bytes.NewBuffer(headerRlp), &header)
|
||||
if err != nil {
|
||||
println(err.Error())
|
||||
log.Error(err)
|
||||
}
|
||||
println("Header")
|
||||
println(header.Hash().Hex())
|
||||
println(header.Number.Int64())
|
||||
}
|
||||
for _, trxRlp := range payload.TransactionsRlp {
|
||||
var trx types.Transaction
|
||||
buff := bytes.NewBuffer(trxRlp)
|
||||
stream := rlp.NewStream(buff, 0)
|
||||
err := trx.DecodeRLP(stream)
|
||||
if err != nil {
|
||||
println(err.Error())
|
||||
log.Error(err)
|
||||
}
|
||||
println("Trx")
|
||||
println(trx.Hash().Hex())
|
||||
}
|
||||
for _, rctRlp := range payload.ReceiptsRlp {
|
||||
var rct types.Receipt
|
||||
buff := bytes.NewBuffer(rctRlp)
|
||||
stream := rlp.NewStream(buff, 0)
|
||||
err = rct.DecodeRLP(stream)
|
||||
if err != nil {
|
||||
println(err.Error())
|
||||
log.Error(err)
|
||||
}
|
||||
println("Rct")
|
||||
for _, l := range rct.Logs {
|
||||
println("log")
|
||||
println(l.BlockHash.Hex())
|
||||
println(l.TxHash.Hex())
|
||||
println(l.Address.Hex())
|
||||
}
|
||||
}
|
||||
for key, stateRlp := range payload.StateNodesRlp {
|
||||
var acct state.Account
|
||||
err = rlp.Decode(bytes.NewBuffer(stateRlp), &acct)
|
||||
if err != nil {
|
||||
println(err.Error())
|
||||
log.Error(err)
|
||||
}
|
||||
println("State")
|
||||
print("key: ")
|
||||
println(key.Hex())
|
||||
print("root: ")
|
||||
println(acct.Root.Hex())
|
||||
print("balance: ")
|
||||
println(acct.Balance.Int64())
|
||||
}
|
||||
for stateKey, mappedRlp := range payload.StorageNodesRlp {
|
||||
println("Storage")
|
||||
print("state key: ")
|
||||
println(stateKey.Hex())
|
||||
for storageKey, storageRlp := range mappedRlp {
|
||||
println("Storage")
|
||||
print("key: ")
|
||||
println(storageKey.Hex())
|
||||
var i []interface{}
|
||||
err := rlp.DecodeBytes(storageRlp, i)
|
||||
if err != nil {
|
||||
println(err.Error())
|
||||
log.Error(err)
|
||||
}
|
||||
print("bytes: ")
|
||||
println(storageRlp)
|
||||
}
|
||||
}
|
||||
case err = <-sub.Err():
|
||||
println(err.Error())
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func subscriptionConfig() {
|
||||
log.Info("loading subscription config")
|
||||
vulcPath = viper.GetString("subscription.path")
|
||||
subConfig = config.Subscription{
|
||||
// Below default to false, which means we do not backfill by default
|
||||
BackFill: viper.GetBool("subscription.backfill"),
|
||||
BackFillOnly: viper.GetBool("subscription.backfillOnly"),
|
||||
|
||||
// Below default to 0
|
||||
// 0 start means we start at the beginning and 0 end means we continue indefinitely
|
||||
StartingBlock: viper.GetInt64("subscription.startingBlock"),
|
||||
EndingBlock: viper.GetInt64("subscription.endingBlock"),
|
||||
|
||||
// Below default to false, which means we get all headers by default
|
||||
HeaderFilter: config.HeaderFilter{
|
||||
Off: viper.GetBool("subscription.headerFilter.off"),
|
||||
FinalOnly: viper.GetBool("subscription.headerFilter.finalOnly"),
|
||||
},
|
||||
|
||||
// Below defaults to false and two slices of length 0
|
||||
// Which means we get all transactions by default
|
||||
TrxFilter: config.TrxFilter{
|
||||
Off: viper.GetBool("subscription.trxFilter.off"),
|
||||
Src: viper.GetStringSlice("subscription.trxFilter.src"),
|
||||
Dst: viper.GetStringSlice("subscription.trxFilter.dst"),
|
||||
},
|
||||
|
||||
// Below defaults to false and one slice of length 0
|
||||
// Which means we get all receipts by default
|
||||
ReceiptFilter: config.ReceiptFilter{
|
||||
Off: viper.GetBool("subscription.receiptFilter.off"),
|
||||
Topic0s: viper.GetStringSlice("subscription.receiptFilter.topic0s"),
|
||||
},
|
||||
|
||||
// Below defaults to two false, and a slice of length 0
|
||||
// Which means we get all state leafs by default, but no intermediate nodes
|
||||
StateFilter: config.StateFilter{
|
||||
Off: viper.GetBool("subscription.stateFilter.off"),
|
||||
IntermediateNodes: viper.GetBool("subscription.stateFilter.intermediateNodes"),
|
||||
Addresses: viper.GetStringSlice("subscription.stateFilter.addresses"),
|
||||
},
|
||||
|
||||
// Below defaults to two false, and two slices of length 0
|
||||
// Which means we get all storage leafs by default, but no intermediate nodes
|
||||
StorageFilter: config.StorageFilter{
|
||||
Off: viper.GetBool("subscription.storageFilter.off"),
|
||||
IntermediateNodes: viper.GetBool("subscription.storageFilter.intermediateNodes"),
|
||||
Addresses: viper.GetStringSlice("subscription.storageFilter.addresses"),
|
||||
StorageKeys: viper.GetStringSlice("subscription.storageFilter.storageKeys"),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func getRpcClient() core.RpcClient {
|
||||
rawRpcClient, err := rpc.Dial(vulcPath)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
return client.NewRpcClient(rawRpcClient, vulcPath)
|
||||
}
|
||||
@@ -39,7 +39,7 @@ var syncAndPublishCmd = &cobra.Command{
|
||||
Short: "Syncs all Ethereum data into IPFS, indexing the CIDs",
|
||||
Long: `This command works alongside a modified geth node which streams
|
||||
all block and state (diff) data over a websocket subscription. This process
|
||||
then converts the eth objects to IPLDs and publishes them IPFS. Additionally,
|
||||
then converts the eth data to IPLD objects and publishes them to IPFS. Additionally,
|
||||
it maintains a local index of the IPLD objects' CIDs in Postgres.`,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
syncAndPublish()
|
||||
|
||||
@@ -29,13 +29,12 @@ import (
|
||||
// syncPublishScreenAndServeCmd represents the syncPublishScreenAndServe command
|
||||
var syncPublishScreenAndServeCmd = &cobra.Command{
|
||||
Use: "syncPublishScreenAndServe",
|
||||
Short: "A brief description of your command",
|
||||
Long: `A longer description that spans multiple lines and likely contains examples
|
||||
and usage of using your command. For example:
|
||||
|
||||
Cobra is a CLI library for Go that empowers applications.
|
||||
This application is a tool to generate the needed files
|
||||
to quickly create a Cobra application.`,
|
||||
Short: "Syncs all Ethereum data into IPFS, indexing the CIDs, and uses this to serve data requests to requesting clients",
|
||||
Long: `This command works alongside a modified geth node which streams
|
||||
all block and state (diff) data over a websocket subscription. This process
|
||||
then converts the eth data to IPLD objects and publishes them to IPFS. Additionally,
|
||||
it maintains a local index of the IPLD objects' CIDs in Postgres. It then opens up a server which
|
||||
relays relevant data to requesting clients.`,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
syncPublishScreenAndServe()
|
||||
},
|
||||
@@ -44,7 +43,7 @@ to quickly create a Cobra application.`,
|
||||
func init() {
|
||||
rootCmd.AddCommand(syncPublishScreenAndServeCmd)
|
||||
syncPublishScreenAndServeCmd.Flags().StringVarP(&ipfsPath, "ipfs-path", "i", "~/.ipfs", "Path for configuring IPFS node")
|
||||
syncPublishScreenAndServeCmd.Flags().StringVarP(&vulcPath, "ipc-path", "p", "~/.vulcanize/vulcanize.ipc", "IPC path for the Vulcanize seed node server")
|
||||
syncPublishScreenAndServeCmd.Flags().StringVarP(&vulcPath, "sub-path", "p", "~/.vulcanize/vulcanize.ipc", "IPC path for the Vulcanize seed node server")
|
||||
}
|
||||
|
||||
func syncPublishScreenAndServe() {
|
||||
|
||||
-144
@@ -1,144 +0,0 @@
|
||||
// Copyright © 2019 Vulcanize, Inc
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
|
||||
"github.com/ethereum/go-ethereum/core/state"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/rlp"
|
||||
"github.com/ethereum/go-ethereum/rpc"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/libraries/shared/streamer"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/geth/client"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
|
||||
)
|
||||
|
||||
// test2Cmd represents the test2 command
|
||||
var test2Cmd = &cobra.Command{
|
||||
Use: "test2",
|
||||
Short: "A brief description of your command",
|
||||
Long: `A longer description that spans multiple lines and likely contains examples
|
||||
and usage of using your command. For example:
|
||||
|
||||
Cobra is a CLI library for Go that empowers applications.
|
||||
This application is a tool to generate the needed files
|
||||
to quickly create a Cobra application.`,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
test2()
|
||||
},
|
||||
}
|
||||
|
||||
func test2() {
|
||||
rpcClient := getRpcClient()
|
||||
str := streamer.NewSeedStreamer(rpcClient)
|
||||
payloadChan := make(chan ipfs.ResponsePayload, 8000)
|
||||
streamFilters := ipfs.StreamFilters{}
|
||||
streamFilters.HeaderFilter.FinalOnly = true
|
||||
streamFilters.ReceiptFilter.Topic0s = []string{
|
||||
"0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef",
|
||||
"0x930a61a57a70a73c2a503615b87e2e54fe5b9cdeacda518270b852296ab1a377",
|
||||
}
|
||||
streamFilters.StateFilter.Addresses = []string{"0xde0B295669a9FD93d5F28D9Ec85E40f4cb697BAe"}
|
||||
streamFilters.StorageFilter.Off = true
|
||||
//streamFilters.TrxFilter.Src = []string{"0xde0B295669a9FD93d5F28D9Ec85E40f4cb697BAe"}
|
||||
//streamFilters.TrxFilter.Dst = []string{"0xde0B295669a9FD93d5F28D9Ec85E40f4cb697BAe"}
|
||||
sub, err := str.Stream(payloadChan, streamFilters)
|
||||
if err != nil {
|
||||
println(err.Error())
|
||||
log.Fatal(err)
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case payload := <-payloadChan:
|
||||
if payload.Err != nil {
|
||||
log.Error(payload.Err)
|
||||
}
|
||||
for _, headerRlp := range payload.HeadersRlp {
|
||||
var header types.Header
|
||||
err = rlp.Decode(bytes.NewBuffer(headerRlp), &header)
|
||||
if err != nil {
|
||||
println(err.Error())
|
||||
log.Error(err)
|
||||
}
|
||||
println("header")
|
||||
println(header.Hash().Hex())
|
||||
println(header.Number.Int64())
|
||||
}
|
||||
for _, trxRlp := range payload.TransactionsRlp {
|
||||
var trx types.Transaction
|
||||
buff := bytes.NewBuffer(trxRlp)
|
||||
stream := rlp.NewStream(buff, 0)
|
||||
err := trx.DecodeRLP(stream)
|
||||
if err != nil {
|
||||
println(err.Error())
|
||||
log.Error(err)
|
||||
}
|
||||
println("trx")
|
||||
println(trx.Hash().Hex())
|
||||
}
|
||||
for _, rctRlp := range payload.ReceiptsRlp {
|
||||
var rct types.Receipt
|
||||
buff := bytes.NewBuffer(rctRlp)
|
||||
stream := rlp.NewStream(buff, 0)
|
||||
err = rct.DecodeRLP(stream)
|
||||
if err != nil {
|
||||
println(err.Error())
|
||||
log.Error(err)
|
||||
}
|
||||
println("rct")
|
||||
for _, l := range rct.Logs {
|
||||
println("log")
|
||||
println(l.BlockHash.Hex())
|
||||
println(l.TxHash.Hex())
|
||||
println(l.Address.Hex())
|
||||
}
|
||||
}
|
||||
for _, stateRlp := range payload.StateNodesRlp {
|
||||
var acct state.Account
|
||||
err = rlp.Decode(bytes.NewBuffer(stateRlp), &acct)
|
||||
if err != nil {
|
||||
println(err.Error())
|
||||
log.Error(err)
|
||||
}
|
||||
println("state")
|
||||
println(acct.Root.Hex())
|
||||
println(acct.Balance.Int64())
|
||||
}
|
||||
case err = <-sub.Err():
|
||||
println(err.Error())
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func init() {
|
||||
rootCmd.AddCommand(test2Cmd)
|
||||
test2Cmd.Flags().StringVarP(&vulcPath, "ipc-path", "p", "~/.vulcanize/vulcanize.ipc", "IPC path for the Vulcanize seed node server")
|
||||
}
|
||||
|
||||
func getRpcClient() core.RpcClient {
|
||||
println(vulcPath)
|
||||
rawRpcClient, err := rpc.Dial(vulcPath)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
return client.NewRpcClient(rawRpcClient, vulcPath)
|
||||
}
|
||||
Reference in New Issue
Block a user