cerc-io/ipld-eth-server
12
0
Fork 1
Code Issues 10 Pull Requests 2 Actions Packages Projects Releases 37 Wiki Activity

goimports + streamSubscribe command for raw access to the seed node data

Browse Source
This commit is contained in:
Ian Norden
2019-12-02 13:24:51 -06:00
parent 8ccdfd4835
commit b1bb646ad5
26 changed files with 362 additions and 234 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/root.go
+1
View File
@@ -40,6 +40,7 @@ var (
cfgFile string
databaseConfig config.Database
genConfig config.Plugin
subConfig config.Subscription
ipc string
levelDbPath string
queueRecheckInterval time.Duration
cmd/streamSubscribe.go
+214
View File
@@ -0,0 +1,214 @@
// Copyright © 2019 Vulcanize, Inc
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"bytes"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/rpc"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"github.com/vulcanize/vulcanizedb/libraries/shared/streamer"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/geth/client"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
)
// streamSubscribeCmd represents the streamSubscribe command
var streamSubscribeCmd = &cobra.Command{
Use: "streamSubscribe",
Short: "This command is used to subscribe to the seed node stream with the provided filters",
Long: ``,
Run: func(cmd *cobra.Command, args []string) {
streamSubscribe()
},
}
func init() {
rootCmd.AddCommand(streamSubscribeCmd)
}
func streamSubscribe() {
// Prep the subscription config/filters to be sent to the server
subscriptionConfig()
// Create a new rpc client and a subscription streamer with that client
rpcClient := getRpcClient()
str := streamer.NewSeedStreamer(rpcClient)
// Buffered channel for reading subscription payloads
payloadChan := make(chan ipfs.ResponsePayload, 8000)
// Subscribe to the seed node service with the given config/filter parameters
sub, err := str.Stream(payloadChan, subConfig)
if err != nil {
println(err.Error())
log.Fatal(err)
}
// Receive response payloads and print out the results
for {
select {
case payload := <-payloadChan:
if payload.Err != nil {
log.Error(payload.Err)
}
for _, headerRlp := range payload.HeadersRlp {
var header types.Header
err = rlp.Decode(bytes.NewBuffer(headerRlp), &header)
if err != nil {
println(err.Error())
log.Error(err)
}
println("Header")
println(header.Hash().Hex())
println(header.Number.Int64())
}
for _, trxRlp := range payload.TransactionsRlp {
var trx types.Transaction
buff := bytes.NewBuffer(trxRlp)
stream := rlp.NewStream(buff, 0)
err := trx.DecodeRLP(stream)
if err != nil {
println(err.Error())
log.Error(err)
}
println("Trx")
println(trx.Hash().Hex())
}
for _, rctRlp := range payload.ReceiptsRlp {
var rct types.Receipt
buff := bytes.NewBuffer(rctRlp)
stream := rlp.NewStream(buff, 0)
err = rct.DecodeRLP(stream)
if err != nil {
println(err.Error())
log.Error(err)
}
println("Rct")
for _, l := range rct.Logs {
println("log")
println(l.BlockHash.Hex())
println(l.TxHash.Hex())
println(l.Address.Hex())
}
}
for key, stateRlp := range payload.StateNodesRlp {
var acct state.Account
err = rlp.Decode(bytes.NewBuffer(stateRlp), &acct)
if err != nil {
println(err.Error())
log.Error(err)
}
println("State")
print("key: ")
println(key.Hex())
print("root: ")
println(acct.Root.Hex())
print("balance: ")
println(acct.Balance.Int64())
}
for stateKey, mappedRlp := range payload.StorageNodesRlp {
println("Storage")
print("state key: ")
println(stateKey.Hex())
for storageKey, storageRlp := range mappedRlp {
println("Storage")
print("key: ")
println(storageKey.Hex())
var i []interface{}
err := rlp.DecodeBytes(storageRlp, i)
if err != nil {
println(err.Error())
log.Error(err)
}
print("bytes: ")
println(storageRlp)
}
}
case err = <-sub.Err():
println(err.Error())
log.Fatal(err)
}
}
}
func subscriptionConfig() {
log.Info("loading subscription config")
vulcPath = viper.GetString("subscription.path")
subConfig = config.Subscription{
// Below default to false, which means we do not backfill by default
BackFill: viper.GetBool("subscription.backfill"),
BackFillOnly: viper.GetBool("subscription.backfillOnly"),
// Below default to 0
// 0 start means we start at the beginning and 0 end means we continue indefinitely
StartingBlock: viper.GetInt64("subscription.startingBlock"),
EndingBlock: viper.GetInt64("subscription.endingBlock"),
// Below default to false, which means we get all headers by default
HeaderFilter: config.HeaderFilter{
Off: viper.GetBool("subscription.headerFilter.off"),
FinalOnly: viper.GetBool("subscription.headerFilter.finalOnly"),
},
// Below defaults to false and two slices of length 0
// Which means we get all transactions by default
TrxFilter: config.TrxFilter{
Off: viper.GetBool("subscription.trxFilter.off"),
Src: viper.GetStringSlice("subscription.trxFilter.src"),
Dst: viper.GetStringSlice("subscription.trxFilter.dst"),
},
// Below defaults to false and one slice of length 0
// Which means we get all receipts by default
ReceiptFilter: config.ReceiptFilter{
Off: viper.GetBool("subscription.receiptFilter.off"),
Topic0s: viper.GetStringSlice("subscription.receiptFilter.topic0s"),
},
// Below defaults to two false, and a slice of length 0
// Which means we get all state leafs by default, but no intermediate nodes
StateFilter: config.StateFilter{
Off: viper.GetBool("subscription.stateFilter.off"),
IntermediateNodes: viper.GetBool("subscription.stateFilter.intermediateNodes"),
Addresses: viper.GetStringSlice("subscription.stateFilter.addresses"),
},
// Below defaults to two false, and two slices of length 0
// Which means we get all storage leafs by default, but no intermediate nodes
StorageFilter: config.StorageFilter{
Off: viper.GetBool("subscription.storageFilter.off"),
IntermediateNodes: viper.GetBool("subscription.storageFilter.intermediateNodes"),
Addresses: viper.GetStringSlice("subscription.storageFilter.addresses"),
StorageKeys: viper.GetStringSlice("subscription.storageFilter.storageKeys"),
},
}
}
func getRpcClient() core.RpcClient {
rawRpcClient, err := rpc.Dial(vulcPath)
if err != nil {
log.Fatal(err)
}
return client.NewRpcClient(rawRpcClient, vulcPath)
}
cmd/syncAndPublish.go
+1 -1
View File
@@ -39,7 +39,7 @@ var syncAndPublishCmd = &cobra.Command{
Short: "Syncs all Ethereum data into IPFS, indexing the CIDs",
Long: `This command works alongside a modified geth node which streams
all block and state (diff) data over a websocket subscription. This process
then converts the eth objects to IPLDs and publishes them IPFS. Additionally,
then converts the eth data to IPLD objects and publishes them to IPFS. Additionally,
it maintains a local index of the IPLD objects' CIDs in Postgres.`,
Run: func(cmd *cobra.Command, args []string) {
syncAndPublish()
cmd/syncPublishScreenAndServe.go
+7 -8
View File
@@ -29,13 +29,12 @@ import (
// syncPublishScreenAndServeCmd represents the syncPublishScreenAndServe command
var syncPublishScreenAndServeCmd = &cobra.Command{
Use: "syncPublishScreenAndServe",
Short: "A brief description of your command",
Long: `A longer description that spans multiple lines and likely contains examples
and usage of using your command. For example:
Cobra is a CLI library for Go that empowers applications.
This application is a tool to generate the needed files
to quickly create a Cobra application.`,
Short: "Syncs all Ethereum data into IPFS, indexing the CIDs, and uses this to serve data requests to requesting clients",
Long: `This command works alongside a modified geth node which streams
all block and state (diff) data over a websocket subscription. This process
then converts the eth data to IPLD objects and publishes them to IPFS. Additionally,
it maintains a local index of the IPLD objects' CIDs in Postgres. It then opens up a server which
relays relevant data to requesting clients.`,
Run: func(cmd *cobra.Command, args []string) {
syncPublishScreenAndServe()
},
@@ -44,7 +43,7 @@ to quickly create a Cobra application.`,
func init() {
rootCmd.AddCommand(syncPublishScreenAndServeCmd)
syncPublishScreenAndServeCmd.Flags().StringVarP(&ipfsPath, "ipfs-path", "i", "~/.ipfs", "Path for configuring IPFS node")
syncPublishScreenAndServeCmd.Flags().StringVarP(&vulcPath, "ipc-path", "p", "~/.vulcanize/vulcanize.ipc", "IPC path for the Vulcanize seed node server")
syncPublishScreenAndServeCmd.Flags().StringVarP(&vulcPath, "sub-path", "p", "~/.vulcanize/vulcanize.ipc", "IPC path for the Vulcanize seed node server")
}
func syncPublishScreenAndServe() {
cmd/test2.go
-144
View File
@@ -1,144 +0,0 @@
// Copyright © 2019 Vulcanize, Inc
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"bytes"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/rpc"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/vulcanize/vulcanizedb/libraries/shared/streamer"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/geth/client"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
)
// test2Cmd represents the test2 command
var test2Cmd = &cobra.Command{
Use: "test2",
Short: "A brief description of your command",
Long: `A longer description that spans multiple lines and likely contains examples
and usage of using your command. For example:
Cobra is a CLI library for Go that empowers applications.
This application is a tool to generate the needed files
to quickly create a Cobra application.`,
Run: func(cmd *cobra.Command, args []string) {
test2()
},
}
func test2() {
rpcClient := getRpcClient()
str := streamer.NewSeedStreamer(rpcClient)
payloadChan := make(chan ipfs.ResponsePayload, 8000)
streamFilters := ipfs.StreamFilters{}
streamFilters.HeaderFilter.FinalOnly = true
streamFilters.ReceiptFilter.Topic0s = []string{
"0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef",
"0x930a61a57a70a73c2a503615b87e2e54fe5b9cdeacda518270b852296ab1a377",
}
streamFilters.StateFilter.Addresses = []string{"0xde0B295669a9FD93d5F28D9Ec85E40f4cb697BAe"}
streamFilters.StorageFilter.Off = true
//streamFilters.TrxFilter.Src = []string{"0xde0B295669a9FD93d5F28D9Ec85E40f4cb697BAe"}
//streamFilters.TrxFilter.Dst = []string{"0xde0B295669a9FD93d5F28D9Ec85E40f4cb697BAe"}
sub, err := str.Stream(payloadChan, streamFilters)
if err != nil {
println(err.Error())
log.Fatal(err)
}
for {
select {
case payload := <-payloadChan:
if payload.Err != nil {
log.Error(payload.Err)
}
for _, headerRlp := range payload.HeadersRlp {
var header types.Header
err = rlp.Decode(bytes.NewBuffer(headerRlp), &header)
if err != nil {
println(err.Error())
log.Error(err)
}
println("header")
println(header.Hash().Hex())
println(header.Number.Int64())
}
for _, trxRlp := range payload.TransactionsRlp {
var trx types.Transaction
buff := bytes.NewBuffer(trxRlp)
stream := rlp.NewStream(buff, 0)
err := trx.DecodeRLP(stream)
if err != nil {
println(err.Error())
log.Error(err)
}
println("trx")
println(trx.Hash().Hex())
}
for _, rctRlp := range payload.ReceiptsRlp {
var rct types.Receipt
buff := bytes.NewBuffer(rctRlp)
stream := rlp.NewStream(buff, 0)
err = rct.DecodeRLP(stream)
if err != nil {
println(err.Error())
log.Error(err)
}
println("rct")
for _, l := range rct.Logs {
println("log")
println(l.BlockHash.Hex())
println(l.TxHash.Hex())
println(l.Address.Hex())
}
}
for _, stateRlp := range payload.StateNodesRlp {
var acct state.Account
err = rlp.Decode(bytes.NewBuffer(stateRlp), &acct)
if err != nil {
println(err.Error())
log.Error(err)
}
println("state")
println(acct.Root.Hex())
println(acct.Balance.Int64())
}
case err = <-sub.Err():
println(err.Error())
log.Fatal(err)
}
}
}
func init() {
rootCmd.AddCommand(test2Cmd)
test2Cmd.Flags().StringVarP(&vulcPath, "ipc-path", "p", "~/.vulcanize/vulcanize.ipc", "IPC path for the Vulcanize seed node server")
}
func getRpcClient() core.RpcClient {
println(vulcPath)
rawRpcClient, err := rpc.Dial(vulcPath)
if err != nil {
log.Fatal(err)
}
return client.NewRpcClient(rawRpcClient, vulcPath)
}