forked from cerc-io/plugeth
cmd/swarm/swarm-snapshot: swarm snapshot generator (#18453)
* cmd/swarm/swarm-snapshot: add binary to create network snapshots * cmd/swarm/swarm-snapshot: refactor and extend tests * p2p/simulations: remove unused triggerChecks func and fix linter * internal/cmdtest: raise the timeout for killing TestCmd * cmd/swarm/swarm-snapshot: add more comments and other minor adjustments * cmd/swarm/swarm-snapshot: remove redundant check in createSnapshot * cmd/swarm/swarm-snapshot: change comment wording * p2p/simulations: revert Simulation.Run from master https://github.com/ethersphere/go-ethereum/pull/1077/files#r247078904 * cmd/swarm/swarm-snapshot: address pr comments * swarm/network/simulations/discovery: removed snapshot write to file * cmd/swarm/swarm-snapshot, swarm/network/simulations: removed redundant connection event check, fixed lint error
This commit is contained in:
parent
f728837ee6
commit
34f11e752f
157
cmd/swarm/swarm-snapshot/create.go
Normal file
157
cmd/swarm/swarm-snapshot/create.go
Normal file
@ -0,0 +1,157 @@
|
||||
// Copyright 2018 The go-ethereum Authors
|
||||
// This file is part of go-ethereum.
|
||||
//
|
||||
// go-ethereum is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// go-ethereum is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/ethereum/go-ethereum/log"
|
||||
"github.com/ethereum/go-ethereum/node"
|
||||
"github.com/ethereum/go-ethereum/p2p/simulations"
|
||||
"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
|
||||
"github.com/ethereum/go-ethereum/swarm/network"
|
||||
"github.com/ethereum/go-ethereum/swarm/network/simulation"
|
||||
cli "gopkg.in/urfave/cli.v1"
|
||||
)
|
||||
|
||||
// create is used as the entry function for "create" app command.
|
||||
func create(ctx *cli.Context) error {
|
||||
log.PrintOrigins(true)
|
||||
log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(ctx.Int("verbosity")), log.StreamHandler(os.Stdout, log.TerminalFormat(true))))
|
||||
|
||||
if len(ctx.Args()) < 1 {
|
||||
return errors.New("argument should be the filename to verify or write-to")
|
||||
}
|
||||
filename, err := touchPath(ctx.Args()[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return createSnapshot(filename, ctx.Int("nodes"), strings.Split(ctx.String("services"), ","))
|
||||
}
|
||||
|
||||
// createSnapshot creates a new snapshot on filesystem with provided filename,
|
||||
// number of nodes and service names.
|
||||
func createSnapshot(filename string, nodes int, services []string) (err error) {
|
||||
log.Debug("create snapshot", "filename", filename, "nodes", nodes, "services", services)
|
||||
|
||||
sim := simulation.New(map[string]simulation.ServiceFunc{
|
||||
"bzz": func(ctx *adapters.ServiceContext, b *sync.Map) (node.Service, func(), error) {
|
||||
addr := network.NewAddr(ctx.Config.Node())
|
||||
kad := network.NewKademlia(addr.Over(), network.NewKadParams())
|
||||
hp := network.NewHiveParams()
|
||||
hp.KeepAliveInterval = time.Duration(200) * time.Millisecond
|
||||
hp.Discovery = true // discovery must be enabled when creating a snapshot
|
||||
|
||||
config := &network.BzzConfig{
|
||||
OverlayAddr: addr.Over(),
|
||||
UnderlayAddr: addr.Under(),
|
||||
HiveParams: hp,
|
||||
}
|
||||
return network.NewBzz(config, kad, nil, nil, nil), nil, nil
|
||||
},
|
||||
})
|
||||
defer sim.Close()
|
||||
|
||||
_, err = sim.AddNodes(nodes)
|
||||
if err != nil {
|
||||
return fmt.Errorf("add nodes: %v", err)
|
||||
}
|
||||
|
||||
err = sim.Net.ConnectNodesRing(nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("connect nodes: %v", err)
|
||||
}
|
||||
|
||||
ctx, cancelSimRun := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
defer cancelSimRun()
|
||||
if _, err := sim.WaitTillHealthy(ctx); err != nil {
|
||||
return fmt.Errorf("wait for healthy kademlia: %v", err)
|
||||
}
|
||||
|
||||
var snap *simulations.Snapshot
|
||||
if len(services) > 0 {
|
||||
// If service names are provided, include them in the snapshot.
|
||||
// But, check if "bzz" service is not among them to remove it
|
||||
// form the snapshot as it exists on snapshot creation.
|
||||
var removeServices []string
|
||||
var wantBzz bool
|
||||
for _, s := range services {
|
||||
if s == "bzz" {
|
||||
wantBzz = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !wantBzz {
|
||||
removeServices = []string{"bzz"}
|
||||
}
|
||||
snap, err = sim.Net.SnapshotWithServices(services, removeServices)
|
||||
} else {
|
||||
snap, err = sim.Net.Snapshot()
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("create snapshot: %v", err)
|
||||
}
|
||||
jsonsnapshot, err := json.Marshal(snap)
|
||||
if err != nil {
|
||||
return fmt.Errorf("json encode snapshot: %v", err)
|
||||
}
|
||||
return ioutil.WriteFile(filename, jsonsnapshot, 0666)
|
||||
}
|
||||
|
||||
// touchPath creates an empty file and all subdirectories
|
||||
// that are missing.
|
||||
func touchPath(filename string) (string, error) {
|
||||
if path.IsAbs(filename) {
|
||||
if _, err := os.Stat(filename); err == nil {
|
||||
// path exists, overwrite
|
||||
return filename, nil
|
||||
}
|
||||
}
|
||||
|
||||
d, f := path.Split(filename)
|
||||
dir, err := filepath.Abs(filepath.Dir(os.Args[0]))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
_, err = os.Stat(path.Join(dir, filename))
|
||||
if err == nil {
|
||||
// path exists, overwrite
|
||||
return filename, nil
|
||||
}
|
||||
|
||||
dirPath := path.Join(dir, d)
|
||||
filePath := path.Join(dirPath, f)
|
||||
if d != "" {
|
||||
err = os.MkdirAll(dirPath, os.ModeDir)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
|
||||
return filePath, nil
|
||||
}
|
138
cmd/swarm/swarm-snapshot/create_test.go
Normal file
138
cmd/swarm/swarm-snapshot/create_test.go
Normal file
@ -0,0 +1,138 @@
|
||||
// Copyright 2018 The go-ethereum Authors
|
||||
// This file is part of go-ethereum.
|
||||
//
|
||||
// go-ethereum is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// go-ethereum is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/ethereum/go-ethereum/p2p/simulations"
|
||||
)
|
||||
|
||||
// TestSnapshotCreate is a high level e2e test that tests for snapshot generation.
|
||||
// It runs a few "create" commands with different flag values and loads generated
|
||||
// snapshot files to validate their content.
|
||||
func TestSnapshotCreate(t *testing.T) {
|
||||
for _, v := range []struct {
|
||||
name string
|
||||
nodes int
|
||||
services string
|
||||
}{
|
||||
{
|
||||
name: "defaults",
|
||||
},
|
||||
{
|
||||
name: "more nodes",
|
||||
nodes: defaultNodes + 5,
|
||||
},
|
||||
{
|
||||
name: "services",
|
||||
services: "stream,pss,zorglub",
|
||||
},
|
||||
{
|
||||
name: "services with bzz",
|
||||
services: "bzz,pss",
|
||||
},
|
||||
} {
|
||||
t.Run(v.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
file, err := ioutil.TempFile("", "swarm-snapshot")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.Remove(file.Name())
|
||||
|
||||
if err = file.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
args := []string{"create"}
|
||||
if v.nodes > 0 {
|
||||
args = append(args, "--nodes", strconv.Itoa(v.nodes))
|
||||
}
|
||||
if v.services != "" {
|
||||
args = append(args, "--services", v.services)
|
||||
}
|
||||
testCmd := runSnapshot(t, append(args, file.Name())...)
|
||||
|
||||
testCmd.ExpectExit()
|
||||
if code := testCmd.ExitStatus(); code != 0 {
|
||||
t.Fatalf("command exit code %v, expected 0", code)
|
||||
}
|
||||
|
||||
f, err := os.Open(file.Name())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
err := f.Close()
|
||||
if err != nil {
|
||||
t.Error("closing snapshot file", "err", err)
|
||||
}
|
||||
}()
|
||||
|
||||
b, err := ioutil.ReadAll(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var snap simulations.Snapshot
|
||||
err = json.Unmarshal(b, &snap)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
wantNodes := v.nodes
|
||||
if wantNodes == 0 {
|
||||
wantNodes = defaultNodes
|
||||
}
|
||||
gotNodes := len(snap.Nodes)
|
||||
if gotNodes != wantNodes {
|
||||
t.Errorf("got %v nodes, want %v", gotNodes, wantNodes)
|
||||
}
|
||||
|
||||
if len(snap.Conns) == 0 {
|
||||
t.Error("no connections in a snapshot")
|
||||
}
|
||||
|
||||
var wantServices []string
|
||||
if v.services != "" {
|
||||
wantServices = strings.Split(v.services, ",")
|
||||
} else {
|
||||
wantServices = []string{"bzz"}
|
||||
}
|
||||
// sort service names so they can be comparable
|
||||
// as strings to every node sorted services
|
||||
sort.Strings(wantServices)
|
||||
|
||||
for i, n := range snap.Nodes {
|
||||
gotServices := n.Node.Config.Services
|
||||
sort.Strings(gotServices)
|
||||
if fmt.Sprint(gotServices) != fmt.Sprint(wantServices) {
|
||||
t.Errorf("got services %v for node %v, want %v", gotServices, i, wantServices)
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
}
|
82
cmd/swarm/swarm-snapshot/main.go
Normal file
82
cmd/swarm/swarm-snapshot/main.go
Normal file
@ -0,0 +1,82 @@
|
||||
// Copyright 2018 The go-ethereum Authors
|
||||
// This file is part of go-ethereum.
|
||||
//
|
||||
// go-ethereum is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// go-ethereum is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"github.com/ethereum/go-ethereum/cmd/utils"
|
||||
"github.com/ethereum/go-ethereum/log"
|
||||
cli "gopkg.in/urfave/cli.v1"
|
||||
)
|
||||
|
||||
var gitCommit string // Git SHA1 commit hash of the release (set via linker flags)
|
||||
|
||||
// default value for "create" command --nodes flag
|
||||
const defaultNodes = 10
|
||||
|
||||
func main() {
|
||||
err := newApp().Run(os.Args)
|
||||
if err != nil {
|
||||
log.Error(err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// newApp construct a new instance of Swarm Snapshot Utility.
|
||||
// Method Run is called on it in the main function and in tests.
|
||||
func newApp() (app *cli.App) {
|
||||
app = utils.NewApp(gitCommit, "Swarm Snapshot Utility")
|
||||
|
||||
app.Name = "swarm-snapshot"
|
||||
app.Usage = ""
|
||||
|
||||
// app flags (for all commands)
|
||||
app.Flags = []cli.Flag{
|
||||
cli.IntFlag{
|
||||
Name: "verbosity",
|
||||
Value: 1,
|
||||
Usage: "verbosity level",
|
||||
},
|
||||
}
|
||||
|
||||
app.Commands = []cli.Command{
|
||||
{
|
||||
Name: "create",
|
||||
Aliases: []string{"c"},
|
||||
Usage: "create a swarm snapshot",
|
||||
Action: create,
|
||||
// Flags only for "create" command.
|
||||
// Allow app flags to be specified after the
|
||||
// command argument.
|
||||
Flags: append(app.Flags,
|
||||
cli.IntFlag{
|
||||
Name: "nodes",
|
||||
Value: defaultNodes,
|
||||
Usage: "number of nodes",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "services",
|
||||
Value: "bzz",
|
||||
Usage: "comma separated list of services to boot the nodes with",
|
||||
},
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
return app
|
||||
}
|
49
cmd/swarm/swarm-snapshot/run_test.go
Normal file
49
cmd/swarm/swarm-snapshot/run_test.go
Normal file
@ -0,0 +1,49 @@
|
||||
// Copyright 2018 The go-ethereum Authors
|
||||
// This file is part of go-ethereum.
|
||||
//
|
||||
// go-ethereum is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// go-ethereum is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/docker/docker/pkg/reexec"
|
||||
"github.com/ethereum/go-ethereum/internal/cmdtest"
|
||||
)
|
||||
|
||||
func init() {
|
||||
reexec.Register("swarm-snapshot", func() {
|
||||
if err := newApp().Run(os.Args); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
os.Exit(0)
|
||||
})
|
||||
}
|
||||
|
||||
func runSnapshot(t *testing.T, args ...string) *cmdtest.TestCmd {
|
||||
tt := cmdtest.NewTestCmd(t, nil)
|
||||
tt.Run("swarm-snapshot", args...)
|
||||
return tt
|
||||
}
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
if reexec.Init() {
|
||||
return
|
||||
}
|
||||
os.Exit(m.Run())
|
||||
}
|
@ -640,6 +640,8 @@ func (k *Kademlia) saturation() int {
|
||||
})
|
||||
// TODO evaluate whether this check cannot just as well be done within the eachbin
|
||||
depth := depthForPot(k.conns, k.NeighbourhoodSize, k.base)
|
||||
|
||||
// if in the iterator above we iterated deeper than the neighbourhood depth - return depth
|
||||
if depth < prev {
|
||||
return depth
|
||||
}
|
||||
|
@ -18,16 +18,12 @@ package discovery
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@ -86,12 +82,10 @@ func getDbStore(nodeID string) (*state.DBStore, error) {
|
||||
}
|
||||
|
||||
var (
|
||||
nodeCount = flag.Int("nodes", 10, "number of nodes to create (default 10)")
|
||||
initCount = flag.Int("conns", 1, "number of originally connected peers (default 1)")
|
||||
snapshotFile = flag.String("snapshot", "", "path to create snapshot file in")
|
||||
loglevel = flag.Int("loglevel", 3, "verbosity of logs")
|
||||
rawlog = flag.Bool("rawlog", false, "remove terminal formatting from logs")
|
||||
serviceOverride = flag.String("services", "", "remove or add services to the node snapshot; prefix with \"+\" to add, \"-\" to remove; example: +pss,-discovery")
|
||||
nodeCount = flag.Int("nodes", 10, "number of nodes to create (default 10)")
|
||||
initCount = flag.Int("conns", 1, "number of originally connected peers (default 1)")
|
||||
loglevel = flag.Int("loglevel", 3, "verbosity of logs")
|
||||
rawlog = flag.Bool("rawlog", false, "remove terminal formatting from logs")
|
||||
)
|
||||
|
||||
func init() {
|
||||
@ -247,25 +241,14 @@ func discoverySimulation(nodes, conns int, adapter adapters.NodeAdapter) (*simul
|
||||
action := func(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
wg := sync.WaitGroup{}
|
||||
for i := range ids {
|
||||
// collect the overlay addresses, to
|
||||
addrs = append(addrs, ids[i].Bytes())
|
||||
for j := 0; j < conns; j++ {
|
||||
var k int
|
||||
if j == 0 {
|
||||
k = (i + 1) % len(ids)
|
||||
} else {
|
||||
k = rand.Intn(len(ids))
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(i, k int) {
|
||||
defer wg.Done()
|
||||
net.Connect(ids[i], ids[k])
|
||||
}(i, k)
|
||||
}
|
||||
}
|
||||
wg.Wait()
|
||||
err := net.ConnectNodesChain(nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
log.Debug(fmt.Sprintf("nodes: %v", len(addrs)))
|
||||
// construct the peer pot, so that kademlia health can be checked
|
||||
ppmap := network.NewPeerPotMap(network.NewKadParams().NeighbourhoodSize, addrs)
|
||||
@ -309,40 +292,6 @@ func discoverySimulation(nodes, conns int, adapter adapters.NodeAdapter) (*simul
|
||||
if result.Error != nil {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
if *snapshotFile != "" {
|
||||
var err error
|
||||
var snap *simulations.Snapshot
|
||||
if len(*serviceOverride) > 0 {
|
||||
var addServices []string
|
||||
var removeServices []string
|
||||
for _, osvc := range strings.Split(*serviceOverride, ",") {
|
||||
if strings.Index(osvc, "+") == 0 {
|
||||
addServices = append(addServices, osvc[1:])
|
||||
} else if strings.Index(osvc, "-") == 0 {
|
||||
removeServices = append(removeServices, osvc[1:])
|
||||
} else {
|
||||
panic("stick to the rules, you know what they are")
|
||||
}
|
||||
}
|
||||
snap, err = net.SnapshotWithServices(addServices, removeServices)
|
||||
} else {
|
||||
snap, err = net.Snapshot()
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, errors.New("no shapshot dude")
|
||||
}
|
||||
jsonsnapshot, err := json.Marshal(snap)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("corrupt json snapshot: %v", err)
|
||||
}
|
||||
log.Info("writing snapshot", "file", *snapshotFile)
|
||||
err = ioutil.WriteFile(*snapshotFile, jsonsnapshot, 0755)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
@ -457,23 +406,7 @@ func discoveryPersistenceSimulation(nodes, conns int, adapter adapters.NodeAdapt
|
||||
|
||||
return nil
|
||||
}
|
||||
//connects in a chain
|
||||
wg := sync.WaitGroup{}
|
||||
//connects in a ring
|
||||
for i := range ids {
|
||||
for j := 1; j <= conns; j++ {
|
||||
k := (i + j) % len(ids)
|
||||
if k == i {
|
||||
k = (k + 1) % len(ids)
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(i, k int) {
|
||||
defer wg.Done()
|
||||
net.Connect(ids[i], ids[k])
|
||||
}(i, k)
|
||||
}
|
||||
}
|
||||
wg.Wait()
|
||||
net.ConnectNodesChain(nil)
|
||||
log.Debug(fmt.Sprintf("nodes: %v", len(addrs)))
|
||||
// construct the peer pot, so that kademlia health can be checked
|
||||
check := func(ctx context.Context, id enode.ID) (bool, error) {
|
||||
|
Loading…
Reference in New Issue
Block a user