cmd/swarm/swarm-snapshot: swarm snapshot generator (#18453)

* cmd/swarm/swarm-snapshot: add binary to create network snapshots

* cmd/swarm/swarm-snapshot: refactor and extend tests

* p2p/simulations: remove unused triggerChecks func and fix linter

* internal/cmdtest: raise the timeout for killing TestCmd

* cmd/swarm/swarm-snapshot: add more comments and other minor adjustments

* cmd/swarm/swarm-snapshot: remove redundant check in createSnapshot

* cmd/swarm/swarm-snapshot: change comment wording

* p2p/simulations: revert Simulation.Run from master

https://github.com/ethersphere/go-ethereum/pull/1077/files#r247078904

* cmd/swarm/swarm-snapshot: address pr comments

* swarm/network/simulations/discovery: removed snapshot write to file

* cmd/swarm/swarm-snapshot, swarm/network/simulations: removed redundant connection event check, fixed lint error

cmd/swarm/swarm-snapshot/create.go

@@ -0,0 +1,157 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/node"
+	"github.com/ethereum/go-ethereum/p2p/simulations"
+	"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
+	"github.com/ethereum/go-ethereum/swarm/network"
+	"github.com/ethereum/go-ethereum/swarm/network/simulation"
+	cli "gopkg.in/urfave/cli.v1"
+)
+
+// create is used as the entry function for "create" app command.
+func create(ctx *cli.Context) error {
+	log.PrintOrigins(true)
+	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(ctx.Int("verbosity")), log.StreamHandler(os.Stdout, log.TerminalFormat(true))))
+
+	if len(ctx.Args()) < 1 {
+		return errors.New("argument should be the filename to verify or write-to")
+	}
+	filename, err := touchPath(ctx.Args()[0])
+	if err != nil {
+		return err
+	}
+	return createSnapshot(filename, ctx.Int("nodes"), strings.Split(ctx.String("services"), ","))
+}
+
+// createSnapshot creates a new snapshot on filesystem with provided filename,
+// number of nodes and service names.
+func createSnapshot(filename string, nodes int, services []string) (err error) {
+	log.Debug("create snapshot", "filename", filename, "nodes", nodes, "services", services)
+
+	sim := simulation.New(map[string]simulation.ServiceFunc{
+		"bzz": func(ctx *adapters.ServiceContext, b *sync.Map) (node.Service, func(), error) {
+			addr := network.NewAddr(ctx.Config.Node())
+			kad := network.NewKademlia(addr.Over(), network.NewKadParams())
+			hp := network.NewHiveParams()
+			hp.KeepAliveInterval = time.Duration(200) * time.Millisecond
+			hp.Discovery = true // discovery must be enabled when creating a snapshot
+
+			config := &network.BzzConfig{
+				OverlayAddr:  addr.Over(),
+				UnderlayAddr: addr.Under(),
+				HiveParams:   hp,
+			}
+			return network.NewBzz(config, kad, nil, nil, nil), nil, nil
+		},
+	})
+	defer sim.Close()
+
+	_, err = sim.AddNodes(nodes)
+	if err != nil {
+		return fmt.Errorf("add nodes: %v", err)
+	}
+
+	err = sim.Net.ConnectNodesRing(nil)
+	if err != nil {
+		return fmt.Errorf("connect nodes: %v", err)
+	}
+
+	ctx, cancelSimRun := context.WithTimeout(context.Background(), 2*time.Minute)
+	defer cancelSimRun()
+	if _, err := sim.WaitTillHealthy(ctx); err != nil {
+		return fmt.Errorf("wait for healthy kademlia: %v", err)
+	}
+
+	var snap *simulations.Snapshot
+	if len(services) > 0 {
+		// If service names are provided, include them in the snapshot.
+		// But, check if "bzz" service is not among them to remove it
+		// form the snapshot as it exists on snapshot creation.
+		var removeServices []string
+		var wantBzz bool
+		for _, s := range services {
+			if s == "bzz" {
+				wantBzz = true
+				break
+			}
+		}
+		if !wantBzz {
+			removeServices = []string{"bzz"}
+		}
+		snap, err = sim.Net.SnapshotWithServices(services, removeServices)
+	} else {
+		snap, err = sim.Net.Snapshot()
+	}
+	if err != nil {
+		return fmt.Errorf("create snapshot: %v", err)
+	}
+	jsonsnapshot, err := json.Marshal(snap)
+	if err != nil {
+		return fmt.Errorf("json encode snapshot: %v", err)
+	}
+	return ioutil.WriteFile(filename, jsonsnapshot, 0666)
+}
+
+// touchPath creates an empty file and all subdirectories
+// that are missing.
+func touchPath(filename string) (string, error) {
+	if path.IsAbs(filename) {
+		if _, err := os.Stat(filename); err == nil {
+			// path exists, overwrite
+			return filename, nil
+		}
+	}
+
+	d, f := path.Split(filename)
+	dir, err := filepath.Abs(filepath.Dir(os.Args[0]))
+	if err != nil {
+		return "", err
+	}
+
+	_, err = os.Stat(path.Join(dir, filename))
+	if err == nil {
+		// path exists, overwrite
+		return filename, nil
+	}
+
+	dirPath := path.Join(dir, d)
+	filePath := path.Join(dirPath, f)
+	if d != "" {
+		err = os.MkdirAll(dirPath, os.ModeDir)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	return filePath, nil
+}

cmd/swarm/swarm-snapshot/create_test.go

@@ -0,0 +1,138 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"sort"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/p2p/simulations"
+)
+
+// TestSnapshotCreate is a high level e2e test that tests for snapshot generation.
+// It runs a few "create" commands with different flag values and loads generated
+// snapshot files to validate their content.
+func TestSnapshotCreate(t *testing.T) {
+	for _, v := range []struct {
+		name     string
+		nodes    int
+		services string
+	}{
+		{
+			name: "defaults",
+		},
+		{
+			name:  "more nodes",
+			nodes: defaultNodes + 5,
+		},
+		{
+			name:     "services",
+			services: "stream,pss,zorglub",
+		},
+		{
+			name:     "services with bzz",
+			services: "bzz,pss",
+		},
+	} {
+		t.Run(v.name, func(t *testing.T) {
+			t.Parallel()
+
+			file, err := ioutil.TempFile("", "swarm-snapshot")
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer os.Remove(file.Name())
+
+			if err = file.Close(); err != nil {
+				t.Error(err)
+			}
+
+			args := []string{"create"}
+			if v.nodes > 0 {
+				args = append(args, "--nodes", strconv.Itoa(v.nodes))
+			}
+			if v.services != "" {
+				args = append(args, "--services", v.services)
+			}
+			testCmd := runSnapshot(t, append(args, file.Name())...)
+
+			testCmd.ExpectExit()
+			if code := testCmd.ExitStatus(); code != 0 {
+				t.Fatalf("command exit code %v, expected 0", code)
+			}
+
+			f, err := os.Open(file.Name())
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer func() {
+				err := f.Close()
+				if err != nil {
+					t.Error("closing snapshot file", "err", err)
+				}
+			}()
+
+			b, err := ioutil.ReadAll(f)
+			if err != nil {
+				t.Fatal(err)
+			}
+			var snap simulations.Snapshot
+			err = json.Unmarshal(b, &snap)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			wantNodes := v.nodes
+			if wantNodes == 0 {
+				wantNodes = defaultNodes
+			}
+			gotNodes := len(snap.Nodes)
+			if gotNodes != wantNodes {
+				t.Errorf("got %v nodes, want %v", gotNodes, wantNodes)
+			}
+
+			if len(snap.Conns) == 0 {
+				t.Error("no connections in a snapshot")
+			}
+
+			var wantServices []string
+			if v.services != "" {
+				wantServices = strings.Split(v.services, ",")
+			} else {
+				wantServices = []string{"bzz"}
+			}
+			// sort service names so they can be comparable
+			// as strings to every node sorted services
+			sort.Strings(wantServices)
+
+			for i, n := range snap.Nodes {
+				gotServices := n.Node.Config.Services
+				sort.Strings(gotServices)
+				if fmt.Sprint(gotServices) != fmt.Sprint(wantServices) {
+					t.Errorf("got services %v for node %v, want %v", gotServices, i, wantServices)
+				}
+			}
+
+		})
+	}
+}

cmd/swarm/swarm-snapshot/main.go

@@ -0,0 +1,82 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"os"
+
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/log"
+	cli "gopkg.in/urfave/cli.v1"
+)
+
+var gitCommit string // Git SHA1 commit hash of the release (set via linker flags)
+
+// default value for "create" command --nodes flag
+const defaultNodes = 10
+
+func main() {
+	err := newApp().Run(os.Args)
+	if err != nil {
+		log.Error(err.Error())
+		os.Exit(1)
+	}
+}
+
+// newApp construct a new instance of Swarm Snapshot Utility.
+// Method Run is called on it in the main function and in tests.
+func newApp() (app *cli.App) {
+	app = utils.NewApp(gitCommit, "Swarm Snapshot Utility")
+
+	app.Name = "swarm-snapshot"
+	app.Usage = ""
+
+	// app flags (for all commands)
+	app.Flags = []cli.Flag{
+		cli.IntFlag{
+			Name:  "verbosity",
+			Value: 1,
+			Usage: "verbosity level",
+		},
+	}
+
+	app.Commands = []cli.Command{
+		{
+			Name:    "create",
+			Aliases: []string{"c"},
+			Usage:   "create a swarm snapshot",
+			Action:  create,
+			// Flags only for "create" command.
+			// Allow app flags to be specified after the
+			// command argument.
+			Flags: append(app.Flags,
+				cli.IntFlag{
+					Name:  "nodes",
+					Value: defaultNodes,
+					Usage: "number of nodes",
+				},
+				cli.StringFlag{
+					Name:  "services",
+					Value: "bzz",
+					Usage: "comma separated list of services to boot the nodes with",
+				},
+			),
+		},
+	}
+
+	return app
+}

cmd/swarm/swarm-snapshot/run_test.go

@@ -0,0 +1,49 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/docker/docker/pkg/reexec"
+	"github.com/ethereum/go-ethereum/internal/cmdtest"
+)
+
+func init() {
+	reexec.Register("swarm-snapshot", func() {
+		if err := newApp().Run(os.Args); err != nil {
+			fmt.Fprintln(os.Stderr, err)
+			os.Exit(1)
+		}
+		os.Exit(0)
+	})
+}
+
+func runSnapshot(t *testing.T, args ...string) *cmdtest.TestCmd {
+	tt := cmdtest.NewTestCmd(t, nil)
+	tt.Run("swarm-snapshot", args...)
+	return tt
+}
+
+func TestMain(m *testing.M) {
+	if reexec.Init() {
+		return
+	}
+	os.Exit(m.Run())
+}

swarm/network/kademlia.go

@@ -640,6 +640,8 @@ func (k *Kademlia) saturation() int {
 	})
 	// TODO evaluate whether this check cannot just as well be done within the eachbin
 	depth := depthForPot(k.conns, k.NeighbourhoodSize, k.base)
+
+	// if in the iterator above we iterated deeper than the neighbourhood depth - return depth
 	if depth < prev {
 		return depth
 	}

swarm/network/simulations/discovery/discovery_test.go

@@ -18,16 +18,12 @@ package discovery
 
 import (
 	"context"
-	"encoding/json"
-	"errors"
 	"flag"
 	"fmt"
 	"io/ioutil"
-	"math/rand"
 	"os"
 	"path"
 	"strings"
-	"sync"
 	"testing"
 	"time"
 
@@ -86,12 +82,10 @@ func getDbStore(nodeID string) (*state.DBStore, error) {
 }
 
 var (
-	nodeCount       = flag.Int("nodes", 10, "number of nodes to create (default 10)")
+	nodeCount = flag.Int("nodes", 10, "number of nodes to create (default 10)")
-	initCount       = flag.Int("conns", 1, "number of originally connected peers	 (default 1)")
+	initCount = flag.Int("conns", 1, "number of originally connected peers	 (default 1)")
-	snapshotFile    = flag.String("snapshot", "", "path to create snapshot file in")
+	loglevel  = flag.Int("loglevel", 3, "verbosity of logs")
-	loglevel        = flag.Int("loglevel", 3, "verbosity of logs")
+	rawlog    = flag.Bool("rawlog", false, "remove terminal formatting from logs")
-	rawlog          = flag.Bool("rawlog", false, "remove terminal formatting from logs")
-	serviceOverride = flag.String("services", "", "remove or add services to the node snapshot; prefix with \"+\" to add, \"-\" to remove; example: +pss,-discovery")
 )
 
 func init() {
@@ -247,25 +241,14 @@ func discoverySimulation(nodes, conns int, adapter adapters.NodeAdapter) (*simul
 	action := func(ctx context.Context) error {
 		return nil
 	}
-	wg := sync.WaitGroup{}
 	for i := range ids {
 		// collect the overlay addresses, to
 		addrs = append(addrs, ids[i].Bytes())
-		for j := 0; j < conns; j++ {
-			var k int
-			if j == 0 {
-				k = (i + 1) % len(ids)
-			} else {
-				k = rand.Intn(len(ids))
-			}
-			wg.Add(1)
-			go func(i, k int) {
-				defer wg.Done()
-				net.Connect(ids[i], ids[k])
-			}(i, k)
-		}
 	}
-	wg.Wait()
+	err := net.ConnectNodesChain(nil)
+	if err != nil {
+		return nil, err
+	}
 	log.Debug(fmt.Sprintf("nodes: %v", len(addrs)))
 	// construct the peer pot, so that kademlia health can be checked
 	ppmap := network.NewPeerPotMap(network.NewKadParams().NeighbourhoodSize, addrs)
@@ -309,40 +292,6 @@ func discoverySimulation(nodes, conns int, adapter adapters.NodeAdapter) (*simul
 	if result.Error != nil {
 		return result, nil
 	}
-
-	if *snapshotFile != "" {
-		var err error
-		var snap *simulations.Snapshot
-		if len(*serviceOverride) > 0 {
-			var addServices []string
-			var removeServices []string
-			for _, osvc := range strings.Split(*serviceOverride, ",") {
-				if strings.Index(osvc, "+") == 0 {
-					addServices = append(addServices, osvc[1:])
-				} else if strings.Index(osvc, "-") == 0 {
-					removeServices = append(removeServices, osvc[1:])
-				} else {
-					panic("stick to the rules, you know what they are")
-				}
-			}
-			snap, err = net.SnapshotWithServices(addServices, removeServices)
-		} else {
-			snap, err = net.Snapshot()
-		}
-
-		if err != nil {
-			return nil, errors.New("no shapshot dude")
-		}
-		jsonsnapshot, err := json.Marshal(snap)
-		if err != nil {
-			return nil, fmt.Errorf("corrupt json snapshot: %v", err)
-		}
-		log.Info("writing snapshot", "file", *snapshotFile)
-		err = ioutil.WriteFile(*snapshotFile, jsonsnapshot, 0755)
-		if err != nil {
-			return nil, err
-		}
-	}
 	return result, nil
 }
 
@@ -457,23 +406,7 @@ func discoveryPersistenceSimulation(nodes, conns int, adapter adapters.NodeAdapt
 
 		return nil
 	}
-	//connects in a chain
+	net.ConnectNodesChain(nil)
-	wg := sync.WaitGroup{}
-	//connects in a ring
-	for i := range ids {
-		for j := 1; j <= conns; j++ {
-			k := (i + j) % len(ids)
-			if k == i {
-				k = (k + 1) % len(ids)
-			}
-			wg.Add(1)
-			go func(i, k int) {
-				defer wg.Done()
-				net.Connect(ids[i], ids[k])
-			}(i, k)
-		}
-	}
-	wg.Wait()
 	log.Debug(fmt.Sprintf("nodes: %v", len(addrs)))
 	// construct the peer pot, so that kademlia health can be checked
 	check := func(ctx context.Context, id enode.ID) (bool, error) {