From 21415a98472f3e7f1f5349b1ca3976096aaa2614 Mon Sep 17 00:00:00 2001 From: Mike Greenberg Date: Thu, 28 May 2020 16:48:06 -0400 Subject: [PATCH 01/31] add chainwatch systemd svc config; make install-chainwatch-service --- Makefile | 8 ++++++++ scripts/chainwatch.service | 12 ++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 scripts/chainwatch.service diff --git a/Makefile b/Makefile index 92772ab19..fdf9e7e09 100644 --- a/Makefile +++ b/Makefile @@ -114,6 +114,7 @@ install-services: install clean-services: rm -f /usr/local/lib/systemd/system/lotus-daemon.service rm -f /usr/local/lib/systemd/system/lotus-miner.service + rm -f /usr/local/lib/systemd/system/chainwatch.service systemctl daemon-reload # TOOLS @@ -160,6 +161,13 @@ chainwatch: .PHONY: chainwatch BINS+=chainwatch +install-chainwatch-service: chainwatch + install -C ./chainwatch /usr/local/bin/chainwatch + install -C -m 0644 ./scripts/chainwatch.service /usr/local/lib/systemd/system/chainwatch.service + systemctl daemon-reload + @echo + @echo "chainwatch installed. Don't forget to 'systemctl enable chainwatch' for it to be enabled on startup." + bench: rm -f bench go build -o bench ./cmd/lotus-bench diff --git a/scripts/chainwatch.service b/scripts/chainwatch.service new file mode 100644 index 000000000..47af6b59e --- /dev/null +++ b/scripts/chainwatch.service @@ -0,0 +1,12 @@ +[Unit] +Description=Chainwatch +After=lotus-daemon.service +Requires=lotus-daemon.service + +[Service] +ExecStart=/usr/local/bin/chainwatch run +Environment=LOTUS_DB="postgres://postgres:password@localhost:5432/postgres?sslmode=disable" +Environment=LOTUS_PATH="/root/.lotus" + +[Install] +WantedBy=multiuser.target From c0cdcbb9e0b51b16a0db5832c36df9ba713d54df Mon Sep 17 00:00:00 2001 From: Mike Greenberg Date: Thu, 4 Jun 2020 08:09:01 -0400 Subject: [PATCH 02/31] Adjust service logging and startup settings --- Makefile | 3 ++- scripts/chainwatch.service | 5 ++++- scripts/lotus-daemon.service | 6 +++--- scripts/lotus-miner.service | 3 ++- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index fdf9e7e09..5db557d7e 100644 --- a/Makefile +++ b/Makefile @@ -105,11 +105,12 @@ install: install-services: install mkdir -p /usr/local/lib/systemd/system + mkdir -p /var/log/lotus install -C -m 0644 ./scripts/lotus-daemon.service /usr/local/lib/systemd/system/lotus-daemon.service install -C -m 0644 ./scripts/lotus-miner.service /usr/local/lib/systemd/system/lotus-miner.service systemctl daemon-reload @echo - @echo "lotus and lotus-miner services installed. Don't forget to 'systemctl enable lotus|lotus-miner' for it to be enabled on startup." + @echo "lotus-daemon and lotus-miner services installed. Don't forget to 'systemctl enable lotus-daemon|lotus-miner' for it to be enabled on startup." clean-services: rm -f /usr/local/lib/systemd/system/lotus-daemon.service diff --git a/scripts/chainwatch.service b/scripts/chainwatch.service index 47af6b59e..e958ec857 100644 --- a/scripts/chainwatch.service +++ b/scripts/chainwatch.service @@ -1,12 +1,15 @@ [Unit] Description=Chainwatch +PartOf=sentinel.service After=lotus-daemon.service Requires=lotus-daemon.service [Service] -ExecStart=/usr/local/bin/chainwatch run +Environment=GOLOG_FILE="/var/log/lotus/chainwatch.log" +Environment=GOLOG_LOG_FMT="json" Environment=LOTUS_DB="postgres://postgres:password@localhost:5432/postgres?sslmode=disable" Environment=LOTUS_PATH="/root/.lotus" +ExecStart=/usr/local/bin/chainwatch run [Install] WantedBy=multiuser.target diff --git a/scripts/lotus-daemon.service b/scripts/lotus-daemon.service index 46bc47bdb..99a94e217 100644 --- a/scripts/lotus-daemon.service +++ b/scripts/lotus-daemon.service @@ -1,15 +1,15 @@ [Unit] Description=Lotus Daemon After=network-online.target -Wants=network-online.target +Requires=network-online.target [Service] -Environment=GOLOG_FILE="/var/log/lotus-daemon" +Environment=GOLOG_FILE="/var/log/lotus/daemon.log" Environment=GOLOG_LOG_FMT="json" ExecStart=/usr/local/bin/lotus daemon #ExecStop=/bin/sh -a -c /root/capture_lotus_heap.sh Restart=always -RestartSec=30 +RestartSec=10 MemoryAccounting=true MemoryHigh=8G diff --git a/scripts/lotus-miner.service b/scripts/lotus-miner.service index 7b866e043..3a460450f 100644 --- a/scripts/lotus-miner.service +++ b/scripts/lotus-miner.service @@ -2,10 +2,11 @@ Description=Lotus Storage Miner After=network.target After=lotus-daemon.service +Requires=lotus-daemon.service [Service] ExecStart=/usr/local/bin/lotus-storage-miner run -Environment=GOLOG_FILE="/var/log/lotus-miner" +Environment=GOLOG_FILE="/var/log/lotus/miner.log" Environment=GOLOG_LOG_FMT="json" [Install] From ce30162907c8d774a6b8e711bfccfa2e26b02651 Mon Sep 17 00:00:00 2001 From: Mike Greenberg Date: Thu, 4 Jun 2020 08:10:08 -0400 Subject: [PATCH 03/31] Fix documentation link for new systemd page --- documentation/en/.library.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/en/.library.json b/documentation/en/.library.json index 018e6189b..293c871ea 100644 --- a/documentation/en/.library.json +++ b/documentation/en/.library.json @@ -58,8 +58,8 @@ }, { "title": "Use Lotus with systemd", - "slug": "en+install-system-services", - "github": "en/install-system-services.md", + "slug": "en+install-systemd-services", + "github": "en/install-systemd-services.md", "value": null }, { From 997a46a90f02db165a86dab0025ae71fe23afe6e Mon Sep 17 00:00:00 2001 From: laser Date: Thu, 18 Jun 2020 10:32:32 -0700 Subject: [PATCH 04/31] wombat --- cmd/lotus-storage-miner/market.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/lotus-storage-miner/market.go b/cmd/lotus-storage-miner/market.go index 4dac236e4..cb49854cd 100644 --- a/cmd/lotus-storage-miner/market.go +++ b/cmd/lotus-storage-miner/market.go @@ -140,6 +140,8 @@ var getAskCmd = &cli.Command{ Action: func(cctx *cli.Context) error { ctx := lcli.DaemonContext(cctx) + fmt.Println("wombat") + fnapi, closer, err := lcli.GetFullNodeAPI(cctx) if err != nil { return err From b0edf924b429a99e25312dd0ee133b55561190fc Mon Sep 17 00:00:00 2001 From: laser Date: Thu, 18 Jun 2020 13:15:18 -0700 Subject: [PATCH 05/31] add commands for manipulating storage deal CID blacklist --- api/api_storage.go | 2 + api/apistruct/struct.go | 10 +++ cmd/lotus-storage-miner/market.go | 126 ++++++++++++++++++++++++++++++ node/builder.go | 2 + node/config/def.go | 4 + node/impl/storminer.go | 12 ++- node/modules/dtypes/miner.go | 11 +++ node/modules/storageminer.go | 83 ++++++++++++++------ 8 files changed, 225 insertions(+), 25 deletions(-) diff --git a/api/api_storage.go b/api/api_storage.go index 90de01fb9..0eda830dc 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -56,6 +56,8 @@ type StorageMiner interface { DealsImportData(ctx context.Context, dealPropCid cid.Cid, file string) error DealsList(ctx context.Context) ([]storagemarket.StorageDeal, error) DealsSetAcceptingStorageDeals(context.Context, bool) error + DealsBlacklist(context.Context) ([]cid.Cid, error) + DealsSetBlacklist(context.Context, []cid.Cid) error StorageAddLocal(ctx context.Context, path string) error } diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 0d69174ab..43c0a423e 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -226,6 +226,8 @@ type StorageMinerStruct struct { DealsImportData func(ctx context.Context, dealPropCid cid.Cid, file string) error `perm:"write"` DealsList func(ctx context.Context) ([]storagemarket.StorageDeal, error) `perm:"read"` DealsSetAcceptingStorageDeals func(context.Context, bool) error `perm:"admin"` + DealsBlacklist func(context.Context) ([]cid.Cid, error) `perm:"admin"` + DealsSetBlacklist func(context.Context, []cid.Cid) error `perm:"read"` StorageAddLocal func(ctx context.Context, path string) error `perm:"admin"` } @@ -872,6 +874,14 @@ func (c *StorageMinerStruct) DealsSetAcceptingStorageDeals(ctx context.Context, return c.Internal.DealsSetAcceptingStorageDeals(ctx, b) } +func (c *StorageMinerStruct) DealsBlacklist(ctx context.Context) ([]cid.Cid, error) { + return c.Internal.DealsBlacklist(ctx) +} + +func (c *StorageMinerStruct) DealsSetBlacklist(ctx context.Context, cids []cid.Cid) error { + return c.Internal.DealsSetBlacklist(ctx, cids) +} + func (c *StorageMinerStruct) StorageAddLocal(ctx context.Context, path string) error { return c.Internal.StorageAddLocal(ctx, path) } diff --git a/cmd/lotus-storage-miner/market.go b/cmd/lotus-storage-miner/market.go index cb49854cd..f12bce62c 100644 --- a/cmd/lotus-storage-miner/market.go +++ b/cmd/lotus-storage-miner/market.go @@ -1,14 +1,18 @@ package main import ( + "bufio" "encoding/json" "fmt" "os" + "path/filepath" "text/tabwriter" "time" "github.com/docker/go-units" "github.com/ipfs/go-cid" + "github.com/ipfs/go-cidutil/cidenc" + "github.com/multiformats/go-multibase" "github.com/urfave/cli/v2" "golang.org/x/xerrors" @@ -20,6 +24,32 @@ import ( lcli "github.com/filecoin-project/lotus/cli" ) +var CidBaseFlag = cli.StringFlag{ + Name: "cid-base", + Hidden: true, + Value: "base32", + Usage: "Multibase encoding used for version 1 CIDs in output.", + DefaultText: "base32", +} + +// GetCidEncoder returns an encoder using the `cid-base` flag if provided, or +// the default (Base32) encoder if not. +func GetCidEncoder(cctx *cli.Context) (cidenc.Encoder, error) { + val := cctx.String("cid-base") + + e := cidenc.Encoder{Base: multibase.MustNewEncoder(multibase.Base32)} + + if val != "" { + var err error + e.Base, err = multibase.EncoderByName(val) + if err != nil { + return e, err + } + } + + return e, nil +} + var enableCmd = &cli.Command{ Name: "enable", Usage: "Configure the miner to consider storage deal proposals", @@ -199,6 +229,9 @@ var dealsCmd = &cli.Command{ disableCmd, setAskCmd, getAskCmd, + setBlacklistCmd, + getBlacklistCmd, + resetBlacklistCmd, }, } @@ -257,3 +290,96 @@ var dealsListCmd = &cli.Command{ return nil }, } + +var getBlacklistCmd = &cli.Command{ + Name: "get-blacklist", + Usage: "List the CIDs in the storage miner's blacklist", + Flags: []cli.Flag{ + &CidBaseFlag, + }, + Action: func(cctx *cli.Context) error { + api, closer, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return err + } + defer closer() + + blacklist, err := api.DealsBlacklist(lcli.DaemonContext(cctx)) + if err != nil { + return err + } + + encoder, err := GetCidEncoder(cctx) + if err != nil { + return err + } + + for idx := range blacklist { + fmt.Println(encoder.Encode(blacklist[idx])) + } + + return nil + }, +} + +var setBlacklistCmd = &cli.Command{ + Name: "set-blacklist", + Usage: "Set the storage miner's list of blacklisted CIDs", + ArgsUsage: "[ (optional, will read from stdin if omitted)]", + Flags: []cli.Flag{}, + Action: func(cctx *cli.Context) error { + api, closer, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return err + } + defer closer() + + scanner := bufio.NewScanner(os.Stdin) + if cctx.Args().Present() && cctx.Args().First() != "-" { + absPath, err := filepath.Abs(cctx.Args().First()) + if err != nil { + return err + } + + file, err := os.Open(absPath) + if err != nil { + log.Fatal(err) + } + defer file.Close() + + scanner = bufio.NewScanner(file) + } + + var blacklist []cid.Cid + for scanner.Scan() { + decoded, err := cid.Decode(scanner.Text()) + if err != nil { + return err + } + + blacklist = append(blacklist, decoded) + } + + err = scanner.Err() + if err != nil { + return err + } + + return api.DealsSetBlacklist(lcli.DaemonContext(cctx), blacklist) + }, +} + +var resetBlacklistCmd = &cli.Command{ + Name: "reset-blacklist", + Usage: "Remove all entries from the storage miner's blacklist", + Flags: []cli.Flag{}, + Action: func(cctx *cli.Context) error { + api, closer, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return err + } + defer closer() + + return api.DealsSetBlacklist(lcli.DaemonContext(cctx), []cid.Cid{}) + }, +} diff --git a/node/builder.go b/node/builder.go index 536d81901..52e089bbe 100644 --- a/node/builder.go +++ b/node/builder.go @@ -314,6 +314,8 @@ func Online() Option { Override(new(dtypes.AcceptingStorageDealsConfigFunc), modules.NewAcceptingStorageDealsConfigFunc), Override(new(dtypes.SetAcceptingStorageDealsConfigFunc), modules.NewSetAcceptingStorageDealsConfigFunc), + Override(new(dtypes.StorageDealCidBlacklistConfigFunc), modules.NewStorageDealCidBlacklistConfigFunc), + Override(new(dtypes.SetStorageDealCidBlacklistConfigFunc), modules.NewSetStorageDealCidBlacklistConfigFunc), ), ) } diff --git a/node/config/def.go b/node/config/def.go index 5c46f77a4..b92ac3b4d 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -4,6 +4,8 @@ import ( "encoding" "time" + "github.com/ipfs/go-cid" + sectorstorage "github.com/filecoin-project/sector-storage" ) @@ -33,6 +35,7 @@ type StorageMiner struct { type DealmakingConfig struct { AcceptingStorageDeals bool + Blacklist []cid.Cid } // API contains configs for API endpoint @@ -121,6 +124,7 @@ func DefaultStorageMiner() *StorageMiner { Dealmaking: DealmakingConfig{ AcceptingStorageDeals: true, + Blacklist: []cid.Cid{}, }, } cfg.Common.API.ListenAddress = "/ip4/127.0.0.1/tcp/2345/http" diff --git a/node/impl/storminer.go b/node/impl/storminer.go index ed94e173d..2863a9d28 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -43,7 +43,9 @@ type StorageMinerAPI struct { StorageMgr *sectorstorage.Manager `optional:"true"` *stores.Index - SetAcceptingStorageDealsConfigFunc dtypes.SetAcceptingStorageDealsConfigFunc + SetAcceptingStorageDealsConfigFunc dtypes.SetAcceptingStorageDealsConfigFunc + StorageDealCidBlacklistConfigFunc dtypes.StorageDealCidBlacklistConfigFunc + SetStorageDealCidBlacklistConfigFunc dtypes.SetStorageDealCidBlacklistConfigFunc } func (sm *StorageMinerAPI) ServeRemote(w http.ResponseWriter, r *http.Request) { @@ -232,6 +234,14 @@ func (sm *StorageMinerAPI) DealsImportData(ctx context.Context, deal cid.Cid, fn return sm.StorageProvider.ImportDataForDeal(ctx, deal, fi) } +func (sm *StorageMinerAPI) DealsBlacklist(ctx context.Context) ([]cid.Cid, error) { + return sm.StorageDealCidBlacklistConfigFunc() +} + +func (sm *StorageMinerAPI) DealsSetBlacklist(ctx context.Context, cids []cid.Cid) error { + return sm.SetStorageDealCidBlacklistConfigFunc(cids) +} + func (sm *StorageMinerAPI) StorageAddLocal(ctx context.Context, path string) error { if sm.StorageMgr == nil { return xerrors.Errorf("no storage manager") diff --git a/node/modules/dtypes/miner.go b/node/modules/dtypes/miner.go index 5c761d3e5..04ae07a53 100644 --- a/node/modules/dtypes/miner.go +++ b/node/modules/dtypes/miner.go @@ -1,6 +1,8 @@ package dtypes import ( + "github.com/ipfs/go-cid" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/specs-actors/actors/abi" ) @@ -15,3 +17,12 @@ type AcceptingStorageDealsConfigFunc func() (bool, error) // SetAcceptingStorageDealsFunc is a function which is used to disable or enable // storage deal acceptance. type SetAcceptingStorageDealsConfigFunc func(bool) error + +// StorageDealCidBlacklistConfigFunc is a function which reads from miner config +// to obtain a list of CIDs for which the storage miner will not accept storage +// proposals. +type StorageDealCidBlacklistConfigFunc func() ([]cid.Cid, error) + +// SetStorageDealCidBlacklistConfigFunc is a function which is used to set a +// list of CIDs for which the storage miner will reject deal proposals. +type SetStorageDealCidBlacklistConfigFunc func([]cid.Cid) error diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index fce9f9e1f..a6ef6304e 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -8,6 +8,7 @@ import ( "github.com/ipfs/go-bitswap" "github.com/ipfs/go-bitswap/network" "github.com/ipfs/go-blockservice" + "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" graphsync "github.com/ipfs/go-graphsync/impl" @@ -380,35 +381,69 @@ func StorageAuth(ctx helpers.MetricsCtx, ca lapi.Common) (sectorstorage.StorageA } func NewAcceptingStorageDealsConfigFunc(r repo.LockedRepo) (dtypes.AcceptingStorageDealsConfigFunc, error) { - return func() (bool, error) { - raw, err := r.Config() - if err != nil { - return false, err - } - - cfg, ok := raw.(*config.StorageMiner) - if !ok { - return false, xerrors.New("expected address of config.StorageMiner") - } - - return cfg.Dealmaking.AcceptingStorageDeals, nil + return func() (out bool, err error) { + err = readCfg(r, func(cfg *config.StorageMiner) { + out = cfg.Dealmaking.AcceptingStorageDeals + }) + return }, nil } func NewSetAcceptingStorageDealsConfigFunc(r repo.LockedRepo) (dtypes.SetAcceptingStorageDealsConfigFunc, error) { - return func(b bool) error { - var typeErr error - - setConfigErr := r.SetConfig(func(raw interface{}) { - cfg, ok := raw.(*config.StorageMiner) - if !ok { - typeErr = errors.New("expected storage miner config") - return - } - + return func(b bool) (err error) { + err = mutateCfg(r, func(cfg *config.StorageMiner) { cfg.Dealmaking.AcceptingStorageDeals = b }) - - return multierr.Combine(typeErr, setConfigErr) + return }, nil } + +func NewStorageDealCidBlacklistConfigFunc(r repo.LockedRepo) (dtypes.StorageDealCidBlacklistConfigFunc, error) { + return func() (out []cid.Cid, err error) { + err = readCfg(r, func(cfg *config.StorageMiner) { + out = cfg.Dealmaking.Blacklist + }) + return + }, nil +} + +func NewSetStorageDealCidBlacklistConfigFunc(r repo.LockedRepo) (dtypes.SetStorageDealCidBlacklistConfigFunc, error) { + return func(blacklist []cid.Cid) (err error) { + err = mutateCfg(r, func(cfg *config.StorageMiner) { + cfg.Dealmaking.Blacklist = blacklist + }) + return + }, nil +} + +func readCfg(r repo.LockedRepo, accessor func(*config.StorageMiner)) error { + raw, err := r.Config() + if err != nil { + return err + } + + cfg, ok := raw.(*config.StorageMiner) + if !ok { + return xerrors.New("expected address of config.StorageMiner") + } + + accessor(cfg) + + return nil +} + +func mutateCfg(r repo.LockedRepo, mutator func(*config.StorageMiner)) error { + var typeErr error + + setConfigErr := r.SetConfig(func(raw interface{}) { + cfg, ok := raw.(*config.StorageMiner) + if !ok { + typeErr = errors.New("expected storage miner config") + return + } + + mutator(cfg) + }) + + return multierr.Combine(typeErr, setConfigErr) +} From aa18d1985e6835a0b4ea7d3b08913064ea82e9ee Mon Sep 17 00:00:00 2001 From: laser Date: Thu, 18 Jun 2020 13:32:20 -0700 Subject: [PATCH 06/31] reject storage deal proposals containing blacklisted piece CIDs --- node/modules/storageminer.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index a6ef6304e..49893c494 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -3,6 +3,7 @@ package modules import ( "context" "errors" + "fmt" "net/http" "github.com/ipfs/go-bitswap" @@ -308,7 +309,7 @@ func NewStorageAsk(ctx helpers.MetricsCtx, fapi lapi.FullNode, ds dtypes.Metadat return storedAsk, nil } -func StorageProvider(minerAddress dtypes.MinerAddress, ffiConfig *ffiwrapper.Config, storedAsk *storedask.StoredAsk, h host.Host, ds dtypes.MetadataDS, ibs dtypes.StagingBlockstore, r repo.LockedRepo, pieceStore dtypes.ProviderPieceStore, dataTransfer dtypes.ProviderDataTransfer, spn storagemarket.StorageProviderNode, isAcceptingFunc dtypes.AcceptingStorageDealsConfigFunc) (storagemarket.StorageProvider, error) { +func StorageProvider(minerAddress dtypes.MinerAddress, ffiConfig *ffiwrapper.Config, storedAsk *storedask.StoredAsk, h host.Host, ds dtypes.MetadataDS, ibs dtypes.StagingBlockstore, r repo.LockedRepo, pieceStore dtypes.ProviderPieceStore, dataTransfer dtypes.ProviderDataTransfer, spn storagemarket.StorageProviderNode, isAcceptingFunc dtypes.AcceptingStorageDealsConfigFunc, blacklistFunc dtypes.StorageDealCidBlacklistConfigFunc) (storagemarket.StorageProvider, error) { net := smnet.NewFromLibp2pHost(h) store, err := piecefilestore.NewLocalFileStore(piecefilestore.OsPath(r.Path())) if err != nil { @@ -326,6 +327,18 @@ func StorageProvider(minerAddress dtypes.MinerAddress, ffiConfig *ffiwrapper.Con return false, "miner is not accepting storage deals", nil } + blacklist, err := blacklistFunc() + if err != nil { + return false, "miner error", err + } + + for idx := range blacklist { + if deal.Proposal.PieceCID.Equals(blacklist[idx]) { + log.Warnf("piece CID in proposal %s is blacklisted; rejecting storage deal proposal from client: %s", deal.Proposal.PieceCID, deal.Client.String()) + return false, fmt.Sprintf("miner has blacklisted piece CID %s", deal.Proposal.PieceCID), nil + } + } + return true, "", nil }) From 0c8d6489980d2ae2d3630ab33e371c47739eb9a2 Mon Sep 17 00:00:00 2001 From: laser Date: Thu, 18 Jun 2020 13:36:48 -0700 Subject: [PATCH 07/31] specify which CID is being blacklisted (it's the piece) --- api/api_storage.go | 4 ++-- api/apistruct/struct.go | 12 ++++++------ cmd/lotus-storage-miner/market.go | 14 +++++++------- node/builder.go | 4 ++-- node/config/def.go | 4 ++-- node/impl/storminer.go | 14 +++++++------- node/modules/dtypes/miner.go | 4 ++-- node/modules/storageminer.go | 10 +++++----- 8 files changed, 33 insertions(+), 33 deletions(-) diff --git a/api/api_storage.go b/api/api_storage.go index 0eda830dc..afc25f1c5 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -56,8 +56,8 @@ type StorageMiner interface { DealsImportData(ctx context.Context, dealPropCid cid.Cid, file string) error DealsList(ctx context.Context) ([]storagemarket.StorageDeal, error) DealsSetAcceptingStorageDeals(context.Context, bool) error - DealsBlacklist(context.Context) ([]cid.Cid, error) - DealsSetBlacklist(context.Context, []cid.Cid) error + DealsPieceCidBlacklist(context.Context) ([]cid.Cid, error) + DealsSetPieceCidBlacklist(context.Context, []cid.Cid) error StorageAddLocal(ctx context.Context, path string) error } diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 43c0a423e..3d2375c87 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -226,8 +226,8 @@ type StorageMinerStruct struct { DealsImportData func(ctx context.Context, dealPropCid cid.Cid, file string) error `perm:"write"` DealsList func(ctx context.Context) ([]storagemarket.StorageDeal, error) `perm:"read"` DealsSetAcceptingStorageDeals func(context.Context, bool) error `perm:"admin"` - DealsBlacklist func(context.Context) ([]cid.Cid, error) `perm:"admin"` - DealsSetBlacklist func(context.Context, []cid.Cid) error `perm:"read"` + DealsPieceCidBlacklist func(context.Context) ([]cid.Cid, error) `perm:"admin"` + DealsSetPieceCidBlacklist func(context.Context, []cid.Cid) error `perm:"read"` StorageAddLocal func(ctx context.Context, path string) error `perm:"admin"` } @@ -874,12 +874,12 @@ func (c *StorageMinerStruct) DealsSetAcceptingStorageDeals(ctx context.Context, return c.Internal.DealsSetAcceptingStorageDeals(ctx, b) } -func (c *StorageMinerStruct) DealsBlacklist(ctx context.Context) ([]cid.Cid, error) { - return c.Internal.DealsBlacklist(ctx) +func (c *StorageMinerStruct) DealsPieceCidBlacklist(ctx context.Context) ([]cid.Cid, error) { + return c.Internal.DealsPieceCidBlacklist(ctx) } -func (c *StorageMinerStruct) DealsSetBlacklist(ctx context.Context, cids []cid.Cid) error { - return c.Internal.DealsSetBlacklist(ctx, cids) +func (c *StorageMinerStruct) DealsSetPieceCidBlacklist(ctx context.Context, cids []cid.Cid) error { + return c.Internal.DealsSetPieceCidBlacklist(ctx, cids) } func (c *StorageMinerStruct) StorageAddLocal(ctx context.Context, path string) error { diff --git a/cmd/lotus-storage-miner/market.go b/cmd/lotus-storage-miner/market.go index f12bce62c..1f7b84371 100644 --- a/cmd/lotus-storage-miner/market.go +++ b/cmd/lotus-storage-miner/market.go @@ -293,7 +293,7 @@ var dealsListCmd = &cli.Command{ var getBlacklistCmd = &cli.Command{ Name: "get-blacklist", - Usage: "List the CIDs in the storage miner's blacklist", + Usage: "List the contents of the storage miner's piece CID blacklist", Flags: []cli.Flag{ &CidBaseFlag, }, @@ -304,7 +304,7 @@ var getBlacklistCmd = &cli.Command{ } defer closer() - blacklist, err := api.DealsBlacklist(lcli.DaemonContext(cctx)) + blacklist, err := api.DealsPieceCidBlacklist(lcli.DaemonContext(cctx)) if err != nil { return err } @@ -324,8 +324,8 @@ var getBlacklistCmd = &cli.Command{ var setBlacklistCmd = &cli.Command{ Name: "set-blacklist", - Usage: "Set the storage miner's list of blacklisted CIDs", - ArgsUsage: "[ (optional, will read from stdin if omitted)]", + Usage: "Set the storage miner's list of blacklisted piece CIDs", + ArgsUsage: "[ (optional, will read from stdin if omitted)]", Flags: []cli.Flag{}, Action: func(cctx *cli.Context) error { api, closer, err := lcli.GetStorageMinerAPI(cctx) @@ -365,13 +365,13 @@ var setBlacklistCmd = &cli.Command{ return err } - return api.DealsSetBlacklist(lcli.DaemonContext(cctx), blacklist) + return api.DealsSetPieceCidBlacklist(lcli.DaemonContext(cctx), blacklist) }, } var resetBlacklistCmd = &cli.Command{ Name: "reset-blacklist", - Usage: "Remove all entries from the storage miner's blacklist", + Usage: "Remove all entries from the storage miner's piece CID blacklist", Flags: []cli.Flag{}, Action: func(cctx *cli.Context) error { api, closer, err := lcli.GetStorageMinerAPI(cctx) @@ -380,6 +380,6 @@ var resetBlacklistCmd = &cli.Command{ } defer closer() - return api.DealsSetBlacklist(lcli.DaemonContext(cctx), []cid.Cid{}) + return api.DealsSetPieceCidBlacklist(lcli.DaemonContext(cctx), []cid.Cid{}) }, } diff --git a/node/builder.go b/node/builder.go index 52e089bbe..bf32089e0 100644 --- a/node/builder.go +++ b/node/builder.go @@ -314,8 +314,8 @@ func Online() Option { Override(new(dtypes.AcceptingStorageDealsConfigFunc), modules.NewAcceptingStorageDealsConfigFunc), Override(new(dtypes.SetAcceptingStorageDealsConfigFunc), modules.NewSetAcceptingStorageDealsConfigFunc), - Override(new(dtypes.StorageDealCidBlacklistConfigFunc), modules.NewStorageDealCidBlacklistConfigFunc), - Override(new(dtypes.SetStorageDealCidBlacklistConfigFunc), modules.NewSetStorageDealCidBlacklistConfigFunc), + Override(new(dtypes.StorageDealPieceCidBlacklistConfigFunc), modules.NewStorageDealPieceCidBlacklistConfigFunc), + Override(new(dtypes.SetStorageDealPieceCidBlacklistConfigFunc), modules.NewSetStorageDealPieceCidBlacklistConfigFunc), ), ) } diff --git a/node/config/def.go b/node/config/def.go index b92ac3b4d..f9928ed3f 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -35,7 +35,7 @@ type StorageMiner struct { type DealmakingConfig struct { AcceptingStorageDeals bool - Blacklist []cid.Cid + PieceCidBlacklist []cid.Cid } // API contains configs for API endpoint @@ -124,7 +124,7 @@ func DefaultStorageMiner() *StorageMiner { Dealmaking: DealmakingConfig{ AcceptingStorageDeals: true, - Blacklist: []cid.Cid{}, + PieceCidBlacklist: []cid.Cid{}, }, } cfg.Common.API.ListenAddress = "/ip4/127.0.0.1/tcp/2345/http" diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 2863a9d28..6eba12501 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -43,9 +43,9 @@ type StorageMinerAPI struct { StorageMgr *sectorstorage.Manager `optional:"true"` *stores.Index - SetAcceptingStorageDealsConfigFunc dtypes.SetAcceptingStorageDealsConfigFunc - StorageDealCidBlacklistConfigFunc dtypes.StorageDealCidBlacklistConfigFunc - SetStorageDealCidBlacklistConfigFunc dtypes.SetStorageDealCidBlacklistConfigFunc + SetAcceptingStorageDealsConfigFunc dtypes.SetAcceptingStorageDealsConfigFunc + StorageDealPieceCidBlacklistConfigFunc dtypes.StorageDealPieceCidBlacklistConfigFunc + SetStorageDealPieceCidBlacklistConfigFunc dtypes.SetStorageDealPieceCidBlacklistConfigFunc } func (sm *StorageMinerAPI) ServeRemote(w http.ResponseWriter, r *http.Request) { @@ -234,12 +234,12 @@ func (sm *StorageMinerAPI) DealsImportData(ctx context.Context, deal cid.Cid, fn return sm.StorageProvider.ImportDataForDeal(ctx, deal, fi) } -func (sm *StorageMinerAPI) DealsBlacklist(ctx context.Context) ([]cid.Cid, error) { - return sm.StorageDealCidBlacklistConfigFunc() +func (sm *StorageMinerAPI) DealsPieceCidBlacklist(ctx context.Context) ([]cid.Cid, error) { + return sm.StorageDealPieceCidBlacklistConfigFunc() } -func (sm *StorageMinerAPI) DealsSetBlacklist(ctx context.Context, cids []cid.Cid) error { - return sm.SetStorageDealCidBlacklistConfigFunc(cids) +func (sm *StorageMinerAPI) DealsSetPieceCidBlacklist(ctx context.Context, cids []cid.Cid) error { + return sm.SetStorageDealPieceCidBlacklistConfigFunc(cids) } func (sm *StorageMinerAPI) StorageAddLocal(ctx context.Context, path string) error { diff --git a/node/modules/dtypes/miner.go b/node/modules/dtypes/miner.go index 04ae07a53..a9d278ca4 100644 --- a/node/modules/dtypes/miner.go +++ b/node/modules/dtypes/miner.go @@ -21,8 +21,8 @@ type SetAcceptingStorageDealsConfigFunc func(bool) error // StorageDealCidBlacklistConfigFunc is a function which reads from miner config // to obtain a list of CIDs for which the storage miner will not accept storage // proposals. -type StorageDealCidBlacklistConfigFunc func() ([]cid.Cid, error) +type StorageDealPieceCidBlacklistConfigFunc func() ([]cid.Cid, error) // SetStorageDealCidBlacklistConfigFunc is a function which is used to set a // list of CIDs for which the storage miner will reject deal proposals. -type SetStorageDealCidBlacklistConfigFunc func([]cid.Cid) error +type SetStorageDealPieceCidBlacklistConfigFunc func([]cid.Cid) error diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 49893c494..73e062c9c 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -309,7 +309,7 @@ func NewStorageAsk(ctx helpers.MetricsCtx, fapi lapi.FullNode, ds dtypes.Metadat return storedAsk, nil } -func StorageProvider(minerAddress dtypes.MinerAddress, ffiConfig *ffiwrapper.Config, storedAsk *storedask.StoredAsk, h host.Host, ds dtypes.MetadataDS, ibs dtypes.StagingBlockstore, r repo.LockedRepo, pieceStore dtypes.ProviderPieceStore, dataTransfer dtypes.ProviderDataTransfer, spn storagemarket.StorageProviderNode, isAcceptingFunc dtypes.AcceptingStorageDealsConfigFunc, blacklistFunc dtypes.StorageDealCidBlacklistConfigFunc) (storagemarket.StorageProvider, error) { +func StorageProvider(minerAddress dtypes.MinerAddress, ffiConfig *ffiwrapper.Config, storedAsk *storedask.StoredAsk, h host.Host, ds dtypes.MetadataDS, ibs dtypes.StagingBlockstore, r repo.LockedRepo, pieceStore dtypes.ProviderPieceStore, dataTransfer dtypes.ProviderDataTransfer, spn storagemarket.StorageProviderNode, isAcceptingFunc dtypes.AcceptingStorageDealsConfigFunc, blacklistFunc dtypes.StorageDealPieceCidBlacklistConfigFunc) (storagemarket.StorageProvider, error) { net := smnet.NewFromLibp2pHost(h) store, err := piecefilestore.NewLocalFileStore(piecefilestore.OsPath(r.Path())) if err != nil { @@ -411,19 +411,19 @@ func NewSetAcceptingStorageDealsConfigFunc(r repo.LockedRepo) (dtypes.SetAccepti }, nil } -func NewStorageDealCidBlacklistConfigFunc(r repo.LockedRepo) (dtypes.StorageDealCidBlacklistConfigFunc, error) { +func NewStorageDealPieceCidBlacklistConfigFunc(r repo.LockedRepo) (dtypes.StorageDealPieceCidBlacklistConfigFunc, error) { return func() (out []cid.Cid, err error) { err = readCfg(r, func(cfg *config.StorageMiner) { - out = cfg.Dealmaking.Blacklist + out = cfg.Dealmaking.PieceCidBlacklist }) return }, nil } -func NewSetStorageDealCidBlacklistConfigFunc(r repo.LockedRepo) (dtypes.SetStorageDealCidBlacklistConfigFunc, error) { +func NewSetStorageDealPieceCidBlacklistConfigFunc(r repo.LockedRepo) (dtypes.SetStorageDealPieceCidBlacklistConfigFunc, error) { return func(blacklist []cid.Cid) (err error) { err = mutateCfg(r, func(cfg *config.StorageMiner) { - cfg.Dealmaking.Blacklist = blacklist + cfg.Dealmaking.PieceCidBlacklist = blacklist }) return }, nil From 99060fbb64f6f14e98e3545b71f344f3e740d133 Mon Sep 17 00:00:00 2001 From: laser Date: Thu, 18 Jun 2020 14:05:59 -0700 Subject: [PATCH 08/31] eliminate errant debug line --- cmd/lotus-storage-miner/market.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/cmd/lotus-storage-miner/market.go b/cmd/lotus-storage-miner/market.go index 1f7b84371..e27bf6081 100644 --- a/cmd/lotus-storage-miner/market.go +++ b/cmd/lotus-storage-miner/market.go @@ -170,8 +170,6 @@ var getAskCmd = &cli.Command{ Action: func(cctx *cli.Context) error { ctx := lcli.DaemonContext(cctx) - fmt.Println("wombat") - fnapi, closer, err := lcli.GetFullNodeAPI(cctx) if err != nil { return err From de7d6c255c00c180860279a6252cb8f4a1fb245e Mon Sep 17 00:00:00 2001 From: laser Date: Thu, 18 Jun 2020 15:42:24 -0700 Subject: [PATCH 09/31] blacklist -> blocklist --- api/api_storage.go | 4 ++-- api/apistruct/struct.go | 12 +++++----- cmd/lotus-storage-miner/market.go | 38 +++++++++++++++---------------- node/builder.go | 4 ++-- node/config/def.go | 4 ++-- node/impl/storminer.go | 12 +++++----- node/modules/dtypes/miner.go | 8 +++---- node/modules/storageminer.go | 22 +++++++++--------- 8 files changed, 52 insertions(+), 52 deletions(-) diff --git a/api/api_storage.go b/api/api_storage.go index afc25f1c5..29ae5ea2e 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -56,8 +56,8 @@ type StorageMiner interface { DealsImportData(ctx context.Context, dealPropCid cid.Cid, file string) error DealsList(ctx context.Context) ([]storagemarket.StorageDeal, error) DealsSetAcceptingStorageDeals(context.Context, bool) error - DealsPieceCidBlacklist(context.Context) ([]cid.Cid, error) - DealsSetPieceCidBlacklist(context.Context, []cid.Cid) error + DealsPieceCidBlocklist(context.Context) ([]cid.Cid, error) + DealsSetPieceCidBlocklist(context.Context, []cid.Cid) error StorageAddLocal(ctx context.Context, path string) error } diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 3d2375c87..fb11cbda2 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -226,8 +226,8 @@ type StorageMinerStruct struct { DealsImportData func(ctx context.Context, dealPropCid cid.Cid, file string) error `perm:"write"` DealsList func(ctx context.Context) ([]storagemarket.StorageDeal, error) `perm:"read"` DealsSetAcceptingStorageDeals func(context.Context, bool) error `perm:"admin"` - DealsPieceCidBlacklist func(context.Context) ([]cid.Cid, error) `perm:"admin"` - DealsSetPieceCidBlacklist func(context.Context, []cid.Cid) error `perm:"read"` + DealsPieceCidBlocklist func(context.Context) ([]cid.Cid, error) `perm:"admin"` + DealsSetPieceCidBlocklist func(context.Context, []cid.Cid) error `perm:"read"` StorageAddLocal func(ctx context.Context, path string) error `perm:"admin"` } @@ -874,12 +874,12 @@ func (c *StorageMinerStruct) DealsSetAcceptingStorageDeals(ctx context.Context, return c.Internal.DealsSetAcceptingStorageDeals(ctx, b) } -func (c *StorageMinerStruct) DealsPieceCidBlacklist(ctx context.Context) ([]cid.Cid, error) { - return c.Internal.DealsPieceCidBlacklist(ctx) +func (c *StorageMinerStruct) DealsPieceCidBlocklist(ctx context.Context) ([]cid.Cid, error) { + return c.Internal.DealsPieceCidBlocklist(ctx) } -func (c *StorageMinerStruct) DealsSetPieceCidBlacklist(ctx context.Context, cids []cid.Cid) error { - return c.Internal.DealsSetPieceCidBlacklist(ctx, cids) +func (c *StorageMinerStruct) DealsSetPieceCidBlocklist(ctx context.Context, cids []cid.Cid) error { + return c.Internal.DealsSetPieceCidBlocklist(ctx, cids) } func (c *StorageMinerStruct) StorageAddLocal(ctx context.Context, path string) error { diff --git a/cmd/lotus-storage-miner/market.go b/cmd/lotus-storage-miner/market.go index e27bf6081..110411bc6 100644 --- a/cmd/lotus-storage-miner/market.go +++ b/cmd/lotus-storage-miner/market.go @@ -227,9 +227,9 @@ var dealsCmd = &cli.Command{ disableCmd, setAskCmd, getAskCmd, - setBlacklistCmd, - getBlacklistCmd, - resetBlacklistCmd, + setBlocklistCmd, + getBlocklistCmd, + resetBlocklistCmd, }, } @@ -289,9 +289,9 @@ var dealsListCmd = &cli.Command{ }, } -var getBlacklistCmd = &cli.Command{ - Name: "get-blacklist", - Usage: "List the contents of the storage miner's piece CID blacklist", +var getBlocklistCmd = &cli.Command{ + Name: "get-blocklist", + Usage: "List the contents of the storage miner's piece CID blocklist", Flags: []cli.Flag{ &CidBaseFlag, }, @@ -302,7 +302,7 @@ var getBlacklistCmd = &cli.Command{ } defer closer() - blacklist, err := api.DealsPieceCidBlacklist(lcli.DaemonContext(cctx)) + blocklist, err := api.DealsPieceCidBlocklist(lcli.DaemonContext(cctx)) if err != nil { return err } @@ -312,17 +312,17 @@ var getBlacklistCmd = &cli.Command{ return err } - for idx := range blacklist { - fmt.Println(encoder.Encode(blacklist[idx])) + for idx := range blocklist { + fmt.Println(encoder.Encode(blocklist[idx])) } return nil }, } -var setBlacklistCmd = &cli.Command{ - Name: "set-blacklist", - Usage: "Set the storage miner's list of blacklisted piece CIDs", +var setBlocklistCmd = &cli.Command{ + Name: "set-blocklist", + Usage: "Set the storage miner's list of blocklisted piece CIDs", ArgsUsage: "[ (optional, will read from stdin if omitted)]", Flags: []cli.Flag{}, Action: func(cctx *cli.Context) error { @@ -348,14 +348,14 @@ var setBlacklistCmd = &cli.Command{ scanner = bufio.NewScanner(file) } - var blacklist []cid.Cid + var blocklist []cid.Cid for scanner.Scan() { decoded, err := cid.Decode(scanner.Text()) if err != nil { return err } - blacklist = append(blacklist, decoded) + blocklist = append(blocklist, decoded) } err = scanner.Err() @@ -363,13 +363,13 @@ var setBlacklistCmd = &cli.Command{ return err } - return api.DealsSetPieceCidBlacklist(lcli.DaemonContext(cctx), blacklist) + return api.DealsSetPieceCidBlocklist(lcli.DaemonContext(cctx), blocklist) }, } -var resetBlacklistCmd = &cli.Command{ - Name: "reset-blacklist", - Usage: "Remove all entries from the storage miner's piece CID blacklist", +var resetBlocklistCmd = &cli.Command{ + Name: "reset-blocklist", + Usage: "Remove all entries from the storage miner's piece CID blocklist", Flags: []cli.Flag{}, Action: func(cctx *cli.Context) error { api, closer, err := lcli.GetStorageMinerAPI(cctx) @@ -378,6 +378,6 @@ var resetBlacklistCmd = &cli.Command{ } defer closer() - return api.DealsSetPieceCidBlacklist(lcli.DaemonContext(cctx), []cid.Cid{}) + return api.DealsSetPieceCidBlocklist(lcli.DaemonContext(cctx), []cid.Cid{}) }, } diff --git a/node/builder.go b/node/builder.go index bf32089e0..e84c4422c 100644 --- a/node/builder.go +++ b/node/builder.go @@ -314,8 +314,8 @@ func Online() Option { Override(new(dtypes.AcceptingStorageDealsConfigFunc), modules.NewAcceptingStorageDealsConfigFunc), Override(new(dtypes.SetAcceptingStorageDealsConfigFunc), modules.NewSetAcceptingStorageDealsConfigFunc), - Override(new(dtypes.StorageDealPieceCidBlacklistConfigFunc), modules.NewStorageDealPieceCidBlacklistConfigFunc), - Override(new(dtypes.SetStorageDealPieceCidBlacklistConfigFunc), modules.NewSetStorageDealPieceCidBlacklistConfigFunc), + Override(new(dtypes.StorageDealPieceCidBlocklistConfigFunc), modules.NewStorageDealPieceCidBlocklistConfigFunc), + Override(new(dtypes.SetStorageDealPieceCidBlocklistConfigFunc), modules.NewSetStorageDealPieceCidBlocklistConfigFunc), ), ) } diff --git a/node/config/def.go b/node/config/def.go index f9928ed3f..76a6a89ea 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -35,7 +35,7 @@ type StorageMiner struct { type DealmakingConfig struct { AcceptingStorageDeals bool - PieceCidBlacklist []cid.Cid + PieceCidBlocklist []cid.Cid } // API contains configs for API endpoint @@ -124,7 +124,7 @@ func DefaultStorageMiner() *StorageMiner { Dealmaking: DealmakingConfig{ AcceptingStorageDeals: true, - PieceCidBlacklist: []cid.Cid{}, + PieceCidBlocklist: []cid.Cid{}, }, } cfg.Common.API.ListenAddress = "/ip4/127.0.0.1/tcp/2345/http" diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 6eba12501..360a9442c 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -44,8 +44,8 @@ type StorageMinerAPI struct { *stores.Index SetAcceptingStorageDealsConfigFunc dtypes.SetAcceptingStorageDealsConfigFunc - StorageDealPieceCidBlacklistConfigFunc dtypes.StorageDealPieceCidBlacklistConfigFunc - SetStorageDealPieceCidBlacklistConfigFunc dtypes.SetStorageDealPieceCidBlacklistConfigFunc + StorageDealPieceCidBlocklistConfigFunc dtypes.StorageDealPieceCidBlocklistConfigFunc + SetStorageDealPieceCidBlocklistConfigFunc dtypes.SetStorageDealPieceCidBlocklistConfigFunc } func (sm *StorageMinerAPI) ServeRemote(w http.ResponseWriter, r *http.Request) { @@ -234,12 +234,12 @@ func (sm *StorageMinerAPI) DealsImportData(ctx context.Context, deal cid.Cid, fn return sm.StorageProvider.ImportDataForDeal(ctx, deal, fi) } -func (sm *StorageMinerAPI) DealsPieceCidBlacklist(ctx context.Context) ([]cid.Cid, error) { - return sm.StorageDealPieceCidBlacklistConfigFunc() +func (sm *StorageMinerAPI) DealsPieceCidBlocklist(ctx context.Context) ([]cid.Cid, error) { + return sm.StorageDealPieceCidBlocklistConfigFunc() } -func (sm *StorageMinerAPI) DealsSetPieceCidBlacklist(ctx context.Context, cids []cid.Cid) error { - return sm.SetStorageDealPieceCidBlacklistConfigFunc(cids) +func (sm *StorageMinerAPI) DealsSetPieceCidBlocklist(ctx context.Context, cids []cid.Cid) error { + return sm.SetStorageDealPieceCidBlocklistConfigFunc(cids) } func (sm *StorageMinerAPI) StorageAddLocal(ctx context.Context, path string) error { diff --git a/node/modules/dtypes/miner.go b/node/modules/dtypes/miner.go index a9d278ca4..3f0d81d6d 100644 --- a/node/modules/dtypes/miner.go +++ b/node/modules/dtypes/miner.go @@ -18,11 +18,11 @@ type AcceptingStorageDealsConfigFunc func() (bool, error) // storage deal acceptance. type SetAcceptingStorageDealsConfigFunc func(bool) error -// StorageDealCidBlacklistConfigFunc is a function which reads from miner config +// StorageDealCidBlocklistConfigFunc is a function which reads from miner config // to obtain a list of CIDs for which the storage miner will not accept storage // proposals. -type StorageDealPieceCidBlacklistConfigFunc func() ([]cid.Cid, error) +type StorageDealPieceCidBlocklistConfigFunc func() ([]cid.Cid, error) -// SetStorageDealCidBlacklistConfigFunc is a function which is used to set a +// SetStorageDealCidBlocklistConfigFunc is a function which is used to set a // list of CIDs for which the storage miner will reject deal proposals. -type SetStorageDealPieceCidBlacklistConfigFunc func([]cid.Cid) error +type SetStorageDealPieceCidBlocklistConfigFunc func([]cid.Cid) error diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 73e062c9c..67e0e0842 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -309,7 +309,7 @@ func NewStorageAsk(ctx helpers.MetricsCtx, fapi lapi.FullNode, ds dtypes.Metadat return storedAsk, nil } -func StorageProvider(minerAddress dtypes.MinerAddress, ffiConfig *ffiwrapper.Config, storedAsk *storedask.StoredAsk, h host.Host, ds dtypes.MetadataDS, ibs dtypes.StagingBlockstore, r repo.LockedRepo, pieceStore dtypes.ProviderPieceStore, dataTransfer dtypes.ProviderDataTransfer, spn storagemarket.StorageProviderNode, isAcceptingFunc dtypes.AcceptingStorageDealsConfigFunc, blacklistFunc dtypes.StorageDealPieceCidBlacklistConfigFunc) (storagemarket.StorageProvider, error) { +func StorageProvider(minerAddress dtypes.MinerAddress, ffiConfig *ffiwrapper.Config, storedAsk *storedask.StoredAsk, h host.Host, ds dtypes.MetadataDS, ibs dtypes.StagingBlockstore, r repo.LockedRepo, pieceStore dtypes.ProviderPieceStore, dataTransfer dtypes.ProviderDataTransfer, spn storagemarket.StorageProviderNode, isAcceptingFunc dtypes.AcceptingStorageDealsConfigFunc, blocklistFunc dtypes.StorageDealPieceCidBlocklistConfigFunc) (storagemarket.StorageProvider, error) { net := smnet.NewFromLibp2pHost(h) store, err := piecefilestore.NewLocalFileStore(piecefilestore.OsPath(r.Path())) if err != nil { @@ -327,15 +327,15 @@ func StorageProvider(minerAddress dtypes.MinerAddress, ffiConfig *ffiwrapper.Con return false, "miner is not accepting storage deals", nil } - blacklist, err := blacklistFunc() + blocklist, err := blocklistFunc() if err != nil { return false, "miner error", err } - for idx := range blacklist { - if deal.Proposal.PieceCID.Equals(blacklist[idx]) { - log.Warnf("piece CID in proposal %s is blacklisted; rejecting storage deal proposal from client: %s", deal.Proposal.PieceCID, deal.Client.String()) - return false, fmt.Sprintf("miner has blacklisted piece CID %s", deal.Proposal.PieceCID), nil + for idx := range blocklist { + if deal.Proposal.PieceCID.Equals(blocklist[idx]) { + log.Warnf("piece CID in proposal %s is blocklisted; rejecting storage deal proposal from client: %s", deal.Proposal.PieceCID, deal.Client.String()) + return false, fmt.Sprintf("miner has blocklisted piece CID %s", deal.Proposal.PieceCID), nil } } @@ -411,19 +411,19 @@ func NewSetAcceptingStorageDealsConfigFunc(r repo.LockedRepo) (dtypes.SetAccepti }, nil } -func NewStorageDealPieceCidBlacklistConfigFunc(r repo.LockedRepo) (dtypes.StorageDealPieceCidBlacklistConfigFunc, error) { +func NewStorageDealPieceCidBlocklistConfigFunc(r repo.LockedRepo) (dtypes.StorageDealPieceCidBlocklistConfigFunc, error) { return func() (out []cid.Cid, err error) { err = readCfg(r, func(cfg *config.StorageMiner) { - out = cfg.Dealmaking.PieceCidBlacklist + out = cfg.Dealmaking.PieceCidBlocklist }) return }, nil } -func NewSetStorageDealPieceCidBlacklistConfigFunc(r repo.LockedRepo) (dtypes.SetStorageDealPieceCidBlacklistConfigFunc, error) { - return func(blacklist []cid.Cid) (err error) { +func NewSetStorageDealPieceCidBlocklistConfigFunc(r repo.LockedRepo) (dtypes.SetStorageDealPieceCidBlocklistConfigFunc, error) { + return func(blocklist []cid.Cid) (err error) { err = mutateCfg(r, func(cfg *config.StorageMiner) { - cfg.Dealmaking.PieceCidBlacklist = blacklist + cfg.Dealmaking.PieceCidBlocklist = blocklist }) return }, nil From 7c4225d01c8239d09c5192bdae568730c79e15b9 Mon Sep 17 00:00:00 2001 From: Mitch Wagner Date: Thu, 18 Jun 2020 21:31:26 -0400 Subject: [PATCH 10/31] Remove redundant glossary item The excised entry previously rendered twice in the glossary under two different keys. This commit removes one of those KV pairs to avoid that. --- documentation/en/.glossary.json | 4 ---- 1 file changed, 4 deletions(-) diff --git a/documentation/en/.glossary.json b/documentation/en/.glossary.json index 41126f974..bdc43e319 100644 --- a/documentation/en/.glossary.json +++ b/documentation/en/.glossary.json @@ -63,10 +63,6 @@ "title": "Proof-of-Spacetime(s)", "value": "Filecoin is a protocol token whose blockchain runs on a novel proof, called Proof-of-Spacetime, where blocks are created by miners that are storing data." }, - "lotus-testnet": { - "title": "Filecoin Testnet", - "value": "Until we launch, we are making lots of changes to Lotus. The Testnet is expected to bring a few significant fixes/improvements. During Testnet, you can retrieve test filecoin from our network faucet to use as collateral to start mining. Test filecoin do not have any value – the official filecoin tokens will not be released until Mainnet launch." - }, "filecoin-testnet": { "title": "Filecoin Testnet", "value": "Until we launch, we are making lots of changes to Lotus. The Testnet is expected to bring a few significant fixes/improvements. During Testnet, you can retrieve test filecoin from our network faucet to use as collateral to start mining. Test filecoin do not have any value – the official filecoin tokens will not be released until Mainnet launch." From 5e77b5da9f2e45377e76087a93193af29821ce35 Mon Sep 17 00:00:00 2001 From: laser Date: Fri, 19 Jun 2020 08:51:06 -0700 Subject: [PATCH 11/31] bump from 3 to 4 digits in format-string Fixes #1993 --- chain/types/bigint.go | 2 +- chain/types/bigint_test.go | 25 ++++++++++++++++++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/chain/types/bigint.go b/chain/types/bigint.go index 22ecf833c..a7b25870a 100644 --- a/chain/types/bigint.go +++ b/chain/types/bigint.go @@ -76,7 +76,7 @@ func SizeStr(bi BigInt) string { } f, _ := r.Float64() - return fmt.Sprintf("%.3g %s", f, byteSizeUnits[i]) + return fmt.Sprintf("%.4g %s", f, byteSizeUnits[i]) } var deciUnits = []string{"", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"} diff --git a/chain/types/bigint_test.go b/chain/types/bigint_test.go index 2f632d1d7..43e5633b2 100644 --- a/chain/types/bigint_test.go +++ b/chain/types/bigint_test.go @@ -3,7 +3,12 @@ package types import ( "bytes" "math/big" + "math/rand" + "strings" "testing" + "time" + + "github.com/docker/go-units" "github.com/stretchr/testify/assert" ) @@ -60,8 +65,10 @@ func TestSizeStr(t *testing.T) { }{ {0, "0 B"}, {1, "1 B"}, + {1016, "1016 B"}, {1024, "1 KiB"}, - {2000, "1.95 KiB"}, + {1000 * 1024, "1000 KiB"}, + {2000, "1.953 KiB"}, {5 << 20, "5 MiB"}, {11 << 60, "11 EiB"}, } @@ -71,6 +78,22 @@ func TestSizeStr(t *testing.T) { } } +func TestSizeStrUnitsSymmetry(t *testing.T) { + s := rand.NewSource(time.Now().UnixNano()) + r := rand.New(s) + + for i := 0; i < 1000000; i++ { + n := r.Uint64() + l := strings.ReplaceAll(units.BytesSize(float64(n)), " ", "") + r := strings.ReplaceAll(SizeStr(NewInt(n)), " ", "") + + assert.NotContains(t, l, "e+") + assert.NotContains(t, r, "e+") + + assert.Equal(t, l, r, "wrong formatting for %d", n) + } +} + func TestSizeStrBig(t *testing.T) { ZiB := big.NewInt(50000) ZiB = ZiB.Lsh(ZiB, 70) From fbeaab466acd181f5da39ffc22cd9aa97609bf8a Mon Sep 17 00:00:00 2001 From: laser Date: Fri, 19 Jun 2020 09:19:46 -0700 Subject: [PATCH 12/31] obey the linter --- cmd/lotus-storage-miner/market.go | 2 +- node/modules/dtypes/miner.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/lotus-storage-miner/market.go b/cmd/lotus-storage-miner/market.go index 110411bc6..c668456f0 100644 --- a/cmd/lotus-storage-miner/market.go +++ b/cmd/lotus-storage-miner/market.go @@ -343,7 +343,7 @@ var setBlocklistCmd = &cli.Command{ if err != nil { log.Fatal(err) } - defer file.Close() + defer file.Close() //nolint:errcheck scanner = bufio.NewScanner(file) } diff --git a/node/modules/dtypes/miner.go b/node/modules/dtypes/miner.go index 3f0d81d6d..584642a3b 100644 --- a/node/modules/dtypes/miner.go +++ b/node/modules/dtypes/miner.go @@ -18,11 +18,11 @@ type AcceptingStorageDealsConfigFunc func() (bool, error) // storage deal acceptance. type SetAcceptingStorageDealsConfigFunc func(bool) error -// StorageDealCidBlocklistConfigFunc is a function which reads from miner config +// StorageDealPieceCidBlocklistConfigFunc is a function which reads from miner config // to obtain a list of CIDs for which the storage miner will not accept storage // proposals. type StorageDealPieceCidBlocklistConfigFunc func() ([]cid.Cid, error) -// SetStorageDealCidBlocklistConfigFunc is a function which is used to set a +// SetStorageDealPieceCidBlocklistConfigFunc is a function which is used to set a // list of CIDs for which the storage miner will reject deal proposals. type SetStorageDealPieceCidBlocklistConfigFunc func([]cid.Cid) error From 49595965279f6ba49092255620b21b55bc0371ae Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Mon, 22 Jun 2020 11:42:26 -0400 Subject: [PATCH 13/31] Clarify expected block win when below minimum power threshold --- cmd/lotus-storage-miner/info.go | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/cmd/lotus-storage-miner/info.go b/cmd/lotus-storage-miner/info.go index 9d98f8569..01077bc83 100644 --- a/cmd/lotus-storage-miner/info.go +++ b/cmd/lotus-storage-miner/info.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "github.com/filecoin-project/specs-actors/actors/builtin/power" "sort" "time" @@ -119,16 +120,20 @@ var infoCmd = &cli.Command{ faultyPercentage) } - expWinChance := float64(types.BigMul(qpercI, types.NewInt(build.BlocksPerEpoch)).Int64()) / 1000000 - if expWinChance > 0 { - if expWinChance > 1 { - expWinChance = 1 - } - winRate := time.Duration(float64(time.Second*build.BlockDelay) / expWinChance) - winPerDay := float64(time.Hour*24) / float64(winRate) + if pow.MinerPower.RawBytePower.LessThan(power.ConsensusMinerMinPower) { + fmt.Print("Below minimum power threshold, no blocks will be won") + } else { + expWinChance := float64(types.BigMul(qpercI, types.NewInt(build.BlocksPerEpoch)).Int64()) / 1000000 + if expWinChance > 0 { + if expWinChance > 1 { + expWinChance = 1 + } + winRate := time.Duration(float64(time.Second*build.BlockDelay) / expWinChance) + winPerDay := float64(time.Hour*24) / float64(winRate) - fmt.Print("Expected block win rate: ") - color.Blue("%.4f/day (every %s)", winPerDay, winRate.Truncate(time.Second)) + fmt.Print("Expected block win rate: ") + color.Blue("%.4f/day (every %s)", winPerDay, winRate.Truncate(time.Second)) + } } fmt.Println() From 8cd2012d27445058fa5730badf94ed9572612e45 Mon Sep 17 00:00:00 2001 From: Lucas Molas Date: Mon, 22 Jun 2020 12:45:24 -0300 Subject: [PATCH 14/31] doc: report a vulnerability --- README.md | 4 ++++ SECURITY.md | 29 +++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 SECURITY.md diff --git a/README.md b/README.md index cd64073a6..a15276ee2 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,10 @@ Lotus is an implementation of the Filecoin Distributed Storage Network. For more For instructions on how to build lotus from source, please visit [https://docs.lotu.sh](https://docs.lotu.sh) or read the source [here](https://github.com/filecoin-project/lotus/tree/master/documentation). +## Reporting a Vulnerability + +Please send an email to security@filecoin.org. See our [security policy](SECURITY.md) for more details. + ## Development All work is tracked via issues. An attempt at keeping an up-to-date view on remaining work is in the [lotus testnet github project board](https://github.com/filecoin-project/lotus/projects/1). diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 000000000..ecb600deb --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,29 @@ +# Security Policy + +## Reporting a Vulnerability + +For *critical* bugs, please send an email to security@filecoin.org. + +The bug reporting process differs between bugs that are critical and may crash the network, and others that are unlikely to cause problems if malicious parties know about it. For non-critical bugs, please simply file a GitHub [issue](https://github.com/filecoin-project/lotus/issues/new?template=bug_report.md). + +Please try to provide a clear description of any bugs reported, along with how to reproduce the bug if possible. More detailed bug reports (especially those with a PoC included) will help us move forward much faster. Additionally, please avoid reporting bugs that already have open issues. Take a moment to search the issue list of the related GitHub repositories before writing up a new report. + +Here are some examples of bugs we would consider 'critical': + +* If you can spend from a `multisig` wallet you do not control the keys for. +* If you can cause a miner to be slashed without them actually misbehaving. +* If you can maintain power without submitting windowed posts regularly. +* If you can craft a message that causes lotus nodes to panic. +* If you can cause your miner to win significantly more blocks than it should. +* If you can craft a message that causes a persistent fork in the network. +* If you can cause the total amount of Filecoin in the network to no longer be 2 billion. + +This is not an exhaustive list, but should provide some idea of what we consider 'critical'. + +## Supported Versions + +* TODO: This should be defined and set up by Mainnet launch. + +| Version | Supported | +| ------- | ------------------ | +| Testnet | :white_check_mark: | From d20255b1b4b27596bbd8a27489dd029a31edc124 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Mon, 22 Jun 2020 19:41:52 +0100 Subject: [PATCH 15/31] provide an option to disable loading of built-in assets. --- build/bootstrap.go | 4 ++++ build/flags.go | 15 +++++++++++++++ node/modules/storageminer.go | 6 ++++++ 3 files changed, 25 insertions(+) create mode 100644 build/flags.go diff --git a/build/bootstrap.go b/build/bootstrap.go index 0710f0dc0..6343a0172 100644 --- a/build/bootstrap.go +++ b/build/bootstrap.go @@ -13,6 +13,10 @@ import ( ) func BuiltinBootstrap() ([]peer.AddrInfo, error) { + if DisableBuiltinAssets { + return nil, nil + } + var out []peer.AddrInfo b := rice.MustFindBox("bootstrap") diff --git a/build/flags.go b/build/flags.go new file mode 100644 index 000000000..33e9f6ede --- /dev/null +++ b/build/flags.go @@ -0,0 +1,15 @@ +package build + +// DisableBuiltinAssets disables the resolution of go.rice boxes that store +// built-in assets, such as proof parameters, bootstrap peers, genesis blocks, +// etc. +// +// When this value is set to true, it is expected that the user will +// provide any such configurations through the Lotus API itself. +// +// This is useful when you're using Lotus as a library, such as to orchestrate +// test scenarios, or for other purposes where you don't need to use the +// defaults shipped with the binary. +// +// For this flag to be effective, it must be enabled _before_ instantiating Lotus. +var DisableBuiltinAssets = false diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 67e0e0842..c6cadec34 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -75,6 +75,12 @@ func GetParams(sbc *ffiwrapper.Config) error { return err } + // If built-in assets are disabled, we expect the user to have placed the right + // parameters in the right location on the filesystem (/var/tmp/filecoin-proof-parameters). + if build.DisableBuiltinAssets { + return nil + } + if err := paramfetch.GetParams(context.TODO(), build.ParametersJSON(), uint64(ssize)); err != nil { return xerrors.Errorf("fetching proof parameters: %w", err) } From 8649baccf7625c61f3619e778926ff4c02e0bb1c Mon Sep 17 00:00:00 2001 From: Mike Greenberg Date: Thu, 18 Jun 2020 22:18:31 -0400 Subject: [PATCH 16/31] Remove dev values from chainwatch.service --- scripts/chainwatch.service | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/chainwatch.service b/scripts/chainwatch.service index e958ec857..74afee0e9 100644 --- a/scripts/chainwatch.service +++ b/scripts/chainwatch.service @@ -1,15 +1,15 @@ [Unit] Description=Chainwatch -PartOf=sentinel.service After=lotus-daemon.service Requires=lotus-daemon.service [Service] Environment=GOLOG_FILE="/var/log/lotus/chainwatch.log" Environment=GOLOG_LOG_FMT="json" -Environment=LOTUS_DB="postgres://postgres:password@localhost:5432/postgres?sslmode=disable" -Environment=LOTUS_PATH="/root/.lotus" +Environment=LOTUS_DB="" +Environment=LOTUS_PATH="%h/.lotus" +EnvironmentFile=-/etc/lotus/chainwatch.env ExecStart=/usr/local/bin/chainwatch run [Install] -WantedBy=multiuser.target +WantedBy=multi-user.target From b71f771acbfcdbaa273a15e126051b79a51fc542 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Mon, 22 Jun 2020 16:09:05 -0700 Subject: [PATCH 17/31] run block validation for tipsets in parallel --- chain/sync.go | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/chain/sync.go b/chain/sync.go index 26f30d95b..19ddac108 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -466,16 +466,25 @@ func (syncer *Syncer) ValidateTipSet(ctx context.Context, fts *store.FullTipSet) return nil } + var futures []async.ErrorFuture for _, b := range fts.Blocks { - if err := syncer.ValidateBlock(ctx, b); err != nil { - if isPermanent(err) { - syncer.bad.Add(b.Cid(), err.Error()) + futures = append(futures, async.Err(func() error { + if err := syncer.ValidateBlock(ctx, b); err != nil { + if isPermanent(err) { + syncer.bad.Add(b.Cid(), err.Error()) + } + return xerrors.Errorf("validating block %s: %w", b.Cid(), err) } - return xerrors.Errorf("validating block %s: %w", b.Cid(), err) - } - if err := syncer.sm.ChainStore().AddToTipSetTracker(b.Header); err != nil { - return xerrors.Errorf("failed to add validated header to tipset tracker: %w", err) + if err := syncer.sm.ChainStore().AddToTipSetTracker(b.Header); err != nil { + return xerrors.Errorf("failed to add validated header to tipset tracker: %w", err) + } + return nil + })) + } + for _, f := range futures { + if err := f.AwaitContext(ctx); err != nil { + return err } } return nil From 3752311f44f016b8111da3fe28ad0c93587be823 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Mon, 22 Jun 2020 16:25:41 -0700 Subject: [PATCH 18/31] update to latest libp2p release --- go.mod | 15 ++++++++------- go.sum | 21 +++++++++++++++++++++ 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 68ac9a884..70fa0d19e 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/GeertJohan/go.rice v1.0.0 github.com/Gurpartap/async v0.0.0-20180927173644-4f7f499dd9ee github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect + github.com/alangpierce/go-forceexport v0.0.0-20160317203124-8f1d6941cd75 // indirect github.com/coreos/go-systemd/v22 v22.0.0 github.com/docker/go-units v0.4.0 github.com/drand/drand v0.9.2-0.20200616080806-a94e9c1636a4 @@ -45,7 +46,7 @@ require ( github.com/ipfs/go-bitswap v0.2.8 github.com/ipfs/go-block-format v0.0.2 github.com/ipfs/go-blockservice v0.1.3 - github.com/ipfs/go-cid v0.0.6-0.20200501230655-7c82f3b81c00 + github.com/ipfs/go-cid v0.0.6 github.com/ipfs/go-cidutil v0.0.2 github.com/ipfs/go-datastore v0.4.4 github.com/ipfs/go-ds-badger2 v0.1.0 @@ -76,20 +77,20 @@ require ( github.com/kelseyhightower/envconfig v1.4.0 github.com/lib/pq v1.2.0 github.com/libp2p/go-eventbus v0.2.1 - github.com/libp2p/go-libp2p v0.9.4 + github.com/libp2p/go-libp2p v0.10.0 github.com/libp2p/go-libp2p-connmgr v0.2.4 - github.com/libp2p/go-libp2p-core v0.5.7 + github.com/libp2p/go-libp2p-core v0.6.0 github.com/libp2p/go-libp2p-discovery v0.4.0 github.com/libp2p/go-libp2p-kad-dht v0.8.1 github.com/libp2p/go-libp2p-mplex v0.2.3 github.com/libp2p/go-libp2p-peer v0.2.0 - github.com/libp2p/go-libp2p-peerstore v0.2.4 + github.com/libp2p/go-libp2p-peerstore v0.2.6 github.com/libp2p/go-libp2p-pubsub v0.3.2 github.com/libp2p/go-libp2p-quic-transport v0.5.0 github.com/libp2p/go-libp2p-record v0.1.2 github.com/libp2p/go-libp2p-routing-helpers v0.2.3 github.com/libp2p/go-libp2p-secio v0.2.2 - github.com/libp2p/go-libp2p-swarm v0.2.6 + github.com/libp2p/go-libp2p-swarm v0.2.7 github.com/libp2p/go-libp2p-tls v0.1.3 github.com/libp2p/go-libp2p-yamux v0.2.8 github.com/libp2p/go-maddr-filter v0.1.0 @@ -100,11 +101,11 @@ require ( github.com/multiformats/go-multiaddr v0.2.2 github.com/multiformats/go-multiaddr-dns v0.2.0 github.com/multiformats/go-multiaddr-net v0.1.5 - github.com/multiformats/go-multibase v0.0.2 + github.com/multiformats/go-multibase v0.0.3 github.com/multiformats/go-multihash v0.0.13 github.com/opentracing/opentracing-go v1.1.0 github.com/stretchr/objx v0.2.0 // indirect - github.com/stretchr/testify v1.5.1 + github.com/stretchr/testify v1.6.1 github.com/syndtr/goleveldb v1.0.0 github.com/urfave/cli/v2 v2.2.0 github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba diff --git a/go.sum b/go.sum index e21c27f74..1c9dc6627 100644 --- a/go.sum +++ b/go.sum @@ -472,6 +472,8 @@ github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog= github.com/ipfs/go-cid v0.0.6-0.20200501230655-7c82f3b81c00 h1:QN88Q0kT2QiDaLxpR/SDsqOBtNIEF/F3n96gSDUimkA= github.com/ipfs/go-cid v0.0.6-0.20200501230655-7c82f3b81c00/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog= +github.com/ipfs/go-cid v0.0.6 h1:go0y+GcDOGeJIV01FeBsta4FHngoA4Wz7KMeLkXAhMs= +github.com/ipfs/go-cid v0.0.6/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-cidutil v0.0.2 h1:CNOboQf1t7Qp0nuNh8QMmhJs0+Q//bRL1axtCnIB1Yo= github.com/ipfs/go-cidutil v0.0.2/go.mod h1:ewllrvrxG6AMYStla3GD7Cqn+XYSLqjK0vc+086tB6s= github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= @@ -557,6 +559,8 @@ github.com/ipfs/go-ipfs-routing v0.1.0 h1:gAJTT1cEeeLj6/DlLX6t+NxD9fQe2ymTO6qWRD github.com/ipfs/go-ipfs-routing v0.1.0/go.mod h1:hYoUkJLyAUKhF58tysKpids8RNDPO42BVMgK5dNsoqY= github.com/ipfs/go-ipfs-util v0.0.1 h1:Wz9bL2wB2YBJqggkA4dD7oSmqB4cAnpNbGrlHJulv50= github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= +github.com/ipfs/go-ipfs-util v0.0.2 h1:59Sswnk1MFaiq+VcaknX7aYEyGyGDAA73ilhEK2POp8= +github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ= github.com/ipfs/go-ipld-cbor v0.0.1/go.mod h1:RXHr8s4k0NE0TKhnrxqZC9M888QfsBN9rhS5NjfKzY8= github.com/ipfs/go-ipld-cbor v0.0.2/go.mod h1:wTBtrQZA3SoFKMVkp6cn6HMRteIB1VsmHA0AQFOn7Nc= github.com/ipfs/go-ipld-cbor v0.0.3/go.mod h1:wTBtrQZA3SoFKMVkp6cn6HMRteIB1VsmHA0AQFOn7Nc= @@ -725,6 +729,8 @@ github.com/libp2p/go-libp2p v0.8.3/go.mod h1:EsH1A+8yoWK+L4iKcbPYu6MPluZ+CHWI9El github.com/libp2p/go-libp2p v0.9.2/go.mod h1:cunHNLDVus66Ct9iXXcjKRLdmHdFdHVe1TAnbubJQqQ= github.com/libp2p/go-libp2p v0.9.4 h1:yighwjFvsF/qQaGtHPZfxcF+ph4ydCNnsKvg712lYRo= github.com/libp2p/go-libp2p v0.9.4/go.mod h1:NzQcC2o19xgwGqCmjx7DN+4h2F13qPCZ9UJmweYzsnU= +github.com/libp2p/go-libp2p v0.10.0 h1:7ooOvK1wi8eLpyTppy8TeH43UHy5uI75GAHGJxenUi0= +github.com/libp2p/go-libp2p v0.10.0/go.mod h1:yBJNpb+mGJdgrwbKAKrhPU0u3ogyNFTfjJ6bdM+Q/G8= github.com/libp2p/go-libp2p-autonat v0.0.2/go.mod h1:fs71q5Xk+pdnKU014o2iq1RhMs9/PMaG5zXRFNnIIT4= github.com/libp2p/go-libp2p-autonat v0.0.6/go.mod h1:uZneLdOkZHro35xIhpbtTzLlgYturpu4J5+0cZK3MqE= github.com/libp2p/go-libp2p-autonat v0.1.0/go.mod h1:1tLf2yXxiE/oKGtDwPYWTSYG3PtvYlJmg7NeVtPRqH8= @@ -750,6 +756,8 @@ github.com/libp2p/go-libp2p-circuit v0.1.4/go.mod h1:CY67BrEjKNDhdTk8UgBX1Y/H5c3 github.com/libp2p/go-libp2p-circuit v0.2.1/go.mod h1:BXPwYDN5A8z4OEY9sOfr2DUQMLQvKt/6oku45YUmjIo= github.com/libp2p/go-libp2p-circuit v0.2.2 h1:87RLabJ9lrhoiSDDZyCJ80ZlI5TLJMwfyoGAaWXzWqA= github.com/libp2p/go-libp2p-circuit v0.2.2/go.mod h1:nkG3iE01tR3FoQ2nMm06IUrCpCyJp1Eo4A1xYdpjfs4= +github.com/libp2p/go-libp2p-circuit v0.2.3 h1:3Uw1fPHWrp1tgIhBz0vSOxRUmnKL8L/NGUyEd5WfSGM= +github.com/libp2p/go-libp2p-circuit v0.2.3/go.mod h1:nkG3iE01tR3FoQ2nMm06IUrCpCyJp1Eo4A1xYdpjfs4= github.com/libp2p/go-libp2p-connmgr v0.1.1/go.mod h1:wZxh8veAmU5qdrfJ0ZBLcU8oJe9L82ciVP/fl1VHjXk= github.com/libp2p/go-libp2p-connmgr v0.2.3/go.mod h1:Gqjg29zI8CwXX21zRxy6gOg8VYu3zVerJRt2KyktzH4= github.com/libp2p/go-libp2p-connmgr v0.2.4 h1:TMS0vc0TCBomtQJyWr7fYxcVYYhx+q/2gF++G5Jkl/w= @@ -776,6 +784,8 @@ github.com/libp2p/go-libp2p-core v0.5.5/go.mod h1:vj3awlOr9+GMZJFH9s4mpt9RHHgGqe github.com/libp2p/go-libp2p-core v0.5.6/go.mod h1:txwbVEhHEXikXn9gfC7/UDDw7rkxuX0bJvM49Ykaswo= github.com/libp2p/go-libp2p-core v0.5.7 h1:QK3xRwFxqd0Xd9bSZL+8yZ8ncZZbl6Zngd/+Y+A6sgQ= github.com/libp2p/go-libp2p-core v0.5.7/go.mod h1:txwbVEhHEXikXn9gfC7/UDDw7rkxuX0bJvM49Ykaswo= +github.com/libp2p/go-libp2p-core v0.6.0 h1:u03qofNYTBN+yVg08PuAKylZogVf0xcTEeM8skGf+ak= +github.com/libp2p/go-libp2p-core v0.6.0/go.mod h1:txwbVEhHEXikXn9gfC7/UDDw7rkxuX0bJvM49Ykaswo= github.com/libp2p/go-libp2p-crypto v0.0.1/go.mod h1:yJkNyDmO341d5wwXxDUGO0LykUVT72ImHNUqh5D/dBE= github.com/libp2p/go-libp2p-crypto v0.0.2/go.mod h1:eETI5OUfBnvARGOHrJz2eWNyTUxEGZnBxMcbUjfIj4I= github.com/libp2p/go-libp2p-crypto v0.1.0 h1:k9MFy+o2zGDNGsaoZl0MA3iZ75qXxr9OOoAZF+sD5OQ= @@ -835,6 +845,8 @@ github.com/libp2p/go-libp2p-peerstore v0.2.2/go.mod h1:NQxhNjWxf1d4w6PihR8btWIRj github.com/libp2p/go-libp2p-peerstore v0.2.3/go.mod h1:K8ljLdFn590GMttg/luh4caB/3g0vKuY01psze0upRw= github.com/libp2p/go-libp2p-peerstore v0.2.4 h1:jU9S4jYN30kdzTpDAR7SlHUD+meDUjTODh4waLWF1ws= github.com/libp2p/go-libp2p-peerstore v0.2.4/go.mod h1:ss/TWTgHZTMpsU/oKVVPQCGuDHItOpf2W8RxAi50P2s= +github.com/libp2p/go-libp2p-peerstore v0.2.6 h1:2ACefBX23iMdJU9Ke+dcXt3w86MIryes9v7In4+Qq3U= +github.com/libp2p/go-libp2p-peerstore v0.2.6/go.mod h1:ss/TWTgHZTMpsU/oKVVPQCGuDHItOpf2W8RxAi50P2s= github.com/libp2p/go-libp2p-pnet v0.2.0 h1:J6htxttBipJujEjz1y0a5+eYoiPcFHhSYHH6na5f0/k= github.com/libp2p/go-libp2p-pnet v0.2.0/go.mod h1:Qqvq6JH/oMZGwqs3N1Fqhv8NVhrdYcO0BW4wssv21LA= github.com/libp2p/go-libp2p-protocol v0.0.1/go.mod h1:Af9n4PiruirSDjHycM1QuiMi/1VZNHYcK8cLgFJLZ4s= @@ -873,6 +885,8 @@ github.com/libp2p/go-libp2p-swarm v0.2.3/go.mod h1:P2VO/EpxRyDxtChXz/VPVXyTnszHv github.com/libp2p/go-libp2p-swarm v0.2.4/go.mod h1:/xIpHFPPh3wmSthtxdGbkHZ0OET1h/GGZes8Wku/M5Y= github.com/libp2p/go-libp2p-swarm v0.2.6 h1:UhMXIa+yCOALQyceENEIStMlbTCzOM6aWo6vw8QW17Q= github.com/libp2p/go-libp2p-swarm v0.2.6/go.mod h1:F9hrkZjO7dDbcEiYii/fAB1QdpLuU6h1pa4P5VNsEgc= +github.com/libp2p/go-libp2p-swarm v0.2.7 h1:4lV/sf7f0NuVqunOpt1I11+Z54+xp+m0eeAvxj/LyRc= +github.com/libp2p/go-libp2p-swarm v0.2.7/go.mod h1:ZSJ0Q+oq/B1JgfPHJAT2HTall+xYRNYp1xs4S2FBWKA= github.com/libp2p/go-libp2p-testing v0.0.1/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= github.com/libp2p/go-libp2p-testing v0.0.2/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= github.com/libp2p/go-libp2p-testing v0.0.3/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= @@ -1042,6 +1056,8 @@ github.com/mr-tron/base58 v1.1.3 h1:v+sk57XuaCKGXpWtVBX8YJzO7hMGx4Aajh4TQbdEFdc= github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI= github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= +github.com/multiformats/go-base36 v0.1.0 h1:JR6TyF7JjGd3m6FbLU2cOxhC0Li8z8dLNGQ89tUg4F4= +github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM= github.com/multiformats/go-multiaddr v0.0.1/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= github.com/multiformats/go-multiaddr v0.0.2/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= github.com/multiformats/go-multiaddr v0.0.4/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= @@ -1071,6 +1087,8 @@ github.com/multiformats/go-multiaddr-net v0.1.5/go.mod h1:ilNnaM9HbmVFqsb/qcNysj github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multibase v0.0.2 h1:2pAgScmS1g9XjH7EtAfNhTuyrWYEWcxy0G5Wo85hWDA= github.com/multiformats/go-multibase v0.0.2/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= +github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk= +github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= github.com/multiformats/go-multihash v0.0.7/go.mod h1:XuKXPp8VHcTygube3OWZC+aZrA+H1IhmjoCDtJc7PXM= @@ -1290,6 +1308,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= @@ -1777,6 +1797,7 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= From 9903eba7fb4ca4a827a3d637c133e4da3eecebfd Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Mon, 22 Jun 2020 20:08:19 -0700 Subject: [PATCH 19/31] stream bench import results to disk --- cmd/lotus-bench/import.go | 42 ++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index f9c20ac9a..f7538daec 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "io" "io/ioutil" "math" "os" @@ -119,14 +120,22 @@ var importBenchCmd = &cli.Command{ ts = next } - out := make([]TipSetExec, 0, len(tschain)) + ibj, err := os.Create("import-bench.json") + if err != nil { + return err + } + defer ibj.Close() //nolint:errcheck + + enc := json.NewEncoder(ibj) + + var lastTse *TipSetExec lastState := tschain[len(tschain)-1].ParentState() for i := len(tschain) - 2; i >= 0; i-- { cur := tschain[i] log.Infof("computing state (height: %d, ts=%s)", cur.Height(), cur.Cids()) if cur.ParentState() != lastState { - lastTrace := out[len(out)-1].Trace + lastTrace := lastTse.Trace d, err := json.MarshalIndent(lastTrace, "", " ") if err != nil { panic(err) @@ -140,26 +149,20 @@ var importBenchCmd = &cli.Command{ if err != nil { return err } - out = append(out, TipSetExec{ + + lastTse = &TipSetExec{ TipSet: cur.Key(), Trace: trace, Duration: time.Since(start), - }) + } lastState = st + if err := enc.Encode(lastTse); err != nil { + return xerrors.Errorf("failed to write out tipsetexec: %w", err) + } } pprof.StopCPUProfile() - ibj, err := os.Create("import-bench.json") - if err != nil { - return err - } - defer ibj.Close() //nolint:errcheck - - if err := json.NewEncoder(ibj).Encode(out); err != nil { - return err - } - return nil }, @@ -236,8 +239,15 @@ var importAnalyzeCmd = &cli.Command{ } var results []TipSetExec - if err := json.NewDecoder(fi).Decode(&results); err != nil { - return err + for { + var tse TipSetExec + if err := json.NewDecoder(fi).Decode(&tse); err != nil { + if err != io.EOF { + return err + } + break + } + results = append(results, tse) } chargeDeltas := make(map[string][]float64) From abd400801c1b40b92f3c9442cf86c4d3145340cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 23 Jun 2020 11:46:20 +0200 Subject: [PATCH 20/31] mod tidy --- go.mod | 1 - go.sum | 10 ++-------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 70fa0d19e..655f402c9 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,6 @@ require ( github.com/GeertJohan/go.rice v1.0.0 github.com/Gurpartap/async v0.0.0-20180927173644-4f7f499dd9ee github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect - github.com/alangpierce/go-forceexport v0.0.0-20160317203124-8f1d6941cd75 // indirect github.com/coreos/go-systemd/v22 v22.0.0 github.com/docker/go-units v0.4.0 github.com/drand/drand v0.9.2-0.20200616080806-a94e9c1636a4 diff --git a/go.sum b/go.sum index 1c9dc6627..113c73c53 100644 --- a/go.sum +++ b/go.sum @@ -62,7 +62,6 @@ github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBA github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= github.com/akavel/rsrc v0.8.0 h1:zjWn7ukO9Kc5Q62DOJCcxGpXC18RawVtYAGdz2aLlfw= github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= -github.com/alangpierce/go-forceexport v0.0.0-20160317203124-8f1d6941cd75/go.mod h1:uAXEEpARkRhCZfEvy/y0Jcc888f9tHCc1W7/UeEtreE= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -727,8 +726,6 @@ github.com/libp2p/go-libp2p v0.7.4/go.mod h1:oXsBlTLF1q7pxr+9w6lqzS1ILpyHsaBPniV github.com/libp2p/go-libp2p v0.8.2/go.mod h1:NQDA/F/qArMHGe0J7sDScaKjW8Jh4y/ozQqBbYJ+BnA= github.com/libp2p/go-libp2p v0.8.3/go.mod h1:EsH1A+8yoWK+L4iKcbPYu6MPluZ+CHWI9El8cTaefiM= github.com/libp2p/go-libp2p v0.9.2/go.mod h1:cunHNLDVus66Ct9iXXcjKRLdmHdFdHVe1TAnbubJQqQ= -github.com/libp2p/go-libp2p v0.9.4 h1:yighwjFvsF/qQaGtHPZfxcF+ph4ydCNnsKvg712lYRo= -github.com/libp2p/go-libp2p v0.9.4/go.mod h1:NzQcC2o19xgwGqCmjx7DN+4h2F13qPCZ9UJmweYzsnU= github.com/libp2p/go-libp2p v0.10.0 h1:7ooOvK1wi8eLpyTppy8TeH43UHy5uI75GAHGJxenUi0= github.com/libp2p/go-libp2p v0.10.0/go.mod h1:yBJNpb+mGJdgrwbKAKrhPU0u3ogyNFTfjJ6bdM+Q/G8= github.com/libp2p/go-libp2p-autonat v0.0.2/go.mod h1:fs71q5Xk+pdnKU014o2iq1RhMs9/PMaG5zXRFNnIIT4= @@ -852,12 +849,10 @@ github.com/libp2p/go-libp2p-pnet v0.2.0/go.mod h1:Qqvq6JH/oMZGwqs3N1Fqhv8NVhrdYc github.com/libp2p/go-libp2p-protocol v0.0.1/go.mod h1:Af9n4PiruirSDjHycM1QuiMi/1VZNHYcK8cLgFJLZ4s= github.com/libp2p/go-libp2p-protocol v0.1.0/go.mod h1:KQPHpAabB57XQxGrXCNvbL6UEXfQqUgC/1adR2Xtflk= github.com/libp2p/go-libp2p-pubsub v0.1.1/go.mod h1:ZwlKzRSe1eGvSIdU5bD7+8RZN/Uzw0t1Bp9R1znpR/Q= -github.com/libp2p/go-libp2p-pubsub v0.3.1/go.mod h1:TxPOBuo1FPdsTjFnv+FGZbNbWYsp74Culx+4ViQpato= github.com/libp2p/go-libp2p-pubsub v0.3.2-0.20200527132641-c0712c6e92cf/go.mod h1:TxPOBuo1FPdsTjFnv+FGZbNbWYsp74Culx+4ViQpato= github.com/libp2p/go-libp2p-pubsub v0.3.2 h1:k3cJm5JW5mjaWZkobS50sJLJWaB2mBi0HW4eRlE8mSo= github.com/libp2p/go-libp2p-pubsub v0.3.2/go.mod h1:Uss7/Cfz872KggNb+doCVPHeCDmXB7z500m/R8DaAUk= github.com/libp2p/go-libp2p-quic-transport v0.1.1/go.mod h1:wqG/jzhF3Pu2NrhJEvE+IE0NTHNXslOPn9JQzyCAxzU= -github.com/libp2p/go-libp2p-quic-transport v0.3.7/go.mod h1:Kr4aDtnfHHNeENn5J+sZIVc+t8HpQn9W6BOxhVGHbgI= github.com/libp2p/go-libp2p-quic-transport v0.5.0 h1:BUN1lgYNUrtv4WLLQ5rQmC9MCJ6uEXusezGvYRNoJXE= github.com/libp2p/go-libp2p-quic-transport v0.5.0/go.mod h1:IEcuC5MLxvZ5KuHKjRu+dr3LjCT1Be3rcD/4d8JrX8M= github.com/libp2p/go-libp2p-record v0.0.1/go.mod h1:grzqg263Rug/sRex85QrDOLntdFAymLDLm7lxMgU79Q= @@ -883,8 +878,6 @@ github.com/libp2p/go-libp2p-swarm v0.2.1/go.mod h1:x07b4zkMFo2EvgPV2bMTlNmdQc8i+ github.com/libp2p/go-libp2p-swarm v0.2.2/go.mod h1:fvmtQ0T1nErXym1/aa1uJEyN7JzaTNyBcHImCxRpPKU= github.com/libp2p/go-libp2p-swarm v0.2.3/go.mod h1:P2VO/EpxRyDxtChXz/VPVXyTnszHvokHKRhfkEgFKNM= github.com/libp2p/go-libp2p-swarm v0.2.4/go.mod h1:/xIpHFPPh3wmSthtxdGbkHZ0OET1h/GGZes8Wku/M5Y= -github.com/libp2p/go-libp2p-swarm v0.2.6 h1:UhMXIa+yCOALQyceENEIStMlbTCzOM6aWo6vw8QW17Q= -github.com/libp2p/go-libp2p-swarm v0.2.6/go.mod h1:F9hrkZjO7dDbcEiYii/fAB1QdpLuU6h1pa4P5VNsEgc= github.com/libp2p/go-libp2p-swarm v0.2.7 h1:4lV/sf7f0NuVqunOpt1I11+Z54+xp+m0eeAvxj/LyRc= github.com/libp2p/go-libp2p-swarm v0.2.7/go.mod h1:ZSJ0Q+oq/B1JgfPHJAT2HTall+xYRNYp1xs4S2FBWKA= github.com/libp2p/go-libp2p-testing v0.0.1/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= @@ -985,7 +978,6 @@ github.com/libp2p/go-yamux v1.3.7/go.mod h1:fr7aVgmdNGJK+N1g+b6DW6VxzbRCjCOejR/h github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/lucas-clemente/quic-go v0.11.2/go.mod h1:PpMmPfPKO9nKJ/psF49ESTAGQSdfXxlg1otPbEB2nOw= -github.com/lucas-clemente/quic-go v0.15.7/go.mod h1:Myi1OyS0FOjL3not4BxT7KN29bRkcMUV5JVVFLKtDp8= github.com/lucas-clemente/quic-go v0.16.0 h1:jJw36wfzGJhmOhAOaOC2lS36WgeqXQszH47A7spo1LI= github.com/lucas-clemente/quic-go v0.16.0/go.mod h1:I0+fcNTdb9eS1ZcjQZbDVPGchJ86chcIxPALn9lEJqE= github.com/lufia/iostat v1.1.0/go.mod h1:rEPNA0xXgjHQjuI5Cy05sLlS2oRcSlWHRLrvh/AQ+Pg= @@ -1309,6 +1301,7 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= @@ -1797,6 +1790,7 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= From 9e70e9524231ae98375a2ac93b3be4bd22e78992 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 23 Jun 2020 13:21:01 +0200 Subject: [PATCH 21/31] sync: Correctly pass blocks to ValidateBlock --- chain/sync.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/chain/sync.go b/chain/sync.go index 19ddac108..f20637c9e 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -468,6 +468,8 @@ func (syncer *Syncer) ValidateTipSet(ctx context.Context, fts *store.FullTipSet) var futures []async.ErrorFuture for _, b := range fts.Blocks { + b := b // rebind to a scoped variable + futures = append(futures, async.Err(func() error { if err := syncer.ValidateBlock(ctx, b); err != nil { if isPermanent(err) { From 566a99240d9314f925b7aa25c86812960182a818 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Tue, 23 Jun 2020 13:23:04 -0400 Subject: [PATCH 22/31] allow overriding drand config --- chain/beacon/drand/drand.go | 27 ++++++++++++++++++--------- node/modules/services.go | 3 ++- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/chain/beacon/drand/drand.go b/chain/beacon/drand/drand.go index 21bd2501a..91fc31aad 100644 --- a/chain/beacon/drand/drand.go +++ b/chain/beacon/drand/drand.go @@ -27,18 +27,22 @@ import ( var log = logging.Logger("drand") -var drandServers = []string{ - "https://pl-eu.testnet.drand.sh", - "https://pl-us.testnet.drand.sh", - "https://pl-sin.testnet.drand.sh", +type DrandConfig struct { + Servers []string + ChainInfo *dchain.Info } -var drandChain *dchain.Info +var defaultConfig = DrandConfig{ + Servers: []string{ + "https://pl-eu.testnet.drand.sh", + "https://pl-us.testnet.drand.sh", + "https://pl-sin.testnet.drand.sh", + }, +} func init() { - var err error - drandChain, err = dchain.InfoFromJSON(bytes.NewReader([]byte(build.DrandChain))) + defaultConfig.ChainInfo, err = dchain.InfoFromJSON(bytes.NewReader([]byte(build.DrandChain))) if err != nil { panic("could not unmarshal chain info: " + err.Error()) } @@ -73,16 +77,21 @@ type DrandBeacon struct { localCache map[uint64]types.BeaconEntry } -func NewDrandBeacon(genesisTs, interval uint64, ps *pubsub.PubSub) (*DrandBeacon, error) { +func NewDrandBeacon(genesisTs, interval uint64, ps *pubsub.PubSub, config *DrandConfig) (*DrandBeacon, error) { if genesisTs == 0 { panic("what are you doing this cant be zero") } + if config == nil { + config = &defaultConfig + } + drandChain := config.ChainInfo + dlogger := dlog.NewKitLoggerFrom(kzap.NewZapSugarLogger( log.SugaredLogger.Desugar(), zapcore.InfoLevel)) var clients []dclient.Client - for _, url := range drandServers { + for _, url := range config.Servers { hc, err := hclient.NewWithInfo(url, drandChain, nil) if err != nil { return nil, xerrors.Errorf("could not create http drand client: %w", err) diff --git a/node/modules/services.go b/node/modules/services.go index 27244ad3a..cbd0fb882 100644 --- a/node/modules/services.go +++ b/node/modules/services.go @@ -108,6 +108,7 @@ func RetrievalResolver(l *discovery.Local) retrievalmarket.PeerResolver { type RandomBeaconParams struct { fx.In + DrandConfig *drand.DrandConfig `optional:"true"` PubSub *pubsub.PubSub `optional:"true"` Cs *store.ChainStore } @@ -119,5 +120,5 @@ func RandomBeacon(p RandomBeaconParams, _ dtypes.AfterGenesisSet) (beacon.Random } //return beacon.NewMockBeacon(build.BlockDelay * time.Second) - return drand.NewDrandBeacon(gen.Timestamp, build.BlockDelay, p.PubSub) + return drand.NewDrandBeacon(gen.Timestamp, build.BlockDelay, p.PubSub, p.DrandConfig) } From 8adc831a3174cb47dc90ab8bfee83a01470bdcec Mon Sep 17 00:00:00 2001 From: laser Date: Tue, 23 Jun 2020 11:08:04 -0700 Subject: [PATCH 23/31] return error if retrieval deal rejected --- node/impl/client/client.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/node/impl/client/client.go b/node/impl/client/client.go index df6fb862c..9f8dd6bb9 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -348,6 +348,8 @@ func (a *API) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, ref unsubscribe := a.Retrieval.SubscribeToEvents(func(event retrievalmarket.ClientEvent, state retrievalmarket.ClientDealState) { if state.PayloadCID.Equals(order.Root) { switch state.Status { + case retrievalmarket.DealStatusRejected: + retrievalResult <- xerrors.Errorf("Retrieval Proposal Rejected: %s", state.Message) case retrievalmarket.DealStatusFailed, retrievalmarket.DealStatusErrored: retrievalResult <- xerrors.Errorf("Retrieval Error: %s", state.Message) case retrievalmarket.DealStatusCompleted: From fb04b17fad43ac9c8cd7d08c856f4ce2900d90ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 23 Jun 2020 20:55:13 +0200 Subject: [PATCH 24/31] state: Get correct locked table in StateMarketParticipants --- node/impl/full/state.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/node/impl/full/state.go b/node/impl/full/state.go index e39527201..048ae7858 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -425,7 +425,7 @@ func (a *StateAPI) StateMarketParticipants(ctx context.Context, tsk types.TipSet if err != nil { return nil, err } - locked, err := hamt.LoadNode(ctx, cst, state.EscrowTable, hamt.UseTreeBitWidth(5)) + locked, err := hamt.LoadNode(ctx, cst, state.LockedTable, hamt.UseTreeBitWidth(5)) if err != nil { return nil, err } @@ -489,13 +489,11 @@ func (a *StateAPI) StateMarketDeals(ctx context.Context, tsk types.TipSetKey) (m var s market.DealState if err := sa.Get(ctx, i, &s); err != nil { - if err != nil { - if _, ok := err.(*amt.ErrNotFound); !ok { - return xerrors.Errorf("failed to get state for deal in proposals array: %w", err) - } - - s.SectorStartEpoch = -1 + if _, ok := err.(*amt.ErrNotFound); !ok { + return xerrors.Errorf("failed to get state for deal in proposals array: %w", err) } + + s.SectorStartEpoch = -1 } out[strconv.FormatInt(int64(i), 10)] = api.MarketDeal{ Proposal: d, From 960523f45f87c8925ad9ac268b595917e4527361 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Tue, 23 Jun 2020 15:09:28 -0400 Subject: [PATCH 25/31] fix drand test --- chain/beacon/drand/drand_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chain/beacon/drand/drand_test.go b/chain/beacon/drand/drand_test.go index f35f3e4cc..219c86990 100644 --- a/chain/beacon/drand/drand_test.go +++ b/chain/beacon/drand/drand_test.go @@ -10,7 +10,7 @@ import ( ) func TestPrintGroupInfo(t *testing.T) { - c, err := hclient.New(drandServers[0], nil, nil) + c, err := hclient.New(defaultConfig.Servers[0], nil, nil) assert.NoError(t, err) cg := c.(interface { FetchChainInfo(groupHash []byte) (*dchain.Info, error) From 628872d0e425f3d38af8c19f345a846288dfcbf7 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Tue, 23 Jun 2020 15:10:27 -0400 Subject: [PATCH 26/31] forgot my go fmt hook --- chain/beacon/drand/drand.go | 2 +- node/modules/services.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/chain/beacon/drand/drand.go b/chain/beacon/drand/drand.go index 91fc31aad..c800366d1 100644 --- a/chain/beacon/drand/drand.go +++ b/chain/beacon/drand/drand.go @@ -28,7 +28,7 @@ import ( var log = logging.Logger("drand") type DrandConfig struct { - Servers []string + Servers []string ChainInfo *dchain.Info } diff --git a/node/modules/services.go b/node/modules/services.go index cbd0fb882..c5d61d580 100644 --- a/node/modules/services.go +++ b/node/modules/services.go @@ -109,8 +109,8 @@ type RandomBeaconParams struct { fx.In DrandConfig *drand.DrandConfig `optional:"true"` - PubSub *pubsub.PubSub `optional:"true"` - Cs *store.ChainStore + PubSub *pubsub.PubSub `optional:"true"` + Cs *store.ChainStore } func RandomBeacon(p RandomBeaconParams, _ dtypes.AfterGenesisSet) (beacon.RandomBeacon, error) { From 06162290af86de9cb3a438bbb491e603104a7f0f Mon Sep 17 00:00:00 2001 From: laser Date: Tue, 23 Jun 2020 12:14:41 -0700 Subject: [PATCH 27/31] explicitly handle each deal status, as per PR feedback --- node/impl/client/client.go | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 9f8dd6bb9..72beda7ac 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -348,12 +348,34 @@ func (a *API) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, ref unsubscribe := a.Retrieval.SubscribeToEvents(func(event retrievalmarket.ClientEvent, state retrievalmarket.ClientDealState) { if state.PayloadCID.Equals(order.Root) { switch state.Status { - case retrievalmarket.DealStatusRejected: - retrievalResult <- xerrors.Errorf("Retrieval Proposal Rejected: %s", state.Message) - case retrievalmarket.DealStatusFailed, retrievalmarket.DealStatusErrored: - retrievalResult <- xerrors.Errorf("Retrieval Error: %s", state.Message) - case retrievalmarket.DealStatusCompleted: + case + retrievalmarket.DealStatusCompleted: retrievalResult <- nil + case + retrievalmarket.DealStatusRejected: + retrievalResult <- xerrors.Errorf("Retrieval Proposal Rejected: %s", state.Message) + case + retrievalmarket.DealStatusDealNotFound, + retrievalmarket.DealStatusErrored, + retrievalmarket.DealStatusFailed: + retrievalResult <- xerrors.Errorf("Retrieval Error: %s", state.Message) + case + retrievalmarket.DealStatusAccepted, + retrievalmarket.DealStatusAwaitingAcceptance, + retrievalmarket.DealStatusBlocksComplete, + retrievalmarket.DealStatusFinalizing, + retrievalmarket.DealStatusFundsNeeded, + retrievalmarket.DealStatusFundsNeededLastPayment, + retrievalmarket.DealStatusNew, + retrievalmarket.DealStatusOngoing, + retrievalmarket.DealStatusPaymentChannelAddingFunds, + retrievalmarket.DealStatusPaymentChannelAllocatingLane, + retrievalmarket.DealStatusPaymentChannelCreating, + retrievalmarket.DealStatusPaymentChannelReady, + retrievalmarket.DealStatusVerified: + return + default: + retrievalResult <- xerrors.Errorf("Unhandled Retrieval Status: %+v", state.Status) } } }) From 309fbc15b2399421800a17fc25d4fad19e503f92 Mon Sep 17 00:00:00 2001 From: laser Date: Tue, 23 Jun 2020 12:22:33 -0700 Subject: [PATCH 28/31] import aliasing, for legibility --- node/impl/client/client.go | 62 ++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 32 deletions(-) diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 72beda7ac..6e7f50f26 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -31,7 +31,7 @@ import ( "go.uber.org/fx" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-fil-markets/retrievalmarket" + rm "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/sector-storage/ffiwrapper" "github.com/filecoin-project/specs-actors/actors/abi" @@ -59,8 +59,8 @@ type API struct { paych.PaychAPI SMDealClient storagemarket.StorageClient - RetDiscovery retrievalmarket.PeerResolver - Retrieval retrievalmarket.RetrievalClient + RetDiscovery rm.PeerResolver + Retrieval rm.RetrievalClient Chain *store.ChainStore LocalDAG dtypes.ClientDAG @@ -202,7 +202,7 @@ func (a *API) ClientFindData(ctx context.Context, root cid.Cid) ([]api.QueryOffe out := make([]api.QueryOffer, len(peers)) for k, p := range peers { - out[k] = a.makeRetrievalQuery(ctx, p, root, retrievalmarket.QueryParams{}) + out[k] = a.makeRetrievalQuery(ctx, p, root, rm.QueryParams{}) } return out, nil @@ -213,25 +213,25 @@ func (a *API) ClientMinerQueryOffer(ctx context.Context, payload cid.Cid, miner if err != nil { return api.QueryOffer{}, err } - rp := retrievalmarket.RetrievalPeer{ + rp := rm.RetrievalPeer{ Address: miner, ID: mi.PeerId, } - return a.makeRetrievalQuery(ctx, rp, payload, retrievalmarket.QueryParams{}), nil + return a.makeRetrievalQuery(ctx, rp, payload, rm.QueryParams{}), nil } -func (a *API) makeRetrievalQuery(ctx context.Context, rp retrievalmarket.RetrievalPeer, payload cid.Cid, qp retrievalmarket.QueryParams) api.QueryOffer { +func (a *API) makeRetrievalQuery(ctx context.Context, rp rm.RetrievalPeer, payload cid.Cid, qp rm.QueryParams) api.QueryOffer { queryResponse, err := a.Retrieval.Query(ctx, rp, payload, qp) if err != nil { return api.QueryOffer{Err: err.Error(), Miner: rp.Address, MinerPeerID: rp.ID} } var errStr string switch queryResponse.Status { - case retrievalmarket.QueryResponseAvailable: + case rm.QueryResponseAvailable: errStr = "" - case retrievalmarket.QueryResponseUnavailable: + case rm.QueryResponseUnavailable: errStr = fmt.Sprintf("retrieval query offer was unavailable: %s", queryResponse.Message) - case retrievalmarket.QueryResponseError: + case rm.QueryResponseError: errStr = fmt.Sprintf("retrieval query offer errored: %s", queryResponse.Message) } @@ -345,34 +345,32 @@ func (a *API) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, ref retrievalResult := make(chan error, 1) - unsubscribe := a.Retrieval.SubscribeToEvents(func(event retrievalmarket.ClientEvent, state retrievalmarket.ClientDealState) { + unsubscribe := a.Retrieval.SubscribeToEvents(func(event rm.ClientEvent, state rm.ClientDealState) { if state.PayloadCID.Equals(order.Root) { switch state.Status { - case - retrievalmarket.DealStatusCompleted: + case rm.DealStatusCompleted: retrievalResult <- nil - case - retrievalmarket.DealStatusRejected: + case rm.DealStatusRejected: retrievalResult <- xerrors.Errorf("Retrieval Proposal Rejected: %s", state.Message) case - retrievalmarket.DealStatusDealNotFound, - retrievalmarket.DealStatusErrored, - retrievalmarket.DealStatusFailed: + rm.DealStatusDealNotFound, + rm.DealStatusErrored, + rm.DealStatusFailed: retrievalResult <- xerrors.Errorf("Retrieval Error: %s", state.Message) case - retrievalmarket.DealStatusAccepted, - retrievalmarket.DealStatusAwaitingAcceptance, - retrievalmarket.DealStatusBlocksComplete, - retrievalmarket.DealStatusFinalizing, - retrievalmarket.DealStatusFundsNeeded, - retrievalmarket.DealStatusFundsNeededLastPayment, - retrievalmarket.DealStatusNew, - retrievalmarket.DealStatusOngoing, - retrievalmarket.DealStatusPaymentChannelAddingFunds, - retrievalmarket.DealStatusPaymentChannelAllocatingLane, - retrievalmarket.DealStatusPaymentChannelCreating, - retrievalmarket.DealStatusPaymentChannelReady, - retrievalmarket.DealStatusVerified: + rm.DealStatusAccepted, + rm.DealStatusAwaitingAcceptance, + rm.DealStatusBlocksComplete, + rm.DealStatusFinalizing, + rm.DealStatusFundsNeeded, + rm.DealStatusFundsNeededLastPayment, + rm.DealStatusNew, + rm.DealStatusOngoing, + rm.DealStatusPaymentChannelAddingFunds, + rm.DealStatusPaymentChannelAllocatingLane, + rm.DealStatusPaymentChannelCreating, + rm.DealStatusPaymentChannelReady, + rm.DealStatusVerified: return default: retrievalResult <- xerrors.Errorf("Unhandled Retrieval Status: %+v", state.Status) @@ -385,7 +383,7 @@ func (a *API) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, ref _, err := a.Retrieval.Retrieve( ctx, order.Root, - retrievalmarket.NewParamsV0(ppb, order.PaymentInterval, order.PaymentIntervalIncrease), + rm.NewParamsV0(ppb, order.PaymentInterval, order.PaymentIntervalIncrease), order.Total, order.MinerPeerID, order.Client, From b448de422e1f929b02a9ddd719d594c66046d438 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Tue, 23 Jun 2020 15:56:03 -0400 Subject: [PATCH 29/31] improve DrandConfig dependency injection --- build/params_shared.go | 9 ++++++++- chain/beacon/drand/drand.go | 32 ++++++++------------------------ chain/beacon/drand/drand_test.go | 5 ++++- node/builder.go | 1 + node/modules/dtypes/beacon.go | 6 ++++++ node/modules/lp2p/pubsub.go | 7 ++++--- node/modules/services.go | 14 +++++++++++--- 7 files changed, 42 insertions(+), 32 deletions(-) create mode 100644 node/modules/dtypes/beacon.go diff --git a/build/params_shared.go b/build/params_shared.go index 4da70aaeb..5ab07bd3f 100644 --- a/build/params_shared.go +++ b/build/params_shared.go @@ -121,4 +121,11 @@ const VerifSigCacheSize = 32000 const BlockMessageLimit = 512 const BlockGasLimit = 100_000_000_000 -var DrandChain = `{"public_key":"922a2e93828ff83345bae533f5172669a26c02dc76d6bf59c80892e12ab1455c229211886f35bb56af6d5bea981024df","period":25,"genesis_time":1590445175,"hash":"138a324aa6540f93d0dad002aa89454b1bec2b6e948682cde6bd4db40f4b7c9b"}` +var DrandConfig = dtypes.DrandConfig{ + Servers: []string{ + "https://pl-eu.testnet.drand.sh", + "https://pl-us.testnet.drand.sh", + "https://pl-sin.testnet.drand.sh", + }, + ChainInfoJSON: `{"public_key":"922a2e93828ff83345bae533f5172669a26c02dc76d6bf59c80892e12ab1455c229211886f35bb56af6d5bea981024df","period":25,"genesis_time":1590445175,"hash":"138a324aa6540f93d0dad002aa89454b1bec2b6e948682cde6bd4db40f4b7c9b"}`, +} diff --git a/chain/beacon/drand/drand.go b/chain/beacon/drand/drand.go index c800366d1..9769a8436 100644 --- a/chain/beacon/drand/drand.go +++ b/chain/beacon/drand/drand.go @@ -19,33 +19,17 @@ import ( logging "github.com/ipfs/go-log" pubsub "github.com/libp2p/go-libp2p-pubsub" - "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/lotus/chain/beacon" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/specs-actors/actors/abi" ) var log = logging.Logger("drand") type DrandConfig struct { - Servers []string - ChainInfo *dchain.Info -} - -var defaultConfig = DrandConfig{ - Servers: []string{ - "https://pl-eu.testnet.drand.sh", - "https://pl-us.testnet.drand.sh", - "https://pl-sin.testnet.drand.sh", - }, -} - -func init() { - var err error - defaultConfig.ChainInfo, err = dchain.InfoFromJSON(bytes.NewReader([]byte(build.DrandChain))) - if err != nil { - panic("could not unmarshal chain info: " + err.Error()) - } + Servers []string + ChainInfoJSON string } type drandPeer struct { @@ -77,15 +61,15 @@ type DrandBeacon struct { localCache map[uint64]types.BeaconEntry } -func NewDrandBeacon(genesisTs, interval uint64, ps *pubsub.PubSub, config *DrandConfig) (*DrandBeacon, error) { +func NewDrandBeacon(genesisTs, interval uint64, ps *pubsub.PubSub, config DrandConfig) (*DrandBeacon, error) { if genesisTs == 0 { panic("what are you doing this cant be zero") } - if config == nil { - config = &defaultConfig + drandChain, err := dchain.InfoFromJSON(bytes.NewReader([]byte(config.ChainInfoJSON))) + if err != nil { + return nil, xerrors.Errorf("unable to unmarshal drand chain info: %w", err) } - drandChain := config.ChainInfo dlogger := dlog.NewKitLoggerFrom(kzap.NewZapSugarLogger( log.SugaredLogger.Desugar(), zapcore.InfoLevel)) diff --git a/chain/beacon/drand/drand_test.go b/chain/beacon/drand/drand_test.go index 219c86990..d7d9c4d18 100644 --- a/chain/beacon/drand/drand_test.go +++ b/chain/beacon/drand/drand_test.go @@ -7,10 +7,13 @@ import ( dchain "github.com/drand/drand/chain" hclient "github.com/drand/drand/client/http" "github.com/stretchr/testify/assert" + + "github.com/filecoin-project/lotus/build" ) func TestPrintGroupInfo(t *testing.T) { - c, err := hclient.New(defaultConfig.Servers[0], nil, nil) + server := build.DrandConfig.Servers[0] + c, err := hclient.New(server, nil, nil) assert.NoError(t, err) cg := c.(interface { FetchChainInfo(groupHash []byte) (*dchain.Info, error) diff --git a/node/builder.go b/node/builder.go index e84c4422c..3b6c6fbda 100644 --- a/node/builder.go +++ b/node/builder.go @@ -218,6 +218,7 @@ func Online() Option { Override(new(dtypes.BootstrapPeers), modules.BuiltinBootstrap), Override(new(dtypes.DrandBootstrap), modules.DrandBootstrap), + Override(new(dtypes.DrandConfig), modules.BuiltinDrandConfig), Override(HandleIncomingMessagesKey, modules.HandleIncomingMessages), diff --git a/node/modules/dtypes/beacon.go b/node/modules/dtypes/beacon.go new file mode 100644 index 000000000..f3ebc4fac --- /dev/null +++ b/node/modules/dtypes/beacon.go @@ -0,0 +1,6 @@ +package dtypes + +type DrandConfig struct { + Servers []string + ChainInfoJSON string +} diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go index bf9d88d42..ac23f56a8 100644 --- a/node/modules/lp2p/pubsub.go +++ b/node/modules/lp2p/pubsub.go @@ -44,13 +44,14 @@ type GossipIn struct { Db dtypes.DrandBootstrap Cfg *config.Pubsub Sk *dtypes.ScoreKeeper + Dr dtypes.DrandConfig } -func getDrandTopic() (string, error) { +func getDrandTopic(chainInfoJSON string) (string, error) { var drandInfo = struct { Hash string `json:"hash"` }{} - err := json.Unmarshal([]byte(build.DrandChain), &drandInfo) + err := json.Unmarshal([]byte(chainInfoJSON), &drandInfo) if err != nil { return "", xerrors.Errorf("could not unmarshal drand chain info: %w", err) } @@ -68,7 +69,7 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) { } isBootstrapNode := in.Cfg.Bootstrapper - drandTopic, err := getDrandTopic() + drandTopic, err := getDrandTopic(in.Dr.ChainInfoJSON) if err != nil { return nil, err } diff --git a/node/modules/services.go b/node/modules/services.go index c5d61d580..ad7eb056f 100644 --- a/node/modules/services.go +++ b/node/modules/services.go @@ -108,9 +108,13 @@ func RetrievalResolver(l *discovery.Local) retrievalmarket.PeerResolver { type RandomBeaconParams struct { fx.In - DrandConfig *drand.DrandConfig `optional:"true"` - PubSub *pubsub.PubSub `optional:"true"` + PubSub *pubsub.PubSub `optional:"true"` Cs *store.ChainStore + DrandConfig dtypes.DrandConfig +} + +func BuiltinDrandConfig() dtypes.DrandConfig { + return build.DrandConfig } func RandomBeacon(p RandomBeaconParams, _ dtypes.AfterGenesisSet) (beacon.RandomBeacon, error) { @@ -120,5 +124,9 @@ func RandomBeacon(p RandomBeaconParams, _ dtypes.AfterGenesisSet) (beacon.Random } //return beacon.NewMockBeacon(build.BlockDelay * time.Second) - return drand.NewDrandBeacon(gen.Timestamp, build.BlockDelay, p.PubSub, p.DrandConfig) + config := drand.DrandConfig{ + Servers: p.DrandConfig.Servers, + ChainInfoJSON: p.DrandConfig.ChainInfoJSON, + } + return drand.NewDrandBeacon(gen.Timestamp, build.BlockDelay, p.PubSub, config) } From 5074cf8beb2947c2615a73d5506df8a8974423e8 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Tue, 23 Jun 2020 16:23:06 -0400 Subject: [PATCH 30/31] import DrandConfig from dtypes --- chain/beacon/drand/drand.go | 8 ++------ node/modules/services.go | 6 +----- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/chain/beacon/drand/drand.go b/chain/beacon/drand/drand.go index 9769a8436..eb51a2af3 100644 --- a/chain/beacon/drand/drand.go +++ b/chain/beacon/drand/drand.go @@ -23,15 +23,11 @@ import ( "github.com/filecoin-project/lotus/chain/beacon" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/node/modules/dtypes" ) var log = logging.Logger("drand") -type DrandConfig struct { - Servers []string - ChainInfoJSON string -} - type drandPeer struct { addr string tls bool @@ -61,7 +57,7 @@ type DrandBeacon struct { localCache map[uint64]types.BeaconEntry } -func NewDrandBeacon(genesisTs, interval uint64, ps *pubsub.PubSub, config DrandConfig) (*DrandBeacon, error) { +func NewDrandBeacon(genesisTs, interval uint64, ps *pubsub.PubSub, config dtypes.DrandConfig) (*DrandBeacon, error) { if genesisTs == 0 { panic("what are you doing this cant be zero") } diff --git a/node/modules/services.go b/node/modules/services.go index ad7eb056f..2cba3a0be 100644 --- a/node/modules/services.go +++ b/node/modules/services.go @@ -124,9 +124,5 @@ func RandomBeacon(p RandomBeaconParams, _ dtypes.AfterGenesisSet) (beacon.Random } //return beacon.NewMockBeacon(build.BlockDelay * time.Second) - config := drand.DrandConfig{ - Servers: p.DrandConfig.Servers, - ChainInfoJSON: p.DrandConfig.ChainInfoJSON, - } - return drand.NewDrandBeacon(gen.Timestamp, build.BlockDelay, p.PubSub, config) + return drand.NewDrandBeacon(gen.Timestamp, build.BlockDelay, p.PubSub, p.DrandConfig) } From c8104a03e61f56f17c4a39d5efbf0ad23069dc07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Tue, 23 Jun 2020 22:51:25 +0100 Subject: [PATCH 31/31] some initial godocs. (#2118) --- chain/beacon/beacon.go | 4 + chain/beacon/drand/drand.go | 7 ++ chain/blocksync/blocksync.go | 19 +++++ chain/blocksync/blocksync_client.go | 8 ++ chain/store/fts.go | 3 + chain/store/index.go | 2 +- chain/store/store.go | 19 +++++ chain/sync.go | 127 ++++++++++++++++++++++++++-- miner/miner.go | 6 ++ 9 files changed, 188 insertions(+), 7 deletions(-) diff --git a/chain/beacon/beacon.go b/chain/beacon/beacon.go index 34405f3c8..2be2e7f1c 100644 --- a/chain/beacon/beacon.go +++ b/chain/beacon/beacon.go @@ -17,6 +17,10 @@ type Response struct { Err error } +// RandomBeacon represents a system that provides randomness to Lotus. +// Other components interrogate the RandomBeacon to acquire randomness that's +// valid for a specific chain epoch. Also to verify beacon entries that have +// been posted on chain. type RandomBeacon interface { Entry(context.Context, uint64) <-chan Response VerifyEntry(types.BeaconEntry, types.BeaconEntry) error diff --git a/chain/beacon/drand/drand.go b/chain/beacon/drand/drand.go index eb51a2af3..00ff05f81 100644 --- a/chain/beacon/drand/drand.go +++ b/chain/beacon/drand/drand.go @@ -41,6 +41,13 @@ func (dp *drandPeer) IsTLS() bool { return dp.tls } +// DrandBeacon connects Lotus with a drand network in order to provide +// randomness to the system in a way that's aligned with Filecoin rounds/epochs. +// +// We connect to drand peers via their public HTTP endpoints. The peers are +// enumerated in the drandServers variable. +// +// The root trust for the Drand chain is configured from build.DrandChain. type DrandBeacon struct { client dclient.Client diff --git a/chain/blocksync/blocksync.go b/chain/blocksync/blocksync.go index daca9ce20..a9251c419 100644 --- a/chain/blocksync/blocksync.go +++ b/chain/blocksync/blocksync.go @@ -10,6 +10,7 @@ import ( "golang.org/x/xerrors" cborutil "github.com/filecoin-project/go-cbor-util" + "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" @@ -27,6 +28,24 @@ const BlockSyncProtocolID = "/fil/sync/blk/0.0.1" const BlockSyncMaxRequestLength = 800 +// BlockSyncService is the component that services BlockSync requests from +// peers. +// +// BlockSync is the basic chain synchronization protocol of Filecoin. BlockSync +// is an RPC-oriented protocol, with a single operation to request blocks. +// +// A request contains a start anchor block (referred to with a CID), and a +// amount of blocks requested beyond the anchor (including the anchor itself). +// +// A client can also pass options, encoded as a 64-bit bitfield. Lotus supports +// two options at the moment: +// +// - include block contents +// - include block messages +// +// The response will include a status code, an optional message, and the +// response payload in case of success. The payload is a slice of serialized +// tipsets. type BlockSyncService struct { cs *store.ChainStore } diff --git a/chain/blocksync/blocksync_client.go b/chain/blocksync/blocksync_client.go index 129e8d332..daa4b6335 100644 --- a/chain/blocksync/blocksync_client.go +++ b/chain/blocksync/blocksync_client.go @@ -64,6 +64,11 @@ func (bs *BlockSync) processStatus(req *BlockSyncRequest, res *BlockSyncResponse } } +// GetBlocks fetches count blocks from the network, from the provided tipset +// *backwards*, returning as many tipsets as count. +// +// {hint/usage}: This is used by the Syncer during normal chain syncing and when +// resolving forks. func (bs *BlockSync) GetBlocks(ctx context.Context, tsk types.TipSetKey, count int) ([]*types.TipSet, error) { ctx, span := trace.StartSpan(ctx, "bsync.GetBlocks") defer span.End() @@ -80,7 +85,9 @@ func (bs *BlockSync) GetBlocks(ctx context.Context, tsk types.TipSetKey, count i Options: BSOptBlocks, } + // this peerset is sorted by latency and failure counting. peers := bs.getPeers() + // randomize the first few peers so we don't always pick the same peer shufflePrefix(peers) @@ -356,6 +363,7 @@ func (bs *BlockSync) RemovePeer(p peer.ID) { bs.syncPeers.removePeer(p) } +// getPeers returns a preference-sorted set of peers to query. func (bs *BlockSync) getPeers() []peer.ID { return bs.syncPeers.prefSortedPeers() } diff --git a/chain/store/fts.go b/chain/store/fts.go index f9ec4459e..0324938d7 100644 --- a/chain/store/fts.go +++ b/chain/store/fts.go @@ -32,8 +32,11 @@ func (fts *FullTipSet) Cids() []cid.Cid { return cids } +// TipSet returns a narrower view of this FullTipSet elliding the block +// messages. func (fts *FullTipSet) TipSet() *types.TipSet { if fts.tipset != nil { + // FIXME: fts.tipset is actually never set. Should it memoize? return fts.tipset } diff --git a/chain/store/index.go b/chain/store/index.go index bb363ec18..7edbf251f 100644 --- a/chain/store/index.go +++ b/chain/store/index.go @@ -34,7 +34,7 @@ type lbEntry struct { target types.TipSetKey } -func (ci *ChainIndex) GetTipsetByHeight(ctx context.Context, from *types.TipSet, to abi.ChainEpoch) (*types.TipSet, error) { +func (ci *ChainIndex) GetTipsetByHeight(_ context.Context, from *types.TipSet, to abi.ChainEpoch) (*types.TipSet, error) { if from.Height()-to <= ci.skipLength { return ci.walkBack(from, to) } diff --git a/chain/store/store.go b/chain/store/store.go index 0edccb95c..4dabb96f7 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -52,6 +52,15 @@ var blockValidationCacheKeyPrefix = dstore.NewKey("blockValidation") // ReorgNotifee represents a callback that gets called upon reorgs. type ReorgNotifee func(rev, app []*types.TipSet) error +// ChainStore is the main point of access to chain data. +// +// Raw chain data is stored in the Blockstore, with relevant markers (genesis, +// latest head tipset references) being tracked in the Datastore (key-value +// store). +// +// To alleviate disk access, the ChainStore has two ARC caches: +// 1. a tipset cache +// 2. a block => messages references cache. type ChainStore struct { bs bstore.Blockstore ds dstore.Datastore @@ -266,6 +275,9 @@ func (cs *ChainStore) PutTipSet(ctx context.Context, ts *types.TipSet) error { return nil } +// MaybeTakeHeavierTipSet evaluates the incoming tipset and locks it in our +// internal state as our new head, if and only if it is heavier than the current +// head. func (cs *ChainStore) MaybeTakeHeavierTipSet(ctx context.Context, ts *types.TipSet) error { cs.heaviestLk.Lock() defer cs.heaviestLk.Unlock() @@ -331,6 +343,9 @@ func (cs *ChainStore) reorgWorker(ctx context.Context, initialNotifees []ReorgNo return out } +// takeHeaviestTipSet actually sets the incoming tipset as our head both in +// memory and in the ChainStore. It also sends a notification to deliver to +// ReorgNotifees. func (cs *ChainStore) takeHeaviestTipSet(ctx context.Context, ts *types.TipSet) error { _, span := trace.StartSpan(ctx, "takeHeaviestTipSet") defer span.End() @@ -368,6 +383,7 @@ func (cs *ChainStore) SetHead(ts *types.TipSet) error { return cs.takeHeaviestTipSet(context.TODO(), ts) } +// Contains returns whether our BlockStore has all blocks in the supplied TipSet. func (cs *ChainStore) Contains(ts *types.TipSet) (bool, error) { for _, c := range ts.Cids() { has, err := cs.bs.Has(c) @@ -382,6 +398,8 @@ func (cs *ChainStore) Contains(ts *types.TipSet) (bool, error) { return true, nil } +// GetBlock fetches a BlockHeader with the supplied CID. It returns +// blockstore.ErrNotFound if the block was not found in the BlockStore. func (cs *ChainStore) GetBlock(c cid.Cid) (*types.BlockHeader, error) { sb, err := cs.bs.Get(c) if err != nil { @@ -474,6 +492,7 @@ func (cs *ChainStore) ReorgOps(a, b *types.TipSet) ([]*types.TipSet, []*types.Ti return leftChain, rightChain, nil } +// GetHeaviestTipSet returns the current heaviest tipset known (i.e. our head). func (cs *ChainStore) GetHeaviestTipSet() *types.TipSet { cs.heaviestLk.Lock() defer cs.heaviestLk.Unlock() diff --git a/chain/sync.go b/chain/sync.go index f20637c9e..0a08e8b15 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -53,6 +53,29 @@ var log = logging.Logger("chain") var LocalIncoming = "incoming" +// Syncer is in charge of running the chain synchronization logic. As such, it +// is tasked with these functions, amongst others: +// +// * Fast-forwards the chain as it learns of new TipSets from the network via +// the SyncManager. +// * Applies the fork choice rule to select the correct side when confronted +// with a fork in the network. +// * Requests block headers and messages from other peers when not available +// in our BlockStore. +// * Tracks blocks marked as bad in a cache. +// * Keeps the BlockStore and ChainStore consistent with our view of the world, +// the latter of which in turn informs other components when a reorg has been +// committed. +// +// The Syncer does not run workers itself. It's mainly concerned with +// ensuring a consistent state of chain consensus. The reactive and network- +// interfacing processes are part of other components, such as the SyncManager +// (which owns the sync scheduler and sync workers), BlockSync, the HELLO +// protocol, and the gossipsub block propagation layer. +// +// {hint/concept} The fork-choice rule as it currently stands is: "pick the +// chain with the heaviest weight, so long as it hasn’t deviated one finality +// threshold from our head (900 epochs, parameter determined by spec-actors)". type Syncer struct { // The interface for accessing and putting tipsets into local storage store *store.ChainStore @@ -85,6 +108,7 @@ type Syncer struct { verifier ffiwrapper.Verifier } +// NewSyncer creates a new Syncer object. func NewSyncer(sm *stmgr.StateManager, bsync *blocksync.BlockSync, connmgr connmgr.ConnManager, self peer.ID, beacon beacon.RandomBeacon, verifier ffiwrapper.Verifier) (*Syncer, error) { gen, err := sm.ChainStore().GetGenesis() if err != nil { @@ -182,6 +206,11 @@ func (syncer *Syncer) InformNewHead(from peer.ID, fts *store.FullTipSet) bool { return true } +// IncomingBlocks spawns a goroutine that subscribes to the local eventbus to +// receive new block headers as they arrive from the network, and sends them to +// the returned channel. +// +// These blocks have not necessarily been incorporated to our view of the chain. func (syncer *Syncer) IncomingBlocks(ctx context.Context) (<-chan *types.BlockHeader, error) { sub := syncer.incoming.Sub(LocalIncoming) out := make(chan *types.BlockHeader, 10) @@ -209,11 +238,15 @@ func (syncer *Syncer) IncomingBlocks(ctx context.Context) (<-chan *types.BlockHe return out, nil } +// ValidateMsgMeta performs structural and content hash validation of the +// messages within this block. If validation passes, it stores the messages in +// the underlying IPLD block store. func (syncer *Syncer) ValidateMsgMeta(fblk *types.FullBlock) error { if msgc := len(fblk.BlsMessages) + len(fblk.SecpkMessages); msgc > build.BlockMessageLimit { return xerrors.Errorf("block %s has too many messages (%d)", fblk.Header.Cid(), msgc) } + // Collect the CIDs of both types of messages separately: BLS and Secpk. var bcids, scids []cbg.CBORMarshaler for _, m := range fblk.BlsMessages { c := cbg.CborCid(m.Cid()) @@ -231,11 +264,14 @@ func (syncer *Syncer) ValidateMsgMeta(fblk *types.FullBlock) error { blockstore := syncer.store.Blockstore() bs := cbor.NewCborStore(blockstore) + + // Compute the root CID of the combined message trie. smroot, err := computeMsgMeta(bs, bcids, scids) if err != nil { return xerrors.Errorf("validating msgmeta, compute failed: %w", err) } + // Check that the message trie root matches with what's in the block. if fblk.Header.Messages != smroot { return xerrors.Errorf("messages in full block did not match msgmeta root in header (%s != %s)", fblk.Header.Messages, smroot) } @@ -345,6 +381,8 @@ func zipTipSetAndMessages(bs cbor.IpldStore, ts *types.TipSet, allbmsgs []*types return fts, nil } +// computeMsgMeta computes the root CID of the combined arrays of message CIDs +// of both types (BLS and Secpk). func computeMsgMeta(bs cbor.IpldStore, bmsgCids, smsgCids []cbg.CBORMarshaler) (cid.Cid, error) { ctx := context.TODO() bmroot, err := amt.FromArray(ctx, bs, bmsgCids) @@ -368,14 +406,24 @@ func computeMsgMeta(bs cbor.IpldStore, bmsgCids, smsgCids []cbg.CBORMarshaler) ( return mrcid, nil } +// FetchTipSet tries to load the provided tipset from the store, and falls back +// to the network (BlockSync) by querying the supplied peer if not found +// locally. +// +// {hint/usage} This is used from the HELLO protocol, to fetch the greeting +// peer's heaviest tipset if we don't have it. func (syncer *Syncer) FetchTipSet(ctx context.Context, p peer.ID, tsk types.TipSetKey) (*store.FullTipSet, error) { if fts, err := syncer.tryLoadFullTipSet(tsk); err == nil { return fts, nil } + // fall back to the network. return syncer.Bsync.GetFullTipSet(ctx, p, tsk) } +// tryLoadFullTipSet queries the tipset in the ChainStore, and returns a full +// representation of it containing FullBlocks. If ALL blocks are not found +// locally, it errors entirely with blockstore.ErrNotFound. func (syncer *Syncer) tryLoadFullTipSet(tsk types.TipSetKey) (*store.FullTipSet, error) { ts, err := syncer.store.LoadTipSet(tsk) if err != nil { @@ -400,6 +448,12 @@ func (syncer *Syncer) tryLoadFullTipSet(tsk types.TipSetKey) (*store.FullTipSet, return fts, nil } +// Sync tries to advance our view of the chain to `maybeHead`. It does nothing +// if our current head is heavier than the requested tipset, or if we're already +// at the requested head, or if the head is the genesis. +// +// Most of the heavy-lifting logic happens in syncer#collectChain. Refer to the +// godocs on that method for a more detailed view. func (syncer *Syncer) Sync(ctx context.Context, maybeHead *types.TipSet) error { ctx, span := trace.StartSpan(ctx, "chain.Sync") defer span.End() @@ -1004,6 +1058,39 @@ func extractSyncState(ctx context.Context) *SyncerState { return nil } +// collectHeaders collects the headers from the blocks between any two tipsets. +// +// `from` is the heaviest/projected/target tipset we have learned about, and +// `to` is usually an anchor tipset we already have in our view of the chain +// (which could be the genesis). +// +// collectHeaders checks if portions of the chain are in our ChainStore; falling +// down to the network to retrieve the missing parts. If during the process, any +// portion we receive is in our denylist (bad list), we short-circuit. +// +// {hint/naming}: `from` and `to` is in inverse order. `from` is the highest, +// and `to` is the lowest. This method traverses the chain backwards. +// +// {hint/usage}: This is used by collectChain, which is in turn called from the +// main Sync method (Syncer#Sync), so it's a pretty central method. +// +// {hint/logic}: The logic of this method is as follows: +// +// 1. Check that the from tipset is not linked to a parent block known to be +// bad. +// 2. Check the consistency of beacon entries in the from tipset. We check +// total equality of the BeaconEntries in each block. +// 3. Travers the chain backwards, for each tipset: +// 3a. Load it from the chainstore; if found, it move on to its parent. +// 3b. Query our peers via BlockSync in batches, requesting up to a +// maximum of 500 tipsets every time. +// +// Once we've concluded, if we find a mismatching tipset at the height where the +// anchor tipset should be, we are facing a fork, and we invoke Syncer#syncFork +// to resolve it. Refer to the godocs there. +// +// All throughout the process, we keep checking if the received blocks are in +// the deny list, and short-circuit the process if so. func (syncer *Syncer) collectHeaders(ctx context.Context, from *types.TipSet, to *types.TipSet) ([]*types.TipSet, error) { ctx, span := trace.StartSpan(ctx, "collectHeaders") defer span.End() @@ -1020,6 +1107,8 @@ func (syncer *Syncer) collectHeaders(ctx context.Context, from *types.TipSet, to } } + // Check if the parents of the from block are in the denylist. + // i.e. if a fork of the chain has been requested that we know to be bad. for _, pcid := range from.Parents().Cids() { if reason, ok := syncer.bad.Has(pcid); ok { markBad("linked to %s", pcid) @@ -1090,8 +1179,8 @@ loop: } // NB: GetBlocks validates that the blocks are in-fact the ones we - // requested, and that they are correctly linked to eachother. It does - // not validate any state transitions + // requested, and that they are correctly linked to one another. It does + // not validate any state transitions. window := 500 if gap := int(blockSet[len(blockSet)-1].Height() - untilHeight); gap < window { window = gap @@ -1132,7 +1221,6 @@ loop: at = blks[len(blks)-1].Parents() } - // We have now ascertained that this is *not* a 'fast forward' if !types.CidArrsEqual(blockSet[len(blockSet)-1].Parents().Cids(), to.Cids()) { last := blockSet[len(blockSet)-1] if last.Parents() == to.Parents() { @@ -1140,6 +1228,8 @@ loop: return blockSet, nil } + // We have now ascertained that this is *not* a 'fast forward' + log.Warnf("(fork detected) synced header chain (%s - %d) does not link to our best block (%s - %d)", from.Cids(), from.Height(), to.Cids(), to.Height()) fork, err := syncer.syncFork(ctx, last, to) if err != nil { @@ -1161,6 +1251,12 @@ loop: var ErrForkTooLong = fmt.Errorf("fork longer than threshold") +// syncFork tries to obtain the chain fragment that links a fork into a common +// ancestor in our view of the chain. +// +// If the fork is too long (build.ForkLengthThreshold), we add the entire subchain to the +// denylist. Else, we find the common ancestor, and add the missing chain +// fragment until the fork point to the returned []TipSet. func (syncer *Syncer) syncFork(ctx context.Context, from *types.TipSet, to *types.TipSet) ([]*types.TipSet, error) { tips, err := syncer.Bsync.GetBlocks(ctx, from.Parents(), int(build.ForkLengthThreshold)) if err != nil { @@ -1312,6 +1408,25 @@ func persistMessages(bs bstore.Blockstore, bst *blocksync.BSTipSet) error { return nil } +// collectChain tries to advance our view of the chain to the purported head. +// +// It goes through various stages: +// +// 1. StageHeaders: we proceed in the sync process by requesting block headers +// from our peers, moving back from their heads, until we reach a tipset +// that we have in common (such a common tipset must exist, thought it may +// simply be the genesis block). +// +// If the common tipset is our head, we treat the sync as a "fast-forward", +// else we must drop part of our chain to connect to the peer's head +// (referred to as "forking"). +// +// 2. StagePersistHeaders: now that we've collected the missing headers, +// augmented by those on the other side of a fork, we persist them to the +// BlockStore. +// +// 3. StageMessages: having acquired the headers and found a common tipset, +// we then move forward, requesting the full blocks, including the messages. func (syncer *Syncer) collectChain(ctx context.Context, ts *types.TipSet) error { ctx, span := trace.StartSpan(ctx, "collectChain") defer span.End() @@ -1361,9 +1476,8 @@ func (syncer *Syncer) collectChain(ctx context.Context, ts *types.TipSet) error func VerifyElectionPoStVRF(ctx context.Context, worker address.Address, rand []byte, evrf []byte) error { if build.InsecurePoStValidation { return nil - } else { - return gen.VerifyVRF(ctx, worker, rand, evrf) } + return gen.VerifyVRF(ctx, worker, rand, evrf) } func (syncer *Syncer) State() []SyncerState { @@ -1374,6 +1488,7 @@ func (syncer *Syncer) State() []SyncerState { return out } +// MarkBad manually adds a block to the "bad blocks" cache. func (syncer *Syncer) MarkBad(blk cid.Cid) { syncer.bad.Add(blk, "manually marked bad") } @@ -1381,7 +1496,7 @@ func (syncer *Syncer) MarkBad(blk cid.Cid) { func (syncer *Syncer) CheckBadBlockCache(blk cid.Cid) (string, bool) { return syncer.bad.Has(blk) } -func (syncer *Syncer) getLatestBeaconEntry(ctx context.Context, ts *types.TipSet) (*types.BeaconEntry, error) { +func (syncer *Syncer) getLatestBeaconEntry(_ context.Context, ts *types.TipSet) (*types.BeaconEntry, error) { cur := ts for i := 0; i < 20; i++ { cbe := cur.Blocks()[0].BeaconEntries diff --git a/miner/miner.go b/miner/miner.go index bdeed8ac5..fa97bd265 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -252,6 +252,12 @@ func (m *Miner) hasPower(ctx context.Context, addr address.Address, ts *types.Ti return mpower.MinerPower.QualityAdjPower.GreaterThanEqual(power.ConsensusMinerMinPower), nil } +// mineOne mines a single block, and does so synchronously, if and only if we +// have won the current round. +// +// {hint/landmark}: This method coordinates all the steps involved in mining a +// block, including the condition of whether mine or not at all depending on +// whether we win the round or not. func (m *Miner) mineOne(ctx context.Context, base *MiningBase) (*types.BlockMsg, error) { log.Debugw("attempting to mine a block", "tipset", types.LogCids(base.TipSet.Cids())) start := time.Now()