diff --git a/.circleci/config.yml b/.circleci/config.yml index b9aa747df..9bcd6e8d8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -100,7 +100,7 @@ jobs: description: gotestsum format. https://github.com/gotestyourself/gotestsum#format coverage: type: string - default: -coverprofile=coverage.txt + default: -coverprofile=coverage.txt -coverpkg=github.com/filecoin-project/lotus/... description: Coverage flag. Set to the empty string to disable. codecov-upload: type: boolean @@ -190,7 +190,7 @@ jobs: - install-deps - go/mod-download - run: - command: make buildall + command: make build no_output_timeout: 30m - store_artifacts: path: lotus @@ -253,7 +253,8 @@ workflows: jobs: - lint-changes: args: "--new-from-rev origin/master" - - test + - test: + codecov-upload: true - test-short: go-test-flags: "--timeout 10m --short" - mod-tidy-check diff --git a/.codecov.yml b/.codecov.yml index db2472009..cf409a6b6 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -1 +1,3 @@ comment: off +ignore: + - "cbor_gen.go" diff --git a/Makefile b/Makefile index 083a15d70..b81840767 100644 --- a/Makefile +++ b/Makefile @@ -76,6 +76,7 @@ BINS+=lotus-seal-worker lotus-shed: $(BUILD_DEPS) rm -f lotus-shed go build $(GOFLAGS) -o lotus-shed ./cmd/lotus-shed + go run github.com/GeertJohan/go.rice/rice append --exec lotus-shed -i ./build .PHONY: lotus-seal-worker BINS+=lotus-seal-worker @@ -144,6 +145,7 @@ BINS+=bench stats: rm -f stats go build -o stats ./tools/stats + go run github.com/GeertJohan/go.rice/rice append --exec stats -i ./build .PHONY: stats BINS+=stats diff --git a/api/api_storage.go b/api/api_storage.go index 8d0c39038..3fa2ab6d0 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -13,7 +13,8 @@ type SectorState = uint64 const ( UndefinedSectorState SectorState = iota - Empty // TODO: Is this useful + // happy path + Empty Packing // sector not in sealStore, and not on chain Unsealed // sealing / queued @@ -22,13 +23,32 @@ const ( Committing CommitWait // waiting for message to land on chain Proving + _ // reserved + _ + _ + _ + + // recovery handling + // Reseal + _ + _ + _ + _ + _ + _ + _ + + // error modes + FailedUnrecoverable SealFailed PreCommitFailed SealCommitFailed CommitFailed - - FailedUnrecoverable + _ + _ + _ + _ Faulty // sector is corrupted or gone for some reason FaultReported // sector has been declared as a fault on chain diff --git a/api/cbor_gen.go b/api/cbor_gen.go index 8d309a6cc..a65f97aaf 100644 --- a/api/cbor_gen.go +++ b/api/cbor_gen.go @@ -23,6 +23,10 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error { } // t.Channel (address.Address) (struct) + if len("Channel") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Channel\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Channel")))); err != nil { return err } @@ -35,6 +39,10 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error { } // t.ChannelMessage (cid.Cid) (struct) + if len("ChannelMessage") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"ChannelMessage\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("ChannelMessage")))); err != nil { return err } @@ -53,6 +61,10 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error { } // t.Vouchers ([]*types.SignedVoucher) (slice) + if len("Vouchers") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Vouchers\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Vouchers")))); err != nil { return err } @@ -60,6 +72,10 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error { return err } + if len(t.Vouchers) > cbg.MaxLength { + return xerrors.Errorf("Slice value in field t.Vouchers was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Vouchers)))); err != nil { return err } @@ -200,6 +216,10 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error { } // t.SectorID (uint64) (uint64) + if len("SectorID") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"SectorID\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("SectorID")))); err != nil { return err } @@ -212,6 +232,10 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error { } // t.Offset (uint64) (uint64) + if len("Offset") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Offset\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Offset")))); err != nil { return err } @@ -224,6 +248,10 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error { } // t.Size (uint64) (uint64) + if len("Size") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Size\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Size")))); err != nil { return err } @@ -336,6 +364,10 @@ func (t *SealedRefs) MarshalCBOR(w io.Writer) error { } // t.Refs ([]api.SealedRef) (slice) + if len("Refs") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Refs\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Refs")))); err != nil { return err } @@ -343,6 +375,10 @@ func (t *SealedRefs) MarshalCBOR(w io.Writer) error { return err } + if len(t.Refs) > cbg.MaxLength { + return xerrors.Errorf("Slice value in field t.Refs was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Refs)))); err != nil { return err } diff --git a/api/test/deals.go b/api/test/deals.go index b9ff2611f..4c2fb7439 100644 --- a/api/test/deals.go +++ b/api/test/deals.go @@ -11,7 +11,7 @@ import ( "testing" "time" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" @@ -24,7 +24,7 @@ func init() { build.InsecurePoStValidation = true } -func TestDealFlow(t *testing.T, b APIBuilder) { +func TestDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration) { os.Setenv("BELLMAN_NO_GPU", "1") ctx := context.Background() @@ -64,7 +64,7 @@ func TestDealFlow(t *testing.T, b APIBuilder) { go func() { defer close(done) for mine { - time.Sleep(time.Second) + time.Sleep(blocktime) fmt.Println("mining a block now") if err := sn[0].MineOne(ctx); err != nil { t.Error(err) diff --git a/build/bootstrap/bootstrappers.pi b/build/bootstrap/bootstrappers.pi index 9c619b1bb..e646ccf3f 100644 --- a/build/bootstrap/bootstrappers.pi +++ b/build/bootstrap/bootstrappers.pi @@ -1,6 +1,6 @@ -/dns4/lotus-bootstrap-0.dfw.fil-test.net/tcp/1347/p2p/12D3KooWHwGBSiLR5ts7KW9MgH4BMzC2iXe18kwAQ8Ee3LUd1jeR -/dns4/lotus-bootstrap-1.dfw.fil-test.net/tcp/1347/p2p/12D3KooWCLFaawdhLGcSpiqg43DtZ9QzPQ6HcB8Vvyu2Cnta8UWc -/dns4/lotus-bootstrap-0.fra.fil-test.net/tcp/1347/p2p/12D3KooWMmaL7eaUCF6tVAghVmgozxz4uztbuFUQv6dyFpHRarHR -/dns4/lotus-bootstrap-1.fra.fil-test.net/tcp/1347/p2p/12D3KooWLLpNYoKdf9NgcWudBhXLdTcXncqAsTzozw1scMMu6nS5 -/dns4/lotus-bootstrap-0.sin.fil-test.net/tcp/1347/p2p/12D3KooWCNL9vXaXwNs3Bu8uRAJK4pxpCyPeM7jZLSDpJma1wrV8 -/dns4/lotus-bootstrap-1.sin.fil-test.net/tcp/1347/p2p/12D3KooWNGGxFda1eC5U2YKAgs4ypoFHn3Z3xHCsjmFdrCcytoxm +/dns4/lotus-bootstrap-0.sin.fil-test.net/tcp/1347/p2p/12D3KooWLZs8BWtEzRTYET4yR4jzDtPamaA1YsyPQJq6cf2RfxBD +/dns4/lotus-bootstrap-1.sin.fil-test.net/tcp/1347/p2p/12D3KooWGvrgjWw4Yqo4AFWqYp4g37FpUvUCQBkNWudZVSwR9tY1 +/dns4/lotus-bootstrap-0.fra.fil-test.net/tcp/1347/p2p/12D3KooWSfNcrD1cs5Cj5eSHbK6nHCqJLffAuPqvRMBRgvUdqQhX +/dns4/lotus-bootstrap-1.fra.fil-test.net/tcp/1347/p2p/12D3KooWNkXyVPspUnrHUiSC3VJPMcXvHuNdy3BTCLTPPnDgwwTT +/dns4/lotus-bootstrap-0.dfw.fil-test.net/tcp/1347/p2p/12D3KooWSgJWJZK8LTRtCWzPa5FQheCFJjHpficVYgEQWeimcqCu +/dns4/lotus-bootstrap-1.dfw.fil-test.net/tcp/1347/p2p/12D3KooWFPaC4dyGpbNXCpVHjZucdJnDwmv4ng9tponPx5GrzJkT diff --git a/build/forks.go b/build/forks.go index 8e8f53abe..9cc7b0c9d 100644 --- a/build/forks.go +++ b/build/forks.go @@ -1,4 +1,3 @@ package build -const ForkCCM = 1750 -const ForkNoPowerEPSUpdates = 16450 +// No forks yet \o/ diff --git a/build/genesis.go b/build/genesis.go index cdcf0402e..dc4ded273 100644 --- a/build/genesis.go +++ b/build/genesis.go @@ -2,7 +2,7 @@ package build import ( rice "github.com/GeertJohan/go.rice" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" ) // moved from now-defunct build/paramfetch.go @@ -11,12 +11,12 @@ var log = logging.Logger("build") func MaybeGenesis() []byte { builtinGen, err := rice.FindBox("genesis") if err != nil { - log.Warn("loading built-in genesis: %s", err) + log.Warnf("loading built-in genesis: %s", err) return nil } genBytes, err := builtinGen.Bytes("devnet.car") if err != nil { - log.Warn("loading built-in genesis: %s", err) + log.Warnf("loading built-in genesis: %s", err) } return genBytes diff --git a/build/genesis/devnet.car b/build/genesis/devnet.car index d0aef2486..ef2fe2dab 100644 Binary files a/build/genesis/devnet.car and b/build/genesis/devnet.car differ diff --git a/build/parameters.go b/build/parameters.go new file mode 100644 index 000000000..433fa497b --- /dev/null +++ b/build/parameters.go @@ -0,0 +1,5 @@ +package build + +import rice "github.com/GeertJohan/go.rice" + +var ParametersJson = rice.MustFindBox("proof-params").MustBytes("parameters.json") diff --git a/build/params_shared.go b/build/params_shared.go index ec4982cc1..fd9dbf65d 100644 --- a/build/params_shared.go +++ b/build/params_shared.go @@ -56,6 +56,9 @@ const SealRandomnessLookback = Finality // Epochs const SealRandomnessLookbackLimit = SealRandomnessLookback + 2000 +// Maximum lookback that randomness can be sourced from for a seal proof submission +const MaxSealLookback = SealRandomnessLookbackLimit + 2000 + // ///// // Mining @@ -97,3 +100,8 @@ const BadBlockCacheSize = 1 << 15 // assuming 4000 messages per round, this lets us not lose any messages across a // 10 block reorg. const BlsSignatureCacheSize = 40000 + +// /////// +// Limits + +const BlockMessageLimit = 512 diff --git a/build/params_testnet.go b/build/params_testnet.go index aa75ebcd7..016109770 100644 --- a/build/params_testnet.go +++ b/build/params_testnet.go @@ -3,7 +3,6 @@ package build var SectorSizes = []uint64{ - 1 << 30, 32 << 30, } diff --git a/build/proof-params/parameters.json b/build/proof-params/parameters.json new file mode 100644 index 000000000..7e1988a41 --- /dev/null +++ b/build/proof-params/parameters.json @@ -0,0 +1,103 @@ +{ + "v20-proof-of-spacetime-election-5f585aca354eb68e411c8582ed0efd800792430e4e76d73468c4fc03f1a8d6d2.params": { + "cid": "QmX7tYeNPWae2fjZ3Am6GB9dmHvLqvoz8dKo3PR98VYxH9", + "digest": "39a9edec3355516674f0d12b926be493", + "sector_size": 34359738368 + }, + "v20-proof-of-spacetime-election-5f585aca354eb68e411c8582ed0efd800792430e4e76d73468c4fc03f1a8d6d2.vk": { + "cid": "QmbNGx7pNbGiEr8ykoHxVXHW2LNSmGdsxKtj1onZCyguCX", + "digest": "0227ae7df4f2affe529ebafbbc7540ee", + "sector_size": 34359738368 + }, + "v20-proof-of-spacetime-election-a4e18190d4b4657ba1b4d08a341871b2a6f398e327cb9951b28ab141fbdbf49d.params": { + "cid": "QmRGZsNp4mp1cZshcXqt3VMuWscAEsiMa2iepF4CsWWoiv", + "digest": "991041a354b12c280542741f58c7f2ca", + "sector_size": 1024 + }, + "v20-proof-of-spacetime-election-a4e18190d4b4657ba1b4d08a341871b2a6f398e327cb9951b28ab141fbdbf49d.vk": { + "cid": "QmWpmrhCGVcfqLyqp5oGAnhPmCE5hGTPaauHi25mpQwRSU", + "digest": "91fac550e1f9bccab213830bb0c85bd6", + "sector_size": 1024 + }, + "v20-proof-of-spacetime-election-a9eb6d90b896a282ec2d3a875c6143e3fcff778f0da1460709e051833651559b.params": { + "cid": "QmenSZXh1EsSyHiSRvA6wb8yaPhYBTjrKehJw96Px5HnN4", + "digest": "6322eacd2773163ddd51f9ca7d645fc4", + "sector_size": 1073741824 + }, + "v20-proof-of-spacetime-election-a9eb6d90b896a282ec2d3a875c6143e3fcff778f0da1460709e051833651559b.vk": { + "cid": "QmPvZoMKofw6eDhDg5ESJA2QAZP8HvM6qMQk7fw4pq9bQf", + "digest": "0df62745fceac922e3e70847cfc70b52", + "sector_size": 1073741824 + }, + "v20-proof-of-spacetime-election-bf872523641b1de33553db2a177df13e412d7b3b0103e6696ae0a1cf5d525259.params": { + "cid": "QmVibFqzkZoL8cwQmzj8njPokCQGCCx4pBcUH77bzgJgV9", + "digest": "de9d71e672f286706a1673bd57abdaac", + "sector_size": 16777216 + }, + "v20-proof-of-spacetime-election-bf872523641b1de33553db2a177df13e412d7b3b0103e6696ae0a1cf5d525259.vk": { + "cid": "QmZa5FX27XyiEXQQLQpHqtMJKLzrcY8wMuj3pxzmSimSyu", + "digest": "7f796d3a0f13499181e44b5eee0cc744", + "sector_size": 16777216 + }, + "v20-proof-of-spacetime-election-ffc3fb192364238b60977839d14e3154d4a98313e30d46694a12af54b6874975.params": { + "cid": "Qmbt2SWWAmMcYoY3DAiRDXA8fAuqdqRLWucJMSxYmzBCmN", + "digest": "151ae0ae183fc141e8c2bebc28e5cc10", + "sector_size": 268435456 + }, + "v20-proof-of-spacetime-election-ffc3fb192364238b60977839d14e3154d4a98313e30d46694a12af54b6874975.vk": { + "cid": "QmUxvPu4xdVmjMFihUKoYyEdXBqxsXkvmxRweU7KouWHji", + "digest": "95eb89588e9d1832aca044c3a13178af", + "sector_size": 268435456 + }, + "v20-stacked-proof-of-replication-117839dacd1ef31e5968a6fd13bcd6fa86638d85c40c9241a1d07c2a954eb89b.params": { + "cid": "QmQZe8eLo2xXbhSDxtyYZNqEjqjdcWGdADywECRvNEZQdX", + "digest": "fcd50e2e08a8560a6bb3418e883567ed", + "sector_size": 268435456 + }, + "v20-stacked-proof-of-replication-117839dacd1ef31e5968a6fd13bcd6fa86638d85c40c9241a1d07c2a954eb89b.vk": { + "cid": "Qme1hn6QT1covfoUFGDZkqoE1pMTax9FNW3nWWmTNqFe7y", + "digest": "872e244d86499fd659082e3bcf3f13e7", + "sector_size": 268435456 + }, + "v20-stacked-proof-of-replication-b46f3a1051afbb67f70aae7082da95def62eee943662f3e1bf69837fb08aaae4.params": { + "cid": "QmSfrPDC9jwY4MKrjzhCqDBBAG44wSDM8oE5NuDwWSh2xN", + "digest": "0a338b941c5f17946340de5fc95cab30", + "sector_size": 34359738368 + }, + "v20-stacked-proof-of-replication-b46f3a1051afbb67f70aae7082da95def62eee943662f3e1bf69837fb08aaae4.vk": { + "cid": "QmTDGynCmnbaZNBP3Bv3F3duC3ecKRubCKeMUiQQZYbGpF", + "digest": "c752e070a6b7aa8b79aa661a6b600b55", + "sector_size": 34359738368 + }, + "v20-stacked-proof-of-replication-e71093863cadc71de61f38311ee45816633973bbf34849316b147f8d2e66f199.params": { + "cid": "QmXjSSnMUnc7EjQBYtTHhvLU3kXJTbUyhVhJRSTRehh186", + "digest": "efa407fd09202dffd15799a8518e73d3", + "sector_size": 1024 + }, + "v20-stacked-proof-of-replication-e71093863cadc71de61f38311ee45816633973bbf34849316b147f8d2e66f199.vk": { + "cid": "QmYHW3zhQouDP4okFbXSsRMcZ8bokKGvzxqbv7ZrunPMiG", + "digest": "b2f09a0ccb62da28c890d5b881c8dcd2", + "sector_size": 1024 + }, + "v20-stacked-proof-of-replication-e99a585174b6a45b254ba4780d72c89ad808c305c6d11711009ade4f39dba8e9.params": { + "cid": "QmUhyfNeLb32LfSkjsUwTFYLXQGMj6JQ8daff4DdVMt79q", + "digest": "b53c1916a63839ec345aa2224e9198b7", + "sector_size": 1073741824 + }, + "v20-stacked-proof-of-replication-e99a585174b6a45b254ba4780d72c89ad808c305c6d11711009ade4f39dba8e9.vk": { + "cid": "QmWReGfbuoozNErbskmFvqV4q36BY6F2WWb4cVFc3zoYkA", + "digest": "20d58a3fae7343481f8298a2dd493dd7", + "sector_size": 1073741824 + }, + "v20-stacked-proof-of-replication-f571ee2386f4c65a68e802747f2d78691006fc81a67971c4d9641403fffece16.params": { + "cid": "QmSAHu14Pe8iav6BYCt9XkpHJ73XM7tcpY4d9JK9BST9HU", + "digest": "7698426202c7e07b26ef056d31485b3a", + "sector_size": 16777216 + }, + "v20-stacked-proof-of-replication-f571ee2386f4c65a68e802747f2d78691006fc81a67971c4d9641403fffece16.vk": { + "cid": "QmaKtFLShnhMGVn7P9UsHjkgqtqRFSwCStqqykBN7u8dax", + "digest": "834408e5c3fce6ec5d1bf64e64cee94e", + "sector_size": 16777216 + } +} + diff --git a/build/version.go b/build/version.go index b1fd8b63d..7db75d814 100644 --- a/build/version.go +++ b/build/version.go @@ -5,7 +5,7 @@ import "fmt" var CurrentCommit string // BuildVersion is the local build version, set by build system -const BuildVersion = "0.1.6" +const BuildVersion = "0.2.0" var UserVersion = BuildVersion + CurrentCommit diff --git a/chain/actors/actor_init.go b/chain/actors/actor_init.go index 45da1401e..2bb31ce7f 100644 --- a/chain/actors/actor_init.go +++ b/chain/actors/actor_init.go @@ -15,7 +15,7 @@ import ( "github.com/ipfs/go-cid" "github.com/ipfs/go-hamt-ipld" cbor "github.com/ipfs/go-ipld-cbor" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" mh "github.com/multiformats/go-multihash" ) diff --git a/chain/actors/actor_miner.go b/chain/actors/actor_miner.go index 4e3e33096..62bfd3ab0 100644 --- a/chain/actors/actor_miner.go +++ b/chain/actors/actor_miner.go @@ -19,12 +19,10 @@ import ( cbor "github.com/ipfs/go-ipld-cbor" "github.com/libp2p/go-libp2p-core/peer" cbg "github.com/whyrusleeping/cbor-gen" - "go.opencensus.io/trace" "golang.org/x/xerrors" ) const MaxSectors = 1 << 48 -const RLEMax = 100e3 type StorageMinerActor struct{} @@ -141,10 +139,7 @@ func (sma StorageMinerActor) Exports() []interface{} { return []interface{}{ 1: sma.StorageMinerConstructor, 2: sma.PreCommitSector, - 3: withUpdates( - update{start: 0, method: sma.ProveCommitSectorV0}, - update{start: build.ForkNoPowerEPSUpdates, method: sma.ProveCommitSectorV1}, - ), + 3: sma.ProveCommitSector, 4: sma.SubmitFallbackPoSt, //5: sma.SlashStorageFault, //6: sma.GetCurrentProvingSet, @@ -159,10 +154,7 @@ func (sma StorageMinerActor) Exports() []interface{} { //15: sma.ChangeWorker, 16: sma.IsSlashed, 17: sma.CheckMiner, - 18: withUpdates( - update{start: 0, method: sma.DeclareFaultsV0}, - update{start: build.ForkNoPowerEPSUpdates, method: sma.DeclareFaultsV1}, - ), + 18: sma.DeclareFaults, 19: sma.SlashConsensusFault, 20: sma.SubmitElectionPoSt, } @@ -298,110 +290,7 @@ type SectorProveCommitInfo struct { DealIDs []uint64 } -func (sma StorageMinerActor) ProveCommitSectorV0(act *types.Actor, vmctx types.VMContext, params *SectorProveCommitInfo) ([]byte, ActorError) { - ctx := vmctx.Context() - oldstate, self, err := loadState(vmctx) - if err != nil { - return nil, err - } - - mi, err := loadMinerInfo(vmctx, self) - if err != nil { - return nil, err - } - - us, ok := self.PreCommittedSectors[uintToStringKey(params.SectorID)] - if !ok { - return nil, aerrors.New(1, "no pre-commitment found for sector") - } - - if us.ReceivedEpoch+build.InteractivePoRepDelay >= vmctx.BlockHeight() { - return nil, aerrors.New(2, "too early for proof submission") - } - - delete(self.PreCommittedSectors, uintToStringKey(params.SectorID)) - - // TODO: ensure normalization to ID address - maddr := vmctx.Message().To - - ticket, err := vmctx.GetRandomness(us.Info.SealEpoch - build.SealRandomnessLookback) - if err != nil { - return nil, aerrors.Wrap(err, "failed to get ticket randomness") - } - - seed, err := vmctx.GetRandomness(us.ReceivedEpoch + build.InteractivePoRepDelay) - if err != nil { - return nil, aerrors.Wrap(err, "failed to get randomness for prove sector commitment") - } - - enc, err := SerializeParams(&ComputeDataCommitmentParams{ - DealIDs: params.DealIDs, - SectorSize: mi.SectorSize, - }) - if err != nil { - return nil, aerrors.Wrap(err, "failed to serialize ComputeDataCommitmentParams") - } - - commD, err := vmctx.Send(StorageMarketAddress, SMAMethods.ComputeDataCommitment, types.NewInt(0), enc) - if err != nil { - return nil, aerrors.Wrapf(err, "failed to compute data commitment (sector %d, deals: %v)", params.SectorID, params.DealIDs) - } - - if ok, err := vmctx.Sys().ValidatePoRep(ctx, maddr, mi.SectorSize, commD, us.Info.CommR, ticket, params.Proof, seed, params.SectorID); err != nil { - return nil, err - } else if !ok { - return nil, aerrors.Newf(2, "porep proof was invalid (t:%x; s:%x(%d); p:%s)", ticket, seed, us.ReceivedEpoch+build.InteractivePoRepDelay, truncateHexPrint(params.Proof)) - } - - // Note: There must exist a unique index in the miner's sector set for each - // sector ID. The `faults`, `recovered`, and `done` parameters of the - // SubmitPoSt method express indices into this sector set. - nssroot, err := AddToSectorSet(ctx, types.WrapStorage(vmctx.Storage()), self.Sectors, params.SectorID, us.Info.CommR, commD) - if err != nil { - return nil, err - } - self.Sectors = nssroot - - // if miner is not mining, start their proving period now - // Note: As written here, every miners first PoSt will only be over one sector. - // We could set up a 'grace period' for starting mining that would allow miners - // to submit several sectors for their first proving period. Alternatively, we - // could simply make the 'PreCommitSector' call take multiple sectors at a time. - // - // Note: Proving period is a function of sector size; small sectors take less - // time to prove than large sectors do. Sector size is selected when pledging. - pss, lerr := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.ProvingSet) - if lerr != nil { - return nil, aerrors.HandleExternalError(lerr, "could not load proving set node") - } - - if pss.Count == 0 { - self.ProvingSet = self.Sectors - // TODO: probably want to wait until the miner is above a certain - // threshold before starting this - self.ElectionPeriodStart = vmctx.BlockHeight() - } - - nstate, err := vmctx.Storage().Put(self) - if err != nil { - return nil, err - } - if err := vmctx.Storage().Commit(oldstate, nstate); err != nil { - return nil, err - } - - activateParams, err := SerializeParams(&ActivateStorageDealsParams{ - Deals: params.DealIDs, - }) - if err != nil { - return nil, err - } - - _, err = vmctx.Send(StorageMarketAddress, SMAMethods.ActivateStorageDeals, types.NewInt(0), activateParams) - return nil, aerrors.Wrapf(err, "calling ActivateStorageDeals failed") -} - -func (sma StorageMinerActor) ProveCommitSectorV1(act *types.Actor, vmctx types.VMContext, params *SectorProveCommitInfo) ([]byte, ActorError) { +func (sma StorageMinerActor) ProveCommitSector(act *types.Actor, vmctx types.VMContext, params *SectorProveCommitInfo) ([]byte, ActorError) { ctx := vmctx.Context() oldstate, self, err := loadState(vmctx) if err != nil { @@ -431,6 +320,14 @@ func (sma StorageMinerActor) ProveCommitSectorV1(act *types.Actor, vmctx types.V // TODO: ensure normalization to ID address maddr := vmctx.Message().To + if vmctx.BlockHeight()-us.Info.SealEpoch > build.MaxSealLookback { + return nil, aerrors.Newf(5, "source randomness for sector SealEpoch too far in past (epoch %d)", us.Info.SealEpoch) + } + + if vmctx.BlockHeight()-us.ReceivedEpoch > build.MaxSealLookback { + return nil, aerrors.Newf(6, "source randomness for sector ReceivedEpoch too far in past (epoch %d)", us.ReceivedEpoch) + } + ticket, err := vmctx.GetRandomness(us.Info.SealEpoch - build.SealRandomnessLookback) if err != nil { return nil, aerrors.Wrap(err, "failed to get ticket randomness") @@ -577,25 +474,21 @@ func (sma StorageMinerActor) SubmitFallbackPoSt(act *types.Actor, vmctx types.VM return nil, aerrors.HandleExternalError(lerr, "could not load proving set node") } - { - c, nerr := self.FaultSet.Count() - if nerr != nil { - return nil, aerrors.Absorb(nerr, 6, "invalid bitfield") - } - - if c > RLEMax { - return nil, aerrors.Newf(7, "too many items in bitfield: %d", c) - } + ss, lerr := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.Sectors) + if lerr != nil { + return nil, aerrors.HandleExternalError(lerr, "could not load proving set node") } - faults, nerr := self.FaultSet.AllMap() + faults, nerr := self.FaultSet.AllMap(2 * ss.Count) if nerr != nil { return nil, aerrors.Absorb(err, 5, "RLE+ invalid") } + activeFaults := uint64(0) var sectorInfos []ffi.PublicSectorInfo if err := pss.ForEach(func(id uint64, v *cbg.Deferred) error { if faults[id] { + activeFaults++ return nil } @@ -632,8 +525,8 @@ func (sma StorageMinerActor) SubmitFallbackPoSt(act *types.Actor, vmctx types.VM }) } - if ok, lerr := sectorbuilder.VerifyFallbackPost(vmctx.Context(), mi.SectorSize, - sectorbuilder.NewSortedPublicSectorInfo(sectorInfos), seed[:], params.Proof, candidates, proverID, 0); !ok || lerr != nil { // TODO: FORK - set faults to len(faults) + if ok, lerr := vmctx.Sys().VerifyFallbackPost(vmctx.Context(), mi.SectorSize, + sectorbuilder.NewSortedPublicSectorInfo(sectorInfos), seed[:], params.Proof, candidates, proverID, activeFaults); !ok || lerr != nil { if lerr != nil { // TODO: study PoST errors return nil, aerrors.Absorb(lerr, 4, "PoST error") @@ -644,7 +537,7 @@ func (sma StorageMinerActor) SubmitFallbackPoSt(act *types.Actor, vmctx types.VM } // Post submission is successful! - if err := onSuccessfulPoSt(self, vmctx); err != nil { + if err := onSuccessfulPoSt(self, vmctx, activeFaults); err != nil { return nil, err } @@ -747,17 +640,6 @@ func RemoveFromSectorSet(ctx context.Context, s types.Storage, ss cid.Cid, ids [ return ncid, nil } -func ValidatePoRep(ctx context.Context, maddr address.Address, ssize uint64, commD, commR, ticket, proof, seed []byte, sectorID uint64) (bool, ActorError) { - _, span := trace.StartSpan(ctx, "ValidatePoRep") - defer span.End() - ok, err := sectorbuilder.VerifySeal(ssize, commR, commD, maddr, ticket, seed, sectorID, proof) - if err != nil { - return false, aerrors.Absorb(err, 25, "verify seal failed") - } - - return ok, nil -} - func CollateralForPower(power types.BigInt) types.BigInt { return types.BigMul(power, types.NewInt(10)) /* TODO: this @@ -935,33 +817,7 @@ type DeclareFaultsParams struct { Faults types.BitField } -func (sma StorageMinerActor) DeclareFaultsV0(act *types.Actor, vmctx types.VMContext, params *DeclareFaultsParams) ([]byte, ActorError) { - oldstate, self, aerr := loadState(vmctx) - if aerr != nil { - return nil, aerr - } - - nfaults, err := types.MergeBitFields(params.Faults, self.FaultSet) - if err != nil { - return nil, aerrors.Absorb(err, 1, "failed to merge bitfields") - } - - self.FaultSet = nfaults - - self.LastFaultSubmission = vmctx.BlockHeight() - - nstate, aerr := vmctx.Storage().Put(self) - if err != nil { // TODO: FORK: should be aerr - return nil, aerr - } - if err := vmctx.Storage().Commit(oldstate, nstate); err != nil { - return nil, err - } - - return nil, nil -} - -func (sma StorageMinerActor) DeclareFaultsV1(act *types.Actor, vmctx types.VMContext, params *DeclareFaultsParams) ([]byte, ActorError) { +func (sma StorageMinerActor) DeclareFaults(act *types.Actor, vmctx types.VMContext, params *DeclareFaultsParams) ([]byte, ActorError) { oldstate, self, aerr := loadState(vmctx) if aerr != nil { return nil, aerr @@ -981,6 +837,20 @@ func (sma StorageMinerActor) DeclareFaultsV1(act *types.Actor, vmctx types.VMCon return nil, aerrors.Absorb(err, 1, "failed to merge bitfields") } + ss, nerr := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.Sectors) + if nerr != nil { + return nil, aerrors.HandleExternalError(nerr, "failed to load sector set") + } + + cf, nerr := nfaults.Count() + if nerr != nil { + return nil, aerrors.Absorb(nerr, 2, "could not decode RLE+") + } + + if cf > 2*ss.Count { + return nil, aerrors.Newf(3, "too many declared faults: %d > %d", cf, 2*ss.Count) + } + self.FaultSet = nfaults self.LastFaultSubmission = vmctx.BlockHeight() @@ -1056,7 +926,41 @@ func (sma StorageMinerActor) SubmitElectionPoSt(act *types.Actor, vmctx types.VM return nil, aerrors.New(1, "slashed miners can't perform election PoSt") } - if err := onSuccessfulPoSt(self, vmctx); err != nil { + pss, nerr := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.ProvingSet) + if nerr != nil { + return nil, aerrors.HandleExternalError(nerr, "failed to load proving set") + } + + ss, nerr := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.Sectors) + if nerr != nil { + return nil, aerrors.HandleExternalError(nerr, "failed to load proving set") + } + + faults, nerr := self.FaultSet.AllMap(2 * ss.Count) + if nerr != nil { + return nil, aerrors.Absorb(nerr, 1, "invalid bitfield (fatal?)") + } + + activeFaults := uint64(0) + for f := range faults { + if f > amt.MaxIndex { + continue + } + + var comms [][]byte + err := pss.Get(f, &comms) + if err != nil { + var notfound *amt.ErrNotFound + if !xerrors.As(err, ¬found) { + return nil, aerrors.HandleExternalError(err, "failed to find sector in sector set") + } + continue + } + + activeFaults++ + } + + if err := onSuccessfulPoSt(self, vmctx, activeFaults); err != nil { // TODO return nil, err } @@ -1071,82 +975,7 @@ func (sma StorageMinerActor) SubmitElectionPoSt(act *types.Actor, vmctx types.VM return nil, nil } -func onSuccessfulPoSt(self *StorageMinerActorState, vmctx types.VMContext) aerrors.ActorError { - if vmctx.BlockHeight() >= build.ForkNoPowerEPSUpdates { - return onSuccessfulPoStV1(self, vmctx) - } - - return onSuccessfulPoStV0(self, vmctx) -} - -func onSuccessfulPoStV0(self *StorageMinerActorState, vmctx types.VMContext) aerrors.ActorError { - var mi MinerInfo - if err := vmctx.Storage().Get(self.Info, &mi); err != nil { - return err - } - - pss, nerr := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.ProvingSet) - if nerr != nil { - return aerrors.HandleExternalError(nerr, "failed to load proving set") - } - - faults, nerr := self.FaultSet.All() - if nerr != nil { - return aerrors.Absorb(nerr, 1, "invalid bitfield (fatal?)") - } - - self.FaultSet = types.NewBitField() - - oldPower := self.Power - newPower := types.BigMul(types.NewInt(pss.Count-uint64(len(faults))), types.NewInt(mi.SectorSize)) - - // If below the minimum size requirement, miners have zero power - if newPower.LessThan(types.NewInt(build.MinimumMinerPower)) { - newPower = types.NewInt(0) - } - - self.Power = newPower - - delta := types.BigSub(self.Power, oldPower) - if self.SlashedAt != 0 { - self.SlashedAt = 0 - delta = self.Power - } - - prevSlashingDeadline := self.ElectionPeriodStart + build.SlashablePowerDelay - if !self.Active && newPower.GreaterThan(types.NewInt(0)) { - self.Active = true - prevSlashingDeadline = 0 - } - - if !(oldPower.IsZero() && newPower.IsZero()) { - enc, err := SerializeParams(&UpdateStorageParams{ - Delta: delta, - NextSlashDeadline: vmctx.BlockHeight() + build.SlashablePowerDelay, - PreviousSlashDeadline: prevSlashingDeadline, - }) - if err != nil { - return err - } - - _, err = vmctx.Send(StoragePowerAddress, SPAMethods.UpdateStorage, types.NewInt(0), enc) - if err != nil { - return aerrors.Wrap(err, "updating storage failed") - } - } - - ncid, err := RemoveFromSectorSet(vmctx.Context(), vmctx.Storage(), self.Sectors, faults) - if err != nil { - return err - } - - self.Sectors = ncid - self.ProvingSet = ncid - self.ElectionPeriodStart = vmctx.BlockHeight() - return nil -} - -func onSuccessfulPoStV1(self *StorageMinerActorState, vmctx types.VMContext) aerrors.ActorError { +func onSuccessfulPoSt(self *StorageMinerActorState, vmctx types.VMContext, activeFaults uint64) aerrors.ActorError { // TODO: some sector upkeep stuff that is very haphazard and unclear in the spec var mi MinerInfo @@ -1159,17 +988,12 @@ func onSuccessfulPoStV1(self *StorageMinerActorState, vmctx types.VMContext) aer return aerrors.HandleExternalError(nerr, "failed to load proving set") } - { - c, nerr := self.FaultSet.Count() - if nerr != nil { - return aerrors.Absorb(nerr, 2, "invalid bitfield") - } - if c > RLEMax { - return aerrors.Newf(3, "too many items in bitfield: %d", c) - } + ss, nerr := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.Sectors) + if nerr != nil { + return aerrors.HandleExternalError(nerr, "failed to load sector set") } - faults, nerr := self.FaultSet.All() + faults, nerr := self.FaultSet.All(2 * ss.Count) if nerr != nil { return aerrors.Absorb(nerr, 1, "invalid bitfield (fatal?)") } @@ -1177,7 +1001,7 @@ func onSuccessfulPoStV1(self *StorageMinerActorState, vmctx types.VMContext) aer self.FaultSet = types.NewBitField() oldPower := self.Power - newPower := types.BigMul(types.NewInt(pss.Count-uint64(len(faults))), types.NewInt(mi.SectorSize)) + newPower := types.BigMul(types.NewInt(pss.Count-activeFaults), types.NewInt(mi.SectorSize)) // If below the minimum size requirement, miners have zero power if newPower.LessThan(types.NewInt(build.MinimumMinerPower)) { diff --git a/chain/actors/actor_miner_test.go b/chain/actors/actor_miner_test.go index a522fe349..e08342248 100644 --- a/chain/actors/actor_miner_test.go +++ b/chain/actors/actor_miner_test.go @@ -3,18 +3,22 @@ package actors_test import ( "bytes" "context" + "math" "math/rand" "testing" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/aerrors" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/rlepluslazy" hamt "github.com/ipfs/go-hamt-ipld" blockstore "github.com/ipfs/go-ipfs-blockstore" + "github.com/stretchr/testify/assert" cbg "github.com/whyrusleeping/cbor-gen" ) @@ -50,23 +54,37 @@ func TestMinerCommitSectors(t *testing.T) { addSectorToMiner(h, t, minerAddr, worker, client, 1) assertSectorIDs(h, t, minerAddr, []uint64{1}) + } +type badRuns struct { + done bool +} + +func (br *badRuns) HasNext() bool { + return !br.done +} + +func (br *badRuns) NextRun() (rlepluslazy.Run, error) { + br.done = true + return rlepluslazy.Run{true, math.MaxInt64}, nil +} + +var _ rlepluslazy.RunIterator = (*badRuns)(nil) + func TestMinerSubmitBadFault(t *testing.T) { + oldSS, oldMin := build.SectorSizes, build.MinimumMinerPower + build.SectorSizes, build.MinimumMinerPower = []uint64{1024}, 1024 + defer func() { + build.SectorSizes, build.MinimumMinerPower = oldSS, oldMin + }() + var worker, client address.Address var minerAddr address.Address opts := []HarnessOpt{ HarnessAddr(&worker, 1000000), HarnessAddr(&client, 1000000), - HarnessActor(&minerAddr, &worker, actors.StorageMinerCodeCid, - func() cbg.CBORMarshaler { - return &actors.StorageMinerConstructorParams{ - Owner: worker, - Worker: worker, - SectorSize: 1024, - PeerID: "fakepeerid", - } - }), + HarnessAddMiner(&minerAddr, &worker), } h := NewHarness(t, opts...) @@ -92,18 +110,52 @@ func TestMinerSubmitBadFault(t *testing.T) { ret, _ = h.Invoke(t, actors.NetworkAddress, minerAddr, actors.MAMethods.SubmitElectionPoSt, nil) ApplyOK(t, ret) - assertSectorIDs(h, t, minerAddr, []uint64{1}) + st, err := getMinerState(context.TODO(), h.vm.StateTree(), h.bs, minerAddr) + assert.NoError(t, err) + expectedPower := st.Power + if types.BigCmp(expectedPower, types.NewInt(1024)) != 0 { + t.Errorf("Expected power of 1024, got %s", expectedPower) + } + badnum := uint64(0) badnum-- bf = types.NewBitField() bf.Set(badnum) + bf.Set(badnum - 1) ret, _ = h.Invoke(t, worker, minerAddr, actors.MAMethods.DeclareFaults, &actors.DeclareFaultsParams{bf}) ApplyOK(t, ret) ret, _ = h.Invoke(t, actors.NetworkAddress, minerAddr, actors.MAMethods.SubmitElectionPoSt, nil) + ApplyOK(t, ret) + assertSectorIDs(h, t, minerAddr, []uint64{1}) + + st, err = getMinerState(context.TODO(), h.vm.StateTree(), h.bs, minerAddr) + assert.NoError(t, err) + currentPower := st.Power + if types.BigCmp(expectedPower, currentPower) != 0 { + t.Errorf("power changed and shouldn't have: %s != %s", expectedPower, currentPower) + } + + bf.Set(badnum - 2) + ret, _ = h.Invoke(t, worker, minerAddr, actors.MAMethods.DeclareFaults, &actors.DeclareFaultsParams{bf}) + if ret.ExitCode != 3 { + t.Errorf("expected exit code 3, got %d: %+v", ret.ExitCode, ret.ActorErr) + } + assertSectorIDs(h, t, minerAddr, []uint64{1}) + + rle, err := rlepluslazy.EncodeRuns(&badRuns{}, []byte{}) + assert.NoError(t, err) + + bf, err = types.NewBitFieldFromBytes(rle) + assert.NoError(t, err) + ret, _ = h.Invoke(t, worker, minerAddr, actors.MAMethods.DeclareFaults, &actors.DeclareFaultsParams{bf}) + if ret.ExitCode != 3 { + t.Errorf("expected exit code 3, got %d: %+v", ret.ExitCode, ret.ActorErr) + } + assertSectorIDs(h, t, minerAddr, []uint64{1}) bf = types.NewBitField() bf.Set(1) @@ -175,7 +227,7 @@ func assertSectorIDs(h *Harness, t *testing.T, maddr address.Address, ids []uint } } -func getMinerSectorSet(ctx context.Context, st types.StateTree, bs blockstore.Blockstore, maddr address.Address) ([]*api.ChainSectorInfo, error) { +func getMinerState(ctx context.Context, st types.StateTree, bs blockstore.Blockstore, maddr address.Address) (*actors.StorageMinerActorState, error) { mact, err := st.GetActor(maddr) if err != nil { return nil, err @@ -187,6 +239,14 @@ func getMinerSectorSet(ctx context.Context, st types.StateTree, bs blockstore.Bl if err := cst.Get(ctx, mact.Head, &mstate); err != nil { return nil, err } + return &mstate, nil +} + +func getMinerSectorSet(ctx context.Context, st types.StateTree, bs blockstore.Blockstore, maddr address.Address) ([]*api.ChainSectorInfo, error) { + mstate, err := getMinerState(ctx, st, bs, maddr) + if err != nil { + return nil, err + } return stmgr.LoadSectorsFromSet(ctx, bs, mstate.Sectors) } @@ -202,7 +262,6 @@ func (h *Harness) makeFakeDeal(t *testing.T, miner, worker, client address.Addre prop := actors.StorageDealProposal{ PieceRef: commP[:], PieceSize: size, - //PieceSerialization SerializationMode // Needs to be here as it tells how data in the sector maps to PieceRef cid Client: client, Provider: miner, diff --git a/chain/actors/actor_paych.go b/chain/actors/actor_paych.go index 32181e592..05d9a9913 100644 --- a/chain/actors/actor_paych.go +++ b/chain/actors/actor_paych.go @@ -116,7 +116,7 @@ func (pca PaymentChannelActor) UpdateChannelState(act *types.Actor, vmctx types. vb, nerr := sv.SigningBytes() if nerr != nil { - return nil, aerrors.Escalate(nerr, "failed to serialize signedvoucher") + return nil, aerrors.Absorb(nerr, 1, "failed to serialize signedvoucher") } if err := vmctx.VerifySignature(sv.Signature, self.From, vb); err != nil { diff --git a/chain/actors/actor_storagemarket.go b/chain/actors/actor_storagemarket.go index e4ad58cb5..a3fb269bb 100644 --- a/chain/actors/actor_storagemarket.go +++ b/chain/actors/actor_storagemarket.go @@ -122,15 +122,21 @@ func (sdp *StorageDealProposal) Cid() (cid.Cid, error) { return nd.Cid(), nil } -func (sdp *StorageDealProposal) Verify() error { - unsigned := *sdp - unsigned.ProposerSignature = nil - var buf bytes.Buffer - if err := unsigned.MarshalCBOR(&buf); err != nil { - return err +func (sdp *StorageDealProposal) Verify(worker address.Address) error { + if sdp.Client != worker || worker == address.Undef { + unsigned := *sdp + unsigned.ProposerSignature = nil + var buf bytes.Buffer + if err := unsigned.MarshalCBOR(&buf); err != nil { + return err + } + + if err := sdp.ProposerSignature.Verify(sdp.Client, buf.Bytes()); err != nil { + return err + } } - return sdp.ProposerSignature.Verify(sdp.Client, buf.Bytes()) + return nil } type OnChainDeal struct { @@ -396,7 +402,7 @@ func (st *StorageMarketState) validateDeal(vmctx types.VMContext, deal StorageDe return aerrors.New(2, "Deals must be submitted by the miner worker") } - if err := deal.Verify(); err != nil { + if err := deal.Verify(providerWorker); err != nil { return aerrors.Absorb(err, 3, "verifying proposer signature") } diff --git a/chain/actors/actor_storagepower.go b/chain/actors/actor_storagepower.go index 09a3eb30a..d3c2ad674 100644 --- a/chain/actors/actor_storagepower.go +++ b/chain/actors/actor_storagepower.go @@ -67,7 +67,7 @@ type CreateStorageMinerParams struct { func (spa StoragePowerActor) CreateStorageMiner(act *types.Actor, vmctx types.VMContext, params *CreateStorageMinerParams) ([]byte, ActorError) { if !build.SupportedSectorSize(params.SectorSize) { - return nil, aerrors.New(1, "Unsupported sector size") + return nil, aerrors.Newf(1, "Unsupported sector size: %d", params.SectorSize) } var self StoragePowerState diff --git a/chain/actors/actors_test.go b/chain/actors/actors_test.go index 64207b422..d0f8fba86 100644 --- a/chain/actors/actors_test.go +++ b/chain/actors/actors_test.go @@ -50,10 +50,10 @@ func setupVMTestEnv(t *testing.T) (*vm.VM, []address.Address, bstore.Blockstore) t.Fatal(err) } - cs := store.NewChainStore(bs, nil) + cs := store.NewChainStore(bs, nil, nil) // TODO: should probabaly mock out the randomness bit, nil works for now - vm, err := vm.NewVM(stateroot, 1, nil, maddr, cs.Blockstore()) + vm, err := vm.NewVM(stateroot, 1, nil, maddr, cs.Blockstore(), cs.VMSys()) if err != nil { t.Fatal(err) } diff --git a/chain/actors/aerrors/wrap.go b/chain/actors/aerrors/wrap.go index da12cb7c6..b3dea8f09 100644 --- a/chain/actors/aerrors/wrap.go +++ b/chain/actors/aerrors/wrap.go @@ -171,7 +171,8 @@ func HandleExternalError(err error, msg string) ActorError { } return &actorError{ - fatal: true, + fatal: false, + retCode: 219, msg: msg, frame: xerrors.Caller(1), diff --git a/chain/actors/harness2_test.go b/chain/actors/harness2_test.go index d5d34795b..213d1d48e 100644 --- a/chain/actors/harness2_test.go +++ b/chain/actors/harness2_test.go @@ -3,6 +3,7 @@ package actors_test import ( "bytes" "context" + "github.com/filecoin-project/go-sectorbuilder" "math/rand" "testing" @@ -115,6 +116,32 @@ func HarnessActor(actor *address.Address, creator *address.Address, code cid.Cid } +func HarnessAddMiner(addr *address.Address, creator *address.Address) HarnessOpt { + return func(t testing.TB, h *Harness) error { + if h.Stage != HarnessPostInit { + return nil + } + if !addr.Empty() { + return xerrors.New("actor address should be empty") + } + ret, _ := h.InvokeWithValue(t, *creator, actors.StoragePowerAddress, + actors.SPAMethods.CreateStorageMiner, types.NewInt(3000), &actors.StorageMinerConstructorParams{ + Owner: *creator, + Worker: *creator, + SectorSize: 1024, + PeerID: "fakepeerid", + }) + + if ret.ExitCode != 0 { + return xerrors.Errorf("creating actor: %w", ret.ActorErr) + } + var err error + *addr, err = address.NewFromBytes(ret.Return) + return err + + } +} + func HarnessCtx(ctx context.Context) HarnessOpt { return func(t testing.TB, h *Harness) error { h.ctx = ctx @@ -168,8 +195,8 @@ func NewHarness(t *testing.T, options ...HarnessOpt) *Harness { t.Fatal(err) } - h.cs = store.NewChainStore(h.bs, nil) - h.vm, err = vm.NewVM(stateroot, 1, h.Rand, h.HI.Miner, h.cs.Blockstore()) + h.cs = store.NewChainStore(h.bs, nil, vm.Syscalls(sectorbuilder.ProofVerifier)) + h.vm, err = vm.NewVM(stateroot, 1, h.Rand, h.HI.Miner, h.cs.Blockstore(), h.cs.VMSys()) if err != nil { t.Fatal(err) } @@ -177,7 +204,7 @@ func NewHarness(t *testing.T, options ...HarnessOpt) *Harness { for _, opt := range options { err := opt(t, h) if err != nil { - t.Fatalf("Applying options: %v", err) + t.Fatalf("Applying options: %+v", err) } } diff --git a/chain/blocksync/blocksync.go b/chain/blocksync/blocksync.go index 429ad2ce1..b810c7783 100644 --- a/chain/blocksync/blocksync.go +++ b/chain/blocksync/blocksync.go @@ -14,7 +14,7 @@ import ( "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" inet "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" ) diff --git a/chain/blocksync/cbor_gen.go b/chain/blocksync/cbor_gen.go index 896d8f961..01db7d83a 100644 --- a/chain/blocksync/cbor_gen.go +++ b/chain/blocksync/cbor_gen.go @@ -5,7 +5,7 @@ import ( "io" "github.com/filecoin-project/lotus/chain/types" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" xerrors "golang.org/x/xerrors" ) diff --git a/chain/deals/cbor_gen.go b/chain/deals/cbor_gen.go deleted file mode 100644 index 2a5f1f8a9..000000000 --- a/chain/deals/cbor_gen.go +++ /dev/null @@ -1,883 +0,0 @@ -package deals - -import ( - "fmt" - "io" - - "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/types" - "github.com/libp2p/go-libp2p-core/peer" - cbg "github.com/whyrusleeping/cbor-gen" - xerrors "golang.org/x/xerrors" -) - -// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. - -var _ = xerrors.Errorf - -func (t *AskRequest) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{129}); err != nil { - return err - } - - // t.Miner (address.Address) (struct) - if err := t.Miner.MarshalCBOR(w); err != nil { - return err - } - return nil -} - -func (t *AskRequest) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 1 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Miner (address.Address) (struct) - - { - - if err := t.Miner.UnmarshalCBOR(br); err != nil { - return err - } - - } - return nil -} - -func (t *AskResponse) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{129}); err != nil { - return err - } - - // t.Ask (types.SignedStorageAsk) (struct) - if err := t.Ask.MarshalCBOR(w); err != nil { - return err - } - return nil -} - -func (t *AskResponse) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 1 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Ask (types.SignedStorageAsk) (struct) - - { - - pb, err := br.PeekByte() - if err != nil { - return err - } - if pb == cbg.CborNull[0] { - var nbuf [1]byte - if _, err := br.Read(nbuf[:]); err != nil { - return err - } - } else { - t.Ask = new(types.SignedStorageAsk) - if err := t.Ask.UnmarshalCBOR(br); err != nil { - return err - } - } - - } - return nil -} - -func (t *Proposal) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{130}); err != nil { - return err - } - - // t.DealProposal (actors.StorageDealProposal) (struct) - if err := t.DealProposal.MarshalCBOR(w); err != nil { - return err - } - - // t.Piece (cid.Cid) (struct) - - if err := cbg.WriteCid(w, t.Piece); err != nil { - return xerrors.Errorf("failed to write cid field t.Piece: %w", err) - } - - return nil -} - -func (t *Proposal) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 2 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.DealProposal (actors.StorageDealProposal) (struct) - - { - - pb, err := br.PeekByte() - if err != nil { - return err - } - if pb == cbg.CborNull[0] { - var nbuf [1]byte - if _, err := br.Read(nbuf[:]); err != nil { - return err - } - } else { - t.DealProposal = new(actors.StorageDealProposal) - if err := t.DealProposal.UnmarshalCBOR(br); err != nil { - return err - } - } - - } - // t.Piece (cid.Cid) (struct) - - { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Piece: %w", err) - } - - t.Piece = c - - } - return nil -} - -func (t *Response) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{132}); err != nil { - return err - } - - // t.State (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.State))); err != nil { - return err - } - - // t.Message (string) (string) - if len(t.Message) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.Message was too long") - } - - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Message)))); err != nil { - return err - } - if _, err := w.Write([]byte(t.Message)); err != nil { - return err - } - - // t.Proposal (cid.Cid) (struct) - - if err := cbg.WriteCid(w, t.Proposal); err != nil { - return xerrors.Errorf("failed to write cid field t.Proposal: %w", err) - } - - // t.StorageDealSubmission (types.SignedMessage) (struct) - if err := t.StorageDealSubmission.MarshalCBOR(w); err != nil { - return err - } - return nil -} - -func (t *Response) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 4 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.State (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.State = uint64(extra) - // t.Message (string) (string) - - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - t.Message = string(sval) - } - // t.Proposal (cid.Cid) (struct) - - { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Proposal: %w", err) - } - - t.Proposal = c - - } - // t.StorageDealSubmission (types.SignedMessage) (struct) - - { - - pb, err := br.PeekByte() - if err != nil { - return err - } - if pb == cbg.CborNull[0] { - var nbuf [1]byte - if _, err := br.Read(nbuf[:]); err != nil { - return err - } - } else { - t.StorageDealSubmission = new(types.SignedMessage) - if err := t.StorageDealSubmission.UnmarshalCBOR(br); err != nil { - return err - } - } - - } - return nil -} - -func (t *SignedResponse) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{130}); err != nil { - return err - } - - // t.Response (deals.Response) (struct) - if err := t.Response.MarshalCBOR(w); err != nil { - return err - } - - // t.Signature (types.Signature) (struct) - if err := t.Signature.MarshalCBOR(w); err != nil { - return err - } - return nil -} - -func (t *SignedResponse) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 2 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Response (deals.Response) (struct) - - { - - if err := t.Response.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.Signature (types.Signature) (struct) - - { - - pb, err := br.PeekByte() - if err != nil { - return err - } - if pb == cbg.CborNull[0] { - var nbuf [1]byte - if _, err := br.Read(nbuf[:]); err != nil { - return err - } - } else { - t.Signature = new(types.Signature) - if err := t.Signature.UnmarshalCBOR(br); err != nil { - return err - } - } - - } - return nil -} - -func (t *ClientDealProposal) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{136}); err != nil { - return err - } - - // t.Data (cid.Cid) (struct) - - if err := cbg.WriteCid(w, t.Data); err != nil { - return xerrors.Errorf("failed to write cid field t.Data: %w", err) - } - - // t.PricePerEpoch (types.BigInt) (struct) - if err := t.PricePerEpoch.MarshalCBOR(w); err != nil { - return err - } - - // t.ProposalExpiration (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.ProposalExpiration))); err != nil { - return err - } - - // t.Duration (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Duration))); err != nil { - return err - } - - // t.ProviderAddress (address.Address) (struct) - if err := t.ProviderAddress.MarshalCBOR(w); err != nil { - return err - } - - // t.Client (address.Address) (struct) - if err := t.Client.MarshalCBOR(w); err != nil { - return err - } - - // t.MinerWorker (address.Address) (struct) - if err := t.MinerWorker.MarshalCBOR(w); err != nil { - return err - } - - // t.MinerID (peer.ID) (string) - if len(t.MinerID) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.MinerID was too long") - } - - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.MinerID)))); err != nil { - return err - } - if _, err := w.Write([]byte(t.MinerID)); err != nil { - return err - } - return nil -} - -func (t *ClientDealProposal) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 8 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Data (cid.Cid) (struct) - - { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Data: %w", err) - } - - t.Data = c - - } - // t.PricePerEpoch (types.BigInt) (struct) - - { - - if err := t.PricePerEpoch.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.ProposalExpiration (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.ProposalExpiration = uint64(extra) - // t.Duration (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Duration = uint64(extra) - // t.ProviderAddress (address.Address) (struct) - - { - - if err := t.ProviderAddress.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.Client (address.Address) (struct) - - { - - if err := t.Client.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.MinerWorker (address.Address) (struct) - - { - - if err := t.MinerWorker.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.MinerID (peer.ID) (string) - - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - t.MinerID = peer.ID(sval) - } - return nil -} - -func (t *ClientDeal) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{135}); err != nil { - return err - } - - // t.ProposalCid (cid.Cid) (struct) - - if err := cbg.WriteCid(w, t.ProposalCid); err != nil { - return xerrors.Errorf("failed to write cid field t.ProposalCid: %w", err) - } - - // t.Proposal (actors.StorageDealProposal) (struct) - if err := t.Proposal.MarshalCBOR(w); err != nil { - return err - } - - // t.State (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.State))); err != nil { - return err - } - - // t.Miner (peer.ID) (string) - if len(t.Miner) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.Miner was too long") - } - - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Miner)))); err != nil { - return err - } - if _, err := w.Write([]byte(t.Miner)); err != nil { - return err - } - - // t.MinerWorker (address.Address) (struct) - if err := t.MinerWorker.MarshalCBOR(w); err != nil { - return err - } - - // t.DealID (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.DealID))); err != nil { - return err - } - - // t.PublishMessage (types.SignedMessage) (struct) - if err := t.PublishMessage.MarshalCBOR(w); err != nil { - return err - } - return nil -} - -func (t *ClientDeal) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 7 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.ProposalCid (cid.Cid) (struct) - - { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.ProposalCid: %w", err) - } - - t.ProposalCid = c - - } - // t.Proposal (actors.StorageDealProposal) (struct) - - { - - if err := t.Proposal.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.State (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.State = uint64(extra) - // t.Miner (peer.ID) (string) - - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - t.Miner = peer.ID(sval) - } - // t.MinerWorker (address.Address) (struct) - - { - - if err := t.MinerWorker.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.DealID (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.DealID = uint64(extra) - // t.PublishMessage (types.SignedMessage) (struct) - - { - - pb, err := br.PeekByte() - if err != nil { - return err - } - if pb == cbg.CborNull[0] { - var nbuf [1]byte - if _, err := br.Read(nbuf[:]); err != nil { - return err - } - } else { - t.PublishMessage = new(types.SignedMessage) - if err := t.PublishMessage.UnmarshalCBOR(br); err != nil { - return err - } - } - - } - return nil -} - -func (t *MinerDeal) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{135}); err != nil { - return err - } - - // t.Client (peer.ID) (string) - if len(t.Client) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.Client was too long") - } - - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Client)))); err != nil { - return err - } - if _, err := w.Write([]byte(t.Client)); err != nil { - return err - } - - // t.Proposal (actors.StorageDealProposal) (struct) - if err := t.Proposal.MarshalCBOR(w); err != nil { - return err - } - - // t.ProposalCid (cid.Cid) (struct) - - if err := cbg.WriteCid(w, t.ProposalCid); err != nil { - return xerrors.Errorf("failed to write cid field t.ProposalCid: %w", err) - } - - // t.State (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.State))); err != nil { - return err - } - - // t.Ref (cid.Cid) (struct) - - if err := cbg.WriteCid(w, t.Ref); err != nil { - return xerrors.Errorf("failed to write cid field t.Ref: %w", err) - } - - // t.DealID (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.DealID))); err != nil { - return err - } - - // t.SectorID (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SectorID))); err != nil { - return err - } - return nil -} - -func (t *MinerDeal) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 7 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Client (peer.ID) (string) - - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - t.Client = peer.ID(sval) - } - // t.Proposal (actors.StorageDealProposal) (struct) - - { - - if err := t.Proposal.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.ProposalCid (cid.Cid) (struct) - - { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.ProposalCid: %w", err) - } - - t.ProposalCid = c - - } - // t.State (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.State = uint64(extra) - // t.Ref (cid.Cid) (struct) - - { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Ref: %w", err) - } - - t.Ref = c - - } - // t.DealID (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.DealID = uint64(extra) - // t.SectorID (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.SectorID = uint64(extra) - return nil -} - -func (t *StorageDataTransferVoucher) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{130}); err != nil { - return err - } - - // t.Proposal (cid.Cid) (struct) - - if err := cbg.WriteCid(w, t.Proposal); err != nil { - return xerrors.Errorf("failed to write cid field t.Proposal: %w", err) - } - - // t.DealID (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.DealID))); err != nil { - return err - } - return nil -} - -func (t *StorageDataTransferVoucher) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 2 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Proposal (cid.Cid) (struct) - - { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Proposal: %w", err) - } - - t.Proposal = c - - } - // t.DealID (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.DealID = uint64(extra) - return nil -} diff --git a/chain/deals/client.go b/chain/deals/client.go deleted file mode 100644 index 709432f63..000000000 --- a/chain/deals/client.go +++ /dev/null @@ -1,311 +0,0 @@ -package deals - -import ( - "context" - - "github.com/ipfs/go-cid" - logging "github.com/ipfs/go-log" - "github.com/libp2p/go-libp2p-core/host" - inet "github.com/libp2p/go-libp2p-core/network" - "github.com/libp2p/go-libp2p-core/peer" - "golang.org/x/xerrors" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/go-statestore" - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/events" - "github.com/filecoin-project/lotus/chain/market" - "github.com/filecoin-project/lotus/chain/stmgr" - "github.com/filecoin-project/lotus/chain/store" - "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/chain/wallet" - "github.com/filecoin-project/lotus/node/impl/full" - "github.com/filecoin-project/lotus/node/modules/dtypes" - "github.com/filecoin-project/lotus/retrieval/discovery" -) - -var log = logging.Logger("deals") - -type ClientDeal struct { - ProposalCid cid.Cid - Proposal actors.StorageDealProposal - State api.DealState - Miner peer.ID - MinerWorker address.Address - DealID uint64 - - PublishMessage *types.SignedMessage - - s inet.Stream -} - -type Client struct { - sm *stmgr.StateManager - chain *store.ChainStore - h host.Host - w *wallet.Wallet - // dataTransfer - // TODO: once the data transfer module is complete, the - // client will listen to events on the data transfer module - // Because we are using only a fake DAGService - // implementation, there's no validation or events on the client side - dataTransfer dtypes.ClientDataTransfer - dag dtypes.ClientDAG - discovery *discovery.Local - events *events.Events - fm *market.FundMgr - - deals *statestore.StateStore - conns map[cid.Cid]inet.Stream - - incoming chan *ClientDeal - updated chan clientDealUpdate - - stop chan struct{} - stopped chan struct{} -} - -type clientDealUpdate struct { - newState api.DealState - id cid.Cid - err error - mut func(*ClientDeal) -} - -type clientApi struct { - full.ChainAPI - full.StateAPI -} - -func NewClient(sm *stmgr.StateManager, chain *store.ChainStore, h host.Host, w *wallet.Wallet, dag dtypes.ClientDAG, dataTransfer dtypes.ClientDataTransfer, discovery *discovery.Local, fm *market.FundMgr, deals dtypes.ClientDealStore, chainapi full.ChainAPI, stateapi full.StateAPI) *Client { - c := &Client{ - sm: sm, - chain: chain, - h: h, - w: w, - dataTransfer: dataTransfer, - dag: dag, - discovery: discovery, - fm: fm, - events: events.NewEvents(context.TODO(), &clientApi{chainapi, stateapi}), - - deals: deals, - conns: map[cid.Cid]inet.Stream{}, - - incoming: make(chan *ClientDeal, 16), - updated: make(chan clientDealUpdate, 16), - - stop: make(chan struct{}), - stopped: make(chan struct{}), - } - - return c -} - -func (c *Client) Run(ctx context.Context) { - go func() { - defer close(c.stopped) - - for { - select { - case deal := <-c.incoming: - c.onIncoming(deal) - case update := <-c.updated: - c.onUpdated(ctx, update) - case <-c.stop: - return - } - } - }() -} - -func (c *Client) onIncoming(deal *ClientDeal) { - log.Info("incoming deal") - - if _, ok := c.conns[deal.ProposalCid]; ok { - log.Errorf("tracking deal connection: already tracking connection for deal %s", deal.ProposalCid) - return - } - c.conns[deal.ProposalCid] = deal.s - - if err := c.deals.Begin(deal.ProposalCid, deal); err != nil { - // We may have re-sent the proposal - log.Errorf("deal tracking failed: %s", err) - c.failDeal(deal.ProposalCid, err) - return - } - - go func() { - c.updated <- clientDealUpdate{ - newState: api.DealUnknown, - id: deal.ProposalCid, - err: nil, - } - }() -} - -func (c *Client) onUpdated(ctx context.Context, update clientDealUpdate) { - log.Infof("Client deal %s updated state to %s", update.id, api.DealStates[update.newState]) - var deal ClientDeal - err := c.deals.Mutate(update.id, func(d *ClientDeal) error { - d.State = update.newState - if update.mut != nil { - update.mut(d) - } - deal = *d - return nil - }) - if update.err != nil { - log.Errorf("deal %s failed: %s", update.id, update.err) - c.failDeal(update.id, update.err) - return - } - if err != nil { - c.failDeal(update.id, err) - return - } - - switch update.newState { - case api.DealUnknown: // new - c.handle(ctx, deal, c.new, api.DealAccepted) - case api.DealAccepted: - c.handle(ctx, deal, c.accepted, api.DealStaged) - case api.DealStaged: - c.handle(ctx, deal, c.staged, api.DealSealing) - case api.DealSealing: - c.handle(ctx, deal, c.sealing, api.DealNoUpdate) - // TODO: DealComplete -> watch for faults, expiration, etc. - } -} - -type ClientDealProposal struct { - Data cid.Cid - - PricePerEpoch types.BigInt - ProposalExpiration uint64 - Duration uint64 - - ProviderAddress address.Address - Client address.Address - MinerWorker address.Address - MinerID peer.ID -} - -func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, error) { - if err := c.fm.EnsureAvailable(ctx, p.Client, types.BigMul(p.PricePerEpoch, types.NewInt(p.Duration))); err != nil { - return cid.Undef, xerrors.Errorf("adding market funds failed: %w", err) - } - - commP, pieceSize, err := c.commP(ctx, p.Data) - if err != nil { - return cid.Undef, xerrors.Errorf("computing commP failed: %w", err) - } - - dealProposal := &actors.StorageDealProposal{ - PieceRef: commP, - PieceSize: uint64(pieceSize), - Client: p.Client, - Provider: p.ProviderAddress, - ProposalExpiration: p.ProposalExpiration, - Duration: p.Duration, - StoragePricePerEpoch: p.PricePerEpoch, - StorageCollateral: types.NewInt(uint64(pieceSize)), // TODO: real calc - } - - if err := api.SignWith(ctx, c.w.Sign, p.Client, dealProposal); err != nil { - return cid.Undef, xerrors.Errorf("signing deal proposal failed: %w", err) - } - - proposalNd, err := cborutil.AsIpld(dealProposal) - if err != nil { - return cid.Undef, xerrors.Errorf("getting proposal node failed: %w", err) - } - - s, err := c.h.NewStream(ctx, p.MinerID, DealProtocolID) - if err != nil { - return cid.Undef, xerrors.Errorf("connecting to storage provider failed: %w", err) - } - - proposal := &Proposal{ - DealProposal: dealProposal, - Piece: p.Data, - } - - if err := cborutil.WriteCborRPC(s, proposal); err != nil { - s.Reset() - return cid.Undef, xerrors.Errorf("sending proposal to storage provider failed: %w", err) - } - - deal := &ClientDeal{ - ProposalCid: proposalNd.Cid(), - Proposal: *dealProposal, - State: api.DealUnknown, - Miner: p.MinerID, - MinerWorker: p.MinerWorker, - - s: s, - } - - c.incoming <- deal - - return deal.ProposalCid, c.discovery.AddPeer(p.Data, discovery.RetrievalPeer{ - Address: dealProposal.Provider, - ID: deal.Miner, - }) -} - -func (c *Client) QueryAsk(ctx context.Context, p peer.ID, a address.Address) (*types.SignedStorageAsk, error) { - s, err := c.h.NewStream(ctx, p, AskProtocolID) - if err != nil { - return nil, xerrors.Errorf("failed to open stream to miner: %w", err) - } - - req := &AskRequest{ - Miner: a, - } - if err := cborutil.WriteCborRPC(s, req); err != nil { - return nil, xerrors.Errorf("failed to send ask request: %w", err) - } - - var out AskResponse - if err := cborutil.ReadCborRPC(s, &out); err != nil { - return nil, xerrors.Errorf("failed to read ask response: %w", err) - } - - if out.Ask == nil { - return nil, xerrors.Errorf("got no ask back") - } - - if out.Ask.Ask.Miner != a { - return nil, xerrors.Errorf("got back ask for wrong miner") - } - - if err := c.checkAskSignature(out.Ask); err != nil { - return nil, xerrors.Errorf("ask was not properly signed") - } - - return out.Ask, nil -} - -func (c *Client) List() ([]ClientDeal, error) { - var out []ClientDeal - if err := c.deals.List(&out); err != nil { - return nil, err - } - return out, nil -} - -func (c *Client) GetDeal(d cid.Cid) (*ClientDeal, error) { - var out ClientDeal - if err := c.deals.Get(d, &out); err != nil { - return nil, err - } - return &out, nil -} - -func (c *Client) Stop() { - close(c.stop) - <-c.stopped -} diff --git a/chain/deals/client_states.go b/chain/deals/client_states.go deleted file mode 100644 index d19765f60..000000000 --- a/chain/deals/client_states.go +++ /dev/null @@ -1,233 +0,0 @@ -package deals - -import ( - "bytes" - "context" - - "golang.org/x/xerrors" - - "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/stmgr" - "github.com/filecoin-project/lotus/chain/types" -) - -type clientHandlerFunc func(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) - -func (c *Client) handle(ctx context.Context, deal ClientDeal, cb clientHandlerFunc, next api.DealState) { - go func() { - mut, err := cb(ctx, deal) - if err != nil { - next = api.DealError - } - - if err == nil && next == api.DealNoUpdate { - return - } - - select { - case c.updated <- clientDealUpdate{ - newState: next, - id: deal.ProposalCid, - err: err, - mut: mut, - }: - case <-c.stop: - } - }() -} - -func (c *Client) new(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) { - resp, err := c.readStorageDealResp(deal) - if err != nil { - return nil, err - } - - // TODO: verify StorageDealSubmission - - if err := c.disconnect(deal); err != nil { - return nil, err - } - - /* data transfer happens */ - if resp.State != api.DealAccepted { - return nil, xerrors.Errorf("deal wasn't accepted (State=%d)", resp.State) - } - - return func(info *ClientDeal) { - info.PublishMessage = resp.StorageDealSubmission - }, nil -} - -func (c *Client) accepted(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) { - log.Infow("DEAL ACCEPTED!") - - pubmsg := deal.PublishMessage.Message - pw, err := stmgr.GetMinerWorker(ctx, c.sm, nil, deal.Proposal.Provider) - if err != nil { - return nil, xerrors.Errorf("getting miner worker failed: %w", err) - } - - if pubmsg.From != pw { - return nil, xerrors.Errorf("deal wasn't published by storage provider: from=%s, provider=%s", pubmsg.From, deal.Proposal.Provider) - } - - if pubmsg.To != actors.StorageMarketAddress { - return nil, xerrors.Errorf("deal publish message wasn't set to StorageMarket actor (to=%s)", pubmsg.To) - } - - if pubmsg.Method != actors.SMAMethods.PublishStorageDeals { - return nil, xerrors.Errorf("deal publish message called incorrect method (method=%s)", pubmsg.Method) - } - - var params actors.PublishStorageDealsParams - if err := params.UnmarshalCBOR(bytes.NewReader(pubmsg.Params)); err != nil { - return nil, err - } - - dealIdx := -1 - for i, storageDeal := range params.Deals { - // TODO: make it less hacky - sd := storageDeal - eq, err := cborutil.Equals(&deal.Proposal, &sd) - if err != nil { - return nil, err - } - if eq { - dealIdx = i - break - } - } - - if dealIdx == -1 { - return nil, xerrors.Errorf("deal publish didn't contain our deal (message cid: %s)", deal.PublishMessage.Cid()) - } - - // TODO: timeout - _, ret, err := c.sm.WaitForMessage(ctx, deal.PublishMessage.Cid()) - if err != nil { - return nil, xerrors.Errorf("waiting for deal publish message: %w", err) - } - if ret.ExitCode != 0 { - return nil, xerrors.Errorf("deal publish failed: exit=%d", ret.ExitCode) - } - - var res actors.PublishStorageDealResponse - if err := res.UnmarshalCBOR(bytes.NewReader(ret.Return)); err != nil { - return nil, err - } - - return func(info *ClientDeal) { - info.DealID = res.DealIDs[dealIdx] - }, nil -} - -func (c *Client) staged(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) { - // TODO: Maybe wait for pre-commit - - return nil, nil -} - -func (c *Client) sealing(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) { - checkFunc := func(ts *types.TipSet) (done bool, more bool, err error) { - sd, err := stmgr.GetStorageDeal(ctx, c.sm, deal.DealID, ts) - if err != nil { - // TODO: This may be fine for some errors - return false, false, xerrors.Errorf("failed to look up deal on chain: %w", err) - } - - if sd.ActivationEpoch > 0 { - select { - case c.updated <- clientDealUpdate{ - newState: api.DealComplete, - id: deal.ProposalCid, - }: - case <-c.stop: - } - - return true, false, nil - } - - return false, true, nil - } - - called := func(msg *types.Message, rec *types.MessageReceipt, ts *types.TipSet, curH uint64) (more bool, err error) { - defer func() { - if err != nil { - select { - case c.updated <- clientDealUpdate{ - newState: api.DealComplete, - id: deal.ProposalCid, - err: xerrors.Errorf("handling applied event: %w", err), - }: - case <-c.stop: - } - } - }() - - if msg == nil { - log.Error("timed out waiting for deal activation... what now?") - return false, nil - } - - sd, err := stmgr.GetStorageDeal(ctx, c.sm, deal.DealID, ts) - if err != nil { - return false, xerrors.Errorf("failed to look up deal on chain: %w", err) - } - - if sd.ActivationEpoch == 0 { - return false, xerrors.Errorf("deal wasn't active: deal=%d, parentState=%s, h=%d", deal.DealID, ts.ParentState(), ts.Height()) - } - - log.Infof("Storage deal %d activated at epoch %d", deal.DealID, sd.ActivationEpoch) - - select { - case c.updated <- clientDealUpdate{ - newState: api.DealComplete, - id: deal.ProposalCid, - }: - case <-c.stop: - } - - return false, nil - } - - revert := func(ctx context.Context, ts *types.TipSet) error { - log.Warn("deal activation reverted; TODO: actually handle this!") - // TODO: Just go back to DealSealing? - return nil - } - - matchEvent := func(msg *types.Message) (bool, error) { - if msg.To != deal.Proposal.Provider { - return false, nil - } - - if msg.Method != actors.MAMethods.ProveCommitSector { - return false, nil - } - - var params actors.SectorProveCommitInfo - if err := params.UnmarshalCBOR(bytes.NewReader(msg.Params)); err != nil { - return false, err - } - - var found bool - for _, dealID := range params.DealIDs { - if dealID == deal.DealID { - found = true - break - } - } - - return found, nil - } - - if err := c.events.Called(checkFunc, called, revert, 3, build.SealRandomnessLookbackLimit, matchEvent); err != nil { - return nil, xerrors.Errorf("failed to set up called handler") - } - - return nil, nil -} diff --git a/chain/deals/client_utils.go b/chain/deals/client_utils.go deleted file mode 100644 index 944846ed0..000000000 --- a/chain/deals/client_utils.go +++ /dev/null @@ -1,171 +0,0 @@ -package deals - -import ( - "bytes" - "context" - "runtime" - - sectorbuilder "github.com/filecoin-project/go-sectorbuilder" - "github.com/ipfs/go-cid" - files "github.com/ipfs/go-ipfs-files" - unixfile "github.com/ipfs/go-unixfs/file" - "github.com/ipld/go-ipld-prime" - "github.com/libp2p/go-libp2p-core/peer" - "golang.org/x/xerrors" - - cborutil "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/go-statestore" - - "github.com/filecoin-project/lotus/datatransfer" - "github.com/filecoin-project/lotus/lib/padreader" - "github.com/filecoin-project/lotus/node/modules/dtypes" -) - -func (c *Client) failDeal(id cid.Cid, cerr error) { - if cerr == nil { - _, f, l, _ := runtime.Caller(1) - cerr = xerrors.Errorf("unknown error (fail called at %s:%d)", f, l) - } - - s, ok := c.conns[id] - if ok { - _ = s.Reset() - delete(c.conns, id) - } - - // TODO: store in some sort of audit log - log.Errorf("deal %s failed: %+v", id, cerr) -} - -func (c *Client) commP(ctx context.Context, data cid.Cid) ([]byte, uint64, error) { - root, err := c.dag.Get(ctx, data) - if err != nil { - log.Errorf("failed to get file root for deal: %s", err) - return nil, 0, err - } - - n, err := unixfile.NewUnixfsFile(ctx, c.dag, root) - if err != nil { - log.Errorf("cannot open unixfs file: %s", err) - return nil, 0, err - } - - uf, ok := n.(files.File) - if !ok { - // TODO: we probably got directory, how should we handle this in unixfs mode? - return nil, 0, xerrors.New("unsupported unixfs type") - } - - s, err := uf.Size() - if err != nil { - return nil, 0, err - } - - pr, psize := padreader.New(uf, uint64(s)) - - commp, err := sectorbuilder.GeneratePieceCommitment(pr, psize) - if err != nil { - return nil, 0, xerrors.Errorf("generating CommP: %w", err) - } - - return commp[:], psize, nil -} - -func (c *Client) readStorageDealResp(deal ClientDeal) (*Response, error) { - s, ok := c.conns[deal.ProposalCid] - if !ok { - // TODO: Try to re-establish the connection using query protocol - return nil, xerrors.Errorf("no connection to miner") - } - - var resp SignedResponse - if err := cborutil.ReadCborRPC(s, &resp); err != nil { - log.Errorw("failed to read Response message", "error", err) - return nil, err - } - - if err := resp.Verify(deal.MinerWorker); err != nil { - return nil, xerrors.Errorf("verifying response signature failed", err) - } - - if resp.Response.Proposal != deal.ProposalCid { - return nil, xerrors.Errorf("miner responded to a wrong proposal: %s != %s", resp.Response.Proposal, deal.ProposalCid) - } - - return &resp.Response, nil -} - -func (c *Client) disconnect(deal ClientDeal) error { - s, ok := c.conns[deal.ProposalCid] - if !ok { - return nil - } - - err := s.Close() - delete(c.conns, deal.ProposalCid) - return err -} - -var _ datatransfer.RequestValidator = &ClientRequestValidator{} - -// ClientRequestValidator validates data transfer requests for the client -// in a storage market -type ClientRequestValidator struct { - deals *statestore.StateStore -} - -// NewClientRequestValidator returns a new client request validator for the -// given datastore -func NewClientRequestValidator(deals dtypes.ClientDealStore) *ClientRequestValidator { - crv := &ClientRequestValidator{ - deals: deals, - } - return crv -} - -// ValidatePush validates a push request received from the peer that will send data -// Will always error because clients should not accept push requests from a provider -// in a storage deal (i.e. send data to client). -func (c *ClientRequestValidator) ValidatePush( - sender peer.ID, - voucher datatransfer.Voucher, - baseCid cid.Cid, - Selector ipld.Node) error { - return ErrNoPushAccepted -} - -// ValidatePull validates a pull request received from the peer that will receive data -// Will succeed only if: -// - voucher has correct type -// - voucher references an active deal -// - referenced deal matches the receiver (miner) -// - referenced deal matches the given base CID -// - referenced deal is in an acceptable state -func (c *ClientRequestValidator) ValidatePull( - receiver peer.ID, - voucher datatransfer.Voucher, - baseCid cid.Cid, - Selector ipld.Node) error { - dealVoucher, ok := voucher.(*StorageDataTransferVoucher) - if !ok { - return xerrors.Errorf("voucher type %s: %w", voucher.Identifier(), ErrWrongVoucherType) - } - - var deal ClientDeal - err := c.deals.Get(dealVoucher.Proposal, &deal) - if err != nil { - return xerrors.Errorf("Proposal CID %s: %w", dealVoucher.Proposal.String(), ErrNoDeal) - } - if deal.Miner != receiver { - return xerrors.Errorf("Deal Peer %s, Data Transfer Peer %s: %w", deal.Miner.String(), receiver.String(), ErrWrongPeer) - } - if !bytes.Equal(deal.Proposal.PieceRef, baseCid.Bytes()) { - return xerrors.Errorf("Deal Payload CID %s, Data Transfer CID %s: %w", string(deal.Proposal.PieceRef), baseCid.String(), ErrWrongPiece) - } - for _, state := range DataTransferStates { - if deal.State == state { - return nil - } - } - return xerrors.Errorf("Deal State %s: %w", deal.State, ErrInacceptableDealState) -} diff --git a/chain/deals/provider.go b/chain/deals/provider.go deleted file mode 100644 index 9e45ef8b2..000000000 --- a/chain/deals/provider.go +++ /dev/null @@ -1,294 +0,0 @@ -package deals - -import ( - "context" - "errors" - "sync" - - cid "github.com/ipfs/go-cid" - datastore "github.com/ipfs/go-datastore" - "github.com/ipfs/go-datastore/namespace" - inet "github.com/libp2p/go-libp2p-core/network" - "github.com/libp2p/go-libp2p-core/peer" - "golang.org/x/xerrors" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/go-statestore" - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/datatransfer" - "github.com/filecoin-project/lotus/node/modules/dtypes" - "github.com/filecoin-project/lotus/storage" - "github.com/filecoin-project/lotus/storage/sectorblocks" -) - -var ProviderDsPrefix = "/deals/provider" - -type MinerDeal struct { - Client peer.ID - Proposal actors.StorageDealProposal - ProposalCid cid.Cid - State api.DealState - - Ref cid.Cid - - DealID uint64 - SectorID uint64 // Set when State >= DealStaged - - s inet.Stream -} - -type Provider struct { - pricePerByteBlock types.BigInt // how much we want for storing one byte for one block - minPieceSize uint64 - - ask *types.SignedStorageAsk - askLk sync.Mutex - - secb *sectorblocks.SectorBlocks - sminer *storage.Miner - full api.FullNode - - // TODO: This will go away once storage market module + CAR - // is implemented - dag dtypes.StagingDAG - - // dataTransfer is the manager of data transfers used by this storage provider - dataTransfer dtypes.ProviderDataTransfer - - deals *statestore.StateStore - ds dtypes.MetadataDS - - conns map[cid.Cid]inet.Stream - - actor address.Address - - incoming chan MinerDeal - updated chan minerDealUpdate - stop chan struct{} - stopped chan struct{} -} - -type minerDealUpdate struct { - newState api.DealState - id cid.Cid - err error - mut func(*MinerDeal) -} - -var ( - // ErrDataTransferFailed means a data transfer for a deal failed - ErrDataTransferFailed = errors.New("deal data transfer failed") -) - -func NewProvider(ds dtypes.MetadataDS, sminer *storage.Miner, secb *sectorblocks.SectorBlocks, dag dtypes.StagingDAG, dataTransfer dtypes.ProviderDataTransfer, fullNode api.FullNode) (*Provider, error) { - addr, err := ds.Get(datastore.NewKey("miner-address")) - if err != nil { - return nil, err - } - minerAddress, err := address.NewFromBytes(addr) - if err != nil { - return nil, err - } - - h := &Provider{ - sminer: sminer, - dag: dag, - dataTransfer: dataTransfer, - full: fullNode, - secb: secb, - - pricePerByteBlock: types.NewInt(3), // TODO: allow setting - minPieceSize: 256, // TODO: allow setting (BUT KEEP MIN 256! (because of how we fill sectors up)) - - conns: map[cid.Cid]inet.Stream{}, - - incoming: make(chan MinerDeal), - updated: make(chan minerDealUpdate), - stop: make(chan struct{}), - stopped: make(chan struct{}), - - actor: minerAddress, - - deals: statestore.New(namespace.Wrap(ds, datastore.NewKey(ProviderDsPrefix))), - ds: ds, - } - - if err := h.tryLoadAsk(); err != nil { - return nil, err - } - - if h.ask == nil { - // TODO: we should be fine with this state, and just say it means 'not actively accepting deals' - // for now... lets just set a price - if err := h.SetPrice(types.NewInt(500_000_000), 1000000); err != nil { - return nil, xerrors.Errorf("failed setting a default price: %w", err) - } - } - - // register a data transfer event handler -- this will move deals from - // accepted to staged - h.dataTransfer.SubscribeToEvents(h.onDataTransferEvent) - - return h, nil -} - -func (p *Provider) Run(ctx context.Context) { - // TODO: restore state - - go func() { - defer log.Warn("quitting deal provider loop") - defer close(p.stopped) - - for { - select { - case deal := <-p.incoming: // DealAccepted - p.onIncoming(deal) - case update := <-p.updated: // DealStaged - p.onUpdated(ctx, update) - case <-p.stop: - return - } - } - }() -} - -func (p *Provider) onIncoming(deal MinerDeal) { - log.Info("incoming deal") - - p.conns[deal.ProposalCid] = deal.s - - if err := p.deals.Begin(deal.ProposalCid, &deal); err != nil { - // This can happen when client re-sends proposal - p.failDeal(deal.ProposalCid, err) - log.Errorf("deal tracking failed: %s", err) - return - } - - go func() { - p.updated <- minerDealUpdate{ - newState: api.DealAccepted, - id: deal.ProposalCid, - err: nil, - } - }() -} - -func (p *Provider) onUpdated(ctx context.Context, update minerDealUpdate) { - log.Infof("Deal %s updated state to %s", update.id, api.DealStates[update.newState]) - if update.err != nil { - log.Errorf("deal %s (newSt: %d) failed: %+v", update.id, update.newState, update.err) - p.failDeal(update.id, update.err) - return - } - var deal MinerDeal - err := p.deals.Mutate(update.id, func(d *MinerDeal) error { - d.State = update.newState - if update.mut != nil { - update.mut(d) - } - deal = *d - return nil - }) - if err != nil { - p.failDeal(update.id, err) - return - } - - switch update.newState { - case api.DealAccepted: - p.handle(ctx, deal, p.accept, api.DealNoUpdate) - case api.DealStaged: - p.handle(ctx, deal, p.staged, api.DealSealing) - case api.DealSealing: - p.handle(ctx, deal, p.sealing, api.DealComplete) - case api.DealComplete: - p.handle(ctx, deal, p.complete, api.DealNoUpdate) - } -} - -// onDataTransferEvent is the function called when an event occurs in a data -// transfer -- it reads the voucher to verify this even occurred in a storage -// market deal, then, based on the data transfer event that occurred, it generates -// and update message for the deal -- either moving to staged for a completion -// event or moving to error if a data transfer error occurs -func (p *Provider) onDataTransferEvent(event datatransfer.Event, channelState datatransfer.ChannelState) { - voucher, ok := channelState.Voucher().(*StorageDataTransferVoucher) - // if this event is for a transfer not related to storage, ignore - if !ok { - return - } - - // data transfer events for opening and progress do not affect deal state - var next api.DealState - var err error - var mut func(*MinerDeal) - switch event { - case datatransfer.Complete: - next = api.DealStaged - mut = func(deal *MinerDeal) { - deal.DealID = voucher.DealID - } - case datatransfer.Error: - next = api.DealFailed - err = ErrDataTransferFailed - default: - // the only events we care about are complete and error - return - } - - select { - case p.updated <- minerDealUpdate{ - newState: next, - id: voucher.Proposal, - err: err, - mut: mut, - }: - case <-p.stop: - } -} - -func (p *Provider) newDeal(s inet.Stream, proposal Proposal) (MinerDeal, error) { - proposalNd, err := cborutil.AsIpld(proposal.DealProposal) - if err != nil { - return MinerDeal{}, err - } - - return MinerDeal{ - Client: s.Conn().RemotePeer(), - Proposal: *proposal.DealProposal, - ProposalCid: proposalNd.Cid(), - State: api.DealUnknown, - - Ref: proposal.Piece, - - s: s, - }, nil -} - -func (p *Provider) HandleStream(s inet.Stream) { - log.Info("Handling storage deal proposal!") - - proposal, err := p.readProposal(s) - if err != nil { - log.Error(err) - s.Close() - return - } - - deal, err := p.newDeal(s, proposal) - if err != nil { - log.Errorf("%+v", err) - s.Close() - return - } - - p.incoming <- deal -} - -func (p *Provider) Stop() { - close(p.stop) - <-p.stopped -} diff --git a/chain/deals/provider_asks.go b/chain/deals/provider_asks.go deleted file mode 100644 index 1170eb28b..000000000 --- a/chain/deals/provider_asks.go +++ /dev/null @@ -1,160 +0,0 @@ -package deals - -import ( - "bytes" - "context" - "time" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/lotus/chain/stmgr" - "github.com/filecoin-project/lotus/chain/types" - datastore "github.com/ipfs/go-datastore" - inet "github.com/libp2p/go-libp2p-core/network" - "golang.org/x/xerrors" -) - -func (p *Provider) SetPrice(price types.BigInt, ttlsecs int64) error { - p.askLk.Lock() - defer p.askLk.Unlock() - - var seqno uint64 - if p.ask != nil { - seqno = p.ask.Ask.SeqNo + 1 - } - - now := time.Now().Unix() - ask := &types.StorageAsk{ - Price: price, - Timestamp: uint64(now), - Expiry: uint64(now + ttlsecs), - Miner: p.actor, - SeqNo: seqno, - MinPieceSize: p.minPieceSize, - } - - ssa, err := p.signAsk(ask) - if err != nil { - return err - } - - return p.saveAsk(ssa) -} - -func (p *Provider) getAsk(m address.Address) *types.SignedStorageAsk { - p.askLk.Lock() - defer p.askLk.Unlock() - if m != p.actor { - return nil - } - - return p.ask -} - -func (p *Provider) HandleAskStream(s inet.Stream) { - defer s.Close() - var ar AskRequest - if err := cborutil.ReadCborRPC(s, &ar); err != nil { - log.Errorf("failed to read AskRequest from incoming stream: %s", err) - return - } - - resp := p.processAskRequest(&ar) - - if err := cborutil.WriteCborRPC(s, resp); err != nil { - log.Errorf("failed to write ask response: %s", err) - return - } -} - -func (p *Provider) processAskRequest(ar *AskRequest) *AskResponse { - return &AskResponse{ - Ask: p.getAsk(ar.Miner), - } -} - -var bestAskKey = datastore.NewKey("latest-ask") - -func (p *Provider) tryLoadAsk() error { - p.askLk.Lock() - defer p.askLk.Unlock() - - err := p.loadAsk() - if err != nil { - if xerrors.Is(err, datastore.ErrNotFound) { - log.Warn("no previous ask found, miner will not accept deals until a price is set") - return nil - } - return err - } - - return nil -} - -func (p *Provider) loadAsk() error { - askb, err := p.ds.Get(datastore.NewKey("latest-ask")) - if err != nil { - return xerrors.Errorf("failed to load most recent ask from disk: %w", err) - } - - var ssa types.SignedStorageAsk - if err := cborutil.ReadCborRPC(bytes.NewReader(askb), &ssa); err != nil { - return err - } - - p.ask = &ssa - return nil -} - -func (p *Provider) signAsk(a *types.StorageAsk) (*types.SignedStorageAsk, error) { - b, err := cborutil.Dump(a) - if err != nil { - return nil, err - } - - worker, err := p.getWorker(p.actor) - if err != nil { - return nil, xerrors.Errorf("failed to get worker to sign ask: %w", err) - } - - sig, err := p.full.WalletSign(context.TODO(), worker, b) - if err != nil { - return nil, err - } - - return &types.SignedStorageAsk{ - Ask: a, - Signature: sig, - }, nil -} - -func (p *Provider) saveAsk(a *types.SignedStorageAsk) error { - b, err := cborutil.Dump(a) - if err != nil { - return err - } - - if err := p.ds.Put(bestAskKey, b); err != nil { - return err - } - - p.ask = a - return nil -} - -func (c *Client) checkAskSignature(ask *types.SignedStorageAsk) error { - tss := c.sm.ChainStore().GetHeaviestTipSet().ParentState() - - w, err := stmgr.GetMinerWorkerRaw(context.TODO(), c.sm, tss, ask.Ask.Miner) - if err != nil { - return xerrors.Errorf("failed to get worker for miner in ask", err) - } - - sigb, err := cborutil.Dump(ask.Ask) - if err != nil { - return xerrors.Errorf("failed to re-serialize ask") - } - - return ask.Signature.Verify(w, sigb) - -} diff --git a/chain/deals/provider_states.go b/chain/deals/provider_states.go deleted file mode 100644 index 4faa44973..000000000 --- a/chain/deals/provider_states.go +++ /dev/null @@ -1,215 +0,0 @@ -package deals - -import ( - "bytes" - "context" - - ipldfree "github.com/ipld/go-ipld-prime/impl/free" - "github.com/ipld/go-ipld-prime/traversal/selector" - "github.com/ipld/go-ipld-prime/traversal/selector/builder" - - unixfile "github.com/ipfs/go-unixfs/file" - "golang.org/x/xerrors" - - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/padreader" - "github.com/filecoin-project/lotus/storage/sectorblocks" -) - -type providerHandlerFunc func(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) - -func (p *Provider) handle(ctx context.Context, deal MinerDeal, cb providerHandlerFunc, next api.DealState) { - go func() { - mut, err := cb(ctx, deal) - - if err == nil && next == api.DealNoUpdate { - return - } - - select { - case p.updated <- minerDealUpdate{ - newState: next, - id: deal.ProposalCid, - err: err, - mut: mut, - }: - case <-p.stop: - } - }() -} - -// ACCEPTED -func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) { - - head, err := p.full.ChainHead(ctx) - if err != nil { - return nil, err - } - if head.Height() >= deal.Proposal.ProposalExpiration { - return nil, xerrors.Errorf("deal proposal already expired") - } - - // TODO: check StorageCollateral - - minPrice := types.BigDiv(types.BigMul(p.ask.Ask.Price, types.NewInt(deal.Proposal.PieceSize)), types.NewInt(1<<30)) - if deal.Proposal.StoragePricePerEpoch.LessThan(minPrice) { - return nil, xerrors.Errorf("storage price per epoch less than asking price: %s < %s", deal.Proposal.StoragePricePerEpoch, minPrice) - } - - if deal.Proposal.PieceSize < p.ask.Ask.MinPieceSize { - return nil, xerrors.Errorf("piece size less than minimum required size: %d < %d", deal.Proposal.PieceSize, p.ask.Ask.MinPieceSize) - } - - // check market funds - clientMarketBalance, err := p.full.StateMarketBalance(ctx, deal.Proposal.Client, nil) - if err != nil { - return nil, xerrors.Errorf("getting client market balance failed: %w", err) - } - - // This doesn't guarantee that the client won't withdraw / lock those funds - // but it's a decent first filter - if clientMarketBalance.Available.LessThan(deal.Proposal.TotalStoragePrice()) { - return nil, xerrors.New("clientMarketBalance.Available too small") - } - - waddr, err := p.full.StateMinerWorker(ctx, deal.Proposal.Provider, nil) - if err != nil { - return nil, err - } - - // TODO: check StorageCollateral (may be too large (or too small)) - if err := p.full.MarketEnsureAvailable(ctx, waddr, deal.Proposal.StorageCollateral); err != nil { - return nil, err - } - - log.Info("publishing deal") - - params, err := actors.SerializeParams(&actors.PublishStorageDealsParams{ - Deals: []actors.StorageDealProposal{deal.Proposal}, - }) - if err != nil { - return nil, xerrors.Errorf("serializing PublishStorageDeals params failed: ", err) - } - - // TODO: We may want this to happen after fetching data - smsg, err := p.full.MpoolPushMessage(ctx, &types.Message{ - To: actors.StorageMarketAddress, - From: waddr, - Value: types.NewInt(0), - GasPrice: types.NewInt(0), - GasLimit: types.NewInt(1000000), - Method: actors.SMAMethods.PublishStorageDeals, - Params: params, - }) - if err != nil { - return nil, err - } - r, err := p.full.StateWaitMsg(ctx, smsg.Cid()) - if err != nil { - return nil, err - } - if r.Receipt.ExitCode != 0 { - return nil, xerrors.Errorf("publishing deal failed: exit %d", r.Receipt.ExitCode) - } - var resp actors.PublishStorageDealResponse - if err := resp.UnmarshalCBOR(bytes.NewReader(r.Receipt.Return)); err != nil { - return nil, err - } - if len(resp.DealIDs) != 1 { - return nil, xerrors.Errorf("got unexpected number of DealIDs from SMA") - } - - log.Infof("fetching data for a deal %d", resp.DealIDs[0]) - err = p.sendSignedResponse(&Response{ - State: api.DealAccepted, - - Proposal: deal.ProposalCid, - StorageDealSubmission: smsg, - }) - if err != nil { - return nil, err - } - - if err := p.disconnect(deal); err != nil { - log.Warnf("closing client connection: %+v", err) - } - - ssb := builder.NewSelectorSpecBuilder(ipldfree.NodeBuilder()) - - // this is the selector for "get the whole DAG" - // TODO: support storage deals with custom payload selectors - allSelector := ssb.ExploreRecursive(selector.RecursionLimitNone(), - ssb.ExploreAll(ssb.ExploreRecursiveEdge())).Node() - - // initiate a pull data transfer. This will complete asynchronously and the - // completion of the data transfer will trigger a change in deal state - // (see onDataTransferEvent) - _, err = p.dataTransfer.OpenPullDataChannel(ctx, - deal.Client, - &StorageDataTransferVoucher{Proposal: deal.ProposalCid, DealID: resp.DealIDs[0]}, - deal.Ref, - allSelector, - ) - if err != nil { - return nil, xerrors.Errorf("failed to open pull data channel: %w", err) - } - - return nil, nil -} - -// STAGED - -func (p *Provider) staged(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) { - root, err := p.dag.Get(ctx, deal.Ref) - if err != nil { - return nil, xerrors.Errorf("failed to get file root for deal: %s", err) - } - - // TODO: abstract this away into ReadSizeCloser + implement different modes - n, err := unixfile.NewUnixfsFile(ctx, p.dag, root) - if err != nil { - return nil, xerrors.Errorf("cannot open unixfs file: %s", err) - } - - uf, ok := n.(sectorblocks.UnixfsReader) - if !ok { - // we probably got directory, unsupported for now - return nil, xerrors.Errorf("unsupported unixfs file type") - } - - // TODO: uf.Size() is user input, not trusted - // This won't be useful / here after we migrate to putting CARs into sectors - size, err := uf.Size() - if err != nil { - return nil, xerrors.Errorf("getting unixfs file size: %w", err) - } - if padreader.PaddedSize(uint64(size)) != deal.Proposal.PieceSize { - return nil, xerrors.Errorf("deal.Proposal.PieceSize didn't match padded unixfs file size") - } - - sectorID, err := p.secb.AddUnixfsPiece(ctx, uf, deal.DealID) - if err != nil { - return nil, xerrors.Errorf("AddPiece failed: %s", err) - } - log.Warnf("New Sector: %d (deal %d)", sectorID, deal.DealID) - - return func(deal *MinerDeal) { - deal.SectorID = sectorID - }, nil -} - -// SEALING - -func (p *Provider) sealing(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) { - // TODO: consider waiting for seal to happen - - return nil, nil -} - -func (p *Provider) complete(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) { - // TODO: observe sector lifecycle, status, expiration.. - - return nil, nil -} diff --git a/chain/deals/provider_utils.go b/chain/deals/provider_utils.go deleted file mode 100644 index e1fcfcf32..000000000 --- a/chain/deals/provider_utils.go +++ /dev/null @@ -1,198 +0,0 @@ -package deals - -import ( - "bytes" - "context" - "runtime" - - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/datatransfer" - "github.com/filecoin-project/lotus/node/modules/dtypes" - "github.com/ipld/go-ipld-prime" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/go-statestore" - "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/types" - - "github.com/ipfs/go-cid" - inet "github.com/libp2p/go-libp2p-core/network" - "github.com/libp2p/go-libp2p-core/peer" - "golang.org/x/xerrors" -) - -func (p *Provider) failDeal(id cid.Cid, cerr error) { - if err := p.deals.End(id); err != nil { - log.Warnf("deals.End: %s", err) - } - - if cerr == nil { - _, f, l, _ := runtime.Caller(1) - cerr = xerrors.Errorf("unknown error (fail called at %s:%d)", f, l) - } - - log.Warnf("deal %s failed: %s", id, cerr) - - err := p.sendSignedResponse(&Response{ - State: api.DealFailed, - Message: cerr.Error(), - Proposal: id, - }) - - s, ok := p.conns[id] - if ok { - _ = s.Reset() - delete(p.conns, id) - } - - if err != nil { - log.Warnf("notifying client about deal failure: %s", err) - } -} - -func (p *Provider) readProposal(s inet.Stream) (proposal Proposal, err error) { - if err := cborutil.ReadCborRPC(s, &proposal); err != nil { - log.Errorw("failed to read proposal message", "error", err) - return proposal, err - } - - if err := proposal.DealProposal.Verify(); err != nil { - return proposal, xerrors.Errorf("verifying StorageDealProposal: %w", err) - } - - if proposal.DealProposal.Provider != p.actor { - log.Errorf("proposal with wrong ProviderAddress: %s", proposal.DealProposal.Provider) - return proposal, err - } - - return -} - -func (p *Provider) sendSignedResponse(resp *Response) error { - s, ok := p.conns[resp.Proposal] - if !ok { - return xerrors.New("couldn't send response: not connected") - } - - msg, err := cborutil.Dump(resp) - if err != nil { - return xerrors.Errorf("serializing response: %w", err) - } - - worker, err := p.getWorker(p.actor) - if err != nil { - return err - } - - sig, err := p.full.WalletSign(context.TODO(), worker, msg) - if err != nil { - return xerrors.Errorf("failed to sign response message: %w", err) - } - - signedResponse := &SignedResponse{ - Response: *resp, - Signature: sig, - } - - err = cborutil.WriteCborRPC(s, signedResponse) - if err != nil { - // Assume client disconnected - s.Close() - delete(p.conns, resp.Proposal) - } - return err -} - -func (p *Provider) disconnect(deal MinerDeal) error { - s, ok := p.conns[deal.ProposalCid] - if !ok { - return nil - } - - err := s.Close() - delete(p.conns, deal.ProposalCid) - return err -} - -func (p *Provider) getWorker(miner address.Address) (address.Address, error) { - getworker := &types.Message{ - To: miner, - From: miner, - Method: actors.MAMethods.GetWorkerAddr, - } - r, err := p.full.StateCall(context.TODO(), getworker, nil) - if err != nil { - return address.Undef, xerrors.Errorf("getting worker address: %w", err) - } - - if r.ExitCode != 0 { - return address.Undef, xerrors.Errorf("getWorker call failed: %d", r.ExitCode) - } - - return address.NewFromBytes(r.Return) -} - -var _ datatransfer.RequestValidator = &ProviderRequestValidator{} - -// ProviderRequestValidator validates data transfer requests for the provider -// in a storage market -type ProviderRequestValidator struct { - deals *statestore.StateStore -} - -// NewProviderRequestValidator returns a new client request validator for the -// given datastore -func NewProviderRequestValidator(deals dtypes.ProviderDealStore) *ProviderRequestValidator { - return &ProviderRequestValidator{ - deals: deals, - } -} - -// ValidatePush validates a push request received from the peer that will send data -// Will succeed only if: -// - voucher has correct type -// - voucher references an active deal -// - referenced deal matches the client -// - referenced deal matches the given base CID -// - referenced deal is in an acceptable state -func (m *ProviderRequestValidator) ValidatePush( - sender peer.ID, - voucher datatransfer.Voucher, - baseCid cid.Cid, - Selector ipld.Node) error { - dealVoucher, ok := voucher.(*StorageDataTransferVoucher) - if !ok { - return xerrors.Errorf("voucher type %s: %w", voucher.Identifier(), ErrWrongVoucherType) - } - - var deal MinerDeal - err := m.deals.Get(dealVoucher.Proposal, &deal) - if err != nil { - return xerrors.Errorf("Proposal CID %s: %w", dealVoucher.Proposal.String(), ErrNoDeal) - } - if deal.Client != sender { - return xerrors.Errorf("Deal Peer %s, Data Transfer Peer %s: %w", deal.Client.String(), sender.String(), ErrWrongPeer) - } - - if !bytes.Equal(deal.Proposal.PieceRef, baseCid.Bytes()) { - return xerrors.Errorf("Deal Payload CID %s, Data Transfer CID %s: %w", string(deal.Proposal.PieceRef), baseCid.String(), ErrWrongPiece) - } - for _, state := range DataTransferStates { - if deal.State == state { - return nil - } - } - return xerrors.Errorf("Deal State %s: %w", deal.State, ErrInacceptableDealState) -} - -// ValidatePull validates a pull request received from the peer that will receive data. -// Will always error because providers should not accept pull requests from a client -// in a storage deal (i.e. send data to client). -func (m *ProviderRequestValidator) ValidatePull( - receiver peer.ID, - voucher datatransfer.Voucher, - baseCid cid.Cid, - Selector ipld.Node) error { - return ErrNoPullAccepted -} diff --git a/chain/deals/request_validation_test.go b/chain/deals/request_validation_test.go deleted file mode 100644 index 6da68d219..000000000 --- a/chain/deals/request_validation_test.go +++ /dev/null @@ -1,291 +0,0 @@ -package deals_test - -import ( - "fmt" - "math/rand" - "testing" - - "github.com/ipfs/go-cid" - "github.com/ipfs/go-datastore" - "github.com/ipfs/go-datastore/namespace" - dss "github.com/ipfs/go-datastore/sync" - blocksutil "github.com/ipfs/go-ipfs-blocksutil" - "github.com/libp2p/go-libp2p-core/peer" - xerrors "golang.org/x/xerrors" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/go-statestore" - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/deals" - "github.com/filecoin-project/lotus/chain/types" -) - -var blockGenerator = blocksutil.NewBlockGenerator() - -type wrongDTType struct { -} - -func (wrongDTType) ToBytes() ([]byte, error) { - return []byte{}, nil -} - -func (wrongDTType) FromBytes([]byte) error { - return fmt.Errorf("not implemented") -} - -func (wrongDTType) Identifier() string { - return "WrongDTTYPE" -} - -func uniqueStorageDealProposal() (actors.StorageDealProposal, error) { - clientAddr, err := address.NewIDAddress(uint64(rand.Int())) - if err != nil { - return actors.StorageDealProposal{}, err - } - providerAddr, err := address.NewIDAddress(uint64(rand.Int())) - if err != nil { - return actors.StorageDealProposal{}, err - } - return actors.StorageDealProposal{ - PieceRef: blockGenerator.Next().Cid().Bytes(), - Client: clientAddr, - Provider: providerAddr, - ProposerSignature: &types.Signature{ - Data: []byte("foo bar cat dog"), - Type: types.KTBLS, - }, - }, nil -} - -func newClientDeal(minerID peer.ID, state api.DealState) (deals.ClientDeal, error) { - newProposal, err := uniqueStorageDealProposal() - if err != nil { - return deals.ClientDeal{}, err - } - proposalNd, err := cborutil.AsIpld(&newProposal) - if err != nil { - return deals.ClientDeal{}, err - } - minerAddr, err := address.NewIDAddress(uint64(rand.Int())) - if err != nil { - return deals.ClientDeal{}, err - } - - return deals.ClientDeal{ - Proposal: newProposal, - ProposalCid: proposalNd.Cid(), - Miner: minerID, - MinerWorker: minerAddr, - State: state, - }, nil -} - -func newMinerDeal(clientID peer.ID, state api.DealState) (deals.MinerDeal, error) { - newProposal, err := uniqueStorageDealProposal() - if err != nil { - return deals.MinerDeal{}, err - } - proposalNd, err := cborutil.AsIpld(&newProposal) - if err != nil { - return deals.MinerDeal{}, err - } - ref, err := cid.Cast(newProposal.PieceRef) - if err != nil { - return deals.MinerDeal{}, err - } - - return deals.MinerDeal{ - Proposal: newProposal, - ProposalCid: proposalNd.Cid(), - Client: clientID, - State: state, - Ref: ref, - }, nil -} - -func TestClientRequestValidation(t *testing.T) { - ds := dss.MutexWrap(datastore.NewMapDatastore()) - state := statestore.New(namespace.Wrap(ds, datastore.NewKey("/deals/client"))) - - crv := deals.NewClientRequestValidator(state) - minerID := peer.ID("fakepeerid") - block := blockGenerator.Next() - t.Run("ValidatePush fails", func(t *testing.T) { - if !xerrors.Is(crv.ValidatePush(minerID, wrongDTType{}, block.Cid(), nil), deals.ErrNoPushAccepted) { - t.Fatal("Push should fail for the client request validator for storage deals") - } - }) - t.Run("ValidatePull fails deal not found", func(t *testing.T) { - proposal, err := uniqueStorageDealProposal() - if err != nil { - t.Fatal("error creating proposal") - } - proposalNd, err := cborutil.AsIpld(&proposal) - if err != nil { - t.Fatal("error serializing proposal") - } - pieceRef, err := cid.Cast(proposal.PieceRef) - if err != nil { - t.Fatal("unable to construct piece cid") - } - if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{proposalNd.Cid(), 1}, pieceRef, nil), deals.ErrNoDeal) { - t.Fatal("Pull should fail if there is no deal stored") - } - }) - t.Run("ValidatePull fails wrong client", func(t *testing.T) { - otherMiner := peer.ID("otherminer") - clientDeal, err := newClientDeal(otherMiner, api.DealAccepted) - if err != nil { - t.Fatal("error creating client deal") - } - if err := state.Begin(clientDeal.ProposalCid, &clientDeal); err != nil { - t.Fatal("deal tracking failed") - } - pieceRef, err := cid.Cast(clientDeal.Proposal.PieceRef) - if err != nil { - t.Fatal("unable to construct piece cid") - } - if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid, 1}, pieceRef, nil), deals.ErrWrongPeer) { - t.Fatal("Pull should fail if miner address is incorrect") - } - }) - t.Run("ValidatePull fails wrong piece ref", func(t *testing.T) { - clientDeal, err := newClientDeal(minerID, api.DealAccepted) - if err != nil { - t.Fatal("error creating client deal") - } - if err := state.Begin(clientDeal.ProposalCid, &clientDeal); err != nil { - t.Fatal("deal tracking failed") - } - if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid, 1}, blockGenerator.Next().Cid(), nil), deals.ErrWrongPiece) { - t.Fatal("Pull should fail if piece ref is incorrect") - } - }) - t.Run("ValidatePull fails wrong deal state", func(t *testing.T) { - clientDeal, err := newClientDeal(minerID, api.DealComplete) - if err != nil { - t.Fatal("error creating client deal") - } - if err := state.Begin(clientDeal.ProposalCid, &clientDeal); err != nil { - t.Fatal("deal tracking failed") - } - pieceRef, err := cid.Cast(clientDeal.Proposal.PieceRef) - if err != nil { - t.Fatal("unable to construct piece cid") - } - if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid, 1}, pieceRef, nil), deals.ErrInacceptableDealState) { - t.Fatal("Pull should fail if deal is in a state that cannot be data transferred") - } - }) - t.Run("ValidatePull succeeds", func(t *testing.T) { - clientDeal, err := newClientDeal(minerID, api.DealAccepted) - if err != nil { - t.Fatal("error creating client deal") - } - if err := state.Begin(clientDeal.ProposalCid, &clientDeal); err != nil { - t.Fatal("deal tracking failed") - } - pieceRef, err := cid.Cast(clientDeal.Proposal.PieceRef) - if err != nil { - t.Fatal("unable to construct piece cid") - } - if crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid, 1}, pieceRef, nil) != nil { - t.Fatal("Pull should should succeed when all parameters are correct") - } - }) -} - -func TestProviderRequestValidation(t *testing.T) { - ds := dss.MutexWrap(datastore.NewMapDatastore()) - state := statestore.New(namespace.Wrap(ds, datastore.NewKey("/deals/client"))) - - mrv := deals.NewProviderRequestValidator(state) - clientID := peer.ID("fakepeerid") - block := blockGenerator.Next() - t.Run("ValidatePull fails", func(t *testing.T) { - if !xerrors.Is(mrv.ValidatePull(clientID, wrongDTType{}, block.Cid(), nil), deals.ErrNoPullAccepted) { - t.Fatal("Pull should fail for the provider request validator for storage deals") - } - }) - - t.Run("ValidatePush fails deal not found", func(t *testing.T) { - proposal, err := uniqueStorageDealProposal() - if err != nil { - t.Fatal("error creating proposal") - } - proposalNd, err := cborutil.AsIpld(&proposal) - if err != nil { - t.Fatal("error serializing proposal") - } - pieceRef, err := cid.Cast(proposal.PieceRef) - if err != nil { - t.Fatal("unable to construct piece cid") - } - if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{proposalNd.Cid(), 1}, pieceRef, nil), deals.ErrNoDeal) { - t.Fatal("Push should fail if there is no deal stored") - } - }) - t.Run("ValidatePush fails wrong miner", func(t *testing.T) { - otherClient := peer.ID("otherclient") - minerDeal, err := newMinerDeal(otherClient, api.DealAccepted) - if err != nil { - t.Fatal("error creating client deal") - } - if err := state.Begin(minerDeal.ProposalCid, &minerDeal); err != nil { - t.Fatal("deal tracking failed") - } - pieceRef, err := cid.Cast(minerDeal.Proposal.PieceRef) - if err != nil { - t.Fatal("unable to construct piece cid") - } - if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid, 1}, pieceRef, nil), deals.ErrWrongPeer) { - t.Fatal("Push should fail if miner address is incorrect") - } - }) - t.Run("ValidatePush fails wrong piece ref", func(t *testing.T) { - minerDeal, err := newMinerDeal(clientID, api.DealAccepted) - if err != nil { - t.Fatal("error creating client deal") - } - if err := state.Begin(minerDeal.ProposalCid, &minerDeal); err != nil { - t.Fatal("deal tracking failed") - } - if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid, 1}, blockGenerator.Next().Cid(), nil), deals.ErrWrongPiece) { - t.Fatal("Push should fail if piece ref is incorrect") - } - }) - t.Run("ValidatePush fails wrong deal state", func(t *testing.T) { - minerDeal, err := newMinerDeal(clientID, api.DealComplete) - if err != nil { - t.Fatal("error creating client deal") - } - if err := state.Begin(minerDeal.ProposalCid, &minerDeal); err != nil { - t.Fatal("deal tracking failed") - } - pieceRef, err := cid.Cast(minerDeal.Proposal.PieceRef) - if err != nil { - t.Fatal("unable to construct piece cid") - } - if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid, 1}, pieceRef, nil), deals.ErrInacceptableDealState) { - t.Fatal("Push should fail if deal is in a state that cannot be data transferred") - } - }) - t.Run("ValidatePush succeeds", func(t *testing.T) { - minerDeal, err := newMinerDeal(clientID, api.DealAccepted) - if err != nil { - t.Fatal("error creating client deal") - } - if err := state.Begin(minerDeal.ProposalCid, &minerDeal); err != nil { - t.Fatal("deal tracking failed") - } - pieceRef, err := cid.Cast(minerDeal.Proposal.PieceRef) - if err != nil { - t.Fatal("unable to construct piece cid") - } - if mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid, 1}, pieceRef, nil) != nil { - t.Fatal("Push should should succeed when all parameters are correct") - } - }) -} diff --git a/chain/deals/types.go b/chain/deals/types.go deleted file mode 100644 index 8a368e80d..000000000 --- a/chain/deals/types.go +++ /dev/null @@ -1,114 +0,0 @@ -package deals - -import ( - "bytes" - "errors" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/types" - "github.com/ipfs/go-cid" -) - -var ( - // ErrWrongVoucherType means the voucher was not the correct type can validate against - ErrWrongVoucherType = errors.New("cannot validate voucher type.") - - // ErrNoPushAccepted just means clients do not accept pushes for storage deals - ErrNoPushAccepted = errors.New("client should not receive data for a storage deal.") - - // ErrNoPullAccepted just means providers do not accept pulls for storage deals - ErrNoPullAccepted = errors.New("provider should not send data for a storage deal.") - - // ErrNoDeal means no active deal was found for this vouchers proposal cid - ErrNoDeal = errors.New("no deal found for this proposal.") - - // ErrWrongPeer means that the other peer for this data transfer request does not match - // the other peer for the deal - ErrWrongPeer = errors.New("data Transfer peer id and Deal peer id do not match.") - - // ErrWrongPiece means that the pieceref for this data transfer request does not match - // the one specified in the deal - ErrWrongPiece = errors.New("base CID for deal does not match CID for piece.") - - // ErrInacceptableDealState means the deal for this transfer is not in a deal state - // where transfer can be performed - ErrInacceptableDealState = errors.New("deal is not a in a state where deals are accepted.") - - // DataTransferStates are the states in which it would make sense to actually start a data transfer - DataTransferStates = []api.DealState{api.DealAccepted, api.DealUnknown} -) - -const DealProtocolID = "/fil/storage/mk/1.0.1" -const AskProtocolID = "/fil/storage/ask/1.0.1" - -type Proposal struct { - DealProposal *actors.StorageDealProposal - - Piece cid.Cid // Used for retrieving from the client -} - -type Response struct { - State api.DealState - - // DealProposalRejected - Message string - Proposal cid.Cid - - // DealAccepted - StorageDealSubmission *types.SignedMessage -} - -// TODO: Do we actually need this to be signed? -type SignedResponse struct { - Response Response - - Signature *types.Signature -} - -func (r *SignedResponse) Verify(addr address.Address) error { - b, err := cborutil.Dump(&r.Response) - if err != nil { - return err - } - - return r.Signature.Verify(addr, b) -} - -type AskRequest struct { - Miner address.Address -} - -type AskResponse struct { - Ask *types.SignedStorageAsk -} - -// StorageDataTransferVoucher is the voucher type for data transfers -// used by the storage market -type StorageDataTransferVoucher struct { - Proposal cid.Cid - DealID uint64 -} - -// ToBytes converts the StorageDataTransferVoucher to raw bytes -func (dv *StorageDataTransferVoucher) ToBytes() ([]byte, error) { - var buf bytes.Buffer - err := dv.MarshalCBOR(&buf) - if err != nil { - return nil, err - } - return buf.Bytes(), nil -} - -// FromBytes converts the StorageDataTransferVoucher to raw bytes -func (dv *StorageDataTransferVoucher) FromBytes(raw []byte) error { - r := bytes.NewReader(raw) - return dv.UnmarshalCBOR(r) -} - -// Identifier is the unique string identifier for a StorageDataTransferVoucher -func (dv *StorageDataTransferVoucher) Identifier() string { - return "StorageDataTransferVoucher" -} diff --git a/chain/events/events.go b/chain/events/events.go index 683dab5db..8a860c5ad 100644 --- a/chain/events/events.go +++ b/chain/events/events.go @@ -6,7 +6,7 @@ import ( "time" "github.com/ipfs/go-cid" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" diff --git a/chain/gen/gen.go b/chain/gen/gen.go index d476a0937..bf0c7a2b9 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -6,6 +6,7 @@ import ( "crypto/sha256" "encoding/binary" "fmt" + "github.com/filecoin-project/lotus/chain/vm" "io/ioutil" "sync/atomic" @@ -34,7 +35,7 @@ import ( block "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" blockstore "github.com/ipfs/go-ipfs-blockstore" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" ) var log = logging.Logger("gen") @@ -170,7 +171,9 @@ func NewGenerator() (*ChainGen, error) { MinerAddrs: []address.Address{maddr1, maddr2}, } - genb, err := MakeGenesisBlock(bs, map[address.Address]types.BigInt{ + sys := vm.Syscalls(sectorbuilder.ProofVerifier) + + genb, err := MakeGenesisBlock(bs, sys, map[address.Address]types.BigInt{ mk1: types.FromFil(40000), mk2: types.FromFil(40000), banker: types.FromFil(50000), @@ -179,7 +182,7 @@ func NewGenerator() (*ChainGen, error) { return nil, xerrors.Errorf("make genesis block failed: %w", err) } - cs := store.NewChainStore(bs, ds) + cs := store.NewChainStore(bs, ds, sys) genfb := &types.FullBlock{Header: genb.Genesis} gents := store.NewFullTipSet([]*types.FullBlock{genfb}) diff --git a/chain/gen/utils.go b/chain/gen/utils.go index 3f528075e..8b53e7c7a 100644 --- a/chain/gen/utils.go +++ b/chain/gen/utils.go @@ -275,7 +275,7 @@ func mustEnc(i cbg.CBORMarshaler) []byte { } func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid, gmcfg *GenMinerCfg) (cid.Cid, []actors.StorageDealProposal, error) { - vm, err := vm.NewVM(sroot, 0, nil, actors.NetworkAddress, cs.Blockstore()) + vm, err := vm.NewVM(sroot, 0, nil, actors.NetworkAddress, cs.Blockstore(), cs.VMSys()) if err != nil { return cid.Undef, nil, xerrors.Errorf("failed to create NewVM: %w", err) } @@ -555,7 +555,7 @@ func doExecValue(ctx context.Context, vm *vm.VM, to, from address.Address, value return ret.Return, nil } -func MakeGenesisBlock(bs bstore.Blockstore, balances map[address.Address]types.BigInt, gmcfg *GenMinerCfg, ts uint64) (*GenesisBootstrap, error) { +func MakeGenesisBlock(bs bstore.Blockstore, sys *types.VMSyscalls, balances map[address.Address]types.BigInt, gmcfg *GenMinerCfg, ts uint64) (*GenesisBootstrap, error) { ctx := context.Background() state, err := MakeInitialStateTree(bs, balances) @@ -569,7 +569,7 @@ func MakeGenesisBlock(bs bstore.Blockstore, balances map[address.Address]types.B } // temp chainstore - cs := store.NewChainStore(bs, datastore.NewMapDatastore()) + cs := store.NewChainStore(bs, datastore.NewMapDatastore(), sys) stateroot, deals, err := SetupStorageMiners(ctx, cs, stateroot, gmcfg) if err != nil { return nil, xerrors.Errorf("setup storage miners failed: %w", err) diff --git a/chain/messagepool/messagepool.go b/chain/messagepool/messagepool.go index 8bbf3823a..1ffd46907 100644 --- a/chain/messagepool/messagepool.go +++ b/chain/messagepool/messagepool.go @@ -13,7 +13,7 @@ import ( "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" "github.com/ipfs/go-datastore/query" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" pubsub "github.com/libp2p/go-libp2p-pubsub" lps "github.com/whyrusleeping/pubsub" "go.uber.org/multierr" diff --git a/chain/metrics/consensus.go b/chain/metrics/consensus.go index b0352860e..0635b08fd 100644 --- a/chain/metrics/consensus.go +++ b/chain/metrics/consensus.go @@ -6,7 +6,7 @@ import ( "time" "github.com/ipfs/go-cid" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" pubsub "github.com/libp2p/go-libp2p-pubsub" "go.uber.org/fx" diff --git a/chain/state/statetree.go b/chain/state/statetree.go index fc3da47e4..384af58bb 100644 --- a/chain/state/statetree.go +++ b/chain/state/statetree.go @@ -6,7 +6,7 @@ import ( "github.com/ipfs/go-cid" hamt "github.com/ipfs/go-hamt-ipld" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" diff --git a/chain/stmgr/call.go b/chain/stmgr/call.go index cfbe729c0..bc4a2f9fc 100644 --- a/chain/stmgr/call.go +++ b/chain/stmgr/call.go @@ -18,7 +18,7 @@ func (sm *StateManager) CallRaw(ctx context.Context, msg *types.Message, bstate ctx, span := trace.StartSpan(ctx, "statemanager.CallRaw") defer span.End() - vmi, err := vm.NewVM(bstate, bheight, r, actors.NetworkAddress, sm.cs.Blockstore()) + vmi, err := vm.NewVM(bstate, bheight, r, actors.NetworkAddress, sm.cs.Blockstore(), sm.cs.VMSys()) if err != nil { return nil, xerrors.Errorf("failed to set up vm: %w", err) } diff --git a/chain/stmgr/fork_no_p_eps.go b/chain/stmgr/fork_no_p_eps.go deleted file mode 100644 index 2eb112ca6..000000000 --- a/chain/stmgr/fork_no_p_eps.go +++ /dev/null @@ -1,127 +0,0 @@ -package stmgr - -import ( - "context" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-amt-ipld" - "github.com/ipfs/go-cid" - "github.com/ipfs/go-hamt-ipld" - cbor "github.com/ipfs/go-ipld-cbor" - "github.com/whyrusleeping/cbor-gen" - "golang.org/x/xerrors" - - "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/state" - "github.com/filecoin-project/lotus/chain/types" -) - -func (sm *StateManager) forkNoPowerEPS(ctx context.Context, pstate cid.Cid) (cid.Cid, error) { - cst := hamt.CSTFromBstore(sm.cs.Blockstore()) - st, err := state.LoadStateTree(cst, pstate) - if err != nil { - return cid.Undef, xerrors.Errorf("loading parent state tree: %w", err) - } - - if err := st.MutateActor(actors.StoragePowerAddress, func(spa *types.Actor) error { - var head actors.StoragePowerState - if err := cst.Get(ctx, spa.Head, &head); err != nil { - return xerrors.Errorf("reading StoragePower state: %w", err) - } - - buckets, err := amt.LoadAMT(amt.WrapBlockstore(sm.cs.Blockstore()), head.ProvingBuckets) - if err != nil { - return xerrors.Errorf("opening proving buckets AMT: %w", err) - } - - fixedBuckets := map[uint64]map[address.Address]struct{}{} - - if err := buckets.ForEach(func(bucketId uint64, ent *typegen.Deferred) error { - var bcid cid.Cid - if err := cbor.DecodeInto(ent.Raw, &bcid); err != nil { - return xerrors.Errorf("decoding bucket cid: %w", err) - } - - bucket, err := hamt.LoadNode(ctx, cst, bcid) - if err != nil { - return xerrors.Errorf("loading bucket hamt: %w", err) - } - - return bucket.ForEach(ctx, func(abytes string, _ interface{}) error { - addr, err := address.NewFromBytes([]byte(abytes)) - if err != nil { - return xerrors.Errorf("parsing address in proving bucket: %w", err) - } - - // now find the correct bucket - miner, err := st.GetActor(addr) - if err != nil { - return xerrors.Errorf("getting miner %s: %w", addr, err) - } - - var minerHead actors.StorageMinerActorState - if err := cst.Get(ctx, miner.Head, &minerHead); err != nil { - return xerrors.Errorf("reading miner %s state: %w", addr, err) - } - - correctBucket := minerHead.ElectionPeriodStart % build.SlashablePowerDelay - if correctBucket != bucketId { - log.Warnf("miner %s was in wrong proving bucket %d, putting in %d (eps: %d)", addr, bucketId, correctBucket, minerHead.ElectionPeriodStart) - } - - if _, ok := fixedBuckets[correctBucket]; !ok { - fixedBuckets[correctBucket] = map[address.Address]struct{}{} - } - fixedBuckets[correctBucket][addr] = struct{}{} - - return nil - }) - }); err != nil { - return err - } - - // ///// - // Write fixed buckets - - fixed := amt.NewAMT(amt.WrapBlockstore(sm.cs.Blockstore())) - - for bucketId, addrss := range fixedBuckets { - bucket := hamt.NewNode(cst) - for addr := range addrss { - if err := bucket.Set(ctx, string(addr.Bytes()), actors.CborNull); err != nil { - return xerrors.Errorf("setting address in bucket: %w", err) - } - } - - if err := bucket.Flush(ctx); err != nil { - return xerrors.Errorf("flushing bucket amt: %w", err) - } - - bcid, err := cst.Put(context.TODO(), bucket) - if err != nil { - return xerrors.Errorf("put bucket: %w", err) - } - - if err := fixed.Set(bucketId, bcid); err != nil { - return xerrors.Errorf("set bucket: %w", err) - } - } - - head.ProvingBuckets, err = fixed.Flush() - if err != nil { - return xerrors.Errorf("flushing bucket amt: %w", err) - } - - spa.Head, err = cst.Put(ctx, &head) - if err != nil { - return xerrors.Errorf("putting actor head: %w", err) - } - - return nil - }); err != nil { - return cid.Undef, err - } - - return st.Flush() -} diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index 0f92f3438..c1f48b8f6 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -2,22 +2,13 @@ package stmgr import ( "context" - "github.com/ipfs/go-cid" - "golang.org/x/xerrors" - "github.com/filecoin-project/lotus/build" + "github.com/ipfs/go-cid" ) func (sm *StateManager) handleStateForks(ctx context.Context, pstate cid.Cid, height, parentH uint64) (_ cid.Cid, err error) { for i := parentH; i < height; i++ { switch i { - case build.ForkNoPowerEPSUpdates: - pstate, err = sm.forkNoPowerEPS(ctx, pstate) - if err != nil { - return cid.Undef, xerrors.Errorf("executing state fork in epoch %d: %w", i, err) - } - - log.Infof("forkNoPowerEPS state: %s", pstate) } } diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index b880492f2..310f0a9e9 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -18,7 +18,7 @@ import ( bls "github.com/filecoin-project/filecoin-ffi" "github.com/ipfs/go-cid" hamt "github.com/ipfs/go-hamt-ipld" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "go.opencensus.io/trace" ) @@ -139,7 +139,7 @@ func (sm *StateManager) computeTipSetState(ctx context.Context, blks []*types.Bl r := store.NewChainRand(sm.cs, cids, blks[0].Height) - vmi, err := vm.NewVM(pstate, blks[0].Height, r, address.Undef, sm.cs.Blockstore()) + vmi, err := vm.NewVM(pstate, blks[0].Height, r, address.Undef, sm.cs.Blockstore(), sm.cs.VMSys()) if err != nil { return cid.Undef, cid.Undef, xerrors.Errorf("instantiating VM failed: %w", err) } diff --git a/chain/store/store.go b/chain/store/store.go index d59126518..5bd1bfdf5 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -23,7 +23,7 @@ import ( dstore "github.com/ipfs/go-datastore" hamt "github.com/ipfs/go-hamt-ipld" bstore "github.com/ipfs/go-ipfs-blockstore" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" cbg "github.com/whyrusleeping/cbor-gen" pubsub "github.com/whyrusleeping/pubsub" "golang.org/x/xerrors" @@ -51,9 +51,11 @@ type ChainStore struct { mmCache *lru.ARCCache tsCache *lru.ARCCache + + vmcalls *types.VMSyscalls } -func NewChainStore(bs bstore.Blockstore, ds dstore.Batching) *ChainStore { +func NewChainStore(bs bstore.Blockstore, ds dstore.Batching, vmcalls *types.VMSyscalls) *ChainStore { c, _ := lru.NewARC(2048) tsc, _ := lru.NewARC(4096) cs := &ChainStore{ @@ -63,6 +65,7 @@ func NewChainStore(bs bstore.Blockstore, ds dstore.Batching) *ChainStore { tipsets: make(map[uint64][]cid.Cid), mmCache: c, tsCache: tsc, + vmcalls: vmcalls, } cs.reorgCh = cs.reorgWorker(context.TODO()) @@ -793,6 +796,10 @@ func (cs *ChainStore) Blockstore() bstore.Blockstore { return cs.bs } +func (cs *ChainStore) VMSys() *types.VMSyscalls { + return cs.vmcalls +} + func (cs *ChainStore) TryFillTipSet(ts *types.TipSet) (*FullTipSet, error) { var out []*types.FullBlock diff --git a/chain/store/store_test.go b/chain/store/store_test.go index 4d6d11c1b..c3c7a2ef4 100644 --- a/chain/store/store_test.go +++ b/chain/store/store_test.go @@ -55,7 +55,7 @@ func BenchmarkGetRandomness(b *testing.B) { bs := blockstore.NewBlockstore(bds) - cs := store.NewChainStore(bs, mds) + cs := store.NewChainStore(bs, mds, nil) b.ResetTimer() diff --git a/chain/store/weight.go b/chain/store/weight.go index 901356728..49ef9ef26 100644 --- a/chain/store/weight.go +++ b/chain/store/weight.go @@ -60,7 +60,7 @@ func (cs *ChainStore) call(ctx context.Context, msg *types.Message, ts *types.Ti r := NewChainRand(cs, ts.Cids(), ts.Height()) - vmi, err := vm.NewVM(bstate, ts.Height(), r, actors.NetworkAddress, cs.bs) + vmi, err := vm.NewVM(bstate, ts.Height(), r, actors.NetworkAddress, cs.bs, cs.vmcalls) if err != nil { return nil, xerrors.Errorf("failed to set up vm: %w", err) } diff --git a/chain/sub/incoming.go b/chain/sub/incoming.go index 79dc29b80..fc554f4b3 100644 --- a/chain/sub/incoming.go +++ b/chain/sub/incoming.go @@ -4,10 +4,11 @@ import ( "context" "time" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" connmgr "github.com/libp2p/go-libp2p-core/connmgr" pubsub "github.com/libp2p/go-libp2p-pubsub" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain" "github.com/filecoin-project/lotus/chain/messagepool" "github.com/filecoin-project/lotus/chain/types" @@ -33,6 +34,11 @@ func HandleIncomingBlocks(ctx context.Context, bsub *pubsub.Subscription, s *cha continue } + if len(blk.BlsMessages)+len(blk.SecpkMessages) > build.BlockMessageLimit { + log.Warnf("received block with too many messages over pubsub") + continue + } + go func() { log.Infof("New block over pubsub: %s", blk.Cid()) diff --git a/chain/sync.go b/chain/sync.go index 9065bc099..7c8e13b36 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -18,7 +18,7 @@ import ( dstore "github.com/ipfs/go-datastore" hamt "github.com/ipfs/go-hamt-ipld" bstore "github.com/ipfs/go-ipfs-blockstore" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p-core/connmgr" "github.com/libp2p/go-libp2p-core/peer" cbg "github.com/whyrusleeping/cbor-gen" @@ -189,6 +189,10 @@ func (syncer *Syncer) IncomingBlocks(ctx context.Context) (<-chan *types.BlockHe } func (syncer *Syncer) ValidateMsgMeta(fblk *types.FullBlock) error { + if msgc := len(fblk.BlsMessages) + len(fblk.SecpkMessages); msgc > build.BlockMessageLimit { + return xerrors.Errorf("block %s has too many messages (%d)", fblk.Header.Cid(), msgc) + } + var bcids, scids []cbg.CBORMarshaler for _, m := range fblk.BlsMessages { c := cbg.CborCid(m.Cid()) @@ -295,6 +299,10 @@ func zipTipSetAndMessages(bs amt.Blocks, ts *types.TipSet, allbmsgs []*types.Mes bmsgCids = append(bmsgCids, &c) } + if msgc := len(bmsgCids) + len(smsgCids); msgc > build.BlockMessageLimit { + return nil, fmt.Errorf("block %q has too many messages (%d)", b.Cid(), msgc) + } + mrcid, err := computeMsgMeta(bs, bmsgCids, smsgCids) if err != nil { return nil, err @@ -533,15 +541,17 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) err snum := types.BigDiv(mpow, types.NewInt(ssize)) - // FORK START - if h.Height > build.ForkCCM { - if len(h.EPostProof.Candidates) == 0 { - return xerrors.Errorf("no candidates") - } + if len(h.EPostProof.Candidates) == 0 { + return xerrors.Errorf("no candidates") } - // FORK END + wins := make(map[uint64]bool) for _, t := range h.EPostProof.Candidates { + if wins[t.ChallengeIndex] { + return xerrors.Errorf("block had duplicate epost candidates") + } + wins[t.ChallengeIndex] = true + if !types.IsTicketWinner(t.Partial, ssize, snum.Uint64(), tpow) { return xerrors.Errorf("miner created a block but was not a winner") } @@ -662,13 +672,10 @@ func (syncer *Syncer) VerifyElectionPoStProof(ctx context.Context, h *types.Bloc SectorChallengeIndex: t.ChallengeIndex, }) } - // FORK START - if h.Height > build.ForkCCM { - if len(winners) == 0 { - return xerrors.Errorf("no candidates") - } + + if len(winners) == 0 { + return xerrors.Errorf("no candidates") } - // FORK END sectorInfo, err := stmgr.GetSectorsForElectionPost(ctx, syncer.sm, baseTs, h.Miner) if err != nil { @@ -683,7 +690,7 @@ func (syncer *Syncer) VerifyElectionPoStProof(ctx context.Context, h *types.Bloc } hvrf := sha256.Sum256(h.EPostProof.PostRand) - ok, err := sectorbuilder.VerifyElectionPost(ctx, ssize, *sectorInfo, hvrf[:], h.EPostProof.Proof, winners, h.Miner) + ok, err := sectorbuilder.ProofVerifier.VerifyElectionPost(ctx, ssize, *sectorInfo, hvrf[:], h.EPostProof.Proof, winners, h.Miner) if err != nil { return xerrors.Errorf("failed to verify election post: %w", err) } diff --git a/chain/sync_test.go b/chain/sync_test.go index c9b8e7bbb..9240dea21 100644 --- a/chain/sync_test.go +++ b/chain/sync_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p-core/peer" mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" "github.com/stretchr/testify/require" diff --git a/chain/types/bitfield.go b/chain/types/bitfield.go index d959e4f84..81031085b 100644 --- a/chain/types/bitfield.go +++ b/chain/types/bitfield.go @@ -1,6 +1,7 @@ package types import ( + "errors" "fmt" "io" @@ -9,6 +10,8 @@ import ( "golang.org/x/xerrors" ) +var ErrBitFieldTooMany = errors.New("to many items in RLE") + type BitField struct { rle rlepluslazy.RLE @@ -16,14 +19,23 @@ type BitField struct { } func NewBitField() BitField { - rle, err := rlepluslazy.FromBuf([]byte{}) + bf, err := NewBitFieldFromBytes([]byte{}) if err != nil { - panic(err) + panic(fmt.Sprintf("creating empty rle: %+v", err)) } - return BitField{ - rle: rle, - bits: make(map[uint64]struct{}), + return bf +} + +func NewBitFieldFromBytes(rle []byte) (BitField, error) { + bf := BitField{} + rlep, err := rlepluslazy.FromBuf(rle) + if err != nil { + return BitField{}, xerrors.Errorf("could not decode rle+: %w", err) } + bf.rle = rlep + bf.bits = make(map[uint64]struct{}) + return bf, nil + } func BitFieldFromSet(setBits []uint64) BitField { @@ -106,7 +118,14 @@ func (bf BitField) Count() (uint64, error) { } // All returns all set bits -func (bf BitField) All() ([]uint64, error) { +func (bf BitField) All(max uint64) ([]uint64, error) { + c, err := bf.Count() + if err != nil { + return nil, xerrors.Errorf("count errror: %w", err) + } + if c > max { + return nil, xerrors.Errorf("expected %d, got %d: %w", max, c, ErrBitFieldTooMany) + } runs, err := bf.sum() if err != nil { @@ -121,7 +140,14 @@ func (bf BitField) All() ([]uint64, error) { return res, nil } -func (bf BitField) AllMap() (map[uint64]bool, error) { +func (bf BitField) AllMap(max uint64) (map[uint64]bool, error) { + c, err := bf.Count() + if err != nil { + return nil, xerrors.Errorf("count errror: %w", err) + } + if c > max { + return nil, xerrors.Errorf("expected %d, got %d: %w", max, c, ErrBitFieldTooMany) + } runs, err := bf.sum() if err != nil { diff --git a/chain/types/blockheader.go b/chain/types/blockheader.go index 23fdab0f7..c91d029fc 100644 --- a/chain/types/blockheader.go +++ b/chain/types/blockheader.go @@ -59,6 +59,8 @@ type BlockHeader struct { Timestamp uint64 BlockSig *Signature + + ForkSignaling uint64 } func (b *BlockHeader) ToStorageBlock() (block.Block, error) { @@ -216,7 +218,7 @@ func IsTicketWinner(partialTicket []byte, ssizeI uint64, snum uint64, totpow Big return lhs.Cmp(rhs) < 0 } -func ElectionPostChallengeCount(sectors uint64, faults int) uint64 { +func ElectionPostChallengeCount(sectors uint64, faults uint64) uint64 { return sectorbuilder.ElectionPostChallengeCount(sectors, faults) } diff --git a/chain/types/cbor_gen.go b/chain/types/cbor_gen.go index 407f89809..45c186435 100644 --- a/chain/types/cbor_gen.go +++ b/chain/types/cbor_gen.go @@ -5,7 +5,7 @@ import ( "io" "math" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" xerrors "golang.org/x/xerrors" ) @@ -19,7 +19,7 @@ func (t *BlockHeader) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{140}); err != nil { + if _, err := w.Write([]byte{141}); err != nil { return err } @@ -94,6 +94,11 @@ func (t *BlockHeader) MarshalCBOR(w io.Writer) error { if err := t.BlockSig.MarshalCBOR(w); err != nil { return err } + + // t.ForkSignaling (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.ForkSignaling))); err != nil { + return err + } return nil } @@ -108,7 +113,7 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type array") } - if extra != 12 { + if extra != 13 { return fmt.Errorf("cbor input had wrong number of fields") } @@ -272,6 +277,16 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { } } + // t.ForkSignaling (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.ForkSignaling = uint64(extra) return nil } diff --git a/chain/types/tipset.go b/chain/types/tipset.go index a4fb209cb..b320f3e30 100644 --- a/chain/types/tipset.go +++ b/chain/types/tipset.go @@ -8,7 +8,7 @@ import ( "sort" "github.com/ipfs/go-cid" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" ) diff --git a/chain/types/vmcontext.go b/chain/types/vmcontext.go index 1af2aa3da..b6eac6ca2 100644 --- a/chain/types/vmcontext.go +++ b/chain/types/vmcontext.go @@ -2,6 +2,7 @@ package types import ( "context" + "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-amt-ipld" @@ -46,7 +47,15 @@ type VMContext interface { } type VMSyscalls struct { - ValidatePoRep func(context.Context, address.Address, uint64, []byte, []byte, []byte, []byte, []byte, uint64) (bool, aerrors.ActorError) + ValidatePoRep func(context.Context, address.Address, uint64, []byte, []byte, []byte, []byte, []byte, uint64) (bool, aerrors.ActorError) + VerifyFallbackPost func(ctx context.Context, + sectorSize uint64, + sectorInfo sectorbuilder.SortedPublicSectorInfo, + challengeSeed []byte, + proof []byte, + candidates []sectorbuilder.EPostCandidate, + proverID address.Address, + faults uint64) (bool, error) } type storageWrapper struct { diff --git a/chain/validation/applier.go b/chain/validation/applier.go index 4e21f1228..8caf1b2dd 100644 --- a/chain/validation/applier.go +++ b/chain/validation/applier.go @@ -2,6 +2,7 @@ package validation import ( "context" + "github.com/filecoin-project/go-sectorbuilder" vchain "github.com/filecoin-project/chain-validation/pkg/chain" vstate "github.com/filecoin-project/chain-validation/pkg/state" @@ -32,7 +33,7 @@ func (a *Applier) ApplyMessage(eCtx *vchain.ExecutionContext, state vstate.Wrapp if err != nil { return vchain.MessageReceipt{}, err } - lotusVM, err := vm.NewVM(base, eCtx.Epoch, randSrc, minerAddr, st.bs) + lotusVM, err := vm.NewVM(base, eCtx.Epoch, randSrc, minerAddr, st.bs, vm.Syscalls(sectorbuilder.ProofVerifier)) if err != nil { return vchain.MessageReceipt{}, err } diff --git a/chain/vm/syscalls.go b/chain/vm/syscalls.go index 92f9ea269..fa3ea18c9 100644 --- a/chain/vm/syscalls.go +++ b/chain/vm/syscalls.go @@ -1,14 +1,29 @@ package vm import ( + "context" + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/actors/aerrors" "github.com/filecoin-project/lotus/chain/types" + "go.opencensus.io/trace" ) // Actual type is defined in chain/types/vmcontext.go because the VMContext interface is there -func DefaultSyscalls() *types.VMSyscalls { +func Syscalls(verifier sectorbuilder.Verifier) *types.VMSyscalls { return &types.VMSyscalls{ - ValidatePoRep: actors.ValidatePoRep, + ValidatePoRep: func(ctx context.Context, maddr address.Address, ssize uint64, commD, commR, ticket, proof, seed []byte, sectorID uint64) (bool, actors.ActorError) { + _, span := trace.StartSpan(ctx, "ValidatePoRep") + defer span.End() + ok, err := verifier.VerifySeal(ssize, commR, commD, maddr, ticket, seed, sectorID, proof) + if err != nil { + return false, aerrors.Absorb(err, 25, "verify seal failed") + } + + return ok, nil + }, + VerifyFallbackPost: verifier.VerifyFallbackPost, } } diff --git a/chain/vm/vm.go b/chain/vm/vm.go index 6fa6a432c..a0ab66999 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -10,7 +10,7 @@ import ( cid "github.com/ipfs/go-cid" hamt "github.com/ipfs/go-hamt-ipld" blockstore "github.com/ipfs/go-ipfs-blockstore" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" cbg "github.com/whyrusleeping/cbor-gen" "go.opencensus.io/trace" "golang.org/x/xerrors" @@ -259,7 +259,7 @@ func (bs *gasChargingBlocks) GetBlock(ctx context.Context, c cid.Cid) (block.Blo } blk, err := bs.under.GetBlock(ctx, c) if err != nil { - return nil, err + return nil, aerrors.Escalate(err, "failed to get block from blockstore") } if err := bs.chargeGas(uint64(len(blk.RawData())) * gasGetPerByte); err != nil { return nil, err @@ -272,7 +272,10 @@ func (bs *gasChargingBlocks) AddBlock(blk block.Block) error { if err := bs.chargeGas(gasPutObj + uint64(len(blk.RawData()))*gasPutPerByte); err != nil { return err } - return bs.under.AddBlock(blk) + if err := bs.under.AddBlock(blk); err != nil { + return aerrors.Escalate(err, "failed to write data to disk") + } + return nil } func (vm *VM) makeVMContext(ctx context.Context, sroot cid.Cid, msg *types.Message, origin address.Address, usedGas types.BigInt) *VMContext { @@ -309,7 +312,7 @@ type VM struct { Syscalls *types.VMSyscalls } -func NewVM(base cid.Cid, height uint64, r Rand, maddr address.Address, cbs blockstore.Blockstore) (*VM, error) { +func NewVM(base cid.Cid, height uint64, r Rand, maddr address.Address, cbs blockstore.Blockstore, syscalls *types.VMSyscalls) (*VM, error) { buf := bufbstore.NewBufferedBstore(cbs) cst := hamt.CSTFromBstore(buf) state, err := state.LoadStateTree(cst, base) @@ -325,8 +328,8 @@ func NewVM(base cid.Cid, height uint64, r Rand, maddr address.Address, cbs block blockHeight: height, blockMiner: maddr, inv: newInvoker(), - rand: r, - Syscalls: DefaultSyscalls(), + rand: r, // TODO: Probably should be a syscall + Syscalls: syscalls, }, nil } diff --git a/chain/wallet/wallet.go b/chain/wallet/wallet.go index 9472d36e0..3a70b506a 100644 --- a/chain/wallet/wallet.go +++ b/chain/wallet/wallet.go @@ -9,7 +9,7 @@ import ( bls "github.com/filecoin-project/filecoin-ffi" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "github.com/minio/blake2b-simd" "golang.org/x/xerrors" diff --git a/cli/cmd.go b/cli/cmd.go index 9eff3e541..0c6c51e82 100644 --- a/cli/cmd.go +++ b/cli/cmd.go @@ -9,7 +9,7 @@ import ( "strings" "syscall" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "github.com/mitchellh/go-homedir" "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" diff --git a/cli/params.go b/cli/params.go index f83b95bef..440c2bbdf 100644 --- a/cli/params.go +++ b/cli/params.go @@ -5,6 +5,8 @@ import ( paramfetch "github.com/filecoin-project/go-paramfetch" "golang.org/x/xerrors" "gopkg.in/urfave/cli.v2" + + "github.com/filecoin-project/lotus/build" ) var fetchParamCmd = &cli.Command{ @@ -22,7 +24,7 @@ var fetchParamCmd = &cli.Command{ return err } sectorSize := uint64(sectorSizeInt) - err = paramfetch.GetParams(sectorSize) + err = paramfetch.GetParams(build.ParametersJson, sectorSize) if err != nil { return xerrors.Errorf("fetching proof parameters: %w", err) } diff --git a/cli/wallet.go b/cli/wallet.go index 1f950adb8..ad600eadf 100644 --- a/cli/wallet.go +++ b/cli/wallet.go @@ -41,7 +41,7 @@ var walletNew = &cli.Command{ t := cctx.Args().First() if t == "" { - t = "bls" + t = "secp256k1" } nk, err := api.WalletNew(ctx, t) diff --git a/cmd/lotus-bench/main.go b/cmd/lotus-bench/main.go index 57b963041..2cd059a2c 100644 --- a/cmd/lotus-bench/main.go +++ b/cmd/lotus-bench/main.go @@ -17,7 +17,7 @@ import ( ffi "github.com/filecoin-project/filecoin-ffi" paramfetch "github.com/filecoin-project/go-paramfetch" "github.com/ipfs/go-datastore" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "github.com/mitchellh/go-homedir" "golang.org/x/xerrors" "gopkg.in/urfave/cli.v2" @@ -155,7 +155,7 @@ func main() { } } - if err := paramfetch.GetParams(sectorSize); err != nil { + if err := paramfetch.GetParams(build.ParametersJson, sectorSize); err != nil { return xerrors.Errorf("getting params: %w", err) } sb, err := sectorbuilder.New(cfg, mds) @@ -188,7 +188,7 @@ func main() { log.Info("Running replication...") pieces := []sectorbuilder.PublicPieceInfo{pi} - pco, err := sb.SealPreCommit(i, ticket, pieces) + pco, err := sb.SealPreCommit(context.TODO(), i, ticket, pieces) if err != nil { return xerrors.Errorf("commit: %w", err) } @@ -206,14 +206,14 @@ func main() { } log.Info("Generating PoRep for sector") - proof, err := sb.SealCommit(i, ticket, seed, pieces, pco) + proof, err := sb.SealCommit(context.TODO(), i, ticket, seed, pieces, pco) if err != nil { return err } sealcommit := time.Now() commD := pi.CommP - ok, err := sectorbuilder.VerifySeal(sectorSize, pco.CommR[:], commD[:], maddr, ticket.TicketBytes[:], seed.TicketBytes[:], i, proof) + ok, err := sectorbuilder.ProofVerifier.VerifySeal(sectorSize, pco.CommR[:], commD[:], maddr, ticket.TicketBytes[:], seed.TicketBytes[:], i, proof) if err != nil { return err } @@ -307,7 +307,7 @@ func main() { log.Warn("separate epost calls returned different proof values (this might be bad)") } - ok, err := sectorbuilder.VerifyElectionPost(context.TODO(), sectorSize, sinfos, challenge[:], proof1, candidates[:1], maddr) + ok, err := sectorbuilder.ProofVerifier.VerifyElectionPost(context.TODO(), sectorSize, sinfos, challenge[:], proof1, candidates[:1], maddr) if err != nil { return err } @@ -317,7 +317,7 @@ func main() { verifypost1 := time.Now() - ok, err = sectorbuilder.VerifyElectionPost(context.TODO(), sectorSize, sinfos, challenge[:], proof2, candidates[:1], maddr) + ok, err = sectorbuilder.ProofVerifier.VerifyElectionPost(context.TODO(), sectorSize, sinfos, challenge[:], proof2, candidates[:1], maddr) if err != nil { return err } diff --git a/cmd/lotus-chainwatch/main.go b/cmd/lotus-chainwatch/main.go index 7de753a7a..4e67706fd 100644 --- a/cmd/lotus-chainwatch/main.go +++ b/cmd/lotus-chainwatch/main.go @@ -5,7 +5,7 @@ import ( "net/http" "os" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "golang.org/x/xerrors" "gopkg.in/urfave/cli.v2" diff --git a/cmd/lotus-fountain/main.go b/cmd/lotus-fountain/main.go index 33abeb76b..18e61c97c 100644 --- a/cmd/lotus-fountain/main.go +++ b/cmd/lotus-fountain/main.go @@ -11,7 +11,7 @@ import ( rice "github.com/GeertJohan/go.rice" "github.com/ipfs/go-cid" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" peer "github.com/libp2p/go-libp2p-peer" "golang.org/x/xerrors" "gopkg.in/urfave/cli.v2" diff --git a/cmd/lotus-seal-worker/main.go b/cmd/lotus-seal-worker/main.go index 146c246e8..6170f8239 100644 --- a/cmd/lotus-seal-worker/main.go +++ b/cmd/lotus-seal-worker/main.go @@ -5,7 +5,7 @@ import ( "github.com/mitchellh/go-homedir" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "golang.org/x/xerrors" "gopkg.in/urfave/cli.v2" diff --git a/cmd/lotus-seal-worker/sub.go b/cmd/lotus-seal-worker/sub.go index fa1b00a8a..1043aa5ea 100644 --- a/cmd/lotus-seal-worker/sub.go +++ b/cmd/lotus-seal-worker/sub.go @@ -5,10 +5,11 @@ import ( "net/http" paramfetch "github.com/filecoin-project/go-paramfetch" + "github.com/filecoin-project/go-sectorbuilder" "golang.org/x/xerrors" - "github.com/filecoin-project/go-sectorbuilder" lapi "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/build" ) type worker struct { @@ -40,7 +41,7 @@ func acceptJobs(ctx context.Context, api lapi.StorageMiner, endpoint string, aut return err } - if err := paramfetch.GetParams(ssize); err != nil { + if err := paramfetch.GetParams(build.ParametersJson, ssize); err != nil { return xerrors.Errorf("get params: %w", err) } @@ -102,7 +103,7 @@ func (w *worker) processTask(ctx context.Context, task sectorbuilder.WorkerTask) switch task.Type { case sectorbuilder.WorkerPreCommit: - rspco, err := w.sb.SealPreCommit(task.SectorID, task.SealTicket, task.Pieces) + rspco, err := w.sb.SealPreCommit(ctx, task.SectorID, task.SealTicket, task.Pieces) if err != nil { return errRes(xerrors.Errorf("precomitting: %w", err)) } @@ -116,7 +117,7 @@ func (w *worker) processTask(ctx context.Context, task sectorbuilder.WorkerTask) return errRes(xerrors.Errorf("pushing precommited data: %w", err)) } case sectorbuilder.WorkerCommit: - proof, err := w.sb.SealCommit(task.SectorID, task.SealTicket, task.SealSeed, task.Pieces, task.Rspco) + proof, err := w.sb.SealCommit(ctx, task.SectorID, task.SealTicket, task.SealSeed, task.Pieces, task.Rspco) if err != nil { return errRes(xerrors.Errorf("comitting: %w", err)) } diff --git a/cmd/lotus-seed/main.go b/cmd/lotus-seed/main.go index ee1481d3b..f3f529af0 100644 --- a/cmd/lotus-seed/main.go +++ b/cmd/lotus-seed/main.go @@ -8,8 +8,10 @@ import ( "encoding/json" sectorbuilder "github.com/filecoin-project/go-sectorbuilder" - badger "github.com/ipfs/go-ds-badger" - logging "github.com/ipfs/go-log" + "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/namespace" + badger "github.com/ipfs/go-ds-badger2" + logging "github.com/ipfs/go-log/v2" "github.com/mitchellh/go-homedir" "golang.org/x/xerrors" "gopkg.in/urfave/cli.v2" @@ -196,7 +198,7 @@ var aggregateSectorDirsCmd = &cli.Command{ SectorSize: ssize, Dir: destdir, WorkerThreads: 2, - }, agmds) + }, namespace.Wrap(agmds, datastore.NewKey("/sectorbuilder"))) if err != nil { return err } @@ -257,7 +259,7 @@ var aggregateSectorDirsCmd = &cli.Command{ SectorSize: genm.SectorSize, Dir: dir, WorkerThreads: 2, - }, mds) + }, namespace.Wrap(mds, datastore.NewKey("/sectorbuilder"))) if err != nil { return err } diff --git a/cmd/lotus-seed/seed/seed.go b/cmd/lotus-seed/seed/seed.go index e98abdb0b..d4c00e794 100644 --- a/cmd/lotus-seed/seed/seed.go +++ b/cmd/lotus-seed/seed/seed.go @@ -11,8 +11,10 @@ import ( "path/filepath" sectorbuilder "github.com/filecoin-project/go-sectorbuilder" - badger "github.com/ipfs/go-ds-badger" - logging "github.com/ipfs/go-log" + "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/namespace" + badger "github.com/ipfs/go-ds-badger2" + logging "github.com/ipfs/go-log/v2" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" @@ -43,7 +45,7 @@ func PreSeal(maddr address.Address, ssize uint64, offset uint64, sectors int, sb return nil, err } - sb, err := sectorbuilder.New(cfg, mds) + sb, err := sectorbuilder.New(cfg, namespace.Wrap(mds, datastore.NewKey("/sectorbuilder"))) if err != nil { return nil, err } @@ -69,7 +71,7 @@ func PreSeal(maddr address.Address, ssize uint64, offset uint64, sectors int, sb fmt.Printf("sector-id: %d, piece info: %v", sid, pi) - pco, err := sb.SealPreCommit(sid, ticket, []sectorbuilder.PublicPieceInfo{pi}) + pco, err := sb.SealPreCommit(context.TODO(), sid, ticket, []sectorbuilder.PublicPieceInfo{pi}) if err != nil { return nil, xerrors.Errorf("commit: %w", err) } diff --git a/cmd/lotus-shed/main.go b/cmd/lotus-shed/main.go index 79be74d31..fe8abb233 100644 --- a/cmd/lotus-shed/main.go +++ b/cmd/lotus-shed/main.go @@ -3,7 +3,7 @@ package main import ( "os" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "gopkg.in/urfave/cli.v2" "github.com/filecoin-project/lotus/build" diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index 9bf52fea8..b5bf0c6dc 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -13,7 +13,8 @@ import ( paramfetch "github.com/filecoin-project/go-paramfetch" "github.com/ipfs/go-datastore" - badger "github.com/ipfs/go-ds-badger" + "github.com/ipfs/go-datastore/namespace" + badger "github.com/ipfs/go-ds-badger2" "github.com/libp2p/go-libp2p-core/crypto" "github.com/libp2p/go-libp2p-core/peer" "github.com/mitchellh/go-homedir" @@ -21,15 +22,17 @@ import ( "gopkg.in/urfave/cli.v2" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-cbor-util" + cborutil "github.com/filecoin-project/go-cbor-util" + "github.com/filecoin-project/go-fil-markets/storagemarket" + deals "github.com/filecoin-project/go-fil-markets/storagemarket/impl" "github.com/filecoin-project/go-sectorbuilder" lapi "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/deals" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/genesis" + "github.com/filecoin-project/lotus/markets/utils" "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node/modules" "github.com/filecoin-project/lotus/node/modules/dtypes" @@ -93,7 +96,7 @@ var initCmd = &cli.Command{ } log.Info("Checking proof parameters") - if err := paramfetch.GetParams(ssize); err != nil { + if err := paramfetch.GetParams(build.ParametersJson, ssize); err != nil { return xerrors.Errorf("fetching proof parameters: %w", err) } @@ -174,7 +177,7 @@ var initCmd = &cli.Command{ SectorSize: ssize, WorkerThreads: 2, Dir: pssb, - }, oldmds) + }, namespace.Wrap(oldmds, datastore.NewKey("/sectorbuilder"))) if err != nil { return xerrors.Errorf("failed to open up preseal sectorbuilder: %w", err) } @@ -183,7 +186,7 @@ var initCmd = &cli.Command{ SectorSize: ssize, WorkerThreads: 2, Dir: lr.Path(), - }, mds) + }, namespace.Wrap(mds, datastore.NewKey("/sectorbuilder"))) if err != nil { return xerrors.Errorf("failed to open up sectorbuilder: %w", err) } @@ -281,15 +284,21 @@ func migratePreSealMeta(ctx context.Context, api lapi.FullNode, presealDir strin dealKey := datastore.NewKey(deals.ProviderDsPrefix).ChildString(proposalCid.String()) + proposal, err := utils.ToSharedStorageDealProposal(§or.Deal) + if err != nil { + return err + } deal := &deals.MinerDeal{ - Proposal: sector.Deal, - ProposalCid: proposalCid, - State: lapi.DealComplete, - Ref: proposalCid, // TODO: This is super wrong, but there - // are no params for CommP CIDs, we can't recover unixfs cid easily, - // and this isn't even used after the deal enters Complete state - DealID: dealID, - SectorID: sector.SectorID, + MinerDeal: storagemarket.MinerDeal{ + Proposal: *proposal, + ProposalCid: proposalCid, + State: lapi.DealComplete, + Ref: proposalCid, // TODO: This is super wrong, but there + // are no params for CommP CIDs, we can't recover unixfs cid easily, + // and this isn't even used after the deal enters Complete state + DealID: dealID, + SectorID: sector.SectorID, + }, } b, err = cborutil.Dump(deal) diff --git a/cmd/lotus-storage-miner/main.go b/cmd/lotus-storage-miner/main.go index c05383116..ec91a7f4a 100644 --- a/cmd/lotus-storage-miner/main.go +++ b/cmd/lotus-storage-miner/main.go @@ -3,7 +3,7 @@ package main import ( "os" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "go.opencensus.io/trace" "gopkg.in/urfave/cli.v2" diff --git a/cmd/lotus-townhall/main.go b/cmd/lotus-townhall/main.go index 8e7e78008..a2cc8bcef 100644 --- a/cmd/lotus-townhall/main.go +++ b/cmd/lotus-townhall/main.go @@ -6,7 +6,6 @@ import ( "encoding/json" "fmt" "net/http" - "strings" "time" rice "github.com/GeertJohan/go.rice" @@ -16,17 +15,20 @@ import ( blockstore "github.com/ipfs/go-ipfs-blockstore" "github.com/libp2p/go-libp2p" "github.com/libp2p/go-libp2p-core/peer" - pnet "github.com/libp2p/go-libp2p-pnet" pubsub "github.com/libp2p/go-libp2p-pubsub" "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/node/modules/lp2p" ) var topic = "/fil/headnotifs/" func init() { genBytes := build.MaybeGenesis() + if len(genBytes) == 0 { + topic = "" + return + } + bs := blockstore.NewBlockstore(datastore.NewMapDatastore()) c, err := car.LoadCar(bs, bytes.NewReader(genBytes)) @@ -49,17 +51,16 @@ var upgrader = websocket.Upgrader{ } func main() { - ctx := context.Background() - - protec, err := pnet.NewProtector(strings.NewReader(lp2p.LotusKey)) - if err != nil { - panic(err) + if topic == "" { + fmt.Println("FATAL: No genesis found") + return } + ctx := context.Background() + host, err := libp2p.New( ctx, libp2p.Defaults, - libp2p.PrivateNetwork(protec), ) if err != nil { panic(err) diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 4675453f7..59b17ceab 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -68,7 +68,7 @@ var DaemonCmd = &cli.Command{ return xerrors.Errorf("repo init error: %w", err) } - if err := paramfetch.GetParams(0); err != nil { + if err := paramfetch.GetParams(build.ParametersJson, 0); err != nil { return xerrors.Errorf("fetching proof parameters: %w", err) } diff --git a/cmd/lotus/rpc.go b/cmd/lotus/rpc.go index 5178e1b6c..0ec44b148 100644 --- a/cmd/lotus/rpc.go +++ b/cmd/lotus/rpc.go @@ -17,7 +17,7 @@ import ( "github.com/filecoin-project/lotus/node/impl" "github.com/ipfs/go-cid" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" "golang.org/x/xerrors" diff --git a/datatransfer/dagservice_impl.go b/datatransfer/dagservice_impl.go deleted file mode 100644 index 98c15d062..000000000 --- a/datatransfer/dagservice_impl.go +++ /dev/null @@ -1,78 +0,0 @@ -package datatransfer - -import ( - "context" - "reflect" - - "github.com/ipfs/go-cid" - ipldformat "github.com/ipfs/go-ipld-format" - "github.com/ipfs/go-merkledag" - ipld "github.com/ipld/go-ipld-prime" - "github.com/libp2p/go-libp2p-core/peer" - "golang.org/x/xerrors" -) - -// This file implements a VERY simple, incomplete version of the data transfer -// module that allows us to make the necessary insertions of data transfer -// functionality into the storage market -// It does not: -// -- actually validate requests -// -- support Push requests -// -- support multiple subscribers -// -- do any actual network coordination or use Graphsync - -type dagserviceImpl struct { - dag ipldformat.DAGService - subscriber Subscriber -} - -// NewDAGServiceDataTransfer returns a data transfer manager based on -// an IPLD DAGService -func NewDAGServiceDataTransfer(dag ipldformat.DAGService) Manager { - return &dagserviceImpl{dag, nil} -} - -// RegisterVoucherType registers a validator for the given voucher type -// will error if voucher type does not implement voucher -// or if there is a voucher type registered with an identical identifier -func (impl *dagserviceImpl) RegisterVoucherType(voucherType reflect.Type, validator RequestValidator) error { - return nil -} - -// open a data transfer that will send data to the recipient peer and -// transfer parts of the piece that match the selector -func (impl *dagserviceImpl) OpenPushDataChannel(ctx context.Context, to peer.ID, voucher Voucher, baseCid cid.Cid, Selector ipld.Node) (ChannelID, error) { - return ChannelID{}, xerrors.Errorf("not implemented") -} - -// open a data transfer that will request data from the sending peer and -// transfer parts of the piece that match the selector -func (impl *dagserviceImpl) OpenPullDataChannel(ctx context.Context, to peer.ID, voucher Voucher, baseCid cid.Cid, Selector ipld.Node) (ChannelID, error) { - ctx, cancel := context.WithCancel(ctx) - go func() { - defer cancel() - err := merkledag.FetchGraph(ctx, baseCid, impl.dag) - var event Event - if err != nil { - event = Error - } else { - event = Complete - } - impl.subscriber(event, ChannelState{Channel: Channel{voucher: voucher}}) - }() - return ChannelID{}, nil -} - -// close an open channel (effectively a cancel) -func (impl *dagserviceImpl) CloseDataTransferChannel(x ChannelID) {} - -// get status of a transfer -func (impl *dagserviceImpl) TransferChannelStatus(x ChannelID) Status { return ChannelNotFoundError } - -// get notified when certain types of events happen -func (impl *dagserviceImpl) SubscribeToEvents(subscriber Subscriber) { - impl.subscriber = subscriber -} - -// get all in progress transfers -func (impl *dagserviceImpl) InProgressChannels() map[ChannelID]ChannelState { return nil } diff --git a/datatransfer/types.go b/datatransfer/types.go deleted file mode 100644 index 684b7f7a8..000000000 --- a/datatransfer/types.go +++ /dev/null @@ -1,173 +0,0 @@ -package datatransfer - -import ( - "context" - "reflect" - - "github.com/ipfs/go-cid" - ipld "github.com/ipld/go-ipld-prime" - "github.com/libp2p/go-libp2p-core/peer" -) - -// Voucher is used to validate -// a data transfer request against the underlying storage or retrieval deal -// that precipitated it. The only requirement is a voucher can read and write -// from bytes, and has a string identifier type -type Voucher interface { - // ToBytes converts the Voucher to raw bytes - ToBytes() ([]byte, error) - // FromBytes reads a Voucher from raw bytes - FromBytes([]byte) error - // Identifier is a unique string identifier for this voucher type - Identifier() string -} - -// Status is the status of transfer for a given channel -type Status int - -const ( - // Ongoing means the data transfer is in progress - Ongoing Status = iota - - // Completed means the data transfer is completed successfully - Completed - - // Failed means the data transfer failed - Failed - - // ChannelNotFoundError means the searched for data transfer does not exist - ChannelNotFoundError -) - -// TransferID is an identifier for a data transfer, shared between -// request/responder and unique to the requester -type TransferID uint64 - -// ChannelID is a unique identifier for a channel, distinct by both the other -// party's peer ID + the transfer ID -type ChannelID struct { - to peer.ID - id TransferID -} - -// Channel represents all the parameters for a single data transfer -type Channel struct { - // an identifier for this channel shared by request and responder, set by requester through protocol - transferID TransferID - // base CID for the piece being transferred - baseCid cid.Cid - // portion of Piece to return, specified by an IPLD selector - selector ipld.Node - // used to verify this channel - voucher Voucher - // the party that is sending the data (not who initiated the request) - sender peer.ID - // the party that is receiving the data (not who initiated the request) - recipient peer.ID - // expected amount of data to be transferred - totalSize uint64 -} - -// TransferID returns the transfer id for this channel -func (c Channel) TransferID() TransferID { return c.transferID } - -// BaseCID returns the CID that is at the root of this data transfer -func (c Channel) BaseCID() cid.Cid { return c.baseCid } - -// Selector returns the IPLD selector for this data transfer (represented as -// an IPLD node) -func (c Channel) Selector() ipld.Node { return c.selector } - -// Voucher returns the voucher for this data transfer -func (c Channel) Voucher() Voucher { return c.voucher } - -// Sender returns the peer id for the node that is sending data -func (c Channel) Sender() peer.ID { return c.sender } - -// Recipient returns the peer id for the node that is receiving data -func (c Channel) Recipient() peer.ID { return c.recipient } - -// TotalSize returns the total size for the data being transferred -func (c Channel) TotalSize() uint64 { return c.totalSize } - -// ChannelState is immutable channel data plus mutable state -type ChannelState struct { - Channel - // total bytes sent from this node (0 if receiver) - sent uint64 - // total bytes received by this node (0 if sender) - received uint64 -} - -// Sent returns the number of bytes sent -func (c ChannelState) Sent() uint64 { return c.sent } - -// Received returns the number of bytes received -func (c ChannelState) Received() uint64 { return c.received } - -// Event is a name for an event that occurs on a data transfer channel -type Event int - -const ( - // Open is an event occurs when a channel is first opened - Open Event = iota - - // Progress is an event that gets emitted every time more data is transferred - Progress - - // Error is an event that emits when an error occurs in a data transfer - Error - - // Complete is emitted when a data transfer is complete - Complete -) - -// Subscriber is a callback that is called when events are emitted -type Subscriber func(event Event, channelState ChannelState) - -// RequestValidator is an interface implemented by the client of the -// data transfer module to validate requests -type RequestValidator interface { - // ValidatePush validates a push request received from the peer that will send data - ValidatePush( - sender peer.ID, - voucher Voucher, - baseCid cid.Cid, - selector ipld.Node) error - // ValidatePull validates a pull request received from the peer that will receive data - ValidatePull( - receiver peer.ID, - voucher Voucher, - baseCid cid.Cid, - selector ipld.Node) error -} - -// Manager is the core interface presented by all implementations of -// of the data transfer sub system -type Manager interface { - // RegisterVoucherType registers a validator for the given voucher type - // will error if voucher type does not implement voucher - // or if there is a voucher type registered with an identical identifier - RegisterVoucherType(voucherType reflect.Type, validator RequestValidator) error - - // open a data transfer that will send data to the recipient peer and - // open a data transfer that will send data to the recipient peer and - // transfer parts of the piece that match the selector - OpenPushDataChannel(ctx context.Context, to peer.ID, voucher Voucher, baseCid cid.Cid, selector ipld.Node) (ChannelID, error) - - // open a data transfer that will request data from the sending peer and - // transfer parts of the piece that match the selector - OpenPullDataChannel(ctx context.Context, to peer.ID, voucher Voucher, baseCid cid.Cid, selector ipld.Node) (ChannelID, error) - - // close an open channel (effectively a cancel) - CloseDataTransferChannel(x ChannelID) - - // get status of a transfer - TransferChannelStatus(x ChannelID) Status - - // get notified when certain types of events happen - SubscribeToEvents(subscriber Subscriber) - - // get all in progress transfers - InProgressChannels() map[ChannelID]ChannelState -} diff --git a/documentation/en/join-testnet.md b/documentation/en/join-testnet.md index ae97db49f..4d48af20c 100644 --- a/documentation/en/join-testnet.md +++ b/documentation/en/join-testnet.md @@ -47,16 +47,16 @@ lotus sync wait ## Create your first address -Initialize a wallet using BLS signature formats: +Initialize a new wallet: ```sh -lotus wallet new bls +lotus wallet new ``` Here is an example of the response: ```sh -t3vhfme4qfvegqaz7m7q6o6afjcs67n6kpzv7t2eozio4chwpafwa2y4l7zhwd5eom7jmihzdg4s52dpvnclza +t1aswwvjsae63tcrniz6x5ykvsuotlgkvlulnqpsi ``` - Visit the [faucet](https://lotus-faucet.kittyhawk.wtf/funds.html) to add funds. diff --git a/gen/main.go b/gen/main.go index 52fcf4665..7778135ad 100644 --- a/gen/main.go +++ b/gen/main.go @@ -9,10 +9,8 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/blocksync" - "github.com/filecoin-project/lotus/chain/deals" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/paych" - "github.com/filecoin-project/lotus/retrieval" "github.com/filecoin-project/lotus/storage" ) @@ -59,21 +57,6 @@ func main() { os.Exit(1) } - err = gen.WriteTupleEncodersToFile("./retrieval/cbor_gen.go", "retrieval", - retrieval.RetParams{}, - - retrieval.Query{}, - retrieval.QueryResponse{}, - retrieval.Unixfs0Offer{}, - retrieval.DealProposal{}, - retrieval.DealResponse{}, - retrieval.Block{}, - ) - if err != nil { - fmt.Println(err) - os.Exit(1) - } - err = gen.WriteTupleEncodersToFile("./chain/blocksync/cbor_gen.go", "blocksync", blocksync.BlockSyncRequest{}, blocksync.BlockSyncResponse{}, @@ -138,22 +121,6 @@ func main() { os.Exit(1) } - err = gen.WriteTupleEncodersToFile("./chain/deals/cbor_gen.go", "deals", - deals.AskRequest{}, - deals.AskResponse{}, - deals.Proposal{}, - deals.Response{}, - deals.SignedResponse{}, - deals.ClientDealProposal{}, - deals.ClientDeal{}, - deals.MinerDeal{}, - deals.StorageDataTransferVoucher{}, - ) - if err != nil { - fmt.Println(err) - os.Exit(1) - } - err = gen.WriteMapEncodersToFile("./storage/cbor_gen.go", "storage", storage.SealTicket{}, storage.SealSeed{}, diff --git a/go.mod b/go.mod index 6046a9bcf..84efee20d 100644 --- a/go.mod +++ b/go.mod @@ -15,12 +15,13 @@ require ( github.com/filecoin-project/go-amt-ipld v0.0.0-20191205011053-79efc22d6cdc github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 - github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 - github.com/filecoin-project/go-sectorbuilder v0.0.0-20200107152336-0cbb2c483013 - github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5 + github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce + github.com/filecoin-project/go-fil-markets v0.0.0-20200114015428-74d100f305f8 + github.com/filecoin-project/go-paramfetch v0.0.1 + github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200114015900-4103afa82689 + github.com/filecoin-project/go-statestore v0.1.0 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 github.com/go-ole/go-ole v1.2.4 // indirect - github.com/google/go-cmp v0.3.1 // indirect github.com/gorilla/mux v1.7.3 github.com/gorilla/websocket v1.4.1 github.com/hashicorp/go-multierror v1.0.0 @@ -29,15 +30,15 @@ require ( github.com/ipfs/go-bitswap v0.1.8 github.com/ipfs/go-block-format v0.0.2 github.com/ipfs/go-blockservice v0.1.3-0.20190908200855-f22eea50656c - github.com/ipfs/go-car v0.0.2 + github.com/ipfs/go-car v0.0.3-0.20191203022317-23b0a85fd1b1 github.com/ipfs/go-cid v0.0.4 - github.com/ipfs/go-datastore v0.1.1 - github.com/ipfs/go-ds-badger v0.0.7 + github.com/ipfs/go-datastore v0.3.1 + github.com/ipfs/go-ds-badger2 v0.0.0-20200108185345-7f650e6b2521 github.com/ipfs/go-filestore v0.0.2 github.com/ipfs/go-fs-lock v0.0.1 + github.com/ipfs/go-graphsync v0.0.4 github.com/ipfs/go-hamt-ipld v0.0.14-0.20191218031521-b2c774a54db1 github.com/ipfs/go-ipfs-blockstore v0.1.1 - github.com/ipfs/go-ipfs-blocksutil v0.0.1 github.com/ipfs/go-ipfs-chunker v0.0.1 github.com/ipfs/go-ipfs-ds-help v0.0.1 github.com/ipfs/go-ipfs-exchange-interface v0.0.1 @@ -46,11 +47,11 @@ require ( github.com/ipfs/go-ipfs-routing v0.1.0 github.com/ipfs/go-ipld-cbor v0.0.3 github.com/ipfs/go-ipld-format v0.0.2 - github.com/ipfs/go-log v1.0.0 + github.com/ipfs/go-log v1.0.1 + github.com/ipfs/go-log/v2 v2.0.2 github.com/ipfs/go-merkledag v0.2.4 github.com/ipfs/go-path v0.0.7 github.com/ipfs/go-unixfs v0.2.2-0.20190827150610-868af2e9e5cb - github.com/ipld/go-ipld-prime v0.0.2-0.20191025154717-8dff1cbec43b github.com/lib/pq v1.2.0 github.com/libp2p/go-libp2p v0.4.2 github.com/libp2p/go-libp2p-circuit v0.1.4 @@ -61,7 +62,6 @@ require ( github.com/libp2p/go-libp2p-mplex v0.2.1 github.com/libp2p/go-libp2p-peer v0.2.0 github.com/libp2p/go-libp2p-peerstore v0.1.4 - github.com/libp2p/go-libp2p-pnet v0.1.0 github.com/libp2p/go-libp2p-pubsub v0.2.3 github.com/libp2p/go-libp2p-quic-transport v0.1.1 github.com/libp2p/go-libp2p-record v0.1.1 @@ -79,24 +79,26 @@ require ( github.com/multiformats/go-multiaddr-net v0.1.1 github.com/multiformats/go-multihash v0.0.10 github.com/multiformats/go-varint v0.0.2 - github.com/onsi/ginkgo v1.9.0 // indirect - github.com/onsi/gomega v1.6.0 // indirect github.com/opentracing/opentracing-go v1.1.0 github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a github.com/prometheus/common v0.2.0 github.com/stretchr/testify v1.4.0 github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba - github.com/whyrusleeping/cbor-gen v0.0.0-20191216205031-b047b6acb3c0 + github.com/whyrusleeping/cbor-gen v0.0.0-20200106232624-282db0d37dbe github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d go.opencensus.io v0.22.2 + go.uber.org/atomic v1.5.1 // indirect go.uber.org/dig v1.7.0 // indirect go.uber.org/fx v1.9.0 go.uber.org/goleak v0.10.0 // indirect go.uber.org/multierr v1.4.0 - go.uber.org/zap v1.10.0 + go.uber.org/zap v1.13.0 go4.org v0.0.0-20190313082347-94abd6928b1d // indirect + golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect + golang.org/x/sys v0.0.0-20200107162124-548cf772de50 // indirect golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 + golang.org/x/tools v0.0.0-20200108195415-316d2f248479 // indirect golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 gopkg.in/cheggaaa/pb.v1 v1.0.28 gopkg.in/urfave/cli.v2 v2.0.0-20180128182452-d3ae77c26ac8 diff --git a/go.sum b/go.sum index a552dcd4f..8c0a6aa88 100644 --- a/go.sum +++ b/go.sum @@ -7,6 +7,9 @@ github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9 h1:HD8gA2tkBy github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= +github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/GeertJohan/go.incremental v1.0.0 h1:7AH+pY1XUgQE4Y1HcXYaMqAI0m9yrFqo/jt0CW30vsg= github.com/GeertJohan/go.incremental v1.0.0 h1:7AH+pY1XUgQE4Y1HcXYaMqAI0m9yrFqo/jt0CW30vsg= github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0= github.com/GeertJohan/go.rice v1.0.0 h1:KkI6O9uMaQU3VEKaj01ulavtF7o1fWT7+pk/4voiMLQ= @@ -14,6 +17,8 @@ github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtix github.com/Gurpartap/async v0.0.0-20180927173644-4f7f499dd9ee h1:8doiS7ib3zi6/K172oDhSKU0dJ/miJramo9NITOMyZQ= github.com/Gurpartap/async v0.0.0-20180927173644-4f7f499dd9ee/go.mod h1:W0GbEAA4uFNYOGG2cJpmFJ04E6SD1NLELPYZB57/7AY= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= @@ -34,6 +39,7 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= github.com/btcsuite/btcd v0.0.0-20190523000118-16327141da8c/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= github.com/btcsuite/btcd v0.0.0-20190605094302-a0d1e3e36d50/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= +github.com/btcsuite/btcd v0.0.0-20190629003639-c26ffa870fd8/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3 h1:A/EVblehb75cUgXA5njHPn0kLAsykn6mJGz7rnmW5W0= github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= @@ -44,6 +50,8 @@ github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVa github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE= github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -53,7 +61,10 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-semver v0.2.1-0.20180108230905-e214231b295a/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= github.com/daaku/go.zipexe v1.0.0 h1:VSOgZtH418pH9L16hC/JrgSNJbbAL26pj7lmD1+CGdY= @@ -62,12 +73,14 @@ github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018 h1:6xT9KW8zLC5IlbaIF5Q7JNieBoACT7iW0YTxQHR0in0= -github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018/go.mod h1:rQYf4tfk5sSwFsnDg3qYaBxSjsD9S8+59vW0dKUgme4= github.com/dgraph-io/badger v1.5.5-0.20190226225317-8115aed38f8f/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ= github.com/dgraph-io/badger v1.6.0-rc1/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgraph-io/badger v1.6.0 h1:DshxFxZWXUcO0xX476VJC07Xsr6ZCBVRHKZ93Oh7Evo= github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= +github.com/dgraph-io/badger/v2 v2.0.1 h1:+D6dhIqC6jIeCclnxMHqk4HPuXgrRN5UfBsLR4dNQ3A= +github.com/dgraph-io/badger/v2 v2.0.1/go.mod h1:YoRSIp1LmAJ7zH7tZwRvjNMUYLxB4wl3ebYkaIruZ04= +github.com/dgraph-io/ristretto v0.0.0-20191025175511-c1f00be0418e h1:aeUNgwup7PnDOBAD1BOKAqzb/W/NksOj6r3dwKKuqfg= +github.com/dgraph-io/ristretto v0.0.0-20191025175511-c1f00be0418e/go.mod h1:edzKIzGvqUCMzhTVWbiTSe75zD9Xxq0GtSBtFmaUTZs= github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= @@ -80,6 +93,8 @@ github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.8.0 h1:5bzFgL+oy7JITMTxUPJ00n7VxmYd/PdMp5mHFX40/RY= +github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGjnw8= github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E= github.com/filecoin-project/chain-validation v0.0.3 h1:luT/8kJ0WdMIqQ9Bm31W4JkuYCW0wUb26AvnD4WK59M= github.com/filecoin-project/chain-validation v0.0.3/go.mod h1:NCEGFjcWRjb8akWFSOXvU6n2efkWIqAeOKU6o5WBGQw= @@ -93,14 +108,29 @@ github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMX github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= +github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce h1:Jdejrx6XVSTRy2PiX08HCU5y68p3wx2hNMJJc/J7kZY= +github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce/go.mod h1:b14UWxhxVCAjrQUYvVGrQRRsjAh79wXYejw9RbUcAww= +github.com/filecoin-project/go-fil-markets v0.0.0-20200114015428-74d100f305f8 h1:g3oodvSz+Ou+ObwcVBB2wyt8SHdWpwzMiNJ19U1zZNA= +github.com/filecoin-project/go-fil-markets v0.0.0-20200114015428-74d100f305f8/go.mod h1:c8NTjvFVy1Ud02mmGDjOiMeawY2t6ALfrrdvAB01FQc= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU= -github.com/filecoin-project/go-sectorbuilder v0.0.0-20200107152336-0cbb2c483013 h1:OGpRq3HRxyrxZJtbNKCOsb5YTmc+RBLLwdAgwZfkRnY= -github.com/filecoin-project/go-sectorbuilder v0.0.0-20200107152336-0cbb2c483013/go.mod h1:3OZ4E3B2OuwhJjtxR4r7hPU9bCfB+A+hm4alLEsaeDc= -github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5 h1:NZXq90YlfakSmB2/84dGr0AVmKYFA97+yyViBIgTFbk= -github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= +github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= +github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= +github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= +github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= +github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= +github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU= +github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU= +github.com/filecoin-project/go-paramfetch v0.0.1 h1:gV7bs5YaqlgpGFMiLxInGK2L1FyCXUE0rimz4L7ghoE= +github.com/filecoin-project/go-paramfetch v0.0.1/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc= +github.com/filecoin-project/go-sectorbuilder v0.0.1 h1:yiLSEprWA1E43DFTSCXLSuCstYuDKiI6RCXiYz4GaRs= +github.com/filecoin-project/go-sectorbuilder v0.0.1/go.mod h1:3OZ4E3B2OuwhJjtxR4r7hPU9bCfB+A+hm4alLEsaeDc= +github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200114015900-4103afa82689 h1:2cT5bhm/5I0RY+HBIPdRRrtjCwLj33Qx6DHRs9TCslY= +github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200114015900-4103afa82689/go.mod h1:3OZ4E3B2OuwhJjtxR4r7hPU9bCfB+A+hm4alLEsaeDc= +github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIiWBRilQjQ+5IiwdQ= +github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 h1:EzDjxMg43q1tA2c0MV3tNbaontnHLplHyFF6M5KiVP0= @@ -126,9 +156,12 @@ github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= -github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= @@ -150,6 +183,8 @@ github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= github.com/gxed/pubsub v0.0.0-20180201040156-26ebdf44f824/go.mod h1:OiEWyHgK+CWrmOlVquHaIK1vhpUJydC9m0Je6mhaiNE= +github.com/hannahhoward/cbor-gen-for v0.0.0-20191216214420-3e450425c40c h1:+MSf4NEnLCYZoAgK6fqwc7NH88nM8haFSxKGUGIG3vA= +github.com/hannahhoward/cbor-gen-for v0.0.0-20191216214420-3e450425c40c/go.mod h1:WVPCl0HO/0RAL5+vBH2GMxBomlxBF70MAS78+Lu1//k= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= @@ -184,8 +219,8 @@ github.com/ipfs/go-blockservice v0.0.7/go.mod h1:EOfb9k/Y878ZTRY/CH0x5+ATtaipfbR github.com/ipfs/go-blockservice v0.1.0/go.mod h1:hzmMScl1kXHg3M2BjTymbVPjv627N7sYcvYaKbop39M= github.com/ipfs/go-blockservice v0.1.3-0.20190908200855-f22eea50656c h1:lN5IQA07VtLiTLAp/Scezp1ljFhXErC6yq4O1cu+yJ0= github.com/ipfs/go-blockservice v0.1.3-0.20190908200855-f22eea50656c/go.mod h1:t+411r7psEUhLueM8C7aPA7cxCclv4O3VsUVxt9kz2I= -github.com/ipfs/go-car v0.0.2 h1:j02lzgeijorstzoMl3nQmvvb8wjJUVCiOAl8XEwYMCQ= -github.com/ipfs/go-car v0.0.2/go.mod h1:60pzeu308k5kVFHzq0HIi2kPtITgor+1ll1xuGk5JwQ= +github.com/ipfs/go-car v0.0.3-0.20191203022317-23b0a85fd1b1 h1:Nq8xEW+2KZq7IkRlkOh0rTEUI8FgunhMoLj5EMkJzbQ= +github.com/ipfs/go-car v0.0.3-0.20191203022317-23b0a85fd1b1/go.mod h1:rmd887mJxQRDfndfDEY3Liyx8gQVyfFFRSHdsnDSAlk= github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= @@ -197,18 +232,24 @@ github.com/ipfs/go-datastore v0.0.5/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAK github.com/ipfs/go-datastore v0.1.0/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= github.com/ipfs/go-datastore v0.1.1 h1:F4k0TkTAZGLFzBOrVKDAvch6JZtuN4NHkfdcEZL50aI= github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= +github.com/ipfs/go-datastore v0.3.1 h1:SS1t869a6cctoSYmZXUk8eL6AzVXgASmKIWFNQkQ1jU= +github.com/ipfs/go-datastore v0.3.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= github.com/ipfs/go-ds-badger v0.0.2/go.mod h1:Y3QpeSFWQf6MopLTiZD+VT6IC1yZqaGmjvRcKeSGij8= github.com/ipfs/go-ds-badger v0.0.5/go.mod h1:g5AuuCGmr7efyzQhLL8MzwqcauPojGPUaHzfGTzuE3s= github.com/ipfs/go-ds-badger v0.0.7 h1:NMyh88Q50HG6/S2YD58DLkq0c0/ZQPMbSojONH+PRf4= github.com/ipfs/go-ds-badger v0.0.7/go.mod h1:qt0/fWzZDoPW6jpQeqUjR5kBfhDNB65jd9YlmAvpQBk= +github.com/ipfs/go-ds-badger2 v0.0.0-20200108185345-7f650e6b2521 h1:GSygNXXIRXFVntMR3Yr30imf40H9cYDAw72Qb0za2nE= +github.com/ipfs/go-ds-badger2 v0.0.0-20200108185345-7f650e6b2521/go.mod h1:oDKWqmQbLyyml3mgMtHgB7qAf6cQOEsJPRLFzfwKQ5Q= github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIykR4L04tMOYc= github.com/ipfs/go-ds-leveldb v0.1.0/go.mod h1:hqAW8y4bwX5LWcCtku2rFNX3vjDZCy5LZCg+cSZvYb8= github.com/ipfs/go-filestore v0.0.2 h1:pcYwpjtXXwirtbjBXKVJM9CTa9F7/8v1EkfnDaHTO3s= github.com/ipfs/go-filestore v0.0.2/go.mod h1:KnZ41qJsCt2OX2mxZS0xsK3Psr0/oB93HMMssLujjVc= github.com/ipfs/go-fs-lock v0.0.1 h1:XHX8uW4jQBYWHj59XXcjg7BHlHxV9ZOYs6Y43yb7/l0= github.com/ipfs/go-fs-lock v0.0.1/go.mod h1:DNBekbboPKcxs1aukPSaOtFA3QfSdi5C855v0i9XJ8Y= +github.com/ipfs/go-graphsync v0.0.4 h1:iF98+J8pcqvEb48IM0TemqeGARsCDtwQ73P9ejMZIuU= +github.com/ipfs/go-graphsync v0.0.4/go.mod h1:6UACBjfOXEa8rQL3Q/JpZpWS0nZDCLx134WUkjrmFpQ= github.com/ipfs/go-hamt-ipld v0.0.14-0.20191218031521-b2c774a54db1 h1:0xpCaaXvN8bPyws3ObiCn7G0KIfgbS9E132eL57dHx8= github.com/ipfs/go-hamt-ipld v0.0.14-0.20191218031521-b2c774a54db1/go.mod h1:8yRx0xLUps1Xq8ZDnIwIVdQRp7JjA55gGvCiRHT91Vk= github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= @@ -250,6 +291,16 @@ github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= github.com/ipfs/go-log v1.0.0 h1:BW3LQIiZzpNyolt84yvKNCd3FU+AK4VDw1hnHR+1aiI= github.com/ipfs/go-log v1.0.0/go.mod h1:JO7RzlMK6rA+CIxFMLOuB6Wf5b81GDiKElL7UPSIKjA= +github.com/ipfs/go-log v1.0.1 h1:5lIEEOQTk/vd1WuPFBRqz2mcp+5G1fMVcW+Ib/H5Hfo= +github.com/ipfs/go-log v1.0.1 h1:5lIEEOQTk/vd1WuPFBRqz2mcp+5G1fMVcW+Ib/H5Hfo= +github.com/ipfs/go-log v1.0.1/go.mod h1:HuWlQttfN6FWNHRhlY5yMk/lW7evQC0HHGOxEwMRR8I= +github.com/ipfs/go-log v1.0.1/go.mod h1:HuWlQttfN6FWNHRhlY5yMk/lW7evQC0HHGOxEwMRR8I= +github.com/ipfs/go-log/v2 v2.0.1 h1:mnR9XFltezAtO8A6tj5U7nKkRzhEQNEw/wT11U2HhPM= +github.com/ipfs/go-log/v2 v2.0.1 h1:mnR9XFltezAtO8A6tj5U7nKkRzhEQNEw/wT11U2HhPM= +github.com/ipfs/go-log/v2 v2.0.1/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0= +github.com/ipfs/go-log/v2 v2.0.1/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0= +github.com/ipfs/go-log/v2 v2.0.2 h1:xguurydRdfKMJjKyxNXNU8lYP0VZH1NUwJRwUorjuEw= +github.com/ipfs/go-log/v2 v2.0.2/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0= github.com/ipfs/go-merkledag v0.0.6/go.mod h1:QYPdnlvkOg7GnQRofu9XZimC5ZW5Wi3bKys/4GQQfto= github.com/ipfs/go-merkledag v0.1.0/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= @@ -269,8 +320,10 @@ github.com/ipfs/go-unixfs v0.2.2-0.20190827150610-868af2e9e5cb h1:tmWYgjltxwM7PD github.com/ipfs/go-unixfs v0.2.2-0.20190827150610-868af2e9e5cb/go.mod h1:IwAAgul1UQIcNZzKPYZWOCijryFBeCV79cNubPzol+k= github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E= github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0= -github.com/ipld/go-ipld-prime v0.0.2-0.20191025154717-8dff1cbec43b h1:ACSEK4f1SDQC+FJ4B4pqHFW14d7kEW2ufwXA/c7eLP0= -github.com/ipld/go-ipld-prime v0.0.2-0.20191025154717-8dff1cbec43b/go.mod h1:bDDSvVz7vaK12FNvMeRYnpRFkSUPNQOiCYQezMD/P3w= +github.com/ipld/go-ipld-prime v0.0.2-0.20191108012745-28a82f04c785 h1:fASnkvtR+SmB2y453RxmDD3Uvd4LonVUgFGk9JoDaZs= +github.com/ipld/go-ipld-prime v0.0.2-0.20191108012745-28a82f04c785/go.mod h1:bDDSvVz7vaK12FNvMeRYnpRFkSUPNQOiCYQezMD/P3w= +github.com/ipld/go-ipld-prime-proto v0.0.0-20191113031812-e32bd156a1e5 h1:lSip43rAdyGA+yRQuy6ju0ucZkWpYc1F2CTQtZTVW/4= +github.com/ipld/go-ipld-prime-proto v0.0.0-20191113031812-e32bd156a1e5/go.mod h1:gcvzoEDBjwycpXt3LBE061wT9f46szXGHAmj9uoP6fU= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52/go.mod h1:fdg+/X9Gg4AsAIzWpEHwnqd+QY3b7lajxyjE1m4hkq4= github.com/jackpal/gateway v1.0.4/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= @@ -281,6 +334,8 @@ github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+ github.com/jbenet/go-cienv v0.0.0-20150120210510-1bb1476777ec/go.mod h1:rGaEvXB4uRSZMmzKNLoXvTu1sfx+1kv/DojUlPrSZGs= github.com/jbenet/go-cienv v0.1.0 h1:Vc/s0QbQtoxX8MwwSLWWh+xNNZvM3Lw7NsTcHrvvhMc= github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= +github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c h1:uUx61FiAa1GI6ZmVd2wf2vULeQZIKG66eybjNXKYCz4= +github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c/go.mod h1:sdx1xVM9UuLw1tXnhJWN3piypTUO3vCIHYmG15KE/dU= github.com/jbenet/go-temp-err-catcher v0.0.0-20150120210811-aac704a3f4f2 h1:vhC1OXXiT9R2pczegwz6moDvuRpggaroAXhPIseh57A= github.com/jbenet/go-temp-err-catcher v0.0.0-20150120210811-aac704a3f4f2/go.mod h1:8GXXJV31xl8whumTzdZsTt3RnUIiPqzkyf7mxToRCMs= github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY= @@ -321,6 +376,8 @@ github.com/libp2p/go-conn-security v0.0.1/go.mod h1:bGmu51N0KU9IEjX7kl2PQjgZa40J github.com/libp2p/go-conn-security-multistream v0.0.2/go.mod h1:nc9vud7inQ+d6SO0I/6dSWrdMnHnzZNHeyUQqrAJulE= github.com/libp2p/go-conn-security-multistream v0.1.0 h1:aqGmto+ttL/uJgX0JtQI0tD21CIEy5eYd1Hlp0juHY0= github.com/libp2p/go-conn-security-multistream v0.1.0/go.mod h1:aw6eD7LOsHEX7+2hJkDxw1MteijaVcI+/eP2/x3J1xc= +github.com/libp2p/go-eventbus v0.0.2/go.mod h1:Hr/yGlwxA/stuLnpMiu82lpNKpvRy3EaJxPu40XYOwk= +github.com/libp2p/go-eventbus v0.0.3/go.mod h1:Hr/yGlwxA/stuLnpMiu82lpNKpvRy3EaJxPu40XYOwk= github.com/libp2p/go-eventbus v0.1.0 h1:mlawomSAjjkk97QnYiEmHsLu7E136+2oCWSHRUvMfzQ= github.com/libp2p/go-eventbus v0.1.0/go.mod h1:vROgu5cs5T7cv7POWlWxBaVLxfSegC5UGQf8A2eEmx4= github.com/libp2p/go-flow-metrics v0.0.1 h1:0gxuFd2GuK7IIP5pKljLwps6TvcuYgvG7Atqi3INF5s= @@ -328,6 +385,8 @@ github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZ github.com/libp2p/go-libp2p v0.0.30/go.mod h1:XWT8FGHlhptAv1+3V/+J5mEpzyui/5bvFsNuWYs611A= github.com/libp2p/go-libp2p v0.1.0/go.mod h1:6D/2OBauqLUoqcADOJpn9WbKqvaM07tDw68qHM0BxUM= github.com/libp2p/go-libp2p v0.1.1/go.mod h1:I00BRo1UuUSdpuc8Q2mN7yDF/oTUTRAX6JWpTiK9Rp8= +github.com/libp2p/go-libp2p v0.2.1/go.mod h1:HZbtEOrgZN4F1fGZVvkV+930Wx3DkqlpBlO8dIoZWds= +github.com/libp2p/go-libp2p v0.3.0/go.mod h1:J7DPB1+zB5VLc8v/kKSD8+u2cbyIGI0Dh/Pf3Wprt+0= github.com/libp2p/go-libp2p v0.4.2 h1:p0cthB0jDNHO4gH2HzS8/nAMMXbfUlFHs0jwZ4U+F2g= github.com/libp2p/go-libp2p v0.4.2/go.mod h1:MNmgUxUw5pMsdOzMlT0EE7oKjRasl+WyVwM0IBlpKgQ= github.com/libp2p/go-libp2p-autonat v0.0.6/go.mod h1:uZneLdOkZHro35xIhpbtTzLlgYturpu4J5+0cZK3MqE= @@ -336,10 +395,12 @@ github.com/libp2p/go-libp2p-autonat v0.1.1 h1:WLBZcIRsjZlWdAZj9CiBSvU2wQXoUOiS1Z github.com/libp2p/go-libp2p-autonat v0.1.1/go.mod h1:OXqkeGOY2xJVWKAGV2inNF5aKN/djNA3fdpCWloIudE= github.com/libp2p/go-libp2p-blankhost v0.0.1/go.mod h1:Ibpbw/7cPPYwFb7PACIWdvxxv0t0XCCI10t7czjAjTc= github.com/libp2p/go-libp2p-blankhost v0.1.1/go.mod h1:pf2fvdLJPsC1FsVrNP3DUUvMzUts2dsLLBEpo1vW1ro= +github.com/libp2p/go-libp2p-blankhost v0.1.3/go.mod h1:KML1//wiKR8vuuJO0y3LUd1uLv+tlkGTAr3jC0S5cLg= github.com/libp2p/go-libp2p-blankhost v0.1.4 h1:I96SWjR4rK9irDHcHq3XHN6hawCRTPUADzkJacgZLvk= github.com/libp2p/go-libp2p-blankhost v0.1.4/go.mod h1:oJF0saYsAXQCSfDq254GMNmLNz6ZTHTOvtF4ZydUvwU= github.com/libp2p/go-libp2p-circuit v0.0.9/go.mod h1:uU+IBvEQzCu953/ps7bYzC/D/R0Ho2A9LfKVVCatlqU= github.com/libp2p/go-libp2p-circuit v0.1.0/go.mod h1:Ahq4cY3V9VJcHcn1SBXjr78AbFkZeIRmfunbA7pmFh8= +github.com/libp2p/go-libp2p-circuit v0.1.1/go.mod h1:Ahq4cY3V9VJcHcn1SBXjr78AbFkZeIRmfunbA7pmFh8= github.com/libp2p/go-libp2p-circuit v0.1.4 h1:Phzbmrg3BkVzbqd4ZZ149JxCuUWu2wZcXf/Kr6hZJj8= github.com/libp2p/go-libp2p-circuit v0.1.4/go.mod h1:CY67BrEjKNDhdTk8UgBX1Y/H5c3xkAcs3gnksxY7osU= github.com/libp2p/go-libp2p-connmgr v0.1.0 h1:vp0t0F0EuT3rrlTtnMnIyyzCnly7nIlRoEbhJpgp0qU= @@ -348,6 +409,8 @@ github.com/libp2p/go-libp2p-core v0.0.1/go.mod h1:g/VxnTZ/1ygHxH3dKok7Vno1VfpvGc github.com/libp2p/go-libp2p-core v0.0.2/go.mod h1:9dAcntw/n46XycV4RnlBq3BpgrmyUi9LuoTNdPrbUco= github.com/libp2p/go-libp2p-core v0.0.3/go.mod h1:j+YQMNz9WNSkNezXOsahp9kwZBKBvxLpKD316QWSJXE= github.com/libp2p/go-libp2p-core v0.0.4/go.mod h1:jyuCQP356gzfCFtRKyvAbNkyeuxb7OlyhWZ3nls5d2I= +github.com/libp2p/go-libp2p-core v0.0.6/go.mod h1:0d9xmaYAVY5qmbp/fcgxHT3ZJsLjYeYPMJAUKpaCHrE= +github.com/libp2p/go-libp2p-core v0.0.9/go.mod h1:0d9xmaYAVY5qmbp/fcgxHT3ZJsLjYeYPMJAUKpaCHrE= github.com/libp2p/go-libp2p-core v0.2.0/go.mod h1:X0eyB0Gy93v0DZtSYbEM7RnMChm9Uv3j7yRXjO77xSI= github.com/libp2p/go-libp2p-core v0.2.2/go.mod h1:8fcwTbsG2B+lTgRJ1ICZtiM5GWCWZVoVrLaDRvIRng0= github.com/libp2p/go-libp2p-core v0.2.4 h1:Et6ykkTwI6PU44tr8qUF9k43vP0aduMNniShAbUJJw8= @@ -393,11 +456,11 @@ github.com/libp2p/go-libp2p-peer v0.2.0/go.mod h1:RCffaCvUyW2CJmG2gAWVqwePwW7JMg github.com/libp2p/go-libp2p-peerstore v0.0.1/go.mod h1:RabLyPVJLuNQ+GFyoEkfi8H4Ti6k/HtZJ7YKgtSq+20= github.com/libp2p/go-libp2p-peerstore v0.0.6/go.mod h1:RabLyPVJLuNQ+GFyoEkfi8H4Ti6k/HtZJ7YKgtSq+20= github.com/libp2p/go-libp2p-peerstore v0.1.0/go.mod h1:2CeHkQsr8svp4fZ+Oi9ykN1HBb6u0MOvdJ7YIsmcwtY= +github.com/libp2p/go-libp2p-peerstore v0.1.2/go.mod h1:BJ9sHlm59/80oSkpWgr1MyY1ciXAXV397W6h1GH/uKI= +github.com/libp2p/go-libp2p-peerstore v0.1.3 h1:wMgajt1uM2tMiqf4M+4qWKVyyFc8SfA+84VV9glZq1M= github.com/libp2p/go-libp2p-peerstore v0.1.3/go.mod h1:BJ9sHlm59/80oSkpWgr1MyY1ciXAXV397W6h1GH/uKI= github.com/libp2p/go-libp2p-peerstore v0.1.4 h1:d23fvq5oYMJ/lkkbO4oTwBp/JP+I/1m5gZJobNXCE/k= github.com/libp2p/go-libp2p-peerstore v0.1.4/go.mod h1:+4BDbDiiKf4PzpANZDAT+knVdLxvqh7hXOujessqdzs= -github.com/libp2p/go-libp2p-pnet v0.1.0 h1:kRUES28dktfnHNIRW4Ro78F7rKBHBiw5MJpl0ikrLIA= -github.com/libp2p/go-libp2p-pnet v0.1.0/go.mod h1:ZkyZw3d0ZFOex71halXRihWf9WH/j3OevcJdTmD0lyE= github.com/libp2p/go-libp2p-protocol v0.0.1/go.mod h1:Af9n4PiruirSDjHycM1QuiMi/1VZNHYcK8cLgFJLZ4s= github.com/libp2p/go-libp2p-protocol v0.1.0/go.mod h1:KQPHpAabB57XQxGrXCNvbL6UEXfQqUgC/1adR2Xtflk= github.com/libp2p/go-libp2p-pubsub v0.2.3 h1:qJRnRnM7Z4xnHb4i6EBb3DKQXRPgtFWlKP4AmfJudLQ= @@ -415,11 +478,15 @@ github.com/libp2p/go-libp2p-routing-helpers v0.1.0 h1:BaFvpyv8TyhCN7TihawTiKuzeu github.com/libp2p/go-libp2p-routing-helpers v0.1.0/go.mod h1:oUs0h39vNwYtYXnQWOTU5BaafbedSyWCCal3gqHuoOQ= github.com/libp2p/go-libp2p-secio v0.0.3/go.mod h1:hS7HQ00MgLhRO/Wyu1bTX6ctJKhVpm+j2/S2A5UqYb0= github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8= +github.com/libp2p/go-libp2p-secio v0.1.1/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8= +github.com/libp2p/go-libp2p-secio v0.2.0 h1:ywzZBsWEEz2KNTn5RtzauEDq5RFEefPsttXYwAWqHng= github.com/libp2p/go-libp2p-secio v0.2.0/go.mod h1:2JdZepB8J5V9mBp79BmwsaPQhRPNN2NrnB2lKQcdy6g= github.com/libp2p/go-libp2p-secio v0.2.1 h1:eNWbJTdyPA7NxhP7J3c5lT97DC5d+u+IldkgCYFTPVA= github.com/libp2p/go-libp2p-secio v0.2.1/go.mod h1:cWtZpILJqkqrSkiYcDBh5lA3wbT2Q+hz3rJQq3iftD8= github.com/libp2p/go-libp2p-swarm v0.0.6/go.mod h1:s5GZvzg9xXe8sbeESuFpjt8CJPTCa8mhEusweJqyFy8= github.com/libp2p/go-libp2p-swarm v0.1.0/go.mod h1:wQVsCdjsuZoc730CgOvh5ox6K8evllckjebkdiY5ta4= +github.com/libp2p/go-libp2p-swarm v0.1.1/go.mod h1:4NVJaLwq/dr5kEq79Jo6pMin7ZFwLx73ln1FTefR91Q= +github.com/libp2p/go-libp2p-swarm v0.2.0/go.mod h1:x07b4zkMFo2EvgPV2bMTlNmdQc8i+74Jjio7xGvsTgU= github.com/libp2p/go-libp2p-swarm v0.2.2 h1:T4hUpgEs2r371PweU3DuH7EOmBIdTBCwWs+FLcgx3bQ= github.com/libp2p/go-libp2p-swarm v0.2.2/go.mod h1:fvmtQ0T1nErXym1/aa1uJEyN7JzaTNyBcHImCxRpPKU= github.com/libp2p/go-libp2p-testing v0.0.1/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= @@ -463,6 +530,7 @@ github.com/libp2p/go-reuseport v0.0.1 h1:7PhkfH73VXfPJYKQ6JwS5I/eVcoyYi9IMNGc6FW github.com/libp2p/go-reuseport v0.0.1/go.mod h1:jn6RmB1ufnQwl0Q1f+YxAj8isJgDCQzaaxIFYDhcYEA= github.com/libp2p/go-reuseport-transport v0.0.2 h1:WglMwyXyBu61CMkjCCtnmqNqnjib0GIEjMiHTwR/KN4= github.com/libp2p/go-reuseport-transport v0.0.2/go.mod h1:YkbSDrvjUVDL6b8XqriyA20obEtsW9BLkuOUyQAOCbs= +github.com/libp2p/go-stream-muxer v0.0.1 h1:Ce6e2Pyu+b5MC1k3eeFtAax0pW4gc6MosYSLV05UeLw= github.com/libp2p/go-stream-muxer v0.0.1/go.mod h1:bAo8x7YkSpadMTbtTaxGVHWUQsR/l5MEaHbKaliuT14= github.com/libp2p/go-stream-muxer v0.1.0/go.mod h1:8JAVsjeRBCWwPoZeH0W1imLOcriqXJyFvB0mR4A04sQ= github.com/libp2p/go-stream-muxer-multistream v0.1.1/go.mod h1:zmGdfkQ1AzOECIAcccoL8L//laqawOsO03zX8Sa+eGw= @@ -492,6 +560,9 @@ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -528,6 +599,7 @@ github.com/multiformats/go-multiaddr v0.1.1 h1:rVAztJYMhCQ7vEFr8FvxW3mS+HF2eY/oP github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo= github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= +github.com/multiformats/go-multiaddr-dns v0.0.3/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.2.0 h1:YWJoIDwLePniH7OU5hBnDZV6SWuvJqJ0YtN6pLeH9zA= github.com/multiformats/go-multiaddr-dns v0.2.0/go.mod h1:TJ5pr5bBO7Y1B18djPuRsVkduhQH2YqYSbxWJzYGdK0= github.com/multiformats/go-multiaddr-fmt v0.0.1/go.mod h1:aBYjqL4T/7j4Qx+R73XSv/8JsgnRFlf0w2KGLCmXl3Q= @@ -541,6 +613,7 @@ github.com/multiformats/go-multibase v0.0.1 h1:PN9/v21eLywrFWdFNsFKaU04kLJzuYzmr github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= +github.com/multiformats/go-multihash v0.0.6/go.mod h1:XuKXPp8VHcTygube3OWZC+aZrA+H1IhmjoCDtJc7PXM= github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.0.9/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.0.10 h1:lMoNbh2Ssd9PUF74Nz008KGzGPlfeV6wH3rit5IIGCM= @@ -596,9 +669,14 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM= github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= @@ -607,6 +685,7 @@ github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHei github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2RVY1rIf+2J2o/IM9+vPq9RzmHDSseB7FoXiSNIUsoU= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v0.0.0-20190710185942-9d28bd7c0945/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8= github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY= @@ -624,6 +703,7 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -631,7 +711,11 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli/v2 v2.0.0 h1:+HU9SCbu8GnEUFtIBfuUNXN39ofWViIEJIp6SURMpCg= +github.com/urfave/cli/v2 v2.0.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8= @@ -646,8 +730,11 @@ github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba/go.mod h1:CH github.com/whyrusleeping/cbor-gen v0.0.0-20190910031516-c1cbffdb01bb/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/cbor-gen v0.0.0-20190917003517-d78d67427694/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/cbor-gen v0.0.0-20191116002219-891f55cd449d/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= +github.com/whyrusleeping/cbor-gen v0.0.0-20191212224538-d370462a7e8a/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/cbor-gen v0.0.0-20191216205031-b047b6acb3c0 h1:efb/4CnrubzNGqQOeHErxyQ6rIsJb7GcgeSDF7fqWeI= github.com/whyrusleeping/cbor-gen v0.0.0-20191216205031-b047b6acb3c0/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= +github.com/whyrusleeping/cbor-gen v0.0.0-20200106232624-282db0d37dbe h1:n7En1uyDtknjLRDXebWlPGJoHvwL8AkNcSQzuOoYYYQ= +github.com/whyrusleeping/cbor-gen v0.0.0-20200106232624-282db0d37dbe/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k= @@ -672,12 +759,16 @@ github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7V github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.1 h1:8dP3SGL7MPB94crU3bEPplMPe83FI4EouesJUeFHv50= go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= go.opencensus.io v0.22.2 h1:75k/FF0Q2YM8QYo07VPddOLBslDt1MZOdEslOHvmzAs= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0 h1:OI5t8sDa1Or+q8AeE+yKeB/SDYioSHAgcVljj9JIETY= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.5.1 h1:rsqfU5vBkVknbhUGbAUwQKR2H4ItV8tjJ+6kJX4cxHM= +go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/dig v1.7.0 h1:E5/L92iQTNJTjfgJF2KgU+/JpMaiuvK2DHLBj0+kSZk= go.uber.org/dig v1.7.0/go.mod h1:z+dSd2TP9Usi48jL8M3v63iSBVkiwtVyMKxMZYYauPg= go.uber.org/fx v1.9.0 h1:7OAz8ucp35AU8eydejpYG7QrbE8rLKzGhHbZlJi5LYY= @@ -685,12 +776,15 @@ go.uber.org/fx v1.9.0/go.mod h1:mFdUyAUuJ3w4jAckiKSKbldsxy1ojpAMJ+dVZg5Y0Aw= go.uber.org/goleak v0.10.0 h1:G3eWbSNIskeRqtsN/1uI5B+eP73y3JUuBsv9AZjehb4= go.uber.org/goleak v0.10.0/go.mod h1:VCZuO8V8mFPlL0F5J5GK1rtHV3DrFcQ1R8ryq7FK0aI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.4.0 h1:f3WCSC2KzAcBXGATIxAB1E2XuCpNU255wNKZ505qi3E= go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0 h1:nR6NoDBgAf67s68NhaXbsojM+2gxp3S1hWkHDl27pVU= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go4.org v0.0.0-20190218023631-ce4c26f7be8e/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= go4.org v0.0.0-20190313082347-94abd6928b1d h1:JkRdGP3zvTtTbabWSAC6n67ka30y7gOzWAah4XYJSfw= go4.org v0.0.0-20190313082347-94abd6928b1d/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= @@ -707,6 +801,8 @@ golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 h1:ULYEB3JvPRE/IfO+9uO7vKV/xzVTO7XPAwm8xbf4w2g= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -717,7 +813,10 @@ golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTk golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f h1:J5lckAjkw6qYlOZNj90mLYNTEKDvWeuc1yieZ8qUzUE= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/net v0.0.0-20180524181706-dfa909b99c79/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -730,11 +829,15 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -742,6 +845,7 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180202135801-37707fdb30a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -760,11 +864,14 @@ golang.org/x/sys v0.0.0-20190524152521-dbbf3f1254d4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190526052359-791d8a0f4d09/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191210023423-ac6580df4449 h1:gSbV7h1NRL2G1xTg/owz62CST1oJBmxy4QpMMregXVQ= golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200107162124-548cf772de50 h1:YvQ10rzcqWXLlJZ3XCUoO25savxmscf4+SC+ZqiCHhA= +golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -782,8 +889,12 @@ golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361 h1:RIIXAeV6GvDBuADKumTODatUqANFZ+5BPMnzsy4hulY= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200108195415-316d2f248479 h1:csuS+MHeEA2eWhyjQCMaPMq4z1+/PohkBSjJZHSIbOE= +golang.org/x/tools v0.0.0-20200108195415-316d2f248479/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -800,6 +911,7 @@ google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1 h1:Hz2g2wirWK7H0qIIhGIqRGTuMwTE8HEKFnDZZ7lm9NU= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= diff --git a/lib/auth/handler.go b/lib/auth/handler.go index b6112ce8c..61eb0ff03 100644 --- a/lib/auth/handler.go +++ b/lib/auth/handler.go @@ -5,7 +5,7 @@ import ( "net/http" "strings" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/apistruct" diff --git a/lib/jsonrpc/client.go b/lib/jsonrpc/client.go index 45b52c0e0..018a9f681 100644 --- a/lib/jsonrpc/client.go +++ b/lib/jsonrpc/client.go @@ -12,7 +12,7 @@ import ( "sync/atomic" "github.com/gorilla/websocket" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "go.opencensus.io/trace" "go.opencensus.io/trace/propagation" "golang.org/x/xerrors" diff --git a/lib/lotuslog/levels.go b/lib/lotuslog/levels.go index ddb70535c..4dad111d8 100644 --- a/lib/lotuslog/levels.go +++ b/lib/lotuslog/levels.go @@ -1,6 +1,6 @@ package lotuslog -import logging "github.com/ipfs/go-log" +import logging "github.com/ipfs/go-log/v2" func SetupLogLevels() { logging.SetLogLevel("*", "INFO") diff --git a/lib/tarutil/systar.go b/lib/tarutil/systar.go index 67a87d321..a94354731 100644 --- a/lib/tarutil/systar.go +++ b/lib/tarutil/systar.go @@ -8,7 +8,7 @@ import ( "os" "path/filepath" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" ) var log = logging.Logger("tarutil") diff --git a/markets/retrievaladapter/client.go b/markets/retrievaladapter/client.go new file mode 100644 index 000000000..676398014 --- /dev/null +++ b/markets/retrievaladapter/client.go @@ -0,0 +1,49 @@ +package retrievaladapter + +import ( + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-fil-markets/retrievalmarket" + retrievaltoken "github.com/filecoin-project/go-fil-markets/shared/tokenamount" + retrievaltypes "github.com/filecoin-project/go-fil-markets/shared/types" + "github.com/filecoin-project/lotus/markets/utils" + payapi "github.com/filecoin-project/lotus/node/impl/paych" + "github.com/filecoin-project/lotus/paych" +) + +type retrievalClientNode struct { + pmgr *paych.Manager + payapi payapi.PaychAPI +} + +// NewRetrievalClientNode returns a new node adapter for a retrieval client that talks to the +// Lotus Node +func NewRetrievalClientNode(pmgr *paych.Manager, payapi payapi.PaychAPI) retrievalmarket.RetrievalClientNode { + return &retrievalClientNode{pmgr: pmgr, payapi: payapi} +} + +// GetOrCreatePaymentChannel sets up a new payment channel if one does not exist +// between a client and a miner and insures the client has the given amount of funds available in the channel +func (rcn *retrievalClientNode) GetOrCreatePaymentChannel(ctx context.Context, clientAddress address.Address, minerAddress address.Address, clientFundsAvailable retrievaltoken.TokenAmount) (address.Address, error) { + paych, _, err := rcn.pmgr.GetPaych(ctx, clientAddress, minerAddress, utils.FromSharedTokenAmount(clientFundsAvailable)) + return paych, err +} + +// Allocate late creates a lane within a payment channel so that calls to +// CreatePaymentVoucher will automatically make vouchers only for the difference +// in total +func (rcn *retrievalClientNode) AllocateLane(paymentChannel address.Address) (uint64, error) { + return rcn.pmgr.AllocateLane(paymentChannel) +} + +// CreatePaymentVoucher creates a new payment voucher in the given lane for a +// given payment channel so that all the payment vouchers in the lane add up +// to the given amount (so the payment voucher will be for the difference) +func (rcn *retrievalClientNode) CreatePaymentVoucher(ctx context.Context, paymentChannel address.Address, amount retrievaltoken.TokenAmount, lane uint64) (*retrievaltypes.SignedVoucher, error) { + voucher, err := rcn.payapi.PaychVoucherCreate(ctx, paymentChannel, utils.FromSharedTokenAmount(amount), lane) + if err != nil { + return nil, err + } + return utils.ToSharedSignedVoucher(voucher) +} diff --git a/markets/retrievaladapter/provider.go b/markets/retrievaladapter/provider.go new file mode 100644 index 000000000..82c765123 --- /dev/null +++ b/markets/retrievaladapter/provider.go @@ -0,0 +1,48 @@ +package retrievaladapter + +import ( + "context" + + "github.com/ipfs/go-cid" + blockstore "github.com/ipfs/go-ipfs-blockstore" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-fil-markets/retrievalmarket" + retrievaltoken "github.com/filecoin-project/go-fil-markets/shared/tokenamount" + retrievaltypes "github.com/filecoin-project/go-fil-markets/shared/types" + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/markets/utils" + "github.com/filecoin-project/lotus/storage/sectorblocks" +) + +type retrievalProviderNode struct { + sectorBlocks *sectorblocks.SectorBlocks + full api.FullNode +} + +// NewRetrievalProviderNode returns a new node adapter for a retrieval provider that talks to the +// Lotus Node +func NewRetrievalProviderNode(sectorBlocks *sectorblocks.SectorBlocks, full api.FullNode) retrievalmarket.RetrievalProviderNode { + return &retrievalProviderNode{sectorBlocks, full} +} + +func (rpn *retrievalProviderNode) GetPieceSize(pieceCid []byte) (uint64, error) { + asCid, err := cid.Cast(pieceCid) + if err != nil { + return 0, err + } + return rpn.sectorBlocks.GetSize(asCid) +} + +func (rpn *retrievalProviderNode) SealedBlockstore(approveUnseal func() error) blockstore.Blockstore { + return rpn.sectorBlocks.SealedBlockstore(approveUnseal) +} + +func (rpn *retrievalProviderNode) SavePaymentVoucher(ctx context.Context, paymentChannel address.Address, voucher *retrievaltypes.SignedVoucher, proof []byte, expectedAmount retrievaltoken.TokenAmount) (retrievaltoken.TokenAmount, error) { + localVoucher, err := utils.FromSharedSignedVoucher(voucher) + if err != nil { + return retrievaltoken.FromInt(0), err + } + added, err := rpn.full.PaychVoucherAdd(ctx, paymentChannel, localVoucher, proof, utils.FromSharedTokenAmount(expectedAmount)) + return utils.ToSharedTokenAmount(added), err +} diff --git a/markets/storageadapter/client.go b/markets/storageadapter/client.go new file mode 100644 index 000000000..519e12e57 --- /dev/null +++ b/markets/storageadapter/client.go @@ -0,0 +1,341 @@ +package storageadapter + +// this file implements storagemarket.StorageClientNode + +import ( + "bytes" + "context" + + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + cborutil "github.com/filecoin-project/go-cbor-util" + "github.com/filecoin-project/go-fil-markets/shared/tokenamount" + sharedtypes "github.com/filecoin-project/go-fil-markets/shared/types" + "github.com/filecoin-project/go-fil-markets/storagemarket" + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/events" + "github.com/filecoin-project/lotus/chain/market" + "github.com/filecoin-project/lotus/chain/stmgr" + "github.com/filecoin-project/lotus/chain/store" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/markets/utils" + "github.com/filecoin-project/lotus/node/impl/full" +) + +type ClientNodeAdapter struct { + full.StateAPI + full.ChainAPI + full.MpoolAPI + + sm *stmgr.StateManager + cs *store.ChainStore + fm *market.FundMgr + ev *events.Events +} + +type clientApi struct { + full.ChainAPI + full.StateAPI +} + +func NewClientNodeAdapter(state full.StateAPI, chain full.ChainAPI, mpool full.MpoolAPI, sm *stmgr.StateManager, cs *store.ChainStore, fm *market.FundMgr) storagemarket.StorageClientNode { + return &ClientNodeAdapter{ + StateAPI: state, + ChainAPI: chain, + MpoolAPI: mpool, + + sm: sm, + cs: cs, + fm: fm, + ev: events.NewEvents(context.TODO(), &clientApi{chain, state}), + } +} + +func (n *ClientNodeAdapter) ListStorageProviders(ctx context.Context) ([]*storagemarket.StorageProviderInfo, error) { + ts, err := n.ChainHead(ctx) + if err != nil { + return nil, err + } + + addresses, err := n.StateListMiners(ctx, ts) + if err != nil { + return nil, err + } + + var out []*storagemarket.StorageProviderInfo + + for _, addr := range addresses { + workerAddr, err := n.StateMinerWorker(ctx, addr, ts) + if err != nil { + return nil, err + } + + sectorSize, err := n.StateMinerSectorSize(ctx, addr, ts) + if err != nil { + return nil, err + } + + peerId, err := n.StateMinerPeerID(ctx, addr, ts) + if err != nil { + return nil, err + } + storageProviderInfo := utils.NewStorageProviderInfo(addr, workerAddr, sectorSize, peerId) + out = append(out, &storageProviderInfo) + } + + return out, nil +} + +func (n *ClientNodeAdapter) ListClientDeals(ctx context.Context, addr address.Address) ([]storagemarket.StorageDeal, error) { + allDeals, err := n.StateMarketDeals(ctx, nil) + if err != nil { + return nil, err + } + + var out []storagemarket.StorageDeal + + for _, deal := range allDeals { + storageDeal := utils.FromOnChainDeal(deal) + if storageDeal.Client == addr { + out = append(out, storageDeal) + } + } + + return out, nil +} + +func (n *ClientNodeAdapter) MostRecentStateId(ctx context.Context) (storagemarket.StateKey, error) { + return n.ChainHead(ctx) +} + +// Adds funds with the StorageMinerActor for a storage participant. Used by both providers and clients. +func (n *ClientNodeAdapter) AddFunds(ctx context.Context, addr address.Address, amount tokenamount.TokenAmount) error { + // (Provider Node API) + smsg, err := n.MpoolPushMessage(ctx, &types.Message{ + To: actors.StorageMarketAddress, + From: addr, + Value: utils.FromSharedTokenAmount(amount), + GasPrice: types.NewInt(0), + GasLimit: types.NewInt(1000000), + Method: actors.SMAMethods.AddBalance, + }) + if err != nil { + return err + } + + r, err := n.StateWaitMsg(ctx, smsg.Cid()) + if err != nil { + return err + } + + if r.Receipt.ExitCode != 0 { + return xerrors.Errorf("adding funds to storage miner market actor failed: exit %d", r.Receipt.ExitCode) + } + + return nil +} + +func (n *ClientNodeAdapter) EnsureFunds(ctx context.Context, addr address.Address, amount tokenamount.TokenAmount) error { + return n.fm.EnsureAvailable(ctx, addr, utils.FromSharedTokenAmount(amount)) +} + +func (n *ClientNodeAdapter) GetBalance(ctx context.Context, addr address.Address) (storagemarket.Balance, error) { + bal, err := n.StateMarketBalance(ctx, addr, nil) + if err != nil { + return storagemarket.Balance{}, err + } + + return utils.ToSharedBalance(bal), nil +} + +// ValidatePublishedDeal validates that the provided deal has appeared on chain and references the same ClientDeal +// returns the Deal id if there is no error +func (c *ClientNodeAdapter) ValidatePublishedDeal(ctx context.Context, deal storagemarket.ClientDeal) (uint64, error) { + log.Infow("DEAL ACCEPTED!") + + pubmsg, err := c.cs.GetMessage(*deal.PublishMessage) + if err != nil { + return 0, xerrors.Errorf("getting deal pubsish message: %w", err) + } + + pw, err := stmgr.GetMinerWorker(ctx, c.sm, nil, deal.Proposal.Provider) + if err != nil { + return 0, xerrors.Errorf("getting miner worker failed: %w", err) + } + + if pubmsg.From != pw { + return 0, xerrors.Errorf("deal wasn't published by storage provider: from=%s, provider=%s", pubmsg.From, deal.Proposal.Provider) + } + + if pubmsg.To != actors.StorageMarketAddress { + return 0, xerrors.Errorf("deal publish message wasn't set to StorageMarket actor (to=%s)", pubmsg.To) + } + + if pubmsg.Method != actors.SMAMethods.PublishStorageDeals { + return 0, xerrors.Errorf("deal publish message called incorrect method (method=%s)", pubmsg.Method) + } + + var params actors.PublishStorageDealsParams + if err := params.UnmarshalCBOR(bytes.NewReader(pubmsg.Params)); err != nil { + return 0, err + } + + dealIdx := -1 + for i, storageDeal := range params.Deals { + // TODO: make it less hacky + sd := storageDeal + eq, err := cborutil.Equals(&deal.Proposal, &sd) + if err != nil { + return 0, err + } + if eq { + dealIdx = i + break + } + } + + if dealIdx == -1 { + return 0, xerrors.Errorf("deal publish didn't contain our deal (message cid: %s)", deal.PublishMessage) + } + + // TODO: timeout + _, ret, err := c.sm.WaitForMessage(ctx, *deal.PublishMessage) + if err != nil { + return 0, xerrors.Errorf("waiting for deal publish message: %w", err) + } + if ret.ExitCode != 0 { + return 0, xerrors.Errorf("deal publish failed: exit=%d", ret.ExitCode) + } + + var res actors.PublishStorageDealResponse + if err := res.UnmarshalCBOR(bytes.NewReader(ret.Return)); err != nil { + return 0, err + } + + return res.DealIDs[dealIdx], nil +} + +func (c *ClientNodeAdapter) OnDealSectorCommitted(ctx context.Context, provider address.Address, dealId uint64, cb storagemarket.DealSectorCommittedCallback) error { + checkFunc := func(ts *types.TipSet) (done bool, more bool, err error) { + sd, err := stmgr.GetStorageDeal(ctx, c.StateManager, dealId, ts) + if err != nil { + // TODO: This may be fine for some errors + return false, false, xerrors.Errorf("failed to look up deal on chain: %w", err) + } + + if sd.ActivationEpoch > 0 { + cb(nil) + return true, false, nil + } + + return false, true, nil + } + + called := func(msg *types.Message, rec *types.MessageReceipt, ts *types.TipSet, curH uint64) (more bool, err error) { + defer func() { + if err != nil { + cb(xerrors.Errorf("handling applied event: %w", err)) + } + }() + + if msg == nil { + log.Error("timed out waiting for deal activation... what now?") + return false, nil + } + + sd, err := stmgr.GetStorageDeal(ctx, c.StateManager, dealId, ts) + if err != nil { + return false, xerrors.Errorf("failed to look up deal on chain: %w", err) + } + + if sd.ActivationEpoch == 0 { + return false, xerrors.Errorf("deal wasn't active: deal=%d, parentState=%s, h=%d", dealId, ts.ParentState(), ts.Height()) + } + + log.Infof("Storage deal %d activated at epoch %d", dealId, sd.ActivationEpoch) + + cb(nil) + + return false, nil + } + + revert := func(ctx context.Context, ts *types.TipSet) error { + log.Warn("deal activation reverted; TODO: actually handle this!") + // TODO: Just go back to DealSealing? + return nil + } + + matchEvent := func(msg *types.Message) (bool, error) { + if msg.To != provider { + return false, nil + } + + if msg.Method != actors.MAMethods.ProveCommitSector { + return false, nil + } + + var params actors.SectorProveCommitInfo + if err := params.UnmarshalCBOR(bytes.NewReader(msg.Params)); err != nil { + return false, err + } + + var found bool + for _, dealID := range params.DealIDs { + if dealID == dealId { + found = true + break + } + } + + return found, nil + } + + if err := c.ev.Called(checkFunc, called, revert, 3, build.SealRandomnessLookbackLimit, matchEvent); err != nil { + return xerrors.Errorf("failed to set up called handler") + } + + return nil +} + +func (n *ClientNodeAdapter) SignProposal(ctx context.Context, signer address.Address, proposal *storagemarket.StorageDealProposal) error { + localProposal, err := utils.FromSharedStorageDealProposal(proposal) + if err != nil { + return err + } + err = api.SignWith(ctx, n.Wallet.Sign, signer, localProposal) + if err != nil { + return err + } + signature, err := utils.ToSharedSignature(localProposal.ProposerSignature) + if err != nil { + return err + } + proposal.ProposerSignature = signature + return nil +} + +func (n *ClientNodeAdapter) GetDefaultWalletAddress(ctx context.Context) (address.Address, error) { + addr, err := n.Wallet.GetDefault() + return addr, err +} + +func (n *ClientNodeAdapter) ValidateAskSignature(ask *sharedtypes.SignedStorageAsk) error { + tss := n.cs.GetHeaviestTipSet().ParentState() + + w, err := stmgr.GetMinerWorkerRaw(context.TODO(), n.StateManager, tss, ask.Ask.Miner) + if err != nil { + return xerrors.Errorf("failed to get worker for miner in ask", err) + } + + sigb, err := cborutil.Dump(ask.Ask) + if err != nil { + return xerrors.Errorf("failed to re-serialize ask") + } + + return ask.Signature.Verify(w, sigb) +} + +var _ storagemarket.StorageClientNode = &ClientNodeAdapter{} diff --git a/markets/storageadapter/provider.go b/markets/storageadapter/provider.go new file mode 100644 index 000000000..abe7f66eb --- /dev/null +++ b/markets/storageadapter/provider.go @@ -0,0 +1,209 @@ +package storageadapter + +// this file implements storagemarket.StorageProviderNode + +import ( + "bytes" + "context" + + "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log" + unixfile "github.com/ipfs/go-unixfs/file" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-fil-markets/shared/tokenamount" + sharedtypes "github.com/filecoin-project/go-fil-markets/shared/types" + "github.com/filecoin-project/go-fil-markets/storagemarket" + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/padreader" + "github.com/filecoin-project/lotus/markets/utils" + "github.com/filecoin-project/lotus/node/modules/dtypes" + "github.com/filecoin-project/lotus/storage/sectorblocks" +) + +var log = logging.Logger("provideradapter") + +type ProviderNodeAdapter struct { + api.FullNode + + // this goes away with the data transfer module + dag dtypes.StagingDAG + + secb *sectorblocks.SectorBlocks +} + +func NewProviderNodeAdapter(dag dtypes.StagingDAG, secb *sectorblocks.SectorBlocks, full api.FullNode) storagemarket.StorageProviderNode { + return &ProviderNodeAdapter{ + FullNode: full, + dag: dag, + secb: secb, + } +} + +func (n *ProviderNodeAdapter) PublishDeals(ctx context.Context, deal storagemarket.MinerDeal) (storagemarket.DealID, cid.Cid, error) { + log.Info("publishing deal") + + worker, err := n.StateMinerWorker(ctx, deal.Proposal.Provider, nil) + if err != nil { + return 0, cid.Undef, err + } + + localProposal, err := utils.FromSharedStorageDealProposal(&deal.Proposal) + if err != nil { + return 0, cid.Undef, err + } + params, err := actors.SerializeParams(&actors.PublishStorageDealsParams{ + Deals: []actors.StorageDealProposal{*localProposal}, + }) + + if err != nil { + return 0, cid.Undef, xerrors.Errorf("serializing PublishStorageDeals params failed: ", err) + } + + // TODO: We may want this to happen after fetching data + smsg, err := n.MpoolPushMessage(ctx, &types.Message{ + To: actors.StorageMarketAddress, + From: worker, + Value: types.NewInt(0), + GasPrice: types.NewInt(0), + GasLimit: types.NewInt(1000000), + Method: actors.SMAMethods.PublishStorageDeals, + Params: params, + }) + if err != nil { + return 0, cid.Undef, err + } + r, err := n.StateWaitMsg(ctx, smsg.Cid()) + if err != nil { + return 0, cid.Undef, err + } + if r.Receipt.ExitCode != 0 { + return 0, cid.Undef, xerrors.Errorf("publishing deal failed: exit %d", r.Receipt.ExitCode) + } + var resp actors.PublishStorageDealResponse + if err := resp.UnmarshalCBOR(bytes.NewReader(r.Receipt.Return)); err != nil { + return 0, cid.Undef, err + } + if len(resp.DealIDs) != 1 { + return 0, cid.Undef, xerrors.Errorf("got unexpected number of DealIDs from") + } + + return storagemarket.DealID(resp.DealIDs[0]), smsg.Cid(), nil +} + +func (n *ProviderNodeAdapter) OnDealComplete(ctx context.Context, deal storagemarket.MinerDeal, piecePath string) (uint64, error) { + root, err := n.dag.Get(ctx, deal.Ref) + if err != nil { + return 0, xerrors.Errorf("failed to get file root for deal: %s", err) + } + + // TODO: abstract this away into ReadSizeCloser + implement different modes + node, err := unixfile.NewUnixfsFile(ctx, n.dag, root) + if err != nil { + return 0, xerrors.Errorf("cannot open unixfs file: %s", err) + } + + uf, ok := node.(sectorblocks.UnixfsReader) + if !ok { + // we probably got directory, unsupported for now + return 0, xerrors.Errorf("unsupported unixfs file type") + } + + // TODO: uf.Size() is user input, not trusted + // This won't be useful / here after we migrate to putting CARs into sectors + size, err := uf.Size() + if err != nil { + return 0, xerrors.Errorf("getting unixfs file size: %w", err) + } + if padreader.PaddedSize(uint64(size)) != deal.Proposal.PieceSize { + return 0, xerrors.Errorf("deal.Proposal.PieceSize didn't match padded unixfs file size") + } + + sectorID, err := n.secb.AddUnixfsPiece(ctx, uf, deal.DealID) + if err != nil { + return 0, xerrors.Errorf("AddPiece failed: %s", err) + } + log.Warnf("New Sector: %d (deal %d)", sectorID, deal.DealID) + + return sectorID, nil +} + +func (n *ProviderNodeAdapter) ListProviderDeals(ctx context.Context, addr address.Address) ([]storagemarket.StorageDeal, error) { + allDeals, err := n.StateMarketDeals(ctx, nil) + if err != nil { + return nil, err + } + + var out []storagemarket.StorageDeal + + for _, deal := range allDeals { + sharedDeal := utils.FromOnChainDeal(deal) + if sharedDeal.Provider == addr { + out = append(out, sharedDeal) + } + } + + return out, nil +} + +func (n *ProviderNodeAdapter) GetMinerWorker(ctx context.Context, miner address.Address) (address.Address, error) { + addr, err := n.StateMinerWorker(ctx, miner, nil) + return addr, err +} + +func (n *ProviderNodeAdapter) SignBytes(ctx context.Context, signer address.Address, b []byte) (*sharedtypes.Signature, error) { + localSignature, err := n.WalletSign(ctx, signer, b) + if err != nil { + return nil, err + } + return utils.ToSharedSignature(localSignature) +} + +func (n *ProviderNodeAdapter) EnsureFunds(ctx context.Context, addr address.Address, amt tokenamount.TokenAmount) error { + return n.MarketEnsureAvailable(ctx, addr, utils.FromSharedTokenAmount(amt)) +} + +func (n *ProviderNodeAdapter) MostRecentStateId(ctx context.Context) (storagemarket.StateKey, error) { + return n.ChainHead(ctx) +} + +// Adds funds with the StorageMinerActor for a storage participant. Used by both providers and clients. +func (n *ProviderNodeAdapter) AddFunds(ctx context.Context, addr address.Address, amount tokenamount.TokenAmount) error { + // (Provider Node API) + smsg, err := n.MpoolPushMessage(ctx, &types.Message{ + To: actors.StorageMarketAddress, + From: addr, + Value: utils.FromSharedTokenAmount(amount), + GasPrice: types.NewInt(0), + GasLimit: types.NewInt(1000000), + Method: actors.SMAMethods.AddBalance, + }) + if err != nil { + return err + } + + r, err := n.StateWaitMsg(ctx, smsg.Cid()) + if err != nil { + return err + } + + if r.Receipt.ExitCode != 0 { + return xerrors.Errorf("adding funds to storage miner market actor failed: exit %d", r.Receipt.ExitCode) + } + + return nil +} + +func (n *ProviderNodeAdapter) GetBalance(ctx context.Context, addr address.Address) (storagemarket.Balance, error) { + bal, err := n.StateMarketBalance(ctx, addr, nil) + if err != nil { + return storagemarket.Balance{}, err + } + + return utils.ToSharedBalance(bal), nil +} + +var _ storagemarket.StorageProviderNode = &ProviderNodeAdapter{} diff --git a/markets/utils/converters.go b/markets/utils/converters.go new file mode 100644 index 000000000..58080896f --- /dev/null +++ b/markets/utils/converters.go @@ -0,0 +1,174 @@ +package utils + +import ( + "bytes" + + peer "github.com/libp2p/go-libp2p-peer" + + "github.com/filecoin-project/go-address" + sharedamount "github.com/filecoin-project/go-fil-markets/shared/tokenamount" + sharedtypes "github.com/filecoin-project/go-fil-markets/shared/types" + "github.com/filecoin-project/go-fil-markets/storagemarket" + "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/types" +) + +func FromSharedTokenAmount(in sharedamount.TokenAmount) types.BigInt { + return types.BigInt{Int: in.Int} +} + +func ToSharedTokenAmount(in types.BigInt) sharedamount.TokenAmount { + return sharedamount.TokenAmount{Int: in.Int} +} + +func ToSharedSignedVoucher(in *types.SignedVoucher) (*sharedtypes.SignedVoucher, error) { + var encoded bytes.Buffer + err := in.MarshalCBOR(&encoded) + if err != nil { + return nil, err + } + var out sharedtypes.SignedVoucher + err = out.UnmarshalCBOR(&encoded) + if err != nil { + return nil, err + } + return &out, nil +} + +func FromSharedSignedVoucher(in *sharedtypes.SignedVoucher) (*types.SignedVoucher, error) { + var encoded bytes.Buffer + err := in.MarshalCBOR(&encoded) + if err != nil { + return nil, err + } + var out types.SignedVoucher + err = out.UnmarshalCBOR(&encoded) + if err != nil { + return nil, err + } + return &out, nil +} + +func ToSharedSignature(in *types.Signature) (*sharedtypes.Signature, error) { + var encoded bytes.Buffer + err := in.MarshalCBOR(&encoded) + if err != nil { + return nil, err + } + var out sharedtypes.Signature + err = out.UnmarshalCBOR(&encoded) + if err != nil { + return nil, err + } + return &out, nil +} + +func FromSharedSignature(in *sharedtypes.Signature) (*types.Signature, error) { + var encoded bytes.Buffer + err := in.MarshalCBOR(&encoded) + if err != nil { + return nil, err + } + var out types.Signature + err = out.UnmarshalCBOR(&encoded) + if err != nil { + return nil, err + } + return &out, nil +} + +func ToSharedStorageAsk(in *types.SignedStorageAsk) (*sharedtypes.SignedStorageAsk, error) { + var encoded bytes.Buffer + err := in.MarshalCBOR(&encoded) + if err != nil { + return nil, err + } + var out sharedtypes.SignedStorageAsk + err = out.UnmarshalCBOR(&encoded) + if err != nil { + return nil, err + } + return &out, nil +} + +func FromSignedStorageAsk(in *sharedtypes.SignedStorageAsk) (*types.SignedStorageAsk, error) { + var encoded bytes.Buffer + err := in.MarshalCBOR(&encoded) + if err != nil { + return nil, err + } + var out types.SignedStorageAsk + err = out.UnmarshalCBOR(&encoded) + if err != nil { + return nil, err + } + return &out, nil +} + +func NewStorageProviderInfo(address address.Address, miner address.Address, sectorSize uint64, peer peer.ID) storagemarket.StorageProviderInfo { + return storagemarket.StorageProviderInfo{ + Address: address, + Worker: miner, + SectorSize: sectorSize, + PeerID: peer, + } +} + +func FromOnChainDeal(deal actors.OnChainDeal) storagemarket.StorageDeal { + return storagemarket.StorageDeal{ + PieceRef: deal.PieceRef, + PieceSize: deal.PieceSize, + Client: deal.Client, + Provider: deal.Provider, + StoragePricePerEpoch: ToSharedTokenAmount(deal.StoragePricePerEpoch), + StorageCollateral: ToSharedTokenAmount(deal.StorageCollateral), + ActivationEpoch: deal.ActivationEpoch, + } +} + +func ToOnChainDeal(deal storagemarket.StorageDeal) actors.OnChainDeal { + return actors.OnChainDeal{ + PieceRef: deal.PieceRef, + PieceSize: deal.PieceSize, + Client: deal.Client, + Provider: deal.Provider, + StoragePricePerEpoch: FromSharedTokenAmount(deal.StoragePricePerEpoch), + StorageCollateral: FromSharedTokenAmount(deal.StorageCollateral), + ActivationEpoch: deal.ActivationEpoch, + } +} + +func ToSharedBalance(balance actors.StorageParticipantBalance) storagemarket.Balance { + return storagemarket.Balance{ + Locked: ToSharedTokenAmount(balance.Locked), + Available: ToSharedTokenAmount(balance.Available), + } +} + +func ToSharedStorageDealProposal(proposal *actors.StorageDealProposal) (*storagemarket.StorageDealProposal, error) { + var encoded bytes.Buffer + err := proposal.MarshalCBOR(&encoded) + if err != nil { + return nil, err + } + var out storagemarket.StorageDealProposal + err = out.UnmarshalCBOR(&encoded) + if err != nil { + return nil, err + } + return &out, nil +} + +func FromSharedStorageDealProposal(proposal *storagemarket.StorageDealProposal) (*actors.StorageDealProposal, error) { + var encoded bytes.Buffer + err := proposal.MarshalCBOR(&encoded) + if err != nil { + return nil, err + } + var out actors.StorageDealProposal + err = out.UnmarshalCBOR(&encoded) + if err != nil { + return nil, err + } + return &out, nil +} diff --git a/miner/miner.go b/miner/miner.go index 4c6c3d2a0..d9d4880f3 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -12,13 +12,11 @@ import ( "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/types" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "go.opencensus.io/trace" "golang.org/x/xerrors" ) -const MaxMessagesPerBlock = 4000 - var log = logging.Logger("miner") type waitFunc func(ctx context.Context, baseTime uint64) error @@ -363,6 +361,11 @@ func (m *Miner) createBlock(base *MiningBase, addr address.Address, ticket *type return nil, xerrors.Errorf("message filtering failed: %w", err) } + if len(msgs) > build.BlockMessageLimit { + log.Error("selectMessages returned too many messages: ", len(msgs)) + msgs = msgs[:build.BlockMessageLimit] + } + uts := base.ts.MinTimestamp() + uint64(build.BlockDelay*(base.nullRounds+1)) nheight := base.ts.Height() + base.nullRounds + 1 @@ -383,12 +386,13 @@ func countFrom(msgs []*types.SignedMessage, from address.Address) (out int) { } func selectMessages(ctx context.Context, al actorLookup, base *MiningBase, msgs []*types.SignedMessage) ([]*types.SignedMessage, error) { - out := make([]*types.SignedMessage, 0, len(msgs)) + out := make([]*types.SignedMessage, 0, build.BlockMessageLimit) inclNonces := make(map[address.Address]uint64) inclBalances := make(map[address.Address]types.BigInt) inclCount := make(map[address.Address]int) for _, msg := range msgs { + if msg.Message.To == address.Undef { log.Warnf("message in mempool had bad 'To' address") continue @@ -426,7 +430,7 @@ func selectMessages(ctx context.Context, al actorLookup, base *MiningBase, msgs inclCount[from]++ out = append(out, msg) - if len(out) >= MaxMessagesPerBlock { + if len(out) >= build.BlockMessageLimit { break } } diff --git a/node/builder.go b/node/builder.go index ee34b3600..e2563c81e 100644 --- a/node/builder.go +++ b/node/builder.go @@ -19,10 +19,14 @@ import ( "go.uber.org/fx" "golang.org/x/xerrors" + "github.com/filecoin-project/go-fil-markets/retrievalmarket" + "github.com/filecoin-project/go-fil-markets/retrievalmarket/discovery" + "github.com/filecoin-project/go-fil-markets/storagemarket" + deals "github.com/filecoin-project/go-fil-markets/storagemarket/impl" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain" "github.com/filecoin-project/lotus/chain/blocksync" - "github.com/filecoin-project/lotus/chain/deals" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/market" "github.com/filecoin-project/lotus/chain/messagepool" @@ -30,7 +34,9 @@ import ( "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/vm" "github.com/filecoin-project/lotus/chain/wallet" + "github.com/filecoin-project/lotus/markets/storageadapter" "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node/config" "github.com/filecoin-project/lotus/node/hello" @@ -43,8 +49,6 @@ import ( "github.com/filecoin-project/lotus/node/repo" "github.com/filecoin-project/lotus/paych" "github.com/filecoin-project/lotus/peermgr" - "github.com/filecoin-project/lotus/retrieval" - "github.com/filecoin-project/lotus/retrieval/discovery" "github.com/filecoin-project/lotus/storage" "github.com/filecoin-project/lotus/storage/sectorblocks" ) @@ -56,7 +60,6 @@ type special struct{ id int } //nolint:golint var ( DefaultTransportsKey = special{0} // Libp2p option - PNetKey = special{1} // Option + multiret DiscoveryHandlerKey = special{2} // Private type AddrsFactoryKey = special{3} // Libp2p option SmuxTransportKey = special{4} // Libp2p option @@ -140,7 +143,6 @@ func libp2p() Option { Override(new(peerstore.Peerstore), pstoremem.NewPeerstore), Override(DefaultTransportsKey, lp2p.DefaultTransports), - Override(PNetKey, lp2p.PNet), Override(new(lp2p.RawHost), lp2p.Host), Override(new(host.Host), lp2p.RoutedHost), @@ -193,6 +195,8 @@ func Online() Option { Override(HandleIncomingMessagesKey, modules.HandleIncomingMessages), + Override(new(sectorbuilder.Verifier), sectorbuilder.ProofVerifier), + Override(new(*types.VMSyscalls), vm.Syscalls), Override(new(*store.ChainStore), modules.ChainStore), Override(new(*stmgr.StateManager), stmgr.NewStateManager), Override(new(*wallet.Wallet), wallet.NewWallet), @@ -220,14 +224,15 @@ func Online() Option { Override(RunPeerMgrKey, modules.RunPeerMgr), Override(HandleIncomingBlocksKey, modules.HandleIncomingBlocks), - Override(new(*discovery.Local), discovery.NewLocal), - Override(new(discovery.PeerResolver), modules.RetrievalResolver), + Override(new(*discovery.Local), modules.NewLocalDiscovery), + Override(new(retrievalmarket.PeerResolver), modules.RetrievalResolver), - Override(new(*retrieval.Client), retrieval.NewClient), + Override(new(retrievalmarket.RetrievalClient), modules.RetrievalClient), Override(new(dtypes.ClientDealStore), modules.NewClientDealStore), Override(new(dtypes.ClientDataTransfer), modules.NewClientDAGServiceDataTransfer), - Override(new(*deals.ClientRequestValidator), deals.NewClientRequestValidator), - Override(new(*deals.Client), deals.NewClient), + Override(new(*deals.ClientRequestValidator), modules.NewClientRequestValidator), + Override(new(storagemarket.StorageClient), modules.StorageClient), + Override(new(storagemarket.StorageClientNode), storageadapter.NewClientNodeAdapter), Override(RegisterClientValidatorKey, modules.RegisterClientValidator), Override(RunDealClientKey, modules.RunDealClient), @@ -238,18 +243,20 @@ func Online() Option { // Storage miner ApplyIf(func(s *Settings) bool { return s.nodeType == repo.StorageMiner }, - Override(new(*sectorbuilder.SectorBuilder), modules.SectorBuilder), + Override(new(sectorbuilder.Interface), modules.SectorBuilder), Override(new(*sectorblocks.SectorBlocks), sectorblocks.NewSectorBlocks), Override(new(storage.TicketFn), modules.SealTicketGen), Override(new(*storage.Miner), modules.StorageMiner), + Override(new(dtypes.StagingBlockstore), modules.StagingBlockstore), Override(new(dtypes.StagingDAG), modules.StagingDAG), - - Override(new(*retrieval.Miner), retrieval.NewMiner), + Override(new(dtypes.StagingGraphsync), modules.StagingGraphsync), + Override(new(retrievalmarket.RetrievalProvider), modules.RetrievalProvider), Override(new(dtypes.ProviderDealStore), modules.NewProviderDealStore), Override(new(dtypes.ProviderDataTransfer), modules.NewProviderDAGServiceDataTransfer), - Override(new(*deals.ProviderRequestValidator), deals.NewProviderRequestValidator), - Override(new(*deals.Provider), deals.NewProvider), + Override(new(*deals.ProviderRequestValidator), modules.NewProviderRequestValidator), + Override(new(storagemarket.StorageProvider), modules.StorageProvider), + Override(new(storagemarket.StorageProviderNode), storageadapter.NewProviderNodeAdapter), Override(RegisterProviderValidatorKey, modules.RegisterProviderValidator), Override(HandleRetrievalKey, modules.HandleRetrieval), Override(GetParamsKey, modules.GetParams), @@ -372,6 +379,7 @@ func Repo(r repo.Repo) Option { Override(new(dtypes.ClientFilestore), modules.ClientFstore), Override(new(dtypes.ClientBlockstore), modules.ClientBlockstore), Override(new(dtypes.ClientDAG), modules.ClientDAG), + Override(new(dtypes.ClientGraphsync), modules.ClientGraphsync), Override(new(ci.PrivKey), lp2p.PrivKey), Override(new(ci.PubKey), ci.PrivKey.GetPublic), diff --git a/node/fxlog.go b/node/fxlog.go index 3f0e18d64..d5d1a3e1f 100644 --- a/node/fxlog.go +++ b/node/fxlog.go @@ -1,7 +1,7 @@ package node import ( - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "go.uber.org/fx" ) diff --git a/node/hello/hello.go b/node/hello/hello.go index d05e0a0c8..8dabd3de8 100644 --- a/node/hello/hello.go +++ b/node/hello/hello.go @@ -6,7 +6,7 @@ import ( "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p-core/host" inet "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" diff --git a/node/impl/client/client.go b/node/impl/client/client.go index a0c420e7e..5623f4ae1 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -1,6 +1,7 @@ package client import ( + "bytes" "context" "errors" "io" @@ -17,22 +18,24 @@ import ( files "github.com/ipfs/go-ipfs-files" ipld "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-merkledag" + unixfile "github.com/ipfs/go-unixfs/file" "github.com/ipfs/go-unixfs/importer/balanced" ihelper "github.com/ipfs/go-unixfs/importer/helpers" "github.com/libp2p/go-libp2p-core/peer" "go.uber.org/fx" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-fil-markets/retrievalmarket" + "github.com/filecoin-project/go-fil-markets/shared/tokenamount" + "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/deals" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/markets/utils" "github.com/filecoin-project/lotus/node/impl/full" "github.com/filecoin-project/lotus/node/impl/paych" "github.com/filecoin-project/lotus/node/modules/dtypes" - "github.com/filecoin-project/lotus/retrieval" - "github.com/filecoin-project/lotus/retrieval/discovery" ) type API struct { @@ -43,9 +46,9 @@ type API struct { full.WalletAPI paych.PaychAPI - DealClient *deals.Client - RetDiscovery discovery.PeerResolver - Retrieval *retrieval.Client + SMDealClient storagemarket.StorageClient + RetDiscovery retrievalmarket.PeerResolver + Retrieval retrievalmarket.RetrievalClient Chain *store.ChainStore LocalDAG dtypes.ClientDAG @@ -71,28 +74,26 @@ func (a *API) ClientStartDeal(ctx context.Context, data cid.Cid, addr address.Ad if err != nil { return nil, xerrors.Errorf("failed getting miner worker: %w", err) } + providerInfo := utils.NewStorageProviderInfo(miner, mw, 0, pid) + result, err := a.SMDealClient.ProposeStorageDeal( + ctx, + addr, + &providerInfo, + data, + storagemarket.Epoch(math.MaxUint64), + storagemarket.Epoch(blocksDuration), + utils.ToSharedTokenAmount(epochPrice), + tokenamount.Empty) - proposal := deals.ClientDealProposal{ - Data: data, - PricePerEpoch: epochPrice, - ProposalExpiration: math.MaxUint64, // TODO: set something reasonable - Duration: blocksDuration, - Client: addr, - ProviderAddress: miner, - MinerWorker: mw, - MinerID: pid, - } - - c, err := a.DealClient.Start(ctx, proposal) if err != nil { return nil, xerrors.Errorf("failed to start deal: %w", err) } - return &c, nil + return &result.ProposalCid, nil } func (a *API) ClientListDeals(ctx context.Context) ([]api.DealInfo, error) { - deals, err := a.DealClient.List() + deals, err := a.SMDealClient.ListInProgressDeals(ctx) if err != nil { return nil, err } @@ -107,7 +108,7 @@ func (a *API) ClientListDeals(ctx context.Context) ([]api.DealInfo, error) { PieceRef: v.Proposal.PieceRef, Size: v.Proposal.PieceSize, - PricePerEpoch: v.Proposal.StoragePricePerEpoch, + PricePerEpoch: utils.FromSharedTokenAmount(v.Proposal.StoragePricePerEpoch), Duration: v.Proposal.Duration, } } @@ -116,17 +117,18 @@ func (a *API) ClientListDeals(ctx context.Context) ([]api.DealInfo, error) { } func (a *API) ClientGetDealInfo(ctx context.Context, d cid.Cid) (*api.DealInfo, error) { - v, err := a.DealClient.GetDeal(d) + v, err := a.SMDealClient.GetInProgressDeal(ctx, d) if err != nil { return nil, err } + return &api.DealInfo{ ProposalCid: v.ProposalCid, State: v.State, Provider: v.Proposal.Provider, PieceRef: v.Proposal.PieceRef, Size: v.Proposal.PieceSize, - PricePerEpoch: v.Proposal.StoragePricePerEpoch, + PricePerEpoch: utils.FromSharedTokenAmount(v.Proposal.StoragePricePerEpoch), Duration: v.Proposal.Duration, }, nil } @@ -153,7 +155,18 @@ func (a *API) ClientFindData(ctx context.Context, root cid.Cid) ([]api.QueryOffe out := make([]api.QueryOffer, len(peers)) for k, p := range peers { - out[k] = a.Retrieval.Query(ctx, p, root) + queryResponse, err := a.Retrieval.Query(ctx, p, root.Bytes(), retrievalmarket.QueryParams{}) + if err != nil { + out[k] = api.QueryOffer{Err: err.Error(), Miner: p.Address, MinerPeerID: p.ID} + } else { + out[k] = api.QueryOffer{ + Root: root, + Size: queryResponse.Size, + MinPrice: utils.FromSharedTokenAmount(queryResponse.PieceRetrievalPrice()), + Miner: p.Address, // TODO: check + MinerPeerID: p.ID, + } + } } return out, nil @@ -263,20 +276,54 @@ func (a *API) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, path order.MinerPeerID = pid } - outFile, err := os.OpenFile(path, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0777) - if err != nil { - return err + retrievalResult := make(chan error, 1) + + unsubscribe := a.Retrieval.SubscribeToEvents(func(event retrievalmarket.ClientEvent, state retrievalmarket.ClientDealState) { + if bytes.Equal(state.PieceCID, order.Root.Bytes()) { + switch event { + case retrievalmarket.ClientEventError: + retrievalResult <- xerrors.New("Retrieval Error") + case retrievalmarket.ClientEventComplete: + retrievalResult <- nil + } + } + }) + + a.Retrieval.Retrieve( + ctx, + order.Root.Bytes(), + retrievalmarket.NewParamsV0(types.BigDiv(order.Total, types.NewInt(order.Size)).Int, 0, 0), + utils.ToSharedTokenAmount(order.Total), + order.MinerPeerID, + order.Client, + order.Miner) + select { + case <-ctx.Done(): + return xerrors.New("Retrieval Timed Out") + case err := <-retrievalResult: + if err != nil { + return xerrors.Errorf("RetrieveUnixfs: %w", err) + } } - err = a.Retrieval.RetrieveUnixfs(ctx, order.Root, order.Size, order.Total, order.MinerPeerID, order.Client, order.Miner, outFile) - if err != nil { - _ = outFile.Close() - return xerrors.Errorf("RetrieveUnixfs: %w", err) - } + unsubscribe() - return outFile.Close() + nd, err := a.LocalDAG.Get(ctx, order.Root) + if err != nil { + return xerrors.Errorf("ClientRetrieve: %w", err) + } + file, err := unixfile.NewUnixfsFile(ctx, a.LocalDAG, nd) + if err != nil { + return xerrors.Errorf("ClientRetrieve: %w", err) + } + return files.WriteTo(file, path) } func (a *API) ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*types.SignedStorageAsk, error) { - return a.DealClient.QueryAsk(ctx, p, miner) + info := utils.NewStorageProviderInfo(miner, address.Undef, 0, p) + signedAsk, err := a.SMDealClient.GetAsk(ctx, info) + if err != nil { + return nil, err + } + return utils.FromSignedStorageAsk(signedAsk) } diff --git a/node/impl/full.go b/node/impl/full.go index 2c01674fb..8e7456604 100644 --- a/node/impl/full.go +++ b/node/impl/full.go @@ -1,7 +1,7 @@ package impl import ( - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "github.com/filecoin-project/lotus/node/impl/client" "github.com/filecoin-project/lotus/node/impl/market" diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 5568ed529..9bd5b4c73 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -3,12 +3,13 @@ package impl import ( "context" "encoding/json" - "github.com/filecoin-project/lotus/api/apistruct" "io" "mime" "net/http" "os" + "github.com/filecoin-project/lotus/api/apistruct" + "github.com/gorilla/mux" files "github.com/ipfs/go-ipfs-files" @@ -25,7 +26,7 @@ type StorageMinerAPI struct { CommonAPI SectorBuilderConfig *sectorbuilder.Config - SectorBuilder *sectorbuilder.SectorBuilder + SectorBuilder sectorbuilder.Interface SectorBlocks *sectorblocks.SectorBlocks Miner *storage.Miner diff --git a/node/modules/chain.go b/node/modules/chain.go index bc0ff3ce2..b70cb6892 100644 --- a/node/modules/chain.go +++ b/node/modules/chain.go @@ -73,8 +73,8 @@ func ChainBlockservice(bs dtypes.ChainBlockstore, rem dtypes.ChainExchange) dtyp return blockservice.New(bs, rem) } -func ChainStore(lc fx.Lifecycle, bs dtypes.ChainBlockstore, ds dtypes.MetadataDS) *store.ChainStore { - chain := store.NewChainStore(bs, ds) +func ChainStore(lc fx.Lifecycle, bs dtypes.ChainBlockstore, ds dtypes.MetadataDS, syscalls *types.VMSyscalls) *store.ChainStore { + chain := store.NewChainStore(bs, ds, syscalls) if err := chain.Load(); err != nil { log.Warnf("loading chain state from disk: %s", err) diff --git a/node/modules/client.go b/node/modules/client.go index 205d39a6d..b113a5968 100644 --- a/node/modules/client.go +++ b/node/modules/client.go @@ -5,25 +5,36 @@ import ( "path/filepath" "reflect" + "github.com/filecoin-project/go-data-transfer/impl/graphsync" + "github.com/filecoin-project/go-fil-markets/retrievalmarket" + "github.com/filecoin-project/go-fil-markets/retrievalmarket/discovery" + retrievalimpl "github.com/filecoin-project/go-fil-markets/retrievalmarket/impl" + "github.com/filecoin-project/go-fil-markets/storagemarket" + deals "github.com/filecoin-project/go-fil-markets/storagemarket/impl" + storageimpl "github.com/filecoin-project/go-fil-markets/storagemarket/impl" "github.com/filecoin-project/go-statestore" - "github.com/filecoin-project/lotus/node/modules/helpers" "github.com/ipfs/go-bitswap" "github.com/ipfs/go-bitswap/network" - "github.com/libp2p/go-libp2p-core/host" - "github.com/libp2p/go-libp2p-core/routing" - "github.com/ipfs/go-blockservice" "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" "github.com/ipfs/go-filestore" + graphsync "github.com/ipfs/go-graphsync/impl" + "github.com/ipfs/go-graphsync/ipldbridge" + gsnet "github.com/ipfs/go-graphsync/network" + "github.com/ipfs/go-graphsync/storeutil" blockstore "github.com/ipfs/go-ipfs-blockstore" "github.com/ipfs/go-merkledag" + "github.com/libp2p/go-libp2p-core/host" + "github.com/libp2p/go-libp2p-core/routing" "go.uber.org/fx" - "github.com/filecoin-project/lotus/chain/deals" - "github.com/filecoin-project/lotus/datatransfer" + "github.com/filecoin-project/lotus/markets/retrievaladapter" + payapi "github.com/filecoin-project/lotus/node/impl/paych" "github.com/filecoin-project/lotus/node/modules/dtypes" + "github.com/filecoin-project/lotus/node/modules/helpers" "github.com/filecoin-project/lotus/node/repo" + "github.com/filecoin-project/lotus/paych" ) func ClientFstore(r repo.LockedRepo) (dtypes.ClientFilestore, error) { @@ -49,15 +60,15 @@ func ClientBlockstore(fstore dtypes.ClientFilestore) dtypes.ClientBlockstore { // request validator with the data transfer module as the validator for // StorageDataTransferVoucher types func RegisterClientValidator(crv *deals.ClientRequestValidator, dtm dtypes.ClientDataTransfer) { - if err := dtm.RegisterVoucherType(reflect.TypeOf(deals.StorageDataTransferVoucher{}), crv); err != nil { + if err := dtm.RegisterVoucherType(reflect.TypeOf(&deals.StorageDataTransferVoucher{}), crv); err != nil { panic(err) } } // NewClientDAGServiceDataTransfer returns a data transfer manager that just // uses the clients's Client DAG service for transfers -func NewClientDAGServiceDataTransfer(dag dtypes.ClientDAG) dtypes.ClientDataTransfer { - return datatransfer.NewDAGServiceDataTransfer(dag) +func NewClientDAGServiceDataTransfer(h host.Host, gs dtypes.ClientGraphsync) dtypes.ClientDataTransfer { + return graphsyncimpl.NewGraphSyncDataTransfer(h, gs) } // NewClientDealStore creates a statestore for the client to store its deals @@ -65,6 +76,7 @@ func NewClientDealStore(ds dtypes.MetadataDS) dtypes.ClientDealStore { return statestore.New(namespace.Wrap(ds, datastore.NewKey("/deals/client"))) } +// ClientDAG is a DAGService for the ClientBlockstore func ClientDAG(mctx helpers.MetricsCtx, lc fx.Lifecycle, ibs dtypes.ClientBlockstore, rt routing.Routing, h host.Host) dtypes.ClientDAG { bitswapNetwork := network.NewFromIpfsHost(h, rt) exch := bitswap.New(helpers.LifecycleCtx(mctx, lc), bitswapNetwork, ibs) @@ -80,3 +92,29 @@ func ClientDAG(mctx helpers.MetricsCtx, lc fx.Lifecycle, ibs dtypes.ClientBlocks return dag } + +// ClientGraphsync creates a graphsync instance which reads and writes blocks +// to the ClientBlockstore +func ClientGraphsync(mctx helpers.MetricsCtx, lc fx.Lifecycle, ibs dtypes.ClientBlockstore, h host.Host) dtypes.ClientGraphsync { + graphsyncNetwork := gsnet.NewFromLibp2pHost(h) + ipldBridge := ipldbridge.NewIPLDBridge() + loader := storeutil.LoaderForBlockstore(ibs) + storer := storeutil.StorerForBlockstore(ibs) + gs := graphsync.New(helpers.LifecycleCtx(mctx, lc), graphsyncNetwork, ipldBridge, loader, storer) + + return gs +} + +func NewClientRequestValidator(deals dtypes.ClientDealStore) *storageimpl.ClientRequestValidator { + return storageimpl.NewClientRequestValidator(deals) +} + +func StorageClient(h host.Host, dag dtypes.ClientDAG, dataTransfer dtypes.ClientDataTransfer, discovery *discovery.Local, deals dtypes.ClientDealStore, scn storagemarket.StorageClientNode) storagemarket.StorageClient { + return storageimpl.NewClient(h, dag, dataTransfer, discovery, deals, scn) +} + +// RetrievalClient creates a new retrieval client attached to the client blockstore +func RetrievalClient(h host.Host, bs dtypes.ClientBlockstore, pmgr *paych.Manager, payapi payapi.PaychAPI) retrievalmarket.RetrievalClient { + adapter := retrievaladapter.NewRetrievalClientNode(pmgr, payapi) + return retrievalimpl.NewClient(h, bs, adapter) +} diff --git a/node/modules/core.go b/node/modules/core.go index 186ad9936..23adc23bc 100644 --- a/node/modules/core.go +++ b/node/modules/core.go @@ -13,7 +13,7 @@ import ( "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/repo" "github.com/gbrlsnchs/jwt/v3" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p-core/peerstore" record "github.com/libp2p/go-libp2p-record" "golang.org/x/xerrors" diff --git a/node/modules/dtypes/storage.go b/node/modules/dtypes/storage.go index 32b97767d..d80f86d24 100644 --- a/node/modules/dtypes/storage.go +++ b/node/modules/dtypes/storage.go @@ -4,12 +4,13 @@ import ( bserv "github.com/ipfs/go-blockservice" "github.com/ipfs/go-datastore" "github.com/ipfs/go-filestore" + "github.com/ipfs/go-graphsync" blockstore "github.com/ipfs/go-ipfs-blockstore" exchange "github.com/ipfs/go-ipfs-exchange-interface" ipld "github.com/ipfs/go-ipld-format" + "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/go-statestore" - "github.com/filecoin-project/lotus/datatransfer" ) // MetadataDS stores metadata @@ -26,6 +27,7 @@ type ChainBlockService bserv.BlockService type ClientFilestore *filestore.Filestore type ClientBlockstore blockstore.Blockstore type ClientDAG ipld.DAGService +type ClientGraphsync graphsync.GraphExchange type ClientDealStore *statestore.StateStore // ClientDataTransfer is a data transfer manager for the client @@ -37,3 +39,5 @@ type ProviderDealStore *statestore.StateStore type ProviderDataTransfer datatransfer.Manager type StagingDAG ipld.DAGService +type StagingBlockstore blockstore.Blockstore +type StagingGraphsync graphsync.GraphExchange diff --git a/node/modules/lp2p/host.go b/node/modules/lp2p/host.go index ccb579763..6809d4a5c 100644 --- a/node/modules/lp2p/host.go +++ b/node/modules/lp2p/host.go @@ -79,6 +79,7 @@ func DHTRouting(client bool) interface{} { dhtopts.Client(client), dhtopts.Datastore(dstore), dhtopts.Validator(validator), + dhtopts.Protocols("/lotus/kad/1.0.0"), ) if err != nil { diff --git a/node/modules/lp2p/libp2p.go b/node/modules/lp2p/libp2p.go index f14368aa4..c74d05e60 100644 --- a/node/modules/lp2p/libp2p.go +++ b/node/modules/lp2p/libp2p.go @@ -8,7 +8,7 @@ import ( "github.com/filecoin-project/lotus/chain/types" "golang.org/x/xerrors" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p" connmgr "github.com/libp2p/go-libp2p-connmgr" "github.com/libp2p/go-libp2p-core/crypto" diff --git a/node/modules/lp2p/pnet.go b/node/modules/lp2p/pnet.go deleted file mode 100644 index f182be125..000000000 --- a/node/modules/lp2p/pnet.go +++ /dev/null @@ -1,63 +0,0 @@ -package lp2p - -import ( - "fmt" - "strings" - - "github.com/libp2p/go-libp2p" - pnet "github.com/libp2p/go-libp2p-pnet" -) - -var LotusKey = "/key/swarm/psk/1.0.0/\n/base16/\n20c72388e6299c7bbc1b501fdcc8abe4f89f798e9b93b2d2bc02e3c29b6a088e" - -type PNetFingerprint []byte - -func PNet() (opts Libp2pOpts, fp PNetFingerprint, err error) { - protec, err := pnet.NewProtector(strings.NewReader(LotusKey)) - if err != nil { - return opts, nil, fmt.Errorf("failed to configure private network: %s", err) - } - fp = protec.Fingerprint() - - opts.Opts = append(opts.Opts, libp2p.PrivateNetwork(protec)) - return opts, fp, nil -} - -/* -func PNetChecker(repo repo.Repo, ph host.Host, lc fx.Lifecycle) error { - // TODO: better check? - swarmkey, err := repo.SwarmKey() - if err != nil || swarmkey == nil { - return err - } - - done := make(chan struct{}) - lc.Append(fx.Hook{ - OnStart: func(_ context.Context) error { - go func() { - t := time.NewTicker(30 * time.Second) - defer t.Stop() - - <-t.C // swallow one tick - for { - select { - case <-t.C: - if len(ph.Network().Peers()) == 0 { - log.Warn("We are in private network and have no peers.") - log.Warn("This might be configuration mistake.") - } - case <-done: - return - } - } - }() - return nil - }, - OnStop: func(_ context.Context) error { - close(done) - return nil - }, - }) - return nil -} -*/ diff --git a/node/modules/services.go b/node/modules/services.go index 92b6264e8..33f2733b5 100644 --- a/node/modules/services.go +++ b/node/modules/services.go @@ -8,15 +8,17 @@ import ( pubsub "github.com/libp2p/go-libp2p-pubsub" "go.uber.org/fx" + "github.com/filecoin-project/go-fil-markets/retrievalmarket" + "github.com/filecoin-project/go-fil-markets/retrievalmarket/discovery" + "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/lotus/chain" "github.com/filecoin-project/lotus/chain/blocksync" - "github.com/filecoin-project/lotus/chain/deals" "github.com/filecoin-project/lotus/chain/messagepool" "github.com/filecoin-project/lotus/chain/sub" "github.com/filecoin-project/lotus/node/hello" + "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/helpers" "github.com/filecoin-project/lotus/peermgr" - "github.com/filecoin-project/lotus/retrieval/discovery" ) func RunHello(mctx helpers.MetricsCtx, lc fx.Lifecycle, h host.Host, svc *hello.Service) { @@ -65,7 +67,7 @@ func HandleIncomingMessages(mctx helpers.MetricsCtx, lc fx.Lifecycle, pubsub *pu go sub.HandleIncomingMessages(ctx, mpool, msgsub) } -func RunDealClient(mctx helpers.MetricsCtx, lc fx.Lifecycle, c *deals.Client) { +func RunDealClient(mctx helpers.MetricsCtx, lc fx.Lifecycle, c storagemarket.StorageClient) { ctx := helpers.LifecycleCtx(mctx, lc) lc.Append(fx.Hook{ @@ -80,6 +82,10 @@ func RunDealClient(mctx helpers.MetricsCtx, lc fx.Lifecycle, c *deals.Client) { }) } -func RetrievalResolver(l *discovery.Local) discovery.PeerResolver { +func NewLocalDiscovery(ds dtypes.MetadataDS) *discovery.Local { + return discovery.NewLocal(ds) +} + +func RetrievalResolver(l *discovery.Local) retrievalmarket.PeerResolver { return discovery.Multi(l) } diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 23fb35455..ae9780560 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -11,6 +11,10 @@ import ( "github.com/ipfs/go-blockservice" "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" + graphsync "github.com/ipfs/go-graphsync/impl" + "github.com/ipfs/go-graphsync/ipldbridge" + gsnet "github.com/ipfs/go-graphsync/network" + "github.com/ipfs/go-graphsync/storeutil" blockstore "github.com/ipfs/go-ipfs-blockstore" "github.com/ipfs/go-merkledag" "github.com/libp2p/go-libp2p-core/host" @@ -20,19 +24,25 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-address" + dtgraphsync "github.com/filecoin-project/go-data-transfer/impl/graphsync" + "github.com/filecoin-project/go-fil-markets/retrievalmarket" + retrievalimpl "github.com/filecoin-project/go-fil-markets/retrievalmarket/impl" + "github.com/filecoin-project/go-fil-markets/storagemarket" + deals "github.com/filecoin-project/go-fil-markets/storagemarket/impl" + storageimpl "github.com/filecoin-project/go-fil-markets/storagemarket/impl" "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/deals" "github.com/filecoin-project/lotus/chain/gen" - "github.com/filecoin-project/lotus/datatransfer" "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/helpers" "github.com/filecoin-project/lotus/node/repo" - "github.com/filecoin-project/lotus/retrieval" + + "github.com/filecoin-project/lotus/markets/retrievaladapter" "github.com/filecoin-project/lotus/storage" + "github.com/filecoin-project/lotus/storage/sectorblocks" ) func minerAddrFromDS(ds dtypes.MetadataDS) (address.Address, error) { @@ -45,7 +55,7 @@ func minerAddrFromDS(ds dtypes.MetadataDS) (address.Address, error) { } func GetParams(sbc *sectorbuilder.Config) error { - if err := paramfetch.GetParams(sbc.SectorSize); err != nil { + if err := paramfetch.GetParams(build.ParametersJson, sbc.SectorSize); err != nil { return xerrors.Errorf("fetching proof parameters: %w", err) } @@ -88,7 +98,7 @@ func SectorBuilderConfig(storagePath string, threads uint, noprecommit, nocommit } } -func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api api.FullNode, h host.Host, ds dtypes.MetadataDS, sb *sectorbuilder.SectorBuilder, tktFn storage.TicketFn) (*storage.Miner, error) { +func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api api.FullNode, h host.Host, ds dtypes.MetadataDS, sb sectorbuilder.Interface, tktFn storage.TicketFn) (*storage.Miner, error) { maddr, err := minerAddrFromDS(ds) if err != nil { return nil, err @@ -111,24 +121,21 @@ func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api api.FullNode, h return sm, nil } -func HandleRetrieval(host host.Host, lc fx.Lifecycle, m *retrieval.Miner) { +func HandleRetrieval(host host.Host, lc fx.Lifecycle, m retrievalmarket.RetrievalProvider) { lc.Append(fx.Hook{ OnStart: func(context.Context) error { - host.SetStreamHandler(retrieval.QueryProtocolID, m.HandleQueryStream) - host.SetStreamHandler(retrieval.ProtocolID, m.HandleDealStream) + m.Start(host) return nil }, }) } -func HandleDeals(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, h *deals.Provider) { +func HandleDeals(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, h storagemarket.StorageProvider) { ctx := helpers.LifecycleCtx(mctx, lc) lc.Append(fx.Hook{ OnStart: func(context.Context) error { - h.Run(ctx) - host.SetStreamHandler(deals.DealProtocolID, h.HandleStream) - host.SetStreamHandler(deals.AskProtocolID, h.HandleAskStream) + h.Run(ctx, host) return nil }, OnStop: func(context.Context) error { @@ -142,15 +149,15 @@ func HandleDeals(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, h *de // request validator with the data transfer module as the validator for // StorageDataTransferVoucher types func RegisterProviderValidator(mrv *deals.ProviderRequestValidator, dtm dtypes.ProviderDataTransfer) { - if err := dtm.RegisterVoucherType(reflect.TypeOf(deals.StorageDataTransferVoucher{}), mrv); err != nil { + if err := dtm.RegisterVoucherType(reflect.TypeOf(&deals.StorageDataTransferVoucher{}), mrv); err != nil { panic(err) } } // NewProviderDAGServiceDataTransfer returns a data transfer manager that just // uses the provider's Staging DAG service for transfers -func NewProviderDAGServiceDataTransfer(dag dtypes.StagingDAG) dtypes.ProviderDataTransfer { - return datatransfer.NewDAGServiceDataTransfer(dag) +func NewProviderDAGServiceDataTransfer(h host.Host, gs dtypes.StagingGraphsync) dtypes.ProviderDataTransfer { + return dtgraphsync.NewGraphSyncDataTransfer(h, gs) } // NewProviderDealStore creates a statestore for the client to store its deals @@ -158,7 +165,9 @@ func NewProviderDealStore(ds dtypes.MetadataDS) dtypes.ProviderDealStore { return statestore.New(namespace.Wrap(ds, datastore.NewKey("/deals/client"))) } -func StagingDAG(mctx helpers.MetricsCtx, lc fx.Lifecycle, r repo.LockedRepo, rt routing.Routing, h host.Host) (dtypes.StagingDAG, error) { +// StagingBlockstore creates a blockstore for staging blocks for a miner +// in a storage deal, prior to sealing +func StagingBlockstore(r repo.LockedRepo) (dtypes.StagingBlockstore, error) { stagingds, err := r.Datastore("/staging") if err != nil { return nil, err @@ -167,8 +176,14 @@ func StagingDAG(mctx helpers.MetricsCtx, lc fx.Lifecycle, r repo.LockedRepo, rt bs := blockstore.NewBlockstore(stagingds) ibs := blockstore.NewIdStore(bs) + return ibs, nil +} + +// StagingDAG is a DAGService for the StagingBlockstore +func StagingDAG(mctx helpers.MetricsCtx, lc fx.Lifecycle, ibs dtypes.StagingBlockstore, rt routing.Routing, h host.Host) (dtypes.StagingDAG, error) { + bitswapNetwork := network.NewFromIpfsHost(h, rt) - exch := bitswap.New(helpers.LifecycleCtx(mctx, lc), bitswapNetwork, bs) + exch := bitswap.New(helpers.LifecycleCtx(mctx, lc), bitswapNetwork, ibs) bsvc := blockservice.New(ibs, exch) dag := merkledag.NewDAGService(bsvc) @@ -182,6 +197,18 @@ func StagingDAG(mctx helpers.MetricsCtx, lc fx.Lifecycle, r repo.LockedRepo, rt return dag, nil } +// StagingGraphsync creates a graphsync instance which reads and writes blocks +// to the StagingBlockstore +func StagingGraphsync(mctx helpers.MetricsCtx, lc fx.Lifecycle, ibs dtypes.StagingBlockstore, h host.Host) dtypes.StagingGraphsync { + graphsyncNetwork := gsnet.NewFromLibp2pHost(h) + ipldBridge := ipldbridge.NewIPLDBridge() + loader := storeutil.LoaderForBlockstore(ibs) + storer := storeutil.StorerForBlockstore(ibs) + gs := graphsync.New(helpers.LifecycleCtx(mctx, lc), graphsyncNetwork, ipldBridge, loader, storer) + + return gs +} + func SetupBlockProducer(lc fx.Lifecycle, ds dtypes.MetadataDS, api api.FullNode, epp gen.ElectionPoStProver) (*miner.Miner, error) { minerAddr, err := minerAddrFromDS(ds) if err != nil { @@ -206,7 +233,7 @@ func SetupBlockProducer(lc fx.Lifecycle, ds dtypes.MetadataDS, api api.FullNode, } func SectorBuilder(cfg *sectorbuilder.Config, ds dtypes.MetadataDS) (*sectorbuilder.SectorBuilder, error) { - sb, err := sectorbuilder.New(cfg, ds) + sb, err := sectorbuilder.New(cfg, namespace.Wrap(ds, datastore.NewKey("/sectorbuilder"))) if err != nil { return nil, err } @@ -237,3 +264,17 @@ func SealTicketGen(api api.FullNode) storage.TicketFn { }, nil } } + +func NewProviderRequestValidator(deals dtypes.ProviderDealStore) *storageimpl.ProviderRequestValidator { + return storageimpl.NewProviderRequestValidator(deals) +} + +func StorageProvider(ds dtypes.MetadataDS, dag dtypes.StagingDAG, dataTransfer dtypes.ProviderDataTransfer, spn storagemarket.StorageProviderNode) (storagemarket.StorageProvider, error) { + return storageimpl.NewProvider(ds, dag, dataTransfer, spn) +} + +// RetrievalProvider creates a new retrieval provider attached to the provider blockstore +func RetrievalProvider(sblks *sectorblocks.SectorBlocks, full api.FullNode) retrievalmarket.RetrievalProvider { + adapter := retrievaladapter.NewRetrievalProviderNode(sblks, full) + return retrievalimpl.NewProvider(adapter) +} diff --git a/node/modules/testing/genesis.go b/node/modules/testing/genesis.go index 6d13612ab..a5cc2e68b 100644 --- a/node/modules/testing/genesis.go +++ b/node/modules/testing/genesis.go @@ -13,7 +13,7 @@ import ( "github.com/ipfs/go-car" "github.com/ipfs/go-cid" offline "github.com/ipfs/go-ipfs-exchange-offline" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "github.com/ipfs/go-merkledag" "github.com/libp2p/go-libp2p-core/peer" "github.com/mitchellh/go-homedir" @@ -30,8 +30,8 @@ import ( var glog = logging.Logger("genesis") -func MakeGenesisMem(out io.Writer, gmc *gen.GenMinerCfg) func(bs dtypes.ChainBlockstore, w *wallet.Wallet) modules.Genesis { - return func(bs dtypes.ChainBlockstore, w *wallet.Wallet) modules.Genesis { +func MakeGenesisMem(out io.Writer, gmc *gen.GenMinerCfg) func(bs dtypes.ChainBlockstore, w *wallet.Wallet, syscalls *types.VMSyscalls) modules.Genesis { + return func(bs dtypes.ChainBlockstore, w *wallet.Wallet, syscalls *types.VMSyscalls) modules.Genesis { return func() (*types.BlockHeader, error) { glog.Warn("Generating new random genesis block, note that this SHOULD NOT happen unless you are setting up new network") defk, err := w.GenerateKey(types.KTBLS) @@ -51,7 +51,7 @@ func MakeGenesisMem(out io.Writer, gmc *gen.GenMinerCfg) func(bs dtypes.ChainBlo alloc[waddr] = types.FromFil(10000) } - b, err := gen.MakeGenesisBlock(bs, alloc, gmc, 100000) + b, err := gen.MakeGenesisBlock(bs, syscalls, alloc, gmc, 100000) if err != nil { return nil, err } @@ -68,8 +68,8 @@ func MakeGenesisMem(out io.Writer, gmc *gen.GenMinerCfg) func(bs dtypes.ChainBlo } } -func MakeGenesis(outFile, presealInfo, timestamp string) func(bs dtypes.ChainBlockstore, w *wallet.Wallet) modules.Genesis { - return func(bs dtypes.ChainBlockstore, w *wallet.Wallet) modules.Genesis { +func MakeGenesis(outFile, presealInfo, timestamp string) func(bs dtypes.ChainBlockstore, w *wallet.Wallet, syscalls *types.VMSyscalls) modules.Genesis { + return func(bs dtypes.ChainBlockstore, w *wallet.Wallet, syscalls *types.VMSyscalls) modules.Genesis { return func() (*types.BlockHeader, error) { glog.Warn("Generating new random genesis block, note that this SHOULD NOT happen unless you are setting up new network") presealInfo, err := homedir.Expand(presealInfo) @@ -130,7 +130,7 @@ func MakeGenesis(outFile, presealInfo, timestamp string) func(bs dtypes.ChainBlo ts = uint64(t.Unix()) } - b, err := gen.MakeGenesisBlock(bs, addrs, gmc, ts) + b, err := gen.MakeGenesisBlock(bs, syscalls, addrs, gmc, ts) if err != nil { return nil, err } diff --git a/node/node_test.go b/node/node_test.go index f8592b889..ea92a92b6 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -8,21 +8,20 @@ import ( "net/http/httptest" "path/filepath" "testing" + "time" - "github.com/filecoin-project/lotus/build" - - "github.com/libp2p/go-libp2p-core/crypto" - + "github.com/filecoin-project/go-address" sectorbuilder "github.com/filecoin-project/go-sectorbuilder" + "github.com/filecoin-project/lotus/build" "github.com/ipfs/go-datastore" - badger "github.com/ipfs/go-ds-badger" - logging "github.com/ipfs/go-log" + "github.com/ipfs/go-datastore/namespace" + badger "github.com/ipfs/go-ds-badger2" + logging "github.com/ipfs/go-log/v2" + "github.com/libp2p/go-libp2p-core/crypto" "github.com/libp2p/go-libp2p-core/peer" mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" "github.com/stretchr/testify/require" - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/client" "github.com/filecoin-project/lotus/api/test" @@ -38,6 +37,7 @@ import ( "github.com/filecoin-project/lotus/node/modules" modtest "github.com/filecoin-project/lotus/node/modules/testing" "github.com/filecoin-project/lotus/node/repo" + "github.com/filecoin-project/lotus/storage/sbmock" ) func init() { @@ -47,7 +47,7 @@ func init() { build.MinimumMinerPower = 1024 } -func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, act address.Address, pk crypto.PrivKey, tnd test.TestNode, mn mocknet.Mocknet) test.TestStorageNode { +func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, act address.Address, pk crypto.PrivKey, tnd test.TestNode, mn mocknet.Mocknet, opts node.Option) test.TestStorageNode { r := repo.NewMemory(nil) lr, err := r.Lock(repo.StorageMiner) @@ -107,6 +107,8 @@ func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, a node.Override(new(api.FullNode), tnd), node.Override(new(*miner.Miner), miner.NewTestMiner(mineBlock, act)), + + opts, ) if err != nil { t.Fatalf("failed to construct node: %v", err) @@ -216,7 +218,7 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te genMiner := gmc.MinerAddrs[i] wa := gmc.PreSeals[genMiner.String()].Worker - storers[i] = testStorageNode(ctx, t, wa, genMiner, pk, f, mn) + storers[i] = testStorageNode(ctx, t, wa, genMiner, pk, f, mn, node.Options()) sma := storers[i].StorageMiner.(*impl.StorageMinerAPI) @@ -231,12 +233,12 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te WorkerThreads: 2, Miner: genMiner, Dir: psd, - }, mds) + }, namespace.Wrap(mds, datastore.NewKey("/sectorbuilder"))) if err != nil { t.Fatal(err) } - if err := sma.SectorBuilder.ImportFrom(osb, false); err != nil { + if err := sma.SectorBuilder.(*sectorbuilder.SectorBuilder).ImportFrom(osb, false); err != nil { t.Fatal(err) } @@ -249,6 +251,106 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te return fulls, storers } +func mockSbBuilder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.TestStorageNode) { + ctx := context.Background() + mn := mocknet.New(ctx) + + fulls := make([]test.TestNode, nFull) + storers := make([]test.TestStorageNode, len(storage)) + + pk, _, err := crypto.GenerateEd25519Key(rand.Reader) + require.NoError(t, err) + + minerPid, err := peer.IDFromPrivateKey(pk) + require.NoError(t, err) + + var genbuf bytes.Buffer + + if len(storage) > 1 { + panic("need more peer IDs") + } + // PRESEAL SECTION, TRY TO REPLACE WITH BETTER IN THE FUTURE + // TODO: would be great if there was a better way to fake the preseals + gmc := &gen.GenMinerCfg{ + PeerIDs: []peer.ID{minerPid}, // TODO: if we have more miners, need more peer IDs + PreSeals: map[string]genesis.GenesisMiner{}, + } + + var presealDirs []string + for i := 0; i < len(storage); i++ { + maddr, err := address.NewIDAddress(300 + uint64(i)) + if err != nil { + t.Fatal(err) + } + tdir, err := ioutil.TempDir("", "preseal-memgen") + if err != nil { + t.Fatal(err) + } + genm, err := sbmock.PreSeal(1024, maddr, 1) + if err != nil { + t.Fatal(err) + } + + presealDirs = append(presealDirs, tdir) + gmc.MinerAddrs = append(gmc.MinerAddrs, maddr) + gmc.PreSeals[maddr.String()] = *genm + } + + // END PRESEAL SECTION + + for i := 0; i < nFull; i++ { + var genesis node.Option + if i == 0 { + genesis = node.Override(new(modules.Genesis), modtest.MakeGenesisMem(&genbuf, gmc)) + } else { + genesis = node.Override(new(modules.Genesis), modules.LoadGenesis(genbuf.Bytes())) + } + + var err error + // TODO: Don't ignore stop + _, err = node.New(ctx, + node.FullAPI(&fulls[i].FullNode), + node.Online(), + node.Repo(repo.NewMemory(nil)), + node.MockHost(mn), + node.Test(), + + node.Override(new(sectorbuilder.Verifier), sbmock.MockVerifier), + + genesis, + ) + if err != nil { + t.Fatal(err) + } + + } + + for i, full := range storage { + // TODO: support non-bootstrap miners + if i != 0 { + t.Fatal("only one storage node supported") + } + if full != 0 { + t.Fatal("storage nodes only supported on the first full node") + } + + f := fulls[full] + + genMiner := gmc.MinerAddrs[i] + wa := gmc.PreSeals[genMiner.String()].Worker + + storers[i] = testStorageNode(ctx, t, wa, genMiner, pk, f, mn, node.Options( + node.Override(new(sectorbuilder.Interface), sbmock.NewMockSectorBuilder(5, build.SectorSizes[0])), + )) + } + + if err := mn.LinkAll(); err != nil { + t.Fatal(err) + } + + return fulls, storers +} + func TestAPI(t *testing.T) { test.TestApis(t, builder) } @@ -291,8 +393,12 @@ func TestAPIRPC(t *testing.T) { } func TestAPIDealFlow(t *testing.T) { + test.TestDealFlow(t, mockSbBuilder, 10 * time.Millisecond) +} + +func TestAPIDealFlowReal(t *testing.T) { if testing.Short() { t.Skip("skipping test in short mode") } - test.TestDealFlow(t, builder) + test.TestDealFlow(t, builder, time.Second) } diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index b41024e5d..b0d9de1ec 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -12,9 +12,9 @@ import ( "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" - badger "github.com/ipfs/go-ds-badger" + badger "github.com/ipfs/go-ds-badger2" fslock "github.com/ipfs/go-fs-lock" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "github.com/mitchellh/go-homedir" "github.com/multiformats/go-base32" "github.com/multiformats/go-multiaddr" diff --git a/paych/paych.go b/paych/paych.go index 255d6c786..28f4c0fca 100644 --- a/paych/paych.go +++ b/paych/paych.go @@ -9,7 +9,7 @@ import ( "golang.org/x/xerrors" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "go.uber.org/fx" "github.com/filecoin-project/go-address" diff --git a/peermgr/peermgr.go b/peermgr/peermgr.go index fe87e723f..3d600fcbc 100644 --- a/peermgr/peermgr.go +++ b/peermgr/peermgr.go @@ -13,7 +13,7 @@ import ( peer "github.com/libp2p/go-libp2p-core/peer" dht "github.com/libp2p/go-libp2p-kad-dht" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" ) var log = logging.Logger("peermgr") diff --git a/retrieval/cbor_gen.go b/retrieval/cbor_gen.go deleted file mode 100644 index c80ab66d6..000000000 --- a/retrieval/cbor_gen.go +++ /dev/null @@ -1,476 +0,0 @@ -package retrieval - -import ( - "fmt" - "io" - - cbg "github.com/whyrusleeping/cbor-gen" - xerrors "golang.org/x/xerrors" -) - -// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. - -var _ = xerrors.Errorf - -func (t *RetParams) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{129}); err != nil { - return err - } - - // t.Unixfs0 (retrieval.Unixfs0Offer) (struct) - if err := t.Unixfs0.MarshalCBOR(w); err != nil { - return err - } - return nil -} - -func (t *RetParams) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 1 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Unixfs0 (retrieval.Unixfs0Offer) (struct) - - { - - pb, err := br.PeekByte() - if err != nil { - return err - } - if pb == cbg.CborNull[0] { - var nbuf [1]byte - if _, err := br.Read(nbuf[:]); err != nil { - return err - } - } else { - t.Unixfs0 = new(Unixfs0Offer) - if err := t.Unixfs0.UnmarshalCBOR(br); err != nil { - return err - } - } - - } - return nil -} - -func (t *Query) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{129}); err != nil { - return err - } - - // t.Piece (cid.Cid) (struct) - - if err := cbg.WriteCid(w, t.Piece); err != nil { - return xerrors.Errorf("failed to write cid field t.Piece: %w", err) - } - - return nil -} - -func (t *Query) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 1 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Piece (cid.Cid) (struct) - - { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Piece: %w", err) - } - - t.Piece = c - - } - return nil -} - -func (t *QueryResponse) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{131}); err != nil { - return err - } - - // t.Status (retrieval.QueryResponseStatus) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Status))); err != nil { - return err - } - - // t.Size (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Size))); err != nil { - return err - } - - // t.MinPrice (types.BigInt) (struct) - if err := t.MinPrice.MarshalCBOR(w); err != nil { - return err - } - return nil -} - -func (t *QueryResponse) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 3 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Status (retrieval.QueryResponseStatus) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Status = QueryResponseStatus(extra) - // t.Size (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Size = uint64(extra) - // t.MinPrice (types.BigInt) (struct) - - { - - if err := t.MinPrice.UnmarshalCBOR(br); err != nil { - return err - } - - } - return nil -} - -func (t *Unixfs0Offer) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{130}); err != nil { - return err - } - - // t.Offset (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Offset))); err != nil { - return err - } - - // t.Size (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Size))); err != nil { - return err - } - return nil -} - -func (t *Unixfs0Offer) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 2 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Offset (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Offset = uint64(extra) - // t.Size (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Size = uint64(extra) - return nil -} - -func (t *DealProposal) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{131}); err != nil { - return err - } - - // t.Payment (api.PaymentInfo) (struct) - if err := t.Payment.MarshalCBOR(w); err != nil { - return err - } - - // t.Ref (cid.Cid) (struct) - - if err := cbg.WriteCid(w, t.Ref); err != nil { - return xerrors.Errorf("failed to write cid field t.Ref: %w", err) - } - - // t.Params (retrieval.RetParams) (struct) - if err := t.Params.MarshalCBOR(w); err != nil { - return err - } - return nil -} - -func (t *DealProposal) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 3 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Payment (api.PaymentInfo) (struct) - - { - - if err := t.Payment.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.Ref (cid.Cid) (struct) - - { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Ref: %w", err) - } - - t.Ref = c - - } - // t.Params (retrieval.RetParams) (struct) - - { - - if err := t.Params.UnmarshalCBOR(br); err != nil { - return err - } - - } - return nil -} - -func (t *DealResponse) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{130}); err != nil { - return err - } - - // t.Status (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Status))); err != nil { - return err - } - - // t.Message (string) (string) - if len(t.Message) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.Message was too long") - } - - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Message)))); err != nil { - return err - } - if _, err := w.Write([]byte(t.Message)); err != nil { - return err - } - return nil -} - -func (t *DealResponse) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 2 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Status (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Status = uint64(extra) - // t.Message (string) (string) - - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - t.Message = string(sval) - } - return nil -} - -func (t *Block) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{130}); err != nil { - return err - } - - // t.Prefix ([]uint8) (slice) - if len(t.Prefix) > cbg.ByteArrayMaxLen { - return xerrors.Errorf("Byte array in field t.Prefix was too long") - } - - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Prefix)))); err != nil { - return err - } - if _, err := w.Write(t.Prefix); err != nil { - return err - } - - // t.Data ([]uint8) (slice) - if len(t.Data) > cbg.ByteArrayMaxLen { - return xerrors.Errorf("Byte array in field t.Data was too long") - } - - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Data)))); err != nil { - return err - } - if _, err := w.Write(t.Data); err != nil { - return err - } - return nil -} - -func (t *Block) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 2 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Prefix ([]uint8) (slice) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - - if extra > cbg.ByteArrayMaxLen { - return fmt.Errorf("t.Prefix: byte array too large (%d)", extra) - } - if maj != cbg.MajByteString { - return fmt.Errorf("expected byte array") - } - t.Prefix = make([]byte, extra) - if _, err := io.ReadFull(br, t.Prefix); err != nil { - return err - } - // t.Data ([]uint8) (slice) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - - if extra > cbg.ByteArrayMaxLen { - return fmt.Errorf("t.Data: byte array too large (%d)", extra) - } - if maj != cbg.MajByteString { - return fmt.Errorf("expected byte array") - } - t.Data = make([]byte, extra) - if _, err := io.ReadFull(br, t.Data); err != nil { - return err - } - return nil -} diff --git a/retrieval/client.go b/retrieval/client.go deleted file mode 100644 index bd06c4d3b..000000000 --- a/retrieval/client.go +++ /dev/null @@ -1,272 +0,0 @@ -package retrieval - -import ( - "context" - "io" - - blocks "github.com/ipfs/go-block-format" - "github.com/ipfs/go-cid" - logging "github.com/ipfs/go-log" - "github.com/libp2p/go-libp2p-core/host" - "github.com/libp2p/go-libp2p-core/network" - "github.com/libp2p/go-libp2p-core/peer" - cbg "github.com/whyrusleeping/cbor-gen" - "golang.org/x/xerrors" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/types" - payapi "github.com/filecoin-project/lotus/node/impl/paych" - "github.com/filecoin-project/lotus/paych" - "github.com/filecoin-project/lotus/retrieval/discovery" -) - -var log = logging.Logger("retrieval") - -type Client struct { - h host.Host - - pmgr *paych.Manager - payapi payapi.PaychAPI -} - -func NewClient(h host.Host, pmgr *paych.Manager, payapi payapi.PaychAPI) *Client { - return &Client{h: h, pmgr: pmgr, payapi: payapi} -} - -func (c *Client) Query(ctx context.Context, p discovery.RetrievalPeer, data cid.Cid) api.QueryOffer { - s, err := c.h.NewStream(ctx, p.ID, QueryProtocolID) - if err != nil { - log.Warn(err) - return api.QueryOffer{Err: err.Error(), Miner: p.Address, MinerPeerID: p.ID} - } - defer s.Close() - - err = cborutil.WriteCborRPC(s, &Query{ - Piece: data, - }) - if err != nil { - log.Warn(err) - return api.QueryOffer{Err: err.Error(), Miner: p.Address, MinerPeerID: p.ID} - } - - var resp QueryResponse - if err := resp.UnmarshalCBOR(s); err != nil { - log.Warn(err) - return api.QueryOffer{Err: err.Error(), Miner: p.Address, MinerPeerID: p.ID} - } - - return api.QueryOffer{ - Root: data, - Size: resp.Size, - MinPrice: resp.MinPrice, - Miner: p.Address, // TODO: check - MinerPeerID: p.ID, - } -} - -type clientStream struct { - payapi payapi.PaychAPI - stream network.Stream - peeker cbg.BytePeeker - - root cid.Cid - size types.BigInt - offset uint64 - - paych address.Address - lane uint64 - total types.BigInt - transferred types.BigInt - - windowSize uint64 // how much we "trust" the peer - verifier BlockVerifier -} - -// C > S -// -// Offset MUST be aligned on chunking boundaries, size is rounded up to leaf size -// -// > DealProposal{Mode: Unixfs0, RootCid, Offset, Size, Payment(nil if free)} -// < Resp{Accept} -// < ..(Intermediate Block) -// < ..Blocks -// < ..(Intermediate Block) -// < ..Blocks -// > DealProposal(...) -// < ... -func (c *Client) RetrieveUnixfs(ctx context.Context, root cid.Cid, size uint64, total types.BigInt, miner peer.ID, client, minerAddr address.Address, out io.Writer) error { - s, err := c.h.NewStream(ctx, miner, ProtocolID) - if err != nil { - return xerrors.Errorf("failed to open stream to miner for retrieval query: %w", err) - } - defer s.Close() - - initialOffset := uint64(0) // TODO: Check how much data we have locally - // TODO: Support in handler - // TODO: Allow client to specify this - - paych, _, err := c.pmgr.GetPaych(ctx, client, minerAddr, total) - if err != nil { - return xerrors.Errorf("getting payment channel: %w", err) - } - lane, err := c.pmgr.AllocateLane(paych) - if err != nil { - return xerrors.Errorf("allocating payment lane: %w", err) - } - - cst := clientStream{ - payapi: c.payapi, - stream: s, - peeker: cbg.GetPeeker(s), - - root: root, - size: types.NewInt(size), - offset: initialOffset, - - paych: paych, - lane: lane, - total: total, - transferred: types.NewInt(0), - - windowSize: build.UnixfsChunkSize, - verifier: &UnixFs0Verifier{Root: root}, - } - - for cst.offset != size+initialOffset { - toFetch := cst.windowSize - if toFetch+cst.offset > size { - toFetch = size - cst.offset - } - log.Infof("Retrieve %dB @%d", toFetch, cst.offset) - - err := cst.doOneExchange(ctx, toFetch, out) - if err != nil { - return xerrors.Errorf("retrieval exchange: %w", err) - } - - cst.offset += toFetch - } - return nil -} - -func (cst *clientStream) doOneExchange(ctx context.Context, toFetch uint64, out io.Writer) error { - payAmount := types.BigDiv(types.BigMul(cst.total, types.NewInt(toFetch)), cst.size) - - payment, err := cst.setupPayment(ctx, payAmount) - if err != nil { - return xerrors.Errorf("setting up retrieval payment: %w", err) - } - - deal := &DealProposal{ - Payment: payment, - Ref: cst.root, - Params: RetParams{ - Unixfs0: &Unixfs0Offer{ - Offset: cst.offset, - Size: toFetch, - }, - }, - } - - if err := cborutil.WriteCborRPC(cst.stream, deal); err != nil { - return xerrors.Errorf("sending incremental retrieval request: %w", err) - } - - var resp DealResponse - if err := cborutil.ReadCborRPC(cst.peeker, &resp); err != nil { - return xerrors.Errorf("reading retrieval response: %w", err) - } - - if resp.Status != Accepted { - cst.windowSize = build.UnixfsChunkSize - // TODO: apply some 'penalty' to miner 'reputation' (needs to be the same in both cases) - - if resp.Status == Error { - return xerrors.Errorf("storage deal error: %s", resp.Message) - } - if resp.Status == Rejected { - return xerrors.Errorf("storage deal rejected: %s", resp.Message) - } - return xerrors.New("storage deal response had no Accepted section") - } - - log.Info("Retrieval accepted, fetching blocks") - - return cst.fetchBlocks(toFetch, out) - - // TODO: maybe increase miner window size after success -} - -func (cst *clientStream) fetchBlocks(toFetch uint64, out io.Writer) error { - blocksToFetch := (toFetch + build.UnixfsChunkSize - 1) / build.UnixfsChunkSize - - for i := uint64(0); i < blocksToFetch; { - log.Infof("block %d of %d", i+1, blocksToFetch) - - var block Block - if err := cborutil.ReadCborRPC(cst.peeker, &block); err != nil { - return xerrors.Errorf("reading fetchBlock response: %w", err) - } - - dataBlocks, err := cst.consumeBlockMessage(block, out) - if err != nil { - return xerrors.Errorf("consuming retrieved blocks: %w", err) - } - - i += dataBlocks - } - - return nil -} - -func (cst *clientStream) consumeBlockMessage(block Block, out io.Writer) (uint64, error) { - prefix, err := cid.PrefixFromBytes(block.Prefix) - if err != nil { - return 0, err - } - - cid, err := prefix.Sum(block.Data) - if err != nil { - return 0, err - } - - blk, err := blocks.NewBlockWithCid(block.Data, cid) - if err != nil { - return 0, err - } - - internal, err := cst.verifier.Verify(context.TODO(), blk, out) - if err != nil { - log.Warnf("block verify failed: %s", err) - return 0, err - } - - // TODO: Smarter out, maybe add to filestore automagically - // (Also, persist intermediate nodes) - - if internal { - return 0, nil - } - - return 1, nil -} - -func (cst *clientStream) setupPayment(ctx context.Context, toSend types.BigInt) (api.PaymentInfo, error) { - amount := types.BigAdd(cst.transferred, toSend) - - sv, err := cst.payapi.PaychVoucherCreate(ctx, cst.paych, amount, cst.lane) - if err != nil { - return api.PaymentInfo{}, err - } - - cst.transferred = amount - - return api.PaymentInfo{ - Channel: cst.paych, - ChannelMessage: nil, - Vouchers: []*types.SignedVoucher{sv}, - }, nil -} diff --git a/retrieval/discovery/discovery.go b/retrieval/discovery/discovery.go deleted file mode 100644 index 9933ad793..000000000 --- a/retrieval/discovery/discovery.go +++ /dev/null @@ -1,25 +0,0 @@ -package discovery - -import ( - "github.com/filecoin-project/go-address" - "github.com/ipfs/go-cid" - cbor "github.com/ipfs/go-ipld-cbor" - "github.com/libp2p/go-libp2p-core/peer" -) - -func init() { - cbor.RegisterCborType(RetrievalPeer{}) -} - -type RetrievalPeer struct { - Address address.Address - ID peer.ID // optional -} - -type PeerResolver interface { - GetPeers(data cid.Cid) ([]RetrievalPeer, error) // TODO: channel -} - -func Multi(r PeerResolver) PeerResolver { // TODO: actually support multiple mechanisms - return r -} diff --git a/retrieval/discovery/local.go b/retrieval/discovery/local.go deleted file mode 100644 index 1ca99db9c..000000000 --- a/retrieval/discovery/local.go +++ /dev/null @@ -1,53 +0,0 @@ -package discovery - -import ( - "github.com/ipfs/go-cid" - "github.com/ipfs/go-datastore" - "github.com/ipfs/go-datastore/namespace" - dshelp "github.com/ipfs/go-ipfs-ds-help" - cbor "github.com/ipfs/go-ipld-cbor" - logging "github.com/ipfs/go-log" - - "github.com/filecoin-project/lotus/node/modules/dtypes" -) - -var log = logging.Logger("ret-discovery") - -type Local struct { - ds datastore.Datastore -} - -func NewLocal(ds dtypes.MetadataDS) *Local { - return &Local{ds: namespace.Wrap(ds, datastore.NewKey("/deals/local"))} -} - -func (l *Local) AddPeer(cid cid.Cid, peer RetrievalPeer) error { - // TODO: allow multiple peers here - // (implement an util for tracking map[thing][]otherThing, use in sectorBlockstore too) - - log.Warn("Tracking multiple retrieval peers not implemented") - - entry, err := cbor.DumpObject(peer) - if err != nil { - return err - } - - return l.ds.Put(dshelp.CidToDsKey(cid), entry) -} - -func (l *Local) GetPeers(data cid.Cid) ([]RetrievalPeer, error) { - entry, err := l.ds.Get(dshelp.CidToDsKey(data)) - if err == datastore.ErrNotFound { - return []RetrievalPeer{}, nil - } - if err != nil { - return nil, err - } - var peer RetrievalPeer - if err := cbor.DecodeInto(entry, &peer); err != nil { - return nil, err - } - return []RetrievalPeer{peer}, nil -} - -var _ PeerResolver = &Local{} diff --git a/retrieval/miner.go b/retrieval/miner.go deleted file mode 100644 index c9b54c5ec..000000000 --- a/retrieval/miner.go +++ /dev/null @@ -1,245 +0,0 @@ -package retrieval - -import ( - "context" - "io" - - "github.com/ipfs/go-blockservice" - "github.com/ipfs/go-cid" - "github.com/ipfs/go-merkledag" - unixfile "github.com/ipfs/go-unixfs/file" - "github.com/libp2p/go-libp2p-core/network" - "golang.org/x/xerrors" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/storage/sectorblocks" -) - -type RetrMinerApi interface { - PaychVoucherAdd(context.Context, address.Address, *types.SignedVoucher, []byte, types.BigInt) (types.BigInt, error) -} - -type Miner struct { - sectorBlocks *sectorblocks.SectorBlocks - full RetrMinerApi - - pricePerByte types.BigInt - // TODO: Unseal price -} - -func NewMiner(sblks *sectorblocks.SectorBlocks, full api.FullNode) *Miner { - return &Miner{ - sectorBlocks: sblks, - full: full, - - pricePerByte: types.NewInt(2), // TODO: allow setting - } -} - -func writeErr(stream network.Stream, err error) { - log.Errorf("Retrieval deal error: %+v", err) - _ = cborutil.WriteCborRPC(stream, &DealResponse{ - Status: Error, - Message: err.Error(), - }) -} - -func (m *Miner) HandleQueryStream(stream network.Stream) { - defer stream.Close() - - var query Query - if err := cborutil.ReadCborRPC(stream, &query); err != nil { - writeErr(stream, err) - return - } - - size, err := m.sectorBlocks.GetSize(query.Piece) - if err != nil && err != sectorblocks.ErrNotFound { - log.Errorf("Retrieval query: GetRefs: %s", err) - return - } - - answer := &QueryResponse{ - Status: Unavailable, - } - if err == nil { - answer.Status = Available - - // TODO: get price, look for already unsealed ref to reduce work - answer.MinPrice = types.BigMul(types.NewInt(uint64(size)), m.pricePerByte) - answer.Size = uint64(size) // TODO: verify on intermediate - } - - if err := cborutil.WriteCborRPC(stream, answer); err != nil { - log.Errorf("Retrieval query: WriteCborRPC: %s", err) - return - } -} - -type handlerDeal struct { - m *Miner - stream network.Stream - - ufsr sectorblocks.UnixfsReader - open cid.Cid - at uint64 - size uint64 -} - -func (m *Miner) HandleDealStream(stream network.Stream) { - defer stream.Close() - - hnd := &handlerDeal{ - m: m, - - stream: stream, - } - - var err error - more := true - - for more { - more, err = hnd.handleNext() // TODO: 'more' bool - if err != nil { - writeErr(stream, err) - return - } - } - -} - -func (hnd *handlerDeal) handleNext() (bool, error) { - var deal DealProposal - if err := cborutil.ReadCborRPC(hnd.stream, &deal); err != nil { - if err == io.EOF { // client sent all deals - err = nil - } - return false, err - } - - if deal.Params.Unixfs0 == nil { - return false, xerrors.New("unknown deal type") - } - - unixfs0 := deal.Params.Unixfs0 - - if len(deal.Payment.Vouchers) != 1 { - return false, xerrors.Errorf("expected one signed voucher, got %d", len(deal.Payment.Vouchers)) - } - - expPayment := types.BigMul(hnd.m.pricePerByte, types.NewInt(deal.Params.Unixfs0.Size)) - if _, err := hnd.m.full.PaychVoucherAdd(context.TODO(), deal.Payment.Channel, deal.Payment.Vouchers[0], nil, expPayment); err != nil { - return false, xerrors.Errorf("processing retrieval payment: %w", err) - } - - // If the file isn't open (new deal stream), isn't the right file, or isn't - // at the right offset, (re)open it - if hnd.open != deal.Ref || hnd.at != unixfs0.Offset { - log.Infof("opening file for sending (open '%s') (@%d, want %d)", deal.Ref, hnd.at, unixfs0.Offset) - if err := hnd.openFile(deal); err != nil { - return false, err - } - } - - if unixfs0.Offset+unixfs0.Size > hnd.size { - return false, xerrors.Errorf("tried to read too much %d+%d > %d", unixfs0.Offset, unixfs0.Size, hnd.size) - } - - err := hnd.accept(deal) - if err != nil { - return false, err - } - return true, nil -} - -func (hnd *handlerDeal) openFile(deal DealProposal) error { - unixfs0 := deal.Params.Unixfs0 - - if unixfs0.Offset != 0 { - // TODO: Implement SeekBlock (like ReadBlock) in go-unixfs - return xerrors.New("sending merkle proofs for nonzero offset not supported yet") - } - hnd.at = unixfs0.Offset - - bstore := hnd.m.sectorBlocks.SealedBlockstore(func() error { - return nil // TODO: approve unsealing based on amount paid - }) - - ds := merkledag.NewDAGService(blockservice.New(bstore, nil)) - rootNd, err := ds.Get(context.TODO(), deal.Ref) - if err != nil { - return err - } - - fsr, err := unixfile.NewUnixfsFile(context.TODO(), ds, rootNd) - if err != nil { - return err - } - - var ok bool - hnd.ufsr, ok = fsr.(sectorblocks.UnixfsReader) - if !ok { - return xerrors.Errorf("file %s didn't implement sectorblocks.UnixfsReader", deal.Ref) - } - - isize, err := hnd.ufsr.Size() - if err != nil { - return err - } - hnd.size = uint64(isize) - - hnd.open = deal.Ref - - return nil -} - -func (hnd *handlerDeal) accept(deal DealProposal) error { - unixfs0 := deal.Params.Unixfs0 - - resp := &DealResponse{ - Status: Accepted, - } - if err := cborutil.WriteCborRPC(hnd.stream, resp); err != nil { - log.Errorf("Retrieval query: Write Accepted resp: %s", err) - return err - } - - blocksToSend := (unixfs0.Size + build.UnixfsChunkSize - 1) / build.UnixfsChunkSize - for i := uint64(0); i < blocksToSend; { - data, offset, nd, err := hnd.ufsr.ReadBlock(context.TODO()) - if err != nil { - return err - } - - log.Infof("sending block for a deal: %s", nd.Cid()) - - if offset != unixfs0.Offset { - return xerrors.Errorf("ReadBlock on wrong offset: want %d, got %d", unixfs0.Offset, offset) - } - - /*if uint64(len(data)) != deal.Unixfs0.Size { // TODO: Fix for internal nodes (and any other node too) - writeErr(stream, xerrors.Errorf("ReadBlock data with wrong size: want %d, got %d", deal.Unixfs0.Size, len(data))) - return - }*/ - - block := &Block{ - Prefix: nd.Cid().Prefix().Bytes(), - Data: nd.RawData(), - } - - if err := cborutil.WriteCborRPC(hnd.stream, block); err != nil { - return err - } - - if len(data) > 0 { // don't count internal nodes - hnd.at += uint64(len(data)) - i++ - } - } - - return nil -} diff --git a/retrieval/types.go b/retrieval/types.go deleted file mode 100644 index 28343e6b2..000000000 --- a/retrieval/types.go +++ /dev/null @@ -1,66 +0,0 @@ -package retrieval - -import ( - "github.com/filecoin-project/lotus/api" - "github.com/ipfs/go-cid" - - "github.com/filecoin-project/lotus/chain/types" -) - -const ProtocolID = "/fil/retrieval/-1.0.0" // TODO: spec -const QueryProtocolID = "/fil/retrieval/qry/-1.0.0" // TODO: spec - -type QueryResponseStatus uint64 - -const ( - Available QueryResponseStatus = iota - Unavailable -) - -const ( - Accepted = iota - Error - Rejected - Unsealing -) - -type Query struct { - Piece cid.Cid - // TODO: payment -} - -type QueryResponse struct { - Status QueryResponseStatus - - Size uint64 // TODO: spec - // TODO: unseal price (+spec) - // TODO: sectors to unseal - // TODO: address to send money for the deal? - MinPrice types.BigInt -} - -type Unixfs0Offer struct { - Offset uint64 - Size uint64 -} - -type RetParams struct { - Unixfs0 *Unixfs0Offer -} - -type DealProposal struct { - Payment api.PaymentInfo - - Ref cid.Cid - Params RetParams -} - -type DealResponse struct { - Status uint64 - Message string -} - -type Block struct { // TODO: put in spec - Prefix []byte // TODO: fix cid.Prefix marshaling somehow - Data []byte -} diff --git a/retrieval/verify.go b/retrieval/verify.go deleted file mode 100644 index 88fa0dfed..000000000 --- a/retrieval/verify.go +++ /dev/null @@ -1,143 +0,0 @@ -package retrieval - -import ( - "context" - "io" - - blocks "github.com/ipfs/go-block-format" - "github.com/ipfs/go-cid" - ipld "github.com/ipfs/go-ipld-format" - "github.com/ipfs/go-merkledag" - "github.com/ipfs/go-unixfs" - pb "github.com/ipfs/go-unixfs/pb" - "golang.org/x/xerrors" - - "github.com/filecoin-project/lotus/build" -) - -type BlockVerifier interface { - Verify(context.Context, blocks.Block, io.Writer) (internal bool, err error) -} - -type OptimisticVerifier struct { -} - -func (o *OptimisticVerifier) Verify(context.Context, blocks.Block, io.Writer) (bool, error) { - // It's probably fine - return false, nil -} - -type UnixFs0Verifier struct { - Root cid.Cid - rootBlk blocks.Block - - expect int - seen int - - sub *UnixFs0Verifier -} - -func (b *UnixFs0Verifier) verify(ctx context.Context, blk blocks.Block, out io.Writer) (last bool, internal bool, err error) { - if b.sub != nil { - // TODO: check links here (iff b.sub.sub == nil) - - subLast, internal, err := b.sub.verify(ctx, blk, out) - if err != nil { - return false, false, err - } - if subLast { - b.sub = nil - b.seen++ - } - - return b.seen == b.expect, internal, nil - } - - if b.seen >= b.expect { // this is probably impossible - return false, false, xerrors.New("unixfs verifier: too many nodes in level") - } - - links, err := b.checkInternal(blk, out) - if err != nil { - return false, false, err - } - - if links > 0 { // TODO: check if all links are intermediate (or all aren't) - if links > build.UnixfsLinksPerLevel { - return false, false, xerrors.New("unixfs verifier: too many links in intermediate node") - } - - if b.seen+1 == b.expect && links != build.UnixfsLinksPerLevel { - return false, false, xerrors.New("unixfs verifier: too few nodes in level") - } - - b.sub = &UnixFs0Verifier{ - Root: blk.Cid(), - rootBlk: blk, - expect: links, - } - - // don't mark as seen yet - return false, true, nil - } - - b.seen++ - return b.seen == b.expect, false, nil -} - -func (b *UnixFs0Verifier) checkInternal(blk blocks.Block, out io.Writer) (int, error) { - nd, err := ipld.Decode(blk) - if err != nil { - log.Warnf("IPLD Decode failed: %s", err) - return 0, err - } - - // TODO: check size - switch nd := nd.(type) { - case *merkledag.ProtoNode: - fsn, err := unixfs.FSNodeFromBytes(nd.Data()) - if err != nil { - log.Warnf("unixfs.FSNodeFromBytes failed: %s", err) - return 0, err - } - if fsn.Type() != pb.Data_File { - return 0, xerrors.New("internal nodes must be a file") - } - if len(fsn.Data()) > 0 { - return 0, xerrors.New("internal node with data") - } - if len(nd.Links()) == 0 { - return 0, xerrors.New("internal node with no links") - } - return len(nd.Links()), nil - - case *merkledag.RawNode: - _, err := out.Write(nd.RawData()) - return 0, err - default: - return 0, xerrors.New("verifier: unknown node type") - } -} - -func (b *UnixFs0Verifier) Verify(ctx context.Context, blk blocks.Block, w io.Writer) (bool, error) { - // root is special - if b.rootBlk == nil { - if !b.Root.Equals(blk.Cid()) { - return false, xerrors.Errorf("unixfs verifier: root block CID didn't match: valid %s, got %s", b.Root, blk.Cid()) - } - b.rootBlk = blk - links, err := b.checkInternal(blk, w) - if err != nil { - return false, err - } - - b.expect = links - return links != 0, nil - } - - _, internal, err := b.verify(ctx, blk, w) - return internal, err -} - -var _ BlockVerifier = &OptimisticVerifier{} -var _ BlockVerifier = &UnixFs0Verifier{} diff --git a/storage/cbor_gen.go b/storage/cbor_gen.go index 43262ad2b..d2a6b9ee3 100644 --- a/storage/cbor_gen.go +++ b/storage/cbor_gen.go @@ -22,6 +22,10 @@ func (t *SealTicket) MarshalCBOR(w io.Writer) error { } // t.BlockHeight (uint64) (uint64) + if len("BlockHeight") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"BlockHeight\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("BlockHeight")))); err != nil { return err } @@ -34,6 +38,10 @@ func (t *SealTicket) MarshalCBOR(w io.Writer) error { } // t.TicketBytes ([]uint8) (slice) + if len("TicketBytes") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"TicketBytes\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("TicketBytes")))); err != nil { return err } @@ -41,6 +49,10 @@ func (t *SealTicket) MarshalCBOR(w io.Writer) error { return err } + if len(t.TicketBytes) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.TicketBytes was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.TicketBytes)))); err != nil { return err } @@ -133,6 +145,10 @@ func (t *SealSeed) MarshalCBOR(w io.Writer) error { } // t.BlockHeight (uint64) (uint64) + if len("BlockHeight") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"BlockHeight\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("BlockHeight")))); err != nil { return err } @@ -145,6 +161,10 @@ func (t *SealSeed) MarshalCBOR(w io.Writer) error { } // t.TicketBytes ([]uint8) (slice) + if len("TicketBytes") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"TicketBytes\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("TicketBytes")))); err != nil { return err } @@ -152,6 +172,10 @@ func (t *SealSeed) MarshalCBOR(w io.Writer) error { return err } + if len(t.TicketBytes) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.TicketBytes was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.TicketBytes)))); err != nil { return err } @@ -244,6 +268,10 @@ func (t *Piece) MarshalCBOR(w io.Writer) error { } // t.DealID (uint64) (uint64) + if len("DealID") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"DealID\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("DealID")))); err != nil { return err } @@ -256,6 +284,10 @@ func (t *Piece) MarshalCBOR(w io.Writer) error { } // t.Size (uint64) (uint64) + if len("Size") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Size\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Size")))); err != nil { return err } @@ -268,6 +300,10 @@ func (t *Piece) MarshalCBOR(w io.Writer) error { } // t.CommP ([]uint8) (slice) + if len("CommP") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"CommP\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("CommP")))); err != nil { return err } @@ -275,6 +311,10 @@ func (t *Piece) MarshalCBOR(w io.Writer) error { return err } + if len(t.CommP) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.CommP was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.CommP)))); err != nil { return err } @@ -390,6 +430,10 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } // t.State (uint64) (uint64) + if len("State") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"State\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("State")))); err != nil { return err } @@ -402,6 +446,10 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } // t.SectorID (uint64) (uint64) + if len("SectorID") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"SectorID\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("SectorID")))); err != nil { return err } @@ -414,6 +462,10 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } // t.Nonce (uint64) (uint64) + if len("Nonce") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Nonce\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Nonce")))); err != nil { return err } @@ -426,6 +478,10 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } // t.Pieces ([]storage.Piece) (slice) + if len("Pieces") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Pieces\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Pieces")))); err != nil { return err } @@ -433,6 +489,10 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return err } + if len(t.Pieces) > cbg.MaxLength { + return xerrors.Errorf("Slice value in field t.Pieces was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Pieces)))); err != nil { return err } @@ -443,6 +503,10 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } // t.CommD ([]uint8) (slice) + if len("CommD") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"CommD\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("CommD")))); err != nil { return err } @@ -450,6 +514,10 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return err } + if len(t.CommD) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.CommD was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.CommD)))); err != nil { return err } @@ -458,6 +526,10 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } // t.CommR ([]uint8) (slice) + if len("CommR") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"CommR\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("CommR")))); err != nil { return err } @@ -465,6 +537,10 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return err } + if len(t.CommR) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.CommR was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.CommR)))); err != nil { return err } @@ -473,6 +549,10 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } // t.Proof ([]uint8) (slice) + if len("Proof") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Proof\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Proof")))); err != nil { return err } @@ -480,6 +560,10 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return err } + if len(t.Proof) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.Proof was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Proof)))); err != nil { return err } @@ -488,6 +572,10 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } // t.Ticket (storage.SealTicket) (struct) + if len("Ticket") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Ticket\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Ticket")))); err != nil { return err } @@ -500,6 +588,10 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } // t.PreCommitMessage (cid.Cid) (struct) + if len("PreCommitMessage") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"PreCommitMessage\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("PreCommitMessage")))); err != nil { return err } @@ -518,6 +610,10 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } // t.Seed (storage.SealSeed) (struct) + if len("Seed") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Seed\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Seed")))); err != nil { return err } @@ -530,6 +626,10 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } // t.CommitMessage (cid.Cid) (struct) + if len("CommitMessage") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"CommitMessage\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("CommitMessage")))); err != nil { return err } @@ -548,6 +648,10 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } // t.FaultReportMsg (cid.Cid) (struct) + if len("FaultReportMsg") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"FaultReportMsg\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("FaultReportMsg")))); err != nil { return err } @@ -566,6 +670,10 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } // t.LastErr (string) (string) + if len("LastErr") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"LastErr\" was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("LastErr")))); err != nil { return err } @@ -573,6 +681,10 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return err } + if len(t.LastErr) > cbg.MaxLength { + return xerrors.Errorf("Value in field t.LastErr was too long") + } + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.LastErr)))); err != nil { return err } diff --git a/storage/fpost_sched.go b/storage/fpost_sched.go index fc192ebee..cafb212b6 100644 --- a/storage/fpost_sched.go +++ b/storage/fpost_sched.go @@ -4,11 +4,12 @@ import ( "context" "sync" - sectorbuilder "github.com/filecoin-project/go-sectorbuilder" "go.opencensus.io/trace" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-sectorbuilder" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" @@ -20,7 +21,7 @@ const StartConfidence = 4 // TODO: config type fpostScheduler struct { api storageMinerApi - sb *sectorbuilder.SectorBuilder + sb sectorbuilder.Interface actor address.Address worker address.Address diff --git a/storage/garbage.go b/storage/garbage.go index d128413bd..a5470cdaa 100644 --- a/storage/garbage.go +++ b/storage/garbage.go @@ -10,7 +10,6 @@ import ( sectorbuilder "github.com/filecoin-project/go-sectorbuilder" "golang.org/x/xerrors" - "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/types" ) @@ -39,11 +38,7 @@ func (m *Miner) pledgeSector(ctx context.Context, sectorID uint64, existingPiece Duration: math.MaxUint64 / 2, // /2 because overflows StoragePricePerEpoch: types.NewInt(0), StorageCollateral: types.NewInt(0), - ProposerSignature: nil, - } - - if err := api.SignWith(ctx, m.api.WalletSign, m.worker, &sdp); err != nil { - return nil, xerrors.Errorf("signing storage deal failed: ", err) + ProposerSignature: nil, // nil because self dealing } deals[i] = sdp diff --git a/storage/miner.go b/storage/miner.go index 3265aa2ae..02022d03f 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -8,7 +8,7 @@ import ( "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p-core/host" "golang.org/x/xerrors" @@ -36,7 +36,7 @@ type Miner struct { worker address.Address // Sealing - sb *sectorbuilder.SectorBuilder + sb sectorbuilder.Interface sectors *statestore.StateStore tktFn TicketFn @@ -71,7 +71,7 @@ type storageMinerApi interface { WalletHas(context.Context, address.Address) (bool, error) } -func NewMiner(api storageMinerApi, addr address.Address, h host.Host, ds datastore.Batching, sb *sectorbuilder.SectorBuilder, tktFn TicketFn) (*Miner, error) { +func NewMiner(api storageMinerApi, addr address.Address, h host.Host, ds datastore.Batching, sb sectorbuilder.Interface, tktFn TicketFn) (*Miner, error) { return &Miner{ api: api, @@ -144,10 +144,10 @@ func (m *Miner) runPreflightChecks(ctx context.Context) error { } type SectorBuilderEpp struct { - sb *sectorbuilder.SectorBuilder + sb sectorbuilder.Interface } -func NewElectionPoStProver(sb *sectorbuilder.SectorBuilder) *SectorBuilderEpp { +func NewElectionPoStProver(sb sectorbuilder.Interface) *SectorBuilderEpp { return &SectorBuilderEpp{sb} } diff --git a/storage/sbmock/preseal.go b/storage/sbmock/preseal.go new file mode 100644 index 000000000..ad4fc4f63 --- /dev/null +++ b/storage/sbmock/preseal.go @@ -0,0 +1,52 @@ +package sbmock + +import ( + "math" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-sectorbuilder" + + "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/wallet" + "github.com/filecoin-project/lotus/genesis" +) + +func PreSeal(ssize uint64, maddr address.Address, sectors int) (*genesis.GenesisMiner, error) { + k, err := wallet.GenerateKey(types.KTBLS) + if err != nil { + return nil, err + } + + genm := &genesis.GenesisMiner{ + Owner: k.Address, + Worker: k.Address, + SectorSize: ssize, + Sectors: make([]*genesis.PreSeal, sectors), + Key: k.KeyInfo, + } + + for i := range genm.Sectors { + preseal := &genesis.PreSeal{} + sdata := randB(sectorbuilder.UserBytesForSectorSize(ssize)) + + preseal.CommD = commD(sdata) + preseal.CommR = commDR(preseal.CommD[:]) + preseal.SectorID = uint64(i + 1) + preseal.Deal = actors.StorageDealProposal{ + PieceRef: preseal.CommD[:], + PieceSize: sectorbuilder.UserBytesForSectorSize(ssize), + Client: maddr, + Provider: maddr, + ProposalExpiration: math.MaxUint64, + Duration: math.MaxUint64, + StoragePricePerEpoch: types.NewInt(0), + StorageCollateral: types.NewInt(0), + ProposerSignature: nil, + } + + genm.Sectors[i] = preseal + } + + return genm, nil +} diff --git a/storage/sbmock/sbmock.go b/storage/sbmock/sbmock.go new file mode 100644 index 000000000..b8ca9b69e --- /dev/null +++ b/storage/sbmock/sbmock.go @@ -0,0 +1,343 @@ +package sbmock + +import ( + "bytes" + "context" + "fmt" + "io" + "io/ioutil" + "math/big" + "math/rand" + "sync" + + ffi "github.com/filecoin-project/filecoin-ffi" + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-sectorbuilder" + "golang.org/x/xerrors" +) + +type SBMock struct { + sectors map[uint64]*sectorState + sectorSize uint64 + nextSectorID uint64 + rateLimit chan struct{} + + lk sync.Mutex +} + +type mockVerif struct{} + +func NewMockSectorBuilder(threads int, ssize uint64) *SBMock { + return &SBMock{ + sectors: make(map[uint64]*sectorState), + sectorSize: ssize, + nextSectorID: 5, + rateLimit: make(chan struct{}, threads), + } +} + +const ( + statePacking = iota + statePreCommit + stateCommit +) + +type sectorState struct { + pieces [][]byte + failed bool + + state int + + lk sync.Mutex +} + +func (sb *SBMock) RateLimit() func() { + sb.rateLimit <- struct{}{} + + // TODO: probably want to copy over rate limit code + return func() { + <-sb.rateLimit + } +} + +func (sb *SBMock) AddPiece(size uint64, sectorId uint64, r io.Reader, existingPieces []uint64) (sectorbuilder.PublicPieceInfo, error) { + sb.lk.Lock() + ss, ok := sb.sectors[sectorId] + if !ok { + ss = §orState{ + state: statePacking, + } + sb.sectors[sectorId] = ss + } + sb.lk.Unlock() + ss.lk.Lock() + defer ss.lk.Unlock() + + b, err := ioutil.ReadAll(r) + if err != nil { + return sectorbuilder.PublicPieceInfo{}, err + } + + ss.pieces = append(ss.pieces, b) + return sectorbuilder.PublicPieceInfo{ + Size: size, + CommP: commD(b), + }, nil +} + +func (sb *SBMock) SectorSize() uint64 { + return sb.sectorSize +} + +func (sb *SBMock) AcquireSectorId() (uint64, error) { + sb.lk.Lock() + defer sb.lk.Unlock() + id := sb.nextSectorID + sb.nextSectorID++ + return id, nil +} + +func (sb *SBMock) Scrub(sectorbuilder.SortedPublicSectorInfo) []*sectorbuilder.Fault { + sb.lk.Lock() + mcopy := make(map[uint64]*sectorState) + for k, v := range sb.sectors { + mcopy[k] = v + } + sb.lk.Unlock() + + var out []*sectorbuilder.Fault + for sid, ss := range mcopy { + ss.lk.Lock() + if ss.failed { + out = append(out, §orbuilder.Fault{ + SectorID: sid, + Err: fmt.Errorf("mock sector failed"), + }) + + } + ss.lk.Unlock() + } + + return out +} + +func (sb *SBMock) GenerateFallbackPoSt(sectorbuilder.SortedPublicSectorInfo, [sectorbuilder.CommLen]byte, []uint64) ([]sectorbuilder.EPostCandidate, []byte, error) { + panic("NYI") +} + +func (sb *SBMock) SealPreCommit(ctx context.Context, sid uint64, ticket sectorbuilder.SealTicket, pieces []sectorbuilder.PublicPieceInfo) (sectorbuilder.RawSealPreCommitOutput, error) { + sb.lk.Lock() + ss, ok := sb.sectors[sid] + sb.lk.Unlock() + if !ok { + return sectorbuilder.RawSealPreCommitOutput{}, xerrors.Errorf("no sector with id %d in sectorbuilder", sid) + } + + ss.lk.Lock() + defer ss.lk.Unlock() + + ussize := sectorbuilder.UserBytesForSectorSize(sb.sectorSize) + + // TODO: verify pieces in sinfo.pieces match passed in pieces + + var sum uint64 + for _, p := range pieces { + sum += p.Size + } + + if sum != ussize { + return sectorbuilder.RawSealPreCommitOutput{}, xerrors.Errorf("aggregated piece sizes don't match up: %d != %d", sum, ussize) + } + + if ss.state != statePacking { + return sectorbuilder.RawSealPreCommitOutput{}, xerrors.Errorf("cannot call pre-seal on sector not in 'packing' state") + } + + opFinishWait(ctx) + + ss.state = statePreCommit + + pis := make([]ffi.PublicPieceInfo, len(ss.pieces)) + for i, piece := range ss.pieces { + pis[i] = ffi.PublicPieceInfo{ + Size: uint64(len(piece)), + CommP: commD(piece), + } + } + + commd, err := MockVerifier.GenerateDataCommitment(sb.sectorSize, pis) + if err != nil { + return sectorbuilder.RawSealPreCommitOutput{}, err + } + + return sectorbuilder.RawSealPreCommitOutput{ + CommD: commd, + CommR: commDR(commd[:]), + }, nil +} + +func (sb *SBMock) SealCommit(ctx context.Context, sid uint64, ticket sectorbuilder.SealTicket, seed sectorbuilder.SealSeed, pieces []sectorbuilder.PublicPieceInfo, precommit sectorbuilder.RawSealPreCommitOutput) ([]byte, error) { + sb.lk.Lock() + ss, ok := sb.sectors[sid] + sb.lk.Unlock() + if !ok { + return nil, xerrors.Errorf("no such sector %d", sid) + } + ss.lk.Lock() + defer ss.lk.Unlock() + + if ss.failed { + return nil, xerrors.Errorf("[mock] cannot commit failed sector %d", sid) + } + + if ss.state != statePreCommit { + return nil, xerrors.Errorf("cannot commit sector that has not been precommitted") + } + + opFinishWait(ctx) + + var out [32]byte + for i := range out { + out[i] = precommit.CommD[i] + precommit.CommR[31-i] - ticket.TicketBytes[i]*seed.TicketBytes[i] + } + return out[:], nil +} + +func (sb *SBMock) GetPath(string, string) (string, error) { + panic("nyi") +} + +func (sb *SBMock) WorkerStats() sectorbuilder.WorkerStats { + panic("nyi") +} + +func (sb *SBMock) AddWorker(context.Context, sectorbuilder.WorkerCfg) (<-chan sectorbuilder.WorkerTask, error) { + panic("nyi") +} + +func (sb *SBMock) TaskDone(context.Context, uint64, sectorbuilder.SealRes) error { + panic("nyi") +} + +// Test Instrumentation Methods + +func (sb *SBMock) FailSector(sid uint64) error { + sb.lk.Lock() + defer sb.lk.Unlock() + ss, ok := sb.sectors[sid] + if !ok { + return fmt.Errorf("no such sector in sectorbuilder") + } + + ss.failed = true + return nil +} + +func opFinishWait(ctx context.Context) { + val, ok := ctx.Value("opfinish").(chan struct{}) + if !ok { + return + } + <-val +} + +func AddOpFinish(ctx context.Context) (context.Context, func()) { + done := make(chan struct{}) + + return context.WithValue(ctx, "opfinish", done), func() { + close(done) + } +} + +func (sb *SBMock) ComputeElectionPoSt(sectorInfo sectorbuilder.SortedPublicSectorInfo, challengeSeed []byte, winners []sectorbuilder.EPostCandidate) ([]byte, error) { + panic("implement me") +} + +func (sb *SBMock) GenerateEPostCandidates(sectorInfo sectorbuilder.SortedPublicSectorInfo, challengeSeed [sectorbuilder.CommLen]byte, faults []uint64) ([]sectorbuilder.EPostCandidate, error) { + if len(faults) > 0 { + panic("todo") + } + + n := sectorbuilder.ElectionPostChallengeCount(uint64(len(sectorInfo.Values())), uint64(len(faults))) + if n > uint64(len(sectorInfo.Values())) { + n = uint64(len(sectorInfo.Values())) + } + + out := make([]sectorbuilder.EPostCandidate, n) + + seed := big.NewInt(0).SetBytes(challengeSeed[:]) + start := seed.Mod(seed, big.NewInt(int64(len(sectorInfo.Values())))).Int64() + + for i := range out { + out[i] = sectorbuilder.EPostCandidate{ + SectorID: uint64((int(start) + i) % len(sectorInfo.Values())), + PartialTicket: challengeSeed, + Ticket: commDR(challengeSeed[:]), + SectorChallengeIndex: 0, + } + } + + return out, nil +} + +func (sb *SBMock) ReadPieceFromSealedSector(sectorID uint64, offset uint64, size uint64, ticket []byte, commD []byte) (io.ReadCloser, error) { + if len(sb.sectors[sectorID].pieces) > 1 { + panic("implme") + } + return ioutil.NopCloser(io.LimitReader(bytes.NewReader(sb.sectors[sectorID].pieces[0][offset:]), int64(size))), nil +} + +func (sb *SBMock) StageFakeData() (uint64, []sectorbuilder.PublicPieceInfo, error) { + usize := sectorbuilder.UserBytesForSectorSize(sb.sectorSize) + sid, err := sb.AcquireSectorId() + if err != nil { + return 0, nil, err + } + + buf := make([]byte, usize) + rand.Read(buf) + + pi, err := sb.AddPiece(usize, sid, bytes.NewReader(buf), nil) + if err != nil { + return 0, nil, err + } + + return sid, []sectorbuilder.PublicPieceInfo{pi}, nil +} + +func (m mockVerif) VerifyElectionPost(ctx context.Context, sectorSize uint64, sectorInfo sectorbuilder.SortedPublicSectorInfo, challengeSeed []byte, proof []byte, candidates []sectorbuilder.EPostCandidate, proverID address.Address) (bool, error) { + panic("implement me") +} + +func (m mockVerif) VerifyFallbackPost(ctx context.Context, sectorSize uint64, sectorInfo sectorbuilder.SortedPublicSectorInfo, challengeSeed []byte, proof []byte, candidates []sectorbuilder.EPostCandidate, proverID address.Address, faults uint64) (bool, error) { + panic("implement me") +} + +func (m mockVerif) VerifySeal(sectorSize uint64, commR, commD []byte, proverID address.Address, ticket []byte, seed []byte, sectorID uint64, proof []byte) (bool, error) { + if len(proof) != 32 { // Real ones are longer, but this should be fine + return false, nil + } + + for i, b := range proof { + if b != commD[i]+commR[31-i]-ticket[i]*seed[i] { + return false, nil + } + } + + return true, nil +} + +func (m mockVerif) GenerateDataCommitment(ssize uint64, pieces []ffi.PublicPieceInfo) ([sectorbuilder.CommLen]byte, error) { + if len(pieces) != 1 { + panic("todo") + } + if pieces[0].Size != sectorbuilder.UserBytesForSectorSize(ssize) { + panic("todo") + } + return pieces[0].CommP, nil +} + +var MockVerifier = mockVerif{} + +var _ sectorbuilder.Verifier = MockVerifier +var _ sectorbuilder.Interface = &SBMock{} diff --git a/storage/sbmock/sbmock_test.go b/storage/sbmock/sbmock_test.go new file mode 100644 index 000000000..d07a4eee4 --- /dev/null +++ b/storage/sbmock/sbmock_test.go @@ -0,0 +1,45 @@ +package sbmock + +import ( + "context" + "testing" + "time" + + "github.com/filecoin-project/go-sectorbuilder" +) + +func TestOpFinish(t *testing.T) { + sb := NewMockSectorBuilder(1, 1024) + + sid, pieces, err := sb.StageFakeData() + if err != nil { + t.Fatal(err) + } + + ctx, done := AddOpFinish(context.TODO()) + + finished := make(chan struct{}) + go func() { + _, err := sb.SealPreCommit(ctx, sid, sectorbuilder.SealTicket{}, pieces) + if err != nil { + t.Error(err) + return + } + + close(finished) + }() + + select { + case <-finished: + t.Fatal("should not finish until we tell it to") + case <-time.After(time.Second / 2): + } + + done() + + select { + case <-finished: + case <-time.After(time.Second / 2): + t.Fatal("should finish after we tell it to") + } +} diff --git a/storage/sbmock/util.go b/storage/sbmock/util.go new file mode 100644 index 000000000..1fb5206e7 --- /dev/null +++ b/storage/sbmock/util.go @@ -0,0 +1,34 @@ +package sbmock + +import ( + "bytes" + "crypto/rand" + "io" + "io/ioutil" + + "github.com/filecoin-project/go-sectorbuilder" +) + +func randB(n uint64) []byte { + b, err := ioutil.ReadAll(io.LimitReader(rand.Reader, int64(n))) + if err != nil { + panic(err) + } + return b +} + +func commDR(in []byte) (out [32]byte) { + for i, b := range in { + out[i] = ^b + } + + return out +} + +func commD(b []byte) [32]byte { + c, err := sectorbuilder.GeneratePieceCommitment(bytes.NewReader(b), uint64(len(b))) + if err != nil { + panic(err) + } + return c +} diff --git a/storage/sector_states.go b/storage/sector_states.go index 0b86a3059..f3c2a4b44 100644 --- a/storage/sector_states.go +++ b/storage/sector_states.go @@ -69,7 +69,7 @@ func (m *Miner) handleUnsealed(ctx context.Context, sector SectorInfo) *sectorUp return sector.upd().fatal(err) } - rspco, err := m.sb.SealPreCommit(sector.SectorID, *ticket, sector.pieceInfos()) + rspco, err := m.sb.SealPreCommit(ctx, sector.SectorID, *ticket, sector.pieceInfos()) if err != nil { return sector.upd().to(api.SealFailed).error(xerrors.Errorf("seal pre commit failed: %w", err)) } @@ -173,7 +173,7 @@ func (m *Miner) handlePreCommitted(ctx context.Context, sector SectorInfo) *sect func (m *Miner) handleCommitting(ctx context.Context, sector SectorInfo) *sectorUpdate { log.Info("scheduling seal proof computation...") - proof, err := m.sb.SealCommit(sector.SectorID, sector.Ticket.SB(), sector.Seed.SB(), sector.pieceInfos(), sector.rspco()) + proof, err := m.sb.SealCommit(ctx, sector.SectorID, sector.Ticket.SB(), sector.Seed.SB(), sector.pieceInfos(), sector.rspco()) if err != nil { return sector.upd().to(api.SealCommitFailed).error(xerrors.Errorf("computing seal proof failed: %w", err)) } diff --git a/storage/sector_utils.go b/storage/sector_utils.go index 664223a20..96fcbf891 100644 --- a/storage/sector_utils.go +++ b/storage/sector_utils.go @@ -52,6 +52,6 @@ func (m *Miner) ListSectors() ([]SectorInfo, error) { func (m *Miner) GetSectorInfo(sid uint64) (SectorInfo, error) { var out SectorInfo - err := m.sectors.Get(sid, &out) + err := m.sectors.Get(sid).Get(&out) return out, err } diff --git a/storage/sectorblocks/blocks.go b/storage/sectorblocks/blocks.go index 9d12e0537..ed3a718df 100644 --- a/storage/sectorblocks/blocks.go +++ b/storage/sectorblocks/blocks.go @@ -39,7 +39,7 @@ var ErrNotFound = errors.New("not found") type SectorBlocks struct { *storage.Miner - sb *sectorbuilder.SectorBuilder + sb sectorbuilder.Interface intermediate blockstore.Blockstore // holds intermediate nodes TODO: consider combining with the staging blockstore @@ -47,7 +47,7 @@ type SectorBlocks struct { keyLk sync.Mutex } -func NewSectorBlocks(miner *storage.Miner, ds dtypes.MetadataDS, sb *sectorbuilder.SectorBuilder) *SectorBlocks { +func NewSectorBlocks(miner *storage.Miner, ds dtypes.MetadataDS, sb sectorbuilder.Interface) *SectorBlocks { sbc := &SectorBlocks{ Miner: miner, sb: sb, diff --git a/storage/sectorblocks/blockstore.go b/storage/sectorblocks/blockstore.go index b0807ef3a..3cf497d59 100644 --- a/storage/sectorblocks/blockstore.go +++ b/storage/sectorblocks/blockstore.go @@ -9,7 +9,7 @@ import ( blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" blockstore "github.com/ipfs/go-ipfs-blockstore" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "golang.org/x/xerrors" ) diff --git a/storage/sectors.go b/storage/sectors.go index e8894749e..bc07902b9 100644 --- a/storage/sectors.go +++ b/storage/sectors.go @@ -161,7 +161,7 @@ func (m *Miner) onSectorIncoming(sector *SectorInfo) { func (m *Miner) onSectorUpdated(ctx context.Context, update sectorUpdate) { log.Infof("Sector %d updated state to %s", update.id, api.SectorStates[update.newState]) var sector SectorInfo - err := m.sectors.Mutate(update.id, func(s *SectorInfo) error { + err := m.sectors.Get(update.id).Mutate(func(s *SectorInfo) error { if update.nonce < s.Nonce { return xerrors.Errorf("update nonce too low, ignoring (%d < %d)", update.nonce, s.Nonce) } diff --git a/tools/stats/main.go b/tools/stats/main.go index 7c0f12e97..fb383e8e1 100644 --- a/tools/stats/main.go +++ b/tools/stats/main.go @@ -6,7 +6,7 @@ import ( "os" "time" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" ) var log = logging.Logger("stats") diff --git a/tracing/setup.go b/tracing/setup.go index ee8e4c8dd..141683b39 100644 --- a/tracing/setup.go +++ b/tracing/setup.go @@ -4,7 +4,7 @@ import ( "os" "contrib.go.opencensus.io/exporter/jaeger" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "go.opencensus.io/trace" )