diff --git a/.circleci/config.yml b/.circleci/config.yml index e74cb7736..a4a88a090 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -976,19 +976,10 @@ workflows: tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - - build-appimage: - filters: - branches: - ignore: - - /.*/ - tags: - only: - - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - publish: requires: - build-all - build-macos - - build-appimage filters: branches: ignore: diff --git a/.circleci/template.yml b/.circleci/template.yml index 4b954391b..666ad39f5 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -816,19 +816,10 @@ workflows: tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - - build-appimage: - filters: - branches: - ignore: - - /.*/ - tags: - only: - - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - publish: requires: - build-all - build-macos - - build-appimage filters: branches: ignore: diff --git a/CHANGELOG.md b/CHANGELOG.md index 338410642..28814d3c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,58 +1,68 @@ # Lotus changelog -# v1.13.2-rc1 / 2021-11-30 +# v1.13.2-rc2 / 2021-12-08 -This is the first RC for lotus v1.13.2. This is a highly recommended release with sealing pipeline fixes, worker management and scheduler enhancement, retrieval improvements and so on. More detailed changelog will be updated later. +This is the second RC for lotus v1.13.2, with further optimization on retrieval. This is a highly recommended release +with sealing pipeline fixes, worker management and scheduler enhancement, retrieval improvements and so on. More detailed changelog will be updated later. - github.com/filecoin-project/lotus: - - Address Scheduler enhancements (#7703) review ([filecoin-project/lotus#7714](https://github.com/filecoin-project/lotus/pull/7714)) - - Scheduler enhancements ([filecoin-project/lotus#7703](https://github.com/filecoin-project/lotus/pull/7703)) - - ffiwrapper: Validate PC2 by calling C1 with random seeds ([filecoin-project/lotus#7710](https://github.com/filecoin-project/lotus/pull/7710)) - - fix logic error ([filecoin-project/lotus#7709](https://github.com/filecoin-project/lotus/pull/7709)) - - Update go-graphsync v0.10.6 ([filecoin-project/lotus#7708](https://github.com/filecoin-project/lotus/pull/7708)) - - Add verbose mode to lotus-miner pieces list-cids ([filecoin-project/lotus#7699](https://github.com/filecoin-project/lotus/pull/7699)) - - retrieval: Only output matching nodes, MatchPath dagspec ([filecoin-project/lotus#7706](https://github.com/filecoin-project/lotus/pull/7706)) - - disable mplex stream muxer ([filecoin-project/lotus#7689](https://github.com/filecoin-project/lotus/pull/7689)) - - Cleanup partial retrieval codepaths ( zero functional changes ) ([filecoin-project/lotus#7688](https://github.com/filecoin-project/lotus/pull/7688)) - - Make small retrieval 200x faster ([filecoin-project/lotus#7693](https://github.com/filecoin-project/lotus/pull/7693)) - - Releases back to master ([filecoin-project/lotus#7698](https://github.com/filecoin-project/lotus/pull/7698)) - - Add RLE dump code ([filecoin-project/lotus#7691](https://github.com/filecoin-project/lotus/pull/7691)) - - Update archive script ([filecoin-project/lotus#7690](https://github.com/filecoin-project/lotus/pull/7690)) - - storage: Use 1M buffers for Tar transfers ([filecoin-project/lotus#7681](https://github.com/filecoin-project/lotus/pull/7681)) - - Chore/dm level tests plus merkle proof cars ([filecoin-project/lotus#7673](https://github.com/filecoin-project/lotus/pull/7673)) - - Partial retrieval ux improvements ([filecoin-project/lotus#7610](https://github.com/filecoin-project/lotus/pull/7610)) - - Sector storage groups ([filecoin-project/lotus#7453](https://github.com/filecoin-project/lotus/pull/7453)) - - docsgen-cli: Handle commands with no description correctly ([filecoin-project/lotus#7659](https://github.com/filecoin-project/lotus/pull/7659)) - - shed: simple wallet balancer util ([filecoin-project/lotus#7414](https://github.com/filecoin-project/lotus/pull/7414)) - - remote store: Remove debug printf ([filecoin-project/lotus#7664](https://github.com/filecoin-project/lotus/pull/7664)) - - Bump ws from 5.2.2 to 5.2.3 in /lotuspond/front ([filecoin-project/lotus#7660](https://github.com/filecoin-project/lotus/pull/7660)) - - Bump color-string from 1.5.3 to 1.6.0 in /lotuspond/front ([filecoin-project/lotus#7658](https://github.com/filecoin-project/lotus/pull/7658)) - - Bump postcss from 7.0.17 to 7.0.39 in /lotuspond/front ([filecoin-project/lotus#7657](https://github.com/filecoin-project/lotus/pull/7657)) - - Bump path-parse from 1.0.6 to 1.0.7 in /lotuspond/front ([filecoin-project/lotus#7656](https://github.com/filecoin-project/lotus/pull/7656)) - - Bump tmpl from 1.0.4 to 1.0.5 in /lotuspond/front ([filecoin-project/lotus#7655](https://github.com/filecoin-project/lotus/pull/7655)) - - Bump url-parse from 1.4.7 to 1.5.3 in /lotuspond/front ([filecoin-project/lotus#7654](https://github.com/filecoin-project/lotus/pull/7654)) - - Add caches to lotus-stats and splitcode ([filecoin-project/lotus#7329](https://github.com/filecoin-project/lotus/pull/7329)) - - add additional methods to lotus gateway ([filecoin-project/lotus#7644](https://github.com/filecoin-project/lotus/pull/7644)) - - checkCommit should return SectorCommitFailed ([filecoin-project/lotus#7555](https://github.com/filecoin-project/lotus/pull/7555)) - - Wdpost worker: Reduce challenge confidence to 1 epoch ([filecoin-project/lotus#7572](https://github.com/filecoin-project/lotus/pull/7572)) - - remove api and jaeger env from docker file ([filecoin-project/lotus#7624](https://github.com/filecoin-project/lotus/pull/7624)) - - CLI: Add a lotus multisig cancel command ([filecoin-project/lotus#7645](https://github.com/filecoin-project/lotus/pull/7645)) - - remove jaeger envvars ([filecoin-project/lotus#7631](https://github.com/filecoin-project/lotus/pull/7631)) - - lotus-shed msg: Decode submessages/msig proposals ([filecoin-project/lotus#7639](https://github.com/filecoin-project/lotus/pull/7639)) - - add log for restart windows post scheduler ([filecoin-project/lotus#7613](https://github.com/filecoin-project/lotus/pull/7613)) - - Shed: Add a util to list miner faults ([filecoin-project/lotus#7605](https://github.com/filecoin-project/lotus/pull/7605)) - - add timeout flag to wait-api command ([filecoin-project/lotus#7592](https://github.com/filecoin-project/lotus/pull/7592)) - - Shed: Add a util to create miners more easily ([filecoin-project/lotus#7595](https://github.com/filecoin-project/lotus/pull/7595)) - - Update go-state-types ([filecoin-project/lotus#7591](https://github.com/filecoin-project/lotus/pull/7591)) - - add missing NodeType tag ([filecoin-project/lotus#7559](https://github.com/filecoin-project/lotus/pull/7559)) - - update go-libp2p-pubsub to v0.5.6 ([filecoin-project/lotus#7581](https://github.com/filecoin-project/lotus/pull/7581)) - - bump the master version to v1.13.2-dev ([filecoin-project/lotus#7568](https://github.com/filecoin-project/lotus/pull/7568)) + - CARv2 v2.1.0 + - dagstore pieceReader: Fix wrong ErrUnexpectedEOF return in ReadAt + - dagstore pieceReader: Always read full in ReadAt + - Add metrics to dagstore piecereader + - disable building of appimage on release + - v1.13.2-rc1 ([filecoin-project/lotus#7718](https://github.com/filecoin-project/lotus/pull/7718)) + - Address Scheduler enhancements (#7703) review ([filecoin-project/lotus#7714](https://github.com/filecoin-project/lotus/pull/7714)) + - Scheduler enhancements ([filecoin-project/lotus#7703](https://github.com/filecoin-project/lotus/pull/7703)) + - ffiwrapper: Validate PC2 by calling C1 with random seeds ([filecoin-project/lotus#7710](https://github.com/filecoin-project/lotus/pull/7710)) + - fix logic error ([filecoin-project/lotus#7709](https://github.com/filecoin-project/lotus/pull/7709)) + - Update go-graphsync v0.10.6 ([filecoin-project/lotus#7708](https://github.com/filecoin-project/lotus/pull/7708)) + - Add verbose mode to lotus-miner pieces list-cids ([filecoin-project/lotus#7699](https://github.com/filecoin-project/lotus/pull/7699)) + - retrieval: Only output matching nodes, MatchPath dagspec ([filecoin-project/lotus#7706](https://github.com/filecoin-project/lotus/pull/7706)) + - disable mplex stream muxer ([filecoin-project/lotus#7689](https://github.com/filecoin-project/lotus/pull/7689)) + - Cleanup partial retrieval codepaths ( zero functional changes ) ([filecoin-project/lotus#7688](https://github.com/filecoin-project/lotus/pull/7688)) + - Make small retrieval 200x faster ([filecoin-project/lotus#7693](https://github.com/filecoin-project/lotus/pull/7693)) + - Releases back to master ([filecoin-project/lotus#7698](https://github.com/filecoin-project/lotus/pull/7698)) + - Add RLE dump code ([filecoin-project/lotus#7691](https://github.com/filecoin-project/lotus/pull/7691)) + - Update archive script ([filecoin-project/lotus#7690](https://github.com/filecoin-project/lotus/pull/7690)) + - storage: Use 1M buffers for Tar transfers ([filecoin-project/lotus#7681](https://github.com/filecoin-project/lotus/pull/7681)) + - Chore/dm level tests plus merkle proof cars ([filecoin-project/lotus#7673](https://github.com/filecoin-project/lotus/pull/7673)) + - Partial retrieval ux improvements ([filecoin-project/lotus#7610](https://github.com/filecoin-project/lotus/pull/7610)) + - Sector storage groups ([filecoin-project/lotus#7453](https://github.com/filecoin-project/lotus/pull/7453)) + - docsgen-cli: Handle commands with no description correctly ([filecoin-project/lotus#7659](https://github.com/filecoin-project/lotus/pull/7659)) + - shed: simple wallet balancer util ([filecoin-project/lotus#7414](https://github.com/filecoin-project/lotus/pull/7414)) + - remote store: Remove debug printf ([filecoin-project/lotus#7664](https://github.com/filecoin-project/lotus/pull/7664)) + - Bump ws from 5.2.2 to 5.2.3 in /lotuspond/front ([filecoin-project/lotus#7660](https://github.com/filecoin-project/lotus/pull/7660)) + - Bump color-string from 1.5.3 to 1.6.0 in /lotuspond/front ([filecoin-project/lotus#7658](https://github.com/filecoin-project/lotus/pull/7658)) + - Bump postcss from 7.0.17 to 7.0.39 in /lotuspond/front ([filecoin-project/lotus#7657](https://github.com/filecoin-project/lotus/pull/7657)) + - Bump path-parse from 1.0.6 to 1.0.7 in /lotuspond/front ([filecoin-project/lotus#7656](https://github.com/filecoin-project/lotus/pull/7656)) + - Bump tmpl from 1.0.4 to 1.0.5 in /lotuspond/front ([filecoin-project/lotus#7655](https://github.com/filecoin-project/lotus/pull/7655)) + - Bump url-parse from 1.4.7 to 1.5.3 in /lotuspond/front ([filecoin-project/lotus#7654](https://github.com/filecoin-project/lotus/pull/7654)) + - Add caches to lotus-stats and splitcode ([filecoin-project/lotus#7329](https://github.com/filecoin-project/lotus/pull/7329)) + - add additional methods to lotus gateway ([filecoin-project/lotus#7644](https://github.com/filecoin-project/lotus/pull/7644)) + - checkCommit should return SectorCommitFailed ([filecoin-project/lotus#7555](https://github.com/filecoin-project/lotus/pull/7555)) + - Wdpost worker: Reduce challenge confidence to 1 epoch ([filecoin-project/lotus#7572](https://github.com/filecoin-project/lotus/pull/7572)) + - remove api and jaeger env from docker file ([filecoin-project/lotus#7624](https://github.com/filecoin-project/lotus/pull/7624)) + - CLI: Add a lotus multisig cancel command ([filecoin-project/lotus#7645](https://github.com/filecoin-project/lotus/pull/7645)) + - remove jaeger envvars ([filecoin-project/lotus#7631](https://github.com/filecoin-project/lotus/pull/7631)) + - lotus-shed msg: Decode submessages/msig proposals ([filecoin-project/lotus#7639](https://github.com/filecoin-project/lotus/pull/7639)) + - add log for restart windows post scheduler ([filecoin-project/lotus#7613](https://github.com/filecoin-project/lotus/pull/7613)) + - Shed: Add a util to list miner faults ([filecoin-project/lotus#7605](https://github.com/filecoin-project/lotus/pull/7605)) + - add timeout flag to wait-api command ([filecoin-project/lotus#7592](https://github.com/filecoin-project/lotus/pull/7592)) + - Shed: Add a util to create miners more easily ([filecoin-project/lotus#7595](https://github.com/filecoin-project/lotus/pull/7595)) + - Update go-state-types ([filecoin-project/lotus#7591](https://github.com/filecoin-project/lotus/pull/7591)) + - add missing NodeType tag ([filecoin-project/lotus#7559](https://github.com/filecoin-project/lotus/pull/7559)) + - update go-libp2p-pubsub to v0.5.6 ([filecoin-project/lotus#7581](https://github.com/filecoin-project/lotus/pull/7581)) + - bump the master version to v1.13.2-dev ([filecoin-project/lotus#7568](https://github.com/filecoin-project/lotus/pull/7568)) +- github.com/filecoin-project/go-fil-markets (v1.13.3 -> v1.13.4): + failed to fetch repo - github.com/filecoin-project/go-state-types (v0.1.1-0.20210915140513-d354ccf10379 -> v0.1.1): - - Add v14 - - Add PoRep -> UpdateProof mapping - - Intoduce update proofs enums - - Update golangci-lint for comatibility with Go 1.17 - - Update execution image to maybe update debian version + - Add v14 + - Add PoRep -> UpdateProof mapping + - Intoduce update proofs enums + - Update golangci-lint for comatibility with Go 1.17 + - Update execution image to maybe update debian version + # v1.13.1 / 2021-11-26 diff --git a/build/openrpc/full.json.gz b/build/openrpc/full.json.gz index fa2424f8d..9ee5ab7c7 100644 Binary files a/build/openrpc/full.json.gz and b/build/openrpc/full.json.gz differ diff --git a/build/openrpc/miner.json.gz b/build/openrpc/miner.json.gz index 2c4c48781..df59b1559 100644 Binary files a/build/openrpc/miner.json.gz and b/build/openrpc/miner.json.gz differ diff --git a/build/openrpc/worker.json.gz b/build/openrpc/worker.json.gz index 620250665..ba7831d24 100644 Binary files a/build/openrpc/worker.json.gz and b/build/openrpc/worker.json.gz differ diff --git a/build/version.go b/build/version.go index 3cc154a70..edd63244b 100644 --- a/build/version.go +++ b/build/version.go @@ -37,7 +37,7 @@ func BuildTypeString() string { } // BuildVersion is the local build version -const BuildVersion = "1.13.2-rc1" +const BuildVersion = "1.13.2-rc2" func UserVersion() string { if os.Getenv("LOTUS_VERSION_IGNORE_COMMIT") == "1" { diff --git a/documentation/en/cli-lotus-miner.md b/documentation/en/cli-lotus-miner.md index e94ac12c9..ac8567ebb 100644 --- a/documentation/en/cli-lotus-miner.md +++ b/documentation/en/cli-lotus-miner.md @@ -7,7 +7,7 @@ USAGE: lotus-miner [global options] command [command options] [arguments...] VERSION: - 1.13.2-rc1 + 1.13.2-rc2 COMMANDS: init Initialize a lotus miner repo diff --git a/documentation/en/cli-lotus-worker.md b/documentation/en/cli-lotus-worker.md index d4b097747..1116f5bfb 100644 --- a/documentation/en/cli-lotus-worker.md +++ b/documentation/en/cli-lotus-worker.md @@ -7,7 +7,7 @@ USAGE: lotus-worker [global options] command [command options] [arguments...] VERSION: - 1.13.2-rc1 + 1.13.2-rc2 COMMANDS: run Start lotus worker diff --git a/documentation/en/cli-lotus.md b/documentation/en/cli-lotus.md index c2a93c335..2a99fc791 100644 --- a/documentation/en/cli-lotus.md +++ b/documentation/en/cli-lotus.md @@ -7,7 +7,7 @@ USAGE: lotus [global options] command [command options] [arguments...] VERSION: - 1.13.2-rc1 + 1.13.2-rc2 COMMANDS: daemon Start a lotus daemon process diff --git a/go.mod b/go.mod index ba29f8cdf..21536e3b5 100644 --- a/go.mod +++ b/go.mod @@ -37,7 +37,7 @@ require ( github.com/filecoin-project/go-data-transfer v1.11.4 github.com/filecoin-project/go-fil-commcid v0.1.0 github.com/filecoin-project/go-fil-commp-hashhash v0.1.0 - github.com/filecoin-project/go-fil-markets v1.13.3 + github.com/filecoin-project/go-fil-markets v1.13.4 github.com/filecoin-project/go-jsonrpc v0.1.5 github.com/filecoin-project/go-padreader v0.0.1 github.com/filecoin-project/go-paramfetch v0.0.2 @@ -99,7 +99,7 @@ require ( github.com/ipfs/go-unixfs v0.2.6 github.com/ipfs/interface-go-ipfs-core v0.4.0 github.com/ipld/go-car v0.3.2-0.20211001225732-32d0d9933823 - github.com/ipld/go-car/v2 v2.0.3-0.20210811121346-c514a30114d7 + github.com/ipld/go-car/v2 v2.1.0 github.com/ipld/go-codec-dagpb v1.3.0 github.com/ipld/go-ipld-prime v0.12.3 github.com/ipld/go-ipld-selector-text-lite v0.0.1 diff --git a/go.sum b/go.sum index c0ceb01ae..1a5e9d368 100644 --- a/go.sum +++ b/go.sum @@ -339,8 +339,8 @@ github.com/filecoin-project/go-fil-commcid v0.1.0/go.mod h1:Eaox7Hvus1JgPrL5+M3+ github.com/filecoin-project/go-fil-commp-hashhash v0.1.0 h1:imrrpZWEHRnNqqv0tN7LXep5bFEVOVmQWHJvl2mgsGo= github.com/filecoin-project/go-fil-commp-hashhash v0.1.0/go.mod h1:73S8WSEWh9vr0fDJVnKADhfIv/d6dCbAGaAGWbdJEI8= github.com/filecoin-project/go-fil-markets v1.0.5-0.20201113164554-c5eba40d5335/go.mod h1:AJySOJC00JRWEZzRG2KsfUnqEf5ITXxeX09BE9N4f9c= -github.com/filecoin-project/go-fil-markets v1.13.3 h1:iMCpG7I4fb+YLcgDnMaqZiZiyFZWNvrwHqiFPHB0/tQ= -github.com/filecoin-project/go-fil-markets v1.13.3/go.mod h1:38zuj8AgDvOfdakFLpC/syYIYgXTzkq7xqBJ6T1AuG4= +github.com/filecoin-project/go-fil-markets v1.13.4 h1:NAu+ACelR2mYsj+yJ4iLu8FGqWK50OnU5VF8axkLsSc= +github.com/filecoin-project/go-fil-markets v1.13.4/go.mod h1:aANjXD2XMHWnT2zWpyGWLsWLC24C4mHm0gRm85OpPWE= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24= github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 h1:b3UDemBYN2HNfk3KOXNuxgTTxlWi3xVvbQP0IT38fvM= @@ -866,8 +866,8 @@ github.com/ipld/go-car v0.3.2-0.20211001225732-32d0d9933823 h1:8JMSJ0k71fU9lIUrp github.com/ipld/go-car v0.3.2-0.20211001225732-32d0d9933823/go.mod h1:jSlTph+i/q1jLFoiKKeN69KGG0fXpwrcD0izu5C1Tpo= github.com/ipld/go-car/v2 v2.0.0-beta1.0.20210721090610-5a9d1b217d25/go.mod h1:I2ACeeg6XNBe5pdh5TaR7Ambhfa7If9KXxmXgZsYENU= github.com/ipld/go-car/v2 v2.0.2/go.mod h1:I2ACeeg6XNBe5pdh5TaR7Ambhfa7If9KXxmXgZsYENU= -github.com/ipld/go-car/v2 v2.0.3-0.20210811121346-c514a30114d7 h1:6Z0beJSZNsRY+7udoqUl4gQ/tqtrPuRvDySrlsvbqZA= -github.com/ipld/go-car/v2 v2.0.3-0.20210811121346-c514a30114d7/go.mod h1:I2ACeeg6XNBe5pdh5TaR7Ambhfa7If9KXxmXgZsYENU= +github.com/ipld/go-car/v2 v2.1.0 h1:t8R/WXUSkfu1K1gpPk76mytCxsEdMjGcMIgpOq3/Cnw= +github.com/ipld/go-car/v2 v2.1.0/go.mod h1:Xr6GwkDhv8dtOtgHzOynAkIOg0t0YiPc5DxBPppWqZA= github.com/ipld/go-codec-dagpb v1.2.0/go.mod h1:6nBN7X7h8EOsEejZGqC7tej5drsdBAXbMHyBT+Fne5s= github.com/ipld/go-codec-dagpb v1.3.0 h1:czTcaoAuNNyIYWs6Qe01DJ+sEX7B+1Z0LcXjSatMGe8= github.com/ipld/go-codec-dagpb v1.3.0/go.mod h1:ga4JTU3abYApDC3pZ00BC2RSvC3qfBb9MSJkMLSwnhA= @@ -1483,8 +1483,9 @@ github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPw github.com/multiformats/go-multicodec v0.2.0/go.mod h1:/y4YVwkfMyry5kFbMTbLJKErhycTIftytRV+llXdyS4= github.com/multiformats/go-multicodec v0.2.1-0.20210713081508-b421db6850ae/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ= github.com/multiformats/go-multicodec v0.2.1-0.20210714093213-b2b5bd6fe68b/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ= -github.com/multiformats/go-multicodec v0.3.0 h1:tstDwfIjiHbnIjeM5Lp+pMrSeN+LCMsEwOrkPmWm03A= github.com/multiformats/go-multicodec v0.3.0/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ= +github.com/multiformats/go-multicodec v0.3.1-0.20210902112759-1539a079fd61 h1:ZrUuMKNgJ52qHPoQ+bx0h0uBfcWmN7Px+4uKSZeesiI= +github.com/multiformats/go-multicodec v0.3.1-0.20210902112759-1539a079fd61/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= diff --git a/markets/dagstore/mount_test.go b/markets/dagstore/mount_test.go index ec5e8a086..820058786 100644 --- a/markets/dagstore/mount_test.go +++ b/markets/dagstore/mount_test.go @@ -30,7 +30,6 @@ func TestLotusMount(t *testing.T) { mockLotusMountAPI := mock_dagstore.NewMockMinerAPI(mockCtrl) mockLotusMountAPI.EXPECT().IsUnsealed(gomock.Any(), cid).Return(true, nil).Times(1) - mockLotusMountAPI.EXPECT().FetchUnsealedPiece(gomock.Any(), cid, uint64(0)).Return(ioutil.NopCloser(strings.NewReader("testing")), abi.UnpaddedPieceSize(7), nil).Times(1) mockLotusMountAPI.EXPECT().FetchUnsealedPiece(gomock.Any(), cid, uint64(0)).Return(ioutil.NopCloser(strings.NewReader("testing")), abi.UnpaddedPieceSize(7), nil).Times(1) mockLotusMountAPI.EXPECT().GetUnpaddedCARSize(ctx, cid).Return(uint64(100), nil).Times(1) diff --git a/markets/dagstore/piecereader.go b/markets/dagstore/piecereader.go index 6ef69dfbe..67cd10c27 100644 --- a/markets/dagstore/piecereader.go +++ b/markets/dagstore/piecereader.go @@ -1,19 +1,23 @@ package dagstore import ( + "bufio" "context" "io" "github.com/ipfs/go-cid" + "go.opencensus.io/stats" "golang.org/x/xerrors" "github.com/filecoin-project/dagstore/mount" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/metrics" ) // For small read skips, it's faster to "burn" some bytes than to setup new sector reader. // Assuming 1ms stream seek latency, and 1G/s stream rate, we're willing to discard up to 1 MiB. var MaxPieceReaderBurnBytes int64 = 1 << 20 // 1M +var ReadBuf = 128 * (127 * 8) // unpadded(128k) type pieceReader struct { ctx context.Context @@ -25,15 +29,19 @@ type pieceReader struct { seqAt int64 // next byte to be read by io.Reader r io.ReadCloser + br *bufio.Reader rAt int64 } func (p *pieceReader) init() (_ *pieceReader, err error) { + stats.Record(p.ctx, metrics.DagStorePRInitCount.M(1)) + p.rAt = 0 p.r, p.len, err = p.api.FetchUnsealedPiece(p.ctx, p.pieceCid, uint64(p.rAt)) if err != nil { return nil, err } + p.br = bufio.NewReaderSize(p.r, ReadBuf) return p, nil } @@ -95,6 +103,8 @@ func (p *pieceReader) ReadAt(b []byte, off int64) (n int, err error) { return 0, err } + stats.Record(p.ctx, metrics.DagStorePRBytesRequested.M(int64(len(b)))) + // 1. Get the backing reader into the correct position // if the backing reader is ahead of the offset we want, or more than @@ -105,12 +115,20 @@ func (p *pieceReader) ReadAt(b []byte, off int64) (n int, err error) { return 0, xerrors.Errorf("closing backing reader: %w", err) } p.r = nil + p.br = nil } - log.Debugw("pieceReader new stream", "piece", p.pieceCid, "at", p.rAt, "off", off-p.rAt) + log.Debugw("pieceReader new stream", "piece", p.pieceCid, "at", p.rAt, "off", off-p.rAt, "n", len(b)) + + if off > p.rAt { + stats.Record(p.ctx, metrics.DagStorePRSeekForwardBytes.M(off-p.rAt), metrics.DagStorePRSeekForwardCount.M(1)) + } else { + stats.Record(p.ctx, metrics.DagStorePRSeekBackBytes.M(p.rAt-off), metrics.DagStorePRSeekBackCount.M(1)) + } p.rAt = off p.r, _, err = p.api.FetchUnsealedPiece(p.ctx, p.pieceCid, uint64(p.rAt)) + p.br = bufio.NewReaderSize(p.r, ReadBuf) if err != nil { return 0, xerrors.Errorf("getting backing reader: %w", err) } @@ -118,7 +136,9 @@ func (p *pieceReader) ReadAt(b []byte, off int64) (n int, err error) { // 2. Check if we need to burn some bytes if off > p.rAt { - n, err := io.CopyN(io.Discard, p.r, off-p.rAt) + stats.Record(p.ctx, metrics.DagStorePRBytesDiscarded.M(off-p.rAt), metrics.DagStorePRDiscardCount.M(1)) + + n, err := io.CopyN(io.Discard, p.br, off-p.rAt) p.rAt += n if err != nil { return 0, xerrors.Errorf("discarding read gap: %w", err) @@ -131,7 +151,14 @@ func (p *pieceReader) ReadAt(b []byte, off int64) (n int, err error) { } // 4. Read! - n, err = p.r.Read(b) + n, err = io.ReadFull(p.br, b) + if n < len(b) { + log.Debugw("pieceReader short read", "piece", p.pieceCid, "at", p.rAt, "toEnd", int64(p.len)-p.rAt, "n", len(b), "read", n, "err", err) + } + if err == io.ErrUnexpectedEOF { + err = io.EOF + } + p.rAt += int64(n) return n, err } diff --git a/metrics/metrics.go b/metrics/metrics.go index ddd149d8d..b4032bb1d 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -128,6 +128,15 @@ var ( StorageLimitUsedBytes = stats.Int64("storage/path_limit_used_bytes", "used optional storage limit bytes", stats.UnitBytes) StorageLimitMaxBytes = stats.Int64("storage/path_limit_max_bytes", "optional storage limit", stats.UnitBytes) + DagStorePRInitCount = stats.Int64("dagstore/pr_init_count", "PieceReader init count", stats.UnitDimensionless) + DagStorePRBytesRequested = stats.Int64("dagstore/pr_requested_bytes", "PieceReader requested bytes", stats.UnitBytes) + DagStorePRBytesDiscarded = stats.Int64("dagstore/pr_discarded_bytes", "PieceReader discarded bytes", stats.UnitBytes) + DagStorePRDiscardCount = stats.Int64("dagstore/pr_discard_count", "PieceReader discard count", stats.UnitDimensionless) + DagStorePRSeekBackCount = stats.Int64("dagstore/pr_seek_back_count", "PieceReader seek back count", stats.UnitDimensionless) + DagStorePRSeekForwardCount = stats.Int64("dagstore/pr_seek_forward_count", "PieceReader seek forward count", stats.UnitDimensionless) + DagStorePRSeekBackBytes = stats.Int64("dagstore/pr_seek_back_bytes", "PieceReader seek back bytes", stats.UnitBytes) + DagStorePRSeekForwardBytes = stats.Int64("dagstore/pr_seek_forward_bytes", "PieceReader seek forward bytes", stats.UnitBytes) + // splitstore SplitstoreMiss = stats.Int64("splitstore/miss", "Number of misses in hotstre access", stats.UnitDimensionless) SplitstoreCompactionTimeSeconds = stats.Float64("splitstore/compaction_time", "Compaction time in seconds", stats.UnitSeconds) @@ -383,6 +392,39 @@ var ( TagKeys: []tag.Key{StorageID}, } + DagStorePRInitCountView = &view.View{ + Measure: DagStorePRInitCount, + Aggregation: view.Count(), + } + DagStorePRBytesRequestedView = &view.View{ + Measure: DagStorePRBytesRequested, + Aggregation: view.Sum(), + } + DagStorePRBytesDiscardedView = &view.View{ + Measure: DagStorePRBytesDiscarded, + Aggregation: view.Sum(), + } + DagStorePRDiscardCountView = &view.View{ + Measure: DagStorePRDiscardCount, + Aggregation: view.Count(), + } + DagStorePRSeekBackCountView = &view.View{ + Measure: DagStorePRSeekBackCount, + Aggregation: view.Count(), + } + DagStorePRSeekForwardCountView = &view.View{ + Measure: DagStorePRSeekForwardCount, + Aggregation: view.Count(), + } + DagStorePRSeekBackBytesView = &view.View{ + Measure: DagStorePRSeekBackBytes, + Aggregation: view.Sum(), + } + DagStorePRSeekForwardBytesView = &view.View{ + Measure: DagStorePRSeekForwardBytes, + Aggregation: view.Sum(), + } + // splitstore SplitstoreMissView = &view.View{ Measure: SplitstoreMiss, @@ -539,6 +581,14 @@ var MinerNodeViews = append([]*view.View{ StorageReservedBytesView, StorageLimitUsedBytesView, StorageLimitMaxBytesView, + DagStorePRInitCountView, + DagStorePRBytesRequestedView, + DagStorePRBytesDiscardedView, + DagStorePRDiscardCountView, + DagStorePRSeekBackCountView, + DagStorePRSeekForwardCountView, + DagStorePRSeekBackBytesView, + DagStorePRSeekForwardBytesView, }, DefaultViews...) // SinceInMilliseconds returns the duration of time since the provide time as a float64.