From c9c437cb3bb0098a2cac12544f915f35645c8e8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 25 Jan 2020 12:15:28 +0100 Subject: [PATCH 01/11] sealing: Parallel CommP calc in pledge sector --- storage/sealing/garbage.go | 54 ++++++++++++++++++++++++++++++++++---- 1 file changed, 49 insertions(+), 5 deletions(-) diff --git a/storage/sealing/garbage.go b/storage/sealing/garbage.go index 1c3925671..a6b439258 100644 --- a/storage/sealing/garbage.go +++ b/storage/sealing/garbage.go @@ -6,14 +6,61 @@ import ( "io" "math" "math/rand" + "runtime" + "sync" sectorbuilder "github.com/filecoin-project/go-sectorbuilder" + "github.com/hashicorp/go-multierror" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/types" ) +func (m *Sealing) fastPledgeCommitment(size uint64, parts uint64) (commP [sectorbuilder.CommLen]byte, err error) { + piece := sectorbuilder.UserBytesForSectorSize((size / 127 + size) / parts) + out := make([]sectorbuilder.PublicPieceInfo, parts) + var lk sync.Mutex + + var wg sync.WaitGroup + wg.Add(int(parts)) + for i := uint64(0); i < parts; i++ { + go func(i uint64) { + defer wg.Done() + + commP, perr := sectorbuilder.GeneratePieceCommitment(io.LimitReader(rand.New(rand.NewSource(42 + int64(i))), int64(piece)), piece) + + lk.Lock() + if perr != nil { + err = multierror.Append(err, perr) + } + out[i] = sectorbuilder.PublicPieceInfo{ + Size: piece, + CommP: commP, + } + lk.Unlock() + }(i) + } + wg.Wait() + + if err != nil { + return [32]byte{}, err + } + + return sectorbuilder.GenerateDataCommitment(m.sb.SectorSize(), out) +} + +func (m *Sealing) pledgeReader(size uint64, parts uint64) io.Reader { + piece := sectorbuilder.UserBytesForSectorSize((size / 127 + size) / parts) + + readers := make([]io.Reader, parts) + for i := range readers { + readers[i] = io.LimitReader(rand.New(rand.NewSource(42 + int64(i))), int64(piece)) + } + + return io.MultiReader(readers...) +} + func (m *Sealing) pledgeSector(ctx context.Context, sectorID uint64, existingPieceSizes []uint64, sizes ...uint64) ([]Piece, error) { if len(sizes) == 0 { return nil, nil @@ -21,10 +68,7 @@ func (m *Sealing) pledgeSector(ctx context.Context, sectorID uint64, existingPie deals := make([]actors.StorageDealProposal, len(sizes)) for i, size := range sizes { - release := m.sb.RateLimit() - commP, err := sectorbuilder.GeneratePieceCommitment(io.LimitReader(rand.New(rand.NewSource(42)), int64(size)), size) - release() - + commP, err := m.fastPledgeCommitment(size, uint64(runtime.NumCPU())) if err != nil { return nil, err } @@ -81,7 +125,7 @@ func (m *Sealing) pledgeSector(ctx context.Context, sectorID uint64, existingPie out := make([]Piece, len(sizes)) for i, size := range sizes { - ppi, err := m.sb.AddPiece(size, sectorID, io.LimitReader(rand.New(rand.NewSource(42)), int64(size)), existingPieceSizes) + ppi, err := m.sb.AddPiece(size, sectorID, m.pledgeReader(size, uint64(runtime.NumCPU())), existingPieceSizes) if err != nil { return nil, err } From dff6e87ce9bc90ba403a079680dd51dad0577bec Mon Sep 17 00:00:00 2001 From: jsign Date: Mon, 27 Jan 2020 09:16:13 -0300 Subject: [PATCH 02/11] Dockerfile: fixes Signed-off-by: jsign --- Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index dab96ea34..03a07b3ad 100644 --- a/Dockerfile +++ b/Dockerfile @@ -79,13 +79,15 @@ EXPOSE 5678 ENV HOME_PATH /data ENV PARAMCACHE_PATH /var/tmp/filecoin-proof-parameters -RUN mkdir -p $HOME_PATH \ +RUN mkdir -p $HOME_PATH $PARAMCACHE_PATH \ && adduser -D -h $HOME_PATH -u 1000 -G users lotus \ - && chown lotus:users $HOME_PATH + && chown lotus:users $HOME_PATH $PARAMCACHE_PATH VOLUME $HOME_PATH VOLUME $PARAMCACHE_PATH +USER lotus + # Execute the daemon subcommand by default CMD ["/sbin/tini", "--", "lotus", "daemon"] From c6e63db7b492b37ad9182f0a61278c0240470a01 Mon Sep 17 00:00:00 2001 From: jsign Date: Mon, 27 Jan 2020 15:08:08 -0300 Subject: [PATCH 03/11] rebase and reapply Signed-off-by: jsign --- .../dockers/docker-examples/basic-miner-busybox/Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/dockers/docker-examples/basic-miner-busybox/Dockerfile b/tools/dockers/docker-examples/basic-miner-busybox/Dockerfile index dab96ea34..03a07b3ad 100644 --- a/tools/dockers/docker-examples/basic-miner-busybox/Dockerfile +++ b/tools/dockers/docker-examples/basic-miner-busybox/Dockerfile @@ -79,13 +79,15 @@ EXPOSE 5678 ENV HOME_PATH /data ENV PARAMCACHE_PATH /var/tmp/filecoin-proof-parameters -RUN mkdir -p $HOME_PATH \ +RUN mkdir -p $HOME_PATH $PARAMCACHE_PATH \ && adduser -D -h $HOME_PATH -u 1000 -G users lotus \ - && chown lotus:users $HOME_PATH + && chown lotus:users $HOME_PATH $PARAMCACHE_PATH VOLUME $HOME_PATH VOLUME $PARAMCACHE_PATH +USER lotus + # Execute the daemon subcommand by default CMD ["/sbin/tini", "--", "lotus", "daemon"] From dc076c64417f8c021e2325b1adf4d6933ef1651f Mon Sep 17 00:00:00 2001 From: jsign Date: Mon, 27 Jan 2020 15:14:24 -0300 Subject: [PATCH 04/11] remove Dockerfile in base Signed-off-by: jsign --- Dockerfile | 93 ------------------------------------------------------ 1 file changed, 93 deletions(-) delete mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 03a07b3ad..000000000 --- a/Dockerfile +++ /dev/null @@ -1,93 +0,0 @@ -FROM golang:1.13.4-buster -MAINTAINER ldoublewood - -ENV SRC_DIR /lotus - -RUN apt-get update && apt-get install -y && apt-get install -y ca-certificates llvm clang mesa-opencl-icd ocl-icd-opencl-dev - -RUN curl -sSf https://sh.rustup.rs | sh -s -- -y - - -# Get su-exec, a very minimal tool for dropping privileges, -# and tini, a very minimal init daemon for containers -ENV SUEXEC_VERSION v0.2 -ENV TINI_VERSION v0.18.0 -RUN set -x \ - && cd /tmp \ - && git clone https://github.com/ncopa/su-exec.git \ - && cd su-exec \ - && git checkout -q $SUEXEC_VERSION \ - && make \ - && cd /tmp \ - && wget -q -O tini https://github.com/krallin/tini/releases/download/$TINI_VERSION/tini \ - && chmod +x tini - -# Download packages first so they can be cached. -COPY go.mod go.sum $SRC_DIR/ -COPY extern/ $SRC_DIR/extern/ -RUN cd $SRC_DIR \ - && go mod download - -COPY Makefile $SRC_DIR - -# Because extern/filecoin-ffi building script need to get version number from git -COPY .git/ $SRC_DIR/.git/ -COPY .gitmodules $SRC_DIR/ - -# Download dependence first -RUN cd $SRC_DIR \ - && mkdir $SRC_DIR/build \ - && . $HOME/.cargo/env \ - && make clean \ - && make deps - - -COPY . $SRC_DIR - -# Build the thing. -RUN cd $SRC_DIR \ - && . $HOME/.cargo/env \ - && make - -# Now comes the actual target image, which aims to be as small as possible. -FROM busybox:1-glibc -MAINTAINER ldoublewood - -# Get the executable binary and TLS CAs from the build container. -ENV SRC_DIR /lotus -COPY --from=0 $SRC_DIR/lotus /usr/local/bin/lotus -COPY --from=0 $SRC_DIR/lotus-storage-miner /usr/local/bin/lotus-storage-miner -COPY --from=0 /tmp/su-exec/su-exec /sbin/su-exec -COPY --from=0 /tmp/tini /sbin/tini -COPY --from=0 /etc/ssl/certs /etc/ssl/certs - - -# This shared lib (part of glibc) doesn't seem to be included with busybox. -COPY --from=0 /lib/x86_64-linux-gnu/libdl-2.28.so /lib/libdl.so.2 -COPY --from=0 /lib/x86_64-linux-gnu/libutil-2.28.so /lib/libutil.so.1 -COPY --from=0 /usr/lib/x86_64-linux-gnu/libOpenCL.so.1.0.0 /lib/libOpenCL.so.1 -COPY --from=0 /lib/x86_64-linux-gnu/librt-2.28.so /lib/librt.so.1 -COPY --from=0 /lib/x86_64-linux-gnu/libgcc_s.so.1 /lib/libgcc_s.so.1 - -# WS port -EXPOSE 1234 -# P2P port -EXPOSE 5678 - - -# Create the home directory and switch to a non-privileged user. -ENV HOME_PATH /data -ENV PARAMCACHE_PATH /var/tmp/filecoin-proof-parameters - -RUN mkdir -p $HOME_PATH $PARAMCACHE_PATH \ - && adduser -D -h $HOME_PATH -u 1000 -G users lotus \ - && chown lotus:users $HOME_PATH $PARAMCACHE_PATH - - -VOLUME $HOME_PATH -VOLUME $PARAMCACHE_PATH - -USER lotus - -# Execute the daemon subcommand by default -CMD ["/sbin/tini", "--", "lotus", "daemon"] From 80903e86fa1de9d56897d448ff24cc477cd1fc78 Mon Sep 17 00:00:00 2001 From: Richard Patel Date: Mon, 27 Jan 2020 20:02:56 +0100 Subject: [PATCH 05/11] Dockerfile: more binaries --- .../docker-examples/basic-miner-busybox/Dockerfile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/dockers/docker-examples/basic-miner-busybox/Dockerfile b/tools/dockers/docker-examples/basic-miner-busybox/Dockerfile index dab96ea34..5374e42fe 100644 --- a/tools/dockers/docker-examples/basic-miner-busybox/Dockerfile +++ b/tools/dockers/docker-examples/basic-miner-busybox/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.13.4-buster +FROM golang:1.13-buster MAINTAINER ldoublewood ENV SRC_DIR /lotus @@ -44,10 +44,12 @@ RUN cd $SRC_DIR \ COPY . $SRC_DIR +ARG MAKE_TARGET=all + # Build the thing. RUN cd $SRC_DIR \ && . $HOME/.cargo/env \ - && make + && make $MAKE_TARGET # Now comes the actual target image, which aims to be as small as possible. FROM busybox:1-glibc @@ -56,7 +58,7 @@ MAINTAINER ldoublewood # Get the executable binary and TLS CAs from the build container. ENV SRC_DIR /lotus COPY --from=0 $SRC_DIR/lotus /usr/local/bin/lotus -COPY --from=0 $SRC_DIR/lotus-storage-miner /usr/local/bin/lotus-storage-miner +COPY --from=0 $SRC_DIR/lotus-* /usr/local/bin/ COPY --from=0 /tmp/su-exec/su-exec /sbin/su-exec COPY --from=0 /tmp/tini /sbin/tini COPY --from=0 /etc/ssl/certs /etc/ssl/certs From cb665caf9ed0120064229a9680e48244e344d9bd Mon Sep 17 00:00:00 2001 From: laser Date: Tue, 28 Jan 2020 11:46:26 -0800 Subject: [PATCH 06/11] planCommitting must handle SectorCommitFailed The SectorCommitFailed struct can be created from within Sealing#handleCommitting, and is created if actors.SerializeParams(params) produces an error or if m.api.MpoolPushMessage(ctx.Context(), msg) produces an error. --- storage/sealing/fsm.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/storage/sealing/fsm.go b/storage/sealing/fsm.go index ad0803488..4fdd81d35 100644 --- a/storage/sealing/fsm.go +++ b/storage/sealing/fsm.go @@ -205,6 +205,8 @@ func planCommitting(events []statemachine.Event, state *SectorInfo) error { state.State = api.SealCommitFailed case SectorSealFailed: state.State = api.CommitFailed + case SectorCommitFailed: + state.State = api.CommitFailed default: return xerrors.Errorf("planCommitting got event of unknown type %T, events: %+v", event.User, events) } From 1cdf29244c8f92d8ed21e52a70b9284476823289 Mon Sep 17 00:00:00 2001 From: laser Date: Tue, 28 Jan 2020 12:39:07 -0800 Subject: [PATCH 07/11] write basic test affirming state change --- storage/sealing/fsm_test.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/storage/sealing/fsm_test.go b/storage/sealing/fsm_test.go index 2dada5470..7430bb634 100644 --- a/storage/sealing/fsm_test.go +++ b/storage/sealing/fsm_test.go @@ -83,3 +83,17 @@ func TestSeedRevert(t *testing.T) { m.planSingle(SectorProving{}) require.Equal(m.t, m.state.State, api.Proving) } + +func TestPlanCommittingHandlesSectorCommitFailed(t *testing.T) { + m := test{ + s: &Sealing{}, + t: t, + state: &SectorInfo{State: api.Committing}, + } + + events := []statemachine.Event{{SectorCommitFailed{}}} + + require.NoError(t, planCommitting(events, m.state)) + + require.Equal(t, api.SectorStates[api.CommitFailed], api.SectorStates[m.state.State]) +} From 66bfadb71a9f0505517ceb1a1eec762e282467a9 Mon Sep 17 00:00:00 2001 From: jimmylee Date: Wed, 29 Jan 2020 00:51:05 -0800 Subject: [PATCH 08/11] documentation: adds lotus client list-deals for checking the status of a deal --- documentation/en/storing-data.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/documentation/en/storing-data.md b/documentation/en/storing-data.md index 410a06553..5ee4af354 100644 --- a/documentation/en/storing-data.md +++ b/documentation/en/storing-data.md @@ -49,6 +49,12 @@ Store a **Data CID** with a miner: lotus client deal ``` +Check the status of a deal: + +```sh +lotus client list-deals +``` + - Price is in attoFIL. - The `duration`, which represents how long the miner will keep your file hosted, is represented in blocks. Each block represents 45 seconds. From 53da298c4fc4452449ff1c425b34afc7a88db258 Mon Sep 17 00:00:00 2001 From: jimmylee Date: Wed, 29 Jan 2020 01:04:43 -0800 Subject: [PATCH 09/11] documentation: updates library schema with fedora instructions --- documentation/en/.library.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/documentation/en/.library.json b/documentation/en/.library.json index 65dbff359..b9e211902 100644 --- a/documentation/en/.library.json +++ b/documentation/en/.library.json @@ -32,6 +32,12 @@ "github": "en/install-lotus-ubuntu.md", "value": null }, + { + "title": "Fedora Installation", + "slug": "en+install-lotus-fedora", + "github": "en/install-lotus-fedora.md", + "value": null + }, { "title": "MacOS Installation", "slug": "en+install-lotus-macos", From 25f544f3db5f3683efcf5241bd07d3232b942af6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 29 Jan 2020 20:12:08 +0100 Subject: [PATCH 10/11] sealing: round parts in fastPledgeCommitment --- storage/sealing/garbage.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/storage/sealing/garbage.go b/storage/sealing/garbage.go index a6b439258..d33b63276 100644 --- a/storage/sealing/garbage.go +++ b/storage/sealing/garbage.go @@ -5,6 +5,7 @@ import ( "context" "io" "math" + "math/bits" "math/rand" "runtime" "sync" @@ -18,7 +19,9 @@ import ( ) func (m *Sealing) fastPledgeCommitment(size uint64, parts uint64) (commP [sectorbuilder.CommLen]byte, err error) { - piece := sectorbuilder.UserBytesForSectorSize((size / 127 + size) / parts) + parts = 1 << bits.Len64(parts) // round down to nearest power of 2 + + piece := sectorbuilder.UserBytesForSectorSize((size/127 + size) / parts) out := make([]sectorbuilder.PublicPieceInfo, parts) var lk sync.Mutex @@ -28,7 +31,7 @@ func (m *Sealing) fastPledgeCommitment(size uint64, parts uint64) (commP [sector go func(i uint64) { defer wg.Done() - commP, perr := sectorbuilder.GeneratePieceCommitment(io.LimitReader(rand.New(rand.NewSource(42 + int64(i))), int64(piece)), piece) + commP, perr := sectorbuilder.GeneratePieceCommitment(io.LimitReader(rand.New(rand.NewSource(42+int64(i))), int64(piece)), piece) lk.Lock() if perr != nil { @@ -51,11 +54,11 @@ func (m *Sealing) fastPledgeCommitment(size uint64, parts uint64) (commP [sector } func (m *Sealing) pledgeReader(size uint64, parts uint64) io.Reader { - piece := sectorbuilder.UserBytesForSectorSize((size / 127 + size) / parts) + piece := sectorbuilder.UserBytesForSectorSize((size/127 + size) / parts) readers := make([]io.Reader, parts) for i := range readers { - readers[i] = io.LimitReader(rand.New(rand.NewSource(42 + int64(i))), int64(piece)) + readers[i] = io.LimitReader(rand.New(rand.NewSource(42+int64(i))), int64(piece)) } return io.MultiReader(readers...) From ba542cf67a97d105bdc0875da8da29891f3df72c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 29 Jan 2020 21:01:20 +0100 Subject: [PATCH 11/11] test fastPledgeCommitment --- storage/sealing/garbage.go | 38 --------------------------------- storage/sealing/utils.go | 40 +++++++++++++++++++++++++++++++++++ storage/sealing/utils_test.go | 11 +++++++++- 3 files changed, 50 insertions(+), 39 deletions(-) diff --git a/storage/sealing/garbage.go b/storage/sealing/garbage.go index d33b63276..4a3b1331b 100644 --- a/storage/sealing/garbage.go +++ b/storage/sealing/garbage.go @@ -5,54 +5,16 @@ import ( "context" "io" "math" - "math/bits" "math/rand" "runtime" - "sync" sectorbuilder "github.com/filecoin-project/go-sectorbuilder" - "github.com/hashicorp/go-multierror" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/types" ) -func (m *Sealing) fastPledgeCommitment(size uint64, parts uint64) (commP [sectorbuilder.CommLen]byte, err error) { - parts = 1 << bits.Len64(parts) // round down to nearest power of 2 - - piece := sectorbuilder.UserBytesForSectorSize((size/127 + size) / parts) - out := make([]sectorbuilder.PublicPieceInfo, parts) - var lk sync.Mutex - - var wg sync.WaitGroup - wg.Add(int(parts)) - for i := uint64(0); i < parts; i++ { - go func(i uint64) { - defer wg.Done() - - commP, perr := sectorbuilder.GeneratePieceCommitment(io.LimitReader(rand.New(rand.NewSource(42+int64(i))), int64(piece)), piece) - - lk.Lock() - if perr != nil { - err = multierror.Append(err, perr) - } - out[i] = sectorbuilder.PublicPieceInfo{ - Size: piece, - CommP: commP, - } - lk.Unlock() - }(i) - } - wg.Wait() - - if err != nil { - return [32]byte{}, err - } - - return sectorbuilder.GenerateDataCommitment(m.sb.SectorSize(), out) -} - func (m *Sealing) pledgeReader(size uint64, parts uint64) io.Reader { piece := sectorbuilder.UserBytesForSectorSize((size/127 + size) / parts) diff --git a/storage/sealing/utils.go b/storage/sealing/utils.go index 8fa887d3c..b235a8a83 100644 --- a/storage/sealing/utils.go +++ b/storage/sealing/utils.go @@ -1,7 +1,12 @@ package sealing import ( + "io" "math/bits" + "math/rand" + "sync" + + "github.com/hashicorp/go-multierror" sectorbuilder "github.com/filecoin-project/go-sectorbuilder" ) @@ -42,6 +47,41 @@ func fillersFromRem(toFill uint64) ([]uint64, error) { return out, nil } +func (m *Sealing) fastPledgeCommitment(size uint64, parts uint64) (commP [sectorbuilder.CommLen]byte, err error) { + parts = 1 << bits.Len64(parts) // round down to nearest power of 2 + + piece := sectorbuilder.UserBytesForSectorSize(size / parts) + out := make([]sectorbuilder.PublicPieceInfo, parts) + var lk sync.Mutex + + var wg sync.WaitGroup + wg.Add(int(parts)) + for i := uint64(0); i < parts; i++ { + go func(i uint64) { + defer wg.Done() + + commP, perr := sectorbuilder.GeneratePieceCommitment(io.LimitReader(rand.New(rand.NewSource(42+int64(i))), int64(piece)), piece) + + lk.Lock() + if perr != nil { + err = multierror.Append(err, perr) + } + out[i] = sectorbuilder.PublicPieceInfo{ + Size: piece, + CommP: commP, + } + lk.Unlock() + }(i) + } + wg.Wait() + + if err != nil { + return [32]byte{}, err + } + + return sectorbuilder.GenerateDataCommitment(m.sb.SectorSize(), out) +} + func (m *Sealing) ListSectors() ([]SectorInfo, error) { var sectors []SectorInfo if err := m.sectors.List(§ors); err != nil { diff --git a/storage/sealing/utils_test.go b/storage/sealing/utils_test.go index 02746a3d8..14d512a52 100644 --- a/storage/sealing/utils_test.go +++ b/storage/sealing/utils_test.go @@ -1,6 +1,7 @@ package sealing import ( + "github.com/filecoin-project/lotus/storage/sbmock" "testing" "github.com/stretchr/testify/assert" @@ -42,5 +43,13 @@ func TestFillersFromRem(t *testing.T) { ub = sectorbuilder.UserBytesForSectorSize(uint64(9) << i) testFill(t, ub, []uint64{ub1, ub4}) } - +} + +func TestFastPledge(t *testing.T) { + sz := uint64(16 << 20) + + s := Sealing{sb: sbmock.NewMockSectorBuilder(0, sz)} + if _, err := s.fastPledgeCommitment(sz, 5); err != nil { + t.Fatalf("%+v", err) + } }