Merge branch 'master' into feat/new-sb-fs

Dockerfile

@@ -1,91 +0,0 @@
-FROM golang:1.13.4-buster
-MAINTAINER ldoublewood <ldoublewood@gmail.com>
-
-ENV SRC_DIR /lotus
-
-RUN apt-get update && apt-get install -y && apt-get install -y ca-certificates llvm clang mesa-opencl-icd ocl-icd-opencl-dev
-
-RUN curl -sSf https://sh.rustup.rs | sh -s -- -y
-
-
-# Get su-exec, a very minimal tool for dropping privileges,
-# and tini, a very minimal init daemon for containers
-ENV SUEXEC_VERSION v0.2
-ENV TINI_VERSION v0.18.0
-RUN set -x \
-  && cd /tmp \
-  && git clone https://github.com/ncopa/su-exec.git \
-  && cd su-exec \
-  && git checkout -q $SUEXEC_VERSION \
-  && make \
-  && cd /tmp \
-  && wget -q -O tini https://github.com/krallin/tini/releases/download/$TINI_VERSION/tini \
-  && chmod +x tini
-
-# Download packages first so they can be cached.
-COPY go.mod go.sum $SRC_DIR/
-COPY extern/ $SRC_DIR/extern/
-RUN cd $SRC_DIR \
-  && go mod download
-
-COPY Makefile $SRC_DIR
-
-# Because extern/filecoin-ffi building script need to get version number from git
-COPY .git/ $SRC_DIR/.git/
-COPY .gitmodules $SRC_DIR/
-
-# Download dependence first
-RUN cd $SRC_DIR \
-  && mkdir $SRC_DIR/build \
-  && . $HOME/.cargo/env \
-  && make clean \
-  && make deps
-
-
-COPY . $SRC_DIR
-
-# Build the thing.
-RUN cd $SRC_DIR \
-  && . $HOME/.cargo/env \
-  && make
-
-# Now comes the actual target image, which aims to be as small as possible.
-FROM busybox:1-glibc
-MAINTAINER ldoublewood <ldoublewood@gmail.com>
-
-# Get the executable binary and TLS CAs from the build container.
-ENV SRC_DIR /lotus
-COPY --from=0 $SRC_DIR/lotus /usr/local/bin/lotus
-COPY --from=0 $SRC_DIR/lotus-storage-miner /usr/local/bin/lotus-storage-miner
-COPY --from=0 /tmp/su-exec/su-exec /sbin/su-exec
-COPY --from=0 /tmp/tini /sbin/tini
-COPY --from=0 /etc/ssl/certs /etc/ssl/certs
-
-
-# This shared lib (part of glibc) doesn't seem to be included with busybox.
-COPY --from=0 /lib/x86_64-linux-gnu/libdl-2.28.so /lib/libdl.so.2
-COPY --from=0 /lib/x86_64-linux-gnu/libutil-2.28.so /lib/libutil.so.1 
-COPY --from=0 /usr/lib/x86_64-linux-gnu/libOpenCL.so.1.0.0 /lib/libOpenCL.so.1
-COPY --from=0 /lib/x86_64-linux-gnu/librt-2.28.so /lib/librt.so.1
-COPY --from=0 /lib/x86_64-linux-gnu/libgcc_s.so.1 /lib/libgcc_s.so.1
-
-# WS port
-EXPOSE 1234
-# P2P port
-EXPOSE 5678
-
-
-# Create the home directory and switch to a non-privileged user.
-ENV HOME_PATH /data
-ENV PARAMCACHE_PATH /var/tmp/filecoin-proof-parameters
-
-RUN mkdir -p $HOME_PATH \
-  && adduser -D -h $HOME_PATH -u 1000 -G users lotus \
-  && chown lotus:users $HOME_PATH
-
-
-VOLUME $HOME_PATH
-VOLUME $PARAMCACHE_PATH
-
-# Execute the daemon subcommand by default
-CMD ["/sbin/tini", "--", "lotus", "daemon"]

documentation/en/.library.json

@@ -32,6 +32,12 @@
           "github": "en/install-lotus-ubuntu.md",
           "value": null
         },
+        {
+          "title": "Fedora Installation",
+          "slug": "en+install-lotus-fedora",
+          "github": "en/install-lotus-fedora.md",
+          "value": null
+        },
         {
           "title": "MacOS Installation",
           "slug": "en+install-lotus-macos",

documentation/en/storing-data.md

@@ -49,6 +49,12 @@ Store a **Data CID** with a miner:
 lotus client deal <Data CID> <miner> <price> <duration>
 ```
 
+Check the status of a deal:
+
+```sh
+lotus client list-deals
+```
+
 - Price is in attoFIL.
 - The `duration`, which represents how long the miner will keep your file hosted, is represented in blocks. Each block represents 45 seconds.
 

storage/sealing/fsm.go

@@ -211,6 +211,8 @@ func planCommitting(events []statemachine.Event, state *SectorInfo) error {
 			state.State = api.SealCommitFailed
 		case SectorSealFailed:
 			state.State = api.CommitFailed
+		case SectorCommitFailed:
+			state.State = api.CommitFailed
 		default:
 			return xerrors.Errorf("planCommitting got event of unknown type %T, events: %+v", event.User, events)
 		}

storage/sealing/fsm_test.go

@@ -83,3 +83,17 @@ func TestSeedRevert(t *testing.T) {
 	m.planSingle(SectorProving{})
 	require.Equal(m.t, m.state.State, api.Proving)
 }
+
+func TestPlanCommittingHandlesSectorCommitFailed(t *testing.T) {
+	m := test{
+		s:     &Sealing{},
+		t:     t,
+		state: &SectorInfo{State: api.Committing},
+	}
+
+	events := []statemachine.Event{{SectorCommitFailed{}}}
+
+	require.NoError(t, planCommitting(events, m.state))
+
+	require.Equal(t, api.SectorStates[api.CommitFailed], api.SectorStates[m.state.State])
+}

storage/sealing/garbage.go

@@ -6,6 +6,7 @@ import (
 	"io"
 	"math"
 	"math/rand"
+	"runtime"
 
 	sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
 	"golang.org/x/xerrors"
@@ -14,6 +15,17 @@ import (
 	"github.com/filecoin-project/lotus/chain/types"
 )
 
+func (m *Sealing) pledgeReader(size uint64, parts uint64) io.Reader {
+	piece := sectorbuilder.UserBytesForSectorSize((size/127 + size) / parts)
+
+	readers := make([]io.Reader, parts)
+	for i := range readers {
+		readers[i] = io.LimitReader(rand.New(rand.NewSource(42+int64(i))), int64(piece))
+	}
+
+	return io.MultiReader(readers...)
+}
+
 func (m *Sealing) pledgeSector(ctx context.Context, sectorID uint64, existingPieceSizes []uint64, sizes ...uint64) ([]Piece, error) {
 	if len(sizes) == 0 {
 		return nil, nil
@@ -21,10 +33,7 @@ func (m *Sealing) pledgeSector(ctx context.Context, sectorID uint64, existingPie
 
 	deals := make([]actors.StorageDealProposal, len(sizes))
 	for i, size := range sizes {
-		release := m.sb.RateLimit()
+		commP, err := m.fastPledgeCommitment(size, uint64(runtime.NumCPU()))
-		commP, err := sectorbuilder.GeneratePieceCommitment(io.LimitReader(rand.New(rand.NewSource(42)), int64(size)), size)
-		release()
-
 		if err != nil {
 			return nil, err
 		}
@@ -81,7 +90,7 @@ func (m *Sealing) pledgeSector(ctx context.Context, sectorID uint64, existingPie
 	out := make([]Piece, len(sizes))
 
 	for i, size := range sizes {
-		ppi, err := m.sb.AddPiece(ctx, size, sectorID, io.LimitReader(rand.New(rand.NewSource(42)), int64(size)), existingPieceSizes)
+		ppi, err := m.sb.AddPiece(ctx, size, sectorID, m.pledgeReader(size, uint64(runtime.NumCPU())), existingPieceSizes)
 		if err != nil {
 			return nil, xerrors.Errorf("add piece: %w", err)
 		}

storage/sealing/utils.go

@@ -1,7 +1,12 @@
 package sealing
 
 import (
+	"io"
 	"math/bits"
+	"math/rand"
+	"sync"
+
+	"github.com/hashicorp/go-multierror"
 
 	sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
 )
@@ -42,6 +47,41 @@ func fillersFromRem(toFill uint64) ([]uint64, error) {
 	return out, nil
 }
 
+func (m *Sealing) fastPledgeCommitment(size uint64, parts uint64) (commP [sectorbuilder.CommLen]byte, err error) {
+	parts = 1 << bits.Len64(parts) // round down to nearest power of 2
+
+	piece := sectorbuilder.UserBytesForSectorSize(size / parts)
+	out := make([]sectorbuilder.PublicPieceInfo, parts)
+	var lk sync.Mutex
+
+	var wg sync.WaitGroup
+	wg.Add(int(parts))
+	for i := uint64(0); i < parts; i++ {
+		go func(i uint64) {
+			defer wg.Done()
+
+			commP, perr := sectorbuilder.GeneratePieceCommitment(io.LimitReader(rand.New(rand.NewSource(42+int64(i))), int64(piece)), piece)
+
+			lk.Lock()
+			if perr != nil {
+				err = multierror.Append(err, perr)
+			}
+			out[i] = sectorbuilder.PublicPieceInfo{
+				Size:  piece,
+				CommP: commP,
+			}
+			lk.Unlock()
+		}(i)
+	}
+	wg.Wait()
+
+	if err != nil {
+		return [32]byte{}, err
+	}
+
+	return sectorbuilder.GenerateDataCommitment(m.sb.SectorSize(), out)
+}
+
 func (m *Sealing) ListSectors() ([]SectorInfo, error) {
 	var sectors []SectorInfo
 	if err := m.sectors.List(&sectors); err != nil {

storage/sealing/utils_test.go

@@ -1,6 +1,7 @@
 package sealing
 
 import (
+	"github.com/filecoin-project/lotus/storage/sbmock"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -42,5 +43,13 @@ func TestFillersFromRem(t *testing.T) {
 		ub = sectorbuilder.UserBytesForSectorSize(uint64(9) << i)
 		testFill(t, ub, []uint64{ub1, ub4})
 	}
-
+}
+
+func TestFastPledge(t *testing.T) {
+	sz := uint64(16 << 20)
+
+	s := Sealing{sb: sbmock.NewMockSectorBuilder(0, sz)}
+	if _, err := s.fastPledgeCommitment(sz, 5); err != nil {
+		t.Fatalf("%+v", err)
+	}
 }

tools/dockers/docker-examples/basic-miner-busybox/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.13.4-buster
+FROM golang:1.13-buster
 MAINTAINER ldoublewood <ldoublewood@gmail.com>
 
 ENV SRC_DIR /lotus
@@ -44,10 +44,12 @@ RUN cd $SRC_DIR \
 
 COPY . $SRC_DIR
 
+ARG MAKE_TARGET=all
+
 # Build the thing.
 RUN cd $SRC_DIR \
   && . $HOME/.cargo/env \
-  && make
+  && make $MAKE_TARGET
 
 # Now comes the actual target image, which aims to be as small as possible.
 FROM busybox:1-glibc
@@ -56,7 +58,7 @@ MAINTAINER ldoublewood <ldoublewood@gmail.com>
 # Get the executable binary and TLS CAs from the build container.
 ENV SRC_DIR /lotus
 COPY --from=0 $SRC_DIR/lotus /usr/local/bin/lotus
-COPY --from=0 $SRC_DIR/lotus-storage-miner /usr/local/bin/lotus-storage-miner
+COPY --from=0 $SRC_DIR/lotus-* /usr/local/bin/
 COPY --from=0 /tmp/su-exec/su-exec /sbin/su-exec
 COPY --from=0 /tmp/tini /sbin/tini
 COPY --from=0 /etc/ssl/certs /etc/ssl/certs
@@ -79,13 +81,15 @@ EXPOSE 5678
 ENV HOME_PATH /data
 ENV PARAMCACHE_PATH /var/tmp/filecoin-proof-parameters
 
-RUN mkdir -p $HOME_PATH \
+RUN mkdir -p $HOME_PATH $PARAMCACHE_PATH \
   && adduser -D -h $HOME_PATH -u 1000 -G users lotus \
-  && chown lotus:users $HOME_PATH
+  && chown lotus:users $HOME_PATH $PARAMCACHE_PATH
 
 
 VOLUME $HOME_PATH
 VOLUME $PARAMCACHE_PATH
 
+USER lotus
+
 # Execute the daemon subcommand by default
 CMD ["/sbin/tini", "--", "lotus", "daemon"]