cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge branch 'master' into feat/new-sb-fs

Browse Source
This commit is contained in:
Łukasz Magiera 2020-01-30 07:39:52 +01:00
commit 75ee71e35a
9 changed files with 101 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
Dockerfile
View File

@ -1,91 +0,0 @@
FROM golang:1.13.4-buster
MAINTAINER ldoublewood <ldoublewood@gmail.com>
ENV SRC_DIR /lotus
RUN apt-get update && apt-get install -y && apt-get install -y ca-certificates llvm clang mesa-opencl-icd ocl-icd-opencl-dev
RUN curl -sSf https://sh.rustup.rs | sh -s -- -y
# Get su-exec, a very minimal tool for dropping privileges,
# and tini, a very minimal init daemon for containers
ENV SUEXEC_VERSION v0.2
ENV TINI_VERSION v0.18.0
RUN set -x \
&& cd /tmp \
&& git clone https://github.com/ncopa/su-exec.git \
&& cd su-exec \
&& git checkout -q $SUEXEC_VERSION \
&& make \
&& cd /tmp \
&& wget -q -O tini https://github.com/krallin/tini/releases/download/$TINI_VERSION/tini \
&& chmod +x tini
# Download packages first so they can be cached.
COPY go.mod go.sum $SRC_DIR/
COPY extern/ $SRC_DIR/extern/
RUN cd $SRC_DIR \
&& go mod download
COPY Makefile $SRC_DIR
# Because extern/filecoin-ffi building script need to get version number from git
COPY .git/ $SRC_DIR/.git/
COPY .gitmodules $SRC_DIR/
# Download dependence first
RUN cd $SRC_DIR \
&& mkdir $SRC_DIR/build \
&& . $HOME/.cargo/env \
&& make clean \
&& make deps
COPY . $SRC_DIR
# Build the thing.
RUN cd $SRC_DIR \
&& . $HOME/.cargo/env \
&& make
# Now comes the actual target image, which aims to be as small as possible.
FROM busybox:1-glibc
MAINTAINER ldoublewood <ldoublewood@gmail.com>
# Get the executable binary and TLS CAs from the build container.
ENV SRC_DIR /lotus
COPY --from=0 $SRC_DIR/lotus /usr/local/bin/lotus
COPY --from=0 $SRC_DIR/lotus-storage-miner /usr/local/bin/lotus-storage-miner
COPY --from=0 /tmp/su-exec/su-exec /sbin/su-exec
COPY --from=0 /tmp/tini /sbin/tini
COPY --from=0 /etc/ssl/certs /etc/ssl/certs
# This shared lib (part of glibc) doesn't seem to be included with busybox.
COPY --from=0 /lib/x86_64-linux-gnu/libdl-2.28.so /lib/libdl.so.2
COPY --from=0 /lib/x86_64-linux-gnu/libutil-2.28.so /lib/libutil.so.1
COPY --from=0 /usr/lib/x86_64-linux-gnu/libOpenCL.so.1.0.0 /lib/libOpenCL.so.1
COPY --from=0 /lib/x86_64-linux-gnu/librt-2.28.so /lib/librt.so.1
COPY --from=0 /lib/x86_64-linux-gnu/libgcc_s.so.1 /lib/libgcc_s.so.1
# WS port
EXPOSE 1234
# P2P port
EXPOSE 5678
# Create the home directory and switch to a non-privileged user.
ENV HOME_PATH /data
ENV PARAMCACHE_PATH /var/tmp/filecoin-proof-parameters
RUN mkdir -p $HOME_PATH \
&& adduser -D -h $HOME_PATH -u 1000 -G users lotus \
&& chown lotus:users $HOME_PATH
VOLUME $HOME_PATH
VOLUME $PARAMCACHE_PATH
# Execute the daemon subcommand by default
CMD ["/sbin/tini", "--", "lotus", "daemon"]

6
documentation/en/.library.json
View File

@ -32,6 +32,12 @@
"github": "en/install-lotus-ubuntu.md",
"value": null
},
{
"title": "Fedora Installation",
"slug": "en+install-lotus-fedora",
"github": "en/install-lotus-fedora.md",
"value": null
},
{
"title": "MacOS Installation",
"slug": "en+install-lotus-macos",

6
documentation/en/storing-data.md
View File

@ -49,6 +49,12 @@ Store a **Data CID** with a miner:
lotus client deal <Data CID> <miner> <price> <duration>
```
Check the status of a deal:
```sh
lotus client list-deals
```
- Price is in attoFIL.
- The `duration`, which represents how long the miner will keep your file hosted, is represented in blocks. Each block represents 45 seconds.

2
storage/sealing/fsm.go
View File

@ -211,6 +211,8 @@ func planCommitting(events []statemachine.Event, state *SectorInfo) error {
state.State = api.SealCommitFailed
case SectorSealFailed:
state.State = api.CommitFailed
case SectorCommitFailed:
state.State = api.CommitFailed
default:
return xerrors.Errorf("planCommitting got event of unknown type %T, events: %+v", event.User, events)
}

14
storage/sealing/fsm_test.go
View File

@ -83,3 +83,17 @@ func TestSeedRevert(t *testing.T) {
m.planSingle(SectorProving{})
require.Equal(m.t, m.state.State, api.Proving)
}
func TestPlanCommittingHandlesSectorCommitFailed(t *testing.T) {
m := test{
s: &Sealing{},
t: t,
state: &SectorInfo{State: api.Committing},
}
events := []statemachine.Event{{SectorCommitFailed{}}}
require.NoError(t, planCommitting(events, m.state))
require.Equal(t, api.SectorStates[api.CommitFailed], api.SectorStates[m.state.State])
}

19
storage/sealing/garbage.go
View File

@ -6,6 +6,7 @@ import (
"io"
"math"
"math/rand"
"runtime"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
"golang.org/x/xerrors"
@ -14,6 +15,17 @@ import (
"github.com/filecoin-project/lotus/chain/types"
)
func (m *Sealing) pledgeReader(size uint64, parts uint64) io.Reader {
piece := sectorbuilder.UserBytesForSectorSize((size/127 + size) / parts)
readers := make([]io.Reader, parts)
for i := range readers {
readers[i] = io.LimitReader(rand.New(rand.NewSource(42+int64(i))), int64(piece))
}
return io.MultiReader(readers...)
}
func (m *Sealing) pledgeSector(ctx context.Context, sectorID uint64, existingPieceSizes []uint64, sizes ...uint64) ([]Piece, error) {
if len(sizes) == 0 {
return nil, nil
@ -21,10 +33,7 @@ func (m *Sealing) pledgeSector(ctx context.Context, sectorID uint64, existingPie
deals := make([]actors.StorageDealProposal, len(sizes))
for i, size := range sizes {
release := m.sb.RateLimit()
commP, err := sectorbuilder.GeneratePieceCommitment(io.LimitReader(rand.New(rand.NewSource(42)), int64(size)), size)
release()
commP, err := m.fastPledgeCommitment(size, uint64(runtime.NumCPU()))
if err != nil {
return nil, err
}
@ -81,7 +90,7 @@ func (m *Sealing) pledgeSector(ctx context.Context, sectorID uint64, existingPie
out := make([]Piece, len(sizes))
for i, size := range sizes {
ppi, err := m.sb.AddPiece(ctx, size, sectorID, io.LimitReader(rand.New(rand.NewSource(42)), int64(size)), existingPieceSizes)
ppi, err := m.sb.AddPiece(ctx, size, sectorID, m.pledgeReader(size, uint64(runtime.NumCPU())), existingPieceSizes)
if err != nil {
return nil, xerrors.Errorf("add piece: %w", err)
}

40
storage/sealing/utils.go
View File

@ -1,7 +1,12 @@
package sealing
import (
"io"
"math/bits"
"math/rand"
"sync"
"github.com/hashicorp/go-multierror"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
)
@ -42,6 +47,41 @@ func fillersFromRem(toFill uint64) ([]uint64, error) {
return out, nil
}
func (m *Sealing) fastPledgeCommitment(size uint64, parts uint64) (commP [sectorbuilder.CommLen]byte, err error) {
parts = 1 << bits.Len64(parts) // round down to nearest power of 2
piece := sectorbuilder.UserBytesForSectorSize(size / parts)
out := make([]sectorbuilder.PublicPieceInfo, parts)
var lk sync.Mutex
var wg sync.WaitGroup
wg.Add(int(parts))
for i := uint64(0); i < parts; i++ {
go func(i uint64) {
defer wg.Done()
commP, perr := sectorbuilder.GeneratePieceCommitment(io.LimitReader(rand.New(rand.NewSource(42+int64(i))), int64(piece)), piece)
lk.Lock()
if perr != nil {
err = multierror.Append(err, perr)
}
out[i] = sectorbuilder.PublicPieceInfo{
Size: piece,
CommP: commP,
}
lk.Unlock()
}(i)
}
wg.Wait()
if err != nil {
return [32]byte{}, err
}
return sectorbuilder.GenerateDataCommitment(m.sb.SectorSize(), out)
}
func (m *Sealing) ListSectors() ([]SectorInfo, error) {
var sectors []SectorInfo
if err := m.sectors.List(&sectors); err != nil {

11
storage/sealing/utils_test.go
View File

@ -1,6 +1,7 @@
package sealing
import (
"github.com/filecoin-project/lotus/storage/sbmock"
"testing"
"github.com/stretchr/testify/assert"
@ -42,5 +43,13 @@ func TestFillersFromRem(t *testing.T) {
ub = sectorbuilder.UserBytesForSectorSize(uint64(9) << i)
testFill(t, ub, []uint64{ub1, ub4})
}
}
func TestFastPledge(t *testing.T) {
sz := uint64(16 << 20)
s := Sealing{sb: sbmock.NewMockSectorBuilder(0, sz)}
if _, err := s.fastPledgeCommitment(sz, 5); err != nil {
t.Fatalf("%+v", err)
}
}

14
tools/dockers/docker-examples/basic-miner-busybox/Dockerfile
View File

@ -1,4 +1,4 @@
FROM golang:1.13.4-buster
FROM golang:1.13-buster
MAINTAINER ldoublewood <ldoublewood@gmail.com>
ENV SRC_DIR /lotus
@ -44,10 +44,12 @@ RUN cd $SRC_DIR \
COPY . $SRC_DIR
ARG MAKE_TARGET=all
# Build the thing.
RUN cd $SRC_DIR \
&& . $HOME/.cargo/env \
&& make
&& make $MAKE_TARGET
# Now comes the actual target image, which aims to be as small as possible.
FROM busybox:1-glibc
@ -56,7 +58,7 @@ MAINTAINER ldoublewood <ldoublewood@gmail.com>
# Get the executable binary and TLS CAs from the build container.
ENV SRC_DIR /lotus
COPY --from=0 $SRC_DIR/lotus /usr/local/bin/lotus
COPY --from=0 $SRC_DIR/lotus-storage-miner /usr/local/bin/lotus-storage-miner
COPY --from=0 $SRC_DIR/lotus-* /usr/local/bin/
COPY --from=0 /tmp/su-exec/su-exec /sbin/su-exec
COPY --from=0 /tmp/tini /sbin/tini
COPY --from=0 /etc/ssl/certs /etc/ssl/certs
@ -79,13 +81,15 @@ EXPOSE 5678
ENV HOME_PATH /data
ENV PARAMCACHE_PATH /var/tmp/filecoin-proof-parameters
RUN mkdir -p $HOME_PATH \
RUN mkdir -p $HOME_PATH $PARAMCACHE_PATH \
&& adduser -D -h $HOME_PATH -u 1000 -G users lotus \
&& chown lotus:users $HOME_PATH
&& chown lotus:users $HOME_PATH $PARAMCACHE_PATH
VOLUME $HOME_PATH
VOLUME $PARAMCACHE_PATH
USER lotus
# Execute the daemon subcommand by default
CMD ["/sbin/tini", "--", "lotus", "daemon"]