v1.27.0-a #10

Closed
jonathanface wants to merge 473 commits from v1.27.0-a into master
26 changed files with 238 additions and 54 deletions
Showing only changes of commit 4c2235a686 - Show all commits

View File

@ -1018,7 +1018,7 @@ workflows:
requires:
- build
suite: utest-unit-rest
target: "./blockstore/... ./build/... ./chain/... ./conformance/... ./gateway/... ./journal/... ./lib/... ./markets/... ./paychmgr/... ./tools/..."
target: "./blockstore/... ./build/... ./chain/... ./conformance/... ./gateway/... ./journal/... ./lib/... ./markets/... ./paychmgr/... ./provider/... ./tools/..."
resource_class: 2xlarge
- test:
name: test-unit-storage

View File

@ -2,16 +2,19 @@ package main
import (
"fmt"
"github.com/urfave/cli/v2"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/types"
lcli "github.com/filecoin-project/lotus/cli"
"github.com/filecoin-project/lotus/cmd/lotus-provider/deps"
"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
"github.com/filecoin-project/lotus/provider/lpseal"
"github.com/urfave/cli/v2"
"golang.org/x/xerrors"
)
var pipelineCmd = &cli.Command{

View File

@ -3,9 +3,6 @@ package tasks
import (
"context"
"github.com/filecoin-project/lotus/provider/chainsched"
"github.com/filecoin-project/lotus/provider/lpffi"
"github.com/filecoin-project/lotus/provider/lpseal"
logging "github.com/ipfs/go-log/v2"
"github.com/samber/lo"
@ -13,7 +10,10 @@ import (
"github.com/filecoin-project/lotus/cmd/lotus-provider/deps"
"github.com/filecoin-project/lotus/lib/harmony/harmonytask"
"github.com/filecoin-project/lotus/provider"
"github.com/filecoin-project/lotus/provider/chainsched"
"github.com/filecoin-project/lotus/provider/lpffi"
"github.com/filecoin-project/lotus/provider/lpmessage"
"github.com/filecoin-project/lotus/provider/lpseal"
"github.com/filecoin-project/lotus/provider/lpwinning"
)

View File

@ -15,6 +15,7 @@ COMMANDS:
config Manage node config by layers. The layer 'base' will always be applied.
test Utility functions for testing
web Start lotus provider web interface
pipeline Manage the sealing pipeline
version Print version
help, h Shows a list of commands or help for one command
DEVELOPER:
@ -267,6 +268,40 @@ OPTIONS:
--help, -h show help
```
## lotus-provider pipeline
```
NAME:
lotus-provider pipeline - Manage the sealing pipeline
USAGE:
lotus-provider pipeline command [command options] [arguments...]
COMMANDS:
start Start new sealing operations manually
help, h Shows a list of commands or help for one command
OPTIONS:
--help, -h show help
```
### lotus-provider pipeline start
```
NAME:
lotus-provider pipeline start - Start new sealing operations manually
USAGE:
lotus-provider pipeline start [command options] [arguments...]
OPTIONS:
--actor value Specify actor address to start sealing sectors for
--now Start sealing sectors for all actors now (not on schedule) (default: false)
--cc Start sealing new CC sectors (default: false)
--count value Number of sectors to start (default: 1)
--synthetic Use synthetic PoRep (default: false)
--layers value [ --layers value ] list of layers to be interpreted (atop defaults). Default: base (default: "base")
--help, -h show help
```
## lotus-provider version
```
NAME:

View File

@ -11,6 +11,48 @@
# type: int
#WinningPostMaxTasks = 0
# EnableSealSDR enables SDR tasks to run. SDR is the long sequential computation
# creating layers.
# In lotus-miner this was run as part of PreCommit1.
#
# type: bool
#EnableSealSDR = false
# type: int
#SealSDRMaxTasks = 0
# EnableSealSDRTrees enables the SDR pipeline tree-building task to run.
# This task handles encoding of unsealed data into last sdr layer and building
# of TreeR, TreeC and TreeD.
# In lotus-miner this was run as part of PreCommit2 (TreeD was run in PreCommit1).
#
# type: bool
#EnableSealSDRTrees = false
# type: int
#SealSDRTreesMaxTasks = 0
# EnableSendPrecommitMsg enables the sending of precommit messages to the chain
# from this lotus-provider instance.
#
# type: bool
#EnableSendPrecommitMsg = false
# EnableSendCommitMsg enables the computation of the porep proof
# In lotus-miner this was Commit1 / Commit2
#
# type: bool
#EnablePoRepProof = false
# type: int
#PoRepProofMaxTasks = 0
# EnableSendCommitMsg enables the sending of commit messages to the chain
# from this lotus-provider instance.
#
# type: bool
#EnableSendCommitMsg = false
# type: bool
#EnableWebGui = false

4
go.mod
View File

@ -117,6 +117,7 @@ require (
github.com/mattn/go-isatty v0.0.19
github.com/mattn/go-sqlite3 v1.14.16
github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1
github.com/minio/sha256-simd v1.0.1
github.com/mitchellh/go-homedir v1.1.0
github.com/multiformats/go-base32 v0.1.0
github.com/multiformats/go-multiaddr v0.12.1
@ -126,7 +127,6 @@ require (
github.com/multiformats/go-multihash v0.2.3
github.com/multiformats/go-varint v0.0.7
github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58
github.com/pkg/errors v0.9.1
github.com/polydawn/refmt v0.89.0
github.com/prometheus/client_golang v1.16.0
@ -276,7 +276,6 @@ require (
github.com/miekg/dns v1.1.55 // indirect
github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect
github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect
github.com/minio/sha256-simd v1.0.1 // indirect
github.com/mr-tron/base58 v1.2.0 // indirect
github.com/multiformats/go-base36 v0.2.0 // indirect
github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect
@ -286,6 +285,7 @@ require (
github.com/onsi/ginkgo/v2 v2.11.0 // indirect
github.com/opencontainers/runtime-spec v1.1.0 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_model v0.4.0 // indirect

View File

@ -3,13 +3,13 @@ package resources
import (
"bytes"
"context"
"github.com/elastic/go-sysinfo"
"os/exec"
"regexp"
"runtime"
"sync/atomic"
"time"
"github.com/elastic/go-sysinfo"
logging "github.com/ipfs/go-log/v2"
"golang.org/x/sys/unix"
"golang.org/x/xerrors"

View File

@ -1007,6 +1007,62 @@ block rewards will be missed!`,
Comment: ``,
},
{
Name: "EnableSealSDR",
Type: "bool",
Comment: `EnableSealSDR enables SDR tasks to run. SDR is the long sequential computation
creating layers.
In lotus-miner this was run as part of PreCommit1.`,
},
{
Name: "SealSDRMaxTasks",
Type: "int",
Comment: ``,
},
{
Name: "EnableSealSDRTrees",
Type: "bool",
Comment: `EnableSealSDRTrees enables the SDR pipeline tree-building task to run.
This task handles encoding of unsealed data into last sdr layer and building
of TreeR, TreeC and TreeD.
In lotus-miner this was run as part of PreCommit2 (TreeD was run in PreCommit1).`,
},
{
Name: "SealSDRTreesMaxTasks",
Type: "int",
Comment: ``,
},
{
Name: "EnableSendPrecommitMsg",
Type: "bool",
Comment: `EnableSendPrecommitMsg enables the sending of precommit messages to the chain
from this lotus-provider instance.`,
},
{
Name: "EnablePoRepProof",
Type: "bool",
Comment: `EnableSendCommitMsg enables the computation of the porep proof
In lotus-miner this was Commit1 / Commit2`,
},
{
Name: "PoRepProofMaxTasks",
Type: "int",
Comment: ``,
},
{
Name: "EnableSendCommitMsg",
Type: "bool",
Comment: `EnableSendCommitMsg enables the sending of commit messages to the chain
from this lotus-provider instance.`,
},
{
Name: "EnableWebGui",
Type: "bool",

View File

@ -3,12 +3,12 @@ package config
import (
"encoding/json"
"errors"
"github.com/mitchellh/go-homedir"
"io"
"io/fs"
"os"
"path"
"github.com/mitchellh/go-homedir"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/storage/sealer/storiface"

View File

@ -98,10 +98,6 @@ type ProviderSubsystemsConfig struct {
EnableWinningPost bool
WinningPostMaxTasks int
EnableWebGui bool
// The address that should listen for Web GUI requests.
GuiAddress string
// EnableSealSDR enables SDR tasks to run. SDR is the long sequential computation
// creating layers.
// In lotus-miner this was run as part of PreCommit1.
@ -127,6 +123,10 @@ type ProviderSubsystemsConfig struct {
// EnableSendCommitMsg enables the sending of commit messages to the chain
// from this lotus-provider instance.
EnableSendCommitMsg bool
EnableWebGui bool
// The address that should listen for Web GUI requests.
GuiAddress string
}
type DAGStoreConfig struct {

View File

@ -4,20 +4,23 @@ import (
"context"
"encoding/json"
"fmt"
"io"
"os"
"path/filepath"
"github.com/ipfs/go-cid"
logging "github.com/ipfs/go-log/v2"
"golang.org/x/xerrors"
ffi "github.com/filecoin-project/filecoin-ffi"
commcid "github.com/filecoin-project/go-fil-commcid"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/provider/lpproof"
"github.com/filecoin-project/lotus/storage/paths"
"github.com/filecoin-project/lotus/storage/pipeline/lib/nullreader"
"github.com/filecoin-project/lotus/storage/sealer/proofpaths"
"github.com/filecoin-project/lotus/storage/sealer/storiface"
"github.com/ipfs/go-cid"
logging "github.com/ipfs/go-log/v2"
"golang.org/x/xerrors"
"io"
"os"
"path/filepath"
)
var log = logging.Logger("lpffi")

View File

@ -3,15 +3,18 @@ package lpmessage
import (
"context"
"encoding/json"
"sync/atomic"
"github.com/ipfs/go-cid"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
"github.com/filecoin-project/lotus/lib/harmony/harmonytask"
"github.com/filecoin-project/lotus/provider/chainsched"
"github.com/ipfs/go-cid"
"sync/atomic"
)
const MinConfidence = 6

View File

@ -1,19 +1,21 @@
package lpproof
import (
commcid "github.com/filecoin-project/go-fil-commcid"
"github.com/filecoin-project/go-state-types/abi"
"github.com/hashicorp/go-multierror"
"github.com/ipfs/go-cid"
pool "github.com/libp2p/go-buffer-pool"
"github.com/minio/sha256-simd"
"golang.org/x/xerrors"
"io"
"math/bits"
"os"
"runtime"
"sync"
"time"
"github.com/hashicorp/go-multierror"
"github.com/ipfs/go-cid"
pool "github.com/libp2p/go-buffer-pool"
"github.com/minio/sha256-simd"
"golang.org/x/xerrors"
commcid "github.com/filecoin-project/go-fil-commcid"
"github.com/filecoin-project/go-state-types/abi"
)
const nodeSize = 32

View File

@ -4,14 +4,16 @@ import (
"bytes"
"crypto/rand"
"fmt"
"github.com/filecoin-project/go-state-types/abi"
pool "github.com/libp2p/go-buffer-pool"
"github.com/stretchr/testify/require"
"io/ioutil"
"os"
"path/filepath"
"runtime"
"testing"
pool "github.com/libp2p/go-buffer-pool"
"github.com/stretchr/testify/require"
"github.com/filecoin-project/go-state-types/abi"
)
func TestTreeSize(t *testing.T) {

View File

@ -2,17 +2,20 @@ package lpseal
import (
"context"
"time"
logging "github.com/ipfs/go-log/v2"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/actors/policy"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
"github.com/filecoin-project/lotus/lib/harmony/harmonytask"
"github.com/filecoin-project/lotus/lib/promise"
logging "github.com/ipfs/go-log/v2"
"golang.org/x/xerrors"
"time"
)
var log = logging.Logger("lpseal")

View File

@ -2,13 +2,16 @@ package lpseal
import (
"context"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-bitfield"
rlepluslazy "github.com/filecoin-project/go-bitfield/rle"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
"golang.org/x/xerrors"
)
type AllocAPI interface {

View File

@ -3,17 +3,20 @@ package lpseal
import (
"bytes"
"context"
"github.com/ipfs/go-cid"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/crypto"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
"github.com/filecoin-project/lotus/lib/harmony/harmonytask"
"github.com/filecoin-project/lotus/lib/harmony/resources"
"github.com/filecoin-project/lotus/provider/lpffi"
"github.com/filecoin-project/lotus/storage/sealer/storiface"
"github.com/ipfs/go-cid"
"golang.org/x/xerrors"
)
type PoRepAPI interface {

View File

@ -3,10 +3,14 @@ package lpseal
import (
"bytes"
"context"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-commp-utils/zerocomm"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/crypto"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors/policy"
"github.com/filecoin-project/lotus/chain/types"
@ -15,7 +19,6 @@ import (
"github.com/filecoin-project/lotus/lib/harmony/resources"
"github.com/filecoin-project/lotus/provider/lpffi"
"github.com/filecoin-project/lotus/storage/sealer/storiface"
"golang.org/x/xerrors"
)
var isDevnet = build.BlockDelaySecs < 30

View File

@ -3,10 +3,14 @@ package lpseal
import (
"bytes"
"context"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/go-state-types/builtin"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/types"
@ -16,7 +20,6 @@ import (
"github.com/filecoin-project/lotus/provider/lpmessage"
"github.com/filecoin-project/lotus/storage/ctladdr"
sealing "github.com/filecoin-project/lotus/storage/pipeline"
"golang.org/x/xerrors"
)
type SubmitCommitAPI interface {

View File

@ -3,11 +3,16 @@ package lpseal
import (
"bytes"
"context"
"github.com/ipfs/go-cid"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/go-state-types/builtin"
miner12 "github.com/filecoin-project/go-state-types/builtin/v12/miner"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/types"
@ -17,8 +22,6 @@ import (
"github.com/filecoin-project/lotus/provider/lpmessage"
"github.com/filecoin-project/lotus/storage/ctladdr"
sealing "github.com/filecoin-project/lotus/storage/pipeline"
"github.com/ipfs/go-cid"
"golang.org/x/xerrors"
)
type SubmitPrecommitTaskApi interface {

View File

@ -2,15 +2,18 @@ package lpseal
import (
"context"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-commp-utils/zerocomm"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
"github.com/filecoin-project/lotus/lib/harmony/harmonytask"
"github.com/filecoin-project/lotus/lib/harmony/resources"
"github.com/filecoin-project/lotus/provider/lpffi"
"github.com/filecoin-project/lotus/storage/pipeline/lib/nullreader"
"github.com/filecoin-project/lotus/storage/sealer/storiface"
"golang.org/x/xerrors"
)
type TreesTask struct {

View File

@ -3,7 +3,6 @@ package paths
import (
"bytes"
"encoding/json"
"github.com/ipfs/go-cid"
"io"
"net/http"
"os"
@ -11,6 +10,7 @@ import (
"time"
"github.com/gorilla/mux"
"github.com/ipfs/go-cid"
logging "github.com/ipfs/go-log/v2"
"golang.org/x/xerrors"

View File

@ -2,9 +2,10 @@ package paths
import (
"context"
"github.com/ipfs/go-cid"
"io"
"github.com/ipfs/go-cid"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/storage/sealer/fsutil"

View File

@ -3,7 +3,6 @@ package paths
import (
"context"
"encoding/json"
"github.com/ipfs/go-cid"
"math/bits"
"math/rand"
"os"
@ -11,6 +10,7 @@ import (
"sync"
"time"
"github.com/ipfs/go-cid"
"golang.org/x/xerrors"
ffi "github.com/filecoin-project/filecoin-ffi"

View File

@ -9,6 +9,7 @@ import (
reflect "reflect"
gomock "github.com/golang/mock/gomock"
cid "github.com/ipfs/go-cid"
abi "github.com/filecoin-project/go-state-types/abi"
@ -85,6 +86,21 @@ func (mr *MockStoreMockRecorder) GenerateSingleVanillaProof(arg0, arg1, arg2, ar
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GenerateSingleVanillaProof", reflect.TypeOf((*MockStore)(nil).GenerateSingleVanillaProof), arg0, arg1, arg2, arg3)
}
// GenetartePoRepVanillaProof mocks base method.
func (m *MockStore) GenetartePoRepVanillaProof(arg0 context.Context, arg1 storiface.SectorRef, arg2, arg3 cid.Cid, arg4 abi.SealRandomness, arg5 abi.InteractiveSealRandomness) ([]byte, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "GenetartePoRepVanillaProof", arg0, arg1, arg2, arg3, arg4, arg5)
ret0, _ := ret[0].([]byte)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// GenetartePoRepVanillaProof indicates an expected call of GenetartePoRepVanillaProof.
func (mr *MockStoreMockRecorder) GenetartePoRepVanillaProof(arg0, arg1, arg2, arg3, arg4, arg5 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GenetartePoRepVanillaProof", reflect.TypeOf((*MockStore)(nil).GenetartePoRepVanillaProof), arg0, arg1, arg2, arg3, arg4, arg5)
}
// MoveStorage mocks base method.
func (m *MockStore) MoveStorage(arg0 context.Context, arg1 storiface.SectorRef, arg2 storiface.SectorFileType) error {
m.ctrl.T.Helper()

View File

@ -4,7 +4,6 @@ import (
"context"
"encoding/json"
"fmt"
"github.com/ipfs/go-cid"
"io"
"math/bits"
"net/http"
@ -18,6 +17,7 @@ import (
"time"
"github.com/hashicorp/go-multierror"
"github.com/ipfs/go-cid"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-state-types/abi"