Merge branch 'master' into feat/event-states
This commit is contained in:
commit
8a9707788c
2
Makefile
2
Makefile
@ -76,6 +76,7 @@ BINS+=lotus-seal-worker
|
||||
lotus-shed: $(BUILD_DEPS)
|
||||
rm -f lotus-shed
|
||||
go build $(GOFLAGS) -o lotus-shed ./cmd/lotus-shed
|
||||
go run github.com/GeertJohan/go.rice/rice append --exec lotus-shed -i ./build
|
||||
.PHONY: lotus-seal-worker
|
||||
BINS+=lotus-seal-worker
|
||||
|
||||
@ -144,6 +145,7 @@ BINS+=bench
|
||||
stats:
|
||||
rm -f stats
|
||||
go build -o stats ./tools/stats
|
||||
go run github.com/GeertJohan/go.rice/rice append --exec stats -i ./build
|
||||
.PHONY: stats
|
||||
BINS+=stats
|
||||
|
||||
|
@ -13,7 +13,8 @@ type SectorState = uint64
|
||||
const (
|
||||
UndefinedSectorState SectorState = iota
|
||||
|
||||
Empty // TODO: Is this useful
|
||||
// happy path
|
||||
Empty
|
||||
Packing // sector not in sealStore, and not on chain
|
||||
|
||||
Unsealed // sealing / queued
|
||||
@ -22,13 +23,32 @@ const (
|
||||
Committing
|
||||
CommitWait // waiting for message to land on chain
|
||||
Proving
|
||||
_ // reserved
|
||||
_
|
||||
_
|
||||
_
|
||||
|
||||
// recovery handling
|
||||
// Reseal
|
||||
_
|
||||
_
|
||||
_
|
||||
_
|
||||
_
|
||||
_
|
||||
_
|
||||
|
||||
// error modes
|
||||
FailedUnrecoverable
|
||||
|
||||
SealFailed
|
||||
PreCommitFailed
|
||||
SealCommitFailed
|
||||
CommitFailed
|
||||
|
||||
FailedUnrecoverable
|
||||
_
|
||||
_
|
||||
_
|
||||
_
|
||||
|
||||
Faulty // sector is corrupted or gone for some reason
|
||||
FaultReported // sector has been declared as a fault on chain
|
||||
|
@ -24,7 +24,7 @@ func init() {
|
||||
build.InsecurePoStValidation = true
|
||||
}
|
||||
|
||||
func TestDealFlow(t *testing.T, b APIBuilder) {
|
||||
func TestDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration) {
|
||||
os.Setenv("BELLMAN_NO_GPU", "1")
|
||||
|
||||
ctx := context.Background()
|
||||
@ -64,7 +64,7 @@ func TestDealFlow(t *testing.T, b APIBuilder) {
|
||||
go func() {
|
||||
defer close(done)
|
||||
for mine {
|
||||
time.Sleep(time.Second)
|
||||
time.Sleep(blocktime)
|
||||
fmt.Println("mining a block now")
|
||||
if err := sn[0].MineOne(ctx); err != nil {
|
||||
t.Error(err)
|
||||
|
6
build/bootstrap/bootstrappers.pi
Normal file
6
build/bootstrap/bootstrappers.pi
Normal file
@ -0,0 +1,6 @@
|
||||
/dns4/lotus-bootstrap-0.sin.fil-test.net/tcp/1347/p2p/12D3KooWLZs8BWtEzRTYET4yR4jzDtPamaA1YsyPQJq6cf2RfxBD
|
||||
/dns4/lotus-bootstrap-1.sin.fil-test.net/tcp/1347/p2p/12D3KooWGvrgjWw4Yqo4AFWqYp4g37FpUvUCQBkNWudZVSwR9tY1
|
||||
/dns4/lotus-bootstrap-0.fra.fil-test.net/tcp/1347/p2p/12D3KooWSfNcrD1cs5Cj5eSHbK6nHCqJLffAuPqvRMBRgvUdqQhX
|
||||
/dns4/lotus-bootstrap-1.fra.fil-test.net/tcp/1347/p2p/12D3KooWNkXyVPspUnrHUiSC3VJPMcXvHuNdy3BTCLTPPnDgwwTT
|
||||
/dns4/lotus-bootstrap-0.dfw.fil-test.net/tcp/1347/p2p/12D3KooWSgJWJZK8LTRtCWzPa5FQheCFJjHpficVYgEQWeimcqCu
|
||||
/dns4/lotus-bootstrap-1.dfw.fil-test.net/tcp/1347/p2p/12D3KooWFPaC4dyGpbNXCpVHjZucdJnDwmv4ng9tponPx5GrzJkT
|
BIN
build/genesis/devnet.car
Normal file
BIN
build/genesis/devnet.car
Normal file
Binary file not shown.
@ -3,7 +3,6 @@
|
||||
package build
|
||||
|
||||
var SectorSizes = []uint64{
|
||||
1 << 30,
|
||||
32 << 30,
|
||||
}
|
||||
|
||||
|
@ -5,7 +5,7 @@ import "fmt"
|
||||
var CurrentCommit string
|
||||
|
||||
// BuildVersion is the local build version, set by build system
|
||||
const BuildVersion = "0.2.0"
|
||||
const BuildVersion = "0.2.1"
|
||||
|
||||
var UserVersion = BuildVersion + CurrentCommit
|
||||
|
||||
|
@ -19,7 +19,6 @@ import (
|
||||
cbor "github.com/ipfs/go-ipld-cbor"
|
||||
"github.com/libp2p/go-libp2p-core/peer"
|
||||
cbg "github.com/whyrusleeping/cbor-gen"
|
||||
"go.opencensus.io/trace"
|
||||
"golang.org/x/xerrors"
|
||||
)
|
||||
|
||||
@ -526,7 +525,7 @@ func (sma StorageMinerActor) SubmitFallbackPoSt(act *types.Actor, vmctx types.VM
|
||||
})
|
||||
}
|
||||
|
||||
if ok, lerr := sectorbuilder.VerifyFallbackPost(vmctx.Context(), mi.SectorSize,
|
||||
if ok, lerr := vmctx.Sys().VerifyFallbackPost(vmctx.Context(), mi.SectorSize,
|
||||
sectorbuilder.NewSortedPublicSectorInfo(sectorInfos), seed[:], params.Proof, candidates, proverID, activeFaults); !ok || lerr != nil {
|
||||
if lerr != nil {
|
||||
// TODO: study PoST errors
|
||||
@ -641,17 +640,6 @@ func RemoveFromSectorSet(ctx context.Context, s types.Storage, ss cid.Cid, ids [
|
||||
return ncid, nil
|
||||
}
|
||||
|
||||
func ValidatePoRep(ctx context.Context, maddr address.Address, ssize uint64, commD, commR, ticket, proof, seed []byte, sectorID uint64) (bool, ActorError) {
|
||||
_, span := trace.StartSpan(ctx, "ValidatePoRep")
|
||||
defer span.End()
|
||||
ok, err := sectorbuilder.VerifySeal(ssize, commR, commD, maddr, ticket, seed, sectorID, proof)
|
||||
if err != nil {
|
||||
return false, aerrors.Absorb(err, 25, "verify seal failed")
|
||||
}
|
||||
|
||||
return ok, nil
|
||||
}
|
||||
|
||||
func CollateralForPower(power types.BigInt) types.BigInt {
|
||||
return types.BigMul(power, types.NewInt(10))
|
||||
/* TODO: this
|
||||
|
@ -262,7 +262,6 @@ func (h *Harness) makeFakeDeal(t *testing.T, miner, worker, client address.Addre
|
||||
prop := actors.StorageDealProposal{
|
||||
PieceRef: commP[:],
|
||||
PieceSize: size,
|
||||
//PieceSerialization SerializationMode // Needs to be here as it tells how data in the sector maps to PieceRef cid
|
||||
|
||||
Client: client,
|
||||
Provider: miner,
|
||||
|
@ -130,6 +130,10 @@ type ArbitrateConsensusFaultParams struct {
|
||||
}
|
||||
|
||||
func (spa StoragePowerActor) ArbitrateConsensusFault(act *types.Actor, vmctx types.VMContext, params *ArbitrateConsensusFaultParams) ([]byte, ActorError) {
|
||||
if params == nil || params.Block1 == nil || params.Block2 == nil {
|
||||
return nil, aerrors.New(1, "failed to parse params")
|
||||
}
|
||||
|
||||
if params.Block1.Miner != params.Block2.Miner {
|
||||
return nil, aerrors.New(2, "blocks must be from the same miner")
|
||||
}
|
||||
|
@ -50,10 +50,10 @@ func setupVMTestEnv(t *testing.T) (*vm.VM, []address.Address, bstore.Blockstore)
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
cs := store.NewChainStore(bs, nil)
|
||||
cs := store.NewChainStore(bs, nil, nil)
|
||||
|
||||
// TODO: should probabaly mock out the randomness bit, nil works for now
|
||||
vm, err := vm.NewVM(stateroot, 1, nil, maddr, cs.Blockstore())
|
||||
vm, err := vm.NewVM(stateroot, 1, nil, maddr, cs.Blockstore(), cs.VMSys())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
@ -3,6 +3,7 @@ package actors_test
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"github.com/filecoin-project/go-sectorbuilder"
|
||||
"math/rand"
|
||||
"testing"
|
||||
|
||||
@ -194,8 +195,8 @@ func NewHarness(t *testing.T, options ...HarnessOpt) *Harness {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
h.cs = store.NewChainStore(h.bs, nil)
|
||||
h.vm, err = vm.NewVM(stateroot, 1, h.Rand, h.HI.Miner, h.cs.Blockstore())
|
||||
h.cs = store.NewChainStore(h.bs, nil, vm.Syscalls(sectorbuilder.ProofVerifier))
|
||||
h.vm, err = vm.NewVM(stateroot, 1, h.Rand, h.HI.Miner, h.cs.Blockstore(), h.cs.VMSys())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
@ -6,6 +6,7 @@ import (
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"github.com/filecoin-project/lotus/chain/vm"
|
||||
"io/ioutil"
|
||||
"sync/atomic"
|
||||
|
||||
@ -170,7 +171,9 @@ func NewGenerator() (*ChainGen, error) {
|
||||
MinerAddrs: []address.Address{maddr1, maddr2},
|
||||
}
|
||||
|
||||
genb, err := MakeGenesisBlock(bs, map[address.Address]types.BigInt{
|
||||
sys := vm.Syscalls(sectorbuilder.ProofVerifier)
|
||||
|
||||
genb, err := MakeGenesisBlock(bs, sys, map[address.Address]types.BigInt{
|
||||
mk1: types.FromFil(40000),
|
||||
mk2: types.FromFil(40000),
|
||||
banker: types.FromFil(50000),
|
||||
@ -179,7 +182,7 @@ func NewGenerator() (*ChainGen, error) {
|
||||
return nil, xerrors.Errorf("make genesis block failed: %w", err)
|
||||
}
|
||||
|
||||
cs := store.NewChainStore(bs, ds)
|
||||
cs := store.NewChainStore(bs, ds, sys)
|
||||
|
||||
genfb := &types.FullBlock{Header: genb.Genesis}
|
||||
gents := store.NewFullTipSet([]*types.FullBlock{genfb})
|
||||
|
@ -275,7 +275,7 @@ func mustEnc(i cbg.CBORMarshaler) []byte {
|
||||
}
|
||||
|
||||
func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid, gmcfg *GenMinerCfg) (cid.Cid, []actors.StorageDealProposal, error) {
|
||||
vm, err := vm.NewVM(sroot, 0, nil, actors.NetworkAddress, cs.Blockstore())
|
||||
vm, err := vm.NewVM(sroot, 0, nil, actors.NetworkAddress, cs.Blockstore(), cs.VMSys())
|
||||
if err != nil {
|
||||
return cid.Undef, nil, xerrors.Errorf("failed to create NewVM: %w", err)
|
||||
}
|
||||
@ -555,7 +555,7 @@ func doExecValue(ctx context.Context, vm *vm.VM, to, from address.Address, value
|
||||
return ret.Return, nil
|
||||
}
|
||||
|
||||
func MakeGenesisBlock(bs bstore.Blockstore, balances map[address.Address]types.BigInt, gmcfg *GenMinerCfg, ts uint64) (*GenesisBootstrap, error) {
|
||||
func MakeGenesisBlock(bs bstore.Blockstore, sys *types.VMSyscalls, balances map[address.Address]types.BigInt, gmcfg *GenMinerCfg, ts uint64) (*GenesisBootstrap, error) {
|
||||
ctx := context.Background()
|
||||
|
||||
state, err := MakeInitialStateTree(bs, balances)
|
||||
@ -569,7 +569,7 @@ func MakeGenesisBlock(bs bstore.Blockstore, balances map[address.Address]types.B
|
||||
}
|
||||
|
||||
// temp chainstore
|
||||
cs := store.NewChainStore(bs, datastore.NewMapDatastore())
|
||||
cs := store.NewChainStore(bs, datastore.NewMapDatastore(), sys)
|
||||
stateroot, deals, err := SetupStorageMiners(ctx, cs, stateroot, gmcfg)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("setup storage miners failed: %w", err)
|
||||
|
@ -18,7 +18,7 @@ func (sm *StateManager) CallRaw(ctx context.Context, msg *types.Message, bstate
|
||||
ctx, span := trace.StartSpan(ctx, "statemanager.CallRaw")
|
||||
defer span.End()
|
||||
|
||||
vmi, err := vm.NewVM(bstate, bheight, r, actors.NetworkAddress, sm.cs.Blockstore())
|
||||
vmi, err := vm.NewVM(bstate, bheight, r, actors.NetworkAddress, sm.cs.Blockstore(), sm.cs.VMSys())
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("failed to set up vm: %w", err)
|
||||
}
|
||||
|
@ -139,7 +139,7 @@ func (sm *StateManager) computeTipSetState(ctx context.Context, blks []*types.Bl
|
||||
|
||||
r := store.NewChainRand(sm.cs, cids, blks[0].Height)
|
||||
|
||||
vmi, err := vm.NewVM(pstate, blks[0].Height, r, address.Undef, sm.cs.Blockstore())
|
||||
vmi, err := vm.NewVM(pstate, blks[0].Height, r, address.Undef, sm.cs.Blockstore(), sm.cs.VMSys())
|
||||
if err != nil {
|
||||
return cid.Undef, cid.Undef, xerrors.Errorf("instantiating VM failed: %w", err)
|
||||
}
|
||||
|
@ -51,9 +51,11 @@ type ChainStore struct {
|
||||
|
||||
mmCache *lru.ARCCache
|
||||
tsCache *lru.ARCCache
|
||||
|
||||
vmcalls *types.VMSyscalls
|
||||
}
|
||||
|
||||
func NewChainStore(bs bstore.Blockstore, ds dstore.Batching) *ChainStore {
|
||||
func NewChainStore(bs bstore.Blockstore, ds dstore.Batching, vmcalls *types.VMSyscalls) *ChainStore {
|
||||
c, _ := lru.NewARC(2048)
|
||||
tsc, _ := lru.NewARC(4096)
|
||||
cs := &ChainStore{
|
||||
@ -63,6 +65,7 @@ func NewChainStore(bs bstore.Blockstore, ds dstore.Batching) *ChainStore {
|
||||
tipsets: make(map[uint64][]cid.Cid),
|
||||
mmCache: c,
|
||||
tsCache: tsc,
|
||||
vmcalls: vmcalls,
|
||||
}
|
||||
|
||||
cs.reorgCh = cs.reorgWorker(context.TODO())
|
||||
@ -793,6 +796,10 @@ func (cs *ChainStore) Blockstore() bstore.Blockstore {
|
||||
return cs.bs
|
||||
}
|
||||
|
||||
func (cs *ChainStore) VMSys() *types.VMSyscalls {
|
||||
return cs.vmcalls
|
||||
}
|
||||
|
||||
func (cs *ChainStore) TryFillTipSet(ts *types.TipSet) (*FullTipSet, error) {
|
||||
var out []*types.FullBlock
|
||||
|
||||
|
@ -55,7 +55,7 @@ func BenchmarkGetRandomness(b *testing.B) {
|
||||
|
||||
bs := blockstore.NewBlockstore(bds)
|
||||
|
||||
cs := store.NewChainStore(bs, mds)
|
||||
cs := store.NewChainStore(bs, mds, nil)
|
||||
|
||||
b.ResetTimer()
|
||||
|
||||
|
@ -60,7 +60,7 @@ func (cs *ChainStore) call(ctx context.Context, msg *types.Message, ts *types.Ti
|
||||
|
||||
r := NewChainRand(cs, ts.Cids(), ts.Height())
|
||||
|
||||
vmi, err := vm.NewVM(bstate, ts.Height(), r, actors.NetworkAddress, cs.bs)
|
||||
vmi, err := vm.NewVM(bstate, ts.Height(), r, actors.NetworkAddress, cs.bs, cs.vmcalls)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("failed to set up vm: %w", err)
|
||||
}
|
||||
|
@ -690,7 +690,7 @@ func (syncer *Syncer) VerifyElectionPoStProof(ctx context.Context, h *types.Bloc
|
||||
}
|
||||
hvrf := sha256.Sum256(h.EPostProof.PostRand)
|
||||
|
||||
ok, err := sectorbuilder.VerifyElectionPost(ctx, ssize, *sectorInfo, hvrf[:], h.EPostProof.Proof, winners, h.Miner)
|
||||
ok, err := sectorbuilder.ProofVerifier.VerifyElectionPost(ctx, ssize, *sectorInfo, hvrf[:], h.EPostProof.Proof, winners, h.Miner)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("failed to verify election post: %w", err)
|
||||
}
|
||||
|
@ -2,6 +2,7 @@ package types
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/filecoin-project/go-sectorbuilder"
|
||||
|
||||
"github.com/filecoin-project/go-address"
|
||||
"github.com/filecoin-project/go-amt-ipld"
|
||||
@ -46,7 +47,15 @@ type VMContext interface {
|
||||
}
|
||||
|
||||
type VMSyscalls struct {
|
||||
ValidatePoRep func(context.Context, address.Address, uint64, []byte, []byte, []byte, []byte, []byte, uint64) (bool, aerrors.ActorError)
|
||||
ValidatePoRep func(context.Context, address.Address, uint64, []byte, []byte, []byte, []byte, []byte, uint64) (bool, aerrors.ActorError)
|
||||
VerifyFallbackPost func(ctx context.Context,
|
||||
sectorSize uint64,
|
||||
sectorInfo sectorbuilder.SortedPublicSectorInfo,
|
||||
challengeSeed []byte,
|
||||
proof []byte,
|
||||
candidates []sectorbuilder.EPostCandidate,
|
||||
proverID address.Address,
|
||||
faults uint64) (bool, error)
|
||||
}
|
||||
|
||||
type storageWrapper struct {
|
||||
|
@ -2,6 +2,7 @@ package validation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/filecoin-project/go-sectorbuilder"
|
||||
|
||||
vchain "github.com/filecoin-project/chain-validation/pkg/chain"
|
||||
vstate "github.com/filecoin-project/chain-validation/pkg/state"
|
||||
@ -32,7 +33,7 @@ func (a *Applier) ApplyMessage(eCtx *vchain.ExecutionContext, state vstate.Wrapp
|
||||
if err != nil {
|
||||
return vchain.MessageReceipt{}, err
|
||||
}
|
||||
lotusVM, err := vm.NewVM(base, eCtx.Epoch, randSrc, minerAddr, st.bs)
|
||||
lotusVM, err := vm.NewVM(base, eCtx.Epoch, randSrc, minerAddr, st.bs, vm.Syscalls(sectorbuilder.ProofVerifier))
|
||||
if err != nil {
|
||||
return vchain.MessageReceipt{}, err
|
||||
}
|
||||
|
@ -1,14 +1,29 @@
|
||||
package vm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/filecoin-project/go-address"
|
||||
"github.com/filecoin-project/go-sectorbuilder"
|
||||
"github.com/filecoin-project/lotus/chain/actors"
|
||||
"github.com/filecoin-project/lotus/chain/actors/aerrors"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
"go.opencensus.io/trace"
|
||||
)
|
||||
|
||||
// Actual type is defined in chain/types/vmcontext.go because the VMContext interface is there
|
||||
|
||||
func DefaultSyscalls() *types.VMSyscalls {
|
||||
func Syscalls(verifier sectorbuilder.Verifier) *types.VMSyscalls {
|
||||
return &types.VMSyscalls{
|
||||
ValidatePoRep: actors.ValidatePoRep,
|
||||
ValidatePoRep: func(ctx context.Context, maddr address.Address, ssize uint64, commD, commR, ticket, proof, seed []byte, sectorID uint64) (bool, actors.ActorError) {
|
||||
_, span := trace.StartSpan(ctx, "ValidatePoRep")
|
||||
defer span.End()
|
||||
ok, err := verifier.VerifySeal(ssize, commR, commD, maddr, ticket, seed, sectorID, proof)
|
||||
if err != nil {
|
||||
return false, aerrors.Absorb(err, 25, "verify seal failed")
|
||||
}
|
||||
|
||||
return ok, nil
|
||||
},
|
||||
VerifyFallbackPost: verifier.VerifyFallbackPost,
|
||||
}
|
||||
}
|
||||
|
@ -312,7 +312,7 @@ type VM struct {
|
||||
Syscalls *types.VMSyscalls
|
||||
}
|
||||
|
||||
func NewVM(base cid.Cid, height uint64, r Rand, maddr address.Address, cbs blockstore.Blockstore) (*VM, error) {
|
||||
func NewVM(base cid.Cid, height uint64, r Rand, maddr address.Address, cbs blockstore.Blockstore, syscalls *types.VMSyscalls) (*VM, error) {
|
||||
buf := bufbstore.NewBufferedBstore(cbs)
|
||||
cst := hamt.CSTFromBstore(buf)
|
||||
state, err := state.LoadStateTree(cst, base)
|
||||
@ -328,8 +328,8 @@ func NewVM(base cid.Cid, height uint64, r Rand, maddr address.Address, cbs block
|
||||
blockHeight: height,
|
||||
blockMiner: maddr,
|
||||
inv: newInvoker(),
|
||||
rand: r,
|
||||
Syscalls: DefaultSyscalls(),
|
||||
rand: r, // TODO: Probably should be a syscall
|
||||
Syscalls: syscalls,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
@ -213,7 +213,7 @@ func main() {
|
||||
|
||||
sealcommit := time.Now()
|
||||
commD := pi.CommP
|
||||
ok, err := sectorbuilder.VerifySeal(sectorSize, pco.CommR[:], commD[:], maddr, ticket.TicketBytes[:], seed.TicketBytes[:], i, proof)
|
||||
ok, err := sectorbuilder.ProofVerifier.VerifySeal(sectorSize, pco.CommR[:], commD[:], maddr, ticket.TicketBytes[:], seed.TicketBytes[:], i, proof)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -307,7 +307,7 @@ func main() {
|
||||
log.Warn("separate epost calls returned different proof values (this might be bad)")
|
||||
}
|
||||
|
||||
ok, err := sectorbuilder.VerifyElectionPost(context.TODO(), sectorSize, sinfos, challenge[:], proof1, candidates[:1], maddr)
|
||||
ok, err := sectorbuilder.ProofVerifier.VerifyElectionPost(context.TODO(), sectorSize, sinfos, challenge[:], proof1, candidates[:1], maddr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -317,7 +317,7 @@ func main() {
|
||||
|
||||
verifypost1 := time.Now()
|
||||
|
||||
ok, err = sectorbuilder.VerifyElectionPost(context.TODO(), sectorSize, sinfos, challenge[:], proof2, candidates[:1], maddr)
|
||||
ok, err = sectorbuilder.ProofVerifier.VerifyElectionPost(context.TODO(), sectorSize, sinfos, challenge[:], proof2, candidates[:1], maddr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
@ -21,7 +21,7 @@ func subBlocks(ctx context.Context, api aapi.FullNode, st *storage) {
|
||||
bh.Cid(): bh,
|
||||
}, false)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
//log.Errorf("%+v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -26,7 +26,7 @@ var dotCmd = &cli.Command{
|
||||
res, err := st.db.Query(`select block, parent, b.miner, b.height, p.height from block_parents
|
||||
inner join blocks b on block_parents.block = b.cid
|
||||
inner join blocks p on block_parents.parent = p.cid
|
||||
where b.height > ? and b.height < ?`, minH, maxH)
|
||||
where b.height > $1 and b.height < $2`, minH, maxH)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
@ -34,6 +34,8 @@ where b.height > ? and b.height < ?`, minH, maxH)
|
||||
|
||||
fmt.Println("digraph D {")
|
||||
|
||||
hl := st.hasList()
|
||||
|
||||
for res.Next() {
|
||||
var block, parent, miner string
|
||||
var height, ph uint64
|
||||
@ -46,7 +48,7 @@ where b.height > ? and b.height < ?`, minH, maxH)
|
||||
return err
|
||||
}
|
||||
|
||||
has := st.hasBlock(bc)
|
||||
_, has := hl[bc]
|
||||
|
||||
col := crc32.Checksum([]byte(miner), crc32.MakeTable(crc32.Castagnoli))&0xc0c0c0c0 + 0x30303030
|
||||
|
||||
|
@ -38,7 +38,7 @@ func main() {
|
||||
&cli.StringFlag{
|
||||
Name: "db",
|
||||
EnvVars: []string{"LOTUS_DB"},
|
||||
Value: "./chainwatch.db",
|
||||
Value: "",
|
||||
},
|
||||
},
|
||||
|
||||
@ -82,8 +82,6 @@ var runCmd = &cli.Command{
|
||||
defer st.close()
|
||||
|
||||
runSyncer(ctx, api, st)
|
||||
go subMpool(ctx, api, st)
|
||||
go subBlocks(ctx, api, st)
|
||||
|
||||
h, err := newHandler(api, st)
|
||||
if err != nil {
|
||||
|
@ -2,6 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/ipfs/go-cid"
|
||||
|
||||
@ -15,24 +16,45 @@ func subMpool(ctx context.Context, api aapi.FullNode, st *storage) {
|
||||
return
|
||||
}
|
||||
|
||||
for change := range sub {
|
||||
if change.Type != aapi.MpoolAdd {
|
||||
continue
|
||||
for {
|
||||
var updates []aapi.MpoolUpdate
|
||||
|
||||
select {
|
||||
case update := <-sub:
|
||||
updates = append(updates, update)
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
|
||||
log.Info("mpool message")
|
||||
loop:
|
||||
for {
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
select {
|
||||
case update := <-sub:
|
||||
updates = append(updates, update)
|
||||
default:
|
||||
break loop
|
||||
}
|
||||
}
|
||||
|
||||
err := st.storeMessages(map[cid.Cid]*types.Message{
|
||||
change.Message.Message.Cid(): &change.Message.Message,
|
||||
})
|
||||
msgs := map[cid.Cid]*types.Message{}
|
||||
for _, v := range updates {
|
||||
if v.Type != aapi.MpoolAdd {
|
||||
continue
|
||||
}
|
||||
|
||||
msgs[v.Message.Message.Cid()] = &v.Message.Message
|
||||
}
|
||||
|
||||
log.Infof("Processing %d mpool updates", len(msgs))
|
||||
|
||||
err := st.storeMessages(msgs)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
continue
|
||||
}
|
||||
|
||||
if err := st.storeMpoolInclusion(change.Message.Message.Cid()); err != nil {
|
||||
if err := st.storeMpoolInclusions(updates); err != nil {
|
||||
log.Error(err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
61
cmd/lotus-chainwatch/site/block.html
Normal file
61
cmd/lotus-chainwatch/site/block.html
Normal file
@ -0,0 +1,61 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>Lotus ChainWatch</title>
|
||||
<link rel="stylesheet" type="text/css" href="main.css">
|
||||
</head>
|
||||
<body>
|
||||
{{$cid := param "cid"}}
|
||||
|
||||
<div class="Index">
|
||||
<div class="Index-header">
|
||||
<div>
|
||||
<span>Lotus ChainWatch - Wallets</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="Index-nodes">
|
||||
<div class="Index-node">
|
||||
<div>Miner: {{index (strings "blocks" "miner" "cid=$1" $cid) 0}}</div>
|
||||
<div>Parents:</div>
|
||||
<div>
|
||||
{{range strings "block_parents" "parent" "block=$1" $cid}}
|
||||
{{$parent := .}}
|
||||
<a href="block.html?cid={{$parent}}">{{. | substr 54 62}}</a>
|
||||
{{end}}
|
||||
</div>
|
||||
<div>Messages:</div>
|
||||
<table>
|
||||
{{range strings "block_messages" "message" "block=$1" $cid}}
|
||||
{{$msg := .}}
|
||||
<tr>
|
||||
<td><a href="message.html?cid={{$msg}}">{{$msg | substr 54 62}}</a></td>
|
||||
<td>
|
||||
{{$from := qstr "select \"from\" from messages where cid=$1" $msg}}
|
||||
{{$nonce := qstr "select nonce from messages where cid=$1" $msg}}
|
||||
<a href="key.html?w={{$from}}">{{$from}}</a> (N:{{$nonce}})
|
||||
</td>
|
||||
<td>-></td>
|
||||
<td>
|
||||
{{$to := qstr "select \"to\" from messages where cid=$1" $msg}}
|
||||
<a href="key.html?w={{$to}}">{{$to}}</a>
|
||||
</td>
|
||||
<td>
|
||||
Method:<b>{{qstr "select method from messages where cid=$1" $msg}}</b>
|
||||
</td>
|
||||
{{$rec := qstrs `select r.exit, r.gas_used from messages
|
||||
inner join block_messages bm on messages.cid = bm.message
|
||||
inner join blocks b on bm.block = b.cid
|
||||
inner join block_parents bp on b.cid = bp.parent
|
||||
inner join blocks chd on bp.block = chd.cid
|
||||
inner join receipts r on messages.cid = r.msg and chd.parentStateRoot = r.state
|
||||
where messages.cid=$1 and b.cid=$2` 2 $msg $cid}}
|
||||
<td>exit:<b>{{index $rec 0}}</b></td>
|
||||
<td>gasUsed:<b>{{index $rec 1}}</b></td>
|
||||
</tr>
|
||||
{{end}}
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
43
cmd/lotus-chainwatch/site/blocks.html
Normal file
43
cmd/lotus-chainwatch/site/blocks.html
Normal file
@ -0,0 +1,43 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>Lotus ChainWatch</title>
|
||||
<link rel="stylesheet" type="text/css" href="main.css">
|
||||
</head>
|
||||
<body>
|
||||
{{$start := param "start" | parseInt}}
|
||||
|
||||
<div class="Index">
|
||||
<div class="Index-header">
|
||||
<div>
|
||||
<span>Lotus ChainWatch - Wallets</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="Index-nodes">
|
||||
<div class="Index-node">
|
||||
<table>
|
||||
{{range pageDown $start 50}}
|
||||
<tr>
|
||||
<td>
|
||||
{{$h := .}}
|
||||
{{$h}};
|
||||
</td>
|
||||
<td>
|
||||
<b>{{qstr `select count(distinct block_messages.message) from block_messages
|
||||
inner join blocks b on block_messages.block = b.cid
|
||||
where b.height = $1` $h}}</b> Msgs
|
||||
</td>
|
||||
<td>
|
||||
{{range strings "blocks" "cid" "height = $1" $h}}
|
||||
<a href="block.html?cid={{.}}">{{. | substr 54 62}}</a>
|
||||
{{end}}
|
||||
</td>
|
||||
</tr>
|
||||
{{end}}
|
||||
</table>
|
||||
<a href="blocks.html?start={{sub $start 50}}">Next 50</a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
@ -26,6 +26,11 @@
|
||||
{{count "id_address_map" "id != address"}} <a href="keys.html">Keys</a>;
|
||||
E% FIL in wallets; F% FIL in miners; M% in market; %G Other actors; %H FIL it treasury
|
||||
</div>
|
||||
<div class="Index-node">
|
||||
{{$maxH := queryNum "select max(height) from blocks inner join blocks_synced bs on blocks.cid = bs.cid"}}
|
||||
|
||||
{{count "blocks"}} <a href="blocks.html?start={{$maxH}}">Blocks</a>; Current Height: {{$maxH}};
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
|
@ -15,18 +15,18 @@
|
||||
</div>
|
||||
<div class="Index-nodes">
|
||||
<div class="Index-node">
|
||||
Balance: {{queryNum "select balance from actors inner join main.id_address_map m on m.address = ? where actors.id = m.id order by nonce desc limit 1" $wallet }}
|
||||
Balance: {{queryNum "select balance from actors inner join id_address_map m on m.address = $1 where actors.id = m.id order by nonce desc limit 1" $wallet }}
|
||||
</div>
|
||||
<div class="Index-node">
|
||||
Messages:
|
||||
<table>
|
||||
<tr><td>Dir</td><td>Peer</td><td>Nonce</td><td>Value</td><td>Block</td><td>Mpool Wait</td></tr>
|
||||
{{ range messages "`from` = ? or `to` = ?" $wallet $wallet $wallet}}
|
||||
{{ range messages "\"from\" = $1 or \"to\" = $1" $wallet}}
|
||||
<tr>
|
||||
{{ if eq .From.String $wallet }}
|
||||
<td>To</td><td>{{.To.String}}</td>
|
||||
<td>To</td><td><a href="key.html?w={{.To.String}}">{{.To.String}}</a></td>
|
||||
{{else}}
|
||||
<td>From</td><td>{{.From.String}}</td>
|
||||
<td>From</td><td><a href="key.html?w={{.From.String}}">{{.From.String}}</a></td>
|
||||
{{end}}
|
||||
<td>{{.Nonce}}</td>
|
||||
<td>{{.Value}}</td>
|
||||
|
@ -14,7 +14,12 @@
|
||||
<div class="Index-nodes">
|
||||
<div class="Index-node">
|
||||
{{range strings "id_address_map" "address" "address != id"}}
|
||||
<div><a href="key.html?w={{.}}">{{.}}</a></div>
|
||||
{{$addr := .}}
|
||||
<div>
|
||||
<a href="key.html?w={{$addr}}">{{$addr}}</a>
|
||||
<span><b>{{qstr "select count(distinct cid) from messages where \"from\"=?" $addr}}</b> outmsgs;</span>
|
||||
<span><b>{{qstr "select count(distinct cid) from messages where \"to\"=?" $addr}}</b> inmsgs</span>
|
||||
</div>
|
||||
{{end}}
|
||||
</div>
|
||||
</div>
|
||||
|
@ -6,6 +6,10 @@ body {
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
b {
|
||||
color: #aff;
|
||||
}
|
||||
|
||||
.Index {
|
||||
width: 100vw;
|
||||
height: 100vh;
|
||||
|
@ -2,11 +2,14 @@ package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"github.com/filecoin-project/lotus/api"
|
||||
"golang.org/x/xerrors"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/ipfs/go-cid"
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
_ "github.com/lib/pq"
|
||||
|
||||
"github.com/filecoin-project/go-address"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
@ -19,11 +22,13 @@ type storage struct {
|
||||
}
|
||||
|
||||
func openStorage(dbSource string) (*storage, error) {
|
||||
db, err := sql.Open("sqlite3", dbSource)
|
||||
db, err := sql.Open("postgres", dbSource)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
db.SetMaxOpenConns(1350)
|
||||
|
||||
st := &storage{db: db}
|
||||
|
||||
return st, st.setup()
|
||||
@ -35,33 +40,77 @@ func (st *storage) setup() error {
|
||||
return err
|
||||
}
|
||||
_, err = tx.Exec(`
|
||||
create table if not exists blocks_synced
|
||||
(
|
||||
cid text not null
|
||||
constraint blocks_synced_pk
|
||||
primary key,
|
||||
add_ts int not null
|
||||
);
|
||||
|
||||
create unique index if not exists blocks_synced_cid_uindex
|
||||
on blocks_synced (cid);
|
||||
|
||||
create table if not exists block_parents
|
||||
(
|
||||
block text not null,
|
||||
parent text not null
|
||||
);
|
||||
|
||||
create unique index if not exists block_parents_block_parent_uindex
|
||||
on block_parents (block, parent);
|
||||
|
||||
create table if not exists blocks
|
||||
(
|
||||
cid text not null
|
||||
constraint blocks_pk
|
||||
primary key,
|
||||
parentWeight numeric not null,
|
||||
parentStateRoot text not null,
|
||||
height bigint not null,
|
||||
miner text not null,
|
||||
timestamp bigint not null,
|
||||
vrfproof bytea,
|
||||
tickets bigint not null,
|
||||
eprof bytea,
|
||||
prand bytea,
|
||||
ep0partial bytea,
|
||||
ep0sector bigint not null,
|
||||
ep0challangei bigint not null
|
||||
);
|
||||
|
||||
create unique index if not exists block_cid_uindex
|
||||
on blocks (cid);
|
||||
|
||||
create table if not exists id_address_map
|
||||
(
|
||||
id text not null,
|
||||
address text not null,
|
||||
constraint id_address_map_pk
|
||||
primary key (id, address)
|
||||
);
|
||||
|
||||
create unique index if not exists id_address_map_id_uindex
|
||||
on id_address_map (id);
|
||||
|
||||
create unique index if not exists id_address_map_address_uindex
|
||||
on id_address_map (address);
|
||||
|
||||
create table if not exists actors
|
||||
(
|
||||
id text not null,
|
||||
id text not null
|
||||
constraint id_address_map_actors_id_fk
|
||||
references id_address_map (id),
|
||||
code text not null,
|
||||
head text not null,
|
||||
nonce int not null,
|
||||
balance text not null,
|
||||
stateroot text
|
||||
constraint actors_blocks_stateroot_fk
|
||||
references blocks (parentStateRoot),
|
||||
constraint actors_pk
|
||||
primary key (id, nonce, balance, stateroot)
|
||||
);
|
||||
|
||||
create index if not exists actors_id_index
|
||||
on actors (id);
|
||||
|
||||
create table if not exists id_address_map
|
||||
(
|
||||
id text not null
|
||||
constraint id_address_map_actors_id_fk
|
||||
references actors (id),
|
||||
address text not null,
|
||||
constraint id_address_map_pk
|
||||
primary key (id, address)
|
||||
);
|
||||
|
||||
create index if not exists id_address_map_address_index
|
||||
on id_address_map (address);
|
||||
|
||||
@ -73,84 +122,23 @@ create table if not exists messages
|
||||
cid text not null
|
||||
constraint messages_pk
|
||||
primary key,
|
||||
"from" text not null
|
||||
constraint messages_id_address_map_from_fk
|
||||
references id_address_map (address),
|
||||
"to" text not null
|
||||
constraint messages_id_address_map_to_fk
|
||||
references id_address_map (address),
|
||||
"from" text not null,
|
||||
"to" text not null,
|
||||
nonce int not null,
|
||||
value text not null,
|
||||
gasprice int not null,
|
||||
gaslimit int not null,
|
||||
method int,
|
||||
params blob
|
||||
params bytea
|
||||
);
|
||||
|
||||
create unique index if not exists messages_cid_uindex
|
||||
on messages (cid);
|
||||
|
||||
create table if not exists blocks
|
||||
(
|
||||
cid text not null
|
||||
constraint blocks_pk
|
||||
primary key,
|
||||
parentWeight numeric not null,
|
||||
parentStateRoot text not null,
|
||||
height int not null,
|
||||
miner text not null
|
||||
constraint blocks_id_address_map_miner_fk
|
||||
references id_address_map (address),
|
||||
timestamp int not null,
|
||||
vrfproof blob,
|
||||
tickets int not null,
|
||||
eprof blob,
|
||||
prand blob,
|
||||
ep0partial blob,
|
||||
ep0sector int not null,
|
||||
ep0challangei int not null
|
||||
);
|
||||
|
||||
create unique index if not exists block_cid_uindex
|
||||
on blocks (cid);
|
||||
|
||||
create table if not exists blocks_synced
|
||||
(
|
||||
cid text not null
|
||||
constraint blocks_synced_pk
|
||||
primary key
|
||||
constraint blocks_synced_blocks_cid_fk
|
||||
references blocks,
|
||||
add_ts int not null
|
||||
);
|
||||
|
||||
create unique index if not exists blocks_synced_cid_uindex
|
||||
on blocks_synced (cid);
|
||||
|
||||
create table if not exists block_parents
|
||||
(
|
||||
block text not null
|
||||
constraint block_parents_blocks_cid_fk
|
||||
references blocks,
|
||||
parent text not null
|
||||
constraint block_parents_blocks_cid_fk_2
|
||||
references blocks
|
||||
);
|
||||
|
||||
create unique index if not exists block_parents_block_parent_uindex
|
||||
on block_parents (block, parent);
|
||||
|
||||
create unique index if not exists blocks_cid_uindex
|
||||
on blocks (cid);
|
||||
|
||||
create table if not exists block_messages
|
||||
(
|
||||
block text not null
|
||||
constraint block_messages_blk_fk
|
||||
references blocks (cid),
|
||||
message text not null
|
||||
constraint block_messages_msg_fk
|
||||
references messages,
|
||||
block text not null,
|
||||
message text not null,
|
||||
constraint block_messages_pk
|
||||
primary key (block, message)
|
||||
);
|
||||
@ -170,16 +158,12 @@ create unique index if not exists mpool_messages_msg_uindex
|
||||
|
||||
create table if not exists receipts
|
||||
(
|
||||
msg text not null
|
||||
constraint receipts_messages_cid_fk
|
||||
references messages,
|
||||
state text not null
|
||||
constraint receipts_blocks_parentStateRoot_fk
|
||||
references blocks (parentStateRoot),
|
||||
msg text not null,
|
||||
state text not null,
|
||||
idx int not null,
|
||||
exit int not null,
|
||||
gas_used int not null,
|
||||
return blob,
|
||||
return bytea,
|
||||
constraint receipts_pk
|
||||
primary key (msg, state)
|
||||
);
|
||||
@ -187,34 +171,23 @@ create table if not exists receipts
|
||||
create index if not exists receipts_msg_state_index
|
||||
on receipts (msg, state);
|
||||
|
||||
|
||||
create table if not exists miner_heads
|
||||
(
|
||||
head text not null
|
||||
constraint miner_heads_actors_head_fk
|
||||
references actors (head),
|
||||
addr text not null
|
||||
constraint miner_heads_actors_id_fk
|
||||
references actors (id),
|
||||
stateroot text not null
|
||||
constraint miner_heads_blocks_stateroot_fk
|
||||
references blocks (parentStateRoot),
|
||||
head text not null,
|
||||
addr text not null,
|
||||
stateroot text not null,
|
||||
sectorset text not null,
|
||||
setsize int not null,
|
||||
setsize decimal not null,
|
||||
provingset text not null,
|
||||
provingsize int not null,
|
||||
provingsize decimal not null,
|
||||
owner text not null,
|
||||
worker text not null,
|
||||
peerid text not null,
|
||||
sectorsize int not null,
|
||||
power text not null,
|
||||
active int,
|
||||
ppe int not null,
|
||||
slashed_at int not null,
|
||||
constraint miner_heads_id_address_map_owner_fk
|
||||
foreign key (owner) references id_address_map (address),
|
||||
constraint miner_heads_id_address_map_worker_fk
|
||||
foreign key (worker) references id_address_map (address),
|
||||
sectorsize bigint not null,
|
||||
power bigint not null,
|
||||
active bool,
|
||||
ppe bigint not null,
|
||||
slashed_at bigint not null,
|
||||
constraint miner_heads_pk
|
||||
primary key (head, addr)
|
||||
);
|
||||
@ -226,14 +199,31 @@ create table if not exists miner_heads
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func (st *storage) hasBlock(bh cid.Cid) bool {
|
||||
var exitsts bool
|
||||
err := st.db.QueryRow(`select exists (select 1 FROM blocks_synced where cid=?)`, bh.String()).Scan(&exitsts)
|
||||
func (st *storage) hasList() map[cid.Cid]struct{} {
|
||||
rws, err := st.db.Query(`select cid FROM blocks_synced`)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
return false
|
||||
return map[cid.Cid]struct{}{}
|
||||
}
|
||||
return exitsts
|
||||
out := map[cid.Cid]struct{}{}
|
||||
|
||||
for rws.Next() {
|
||||
var c string
|
||||
if err := rws.Scan(&c); err != nil {
|
||||
log.Error(err)
|
||||
continue
|
||||
}
|
||||
|
||||
ci, err := cid.Parse(c)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
continue
|
||||
}
|
||||
|
||||
out[ci] = struct{}{}
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
func (st *storage) storeActors(actors map[address.Address]map[types.Actor]cid.Cid) error {
|
||||
@ -241,12 +231,20 @@ func (st *storage) storeActors(actors map[address.Address]map[types.Actor]cid.Ci
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(`
|
||||
|
||||
stmt, err := tx.Prepare(`insert into actors (id, code, head, nonce, balance, stateroot) values (?, ?, ?, ?, ?, ?) on conflict do nothing`)
|
||||
create temp table a (like actors excluding constraints) on commit drop;
|
||||
|
||||
|
||||
`); err != nil {
|
||||
return xerrors.Errorf("prep temp: %w", err)
|
||||
}
|
||||
|
||||
stmt, err := tx.Prepare(`copy a (id, code, head, nonce, balance, stateroot) from stdin `)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer stmt.Close()
|
||||
|
||||
for addr, acts := range actors {
|
||||
for act, st := range acts {
|
||||
if _, err := stmt.Exec(addr.String(), act.Code.String(), act.Head.String(), act.Nonce, act.Balance.String(), st.String()); err != nil {
|
||||
@ -255,6 +253,14 @@ func (st *storage) storeActors(actors map[address.Address]map[types.Actor]cid.Ci
|
||||
}
|
||||
}
|
||||
|
||||
if err := stmt.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := tx.Exec(`insert into actors select * from a on conflict do nothing `); err != nil {
|
||||
return xerrors.Errorf("actor put: %w", err)
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
@ -264,20 +270,28 @@ func (st *storage) storeMiners(miners map[minerKey]*minerInfo) error {
|
||||
return err
|
||||
}
|
||||
|
||||
stmt, err := tx.Prepare(`insert into miner_heads (head, addr, stateroot, sectorset, setsize, provingset, provingsize, owner, worker, peerid, sectorsize, power, active, ppe, slashed_at) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) on conflict do nothing`)
|
||||
if _, err := tx.Exec(`
|
||||
|
||||
create temp table mh (like miner_heads excluding constraints) on commit drop;
|
||||
|
||||
|
||||
`); err != nil {
|
||||
return xerrors.Errorf("prep temp: %w", err)
|
||||
}
|
||||
|
||||
stmt, err := tx.Prepare(`copy mh (head, addr, stateroot, sectorset, setsize, provingset, provingsize, owner, worker, peerid, sectorsize, power, active, ppe, slashed_at) from STDIN`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer stmt.Close()
|
||||
for k, i := range miners {
|
||||
if _, err := stmt.Exec(
|
||||
k.act.Head.String(),
|
||||
k.addr.String(),
|
||||
k.stateroot.String(),
|
||||
i.state.Sectors.String(),
|
||||
i.ssize,
|
||||
fmt.Sprint(i.ssize),
|
||||
i.state.ProvingSet.String(),
|
||||
i.psize,
|
||||
fmt.Sprint(i.psize),
|
||||
i.info.Owner.String(),
|
||||
i.info.Worker.String(),
|
||||
i.info.PeerID.String(),
|
||||
@ -290,6 +304,13 @@ func (st *storage) storeMiners(miners map[minerKey]*minerInfo) error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := stmt.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := tx.Exec(`insert into miner_heads select * from mh on conflict do nothing `); err != nil {
|
||||
return xerrors.Errorf("actor put: %w", err)
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
@ -300,21 +321,77 @@ func (st *storage) storeHeaders(bhs map[cid.Cid]*types.BlockHeader, sync bool) e
|
||||
|
||||
tx, err := st.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
return xerrors.Errorf("begin: %w", err)
|
||||
}
|
||||
|
||||
stmt, err := tx.Prepare(`insert into blocks (cid, parentWeight, parentStateRoot, height, miner, "timestamp", vrfproof, tickets, eprof, prand, ep0partial, ep0sector, ep0challangei) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) on conflict do nothing`)
|
||||
if _, err := tx.Exec(`
|
||||
|
||||
create temp table tbp (like block_parents excluding constraints) on commit drop;
|
||||
create temp table bs (like blocks_synced excluding constraints) on commit drop;
|
||||
create temp table b (like blocks excluding constraints) on commit drop;
|
||||
|
||||
|
||||
`); err != nil {
|
||||
return xerrors.Errorf("prep temp: %w", err)
|
||||
}
|
||||
|
||||
stmt, err := tx.Prepare(`copy tbp (block, parent) from STDIN`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer stmt.Close()
|
||||
|
||||
for _, bh := range bhs {
|
||||
for _, parent := range bh.Parents {
|
||||
if _, err := stmt.Exec(bh.Cid().String(), parent.String()); err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if err := stmt.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := tx.Exec(`insert into block_parents select * from tbp on conflict do nothing `); err != nil {
|
||||
return xerrors.Errorf("parent put: %w", err)
|
||||
}
|
||||
|
||||
if sync {
|
||||
now := time.Now().Unix()
|
||||
|
||||
stmt, err := tx.Prepare(`copy bs (cid, add_ts) from stdin `)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, bh := range bhs {
|
||||
if _, err := stmt.Exec(bh.Cid().String(), now); err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := stmt.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := tx.Exec(`insert into blocks_synced select * from bs on conflict do nothing `); err != nil {
|
||||
return xerrors.Errorf("syncd put: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
stmt2, err := tx.Prepare(`copy b (cid, parentWeight, parentStateRoot, height, miner, "timestamp", vrfproof, tickets, eprof, prand, ep0partial, ep0sector, ep0challangei) from stdin `)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, bh := range bhs {
|
||||
l := len(bh.EPostProof.Candidates)
|
||||
if len(bh.EPostProof.Candidates) == 0 {
|
||||
bh.EPostProof.Candidates = append(bh.EPostProof.Candidates, types.EPostTicket{})
|
||||
}
|
||||
|
||||
if _, err := stmt.Exec(bh.Cid().String(),
|
||||
if _, err := stmt2.Exec(
|
||||
bh.Cid().String(),
|
||||
bh.ParentWeight.String(),
|
||||
bh.ParentStateRoot.String(),
|
||||
bh.Height,
|
||||
@ -326,41 +403,24 @@ func (st *storage) storeHeaders(bhs map[cid.Cid]*types.BlockHeader, sync bool) e
|
||||
bh.EPostProof.PostRand,
|
||||
bh.EPostProof.Candidates[0].Partial,
|
||||
bh.EPostProof.Candidates[0].SectorID,
|
||||
bh.EPostProof.Candidates[0].ChallengeIndex,
|
||||
); err != nil {
|
||||
return err
|
||||
bh.EPostProof.Candidates[0].ChallengeIndex); err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
stmt2, err := tx.Prepare(`insert into block_parents (block, parent) values (?, ?) on conflict do nothing`)
|
||||
if err := stmt2.Close(); err != nil {
|
||||
return xerrors.Errorf("s2 close: %w", err)
|
||||
}
|
||||
|
||||
if _, err := tx.Exec(`insert into blocks select * from b on conflict do nothing `); err != nil {
|
||||
return xerrors.Errorf("blk put: %w", err)
|
||||
}
|
||||
|
||||
err = tx.Commit()
|
||||
if err != nil {
|
||||
return err
|
||||
return xerrors.Errorf("commit: %w", err)
|
||||
}
|
||||
defer stmt2.Close()
|
||||
for _, bh := range bhs {
|
||||
for _, parent := range bh.Parents {
|
||||
if _, err := stmt2.Exec(bh.Cid().String(), parent.String()); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if sync {
|
||||
stmt, err := tx.Prepare(`insert into blocks_synced (cid, add_ts) values (?, ?) on conflict do nothing`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer stmt.Close()
|
||||
now := time.Now().Unix()
|
||||
|
||||
for _, bh := range bhs {
|
||||
if _, err := stmt.Exec(bh.Cid().String(), now); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (st *storage) storeMessages(msgs map[cid.Cid]*types.Message) error {
|
||||
@ -369,11 +429,19 @@ func (st *storage) storeMessages(msgs map[cid.Cid]*types.Message) error {
|
||||
return err
|
||||
}
|
||||
|
||||
stmt, err := tx.Prepare(`insert into messages (cid, "from", "to", nonce, "value", gasprice, gaslimit, method, params) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) on conflict do nothing`)
|
||||
if _, err := tx.Exec(`
|
||||
|
||||
create temp table msgs (like messages excluding constraints) on commit drop;
|
||||
|
||||
|
||||
`); err != nil {
|
||||
return xerrors.Errorf("prep temp: %w", err)
|
||||
}
|
||||
|
||||
stmt, err := tx.Prepare(`copy msgs (cid, "from", "to", nonce, "value", gasprice, gaslimit, method, params) from stdin `)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer stmt.Close()
|
||||
|
||||
for c, m := range msgs {
|
||||
if _, err := stmt.Exec(
|
||||
@ -390,6 +458,13 @@ func (st *storage) storeMessages(msgs map[cid.Cid]*types.Message) error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := stmt.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := tx.Exec(`insert into messages select * from msgs on conflict do nothing `); err != nil {
|
||||
return xerrors.Errorf("actor put: %w", err)
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
@ -400,11 +475,19 @@ func (st *storage) storeReceipts(recs map[mrec]*types.MessageReceipt) error {
|
||||
return err
|
||||
}
|
||||
|
||||
stmt, err := tx.Prepare(`insert into receipts (msg, state, idx, exit, gas_used, return) VALUES (?, ?, ?, ?, ?, ?) on conflict do nothing`)
|
||||
if _, err := tx.Exec(`
|
||||
|
||||
create temp table recs (like receipts excluding constraints) on commit drop;
|
||||
|
||||
|
||||
`); err != nil {
|
||||
return xerrors.Errorf("prep temp: %w", err)
|
||||
}
|
||||
|
||||
stmt, err := tx.Prepare(`copy recs (msg, state, idx, exit, gas_used, return) from stdin `)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer stmt.Close()
|
||||
|
||||
for c, m := range recs {
|
||||
if _, err := stmt.Exec(
|
||||
@ -418,6 +501,13 @@ func (st *storage) storeReceipts(recs map[mrec]*types.MessageReceipt) error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := stmt.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := tx.Exec(`insert into receipts select * from recs on conflict do nothing `); err != nil {
|
||||
return xerrors.Errorf("actor put: %w", err)
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
@ -428,11 +518,19 @@ func (st *storage) storeAddressMap(addrs map[address.Address]address.Address) er
|
||||
return err
|
||||
}
|
||||
|
||||
stmt, err := tx.Prepare(`insert into id_address_map (id, address) VALUES (?, ?) on conflict do nothing`)
|
||||
if _, err := tx.Exec(`
|
||||
|
||||
create temp table iam (like id_address_map excluding constraints) on commit drop;
|
||||
|
||||
|
||||
`); err != nil {
|
||||
return xerrors.Errorf("prep temp: %w", err)
|
||||
}
|
||||
|
||||
stmt, err := tx.Prepare(`copy iam (id, address) from STDIN `)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer stmt.Close()
|
||||
|
||||
for a, i := range addrs {
|
||||
if i == address.Undef {
|
||||
@ -445,6 +543,13 @@ func (st *storage) storeAddressMap(addrs map[address.Address]address.Address) er
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := stmt.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := tx.Exec(`insert into id_address_map select * from iam on conflict do nothing `); err != nil {
|
||||
return xerrors.Errorf("actor put: %w", err)
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
@ -455,11 +560,19 @@ func (st *storage) storeMsgInclusions(incls map[cid.Cid][]cid.Cid) error {
|
||||
return err
|
||||
}
|
||||
|
||||
stmt, err := tx.Prepare(`insert into block_messages (block, message) VALUES (?, ?) on conflict do nothing`)
|
||||
if _, err := tx.Exec(`
|
||||
|
||||
create temp table mi (like block_messages excluding constraints) on commit drop;
|
||||
|
||||
|
||||
`); err != nil {
|
||||
return xerrors.Errorf("prep temp: %w", err)
|
||||
}
|
||||
|
||||
stmt, err := tx.Prepare(`copy mi (block, message) from STDIN `)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer stmt.Close()
|
||||
|
||||
for b, msgs := range incls {
|
||||
for _, msg := range msgs {
|
||||
@ -471,28 +584,58 @@ func (st *storage) storeMsgInclusions(incls map[cid.Cid][]cid.Cid) error {
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := stmt.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := tx.Exec(`insert into block_messages select * from mi on conflict do nothing `); err != nil {
|
||||
return xerrors.Errorf("actor put: %w", err)
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func (st *storage) storeMpoolInclusion(msg cid.Cid) error {
|
||||
func (st *storage) storeMpoolInclusions(msgs []api.MpoolUpdate) error {
|
||||
tx, err := st.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
stmt, err := tx.Prepare(`insert into mpool_messages (msg, add_ts) VALUES (?, ?) on conflict do nothing`)
|
||||
if _, err := tx.Exec(`
|
||||
|
||||
create temp table mi (like mpool_messages excluding constraints) on commit drop;
|
||||
|
||||
|
||||
`); err != nil {
|
||||
return xerrors.Errorf("prep temp: %w", err)
|
||||
}
|
||||
|
||||
stmt, err := tx.Prepare(`copy mi (msg, add_ts) from stdin `)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer stmt.Close()
|
||||
|
||||
if _, err := stmt.Exec(
|
||||
msg.String(),
|
||||
time.Now().Unix(),
|
||||
); err != nil {
|
||||
for _, msg := range msgs {
|
||||
if msg.Type != api.MpoolAdd {
|
||||
continue
|
||||
}
|
||||
|
||||
if _, err := stmt.Exec(
|
||||
msg.Message.Message.Cid().String(),
|
||||
time.Now().Unix(),
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err := stmt.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := tx.Exec(`insert into mpool_messages select * from mi on conflict do nothing `); err != nil {
|
||||
return xerrors.Errorf("actor put: %w", err)
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
|
@ -4,6 +4,7 @@ import (
|
||||
"bytes"
|
||||
"container/list"
|
||||
"context"
|
||||
"math"
|
||||
"sync"
|
||||
|
||||
"github.com/filecoin-project/go-address"
|
||||
@ -16,6 +17,8 @@ import (
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
)
|
||||
|
||||
const maxBatch = 3000
|
||||
|
||||
func runSyncer(ctx context.Context, api api.FullNode, st *storage) {
|
||||
notifs, err := api.ChainNotify(ctx)
|
||||
if err != nil {
|
||||
@ -32,6 +35,12 @@ func runSyncer(ctx context.Context, api api.FullNode, st *storage) {
|
||||
case store.HCRevert:
|
||||
log.Warnf("revert todo")
|
||||
}
|
||||
|
||||
if change.Type == store.HCCurrent {
|
||||
go subMpool(ctx, api, st)
|
||||
go subBlocks(ctx, api, st)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}()
|
||||
@ -56,9 +65,13 @@ func syncHead(ctx context.Context, api api.FullNode, st *storage, ts *types.TipS
|
||||
actors := map[address.Address]map[types.Actor]cid.Cid{}
|
||||
var alk sync.Mutex
|
||||
|
||||
log.Infof("Getting synced block list")
|
||||
|
||||
hazlist := st.hasList()
|
||||
|
||||
log.Infof("Getting headers / actors")
|
||||
|
||||
toSync := map[cid.Cid]*types.BlockHeader{}
|
||||
allToSync := map[cid.Cid]*types.BlockHeader{}
|
||||
toVisit := list.New()
|
||||
|
||||
for _, header := range ts.Blocks() {
|
||||
@ -68,15 +81,16 @@ func syncHead(ctx context.Context, api api.FullNode, st *storage, ts *types.TipS
|
||||
for toVisit.Len() > 0 {
|
||||
bh := toVisit.Remove(toVisit.Back()).(*types.BlockHeader)
|
||||
|
||||
if _, seen := toSync[bh.Cid()]; seen || st.hasBlock(bh.Cid()) {
|
||||
_, has := hazlist[bh.Cid()]
|
||||
if _, seen := allToSync[bh.Cid()]; seen || has {
|
||||
continue
|
||||
}
|
||||
|
||||
toSync[bh.Cid()] = bh
|
||||
allToSync[bh.Cid()] = bh
|
||||
addresses[bh.Miner] = address.Undef
|
||||
|
||||
if len(toSync)%500 == 10 {
|
||||
log.Infof("todo: (%d) %s", len(toSync), bh.Cid())
|
||||
if len(allToSync)%500 == 10 {
|
||||
log.Infof("todo: (%d) %s @%d", len(allToSync), bh.Cid(), bh.Height)
|
||||
}
|
||||
|
||||
if len(bh.Parents) == 0 {
|
||||
@ -94,186 +108,218 @@ func syncHead(ctx context.Context, api api.FullNode, st *storage, ts *types.TipS
|
||||
}
|
||||
}
|
||||
|
||||
log.Infof("Syncing %d blocks", len(toSync))
|
||||
for len(allToSync) > 0 {
|
||||
minH := uint64(math.MaxUint64)
|
||||
|
||||
log.Infof("Persisting actors")
|
||||
|
||||
paDone := 0
|
||||
par(50, maparr(toSync), func(bh *types.BlockHeader) {
|
||||
paDone++
|
||||
if paDone%100 == 0 {
|
||||
log.Infof("pa: %d %d%%", paDone, (paDone*100)/len(toSync))
|
||||
for _, header := range allToSync {
|
||||
if header.Height < minH {
|
||||
minH = header.Height
|
||||
}
|
||||
}
|
||||
|
||||
if len(bh.Parents) == 0 { // genesis case
|
||||
ts, err := types.NewTipSet([]*types.BlockHeader{bh})
|
||||
aadrs, err := api.StateListActors(ctx, ts)
|
||||
toSync := map[cid.Cid]*types.BlockHeader{}
|
||||
for c, header := range allToSync {
|
||||
if header.Height < minH+maxBatch {
|
||||
toSync[c] = header
|
||||
}
|
||||
}
|
||||
for c := range toSync {
|
||||
delete(allToSync, c)
|
||||
}
|
||||
|
||||
log.Infof("Syncing %d blocks", len(toSync))
|
||||
|
||||
paDone := 0
|
||||
par(50, maparr(toSync), func(bh *types.BlockHeader) {
|
||||
paDone++
|
||||
if paDone%100 == 0 {
|
||||
log.Infof("pa: %d %d%%", paDone, (paDone*100)/len(toSync))
|
||||
}
|
||||
|
||||
if len(bh.Parents) == 0 { // genesis case
|
||||
ts, err := types.NewTipSet([]*types.BlockHeader{bh})
|
||||
aadrs, err := api.StateListActors(ctx, ts)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
par(50, aadrs, func(addr address.Address) {
|
||||
act, err := api.StateGetActor(ctx, addr, ts)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
return
|
||||
}
|
||||
alk.Lock()
|
||||
_, ok := actors[addr]
|
||||
if !ok {
|
||||
actors[addr] = map[types.Actor]cid.Cid{}
|
||||
}
|
||||
actors[addr][*act] = bh.ParentStateRoot
|
||||
addresses[addr] = address.Undef
|
||||
alk.Unlock()
|
||||
})
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
pts, err := api.ChainGetTipSet(ctx, types.NewTipSetKey(bh.Parents...))
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
par(50, aadrs, func(addr address.Address) {
|
||||
act, err := api.StateGetActor(ctx, addr, ts)
|
||||
changes, err := api.StateChangedActors(ctx, pts.ParentState(), bh.ParentStateRoot)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
for a, act := range changes {
|
||||
addr, err := address.NewFromString(a)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
alk.Lock()
|
||||
_, ok := actors[addr]
|
||||
if !ok {
|
||||
actors[addr] = map[types.Actor]cid.Cid{}
|
||||
}
|
||||
actors[addr][*act] = bh.ParentStateRoot
|
||||
actors[addr][act] = bh.ParentStateRoot
|
||||
addresses[addr] = address.Undef
|
||||
alk.Unlock()
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
return
|
||||
log.Infof("Getting messages")
|
||||
|
||||
msgs, incls := fetchMessages(ctx, api, toSync)
|
||||
|
||||
log.Infof("Resolving addresses")
|
||||
|
||||
for _, message := range msgs {
|
||||
addresses[message.To] = address.Undef
|
||||
addresses[message.From] = address.Undef
|
||||
}
|
||||
|
||||
pts, err := api.ChainGetTipSet(ctx, types.NewTipSetKey(bh.Parents...))
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
return
|
||||
par(50, kmaparr(addresses), func(addr address.Address) {
|
||||
raddr, err := api.StateLookupID(ctx, addr, nil)
|
||||
if err != nil {
|
||||
log.Warn(err)
|
||||
return
|
||||
}
|
||||
alk.Lock()
|
||||
addresses[addr] = raddr
|
||||
alk.Unlock()
|
||||
})
|
||||
|
||||
log.Infof("Getting miner info")
|
||||
|
||||
miners := map[minerKey]*minerInfo{}
|
||||
|
||||
for addr, m := range actors {
|
||||
for actor, c := range m {
|
||||
if actor.Code != actors2.StorageMinerCodeCid {
|
||||
continue
|
||||
}
|
||||
|
||||
miners[minerKey{
|
||||
addr: addr,
|
||||
act: actor,
|
||||
stateroot: c,
|
||||
}] = &minerInfo{}
|
||||
}
|
||||
}
|
||||
|
||||
changes, err := api.StateChangedActors(ctx, pts.ParentState(), bh.ParentStateRoot)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
return
|
||||
}
|
||||
par(50, kvmaparr(miners), func(it func() (minerKey, *minerInfo)) {
|
||||
k, info := it()
|
||||
|
||||
for a, act := range changes {
|
||||
addr, err := address.NewFromString(a)
|
||||
sszs, err := api.StateMinerSectorCount(ctx, k.addr, nil)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
return
|
||||
}
|
||||
info.psize = sszs.Pset
|
||||
info.ssize = sszs.Sset
|
||||
|
||||
astb, err := api.ChainReadObj(ctx, k.act.Head)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
alk.Lock()
|
||||
_, ok := actors[addr]
|
||||
if !ok {
|
||||
actors[addr] = map[types.Actor]cid.Cid{}
|
||||
}
|
||||
actors[addr][act] = bh.ParentStateRoot
|
||||
addresses[addr] = address.Undef
|
||||
alk.Unlock()
|
||||
}
|
||||
})
|
||||
|
||||
if err := st.storeActors(actors); err != nil {
|
||||
log.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
log.Infof("Persisting miners")
|
||||
|
||||
miners := map[minerKey]*minerInfo{}
|
||||
|
||||
for addr, m := range actors {
|
||||
for actor, c := range m {
|
||||
if actor.Code != actors2.StorageMinerCodeCid {
|
||||
continue
|
||||
if err := info.state.UnmarshalCBOR(bytes.NewReader(astb)); err != nil {
|
||||
log.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
miners[minerKey{
|
||||
addr: addr,
|
||||
act: actor,
|
||||
stateroot: c,
|
||||
}] = &minerInfo{}
|
||||
}
|
||||
}
|
||||
ib, err := api.ChainReadObj(ctx, info.state.Info)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
par(50, kvmaparr(miners), func(it func() (minerKey, *minerInfo)) {
|
||||
k, info := it()
|
||||
if err := info.info.UnmarshalCBOR(bytes.NewReader(ib)); err != nil {
|
||||
log.Error(err)
|
||||
return
|
||||
}
|
||||
})
|
||||
|
||||
sszs, err := api.StateMinerSectorCount(ctx, k.addr, nil)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
log.Info("Getting receipts")
|
||||
|
||||
receipts := fetchParentReceipts(ctx, api, toSync)
|
||||
|
||||
log.Info("Storing headers")
|
||||
|
||||
if err := st.storeHeaders(toSync, true); err != nil {
|
||||
log.Errorf("%+v", err)
|
||||
return
|
||||
}
|
||||
info.psize = sszs.Pset
|
||||
info.ssize = sszs.Sset
|
||||
|
||||
astb, err := api.ChainReadObj(ctx, k.act.Head)
|
||||
if err != nil {
|
||||
log.Info("Storing address mapping")
|
||||
|
||||
if err := st.storeAddressMap(addresses); err != nil {
|
||||
log.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
if err := info.state.UnmarshalCBOR(bytes.NewReader(astb)); err != nil {
|
||||
log.Info("Storing actors")
|
||||
|
||||
if err := st.storeActors(actors); err != nil {
|
||||
log.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
ib, err := api.ChainReadObj(ctx, info.state.Info)
|
||||
if err != nil {
|
||||
log.Info("Storing miners")
|
||||
|
||||
if err := st.storeMiners(miners); err != nil {
|
||||
log.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
if err := info.info.UnmarshalCBOR(bytes.NewReader(ib)); err != nil {
|
||||
log.Infof("Storing messages")
|
||||
|
||||
if err := st.storeMessages(msgs); err != nil {
|
||||
log.Error(err)
|
||||
return
|
||||
}
|
||||
})
|
||||
|
||||
if err := st.storeMiners(miners); err != nil {
|
||||
log.Error(err)
|
||||
return
|
||||
}
|
||||
log.Info("Storing message inclusions")
|
||||
|
||||
log.Infof("Persisting headers")
|
||||
if err := st.storeHeaders(toSync, true); err != nil {
|
||||
log.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
log.Infof("Getting messages")
|
||||
|
||||
msgs, incls := fetchMessages(ctx, api, toSync)
|
||||
|
||||
if err := st.storeMessages(msgs); err != nil {
|
||||
log.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
if err := st.storeMsgInclusions(incls); err != nil {
|
||||
log.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
log.Infof("Getting parent receipts")
|
||||
|
||||
receipts := fetchParentReceipts(ctx, api, toSync)
|
||||
|
||||
if err := st.storeReceipts(receipts); err != nil {
|
||||
log.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
log.Infof("Resolving addresses")
|
||||
|
||||
for _, message := range msgs {
|
||||
addresses[message.To] = address.Undef
|
||||
addresses[message.From] = address.Undef
|
||||
}
|
||||
|
||||
par(50, kmaparr(addresses), func(addr address.Address) {
|
||||
raddr, err := api.StateLookupID(ctx, addr, nil)
|
||||
if err != nil {
|
||||
log.Warn(err)
|
||||
if err := st.storeMsgInclusions(incls); err != nil {
|
||||
log.Error(err)
|
||||
return
|
||||
}
|
||||
alk.Lock()
|
||||
addresses[addr] = raddr
|
||||
alk.Unlock()
|
||||
})
|
||||
|
||||
if err := st.storeAddressMap(addresses); err != nil {
|
||||
log.Error(err)
|
||||
return
|
||||
log.Infof("Storing parent receipts")
|
||||
|
||||
if err := st.storeReceipts(receipts); err != nil {
|
||||
log.Error(err)
|
||||
return
|
||||
}
|
||||
log.Infof("Sync stage done")
|
||||
}
|
||||
|
||||
log.Infof("Sync done")
|
||||
|
@ -6,6 +6,7 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
|
||||
rice "github.com/GeertJohan/go.rice"
|
||||
"github.com/ipfs/go-cid"
|
||||
@ -42,8 +43,15 @@ func newHandler(api api.FullNode, st *storage) (*handler, error) {
|
||||
"queryNum": h.queryNum,
|
||||
"sizeStr": sizeStr,
|
||||
"strings": h.strings,
|
||||
"qstr": h.qstr,
|
||||
"qstrs": h.qstrs,
|
||||
"messages": h.messages,
|
||||
|
||||
"pageDown": pageDown,
|
||||
"parseInt": func(s string) (int, error) { i, e := strconv.ParseInt(s, 10, 64); return int(i), e },
|
||||
"substr": func(i, j int, s string) string { return s[i:j] },
|
||||
"sub": func(a, b int) int { return a - b }, // TODO: really not builtin?
|
||||
|
||||
"param": func(string) string { return "" }, // replaced in request handler
|
||||
}
|
||||
|
||||
@ -153,10 +161,9 @@ func (h *handler) netPower(slashFilt string) (types.BigInt, error) {
|
||||
if slashFilt != "" {
|
||||
slashFilt = " where " + slashFilt
|
||||
}
|
||||
return h.queryNum(`select sum(power) from (
|
||||
select miner_heads.power, miner_heads.slashed_at, max(height) from miner_heads
|
||||
inner join blocks b on miner_heads.stateroot = b.parentStateRoot
|
||||
group by miner_heads.addr)` + slashFilt)
|
||||
return h.queryNum(`select sum(power) from (select distinct on (addr) power, slashed_at from miner_heads
|
||||
inner join blocks b on miner_heads.stateroot = b.parentStateRoot
|
||||
order by addr, height desc) as p` + slashFilt)
|
||||
}
|
||||
|
||||
func (h *handler) queryNum(q string, p ...interface{}) (types.BigInt, error) {
|
||||
@ -193,7 +200,7 @@ func (h *handler) strings(table string, col string, filter string, args ...inter
|
||||
if len(filter) > 0 {
|
||||
filter = " where " + filter
|
||||
}
|
||||
log.Info("strings qstr ", "select "+col+" from "+table+filter)
|
||||
log.Info("strings qstr ", "select "+col+" from "+table+filter, args)
|
||||
rws, err := h.st.db.Query("select "+col+" from "+table+filter, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@ -209,11 +216,40 @@ func (h *handler) strings(table string, col string, filter string, args ...inter
|
||||
return
|
||||
}
|
||||
|
||||
func (h *handler) qstr(q string, p ...interface{}) (string, error) {
|
||||
// explicitly not caring about sql injection too much, this doesn't take user input
|
||||
|
||||
r, err := h.qstrs(q, 1, p...)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return r[0], nil
|
||||
}
|
||||
|
||||
func (h *handler) qstrs(q string, n int, p ...interface{}) ([]string, error) {
|
||||
// explicitly not caring about sql injection too much, this doesn't take user input
|
||||
|
||||
c := make([]string, n)
|
||||
ia := make([]interface{}, n)
|
||||
for i := range c {
|
||||
ia[i] = &c[i]
|
||||
}
|
||||
err := h.st.db.QueryRow(q, p...).Scan(ia...)
|
||||
if err != nil {
|
||||
log.Error("qnum ", q, p, err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func (h *handler) messages(filter string, args ...interface{}) (out []types.Message, err error) {
|
||||
if len(filter) > 0 {
|
||||
filter = " where " + filter
|
||||
}
|
||||
|
||||
log.Info("select * from messages " + filter)
|
||||
|
||||
rws, err := h.st.db.Query("select * from messages "+filter, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@ -250,4 +286,13 @@ func (h *handler) messages(filter string, args ...interface{}) (out []types.Mess
|
||||
return
|
||||
}
|
||||
|
||||
func pageDown(base, n int) []int {
|
||||
out := make([]int, n)
|
||||
for i := range out {
|
||||
out[i] = base - i
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
var _ http.Handler = &handler{}
|
||||
|
@ -15,8 +15,7 @@
|
||||
<span>Enter destination address:</span>
|
||||
<input type='text' name='address' style="width: 300px">
|
||||
<select name="sectorSize">
|
||||
<option selected value="1073741824">1GiB sectors</option>
|
||||
<option value="34359738368">32GiB sectors</option>
|
||||
<option selected value="34359738368">32GiB sectors</option>
|
||||
</select>
|
||||
<button type='submit'>Create Miner</button>
|
||||
</form>
|
||||
|
@ -6,7 +6,6 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
rice "github.com/GeertJohan/go.rice"
|
||||
@ -16,11 +15,9 @@ import (
|
||||
blockstore "github.com/ipfs/go-ipfs-blockstore"
|
||||
"github.com/libp2p/go-libp2p"
|
||||
"github.com/libp2p/go-libp2p-core/peer"
|
||||
pnet "github.com/libp2p/go-libp2p-pnet"
|
||||
pubsub "github.com/libp2p/go-libp2p-pubsub"
|
||||
|
||||
"github.com/filecoin-project/lotus/build"
|
||||
"github.com/filecoin-project/lotus/node/modules/lp2p"
|
||||
)
|
||||
|
||||
var topic = "/fil/headnotifs/"
|
||||
@ -61,15 +58,9 @@ func main() {
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
protec, err := pnet.NewProtector(strings.NewReader(lp2p.LotusKey))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
host, err := libp2p.New(
|
||||
ctx,
|
||||
libp2p.Defaults,
|
||||
libp2p.PrivateNetwork(protec),
|
||||
)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
|
@ -6,6 +6,11 @@ Build the Lotus Binaries in debug mode, This enables the use of 1024 byte sector
|
||||
make debug
|
||||
```
|
||||
|
||||
Download the 1024 byte parameters:
|
||||
```sh
|
||||
./lotus fetch-params --proving-params 1024
|
||||
```
|
||||
|
||||
Pre-seal some sectors:
|
||||
|
||||
```sh
|
||||
|
7
go.mod
7
go.mod
@ -16,9 +16,9 @@ require (
|
||||
github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2
|
||||
github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03
|
||||
github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce
|
||||
github.com/filecoin-project/go-fil-markets v0.0.0-20200113162334-bcb5c10bdf1b
|
||||
github.com/filecoin-project/go-fil-markets v0.0.0-20200114015428-74d100f305f8
|
||||
github.com/filecoin-project/go-paramfetch v0.0.1
|
||||
github.com/filecoin-project/go-sectorbuilder v0.0.0-20200109194458-9656ce473254
|
||||
github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200114015900-4103afa82689
|
||||
github.com/filecoin-project/go-statestore v0.1.0
|
||||
github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1
|
||||
github.com/go-ole/go-ole v1.2.4 // indirect
|
||||
@ -52,6 +52,7 @@ require (
|
||||
github.com/ipfs/go-merkledag v0.2.4
|
||||
github.com/ipfs/go-path v0.0.7
|
||||
github.com/ipfs/go-unixfs v0.2.2-0.20190827150610-868af2e9e5cb
|
||||
github.com/lib/pq v1.2.0
|
||||
github.com/libp2p/go-libp2p v0.4.2
|
||||
github.com/libp2p/go-libp2p-circuit v0.1.4
|
||||
github.com/libp2p/go-libp2p-connmgr v0.1.0
|
||||
@ -61,7 +62,6 @@ require (
|
||||
github.com/libp2p/go-libp2p-mplex v0.2.1
|
||||
github.com/libp2p/go-libp2p-peer v0.2.0
|
||||
github.com/libp2p/go-libp2p-peerstore v0.1.4
|
||||
github.com/libp2p/go-libp2p-pnet v0.1.0
|
||||
github.com/libp2p/go-libp2p-pubsub v0.2.3
|
||||
github.com/libp2p/go-libp2p-quic-transport v0.1.1
|
||||
github.com/libp2p/go-libp2p-record v0.1.1
|
||||
@ -70,7 +70,6 @@ require (
|
||||
github.com/libp2p/go-libp2p-tls v0.1.0
|
||||
github.com/libp2p/go-libp2p-yamux v0.2.1
|
||||
github.com/libp2p/go-maddr-filter v0.0.5
|
||||
github.com/mattn/go-sqlite3 v1.12.0
|
||||
github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1
|
||||
github.com/minio/sha256-simd v0.1.1
|
||||
github.com/mitchellh/go-homedir v1.1.0
|
||||
|
24
go.sum
24
go.sum
@ -10,6 +10,7 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
|
||||
github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM=
|
||||
github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
|
||||
github.com/GeertJohan/go.incremental v1.0.0 h1:7AH+pY1XUgQE4Y1HcXYaMqAI0m9yrFqo/jt0CW30vsg=
|
||||
github.com/GeertJohan/go.incremental v1.0.0 h1:7AH+pY1XUgQE4Y1HcXYaMqAI0m9yrFqo/jt0CW30vsg=
|
||||
github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0=
|
||||
github.com/GeertJohan/go.rice v1.0.0 h1:KkI6O9uMaQU3VEKaj01ulavtF7o1fWT7+pk/4voiMLQ=
|
||||
github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0=
|
||||
@ -72,8 +73,6 @@ github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018 h1:6xT9KW8zLC5IlbaIF5Q7JNieBoACT7iW0YTxQHR0in0=
|
||||
github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018/go.mod h1:rQYf4tfk5sSwFsnDg3qYaBxSjsD9S8+59vW0dKUgme4=
|
||||
github.com/dgraph-io/badger v1.5.5-0.20190226225317-8115aed38f8f/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ=
|
||||
github.com/dgraph-io/badger v1.6.0-rc1/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
|
||||
github.com/dgraph-io/badger v1.6.0 h1:DshxFxZWXUcO0xX476VJC07Xsr6ZCBVRHKZ93Oh7Evo=
|
||||
@ -116,10 +115,18 @@ github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce/
|
||||
github.com/filecoin-project/go-fil-markets v0.0.0-20200113162334-bcb5c10bdf1b h1:UVaUNbV4sViqKN0LfrqLbRhoQIsfy+IY4cQPAid2Hh0=
|
||||
github.com/filecoin-project/go-fil-markets v0.0.0-20200113162334-bcb5c10bdf1b/go.mod h1:ZvtKir+NmP1b6bQZyS2jCX9yqve71iaIwFag+tMT/iA=
|
||||
github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA=
|
||||
github.com/filecoin-project/go-fil-markets v0.0.0-20200114015428-74d100f305f8 h1:g3oodvSz+Ou+ObwcVBB2wyt8SHdWpwzMiNJ19U1zZNA=
|
||||
github.com/filecoin-project/go-fil-markets v0.0.0-20200114015428-74d100f305f8/go.mod h1:c8NTjvFVy1Ud02mmGDjOiMeawY2t6ALfrrdvAB01FQc=
|
||||
github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA=
|
||||
github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA=
|
||||
github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU=
|
||||
github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU=
|
||||
github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus=
|
||||
github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ=
|
||||
github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ=
|
||||
github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA=
|
||||
github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA=
|
||||
github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU=
|
||||
github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU=
|
||||
github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU=
|
||||
github.com/filecoin-project/go-paramfetch v0.0.1 h1:gV7bs5YaqlgpGFMiLxInGK2L1FyCXUE0rimz4L7ghoE=
|
||||
@ -128,6 +135,12 @@ github.com/filecoin-project/go-sectorbuilder v0.0.0-20200109194458-9656ce473254
|
||||
github.com/filecoin-project/go-sectorbuilder v0.0.0-20200109194458-9656ce473254/go.mod h1:3OZ4E3B2OuwhJjtxR4r7hPU9bCfB+A+hm4alLEsaeDc=
|
||||
github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIiWBRilQjQ+5IiwdQ=
|
||||
github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI=
|
||||
github.com/filecoin-project/go-sectorbuilder v0.0.1 h1:yiLSEprWA1E43DFTSCXLSuCstYuDKiI6RCXiYz4GaRs=
|
||||
github.com/filecoin-project/go-sectorbuilder v0.0.1/go.mod h1:3OZ4E3B2OuwhJjtxR4r7hPU9bCfB+A+hm4alLEsaeDc=
|
||||
github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200114015900-4103afa82689 h1:2cT5bhm/5I0RY+HBIPdRRrtjCwLj33Qx6DHRs9TCslY=
|
||||
github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200114015900-4103afa82689/go.mod h1:3OZ4E3B2OuwhJjtxR4r7hPU9bCfB+A+hm4alLEsaeDc=
|
||||
github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIiWBRilQjQ+5IiwdQ=
|
||||
github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI=
|
||||
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 h1:EzDjxMg43q1tA2c0MV3tNbaontnHLplHyFF6M5KiVP0=
|
||||
@ -362,6 +375,8 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
|
||||
github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
|
||||
github.com/libp2p/go-addr-util v0.0.1 h1:TpTQm9cXVRVSKsYbgQ7GKc3KbbHVTnbostgGaDEP+88=
|
||||
github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ=
|
||||
github.com/libp2p/go-buffer-pool v0.0.1/go.mod h1:xtyIz9PMobb13WaxR6Zo1Pd1zXJKYg0a8KiIvDp3TzQ=
|
||||
@ -456,8 +471,6 @@ github.com/libp2p/go-libp2p-peerstore v0.1.3 h1:wMgajt1uM2tMiqf4M+4qWKVyyFc8SfA+
|
||||
github.com/libp2p/go-libp2p-peerstore v0.1.3/go.mod h1:BJ9sHlm59/80oSkpWgr1MyY1ciXAXV397W6h1GH/uKI=
|
||||
github.com/libp2p/go-libp2p-peerstore v0.1.4 h1:d23fvq5oYMJ/lkkbO4oTwBp/JP+I/1m5gZJobNXCE/k=
|
||||
github.com/libp2p/go-libp2p-peerstore v0.1.4/go.mod h1:+4BDbDiiKf4PzpANZDAT+knVdLxvqh7hXOujessqdzs=
|
||||
github.com/libp2p/go-libp2p-pnet v0.1.0 h1:kRUES28dktfnHNIRW4Ro78F7rKBHBiw5MJpl0ikrLIA=
|
||||
github.com/libp2p/go-libp2p-pnet v0.1.0/go.mod h1:ZkyZw3d0ZFOex71halXRihWf9WH/j3OevcJdTmD0lyE=
|
||||
github.com/libp2p/go-libp2p-protocol v0.0.1/go.mod h1:Af9n4PiruirSDjHycM1QuiMi/1VZNHYcK8cLgFJLZ4s=
|
||||
github.com/libp2p/go-libp2p-protocol v0.1.0/go.mod h1:KQPHpAabB57XQxGrXCNvbL6UEXfQqUgC/1adR2Xtflk=
|
||||
github.com/libp2p/go-libp2p-pubsub v0.2.3 h1:qJRnRnM7Z4xnHb4i6EBb3DKQXRPgtFWlKP4AmfJudLQ=
|
||||
@ -567,8 +580,6 @@ github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg
|
||||
github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
|
||||
github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54=
|
||||
github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
|
||||
github.com/mattn/go-sqlite3 v1.12.0 h1:u/x3mp++qUxvYfulZ4HKOvVO0JWhk7HtE8lWhbGz/Do=
|
||||
github.com/mattn/go-sqlite3 v1.12.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
|
||||
github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
|
||||
@ -715,6 +726,7 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb
|
||||
github.com/urfave/cli/v2 v2.0.0 h1:+HU9SCbu8GnEUFtIBfuUNXN39ofWViIEJIp6SURMpCg=
|
||||
github.com/urfave/cli/v2 v2.0.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
|
||||
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
|
||||
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
|
||||
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||
github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8=
|
||||
github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
|
||||
|
@ -23,6 +23,7 @@ import (
|
||||
"github.com/filecoin-project/go-fil-markets/retrievalmarket/discovery"
|
||||
"github.com/filecoin-project/go-fil-markets/storagemarket"
|
||||
deals "github.com/filecoin-project/go-fil-markets/storagemarket/impl"
|
||||
|
||||
"github.com/filecoin-project/lotus/api"
|
||||
"github.com/filecoin-project/lotus/chain"
|
||||
"github.com/filecoin-project/lotus/chain/blocksync"
|
||||
@ -33,6 +34,7 @@ import (
|
||||
"github.com/filecoin-project/lotus/chain/stmgr"
|
||||
"github.com/filecoin-project/lotus/chain/store"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
"github.com/filecoin-project/lotus/chain/vm"
|
||||
"github.com/filecoin-project/lotus/chain/wallet"
|
||||
"github.com/filecoin-project/lotus/markets/storageadapter"
|
||||
"github.com/filecoin-project/lotus/miner"
|
||||
@ -58,7 +60,6 @@ type special struct{ id int }
|
||||
//nolint:golint
|
||||
var (
|
||||
DefaultTransportsKey = special{0} // Libp2p option
|
||||
PNetKey = special{1} // Option + multiret
|
||||
DiscoveryHandlerKey = special{2} // Private type
|
||||
AddrsFactoryKey = special{3} // Libp2p option
|
||||
SmuxTransportKey = special{4} // Libp2p option
|
||||
@ -142,7 +143,6 @@ func libp2p() Option {
|
||||
Override(new(peerstore.Peerstore), pstoremem.NewPeerstore),
|
||||
|
||||
Override(DefaultTransportsKey, lp2p.DefaultTransports),
|
||||
Override(PNetKey, lp2p.PNet),
|
||||
|
||||
Override(new(lp2p.RawHost), lp2p.Host),
|
||||
Override(new(host.Host), lp2p.RoutedHost),
|
||||
@ -195,6 +195,8 @@ func Online() Option {
|
||||
|
||||
Override(HandleIncomingMessagesKey, modules.HandleIncomingMessages),
|
||||
|
||||
Override(new(sectorbuilder.Verifier), sectorbuilder.ProofVerifier),
|
||||
Override(new(*types.VMSyscalls), vm.Syscalls),
|
||||
Override(new(*store.ChainStore), modules.ChainStore),
|
||||
Override(new(*stmgr.StateManager), stmgr.NewStateManager),
|
||||
Override(new(*wallet.Wallet), wallet.NewWallet),
|
||||
@ -241,7 +243,7 @@ func Online() Option {
|
||||
|
||||
// Storage miner
|
||||
ApplyIf(func(s *Settings) bool { return s.nodeType == repo.StorageMiner },
|
||||
Override(new(storage.SectorBuilder), modules.SectorBuilder),
|
||||
Override(new(sectorbuilder.Interface), modules.SectorBuilder),
|
||||
Override(new(*sectorblocks.SectorBlocks), sectorblocks.NewSectorBlocks),
|
||||
Override(new(storage.TicketFn), modules.SealTicketGen),
|
||||
Override(new(*storage.Miner), modules.StorageMiner),
|
||||
|
@ -26,7 +26,7 @@ type StorageMinerAPI struct {
|
||||
CommonAPI
|
||||
|
||||
SectorBuilderConfig *sectorbuilder.Config
|
||||
SectorBuilder storage.SectorBuilder
|
||||
SectorBuilder sectorbuilder.Interface
|
||||
SectorBlocks *sectorblocks.SectorBlocks
|
||||
|
||||
Miner *storage.Miner
|
||||
|
@ -73,8 +73,8 @@ func ChainBlockservice(bs dtypes.ChainBlockstore, rem dtypes.ChainExchange) dtyp
|
||||
return blockservice.New(bs, rem)
|
||||
}
|
||||
|
||||
func ChainStore(lc fx.Lifecycle, bs dtypes.ChainBlockstore, ds dtypes.MetadataDS) *store.ChainStore {
|
||||
chain := store.NewChainStore(bs, ds)
|
||||
func ChainStore(lc fx.Lifecycle, bs dtypes.ChainBlockstore, ds dtypes.MetadataDS, syscalls *types.VMSyscalls) *store.ChainStore {
|
||||
chain := store.NewChainStore(bs, ds, syscalls)
|
||||
|
||||
if err := chain.Load(); err != nil {
|
||||
log.Warnf("loading chain state from disk: %s", err)
|
||||
|
@ -5,40 +5,36 @@ import (
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
|
||||
"github.com/filecoin-project/lotus/markets/retrievaladapter"
|
||||
|
||||
"github.com/filecoin-project/go-data-transfer/impl/graphsync"
|
||||
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
|
||||
"github.com/filecoin-project/go-fil-markets/retrievalmarket/discovery"
|
||||
retrievalimpl "github.com/filecoin-project/go-fil-markets/retrievalmarket/impl"
|
||||
|
||||
"github.com/filecoin-project/go-fil-markets/storagemarket"
|
||||
deals "github.com/filecoin-project/go-fil-markets/storagemarket/impl"
|
||||
storageimpl "github.com/filecoin-project/go-fil-markets/storagemarket/impl"
|
||||
"github.com/filecoin-project/go-statestore"
|
||||
"github.com/filecoin-project/lotus/node/modules/helpers"
|
||||
"github.com/filecoin-project/lotus/paych"
|
||||
|
||||
"github.com/ipfs/go-bitswap"
|
||||
"github.com/ipfs/go-bitswap/network"
|
||||
graphsync "github.com/ipfs/go-graphsync/impl"
|
||||
"github.com/ipfs/go-graphsync/ipldbridge"
|
||||
gsnet "github.com/ipfs/go-graphsync/network"
|
||||
"github.com/ipfs/go-graphsync/storeutil"
|
||||
"github.com/libp2p/go-libp2p-core/host"
|
||||
"github.com/libp2p/go-libp2p-core/routing"
|
||||
|
||||
"github.com/ipfs/go-blockservice"
|
||||
"github.com/ipfs/go-datastore"
|
||||
"github.com/ipfs/go-datastore/namespace"
|
||||
"github.com/ipfs/go-filestore"
|
||||
graphsync "github.com/ipfs/go-graphsync/impl"
|
||||
"github.com/ipfs/go-graphsync/ipldbridge"
|
||||
gsnet "github.com/ipfs/go-graphsync/network"
|
||||
"github.com/ipfs/go-graphsync/storeutil"
|
||||
blockstore "github.com/ipfs/go-ipfs-blockstore"
|
||||
"github.com/ipfs/go-merkledag"
|
||||
"github.com/libp2p/go-libp2p-core/host"
|
||||
"github.com/libp2p/go-libp2p-core/routing"
|
||||
"go.uber.org/fx"
|
||||
|
||||
"github.com/filecoin-project/go-data-transfer/impl/graphsync"
|
||||
deals "github.com/filecoin-project/go-fil-markets/storagemarket/impl"
|
||||
storageimpl "github.com/filecoin-project/go-fil-markets/storagemarket/impl"
|
||||
"github.com/filecoin-project/lotus/markets/retrievaladapter"
|
||||
payapi "github.com/filecoin-project/lotus/node/impl/paych"
|
||||
"github.com/filecoin-project/lotus/node/modules/dtypes"
|
||||
"github.com/filecoin-project/lotus/node/modules/helpers"
|
||||
"github.com/filecoin-project/lotus/node/repo"
|
||||
"github.com/filecoin-project/lotus/paych"
|
||||
)
|
||||
|
||||
func ClientFstore(r repo.LockedRepo) (dtypes.ClientFilestore, error) {
|
||||
|
@ -79,6 +79,7 @@ func DHTRouting(client bool) interface{} {
|
||||
dhtopts.Client(client),
|
||||
dhtopts.Datastore(dstore),
|
||||
dhtopts.Validator(validator),
|
||||
dhtopts.Protocols("/lotus/kad/1.0.0"),
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
|
@ -1,63 +0,0 @@
|
||||
package lp2p
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/libp2p/go-libp2p"
|
||||
pnet "github.com/libp2p/go-libp2p-pnet"
|
||||
)
|
||||
|
||||
var LotusKey = "/key/swarm/psk/1.0.0/\n/base16/\n20c72388e6299c7bbc1b501fdcc8abe4f89f798e9b93b2d2bc02e3c29b6a088e"
|
||||
|
||||
type PNetFingerprint []byte
|
||||
|
||||
func PNet() (opts Libp2pOpts, fp PNetFingerprint, err error) {
|
||||
protec, err := pnet.NewProtector(strings.NewReader(LotusKey))
|
||||
if err != nil {
|
||||
return opts, nil, fmt.Errorf("failed to configure private network: %s", err)
|
||||
}
|
||||
fp = protec.Fingerprint()
|
||||
|
||||
opts.Opts = append(opts.Opts, libp2p.PrivateNetwork(protec))
|
||||
return opts, fp, nil
|
||||
}
|
||||
|
||||
/*
|
||||
func PNetChecker(repo repo.Repo, ph host.Host, lc fx.Lifecycle) error {
|
||||
// TODO: better check?
|
||||
swarmkey, err := repo.SwarmKey()
|
||||
if err != nil || swarmkey == nil {
|
||||
return err
|
||||
}
|
||||
|
||||
done := make(chan struct{})
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(_ context.Context) error {
|
||||
go func() {
|
||||
t := time.NewTicker(30 * time.Second)
|
||||
defer t.Stop()
|
||||
|
||||
<-t.C // swallow one tick
|
||||
for {
|
||||
select {
|
||||
case <-t.C:
|
||||
if len(ph.Network().Peers()) == 0 {
|
||||
log.Warn("We are in private network and have no peers.")
|
||||
log.Warn("This might be configuration mistake.")
|
||||
}
|
||||
case <-done:
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
},
|
||||
OnStop: func(_ context.Context) error {
|
||||
close(done)
|
||||
return nil
|
||||
},
|
||||
})
|
||||
return nil
|
||||
}
|
||||
*/
|
@ -98,7 +98,7 @@ func SectorBuilderConfig(storagePath string, threads uint, noprecommit, nocommit
|
||||
}
|
||||
}
|
||||
|
||||
func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api api.FullNode, h host.Host, ds dtypes.MetadataDS, sb storage.SectorBuilder, tktFn storage.TicketFn) (*storage.Miner, error) {
|
||||
func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api api.FullNode, h host.Host, ds dtypes.MetadataDS, sb sectorbuilder.Interface, tktFn storage.TicketFn) (*storage.Miner, error) {
|
||||
maddr, err := minerAddrFromDS(ds)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
@ -30,8 +30,8 @@ import (
|
||||
|
||||
var glog = logging.Logger("genesis")
|
||||
|
||||
func MakeGenesisMem(out io.Writer, gmc *gen.GenMinerCfg) func(bs dtypes.ChainBlockstore, w *wallet.Wallet) modules.Genesis {
|
||||
return func(bs dtypes.ChainBlockstore, w *wallet.Wallet) modules.Genesis {
|
||||
func MakeGenesisMem(out io.Writer, gmc *gen.GenMinerCfg) func(bs dtypes.ChainBlockstore, w *wallet.Wallet, syscalls *types.VMSyscalls) modules.Genesis {
|
||||
return func(bs dtypes.ChainBlockstore, w *wallet.Wallet, syscalls *types.VMSyscalls) modules.Genesis {
|
||||
return func() (*types.BlockHeader, error) {
|
||||
glog.Warn("Generating new random genesis block, note that this SHOULD NOT happen unless you are setting up new network")
|
||||
defk, err := w.GenerateKey(types.KTBLS)
|
||||
@ -51,7 +51,7 @@ func MakeGenesisMem(out io.Writer, gmc *gen.GenMinerCfg) func(bs dtypes.ChainBlo
|
||||
alloc[waddr] = types.FromFil(10000)
|
||||
}
|
||||
|
||||
b, err := gen.MakeGenesisBlock(bs, alloc, gmc, 100000)
|
||||
b, err := gen.MakeGenesisBlock(bs, syscalls, alloc, gmc, 100000)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -68,8 +68,8 @@ func MakeGenesisMem(out io.Writer, gmc *gen.GenMinerCfg) func(bs dtypes.ChainBlo
|
||||
}
|
||||
}
|
||||
|
||||
func MakeGenesis(outFile, presealInfo, timestamp string) func(bs dtypes.ChainBlockstore, w *wallet.Wallet) modules.Genesis {
|
||||
return func(bs dtypes.ChainBlockstore, w *wallet.Wallet) modules.Genesis {
|
||||
func MakeGenesis(outFile, presealInfo, timestamp string) func(bs dtypes.ChainBlockstore, w *wallet.Wallet, syscalls *types.VMSyscalls) modules.Genesis {
|
||||
return func(bs dtypes.ChainBlockstore, w *wallet.Wallet, syscalls *types.VMSyscalls) modules.Genesis {
|
||||
return func() (*types.BlockHeader, error) {
|
||||
glog.Warn("Generating new random genesis block, note that this SHOULD NOT happen unless you are setting up new network")
|
||||
presealInfo, err := homedir.Expand(presealInfo)
|
||||
@ -130,7 +130,7 @@ func MakeGenesis(outFile, presealInfo, timestamp string) func(bs dtypes.ChainBlo
|
||||
ts = uint64(t.Unix())
|
||||
}
|
||||
|
||||
b, err := gen.MakeGenesisBlock(bs, addrs, gmc, ts)
|
||||
b, err := gen.MakeGenesisBlock(bs, syscalls, addrs, gmc, ts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
@ -8,6 +8,7 @@ import (
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/filecoin-project/go-address"
|
||||
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
|
||||
@ -36,6 +37,7 @@ import (
|
||||
"github.com/filecoin-project/lotus/node/modules"
|
||||
modtest "github.com/filecoin-project/lotus/node/modules/testing"
|
||||
"github.com/filecoin-project/lotus/node/repo"
|
||||
"github.com/filecoin-project/lotus/storage/sbmock"
|
||||
)
|
||||
|
||||
func init() {
|
||||
@ -45,7 +47,7 @@ func init() {
|
||||
build.MinimumMinerPower = 1024
|
||||
}
|
||||
|
||||
func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, act address.Address, pk crypto.PrivKey, tnd test.TestNode, mn mocknet.Mocknet) test.TestStorageNode {
|
||||
func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, act address.Address, pk crypto.PrivKey, tnd test.TestNode, mn mocknet.Mocknet, opts node.Option) test.TestStorageNode {
|
||||
r := repo.NewMemory(nil)
|
||||
|
||||
lr, err := r.Lock(repo.StorageMiner)
|
||||
@ -105,6 +107,8 @@ func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, a
|
||||
|
||||
node.Override(new(api.FullNode), tnd),
|
||||
node.Override(new(*miner.Miner), miner.NewTestMiner(mineBlock, act)),
|
||||
|
||||
opts,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to construct node: %v", err)
|
||||
@ -214,7 +218,7 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te
|
||||
genMiner := gmc.MinerAddrs[i]
|
||||
wa := gmc.PreSeals[genMiner.String()].Worker
|
||||
|
||||
storers[i] = testStorageNode(ctx, t, wa, genMiner, pk, f, mn)
|
||||
storers[i] = testStorageNode(ctx, t, wa, genMiner, pk, f, mn, node.Options())
|
||||
|
||||
sma := storers[i].StorageMiner.(*impl.StorageMinerAPI)
|
||||
|
||||
@ -247,6 +251,106 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te
|
||||
return fulls, storers
|
||||
}
|
||||
|
||||
func mockSbBuilder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.TestStorageNode) {
|
||||
ctx := context.Background()
|
||||
mn := mocknet.New(ctx)
|
||||
|
||||
fulls := make([]test.TestNode, nFull)
|
||||
storers := make([]test.TestStorageNode, len(storage))
|
||||
|
||||
pk, _, err := crypto.GenerateEd25519Key(rand.Reader)
|
||||
require.NoError(t, err)
|
||||
|
||||
minerPid, err := peer.IDFromPrivateKey(pk)
|
||||
require.NoError(t, err)
|
||||
|
||||
var genbuf bytes.Buffer
|
||||
|
||||
if len(storage) > 1 {
|
||||
panic("need more peer IDs")
|
||||
}
|
||||
// PRESEAL SECTION, TRY TO REPLACE WITH BETTER IN THE FUTURE
|
||||
// TODO: would be great if there was a better way to fake the preseals
|
||||
gmc := &gen.GenMinerCfg{
|
||||
PeerIDs: []peer.ID{minerPid}, // TODO: if we have more miners, need more peer IDs
|
||||
PreSeals: map[string]genesis.GenesisMiner{},
|
||||
}
|
||||
|
||||
var presealDirs []string
|
||||
for i := 0; i < len(storage); i++ {
|
||||
maddr, err := address.NewIDAddress(300 + uint64(i))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tdir, err := ioutil.TempDir("", "preseal-memgen")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
genm, err := sbmock.PreSeal(1024, maddr, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
presealDirs = append(presealDirs, tdir)
|
||||
gmc.MinerAddrs = append(gmc.MinerAddrs, maddr)
|
||||
gmc.PreSeals[maddr.String()] = *genm
|
||||
}
|
||||
|
||||
// END PRESEAL SECTION
|
||||
|
||||
for i := 0; i < nFull; i++ {
|
||||
var genesis node.Option
|
||||
if i == 0 {
|
||||
genesis = node.Override(new(modules.Genesis), modtest.MakeGenesisMem(&genbuf, gmc))
|
||||
} else {
|
||||
genesis = node.Override(new(modules.Genesis), modules.LoadGenesis(genbuf.Bytes()))
|
||||
}
|
||||
|
||||
var err error
|
||||
// TODO: Don't ignore stop
|
||||
_, err = node.New(ctx,
|
||||
node.FullAPI(&fulls[i].FullNode),
|
||||
node.Online(),
|
||||
node.Repo(repo.NewMemory(nil)),
|
||||
node.MockHost(mn),
|
||||
node.Test(),
|
||||
|
||||
node.Override(new(sectorbuilder.Verifier), sbmock.MockVerifier),
|
||||
|
||||
genesis,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
for i, full := range storage {
|
||||
// TODO: support non-bootstrap miners
|
||||
if i != 0 {
|
||||
t.Fatal("only one storage node supported")
|
||||
}
|
||||
if full != 0 {
|
||||
t.Fatal("storage nodes only supported on the first full node")
|
||||
}
|
||||
|
||||
f := fulls[full]
|
||||
|
||||
genMiner := gmc.MinerAddrs[i]
|
||||
wa := gmc.PreSeals[genMiner.String()].Worker
|
||||
|
||||
storers[i] = testStorageNode(ctx, t, wa, genMiner, pk, f, mn, node.Options(
|
||||
node.Override(new(sectorbuilder.Interface), sbmock.NewMockSectorBuilder(5, build.SectorSizes[0])),
|
||||
))
|
||||
}
|
||||
|
||||
if err := mn.LinkAll(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
return fulls, storers
|
||||
}
|
||||
|
||||
func TestAPI(t *testing.T) {
|
||||
test.TestApis(t, builder)
|
||||
}
|
||||
@ -289,8 +393,12 @@ func TestAPIRPC(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestAPIDealFlow(t *testing.T) {
|
||||
test.TestDealFlow(t, mockSbBuilder, 10 * time.Millisecond)
|
||||
}
|
||||
|
||||
func TestAPIDealFlowReal(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping test in short mode")
|
||||
}
|
||||
test.TestDealFlow(t, builder)
|
||||
test.TestDealFlow(t, builder, time.Second)
|
||||
}
|
||||
|
@ -8,6 +8,8 @@ import (
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"github.com/filecoin-project/go-address"
|
||||
"github.com/filecoin-project/go-sectorbuilder"
|
||||
|
||||
"github.com/filecoin-project/lotus/build"
|
||||
"github.com/filecoin-project/lotus/chain/store"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
@ -19,7 +21,7 @@ const StartConfidence = 4 // TODO: config
|
||||
|
||||
type fpostScheduler struct {
|
||||
api storageMinerApi
|
||||
sb SectorBuilder
|
||||
sb sectorbuilder.Interface
|
||||
|
||||
actor address.Address
|
||||
worker address.Address
|
||||
|
@ -3,7 +3,6 @@ package storage
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"reflect"
|
||||
"time"
|
||||
|
||||
@ -38,7 +37,7 @@ type Miner struct {
|
||||
worker address.Address
|
||||
|
||||
// Sealing
|
||||
sb SectorBuilder
|
||||
sb sectorbuilder.Interface
|
||||
sectors *statemachine.StateGroup
|
||||
tktFn TicketFn
|
||||
|
||||
@ -72,24 +71,7 @@ type storageMinerApi interface {
|
||||
WalletHas(context.Context, address.Address) (bool, error)
|
||||
}
|
||||
|
||||
type SectorBuilder interface {
|
||||
RateLimit() func()
|
||||
AddPiece(uint64, uint64, io.Reader, []uint64) (sectorbuilder.PublicPieceInfo, error)
|
||||
SectorSize() uint64
|
||||
AcquireSectorId() (uint64, error)
|
||||
Scrub(sectorbuilder.SortedPublicSectorInfo) []*sectorbuilder.Fault
|
||||
GenerateFallbackPoSt(sectorbuilder.SortedPublicSectorInfo, [sectorbuilder.CommLen]byte, []uint64) ([]sectorbuilder.EPostCandidate, []byte, error)
|
||||
SealPreCommit(context.Context, uint64, sectorbuilder.SealTicket, []sectorbuilder.PublicPieceInfo) (sectorbuilder.RawSealPreCommitOutput, error)
|
||||
SealCommit(context.Context, uint64, sectorbuilder.SealTicket, sectorbuilder.SealSeed, []sectorbuilder.PublicPieceInfo, sectorbuilder.RawSealPreCommitOutput) ([]byte, error)
|
||||
|
||||
// Not so sure about these being on the interface
|
||||
GetPath(string, string) (string, error)
|
||||
WorkerStats() sectorbuilder.WorkerStats
|
||||
AddWorker(context.Context, sectorbuilder.WorkerCfg) (<-chan sectorbuilder.WorkerTask, error)
|
||||
TaskDone(context.Context, uint64, sectorbuilder.SealRes) error
|
||||
}
|
||||
|
||||
func NewMiner(api storageMinerApi, addr address.Address, h host.Host, ds datastore.Batching, sb SectorBuilder, tktFn TicketFn) (*Miner, error) {
|
||||
func NewMiner(api storageMinerApi, addr address.Address, h host.Host, ds datastore.Batching, sb sectorbuilder.Interface, tktFn TicketFn) (*Miner, error) {
|
||||
m := &Miner{
|
||||
api: api,
|
||||
|
||||
@ -166,11 +148,11 @@ func (m *Miner) runPreflightChecks(ctx context.Context) error {
|
||||
}
|
||||
|
||||
type SectorBuilderEpp struct {
|
||||
sb *sectorbuilder.SectorBuilder
|
||||
sb sectorbuilder.Interface
|
||||
}
|
||||
|
||||
func NewElectionPoStProver(sb SectorBuilder) *SectorBuilderEpp {
|
||||
return &SectorBuilderEpp{sb.(*sectorbuilder.SectorBuilder)}
|
||||
func NewElectionPoStProver(sb sectorbuilder.Interface) *SectorBuilderEpp {
|
||||
return &SectorBuilderEpp{sb}
|
||||
}
|
||||
|
||||
var _ gen.ElectionPoStProver = (*SectorBuilderEpp)(nil)
|
||||
|
52
storage/sbmock/preseal.go
Normal file
52
storage/sbmock/preseal.go
Normal file
@ -0,0 +1,52 @@
|
||||
package sbmock
|
||||
|
||||
import (
|
||||
"math"
|
||||
|
||||
"github.com/filecoin-project/go-address"
|
||||
"github.com/filecoin-project/go-sectorbuilder"
|
||||
|
||||
"github.com/filecoin-project/lotus/chain/actors"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
"github.com/filecoin-project/lotus/chain/wallet"
|
||||
"github.com/filecoin-project/lotus/genesis"
|
||||
)
|
||||
|
||||
func PreSeal(ssize uint64, maddr address.Address, sectors int) (*genesis.GenesisMiner, error) {
|
||||
k, err := wallet.GenerateKey(types.KTBLS)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
genm := &genesis.GenesisMiner{
|
||||
Owner: k.Address,
|
||||
Worker: k.Address,
|
||||
SectorSize: ssize,
|
||||
Sectors: make([]*genesis.PreSeal, sectors),
|
||||
Key: k.KeyInfo,
|
||||
}
|
||||
|
||||
for i := range genm.Sectors {
|
||||
preseal := &genesis.PreSeal{}
|
||||
sdata := randB(sectorbuilder.UserBytesForSectorSize(ssize))
|
||||
|
||||
preseal.CommD = commD(sdata)
|
||||
preseal.CommR = commDR(preseal.CommD[:])
|
||||
preseal.SectorID = uint64(i + 1)
|
||||
preseal.Deal = actors.StorageDealProposal{
|
||||
PieceRef: preseal.CommD[:],
|
||||
PieceSize: sectorbuilder.UserBytesForSectorSize(ssize),
|
||||
Client: maddr,
|
||||
Provider: maddr,
|
||||
ProposalExpiration: math.MaxUint64,
|
||||
Duration: math.MaxUint64,
|
||||
StoragePricePerEpoch: types.NewInt(0),
|
||||
StorageCollateral: types.NewInt(0),
|
||||
ProposerSignature: nil,
|
||||
}
|
||||
|
||||
genm.Sectors[i] = preseal
|
||||
}
|
||||
|
||||
return genm, nil
|
||||
}
|
@ -6,19 +6,16 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"math/big"
|
||||
"math/rand"
|
||||
"sync"
|
||||
|
||||
ffi "github.com/filecoin-project/filecoin-ffi"
|
||||
"github.com/filecoin-project/go-address"
|
||||
"github.com/filecoin-project/go-sectorbuilder"
|
||||
"golang.org/x/xerrors"
|
||||
)
|
||||
|
||||
func randComm() [sectorbuilder.CommLen]byte {
|
||||
var out [sectorbuilder.CommLen]byte
|
||||
rand.Read(out[:])
|
||||
return out
|
||||
}
|
||||
|
||||
type SBMock struct {
|
||||
sectors map[uint64]*sectorState
|
||||
sectorSize uint64
|
||||
@ -28,11 +25,13 @@ type SBMock struct {
|
||||
lk sync.Mutex
|
||||
}
|
||||
|
||||
type mockVerif struct{}
|
||||
|
||||
func NewMockSectorBuilder(threads int, ssize uint64) *SBMock {
|
||||
return &SBMock{
|
||||
sectors: make(map[uint64]*sectorState),
|
||||
sectorSize: ssize,
|
||||
nextSectorID: 0,
|
||||
nextSectorID: 5,
|
||||
rateLimit: make(chan struct{}, threads),
|
||||
}
|
||||
}
|
||||
@ -81,8 +80,8 @@ func (sb *SBMock) AddPiece(size uint64, sectorId uint64, r io.Reader, existingPi
|
||||
|
||||
ss.pieces = append(ss.pieces, b)
|
||||
return sectorbuilder.PublicPieceInfo{
|
||||
Size: size,
|
||||
// TODO: should we compute a commP? maybe do it when we need it
|
||||
Size: size,
|
||||
CommP: commD(b),
|
||||
}, nil
|
||||
}
|
||||
|
||||
@ -158,9 +157,22 @@ func (sb *SBMock) SealPreCommit(ctx context.Context, sid uint64, ticket sectorbu
|
||||
|
||||
ss.state = statePreCommit
|
||||
|
||||
pis := make([]ffi.PublicPieceInfo, len(ss.pieces))
|
||||
for i, piece := range ss.pieces {
|
||||
pis[i] = ffi.PublicPieceInfo{
|
||||
Size: uint64(len(piece)),
|
||||
CommP: commD(piece),
|
||||
}
|
||||
}
|
||||
|
||||
commd, err := MockVerifier.GenerateDataCommitment(sb.sectorSize, pis)
|
||||
if err != nil {
|
||||
return sectorbuilder.RawSealPreCommitOutput{}, err
|
||||
}
|
||||
|
||||
return sectorbuilder.RawSealPreCommitOutput{
|
||||
CommD: randComm(),
|
||||
CommR: randComm(),
|
||||
CommD: commd,
|
||||
CommR: commDR(commd[:]),
|
||||
}, nil
|
||||
}
|
||||
|
||||
@ -184,9 +196,11 @@ func (sb *SBMock) SealCommit(ctx context.Context, sid uint64, ticket sectorbuild
|
||||
|
||||
opFinishWait(ctx)
|
||||
|
||||
buf := make([]byte, 32)
|
||||
rand.Read(buf)
|
||||
return buf, nil
|
||||
var out [32]byte
|
||||
for i := range out {
|
||||
out[i] = precommit.CommD[i] + precommit.CommR[31-i] - ticket.TicketBytes[i]*seed.TicketBytes[i]
|
||||
}
|
||||
return out[:], nil
|
||||
}
|
||||
|
||||
func (sb *SBMock) GetPath(string, string) (string, error) {
|
||||
@ -235,6 +249,44 @@ func AddOpFinish(ctx context.Context) (context.Context, func()) {
|
||||
}
|
||||
}
|
||||
|
||||
func (sb *SBMock) ComputeElectionPoSt(sectorInfo sectorbuilder.SortedPublicSectorInfo, challengeSeed []byte, winners []sectorbuilder.EPostCandidate) ([]byte, error) {
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (sb *SBMock) GenerateEPostCandidates(sectorInfo sectorbuilder.SortedPublicSectorInfo, challengeSeed [sectorbuilder.CommLen]byte, faults []uint64) ([]sectorbuilder.EPostCandidate, error) {
|
||||
if len(faults) > 0 {
|
||||
panic("todo")
|
||||
}
|
||||
|
||||
n := sectorbuilder.ElectionPostChallengeCount(uint64(len(sectorInfo.Values())), uint64(len(faults)))
|
||||
if n > uint64(len(sectorInfo.Values())) {
|
||||
n = uint64(len(sectorInfo.Values()))
|
||||
}
|
||||
|
||||
out := make([]sectorbuilder.EPostCandidate, n)
|
||||
|
||||
seed := big.NewInt(0).SetBytes(challengeSeed[:])
|
||||
start := seed.Mod(seed, big.NewInt(int64(len(sectorInfo.Values())))).Int64()
|
||||
|
||||
for i := range out {
|
||||
out[i] = sectorbuilder.EPostCandidate{
|
||||
SectorID: uint64((int(start) + i) % len(sectorInfo.Values())),
|
||||
PartialTicket: challengeSeed,
|
||||
Ticket: commDR(challengeSeed[:]),
|
||||
SectorChallengeIndex: 0,
|
||||
}
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (sb *SBMock) ReadPieceFromSealedSector(sectorID uint64, offset uint64, size uint64, ticket []byte, commD []byte) (io.ReadCloser, error) {
|
||||
if len(sb.sectors[sectorID].pieces) > 1 {
|
||||
panic("implme")
|
||||
}
|
||||
return ioutil.NopCloser(io.LimitReader(bytes.NewReader(sb.sectors[sectorID].pieces[0][offset:]), int64(size))), nil
|
||||
}
|
||||
|
||||
func (sb *SBMock) StageFakeData() (uint64, []sectorbuilder.PublicPieceInfo, error) {
|
||||
usize := sectorbuilder.UserBytesForSectorSize(sb.sectorSize)
|
||||
sid, err := sb.AcquireSectorId()
|
||||
@ -252,3 +304,40 @@ func (sb *SBMock) StageFakeData() (uint64, []sectorbuilder.PublicPieceInfo, erro
|
||||
|
||||
return sid, []sectorbuilder.PublicPieceInfo{pi}, nil
|
||||
}
|
||||
|
||||
func (m mockVerif) VerifyElectionPost(ctx context.Context, sectorSize uint64, sectorInfo sectorbuilder.SortedPublicSectorInfo, challengeSeed []byte, proof []byte, candidates []sectorbuilder.EPostCandidate, proverID address.Address) (bool, error) {
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (m mockVerif) VerifyFallbackPost(ctx context.Context, sectorSize uint64, sectorInfo sectorbuilder.SortedPublicSectorInfo, challengeSeed []byte, proof []byte, candidates []sectorbuilder.EPostCandidate, proverID address.Address, faults uint64) (bool, error) {
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (m mockVerif) VerifySeal(sectorSize uint64, commR, commD []byte, proverID address.Address, ticket []byte, seed []byte, sectorID uint64, proof []byte) (bool, error) {
|
||||
if len(proof) != 32 { // Real ones are longer, but this should be fine
|
||||
return false, nil
|
||||
}
|
||||
|
||||
for i, b := range proof {
|
||||
if b != commD[i]+commR[31-i]-ticket[i]*seed[i] {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (m mockVerif) GenerateDataCommitment(ssize uint64, pieces []ffi.PublicPieceInfo) ([sectorbuilder.CommLen]byte, error) {
|
||||
if len(pieces) != 1 {
|
||||
panic("todo")
|
||||
}
|
||||
if pieces[0].Size != sectorbuilder.UserBytesForSectorSize(ssize) {
|
||||
panic("todo")
|
||||
}
|
||||
return pieces[0].CommP, nil
|
||||
}
|
||||
|
||||
var MockVerifier = mockVerif{}
|
||||
|
||||
var _ sectorbuilder.Verifier = MockVerifier
|
||||
var _ sectorbuilder.Interface = &SBMock{}
|
||||
|
34
storage/sbmock/util.go
Normal file
34
storage/sbmock/util.go
Normal file
@ -0,0 +1,34 @@
|
||||
package sbmock
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
|
||||
"github.com/filecoin-project/go-sectorbuilder"
|
||||
)
|
||||
|
||||
func randB(n uint64) []byte {
|
||||
b, err := ioutil.ReadAll(io.LimitReader(rand.Reader, int64(n)))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func commDR(in []byte) (out [32]byte) {
|
||||
for i, b := range in {
|
||||
out[i] = ^b
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
func commD(b []byte) [32]byte {
|
||||
c, err := sectorbuilder.GeneratePieceCommitment(bytes.NewReader(b), uint64(len(b)))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return c
|
||||
}
|
@ -39,7 +39,7 @@ var ErrNotFound = errors.New("not found")
|
||||
|
||||
type SectorBlocks struct {
|
||||
*storage.Miner
|
||||
sb *sectorbuilder.SectorBuilder
|
||||
sb sectorbuilder.Interface
|
||||
|
||||
intermediate blockstore.Blockstore // holds intermediate nodes TODO: consider combining with the staging blockstore
|
||||
|
||||
@ -47,10 +47,10 @@ type SectorBlocks struct {
|
||||
keyLk sync.Mutex
|
||||
}
|
||||
|
||||
func NewSectorBlocks(miner *storage.Miner, ds dtypes.MetadataDS, sb storage.SectorBuilder) *SectorBlocks {
|
||||
func NewSectorBlocks(miner *storage.Miner, ds dtypes.MetadataDS, sb sectorbuilder.Interface) *SectorBlocks {
|
||||
sbc := &SectorBlocks{
|
||||
Miner: miner,
|
||||
sb: sb.(*sectorbuilder.SectorBuilder),
|
||||
sb: sb,
|
||||
|
||||
intermediate: blockstore.NewBlockstore(namespace.Wrap(ds, imBlocksPrefix)),
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user