From a989f60e27496377d107fa85ac304b39c41bf745 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 19 May 2021 13:05:07 +0200 Subject: [PATCH 01/90] add SectorAddPieceToAny and SectorUnsealPiece to StorageMiner iface; model moved to api package - PieceDealInfo, DealSchedule --- api/api_storage.go | 31 +- api/cbor_gen.go | 379 +++++++++++++++++++++++ api/proxy_gen.go | 20 ++ cmd/lotus-storage-miner/init.go | 6 +- cmd/lotus-storage-miner/init_restore.go | 2 +- extern/sector-storage/manager.go | 43 +++ extern/storage-sealing/cbor_gen.go | 386 +----------------------- extern/storage-sealing/input.go | 19 +- extern/storage-sealing/sealing.go | 2 +- extern/storage-sealing/types.go | 23 +- gen/main.go | 2 + markets/storageadapter/provider.go | 12 +- node/impl/storminer.go | 8 + storage/sealing.go | 76 ++++- storage/sectorblocks/blocks.go | 24 +- 15 files changed, 595 insertions(+), 438 deletions(-) diff --git a/api/api_storage.go b/api/api_storage.go index 1131f45a0..e1ef8ab6b 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -54,6 +54,13 @@ type StorageMiner interface { // Get the status of a given sector by ID SectorsStatus(ctx context.Context, sid abi.SectorNumber, showOnChainInfo bool) (SectorInfo, error) //perm:read + // Add piece to an open sector. If no sectors with enough space are open, + // either a new sector will be created, or this call will block until more + // sectors can be created. + SectorAddPieceToAny(ctx context.Context, size abi.UnpaddedPieceSize, r storage.Data, d PieceDealInfo) (SectorOffset, error) //perm:admin + + SectorsUnsealPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, commd *cid.Cid) error //perm:admin + // List all staged sectors SectorsList(context.Context) ([]abi.SectorNumber, error) //perm:read @@ -124,8 +131,8 @@ type StorageMiner interface { StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]stores.StorageInfo, error) //perm:admin StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error //perm:admin StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error) //perm:admin + StorageList(ctx context.Context) (map[stores.ID][]stores.Decl, error) //perm:admin - StorageList(ctx context.Context) (map[stores.ID][]stores.Decl, error) //perm:admin StorageLocal(ctx context.Context) (map[stores.ID]string, error) //perm:admin StorageStat(ctx context.Context, id stores.ID) (fsutil.FsStat, error) //perm:admin @@ -289,3 +296,25 @@ type PendingDealInfo struct { PublishPeriodStart time.Time PublishPeriod time.Duration } + +type SectorOffset struct { + Sector abi.SectorNumber + Offset abi.PaddedPieceSize +} + +// DealInfo is a tuple of deal identity and its schedule +type PieceDealInfo struct { + PublishCid *cid.Cid + DealID abi.DealID + DealProposal *market.DealProposal + DealSchedule DealSchedule + KeepUnsealed bool +} + +// DealSchedule communicates the time interval of a storage deal. The deal must +// appear in a sealed (proven) sector no later than StartEpoch, otherwise it +// is invalid. +type DealSchedule struct { + StartEpoch abi.ChainEpoch + EndEpoch abi.ChainEpoch +} diff --git a/api/cbor_gen.go b/api/cbor_gen.go index 808e516ad..4434b45ed 100644 --- a/api/cbor_gen.go +++ b/api/cbor_gen.go @@ -8,6 +8,7 @@ import ( "sort" abi "github.com/filecoin-project/go-state-types/abi" + market "github.com/filecoin-project/specs-actors/actors/builtin/market" paych "github.com/filecoin-project/specs-actors/actors/builtin/paych" cid "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" @@ -738,3 +739,381 @@ func (t *SealSeed) UnmarshalCBOR(r io.Reader) error { return nil } +func (t *PieceDealInfo) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{165}); err != nil { + return err + } + + scratch := make([]byte, 9) + + // t.PublishCid (cid.Cid) (struct) + if len("PublishCid") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"PublishCid\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("PublishCid"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("PublishCid")); err != nil { + return err + } + + if t.PublishCid == nil { + if _, err := w.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCidBuf(scratch, w, *t.PublishCid); err != nil { + return xerrors.Errorf("failed to write cid field t.PublishCid: %w", err) + } + } + + // t.DealID (abi.DealID) (uint64) + if len("DealID") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"DealID\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("DealID"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("DealID")); err != nil { + return err + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.DealID)); err != nil { + return err + } + + // t.DealProposal (market.DealProposal) (struct) + if len("DealProposal") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"DealProposal\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("DealProposal"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("DealProposal")); err != nil { + return err + } + + if err := t.DealProposal.MarshalCBOR(w); err != nil { + return err + } + + // t.DealSchedule (api.DealSchedule) (struct) + if len("DealSchedule") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"DealSchedule\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("DealSchedule"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("DealSchedule")); err != nil { + return err + } + + if err := t.DealSchedule.MarshalCBOR(w); err != nil { + return err + } + + // t.KeepUnsealed (bool) (bool) + if len("KeepUnsealed") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"KeepUnsealed\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("KeepUnsealed"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("KeepUnsealed")); err != nil { + return err + } + + if err := cbg.WriteBool(w, t.KeepUnsealed); err != nil { + return err + } + return nil +} + +func (t *PieceDealInfo) UnmarshalCBOR(r io.Reader) error { + *t = PieceDealInfo{} + + br := cbg.GetPeeker(r) + scratch := make([]byte, 8) + + maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("PieceDealInfo: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadStringBuf(br, scratch) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.PublishCid (cid.Cid) (struct) + case "PublishCid": + + { + + b, err := br.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := br.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.PublishCid: %w", err) + } + + t.PublishCid = &c + } + + } + // t.DealID (abi.DealID) (uint64) + case "DealID": + + { + + maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.DealID = abi.DealID(extra) + + } + // t.DealProposal (market.DealProposal) (struct) + case "DealProposal": + + { + + b, err := br.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := br.UnreadByte(); err != nil { + return err + } + t.DealProposal = new(market.DealProposal) + if err := t.DealProposal.UnmarshalCBOR(br); err != nil { + return xerrors.Errorf("unmarshaling t.DealProposal pointer: %w", err) + } + } + + } + // t.DealSchedule (api.DealSchedule) (struct) + case "DealSchedule": + + { + + if err := t.DealSchedule.UnmarshalCBOR(br); err != nil { + return xerrors.Errorf("unmarshaling t.DealSchedule: %w", err) + } + + } + // t.KeepUnsealed (bool) (bool) + case "KeepUnsealed": + + maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajOther { + return fmt.Errorf("booleans must be major type 7") + } + switch extra { + case 20: + t.KeepUnsealed = false + case 21: + t.KeepUnsealed = true + default: + return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra) + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *DealSchedule) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{162}); err != nil { + return err + } + + scratch := make([]byte, 9) + + // t.StartEpoch (abi.ChainEpoch) (int64) + if len("StartEpoch") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"StartEpoch\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("StartEpoch"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("StartEpoch")); err != nil { + return err + } + + if t.StartEpoch >= 0 { + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.StartEpoch)); err != nil { + return err + } + } else { + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.StartEpoch-1)); err != nil { + return err + } + } + + // t.EndEpoch (abi.ChainEpoch) (int64) + if len("EndEpoch") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"EndEpoch\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("EndEpoch"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("EndEpoch")); err != nil { + return err + } + + if t.EndEpoch >= 0 { + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.EndEpoch)); err != nil { + return err + } + } else { + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.EndEpoch-1)); err != nil { + return err + } + } + return nil +} + +func (t *DealSchedule) UnmarshalCBOR(r io.Reader) error { + *t = DealSchedule{} + + br := cbg.GetPeeker(r) + scratch := make([]byte, 8) + + maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("DealSchedule: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadStringBuf(br, scratch) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.StartEpoch (abi.ChainEpoch) (int64) + case "StartEpoch": + { + maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.StartEpoch = abi.ChainEpoch(extraI) + } + // t.EndEpoch (abi.ChainEpoch) (int64) + case "EndEpoch": + { + maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.EndEpoch = abi.ChainEpoch(extraI) + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} diff --git a/api/proxy_gen.go b/api/proxy_gen.go index 2d4d41503..eb96851a9 100644 --- a/api/proxy_gen.go +++ b/api/proxy_gen.go @@ -657,6 +657,8 @@ type StorageMinerStruct struct { SealingSchedDiag func(p0 context.Context, p1 bool) (interface{}, error) `perm:"admin"` + SectorAddPieceToAny func(p0 context.Context, p1 abi.UnpaddedPieceSize, p2 storage.Data, p3 PieceDealInfo) (SectorOffset, error) `perm:"admin"` + SectorGetExpectedSealDuration func(p0 context.Context) (time.Duration, error) `perm:"read"` SectorGetSealDelay func(p0 context.Context) (time.Duration, error) `perm:"read"` @@ -687,6 +689,8 @@ type StorageMinerStruct struct { SectorsSummary func(p0 context.Context) (map[SectorState]int, error) `perm:"read"` + SectorsUnsealPiece func(p0 context.Context, p1 storage.SectorRef, p2 storiface.UnpaddedByteIndex, p3 abi.UnpaddedPieceSize, p4 abi.SealRandomness, p5 *cid.Cid) error `perm:"admin"` + SectorsUpdate func(p0 context.Context, p1 abi.SectorNumber, p2 SectorState) error `perm:"admin"` StorageAddLocal func(p0 context.Context, p1 string) error `perm:"admin"` @@ -3107,6 +3111,14 @@ func (s *StorageMinerStub) SealingSchedDiag(p0 context.Context, p1 bool) (interf return nil, xerrors.New("method not supported") } +func (s *StorageMinerStruct) SectorAddPieceToAny(p0 context.Context, p1 abi.UnpaddedPieceSize, p2 storage.Data, p3 PieceDealInfo) (SectorOffset, error) { + return s.Internal.SectorAddPieceToAny(p0, p1, p2, p3) +} + +func (s *StorageMinerStub) SectorAddPieceToAny(p0 context.Context, p1 abi.UnpaddedPieceSize, p2 storage.Data, p3 PieceDealInfo) (SectorOffset, error) { + return *new(SectorOffset), xerrors.New("method not supported") +} + func (s *StorageMinerStruct) SectorGetExpectedSealDuration(p0 context.Context) (time.Duration, error) { return s.Internal.SectorGetExpectedSealDuration(p0) } @@ -3227,6 +3239,14 @@ func (s *StorageMinerStub) SectorsSummary(p0 context.Context) (map[SectorState]i return *new(map[SectorState]int), xerrors.New("method not supported") } +func (s *StorageMinerStruct) SectorsUnsealPiece(p0 context.Context, p1 storage.SectorRef, p2 storiface.UnpaddedByteIndex, p3 abi.UnpaddedPieceSize, p4 abi.SealRandomness, p5 *cid.Cid) error { + return s.Internal.SectorsUnsealPiece(p0, p1, p2, p3, p4, p5) +} + +func (s *StorageMinerStub) SectorsUnsealPiece(p0 context.Context, p1 storage.SectorRef, p2 storiface.UnpaddedByteIndex, p3 abi.UnpaddedPieceSize, p4 abi.SealRandomness, p5 *cid.Cid) error { + return xerrors.New("method not supported") +} + func (s *StorageMinerStruct) SectorsUpdate(p0 context.Context, p1 abi.SectorNumber, p2 SectorState) error { return s.Internal.SectorsUpdate(p0, p1, p2) } diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index a02520116..76132a7a2 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -120,7 +120,7 @@ var initCmd = &cli.Command{ }, }, Subcommands: []*cli.Command{ - initRestoreCmd, + restoreCmd, }, Action: func(cctx *cli.Context) error { log.Info("Initializing lotus miner") @@ -316,10 +316,10 @@ func migratePreSealMeta(ctx context.Context, api v1api.FullNode, metadata string Size: abi.PaddedPieceSize(meta.SectorSize), PieceCID: commD, }, - DealInfo: &sealing.DealInfo{ + DealInfo: &lapi.PieceDealInfo{ DealID: dealID, DealProposal: §or.Deal, - DealSchedule: sealing.DealSchedule{ + DealSchedule: lapi.DealSchedule{ StartEpoch: sector.Deal.StartEpoch, EndEpoch: sector.Deal.EndEpoch, }, diff --git a/cmd/lotus-storage-miner/init_restore.go b/cmd/lotus-storage-miner/init_restore.go index eec7b8413..c609fd2a4 100644 --- a/cmd/lotus-storage-miner/init_restore.go +++ b/cmd/lotus-storage-miner/init_restore.go @@ -30,7 +30,7 @@ import ( "github.com/filecoin-project/lotus/node/repo" ) -var initRestoreCmd = &cli.Command{ +var restoreCmd = &cli.Command{ Name: "restore", Usage: "Initialize a lotus miner repo from a backup", Flags: []cli.Flag{ diff --git a/extern/sector-storage/manager.go b/extern/sector-storage/manager.go index d3fef8533..70d55ee2e 100644 --- a/extern/sector-storage/manager.go +++ b/extern/sector-storage/manager.go @@ -336,6 +336,49 @@ func (m *Manager) ReadPiece(ctx context.Context, sink io.Writer, sector storage. return nil } +func (m *Manager) SectorsUnsealPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, ticket abi.SealRandomness, unsealed *cid.Cid) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + if err := m.index.StorageLock(ctx, sector.ID, storiface.FTSealed|storiface.FTCache, storiface.FTUnsealed); err != nil { + return xerrors.Errorf("acquiring unseal sector lock: %w", err) + } + + unsealFetch := func(ctx context.Context, worker Worker) error { + if _, err := m.waitSimpleCall(ctx)(worker.Fetch(ctx, sector, storiface.FTSealed|storiface.FTCache, storiface.PathSealing, storiface.AcquireCopy)); err != nil { + return xerrors.Errorf("copy sealed/cache sector data: %w", err) + } + + return nil + } + + if unsealed == nil { + return xerrors.Errorf("cannot unseal piece (sector: %d, offset: %d size: %d) - unsealed cid is undefined", sector, offset, size) + } + + ssize, err := sector.ProofType.SectorSize() + if err != nil { + return xerrors.Errorf("getting sector size: %w", err) + } + + selector := newExistingSelector(m.index, sector.ID, storiface.FTSealed|storiface.FTCache, false) + + err = m.sched.Schedule(ctx, sector, sealtasks.TTUnseal, selector, unsealFetch, func(ctx context.Context, w Worker) error { + // TODO: make restartable + + // NOTE: we're unsealing the whole sector here as with SDR we can't really + // unseal the sector partially. Requesting the whole sector here can + // save us some work in case another piece is requested from here + _, err := m.waitSimpleCall(ctx)(w.UnsealPiece(ctx, sector, 0, abi.PaddedPieceSize(ssize).Unpadded(), ticket, *unsealed)) + return err + }) + if err != nil { + return err + } + + return nil +} + func (m *Manager) NewSector(ctx context.Context, sector storage.SectorRef) error { log.Warnf("stub NewSector") return nil diff --git a/extern/storage-sealing/cbor_gen.go b/extern/storage-sealing/cbor_gen.go index 9e12b8649..b71c2863c 100644 --- a/extern/storage-sealing/cbor_gen.go +++ b/extern/storage-sealing/cbor_gen.go @@ -8,7 +8,7 @@ import ( "sort" abi "github.com/filecoin-project/go-state-types/abi" - market "github.com/filecoin-project/specs-actors/actors/builtin/market" + api "github.com/filecoin-project/lotus/api" miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" cid "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" @@ -46,7 +46,7 @@ func (t *Piece) MarshalCBOR(w io.Writer) error { return err } - // t.DealInfo (sealing.DealInfo) (struct) + // t.DealInfo (api.PieceDealInfo) (struct) if len("DealInfo") > cbg.MaxLength { return xerrors.Errorf("Value in field \"DealInfo\" was too long") } @@ -107,7 +107,7 @@ func (t *Piece) UnmarshalCBOR(r io.Reader) error { } } - // t.DealInfo (sealing.DealInfo) (struct) + // t.DealInfo (api.PieceDealInfo) (struct) case "DealInfo": { @@ -120,7 +120,7 @@ func (t *Piece) UnmarshalCBOR(r io.Reader) error { if err := br.UnreadByte(); err != nil { return err } - t.DealInfo = new(DealInfo) + t.DealInfo = new(api.PieceDealInfo) if err := t.DealInfo.UnmarshalCBOR(br); err != nil { return xerrors.Errorf("unmarshaling t.DealInfo pointer: %w", err) } @@ -136,384 +136,6 @@ func (t *Piece) UnmarshalCBOR(r io.Reader) error { return nil } -func (t *DealInfo) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{165}); err != nil { - return err - } - - scratch := make([]byte, 9) - - // t.PublishCid (cid.Cid) (struct) - if len("PublishCid") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"PublishCid\" was too long") - } - - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("PublishCid"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("PublishCid")); err != nil { - return err - } - - if t.PublishCid == nil { - if _, err := w.Write(cbg.CborNull); err != nil { - return err - } - } else { - if err := cbg.WriteCidBuf(scratch, w, *t.PublishCid); err != nil { - return xerrors.Errorf("failed to write cid field t.PublishCid: %w", err) - } - } - - // t.DealID (abi.DealID) (uint64) - if len("DealID") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"DealID\" was too long") - } - - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("DealID"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("DealID")); err != nil { - return err - } - - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.DealID)); err != nil { - return err - } - - // t.DealProposal (market.DealProposal) (struct) - if len("DealProposal") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"DealProposal\" was too long") - } - - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("DealProposal"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("DealProposal")); err != nil { - return err - } - - if err := t.DealProposal.MarshalCBOR(w); err != nil { - return err - } - - // t.DealSchedule (sealing.DealSchedule) (struct) - if len("DealSchedule") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"DealSchedule\" was too long") - } - - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("DealSchedule"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("DealSchedule")); err != nil { - return err - } - - if err := t.DealSchedule.MarshalCBOR(w); err != nil { - return err - } - - // t.KeepUnsealed (bool) (bool) - if len("KeepUnsealed") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"KeepUnsealed\" was too long") - } - - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("KeepUnsealed"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("KeepUnsealed")); err != nil { - return err - } - - if err := cbg.WriteBool(w, t.KeepUnsealed); err != nil { - return err - } - return nil -} - -func (t *DealInfo) UnmarshalCBOR(r io.Reader) error { - *t = DealInfo{} - - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) - - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) - if err != nil { - return err - } - if maj != cbg.MajMap { - return fmt.Errorf("cbor input should be of type map") - } - - if extra > cbg.MaxLength { - return fmt.Errorf("DealInfo: map struct too large (%d)", extra) - } - - var name string - n := extra - - for i := uint64(0); i < n; i++ { - - { - sval, err := cbg.ReadStringBuf(br, scratch) - if err != nil { - return err - } - - name = string(sval) - } - - switch name { - // t.PublishCid (cid.Cid) (struct) - case "PublishCid": - - { - - b, err := br.ReadByte() - if err != nil { - return err - } - if b != cbg.CborNull[0] { - if err := br.UnreadByte(); err != nil { - return err - } - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.PublishCid: %w", err) - } - - t.PublishCid = &c - } - - } - // t.DealID (abi.DealID) (uint64) - case "DealID": - - { - - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.DealID = abi.DealID(extra) - - } - // t.DealProposal (market.DealProposal) (struct) - case "DealProposal": - - { - - b, err := br.ReadByte() - if err != nil { - return err - } - if b != cbg.CborNull[0] { - if err := br.UnreadByte(); err != nil { - return err - } - t.DealProposal = new(market.DealProposal) - if err := t.DealProposal.UnmarshalCBOR(br); err != nil { - return xerrors.Errorf("unmarshaling t.DealProposal pointer: %w", err) - } - } - - } - // t.DealSchedule (sealing.DealSchedule) (struct) - case "DealSchedule": - - { - - if err := t.DealSchedule.UnmarshalCBOR(br); err != nil { - return xerrors.Errorf("unmarshaling t.DealSchedule: %w", err) - } - - } - // t.KeepUnsealed (bool) (bool) - case "KeepUnsealed": - - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) - if err != nil { - return err - } - if maj != cbg.MajOther { - return fmt.Errorf("booleans must be major type 7") - } - switch extra { - case 20: - t.KeepUnsealed = false - case 21: - t.KeepUnsealed = true - default: - return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra) - } - - default: - // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) - } - } - - return nil -} -func (t *DealSchedule) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{162}); err != nil { - return err - } - - scratch := make([]byte, 9) - - // t.StartEpoch (abi.ChainEpoch) (int64) - if len("StartEpoch") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"StartEpoch\" was too long") - } - - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("StartEpoch"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("StartEpoch")); err != nil { - return err - } - - if t.StartEpoch >= 0 { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.StartEpoch)); err != nil { - return err - } - } else { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.StartEpoch-1)); err != nil { - return err - } - } - - // t.EndEpoch (abi.ChainEpoch) (int64) - if len("EndEpoch") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"EndEpoch\" was too long") - } - - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("EndEpoch"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("EndEpoch")); err != nil { - return err - } - - if t.EndEpoch >= 0 { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.EndEpoch)); err != nil { - return err - } - } else { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.EndEpoch-1)); err != nil { - return err - } - } - return nil -} - -func (t *DealSchedule) UnmarshalCBOR(r io.Reader) error { - *t = DealSchedule{} - - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) - - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) - if err != nil { - return err - } - if maj != cbg.MajMap { - return fmt.Errorf("cbor input should be of type map") - } - - if extra > cbg.MaxLength { - return fmt.Errorf("DealSchedule: map struct too large (%d)", extra) - } - - var name string - n := extra - - for i := uint64(0); i < n; i++ { - - { - sval, err := cbg.ReadStringBuf(br, scratch) - if err != nil { - return err - } - - name = string(sval) - } - - switch name { - // t.StartEpoch (abi.ChainEpoch) (int64) - case "StartEpoch": - { - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) - var extraI int64 - if err != nil { - return err - } - switch maj { - case cbg.MajUnsignedInt: - extraI = int64(extra) - if extraI < 0 { - return fmt.Errorf("int64 positive overflow") - } - case cbg.MajNegativeInt: - extraI = int64(extra) - if extraI < 0 { - return fmt.Errorf("int64 negative oveflow") - } - extraI = -1 - extraI - default: - return fmt.Errorf("wrong type for int64 field: %d", maj) - } - - t.StartEpoch = abi.ChainEpoch(extraI) - } - // t.EndEpoch (abi.ChainEpoch) (int64) - case "EndEpoch": - { - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) - var extraI int64 - if err != nil { - return err - } - switch maj { - case cbg.MajUnsignedInt: - extraI = int64(extra) - if extraI < 0 { - return fmt.Errorf("int64 positive overflow") - } - case cbg.MajNegativeInt: - extraI = int64(extra) - if extraI < 0 { - return fmt.Errorf("int64 negative oveflow") - } - extraI = -1 - extraI - default: - return fmt.Errorf("wrong type for int64 field: %d", maj) - } - - t.EndEpoch = abi.ChainEpoch(extraI) - } - - default: - // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) - } - } - - return nil -} func (t *SectorInfo) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) diff --git a/extern/storage-sealing/input.go b/extern/storage-sealing/input.go index 44d2e8275..8f80b257f 100644 --- a/extern/storage-sealing/input.go +++ b/extern/storage-sealing/input.go @@ -14,6 +14,7 @@ import ( "github.com/filecoin-project/go-statemachine" "github.com/filecoin-project/specs-storage/storage" + "github.com/filecoin-project/lotus/api" sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/extern/storage-sealing/sealiface" @@ -224,34 +225,34 @@ func (m *Sealing) handleAddPieceFailed(ctx statemachine.Context, sector SectorIn return nil } -func (m *Sealing) AddPieceToAnySector(ctx context.Context, size abi.UnpaddedPieceSize, data storage.Data, deal DealInfo) (abi.SectorNumber, abi.PaddedPieceSize, error) { +func (m *Sealing) SectorAddPieceToAny(ctx context.Context, size abi.UnpaddedPieceSize, data storage.Data, deal api.PieceDealInfo) (api.SectorOffset, error) { log.Infof("Adding piece for deal %d (publish msg: %s)", deal.DealID, deal.PublishCid) if (padreader.PaddedSize(uint64(size))) != size { - return 0, 0, xerrors.Errorf("cannot allocate unpadded piece") + return api.SectorOffset{}, xerrors.Errorf("cannot allocate unpadded piece") } sp, err := m.currentSealProof(ctx) if err != nil { - return 0, 0, xerrors.Errorf("getting current seal proof type: %w", err) + return api.SectorOffset{}, xerrors.Errorf("getting current seal proof type: %w", err) } ssize, err := sp.SectorSize() if err != nil { - return 0, 0, err + return api.SectorOffset{}, err } if size > abi.PaddedPieceSize(ssize).Unpadded() { - return 0, 0, xerrors.Errorf("piece cannot fit into a sector") + return api.SectorOffset{}, xerrors.Errorf("piece cannot fit into a sector") } if _, err := deal.DealProposal.Cid(); err != nil { - return 0, 0, xerrors.Errorf("getting proposal CID: %w", err) + return api.SectorOffset{}, xerrors.Errorf("getting proposal CID: %w", err) } m.inputLk.Lock() if _, exist := m.pendingPieces[proposalCID(deal)]; exist { m.inputLk.Unlock() - return 0, 0, xerrors.Errorf("piece for deal %s already pending", proposalCID(deal)) + return api.SectorOffset{}, xerrors.Errorf("piece for deal %s already pending", proposalCID(deal)) } resCh := make(chan struct { @@ -283,7 +284,7 @@ func (m *Sealing) AddPieceToAnySector(ctx context.Context, size abi.UnpaddedPiec res := <-resCh - return res.sn, res.offset.Padded(), res.err + return api.SectorOffset{Sector: res.sn, Offset: res.offset.Padded()}, res.err } // called with m.inputLk @@ -425,7 +426,7 @@ func (m *Sealing) StartPacking(sid abi.SectorNumber) error { return m.sectors.Send(uint64(sid), SectorStartPacking{}) } -func proposalCID(deal DealInfo) cid.Cid { +func proposalCID(deal api.PieceDealInfo) cid.Cid { pc, err := deal.DealProposal.Cid() if err != nil { log.Errorf("DealProposal.Cid error: %+v", err) diff --git a/extern/storage-sealing/sealing.go b/extern/storage-sealing/sealing.go index 8feca3b7b..8556a4902 100644 --- a/extern/storage-sealing/sealing.go +++ b/extern/storage-sealing/sealing.go @@ -122,7 +122,7 @@ type openSector struct { type pendingPiece struct { size abi.UnpaddedPieceSize - deal DealInfo + deal api.PieceDealInfo data storage.Data diff --git a/extern/storage-sealing/types.go b/extern/storage-sealing/types.go index 58c35cf36..c5aed505a 100644 --- a/extern/storage-sealing/types.go +++ b/extern/storage-sealing/types.go @@ -11,39 +11,22 @@ import ( "github.com/filecoin-project/go-state-types/exitcode" "github.com/filecoin-project/specs-storage/storage" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" "github.com/filecoin-project/lotus/extern/storage-sealing/sealiface" - "github.com/filecoin-project/specs-actors/v2/actors/builtin/market" ) // Piece is a tuple of piece and deal info type PieceWithDealInfo struct { Piece abi.PieceInfo - DealInfo DealInfo + DealInfo api.PieceDealInfo } // Piece is a tuple of piece info and optional deal type Piece struct { Piece abi.PieceInfo - DealInfo *DealInfo // nil for pieces which do not appear in deals (e.g. filler pieces) -} - -// DealInfo is a tuple of deal identity and its schedule -type DealInfo struct { - PublishCid *cid.Cid - DealID abi.DealID - DealProposal *market.DealProposal - DealSchedule DealSchedule - KeepUnsealed bool -} - -// DealSchedule communicates the time interval of a storage deal. The deal must -// appear in a sealed (proven) sector no later than StartEpoch, otherwise it -// is invalid. -type DealSchedule struct { - StartEpoch abi.ChainEpoch - EndEpoch abi.ChainEpoch + DealInfo *api.PieceDealInfo // nil for pieces which do not appear in deals (e.g. filler pieces) } type Log struct { diff --git a/gen/main.go b/gen/main.go index 9548344fd..0018b241d 100644 --- a/gen/main.go +++ b/gen/main.go @@ -53,6 +53,8 @@ func main() { api.SealedRefs{}, api.SealTicket{}, api.SealSeed{}, + api.PieceDealInfo{}, + api.DealSchedule{}, ) if err != nil { fmt.Println(err) diff --git a/markets/storageadapter/provider.go b/markets/storageadapter/provider.go index fbeaf3b3d..b899c0810 100644 --- a/markets/storageadapter/provider.go +++ b/markets/storageadapter/provider.go @@ -95,11 +95,11 @@ func (n *ProviderNodeAdapter) OnDealComplete(ctx context.Context, deal storagema return nil, xerrors.Errorf("deal.PublishCid can't be nil") } - sdInfo := sealing.DealInfo{ + sdInfo := api.PieceDealInfo{ DealID: deal.DealID, DealProposal: &deal.Proposal, PublishCid: deal.PublishCid, - DealSchedule: sealing.DealSchedule{ + DealSchedule: api.DealSchedule{ StartEpoch: deal.ClientDealProposal.Proposal.StartEpoch, EndEpoch: deal.ClientDealProposal.Proposal.EndEpoch, }, @@ -240,19 +240,19 @@ func (n *ProviderNodeAdapter) LocatePieceForDealWithinSector(ctx context.Context // TODO: better strategy (e.g. look for already unsealed) var best api.SealedRef - var bestSi sealing.SectorInfo + var bestSi api.SectorInfo for _, r := range refs { - si, err := n.secb.Miner.GetSectorInfo(r.SectorID) + si, err := n.secb.SectorBuilder.SectorsStatus(ctx, r.SectorID, false) if err != nil { return 0, 0, 0, xerrors.Errorf("getting sector info: %w", err) } - if si.State == sealing.Proving { + if si.State == api.SectorState(sealing.Proving) { best = r bestSi = si break } } - if bestSi.State == sealing.UndefinedSectorState { + if bestSi.State == api.SectorState(sealing.UndefinedSectorState) { return 0, 0, 0, xerrors.New("no sealed sector found") } return best.SectorID, best.Offset, best.Size.Padded(), nil diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 27ab1af5f..bf1c44fc2 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -237,6 +237,14 @@ func (sm *StorageMinerAPI) SectorsStatus(ctx context.Context, sid abi.SectorNumb return sInfo, nil } +func (sm *StorageMinerAPI) SectorAddPieceToAny(ctx context.Context, size abi.UnpaddedPieceSize, r sto.Data, d api.PieceDealInfo) (api.SectorOffset, error) { + return sm.Miner.SectorAddPieceToAny(ctx, size, r, d) +} + +func (sm *StorageMinerAPI) SectorsUnsealPiece(ctx context.Context, sector sto.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, commd *cid.Cid) error { + return sm.StorageMgr.SectorsUnsealPiece(ctx, sector, offset, size, randomness, commd) +} + // List all staged sectors func (sm *StorageMinerAPI) SectorsList(context.Context) ([]abi.SectorNumber, error) { sectors, err := sm.Miner.ListSectors() diff --git a/storage/sealing.go b/storage/sealing.go index 8981c3738..fdf8f25fa 100644 --- a/storage/sealing.go +++ b/storage/sealing.go @@ -2,14 +2,16 @@ package storage import ( "context" - "io" "github.com/ipfs/go-cid" + "golang.org/x/xerrors" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/specs-storage/storage" + "github.com/filecoin-project/lotus/api" sealing "github.com/filecoin-project/lotus/extern/storage-sealing" ) @@ -19,10 +21,6 @@ func (m *Miner) Address() address.Address { return m.sealing.Address() } -func (m *Miner) AddPieceToAnySector(ctx context.Context, size abi.UnpaddedPieceSize, r io.Reader, d sealing.DealInfo) (abi.SectorNumber, abi.PaddedPieceSize, error) { - return m.sealing.AddPieceToAnySector(ctx, size, r, d) -} - func (m *Miner) StartPackingSector(sectorNum abi.SectorNumber) error { return m.sealing.StartPacking(sectorNum) } @@ -66,3 +64,71 @@ func (m *Miner) MarkForUpgrade(id abi.SectorNumber) error { func (m *Miner) IsMarkedForUpgrade(id abi.SectorNumber) bool { return m.sealing.IsMarkedForUpgrade(id) } + +func (m *Miner) SectorAddPieceToAny(ctx context.Context, size abi.UnpaddedPieceSize, r storage.Data, d api.PieceDealInfo) (api.SectorOffset, error) { + return m.sealing.SectorAddPieceToAny(ctx, size, r, d) +} + +func (m *Miner) SectorsStatus(ctx context.Context, sid abi.SectorNumber, showOnChainInfo bool) (api.SectorInfo, error) { + if showOnChainInfo { + return api.SectorInfo{}, xerrors.Errorf("on-chain info not supported") + } + + info, err := m.sealing.GetSectorInfo(sid) + if err != nil { + return api.SectorInfo{}, err + } + + deals := make([]abi.DealID, len(info.Pieces)) + for i, piece := range info.Pieces { + if piece.DealInfo == nil { + continue + } + deals[i] = piece.DealInfo.DealID + } + + log := make([]api.SectorLog, len(info.Log)) + for i, l := range info.Log { + log[i] = api.SectorLog{ + Kind: l.Kind, + Timestamp: l.Timestamp, + Trace: l.Trace, + Message: l.Message, + } + } + + sInfo := api.SectorInfo{ + SectorID: sid, + State: api.SectorState(info.State), + CommD: info.CommD, + CommR: info.CommR, + Proof: info.Proof, + Deals: deals, + Ticket: api.SealTicket{ + Value: info.TicketValue, + Epoch: info.TicketEpoch, + }, + Seed: api.SealSeed{ + Value: info.SeedValue, + Epoch: info.SeedEpoch, + }, + PreCommitMsg: info.PreCommitMessage, + CommitMsg: info.CommitMessage, + Retries: info.InvalidProofs, + ToUpgrade: m.IsMarkedForUpgrade(sid), + + LastErr: info.LastErr, + Log: log, + // on chain info + SealProof: 0, + Activation: 0, + Expiration: 0, + DealWeight: big.Zero(), + VerifiedDealWeight: big.Zero(), + InitialPledge: big.Zero(), + OnTime: 0, + Early: 0, + } + + return sInfo, nil +} diff --git a/storage/sectorblocks/blocks.go b/storage/sectorblocks/blocks.go index bc8456a1f..ccf2c67d2 100644 --- a/storage/sectorblocks/blocks.go +++ b/storage/sectorblocks/blocks.go @@ -16,11 +16,10 @@ import ( cborutil "github.com/filecoin-project/go-cbor-util" "github.com/filecoin-project/go-state-types/abi" - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" + "github.com/filecoin-project/specs-storage/storage" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/node/modules/dtypes" - "github.com/filecoin-project/lotus/storage" ) type SealSerialization uint8 @@ -48,17 +47,22 @@ func DsKeyToDealID(key datastore.Key) (uint64, error) { return dealID, nil } +type SectorBuilder interface { // todo: apify, make work remote + SectorAddPieceToAny(ctx context.Context, size abi.UnpaddedPieceSize, r storage.Data, d api.PieceDealInfo) (api.SectorOffset, error) + SectorsStatus(ctx context.Context, sid abi.SectorNumber, showOnChainInfo bool) (api.SectorInfo, error) +} + type SectorBlocks struct { - *storage.Miner + SectorBuilder keys datastore.Batching keyLk sync.Mutex } -func NewSectorBlocks(miner *storage.Miner, ds dtypes.MetadataDS) *SectorBlocks { +func NewSectorBlocks(sb SectorBuilder, ds dtypes.MetadataDS) *SectorBlocks { sbc := &SectorBlocks{ - Miner: miner, - keys: namespace.Wrap(ds, dsPrefix), + SectorBuilder: sb, + keys: namespace.Wrap(ds, dsPrefix), } return sbc @@ -96,19 +100,19 @@ func (st *SectorBlocks) writeRef(dealID abi.DealID, sectorID abi.SectorNumber, o return st.keys.Put(DealIDToDsKey(dealID), newRef) // TODO: batch somehow } -func (st *SectorBlocks) AddPiece(ctx context.Context, size abi.UnpaddedPieceSize, r io.Reader, d sealing.DealInfo) (abi.SectorNumber, abi.PaddedPieceSize, error) { - sn, offset, err := st.Miner.AddPieceToAnySector(ctx, size, r, d) +func (st *SectorBlocks) AddPiece(ctx context.Context, size abi.UnpaddedPieceSize, r io.Reader, d api.PieceDealInfo) (abi.SectorNumber, abi.PaddedPieceSize, error) { + so, err := st.SectorBuilder.SectorAddPieceToAny(ctx, size, r, d) if err != nil { return 0, 0, err } // TODO: DealID has very low finality here - err = st.writeRef(d.DealID, sn, offset, size) + err = st.writeRef(d.DealID, so.Sector, so.Offset, size) if err != nil { return 0, 0, xerrors.Errorf("writeRef: %w", err) } - return sn, offset, nil + return so.Sector, so.Offset, nil } func (st *SectorBlocks) List() (map[uint64][]api.SealedRef, error) { From 87ed228fd3e920294287422db8ef1b37faf37603 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 19 May 2021 13:10:56 +0200 Subject: [PATCH 02/90] extract pushUrl and fix return type for worker RPC v0 --- api/client/client.go | 41 ++++++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/api/client/client.go b/api/client/client.go index 90fe714bf..2c18a289b 100644 --- a/api/client/client.go +++ b/api/client/client.go @@ -52,8 +52,30 @@ func NewFullNodeRPCV1(ctx context.Context, addr string, requestHeader http.Heade return &res, closer, err } +func getPushUrl(addr string) (string, error) { + pushUrl, err := url.Parse(addr) + if err != nil { + return "", err + } + switch pushUrl.Scheme { + case "ws": + pushUrl.Scheme = "http" + case "wss": + pushUrl.Scheme = "https" + } + ///rpc/v0 -> /rpc/streams/v0/push + + pushUrl.Path = path.Join(pushUrl.Path, "../streams/v0/push") + return pushUrl.String(), nil +} + // NewStorageMinerRPCV0 creates a new http jsonrpc client for miner func NewStorageMinerRPCV0(ctx context.Context, addr string, requestHeader http.Header, opts ...jsonrpc.Option) (v0api.StorageMiner, jsonrpc.ClientCloser, error) { + pushUrl, err := getPushUrl(addr) + if err != nil { + return nil, nil, err + } + var res v0api.StorageMinerStruct closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin", []interface{}{ @@ -61,26 +83,19 @@ func NewStorageMinerRPCV0(ctx context.Context, addr string, requestHeader http.H &res.Internal, }, requestHeader, - opts..., + append([]jsonrpc.Option{ + rpcenc.ReaderParamEncoder(pushUrl), + }, opts...)..., ) return &res, closer, err } -func NewWorkerRPCV0(ctx context.Context, addr string, requestHeader http.Header) (api.Worker, jsonrpc.ClientCloser, error) { - u, err := url.Parse(addr) +func NewWorkerRPCV0(ctx context.Context, addr string, requestHeader http.Header) (v0api.Worker, jsonrpc.ClientCloser, error) { + pushUrl, err := getPushUrl(addr) if err != nil { return nil, nil, err } - switch u.Scheme { - case "ws": - u.Scheme = "http" - case "wss": - u.Scheme = "https" - } - ///rpc/v0 -> /rpc/streams/v0/push - - u.Path = path.Join(u.Path, "../streams/v0/push") var res api.WorkerStruct closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin", @@ -88,7 +103,7 @@ func NewWorkerRPCV0(ctx context.Context, addr string, requestHeader http.Header) &res.Internal, }, requestHeader, - rpcenc.ReaderParamEncoder(u.String()), + rpcenc.ReaderParamEncoder(pushUrl), jsonrpc.WithNoReconnect(), jsonrpc.WithTimeout(30*time.Second), ) From e275b54f55739bf75ce1245ff3335ea3d6289b30 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 19 May 2021 13:14:16 +0200 Subject: [PATCH 03/90] export PartialFile and OpenPartialFile --- .../sector-storage/ffiwrapper/partialfile.go | 24 +++++++++---------- .../sector-storage/ffiwrapper/sealer_cgo.go | 12 +++++----- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/extern/sector-storage/ffiwrapper/partialfile.go b/extern/sector-storage/ffiwrapper/partialfile.go index e19930ac1..0e8827dd3 100644 --- a/extern/sector-storage/ffiwrapper/partialfile.go +++ b/extern/sector-storage/ffiwrapper/partialfile.go @@ -25,7 +25,7 @@ const veryLargeRle = 1 << 20 // unsealed sector files internally have this structure // [unpadded (raw) data][rle+][4B LE length fo the rle+ field] -type partialFile struct { +type PartialFile struct { maxPiece abi.PaddedPieceSize path string @@ -57,7 +57,7 @@ func writeTrailer(maxPieceSize int64, w *os.File, r rlepluslazy.RunIterator) err return w.Truncate(maxPieceSize + int64(rb) + 4) } -func createPartialFile(maxPieceSize abi.PaddedPieceSize, path string) (*partialFile, error) { +func createPartialFile(maxPieceSize abi.PaddedPieceSize, path string) (*PartialFile, error) { f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0644) // nolint if err != nil { return nil, xerrors.Errorf("openning partial file '%s': %w", path, err) @@ -89,10 +89,10 @@ func createPartialFile(maxPieceSize abi.PaddedPieceSize, path string) (*partialF return nil, xerrors.Errorf("close empty partial file: %w", err) } - return openPartialFile(maxPieceSize, path) + return OpenPartialFile(maxPieceSize, path) } -func openPartialFile(maxPieceSize abi.PaddedPieceSize, path string) (*partialFile, error) { +func OpenPartialFile(maxPieceSize abi.PaddedPieceSize, path string) (*PartialFile, error) { f, err := os.OpenFile(path, os.O_RDWR, 0644) // nolint if err != nil { return nil, xerrors.Errorf("openning partial file '%s': %w", path, err) @@ -165,7 +165,7 @@ func openPartialFile(maxPieceSize abi.PaddedPieceSize, path string) (*partialFil return nil, err } - return &partialFile{ + return &PartialFile{ maxPiece: maxPieceSize, path: path, allocated: rle, @@ -173,11 +173,11 @@ func openPartialFile(maxPieceSize abi.PaddedPieceSize, path string) (*partialFil }, nil } -func (pf *partialFile) Close() error { +func (pf *PartialFile) Close() error { return pf.file.Close() } -func (pf *partialFile) Writer(offset storiface.PaddedByteIndex, size abi.PaddedPieceSize) (io.Writer, error) { +func (pf *PartialFile) Writer(offset storiface.PaddedByteIndex, size abi.PaddedPieceSize) (io.Writer, error) { if _, err := pf.file.Seek(int64(offset), io.SeekStart); err != nil { return nil, xerrors.Errorf("seek piece start: %w", err) } @@ -206,7 +206,7 @@ func (pf *partialFile) Writer(offset storiface.PaddedByteIndex, size abi.PaddedP return pf.file, nil } -func (pf *partialFile) MarkAllocated(offset storiface.PaddedByteIndex, size abi.PaddedPieceSize) error { +func (pf *PartialFile) MarkAllocated(offset storiface.PaddedByteIndex, size abi.PaddedPieceSize) error { have, err := pf.allocated.RunIterator() if err != nil { return err @@ -224,7 +224,7 @@ func (pf *partialFile) MarkAllocated(offset storiface.PaddedByteIndex, size abi. return nil } -func (pf *partialFile) Free(offset storiface.PaddedByteIndex, size abi.PaddedPieceSize) error { +func (pf *PartialFile) Free(offset storiface.PaddedByteIndex, size abi.PaddedPieceSize) error { have, err := pf.allocated.RunIterator() if err != nil { return err @@ -246,7 +246,7 @@ func (pf *partialFile) Free(offset storiface.PaddedByteIndex, size abi.PaddedPie return nil } -func (pf *partialFile) Reader(offset storiface.PaddedByteIndex, size abi.PaddedPieceSize) (*os.File, error) { +func (pf *PartialFile) Reader(offset storiface.PaddedByteIndex, size abi.PaddedPieceSize) (*os.File, error) { if _, err := pf.file.Seek(int64(offset), io.SeekStart); err != nil { return nil, xerrors.Errorf("seek piece start: %w", err) } @@ -275,11 +275,11 @@ func (pf *partialFile) Reader(offset storiface.PaddedByteIndex, size abi.PaddedP return pf.file, nil } -func (pf *partialFile) Allocated() (rlepluslazy.RunIterator, error) { +func (pf *PartialFile) Allocated() (rlepluslazy.RunIterator, error) { return pf.allocated.RunIterator() } -func (pf *partialFile) HasAllocated(offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (bool, error) { +func (pf *PartialFile) HasAllocated(offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (bool, error) { have, err := pf.Allocated() if err != nil { return false, err diff --git a/extern/sector-storage/ffiwrapper/sealer_cgo.go b/extern/sector-storage/ffiwrapper/sealer_cgo.go index 36fbacb30..f7848de6e 100644 --- a/extern/sector-storage/ffiwrapper/sealer_cgo.go +++ b/extern/sector-storage/ffiwrapper/sealer_cgo.go @@ -66,7 +66,7 @@ func (sb *Sealer) AddPiece(ctx context.Context, sector storage.SectorRef, existi } var done func() - var stagedFile *partialFile + var stagedFile *PartialFile defer func() { if done != nil { @@ -97,7 +97,7 @@ func (sb *Sealer) AddPiece(ctx context.Context, sector storage.SectorRef, existi return abi.PieceInfo{}, xerrors.Errorf("acquire unsealed sector: %w", err) } - stagedFile, err = openPartialFile(maxPieceSize, stagedPath.Unsealed) + stagedFile, err = OpenPartialFile(maxPieceSize, stagedPath.Unsealed) if err != nil { return abi.PieceInfo{}, xerrors.Errorf("opening unsealed sector file: %w", err) } @@ -257,7 +257,7 @@ func (sb *Sealer) UnsealPiece(ctx context.Context, sector storage.SectorRef, off // try finding existing unsealedPath, done, err := sb.sectors.AcquireSector(ctx, sector, storiface.FTUnsealed, storiface.FTNone, storiface.PathStorage) - var pf *partialFile + var pf *PartialFile switch { case xerrors.Is(err, storiface.ErrSectorNotFound): @@ -275,7 +275,7 @@ func (sb *Sealer) UnsealPiece(ctx context.Context, sector storage.SectorRef, off case err == nil: defer done() - pf, err = openPartialFile(maxPieceSize, unsealedPath.Unsealed) + pf, err = OpenPartialFile(maxPieceSize, unsealedPath.Unsealed) if err != nil { return xerrors.Errorf("opening partial file: %w", err) } @@ -427,7 +427,7 @@ func (sb *Sealer) ReadPiece(ctx context.Context, writer io.Writer, sector storag } maxPieceSize := abi.PaddedPieceSize(ssize) - pf, err := openPartialFile(maxPieceSize, path.Unsealed) + pf, err := OpenPartialFile(maxPieceSize, path.Unsealed) if err != nil { if xerrors.Is(err, os.ErrNotExist) { return false, nil @@ -611,7 +611,7 @@ func (sb *Sealer) FinalizeSector(ctx context.Context, sector storage.SectorRef, } defer done() - pf, err := openPartialFile(maxPieceSize, paths.Unsealed) + pf, err := OpenPartialFile(maxPieceSize, paths.Unsealed) if err == nil { var at uint64 for sr.HasNext() { From 55401116bb3753fcf384fbd4cd8714b3e775de39 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 19 May 2021 13:16:29 +0200 Subject: [PATCH 04/90] add UnpaddedByteIndex.Valid() --- extern/sector-storage/storiface/ffi.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/extern/sector-storage/storiface/ffi.go b/extern/sector-storage/storiface/ffi.go index f6b2cbdd3..da855d7a7 100644 --- a/extern/sector-storage/storiface/ffi.go +++ b/extern/sector-storage/storiface/ffi.go @@ -5,6 +5,7 @@ import ( "errors" "github.com/ipfs/go-cid" + xerrors "golang.org/x/xerrors" "github.com/filecoin-project/go-state-types/abi" ) @@ -17,6 +18,14 @@ func (i UnpaddedByteIndex) Padded() PaddedByteIndex { return PaddedByteIndex(abi.UnpaddedPieceSize(i).Padded()) } +func (i UnpaddedByteIndex) Valid() error { + if i%127 != 0 { + return xerrors.Errorf("unpadded byte index must be a multiple of 127") + } + + return nil +} + type PaddedByteIndex uint64 type RGetter func(ctx context.Context, id abi.SectorID) (cid.Cid, error) From 83e55dc09e76a33f1bf50c8b3051a7e85a07176d Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 19 May 2021 13:17:59 +0200 Subject: [PATCH 05/90] move handleProvingSector to correct file - states_proving.go --- extern/storage-sealing/states_proving.go | 19 +++++++++++++++++++ extern/storage-sealing/states_sealing.go | 19 ------------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/extern/storage-sealing/states_proving.go b/extern/storage-sealing/states_proving.go index 212fd906f..5e613b20b 100644 --- a/extern/storage-sealing/states_proving.go +++ b/extern/storage-sealing/states_proving.go @@ -126,3 +126,22 @@ func (m *Sealing) handleRemoving(ctx statemachine.Context, sector SectorInfo) er return ctx.Send(SectorRemoved{}) } + +func (m *Sealing) handleProvingSector(ctx statemachine.Context, sector SectorInfo) error { + // TODO: track sector health / expiration + log.Infof("Proving sector %d", sector.SectorNumber) + + cfg, err := m.getConfig() + if err != nil { + return xerrors.Errorf("getting sealing config: %w", err) + } + + if err := m.sealer.ReleaseUnsealed(ctx.Context(), m.minerSector(sector.SectorType, sector.SectorNumber), sector.keepUnsealedRanges(true, cfg.AlwaysKeepUnsealedCopy)); err != nil { + log.Error(err) + } + + // TODO: Watch termination + // TODO: Auto-extend if set + + return nil +} diff --git a/extern/storage-sealing/states_sealing.go b/extern/storage-sealing/states_sealing.go index e371ab33f..4f710e0ae 100644 --- a/extern/storage-sealing/states_sealing.go +++ b/extern/storage-sealing/states_sealing.go @@ -562,22 +562,3 @@ func (m *Sealing) handleFinalizeSector(ctx statemachine.Context, sector SectorIn return ctx.Send(SectorFinalized{}) } - -func (m *Sealing) handleProvingSector(ctx statemachine.Context, sector SectorInfo) error { - // TODO: track sector health / expiration - log.Infof("Proving sector %d", sector.SectorNumber) - - cfg, err := m.getConfig() - if err != nil { - return xerrors.Errorf("getting sealing config: %w", err) - } - - if err := m.sealer.ReleaseUnsealed(ctx.Context(), m.minerSector(sector.SectorType, sector.SectorNumber), sector.keepUnsealedRanges(true, cfg.AlwaysKeepUnsealedCopy)); err != nil { - log.Error(err) - } - - // TODO: Watch termination - // TODO: Auto-extend if set - - return nil -} From 1295c924e1e6bed13acbcf4e7c76dde59cad9d26 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 19 May 2021 13:22:00 +0200 Subject: [PATCH 06/90] confirm that Miner struct satisfies sectorblocks.SectorBuilder --- storage/sealing.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/storage/sealing.go b/storage/sealing.go index fdf8f25fa..453f2d9ba 100644 --- a/storage/sealing.go +++ b/storage/sealing.go @@ -13,6 +13,7 @@ import ( "github.com/filecoin-project/lotus/api" sealing "github.com/filecoin-project/lotus/extern/storage-sealing" + "github.com/filecoin-project/lotus/storage/sectorblocks" ) // TODO: refactor this to be direct somehow @@ -132,3 +133,5 @@ func (m *Miner) SectorsStatus(ctx context.Context, sid abi.SectorNumber, showOnC return sInfo, nil } + +var _ sectorblocks.SectorBuilder = &Miner{} From 2aad7b69796e2b6ee4e1ad089c95ee3b1493328a Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Thu, 20 May 2021 12:32:29 +0200 Subject: [PATCH 07/90] update sectorstorage.New (Manager) interface --- cmd/lotus-storage-miner/init.go | 12 ++++++++++-- extern/sector-storage/manager.go | 9 +-------- extern/sector-storage/stores/local.go | 2 ++ node/modules/storageminer.go | 13 +++++++++++-- 4 files changed, 24 insertions(+), 12 deletions(-) diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index 76132a7a2..71098e725 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -453,14 +453,22 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api v1api.FullNode wsts := statestore.New(namespace.Wrap(mds, modules.WorkerCallsPrefix)) smsts := statestore.New(namespace.Wrap(mds, modules.ManagerWorkPrefix)) - smgr, err := sectorstorage.New(ctx, lr, stores.NewIndex(), sectorstorage.SealerConfig{ + si := stores.NewIndex() + + lstor, err := stores.NewLocal(ctx, lr, si, nil) + if err != nil { + return err + } + stor := stores.NewRemote(lstor, si, nil, 10) + + smgr, err := sectorstorage.New(ctx, lstor, stor, lr, si, sectorstorage.SealerConfig{ ParallelFetchLimit: 10, AllowAddPiece: true, AllowPreCommit1: true, AllowPreCommit2: true, AllowCommit: true, AllowUnseal: true, - }, nil, sa, wsts, smsts) + }, sa, wsts, smsts) if err != nil { return err } diff --git a/extern/sector-storage/manager.go b/extern/sector-storage/manager.go index 70d55ee2e..7eb6df648 100644 --- a/extern/sector-storage/manager.go +++ b/extern/sector-storage/manager.go @@ -105,19 +105,12 @@ type StorageAuth http.Header type WorkerStateStore *statestore.StateStore type ManagerStateStore *statestore.StateStore -func New(ctx context.Context, ls stores.LocalStorage, si stores.SectorIndex, sc SealerConfig, urls URLs, sa StorageAuth, wss WorkerStateStore, mss ManagerStateStore) (*Manager, error) { - lstor, err := stores.NewLocal(ctx, ls, si, urls) - if err != nil { - return nil, err - } - +func New(ctx context.Context, lstor *stores.Local, stor *stores.Remote, ls stores.LocalStorage, si stores.SectorIndex, sc SealerConfig, sa StorageAuth, wss WorkerStateStore, mss ManagerStateStore) (*Manager, error) { prover, err := ffiwrapper.New(&readonlyProvider{stor: lstor, index: si}) if err != nil { return nil, xerrors.Errorf("creating prover instance: %w", err) } - stor := stores.NewRemote(lstor, si, http.Header(sa), sc.ParallelFetchLimit) - m := &Manager{ ls: ls, storage: stor, diff --git a/extern/sector-storage/stores/local.go b/extern/sector-storage/stores/local.go index 5a10b21b9..cac160139 100644 --- a/extern/sector-storage/stores/local.go +++ b/extern/sector-storage/stores/local.go @@ -158,6 +158,8 @@ func (p *path) sectorPath(sid abi.SectorID, fileType storiface.SectorFileType) s return filepath.Join(p.local, fileType.String(), storiface.SectorName(sid)) } +type URLs []string + func NewLocal(ctx context.Context, ls LocalStorage, index SectorIndex, urls []string) (*Local, error) { l := &Local{ localStorage: ls, diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index be949255f..2007470d4 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -661,13 +661,22 @@ func RetrievalProvider(h host.Host, var WorkerCallsPrefix = datastore.NewKey("/worker/calls") var ManagerWorkPrefix = datastore.NewKey("/stmgr/calls") -func SectorStorage(mctx helpers.MetricsCtx, lc fx.Lifecycle, ls stores.LocalStorage, si stores.SectorIndex, sc sectorstorage.SealerConfig, urls sectorstorage.URLs, sa sectorstorage.StorageAuth, ds dtypes.MetadataDS) (*sectorstorage.Manager, error) { +func LocalStorage(mctx helpers.MetricsCtx, lc fx.Lifecycle, ls stores.LocalStorage, si stores.SectorIndex, urls stores.URLs) (*stores.Local, error) { + ctx := helpers.LifecycleCtx(mctx, lc) + return stores.NewLocal(ctx, ls, si, urls) +} + +func RemoteStorage(lstor *stores.Local, si stores.SectorIndex, sa sectorstorage.StorageAuth, sc sectorstorage.SealerConfig) *stores.Remote { + return stores.NewRemote(lstor, si, http.Header(sa), sc.ParallelFetchLimit) +} + +func SectorStorage(mctx helpers.MetricsCtx, lc fx.Lifecycle, lstor *stores.Local, stor *stores.Remote, ls stores.LocalStorage, si stores.SectorIndex, sc sectorstorage.SealerConfig, sa sectorstorage.StorageAuth, ds dtypes.MetadataDS) (*sectorstorage.Manager, error) { ctx := helpers.LifecycleCtx(mctx, lc) wsts := statestore.New(namespace.Wrap(ds, WorkerCallsPrefix)) smsts := statestore.New(namespace.Wrap(ds, ManagerWorkPrefix)) - sst, err := sectorstorage.New(ctx, ls, si, sc, urls, sa, wsts, smsts) + sst, err := sectorstorage.New(ctx, lstor, stor, ls, si, sc, sa, wsts, smsts) if err != nil { return nil, err } From 2562f2e9a6a03a129e8bdd610adfa2670e346e01 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Thu, 20 May 2021 12:36:24 +0200 Subject: [PATCH 08/90] add ReaderParamDecoder to json rpc server to lotus-miner run --- cmd/lotus-storage-miner/run.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/lotus-storage-miner/run.go b/cmd/lotus-storage-miner/run.go index 5d67cf33d..c722cc51e 100644 --- a/cmd/lotus-storage-miner/run.go +++ b/cmd/lotus-storage-miner/run.go @@ -28,6 +28,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" lcli "github.com/filecoin-project/lotus/cli" + "github.com/filecoin-project/lotus/lib/rpcenc" "github.com/filecoin-project/lotus/lib/ulimit" "github.com/filecoin-project/lotus/metrics" "github.com/filecoin-project/lotus/node" @@ -171,10 +172,12 @@ var runCmd = &cli.Command{ mux := mux.NewRouter() - rpcServer := jsonrpc.NewServer() + readerHandler, readerServerOpt := rpcenc.ReaderParamDecoder() + rpcServer := jsonrpc.NewServer(readerServerOpt) rpcServer.Register("Filecoin", api.PermissionedStorMinerAPI(metrics.MetricedStorMinerAPI(minerapi))) mux.Handle("/rpc/v0", rpcServer) + mux.Handle("/rpc/streams/v0/push/{uuid}", readerHandler) mux.PathPrefix("/remote").HandlerFunc(minerapi.(*impl.StorageMinerAPI).ServeRemote) mux.Handle("/debug/metrics", metrics.Exporter()) mux.PathPrefix("/").Handler(http.DefaultServeMux) // pprof From c12d802811393fcd6469f512960d957d1593d3a9 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Thu, 20 May 2021 12:38:38 +0200 Subject: [PATCH 09/90] update reader to use ReadAtLeast --- extern/sector-storage/fr32/readers.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extern/sector-storage/fr32/readers.go b/extern/sector-storage/fr32/readers.go index 20f3e9b31..256821c80 100644 --- a/extern/sector-storage/fr32/readers.go +++ b/extern/sector-storage/fr32/readers.go @@ -51,13 +51,13 @@ func (r *unpadReader) Read(out []byte) (int, error) { r.left -= uint64(todo) - n, err := r.src.Read(r.work[:todo]) + n, err := io.ReadAtLeast(r.src, r.work[:todo], int(todo)) if err != nil && err != io.EOF { return n, err } if n != int(todo) { - return 0, xerrors.Errorf("didn't read enough: %w", err) + return 0, xerrors.Errorf("didn't read enough: %d / %d, left %d, out %d", n, todo, r.left, len(out)) } Unpad(r.work[:todo], out[:todo.Unpadded()]) From cb603c62d93fa7036c2c615be367c68f37290b96 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Thu, 20 May 2021 12:49:44 +0200 Subject: [PATCH 10/90] update retrievaladapter ; add piece_provider --- extern/sector-storage/manager.go | 5 +- extern/sector-storage/piece_provider.go | 110 ++++++++++++++++++ extern/sector-storage/stores/remote.go | 144 ++++++++++++++++++++++++ markets/retrievaladapter/provider.go | 50 ++++---- node/builder.go | 4 +- node/modules/storageminer.go | 24 ++-- 6 files changed, 288 insertions(+), 49 deletions(-) create mode 100644 extern/sector-storage/piece_provider.go diff --git a/extern/sector-storage/manager.go b/extern/sector-storage/manager.go index 7eb6df648..fab003a2a 100644 --- a/extern/sector-storage/manager.go +++ b/extern/sector-storage/manager.go @@ -29,8 +29,6 @@ var log = logging.Logger("advmgr") var ErrNoWorkers = errors.New("no suitable workers found") -type URLs []string - type Worker interface { storiface.WorkerCalls @@ -47,8 +45,6 @@ type Worker interface { } type SectorManager interface { - ReadPiece(context.Context, io.Writer, storage.SectorRef, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) error - ffiwrapper.StorageSealer storage.Prover storiface.WorkerReturn @@ -804,3 +800,4 @@ func (m *Manager) Close(ctx context.Context) error { } var _ SectorManager = &Manager{} +var _ Unsealer = &Manager{} diff --git a/extern/sector-storage/piece_provider.go b/extern/sector-storage/piece_provider.go new file mode 100644 index 000000000..99677121d --- /dev/null +++ b/extern/sector-storage/piece_provider.go @@ -0,0 +1,110 @@ +package sectorstorage + +import ( + "bufio" + "context" + "io" + + "github.com/ipfs/go-cid" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/specs-storage/storage" + + "github.com/filecoin-project/lotus/extern/sector-storage/fr32" + "github.com/filecoin-project/lotus/extern/sector-storage/stores" + "github.com/filecoin-project/lotus/extern/sector-storage/storiface" +) + +type Unsealer interface { + SectorsUnsealPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, commd *cid.Cid) error +} + +type PieceProvider struct { + storage *stores.Remote + index stores.SectorIndex + uns Unsealer +} + +func NewPieceProvider(storage *stores.Remote, index stores.SectorIndex, uns Unsealer) *PieceProvider { + return &PieceProvider{ + storage: storage, + index: index, + uns: uns, + } +} + +func (p *PieceProvider) tryReadUnsealedPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (io.ReadCloser, context.CancelFunc, error) { + // acquire a lock purely for reading unsealed sectors + ctx, cancel := context.WithCancel(ctx) + if err := p.index.StorageLock(ctx, sector.ID, storiface.FTUnsealed, storiface.FTNone); err != nil { + cancel() + return nil, nil, xerrors.Errorf("acquiring read sector lock: %w", err) + } + + r, err := p.storage.Reader(ctx, sector, abi.PaddedPieceSize(offset.Padded()), size.Padded(), storiface.FTUnsealed) + if err != nil { + cancel() + return nil, nil, err + } + if r == nil { + cancel() + } + + return r, cancel, nil +} + +func (p *PieceProvider) ReadPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, ticket abi.SealRandomness, unsealed cid.Cid) (io.ReadCloser, bool, error) { + if err := offset.Valid(); err != nil { + return nil, false, xerrors.Errorf("offset is not valid: %w", err) + } + if err := size.Validate(); err != nil { + return nil, false, xerrors.Errorf("size is not a valid piece size: %w", err) + } + + r, unlock, err := p.tryReadUnsealedPiece(ctx, sector, offset, size) + if err != nil { + return nil, false, err + } + + var uns bool + if r == nil { + uns = true + commd := &unsealed + if unsealed == cid.Undef { + commd = nil + } + if err := p.uns.SectorsUnsealPiece(ctx, sector, offset, size, ticket, commd); err != nil { + return nil, false, xerrors.Errorf("unsealing piece: %w", err) + } + + r, unlock, err = p.tryReadUnsealedPiece(ctx, sector, offset, size) + if err != nil { + return nil, true, xerrors.Errorf("read after unsealing: %w", err) + } + if r == nil { + return nil, true, xerrors.Errorf("got no reader after unsealing piece") + } + } + + upr, err := fr32.NewUnpadReader(r, size.Padded()) + if err != nil { + return nil, uns, xerrors.Errorf("creating unpadded reader: %w", err) + } + + return &funcCloser{ + Reader: bufio.NewReaderSize(upr, 127), + close: func() error { + err = r.Close() + unlock() + return err + }, + }, uns, nil +} + +type funcCloser struct { + io.Reader + close func() error +} + +func (fc *funcCloser) Close() error { return fc.close() } diff --git a/extern/sector-storage/stores/remote.go b/extern/sector-storage/stores/remote.go index 4388a2ffb..280b3760b 100644 --- a/extern/sector-storage/stores/remote.go +++ b/extern/sector-storage/stores/remote.go @@ -3,6 +3,7 @@ package stores import ( "context" "encoding/json" + "fmt" "io" "io/ioutil" "math/bits" @@ -15,6 +16,7 @@ import ( "sort" "sync" + "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/extern/sector-storage/fsutil" "github.com/filecoin-project/lotus/extern/sector-storage/storiface" "github.com/filecoin-project/lotus/extern/sector-storage/tarutil" @@ -293,6 +295,148 @@ func (r *Remote) fetch(ctx context.Context, url, outname string) error { } } +func (r *Remote) checkAllocated(ctx context.Context, url string, spt abi.RegisteredSealProof, offset, size abi.PaddedPieceSize) (bool, error) { + url = fmt.Sprintf("%s/%d/allocated/%d/%d", url, spt, offset.Unpadded(), size.Unpadded()) + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return false, xerrors.Errorf("request: %w", err) + } + req.Header = r.auth.Clone() + req = req.WithContext(ctx) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return false, xerrors.Errorf("do request: %w", err) + } + defer resp.Body.Close() // nolint + + switch resp.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusRequestedRangeNotSatisfiable: + return false, nil + default: + return false, xerrors.Errorf("unexpected http response: %d", resp.StatusCode) + } +} + +func (r *Remote) readRemote(ctx context.Context, url string, spt abi.RegisteredSealProof, offset, size abi.PaddedPieceSize) (io.ReadCloser, error) { + if len(r.limit) >= cap(r.limit) { + log.Infof("Throttling remote read, %d already running", len(r.limit)) + } + + // TODO: Smarter throttling + // * Priority (just going sequentially is still pretty good) + // * Per interface + // * Aware of remote load + select { + case r.limit <- struct{}{}: + defer func() { <-r.limit }() + case <-ctx.Done(): + return nil, xerrors.Errorf("context error while waiting for fetch limiter: %w", ctx.Err()) + } + + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, xerrors.Errorf("request: %w", err) + } + req.Header = r.auth.Clone() + req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+size-1)) + req = req.WithContext(ctx) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, xerrors.Errorf("do request: %w", err) + } + + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { + resp.Body.Close() // nolint + return nil, xerrors.Errorf("non-200 code: %d", resp.StatusCode) + } + + return resp.Body, nil +} + +// Reated gets a reader for unsealed file range. Can return nil in case the requested range isn't allocated in the file +func (r *Remote) Reader(ctx context.Context, s storage.SectorRef, offset, size abi.PaddedPieceSize, ft storiface.SectorFileType) (io.ReadCloser, error) { + if ft != storiface.FTUnsealed { + return nil, xerrors.Errorf("reader only supports unsealed files") + } + + paths, _, err := r.local.AcquireSector(ctx, s, ft, storiface.FTNone, storiface.PathStorage, storiface.AcquireMove) + if err != nil { + return nil, xerrors.Errorf("acquire local: %w", err) + } + + path := storiface.PathByType(paths, ft) + var rd io.ReadCloser + if path == "" { + si, err := r.index.StorageFindSector(ctx, s.ID, ft, 0, false) + if err != nil { + return nil, err + } + + if len(si) == 0 { + return nil, xerrors.Errorf("failed to read sector %v from remote(%d): %w", s, ft, storiface.ErrSectorNotFound) + } + + sort.Slice(si, func(i, j int) bool { + return si[i].Weight < si[j].Weight + }) + + iloop: + for _, info := range si { + for _, url := range info.URLs { + ok, err := r.checkAllocated(ctx, url, s.ProofType, offset, size) + if err != nil { + log.Warnw("check if remote has piece", "url", url, "error", err) + continue + } + if !ok { + continue + } + + rd, err = r.readRemote(ctx, url, s.ProofType, offset, size) + if err != nil { + log.Warnw("reading from remote", "url", url, "error", err) + continue + } + log.Infof("Read remote %s (+%d,%d)", url, offset, size) + break iloop + } + } + } else { + log.Infof("Read local %s (+%d,%d)", path, offset, size) + ssize, err := s.ProofType.SectorSize() + if err != nil { + return nil, err + } + + pf, err := ffiwrapper.OpenPartialFile(abi.PaddedPieceSize(ssize), path) + if err != nil { + return nil, xerrors.Errorf("opening partial file: %w", err) + } + + has, err := pf.HasAllocated(storiface.UnpaddedByteIndex(offset.Unpadded()), size.Unpadded()) + if err != nil { + return nil, xerrors.Errorf("has allocated: %w", err) + } + + if !has { + if err := pf.Close(); err != nil { + return nil, xerrors.Errorf("close partial file: %w", err) + } + + return nil, nil + } + + return pf.Reader(storiface.PaddedByteIndex(offset), size) + } + + // note: rd can be nil + return rd, nil +} + func (r *Remote) MoveStorage(ctx context.Context, s storage.SectorRef, types storiface.SectorFileType) error { // Make sure we have the data local _, _, err := r.AcquireSector(ctx, s, types, storiface.FTNone, storiface.PathStorage, storiface.AcquireMove) diff --git a/markets/retrievaladapter/provider.go b/markets/retrievaladapter/provider.go index e58257c8a..2e255c07c 100644 --- a/markets/retrievaladapter/provider.go +++ b/markets/retrievaladapter/provider.go @@ -5,6 +5,9 @@ import ( "io" "github.com/filecoin-project/lotus/api/v1api" + "github.com/filecoin-project/lotus/node/modules/dtypes" + "github.com/filecoin-project/lotus/storage/sectorblocks" + "golang.org/x/xerrors" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" @@ -13,7 +16,6 @@ import ( "github.com/filecoin-project/lotus/chain/types" sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" "github.com/filecoin-project/lotus/extern/sector-storage/storiface" - "github.com/filecoin-project/lotus/storage" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-fil-markets/retrievalmarket" @@ -25,15 +27,16 @@ import ( var log = logging.Logger("retrievaladapter") type retrievalProviderNode struct { - miner *storage.Miner - sealer sectorstorage.SectorManager - full v1api.FullNode + maddr address.Address + secb sectorblocks.SectorBuilder + pp *sectorstorage.PieceProvider + full v1api.FullNode } // NewRetrievalProviderNode returns a new node adapter for a retrieval provider that talks to the // Lotus Node -func NewRetrievalProviderNode(miner *storage.Miner, sealer sectorstorage.SectorManager, full v1api.FullNode) retrievalmarket.RetrievalProviderNode { - return &retrievalProviderNode{miner, sealer, full} +func NewRetrievalProviderNode(maddr dtypes.MinerAddress, secb sectorblocks.SectorBuilder, pp *sectorstorage.PieceProvider, full v1api.FullNode) retrievalmarket.RetrievalProviderNode { + return &retrievalProviderNode{address.Address(maddr), secb, pp, full} } func (rpn *retrievalProviderNode) GetMinerWorkerAddress(ctx context.Context, miner address.Address, tok shared.TipSetToken) (address.Address, error) { @@ -47,14 +50,12 @@ func (rpn *retrievalProviderNode) GetMinerWorkerAddress(ctx context.Context, min } func (rpn *retrievalProviderNode) UnsealSector(ctx context.Context, sectorID abi.SectorNumber, offset abi.UnpaddedPieceSize, length abi.UnpaddedPieceSize) (io.ReadCloser, error) { - log.Debugf("get sector %d, offset %d, length %d", sectorID, offset, length) - - si, err := rpn.miner.GetSectorInfo(sectorID) + si, err := rpn.secb.SectorsStatus(ctx, sectorID, false) if err != nil { return nil, err } - mid, err := address.IDFromAddress(rpn.miner.Address()) + mid, err := address.IDFromAddress(rpn.maddr) if err != nil { return nil, err } @@ -64,27 +65,20 @@ func (rpn *retrievalProviderNode) UnsealSector(ctx context.Context, sectorID abi Miner: abi.ActorID(mid), Number: sectorID, }, - ProofType: si.SectorType, + ProofType: si.SealProof, } - // Set up a pipe so that data can be written from the unsealing process - // into the reader returned by this function - r, w := io.Pipe() - go func() { - var commD cid.Cid - if si.CommD != nil { - commD = *si.CommD - } + var commD cid.Cid + if si.CommD != nil { + commD = *si.CommD + } - // Read the piece into the pipe's writer, unsealing the piece if necessary - log.Debugf("read piece in sector %d, offset %d, length %d from miner %d", sectorID, offset, length, mid) - err := rpn.sealer.ReadPiece(ctx, w, ref, storiface.UnpaddedByteIndex(offset), length, si.TicketValue, commD) - if err != nil { - log.Errorf("failed to unseal piece from sector %d: %s", sectorID, err) - } - // Close the reader with any error that was returned while reading the piece - _ = w.CloseWithError(err) - }() + // Read the piece into the pipe's writer, unsealing the piece if necessary + r, unsealed, err := rpn.pp.ReadPiece(ctx, ref, storiface.UnpaddedByteIndex(offset), length, si.Ticket.Value, commD) + if err != nil { + return nil, xerrors.Errorf("failed to unseal piece from sector %d: %w", sectorID, err) + } + _ = unsealed // todo: use return r, nil } diff --git a/node/builder.go b/node/builder.go index 34be610f5..4466b39c2 100644 --- a/node/builder.go +++ b/node/builder.go @@ -490,10 +490,10 @@ func ConfigCommon(cfg *config.Common) Option { Override(SetApiEndpointKey, func(lr repo.LockedRepo, e dtypes.APIEndpoint) error { return lr.SetAPIEndpoint(e) }), - Override(new(sectorstorage.URLs), func(e dtypes.APIEndpoint) (sectorstorage.URLs, error) { + Override(new(stores.URLs), func(e dtypes.APIEndpoint) (stores.URLs, error) { ip := cfg.API.RemoteListenAddress - var urls sectorstorage.URLs + var urls stores.URLs urls = append(urls, "http://"+ip+"/remote") // TODO: This makes no assumptions, and probably could... return urls, nil }), diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 2007470d4..b81a15916 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -67,7 +67,6 @@ import ( "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/markets" marketevents "github.com/filecoin-project/lotus/markets/loggers" - "github.com/filecoin-project/lotus/markets/retrievaladapter" lotusminer "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node/config" "github.com/filecoin-project/lotus/node/modules/dtypes" @@ -632,11 +631,15 @@ func RetrievalDealFilter(userFilter dtypes.RetrievalDealFilter) func(onlineOk dt } } +func RetrievalNetwork(h host.Host) rmnet.RetrievalMarketNetwork { + return rmnet.NewFromLibp2pHost(h) +} + // RetrievalProvider creates a new retrieval provider attached to the provider blockstore -func RetrievalProvider(h host.Host, - miner *storage.Miner, - sealer sectorstorage.SectorManager, - full v1api.FullNode, +func RetrievalProvider( + maddr dtypes.MinerAddress, + adapter retrievalmarket.RetrievalProviderNode, + netwk rmnet.RetrievalMarketNetwork, ds dtypes.MetadataDS, pieceStore dtypes.ProviderPieceStore, mds dtypes.StagingMultiDstore, @@ -645,17 +648,8 @@ func RetrievalProvider(h host.Host, offlineOk dtypes.ConsiderOfflineRetrievalDealsConfigFunc, userFilter dtypes.RetrievalDealFilter, ) (retrievalmarket.RetrievalProvider, error) { - adapter := retrievaladapter.NewRetrievalProviderNode(miner, sealer, full) - - maddr, err := minerAddrFromDS(ds) - if err != nil { - return nil, err - } - - netwk := rmnet.NewFromLibp2pHost(h) opt := retrievalimpl.DealDeciderOpt(retrievalimpl.DealDecider(userFilter)) - - return retrievalimpl.NewProvider(maddr, adapter, netwk, pieceStore, mds, dt, namespace.Wrap(ds, datastore.NewKey("/retrievals/provider")), opt) + return retrievalimpl.NewProvider(address.Address(maddr), adapter, netwk, pieceStore, mds, dt, namespace.Wrap(ds, datastore.NewKey("/retrievals/provider")), opt) } var WorkerCallsPrefix = datastore.NewKey("/worker/calls") From 76bb424de09c32a5196a21581aadf40736804f36 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Thu, 20 May 2021 13:01:14 +0200 Subject: [PATCH 11/90] update StorageMinerAPI --- extern/sector-storage/stores/index.go | 2 + node/impl/storminer.go | 147 ++++++++++---------------- storage/sealing.go | 4 - 3 files changed, 57 insertions(+), 96 deletions(-) diff --git a/extern/sector-storage/stores/index.go b/extern/sector-storage/stores/index.go index 4acc2ecdb..200e19f0b 100644 --- a/extern/sector-storage/stores/index.go +++ b/extern/sector-storage/stores/index.go @@ -65,6 +65,8 @@ type SectorIndex interface { // part of storage-miner api // atomically acquire locks on all sector file types. close ctx to unlock StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error) + + StorageList(ctx context.Context) (map[ID][]Decl, error) } type Decl struct { diff --git a/node/impl/storminer.go b/node/impl/storminer.go index bf1c44fc2..527a977c7 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -25,7 +25,6 @@ import ( storagemarket "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-jsonrpc/auth" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/big" sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" "github.com/filecoin-project/lotus/extern/sector-storage/fsutil" @@ -48,44 +47,49 @@ import ( type StorageMinerAPI struct { common.CommonAPI - SectorBlocks *sectorblocks.SectorBlocks + Full api.FullNode + LocalStore *stores.Local + ReboteStore *stores.Remote - PieceStore dtypes.ProviderPieceStore - StorageProvider storagemarket.StorageProvider - RetrievalProvider retrievalmarket.RetrievalProvider - Miner *storage.Miner - BlockMiner *miner.Miner - Full api.FullNode - StorageMgr *sectorstorage.Manager `optional:"true"` - IStorageMgr sectorstorage.SectorManager - *stores.Index - storiface.WorkerReturn - DataTransfer dtypes.ProviderDataTransfer - Host host.Host - AddrSel *storage.AddressSelector - DealPublisher *storageadapter.DealPublisher + // Markets + PieceStore dtypes.ProviderPieceStore `optional:"true"` + StorageProvider storagemarket.StorageProvider `optional:"true"` + RetrievalProvider retrievalmarket.RetrievalProvider `optional:"true"` + DataTransfer dtypes.ProviderDataTransfer `optional:"true"` + DealPublisher *storageadapter.DealPublisher `optional:"true"` + SectorBlocks *sectorblocks.SectorBlocks `optional:"true"` - Epp gen.WinningPoStProver + // Miner / storage + Miner *storage.Miner `optional:"true"` + BlockMiner *miner.Miner `optional:"true"` + StorageMgr *sectorstorage.Manager `optional:"true"` + IStorageMgr sectorstorage.SectorManager `optional:"true"` + stores.SectorIndex + storiface.WorkerReturn `optional:"true"` + Host host.Host + AddrSel *storage.AddressSelector + + Epp gen.WinningPoStProver `optional:"true"` DS dtypes.MetadataDS - ConsiderOnlineStorageDealsConfigFunc dtypes.ConsiderOnlineStorageDealsConfigFunc - SetConsiderOnlineStorageDealsConfigFunc dtypes.SetConsiderOnlineStorageDealsConfigFunc - ConsiderOnlineRetrievalDealsConfigFunc dtypes.ConsiderOnlineRetrievalDealsConfigFunc - SetConsiderOnlineRetrievalDealsConfigFunc dtypes.SetConsiderOnlineRetrievalDealsConfigFunc - StorageDealPieceCidBlocklistConfigFunc dtypes.StorageDealPieceCidBlocklistConfigFunc - SetStorageDealPieceCidBlocklistConfigFunc dtypes.SetStorageDealPieceCidBlocklistConfigFunc - ConsiderOfflineStorageDealsConfigFunc dtypes.ConsiderOfflineStorageDealsConfigFunc - SetConsiderOfflineStorageDealsConfigFunc dtypes.SetConsiderOfflineStorageDealsConfigFunc - ConsiderOfflineRetrievalDealsConfigFunc dtypes.ConsiderOfflineRetrievalDealsConfigFunc - SetConsiderOfflineRetrievalDealsConfigFunc dtypes.SetConsiderOfflineRetrievalDealsConfigFunc - ConsiderVerifiedStorageDealsConfigFunc dtypes.ConsiderVerifiedStorageDealsConfigFunc - SetConsiderVerifiedStorageDealsConfigFunc dtypes.SetConsiderVerifiedStorageDealsConfigFunc - ConsiderUnverifiedStorageDealsConfigFunc dtypes.ConsiderUnverifiedStorageDealsConfigFunc - SetConsiderUnverifiedStorageDealsConfigFunc dtypes.SetConsiderUnverifiedStorageDealsConfigFunc - SetSealingConfigFunc dtypes.SetSealingConfigFunc - GetSealingConfigFunc dtypes.GetSealingConfigFunc - GetExpectedSealDurationFunc dtypes.GetExpectedSealDurationFunc - SetExpectedSealDurationFunc dtypes.SetExpectedSealDurationFunc + ConsiderOnlineStorageDealsConfigFunc dtypes.ConsiderOnlineStorageDealsConfigFunc `optional:"true"` + SetConsiderOnlineStorageDealsConfigFunc dtypes.SetConsiderOnlineStorageDealsConfigFunc `optional:"true"` + ConsiderOnlineRetrievalDealsConfigFunc dtypes.ConsiderOnlineRetrievalDealsConfigFunc `optional:"true"` + SetConsiderOnlineRetrievalDealsConfigFunc dtypes.SetConsiderOnlineRetrievalDealsConfigFunc `optional:"true"` + StorageDealPieceCidBlocklistConfigFunc dtypes.StorageDealPieceCidBlocklistConfigFunc `optional:"true"` + SetStorageDealPieceCidBlocklistConfigFunc dtypes.SetStorageDealPieceCidBlocklistConfigFunc `optional:"true"` + ConsiderOfflineStorageDealsConfigFunc dtypes.ConsiderOfflineStorageDealsConfigFunc `optional:"true"` + SetConsiderOfflineStorageDealsConfigFunc dtypes.SetConsiderOfflineStorageDealsConfigFunc `optional:"true"` + ConsiderOfflineRetrievalDealsConfigFunc dtypes.ConsiderOfflineRetrievalDealsConfigFunc `optional:"true"` + SetConsiderOfflineRetrievalDealsConfigFunc dtypes.SetConsiderOfflineRetrievalDealsConfigFunc `optional:"true"` + ConsiderVerifiedStorageDealsConfigFunc dtypes.ConsiderVerifiedStorageDealsConfigFunc `optional:"true"` + SetConsiderVerifiedStorageDealsConfigFunc dtypes.SetConsiderVerifiedStorageDealsConfigFunc `optional:"true"` + ConsiderUnverifiedStorageDealsConfigFunc dtypes.ConsiderUnverifiedStorageDealsConfigFunc `optional:"true"` + SetConsiderUnverifiedStorageDealsConfigFunc dtypes.SetConsiderUnverifiedStorageDealsConfigFunc `optional:"true"` + SetSealingConfigFunc dtypes.SetSealingConfigFunc `optional:"true"` + GetSealingConfigFunc dtypes.GetSealingConfigFunc `optional:"true"` + GetExpectedSealDurationFunc dtypes.GetExpectedSealDurationFunc `optional:"true"` + SetExpectedSealDurationFunc dtypes.SetExpectedSealDurationFunc `optional:"true"` } func (sm *StorageMinerAPI) ServeRemote(w http.ResponseWriter, r *http.Request) { @@ -135,12 +139,12 @@ func (sm *StorageMinerAPI) PledgeSector(ctx context.Context) (abi.SectorID, erro // wait for the sector to enter the Packing state // TODO: instead of polling implement some pubsub-type thing in storagefsm for { - info, err := sm.Miner.GetSectorInfo(sr.ID.Number) + info, err := sm.Miner.SectorsStatus(ctx, sr.ID.Number, false) if err != nil { return abi.SectorID{}, xerrors.Errorf("getting pledged sector info: %w", err) } - if info.State != sealing.UndefinedSectorState { + if info.State != api.SectorState(sealing.UndefinedSectorState) { return sr.ID, nil } @@ -153,62 +157,11 @@ func (sm *StorageMinerAPI) PledgeSector(ctx context.Context) (abi.SectorID, erro } func (sm *StorageMinerAPI) SectorsStatus(ctx context.Context, sid abi.SectorNumber, showOnChainInfo bool) (api.SectorInfo, error) { - info, err := sm.Miner.GetSectorInfo(sid) + sInfo, err := sm.Miner.SectorsStatus(ctx, sid, false) if err != nil { return api.SectorInfo{}, err } - deals := make([]abi.DealID, len(info.Pieces)) - for i, piece := range info.Pieces { - if piece.DealInfo == nil { - continue - } - deals[i] = piece.DealInfo.DealID - } - - log := make([]api.SectorLog, len(info.Log)) - for i, l := range info.Log { - log[i] = api.SectorLog{ - Kind: l.Kind, - Timestamp: l.Timestamp, - Trace: l.Trace, - Message: l.Message, - } - } - - sInfo := api.SectorInfo{ - SectorID: sid, - State: api.SectorState(info.State), - CommD: info.CommD, - CommR: info.CommR, - Proof: info.Proof, - Deals: deals, - Ticket: api.SealTicket{ - Value: info.TicketValue, - Epoch: info.TicketEpoch, - }, - Seed: api.SealSeed{ - Value: info.SeedValue, - Epoch: info.SeedEpoch, - }, - PreCommitMsg: info.PreCommitMessage, - CommitMsg: info.CommitMessage, - Retries: info.InvalidProofs, - ToUpgrade: sm.Miner.IsMarkedForUpgrade(sid), - - LastErr: info.LastErr, - Log: log, - // on chain info - SealProof: 0, - Activation: 0, - Expiration: 0, - DealWeight: big.Zero(), - VerifiedDealWeight: big.Zero(), - InitialPledge: big.Zero(), - OnTime: 0, - Early: 0, - } - if !showOnChainInfo { return sInfo, nil } @@ -307,7 +260,17 @@ func (sm *StorageMinerAPI) SectorsSummary(ctx context.Context) (map[api.SectorSt } func (sm *StorageMinerAPI) StorageLocal(ctx context.Context) (map[stores.ID]string, error) { - return sm.StorageMgr.StorageLocal(ctx) + l, err := sm.LocalStore.Local(ctx) + if err != nil { + return nil, err + } + + out := map[stores.ID]string{} + for _, st := range l { + out[st.ID] = st.LocalPath + } + + return out, nil } func (sm *StorageMinerAPI) SectorsRefs(context.Context) (map[string][]api.SealedRef, error) { @@ -327,7 +290,7 @@ func (sm *StorageMinerAPI) SectorsRefs(context.Context) (map[string][]api.Sealed } func (sm *StorageMinerAPI) StorageStat(ctx context.Context, id stores.ID) (fsutil.FsStat, error) { - return sm.StorageMgr.FsStat(ctx, id) + return sm.ReboteStore.FsStat(ctx, id) } func (sm *StorageMinerAPI) SectorStartSealing(ctx context.Context, number abi.SectorNumber) error { @@ -674,7 +637,7 @@ func (sm *StorageMinerAPI) CheckProvable(ctx context.Context, pp abi.RegisteredP var rg storiface.RGetter if expensive { rg = func(ctx context.Context, id abi.SectorID) (cid.Cid, error) { - si, err := sm.Miner.GetSectorInfo(id.Number) + si, err := sm.Miner.SectorsStatus(ctx, id.Number, false) if err != nil { return cid.Undef, err } diff --git a/storage/sealing.go b/storage/sealing.go index 453f2d9ba..8c62750b4 100644 --- a/storage/sealing.go +++ b/storage/sealing.go @@ -30,10 +30,6 @@ func (m *Miner) ListSectors() ([]sealing.SectorInfo, error) { return m.sealing.ListSectors() } -func (m *Miner) GetSectorInfo(sid abi.SectorNumber) (sealing.SectorInfo, error) { - return m.sealing.GetSectorInfo(sid) -} - func (m *Miner) PledgeSector(ctx context.Context) (storage.SectorRef, error) { return m.sealing.PledgeSector(ctx) } From 4693c613057151ed05b71a374d7af9545db8aff5 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Thu, 20 May 2021 13:10:14 +0200 Subject: [PATCH 12/90] re-arrange NodeBuilder and add storageminer_svc --- node/builder.go | 346 +------------------------------ node/builder_chain.go | 212 +++++++++++++++++++ node/builder_miner.go | 202 ++++++++++++++++++ node/config/def.go | 11 + node/modules/storageminer_svc.go | 71 +++++++ 5 files changed, 497 insertions(+), 345 deletions(-) create mode 100644 node/builder_chain.go create mode 100644 node/builder_miner.go create mode 100644 node/modules/storageminer_svc.go diff --git a/node/builder.go b/node/builder.go index 4466b39c2..aaaa1247e 100644 --- a/node/builder.go +++ b/node/builder.go @@ -8,14 +8,6 @@ import ( metricsi "github.com/ipfs/go-metrics-interface" - "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/chain" - "github.com/filecoin-project/lotus/chain/exchange" - rpcstmgr "github.com/filecoin-project/lotus/chain/stmgr/rpc" - "github.com/filecoin-project/lotus/chain/store" - "github.com/filecoin-project/lotus/chain/vm" - "github.com/filecoin-project/lotus/chain/wallet" - "github.com/filecoin-project/lotus/node/hello" "github.com/filecoin-project/lotus/system" logging "github.com/ipfs/go-log/v2" @@ -33,52 +25,21 @@ import ( "go.uber.org/fx" "golang.org/x/xerrors" - "github.com/filecoin-project/go-fil-markets/discovery" - discoveryimpl "github.com/filecoin-project/go-fil-markets/discovery/impl" - "github.com/filecoin-project/go-fil-markets/retrievalmarket" - "github.com/filecoin-project/go-fil-markets/storagemarket" - "github.com/filecoin-project/go-fil-markets/storagemarket/impl/storedask" - - storage2 "github.com/filecoin-project/specs-storage/storage" - - "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/beacon" - "github.com/filecoin-project/lotus/chain/gen" - "github.com/filecoin-project/lotus/chain/gen/slashfilter" - "github.com/filecoin-project/lotus/chain/market" - "github.com/filecoin-project/lotus/chain/messagepool" - "github.com/filecoin-project/lotus/chain/messagesigner" - "github.com/filecoin-project/lotus/chain/metrics" - "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/types" - ledgerwallet "github.com/filecoin-project/lotus/chain/wallet/ledger" - "github.com/filecoin-project/lotus/chain/wallet/remotewallet" - sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/extern/sector-storage/stores" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/lib/peermgr" _ "github.com/filecoin-project/lotus/lib/sigs/bls" _ "github.com/filecoin-project/lotus/lib/sigs/secp" - "github.com/filecoin-project/lotus/markets/dealfilter" "github.com/filecoin-project/lotus/markets/storageadapter" - "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node/config" - "github.com/filecoin-project/lotus/node/impl" - "github.com/filecoin-project/lotus/node/impl/common" - "github.com/filecoin-project/lotus/node/impl/full" "github.com/filecoin-project/lotus/node/modules" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/helpers" "github.com/filecoin-project/lotus/node/modules/lp2p" "github.com/filecoin-project/lotus/node/modules/testing" "github.com/filecoin-project/lotus/node/repo" - "github.com/filecoin-project/lotus/paychmgr" - "github.com/filecoin-project/lotus/paychmgr/settler" - "github.com/filecoin-project/lotus/storage" - "github.com/filecoin-project/lotus/storage/sectorblocks" ) //nolint:deadcode,varcheck @@ -246,199 +207,6 @@ func isFullOrLiteNode(s *Settings) bool { return s.nodeType == repo.FullNode } func isFullNode(s *Settings) bool { return s.nodeType == repo.FullNode && !s.Lite } func isLiteNode(s *Settings) bool { return s.nodeType == repo.FullNode && s.Lite } -// Chain node provides access to the Filecoin blockchain, by setting up a full -// validator node, or by delegating some actions to other nodes (lite mode) -var ChainNode = Options( - // Full node or lite node - // TODO: Fix offline mode - - // Consensus settings - Override(new(dtypes.DrandSchedule), modules.BuiltinDrandConfig), - Override(new(stmgr.UpgradeSchedule), stmgr.DefaultUpgradeSchedule()), - Override(new(dtypes.NetworkName), modules.NetworkName), - Override(new(modules.Genesis), modules.ErrorGenesis), - Override(new(dtypes.AfterGenesisSet), modules.SetGenesis), - Override(SetGenesisKey, modules.DoSetGenesis), - Override(new(beacon.Schedule), modules.RandomSchedule), - - // Network bootstrap - Override(new(dtypes.BootstrapPeers), modules.BuiltinBootstrap), - Override(new(dtypes.DrandBootstrap), modules.DrandBootstrap), - - // Consensus: crypto dependencies - Override(new(ffiwrapper.Verifier), ffiwrapper.ProofVerifier), - - // Consensus: VM - Override(new(vm.SyscallBuilder), vm.Syscalls), - - // Consensus: Chain storage/access - Override(new(*store.ChainStore), modules.ChainStore), - Override(new(*stmgr.StateManager), modules.StateManager), - Override(new(dtypes.ChainBitswap), modules.ChainBitswap), - Override(new(dtypes.ChainBlockService), modules.ChainBlockService), // todo: unused - - // Consensus: Chain sync - - // We don't want the SyncManagerCtor to be used as an fx constructor, but rather as a value. - // It will be called implicitly by the Syncer constructor. - Override(new(chain.SyncManagerCtor), func() chain.SyncManagerCtor { return chain.NewSyncManager }), - Override(new(*chain.Syncer), modules.NewSyncer), - Override(new(exchange.Client), exchange.NewClient), - - // Chain networking - Override(new(*hello.Service), hello.NewHelloService), - Override(new(exchange.Server), exchange.NewServer), - Override(new(*peermgr.PeerMgr), peermgr.NewPeerMgr), - - // Chain mining API dependencies - Override(new(*slashfilter.SlashFilter), modules.NewSlashFilter), - - // Service: Message Pool - Override(new(dtypes.DefaultMaxFeeFunc), modules.NewDefaultMaxFeeFunc), - Override(new(*messagepool.MessagePool), modules.MessagePool), - Override(new(*dtypes.MpoolLocker), new(dtypes.MpoolLocker)), - - // Shared graphsync (markets, serving chain) - Override(new(dtypes.Graphsync), modules.Graphsync(config.DefaultFullNode().Client.SimultaneousTransfers)), - - // Service: Wallet - Override(new(*messagesigner.MessageSigner), messagesigner.NewMessageSigner), - Override(new(*wallet.LocalWallet), wallet.NewWallet), - Override(new(wallet.Default), From(new(*wallet.LocalWallet))), - Override(new(api.Wallet), From(new(wallet.MultiWallet))), - - // Service: Payment channels - Override(new(paychmgr.PaychAPI), From(new(modules.PaychAPI))), - Override(new(*paychmgr.Store), modules.NewPaychStore), - Override(new(*paychmgr.Manager), modules.NewManager), - Override(HandlePaymentChannelManagerKey, modules.HandlePaychManager), - Override(SettlePaymentChannelsKey, settler.SettlePaymentChannels), - - // Markets (common) - Override(new(*discoveryimpl.Local), modules.NewLocalDiscovery), - - // Markets (retrieval) - Override(new(discovery.PeerResolver), modules.RetrievalResolver), - Override(new(retrievalmarket.RetrievalClient), modules.RetrievalClient), - Override(new(dtypes.ClientDataTransfer), modules.NewClientGraphsyncDataTransfer), - - // Markets (storage) - Override(new(*market.FundManager), market.NewFundManager), - Override(new(dtypes.ClientDatastore), modules.NewClientDatastore), - Override(new(storagemarket.StorageClient), modules.StorageClient), - Override(new(storagemarket.StorageClientNode), storageadapter.NewClientNodeAdapter), - Override(HandleMigrateClientFundsKey, modules.HandleMigrateClientFunds), - - Override(new(*full.GasPriceCache), full.NewGasPriceCache), - - // Lite node API - ApplyIf(isLiteNode, - Override(new(messagepool.Provider), messagepool.NewProviderLite), - Override(new(messagesigner.MpoolNonceAPI), From(new(modules.MpoolNonceAPI))), - Override(new(full.ChainModuleAPI), From(new(api.Gateway))), - Override(new(full.GasModuleAPI), From(new(api.Gateway))), - Override(new(full.MpoolModuleAPI), From(new(api.Gateway))), - Override(new(full.StateModuleAPI), From(new(api.Gateway))), - Override(new(stmgr.StateManagerAPI), rpcstmgr.NewRPCStateManager), - ), - - // Full node API / service startup - ApplyIf(isFullNode, - Override(new(messagepool.Provider), messagepool.NewProvider), - Override(new(messagesigner.MpoolNonceAPI), From(new(*messagepool.MessagePool))), - Override(new(full.ChainModuleAPI), From(new(full.ChainModule))), - Override(new(full.GasModuleAPI), From(new(full.GasModule))), - Override(new(full.MpoolModuleAPI), From(new(full.MpoolModule))), - Override(new(full.StateModuleAPI), From(new(full.StateModule))), - Override(new(stmgr.StateManagerAPI), From(new(*stmgr.StateManager))), - - Override(RunHelloKey, modules.RunHello), - Override(RunChainExchangeKey, modules.RunChainExchange), - Override(RunPeerMgrKey, modules.RunPeerMgr), - Override(HandleIncomingMessagesKey, modules.HandleIncomingMessages), - Override(HandleIncomingBlocksKey, modules.HandleIncomingBlocks), - ), -) - -var MinerNode = Options( - // API dependencies - Override(new(api.Common), From(new(common.CommonAPI))), - Override(new(sectorstorage.StorageAuth), modules.StorageAuth), - - // Actor config - Override(new(dtypes.MinerAddress), modules.MinerAddress), - Override(new(dtypes.MinerID), modules.MinerID), - Override(new(abi.RegisteredSealProof), modules.SealProofType), - Override(new(dtypes.NetworkName), modules.StorageNetworkName), - - // Sector storage - Override(new(*stores.Index), stores.NewIndex), - Override(new(stores.SectorIndex), From(new(*stores.Index))), - Override(new(stores.LocalStorage), From(new(repo.LockedRepo))), - Override(new(*sectorstorage.Manager), modules.SectorStorage), - Override(new(sectorstorage.SectorManager), From(new(*sectorstorage.Manager))), - Override(new(storiface.WorkerReturn), From(new(sectorstorage.SectorManager))), - - // Sector storage: Proofs - Override(new(ffiwrapper.Verifier), ffiwrapper.ProofVerifier), - Override(new(storage2.Prover), From(new(sectorstorage.SectorManager))), - - // Sealing - Override(new(sealing.SectorIDCounter), modules.SectorIDCounter), - Override(GetParamsKey, modules.GetParams), - - // Mining / proving - Override(new(*slashfilter.SlashFilter), modules.NewSlashFilter), - Override(new(*storage.Miner), modules.StorageMiner(config.DefaultStorageMiner().Fees)), - Override(new(*miner.Miner), modules.SetupBlockProducer), - Override(new(gen.WinningPoStProver), storage.NewWinningPoStProver), - - Override(new(*storage.AddressSelector), modules.AddressSelector(nil)), - - // Markets - Override(new(dtypes.StagingMultiDstore), modules.StagingMultiDatastore), - Override(new(dtypes.StagingBlockstore), modules.StagingBlockstore), - Override(new(dtypes.StagingDAG), modules.StagingDAG), - Override(new(dtypes.StagingGraphsync), modules.StagingGraphsync), - Override(new(dtypes.ProviderPieceStore), modules.NewProviderPieceStore), - Override(new(*sectorblocks.SectorBlocks), sectorblocks.NewSectorBlocks), - - // Markets (retrieval) - Override(new(retrievalmarket.RetrievalProvider), modules.RetrievalProvider), - Override(new(dtypes.RetrievalDealFilter), modules.RetrievalDealFilter(nil)), - Override(HandleRetrievalKey, modules.HandleRetrieval), - - // Markets (storage) - Override(new(dtypes.ProviderDataTransfer), modules.NewProviderDAGServiceDataTransfer), - Override(new(*storedask.StoredAsk), modules.NewStorageAsk), - Override(new(dtypes.StorageDealFilter), modules.BasicDealFilter(nil)), - Override(new(storagemarket.StorageProvider), modules.StorageProvider), - Override(new(*storageadapter.DealPublisher), storageadapter.NewDealPublisher(nil, storageadapter.PublishMsgConfig{})), - Override(new(storagemarket.StorageProviderNode), storageadapter.NewProviderNodeAdapter(nil, nil)), - Override(HandleMigrateProviderFundsKey, modules.HandleMigrateProviderFunds), - Override(HandleDealsKey, modules.HandleDeals), - - // Config (todo: get a real property system) - Override(new(dtypes.ConsiderOnlineStorageDealsConfigFunc), modules.NewConsiderOnlineStorageDealsConfigFunc), - Override(new(dtypes.SetConsiderOnlineStorageDealsConfigFunc), modules.NewSetConsideringOnlineStorageDealsFunc), - Override(new(dtypes.ConsiderOnlineRetrievalDealsConfigFunc), modules.NewConsiderOnlineRetrievalDealsConfigFunc), - Override(new(dtypes.SetConsiderOnlineRetrievalDealsConfigFunc), modules.NewSetConsiderOnlineRetrievalDealsConfigFunc), - Override(new(dtypes.StorageDealPieceCidBlocklistConfigFunc), modules.NewStorageDealPieceCidBlocklistConfigFunc), - Override(new(dtypes.SetStorageDealPieceCidBlocklistConfigFunc), modules.NewSetStorageDealPieceCidBlocklistConfigFunc), - Override(new(dtypes.ConsiderOfflineStorageDealsConfigFunc), modules.NewConsiderOfflineStorageDealsConfigFunc), - Override(new(dtypes.SetConsiderOfflineStorageDealsConfigFunc), modules.NewSetConsideringOfflineStorageDealsFunc), - Override(new(dtypes.ConsiderOfflineRetrievalDealsConfigFunc), modules.NewConsiderOfflineRetrievalDealsConfigFunc), - Override(new(dtypes.SetConsiderOfflineRetrievalDealsConfigFunc), modules.NewSetConsiderOfflineRetrievalDealsConfigFunc), - Override(new(dtypes.ConsiderVerifiedStorageDealsConfigFunc), modules.NewConsiderVerifiedStorageDealsConfigFunc), - Override(new(dtypes.SetConsiderVerifiedStorageDealsConfigFunc), modules.NewSetConsideringVerifiedStorageDealsFunc), - Override(new(dtypes.ConsiderUnverifiedStorageDealsConfigFunc), modules.NewConsiderUnverifiedStorageDealsConfigFunc), - Override(new(dtypes.SetConsiderUnverifiedStorageDealsConfigFunc), modules.NewSetConsideringUnverifiedStorageDealsFunc), - Override(new(dtypes.SetSealingConfigFunc), modules.NewSetSealConfigFunc), - Override(new(dtypes.GetSealingConfigFunc), modules.NewGetSealConfigFunc), - Override(new(dtypes.SetExpectedSealDurationFunc), modules.NewSetExpectedSealDurationFunc), - Override(new(dtypes.GetExpectedSealDurationFunc), modules.NewGetExpectedSealDurationFunc), -) - // Online sets up basic libp2p node func Online() Option { @@ -457,29 +225,6 @@ func Online() Option { ) } -func StorageMiner(out *api.StorageMiner) Option { - return Options( - ApplyIf(func(s *Settings) bool { return s.Config }, - Error(errors.New("the StorageMiner option must be set before Config option")), - ), - ApplyIf(func(s *Settings) bool { return s.Online }, - Error(errors.New("the StorageMiner option must be set before Online option")), - ), - - func(s *Settings) error { - s.nodeType = repo.StorageMiner - return nil - }, - - func(s *Settings) error { - resAPI := &impl.StorageMinerAPI{} - s.invokes[ExtractApiKey] = fx.Populate(resAPI) - *out = resAPI - return nil - }, - ) -} - // Config sets up constructors based on the provided Config func ConfigCommon(cfg *config.Common) Option { return Options( @@ -518,70 +263,6 @@ func ConfigCommon(cfg *config.Common) Option { ) } -func ConfigFullNode(c interface{}) Option { - cfg, ok := c.(*config.FullNode) - if !ok { - return Error(xerrors.Errorf("invalid config from repo, got: %T", c)) - } - - ipfsMaddr := cfg.Client.IpfsMAddr - return Options( - ConfigCommon(&cfg.Common), - - If(cfg.Client.UseIpfs, - Override(new(dtypes.ClientBlockstore), modules.IpfsClientBlockstore(ipfsMaddr, cfg.Client.IpfsOnlineMode)), - If(cfg.Client.IpfsUseForRetrieval, - Override(new(dtypes.ClientRetrievalStoreManager), modules.ClientBlockstoreRetrievalStoreManager), - ), - ), - Override(new(dtypes.Graphsync), modules.Graphsync(cfg.Client.SimultaneousTransfers)), - - If(cfg.Metrics.HeadNotifs, - Override(HeadMetricsKey, metrics.SendHeadNotifs(cfg.Metrics.Nickname)), - ), - - If(cfg.Wallet.RemoteBackend != "", - Override(new(*remotewallet.RemoteWallet), remotewallet.SetupRemoteWallet(cfg.Wallet.RemoteBackend)), - ), - If(cfg.Wallet.EnableLedger, - Override(new(*ledgerwallet.LedgerWallet), ledgerwallet.NewWallet), - ), - If(cfg.Wallet.DisableLocal, - Unset(new(*wallet.LocalWallet)), - Override(new(wallet.Default), wallet.NilDefault), - ), - ) -} - -func ConfigStorageMiner(c interface{}) Option { - cfg, ok := c.(*config.StorageMiner) - if !ok { - return Error(xerrors.Errorf("invalid config from repo, got: %T", c)) - } - - return Options( - ConfigCommon(&cfg.Common), - - If(cfg.Dealmaking.Filter != "", - Override(new(dtypes.StorageDealFilter), modules.BasicDealFilter(dealfilter.CliStorageDealFilter(cfg.Dealmaking.Filter))), - ), - - If(cfg.Dealmaking.RetrievalFilter != "", - Override(new(dtypes.RetrievalDealFilter), modules.RetrievalDealFilter(dealfilter.CliRetrievalDealFilter(cfg.Dealmaking.RetrievalFilter))), - ), - - Override(new(*storageadapter.DealPublisher), storageadapter.NewDealPublisher(&cfg.Fees, storageadapter.PublishMsgConfig{ - Period: time.Duration(cfg.Dealmaking.PublishMsgPeriod), - MaxDealsPerMsg: cfg.Dealmaking.MaxDealsPerPublishMsg, - })), - Override(new(storagemarket.StorageProviderNode), storageadapter.NewProviderNodeAdapter(&cfg.Fees, &cfg.Dealmaking)), - - Override(new(sectorstorage.SealerConfig), cfg.Storage), - Override(new(*storage.AddressSelector), modules.AddressSelector(&cfg.Addresses)), - Override(new(*storage.Miner), modules.StorageMiner(cfg.Fees)), - ) -} - func Repo(r repo.Repo) Option { return func(settings *Settings) error { lr, err := r.Lock(settings.nodeType) @@ -654,31 +335,6 @@ func Repo(r repo.Repo) Option { } } -type FullOption = Option - -func Lite(enable bool) FullOption { - return func(s *Settings) error { - s.Lite = enable - return nil - } -} - -func FullAPI(out *api.FullNode, fopts ...FullOption) Option { - return Options( - func(s *Settings) error { - s.nodeType = repo.FullNode - return nil - }, - Options(fopts...), - func(s *Settings) error { - resAPI := &impl.FullNodeAPI{} - s.invokes[ExtractApiKey] = fx.Populate(resAPI) - *out = resAPI - return nil - }, - ) -} - type StopFunc func(context.Context) error // New builds and starts new Filecoin node @@ -710,7 +366,7 @@ func New(ctx context.Context, opts ...Option) (StopFunc, error) { fx.Options(ctors...), fx.Options(settings.invokes...), - fx.NopLogger, + //fx.NopLogger, ) // TODO: we probably should have a 'firewall' for Closing signal diff --git a/node/builder_chain.go b/node/builder_chain.go new file mode 100644 index 000000000..6fb5b540e --- /dev/null +++ b/node/builder_chain.go @@ -0,0 +1,212 @@ +package node + +import ( + "go.uber.org/fx" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-fil-markets/discovery" + discoveryimpl "github.com/filecoin-project/go-fil-markets/discovery/impl" + "github.com/filecoin-project/go-fil-markets/retrievalmarket" + "github.com/filecoin-project/go-fil-markets/storagemarket" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain" + "github.com/filecoin-project/lotus/chain/beacon" + "github.com/filecoin-project/lotus/chain/exchange" + "github.com/filecoin-project/lotus/chain/gen/slashfilter" + "github.com/filecoin-project/lotus/chain/market" + "github.com/filecoin-project/lotus/chain/messagepool" + "github.com/filecoin-project/lotus/chain/messagesigner" + "github.com/filecoin-project/lotus/chain/metrics" + "github.com/filecoin-project/lotus/chain/stmgr" + rpcstmgr "github.com/filecoin-project/lotus/chain/stmgr/rpc" + "github.com/filecoin-project/lotus/chain/store" + "github.com/filecoin-project/lotus/chain/vm" + "github.com/filecoin-project/lotus/chain/wallet" + ledgerwallet "github.com/filecoin-project/lotus/chain/wallet/ledger" + "github.com/filecoin-project/lotus/chain/wallet/remotewallet" + "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" + "github.com/filecoin-project/lotus/lib/peermgr" + "github.com/filecoin-project/lotus/markets/storageadapter" + "github.com/filecoin-project/lotus/node/config" + "github.com/filecoin-project/lotus/node/hello" + "github.com/filecoin-project/lotus/node/impl" + "github.com/filecoin-project/lotus/node/impl/full" + "github.com/filecoin-project/lotus/node/modules" + "github.com/filecoin-project/lotus/node/modules/dtypes" + "github.com/filecoin-project/lotus/node/repo" + "github.com/filecoin-project/lotus/paychmgr" + "github.com/filecoin-project/lotus/paychmgr/settler" +) + +// Chain node provides access to the Filecoin blockchain, by setting up a full +// validator node, or by delegating some actions to other nodes (lite mode) +var ChainNode = Options( + // Full node or lite node + // TODO: Fix offline mode + + // Consensus settings + Override(new(dtypes.DrandSchedule), modules.BuiltinDrandConfig), + Override(new(stmgr.UpgradeSchedule), stmgr.DefaultUpgradeSchedule()), + Override(new(dtypes.NetworkName), modules.NetworkName), + Override(new(modules.Genesis), modules.ErrorGenesis), + Override(new(dtypes.AfterGenesisSet), modules.SetGenesis), + Override(SetGenesisKey, modules.DoSetGenesis), + Override(new(beacon.Schedule), modules.RandomSchedule), + + // Network bootstrap + Override(new(dtypes.BootstrapPeers), modules.BuiltinBootstrap), + Override(new(dtypes.DrandBootstrap), modules.DrandBootstrap), + + // Consensus: crypto dependencies + Override(new(ffiwrapper.Verifier), ffiwrapper.ProofVerifier), + + // Consensus: VM + Override(new(vm.SyscallBuilder), vm.Syscalls), + + // Consensus: Chain storage/access + Override(new(*store.ChainStore), modules.ChainStore), + Override(new(*stmgr.StateManager), modules.StateManager), + Override(new(dtypes.ChainBitswap), modules.ChainBitswap), + Override(new(dtypes.ChainBlockService), modules.ChainBlockService), // todo: unused + + // Consensus: Chain sync + + // We don't want the SyncManagerCtor to be used as an fx constructor, but rather as a value. + // It will be called implicitly by the Syncer constructor. + Override(new(chain.SyncManagerCtor), func() chain.SyncManagerCtor { return chain.NewSyncManager }), + Override(new(*chain.Syncer), modules.NewSyncer), + Override(new(exchange.Client), exchange.NewClient), + + // Chain networking + Override(new(*hello.Service), hello.NewHelloService), + Override(new(exchange.Server), exchange.NewServer), + Override(new(*peermgr.PeerMgr), peermgr.NewPeerMgr), + + // Chain mining API dependencies + Override(new(*slashfilter.SlashFilter), modules.NewSlashFilter), + + // Service: Message Pool + Override(new(dtypes.DefaultMaxFeeFunc), modules.NewDefaultMaxFeeFunc), + Override(new(*messagepool.MessagePool), modules.MessagePool), + Override(new(*dtypes.MpoolLocker), new(dtypes.MpoolLocker)), + + // Shared graphsync (markets, serving chain) + Override(new(dtypes.Graphsync), modules.Graphsync(config.DefaultFullNode().Client.SimultaneousTransfers)), + + // Service: Wallet + Override(new(*messagesigner.MessageSigner), messagesigner.NewMessageSigner), + Override(new(*wallet.LocalWallet), wallet.NewWallet), + Override(new(wallet.Default), From(new(*wallet.LocalWallet))), + Override(new(api.Wallet), From(new(wallet.MultiWallet))), + + // Service: Payment channels + Override(new(paychmgr.PaychAPI), From(new(modules.PaychAPI))), + Override(new(*paychmgr.Store), modules.NewPaychStore), + Override(new(*paychmgr.Manager), modules.NewManager), + Override(HandlePaymentChannelManagerKey, modules.HandlePaychManager), + Override(SettlePaymentChannelsKey, settler.SettlePaymentChannels), + + // Markets (common) + Override(new(*discoveryimpl.Local), modules.NewLocalDiscovery), + + // Markets (retrieval) + Override(new(discovery.PeerResolver), modules.RetrievalResolver), + Override(new(retrievalmarket.RetrievalClient), modules.RetrievalClient), + Override(new(dtypes.ClientDataTransfer), modules.NewClientGraphsyncDataTransfer), + + // Markets (storage) + Override(new(*market.FundManager), market.NewFundManager), + Override(new(dtypes.ClientDatastore), modules.NewClientDatastore), + Override(new(storagemarket.StorageClient), modules.StorageClient), + Override(new(storagemarket.StorageClientNode), storageadapter.NewClientNodeAdapter), + Override(HandleMigrateClientFundsKey, modules.HandleMigrateClientFunds), + + Override(new(*full.GasPriceCache), full.NewGasPriceCache), + + // Lite node API + ApplyIf(isLiteNode, + Override(new(messagesigner.MpoolNonceAPI), From(new(modules.MpoolNonceAPI))), + Override(new(full.ChainModuleAPI), From(new(api.Gateway))), + Override(new(full.GasModuleAPI), From(new(api.Gateway))), + Override(new(full.MpoolModuleAPI), From(new(api.Gateway))), + Override(new(full.StateModuleAPI), From(new(api.Gateway))), + Override(new(stmgr.StateManagerAPI), rpcstmgr.NewRPCStateManager), + ), + + // Full node API / service startup + ApplyIf(isFullNode, + Override(new(messagesigner.MpoolNonceAPI), From(new(*messagepool.MessagePool))), + Override(new(full.ChainModuleAPI), From(new(full.ChainModule))), + Override(new(full.GasModuleAPI), From(new(full.GasModule))), + Override(new(full.MpoolModuleAPI), From(new(full.MpoolModule))), + Override(new(full.StateModuleAPI), From(new(full.StateModule))), + Override(new(stmgr.StateManagerAPI), From(new(*stmgr.StateManager))), + + Override(RunHelloKey, modules.RunHello), + Override(RunChainExchangeKey, modules.RunChainExchange), + Override(RunPeerMgrKey, modules.RunPeerMgr), + Override(HandleIncomingMessagesKey, modules.HandleIncomingMessages), + Override(HandleIncomingBlocksKey, modules.HandleIncomingBlocks), + ), +) + +func ConfigFullNode(c interface{}) Option { + cfg, ok := c.(*config.FullNode) + if !ok { + return Error(xerrors.Errorf("invalid config from repo, got: %T", c)) + } + + ipfsMaddr := cfg.Client.IpfsMAddr + return Options( + ConfigCommon(&cfg.Common), + + If(cfg.Client.UseIpfs, + Override(new(dtypes.ClientBlockstore), modules.IpfsClientBlockstore(ipfsMaddr, cfg.Client.IpfsOnlineMode)), + If(cfg.Client.IpfsUseForRetrieval, + Override(new(dtypes.ClientRetrievalStoreManager), modules.ClientBlockstoreRetrievalStoreManager), + ), + ), + Override(new(dtypes.Graphsync), modules.Graphsync(cfg.Client.SimultaneousTransfers)), + + If(cfg.Metrics.HeadNotifs, + Override(HeadMetricsKey, metrics.SendHeadNotifs(cfg.Metrics.Nickname)), + ), + + If(cfg.Wallet.RemoteBackend != "", + Override(new(*remotewallet.RemoteWallet), remotewallet.SetupRemoteWallet(cfg.Wallet.RemoteBackend)), + ), + If(cfg.Wallet.EnableLedger, + Override(new(*ledgerwallet.LedgerWallet), ledgerwallet.NewWallet), + ), + If(cfg.Wallet.DisableLocal, + Unset(new(*wallet.LocalWallet)), + Override(new(wallet.Default), wallet.NilDefault), + ), + ) +} + +type FullOption = Option + +func Lite(enable bool) FullOption { + return func(s *Settings) error { + s.Lite = enable + return nil + } +} + +func FullAPI(out *api.FullNode, fopts ...FullOption) Option { + return Options( + func(s *Settings) error { + s.nodeType = repo.FullNode + return nil + }, + Options(fopts...), + func(s *Settings) error { + resAPI := &impl.FullNodeAPI{} + s.invokes[ExtractApiKey] = fx.Populate(resAPI) + *out = resAPI + return nil + }, + ) +} diff --git a/node/builder_miner.go b/node/builder_miner.go new file mode 100644 index 000000000..eeb33749c --- /dev/null +++ b/node/builder_miner.go @@ -0,0 +1,202 @@ +package node + +import ( + "errors" + "time" + + "go.uber.org/fx" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-fil-markets/retrievalmarket" + rmnet "github.com/filecoin-project/go-fil-markets/retrievalmarket/network" + "github.com/filecoin-project/go-fil-markets/storagemarket" + "github.com/filecoin-project/go-fil-markets/storagemarket/impl/storedask" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/markets/retrievaladapter" + storage2 "github.com/filecoin-project/specs-storage/storage" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/gen" + "github.com/filecoin-project/lotus/chain/gen/slashfilter" + sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" + "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" + "github.com/filecoin-project/lotus/extern/sector-storage/stores" + "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + sealing "github.com/filecoin-project/lotus/extern/storage-sealing" + "github.com/filecoin-project/lotus/markets/dealfilter" + "github.com/filecoin-project/lotus/markets/storageadapter" + "github.com/filecoin-project/lotus/miner" + "github.com/filecoin-project/lotus/node/config" + "github.com/filecoin-project/lotus/node/impl" + "github.com/filecoin-project/lotus/node/impl/common" + "github.com/filecoin-project/lotus/node/modules" + "github.com/filecoin-project/lotus/node/modules/dtypes" + "github.com/filecoin-project/lotus/node/repo" + "github.com/filecoin-project/lotus/storage" + "github.com/filecoin-project/lotus/storage/sectorblocks" +) + +var MinerNode = Options( + // API dependencies + Override(new(api.Common), From(new(common.CommonAPI))), + Override(new(sectorstorage.StorageAuth), modules.StorageAuth), + + // Actor config + Override(new(dtypes.MinerAddress), modules.MinerAddress), + Override(new(dtypes.MinerID), modules.MinerID), + Override(new(abi.RegisteredSealProof), modules.SealProofType), + Override(new(dtypes.NetworkName), modules.StorageNetworkName), + + // Mining / proving + Override(new(*storage.AddressSelector), modules.AddressSelector(nil)), +) + +func ConfigStorageMiner(c interface{}) Option { + cfg, ok := c.(*config.StorageMiner) + if !ok { + return Error(xerrors.Errorf("invalid config from repo, got: %T", c)) + } + + return Options( + ConfigCommon(&cfg.Common), + + Override(new(stores.LocalStorage), From(new(repo.LockedRepo))), + Override(new(*stores.Local), modules.LocalStorage), + Override(new(*stores.Remote), modules.RemoteStorage), + + If(!cfg.Subsystems.EnableMining, + If(cfg.Subsystems.EnableSealing, Error(xerrors.Errorf("sealing can only be enabled on a mining node"))), + If(cfg.Subsystems.EnableSectorStorage, Error(xerrors.Errorf("sealing can only be enabled on a mining node"))), + ), + If(cfg.Subsystems.EnableMining, + If(!cfg.Subsystems.EnableSealing, Error(xerrors.Errorf("sealing can't be disabled on a mining node yet"))), + If(!cfg.Subsystems.EnableSectorStorage, Error(xerrors.Errorf("sealing can't be disabled on a mining node yet"))), + + // Sector storage: Proofs + Override(new(ffiwrapper.Verifier), ffiwrapper.ProofVerifier), + Override(new(storage2.Prover), From(new(sectorstorage.SectorManager))), + + // Sealing (todo should be under EnableSealing, but storagefsm is currently bundled with storage.Miner) + Override(new(sealing.SectorIDCounter), modules.SectorIDCounter), + Override(GetParamsKey, modules.GetParams), + + Override(new(dtypes.SetSealingConfigFunc), modules.NewSetSealConfigFunc), + Override(new(dtypes.GetSealingConfigFunc), modules.NewGetSealConfigFunc), + + // Mining / proving + Override(new(*slashfilter.SlashFilter), modules.NewSlashFilter), + Override(new(*storage.Miner), modules.StorageMiner(config.DefaultStorageMiner().Fees)), + Override(new(*miner.Miner), modules.SetupBlockProducer), + Override(new(gen.WinningPoStProver), storage.NewWinningPoStProver), + Override(new(*storage.Miner), modules.StorageMiner(cfg.Fees)), + Override(new(sectorblocks.SectorBuilder), From(new(*storage.Miner))), + ), + + If(cfg.Subsystems.EnableSectorStorage, + // Sector storage + Override(new(*stores.Index), stores.NewIndex), + Override(new(stores.SectorIndex), From(new(*stores.Index))), + Override(new(*sectorstorage.Manager), modules.SectorStorage), + Override(new(sectorstorage.Unsealer), From(new(*sectorstorage.Manager))), + Override(new(sectorstorage.SectorManager), From(new(*sectorstorage.Manager))), + Override(new(storiface.WorkerReturn), From(new(sectorstorage.SectorManager))), + ), + + If(!cfg.Subsystems.EnableSectorStorage, + Override(new(modules.MinerStorageService), modules.ConnectStorageService(cfg.Subsystems.SectorIndexApiInfo)), + Override(new(sectorstorage.Unsealer), From(new(modules.MinerStorageService))), + Override(new(sectorblocks.SectorBuilder), From(new(modules.MinerStorageService))), + ), + If(!cfg.Subsystems.EnableSealing, + Override(new(modules.MinerSealingService), modules.ConnectSealingService(cfg.Subsystems.SealerApiInfo)), + Override(new(stores.SectorIndex), From(new(modules.MinerSealingService))), + ), + + If(cfg.Subsystems.EnableStorageMarket, + // Markets + Override(new(dtypes.StagingMultiDstore), modules.StagingMultiDatastore), + Override(new(dtypes.StagingBlockstore), modules.StagingBlockstore), + Override(new(dtypes.StagingDAG), modules.StagingDAG), + Override(new(dtypes.StagingGraphsync), modules.StagingGraphsync), + Override(new(dtypes.ProviderPieceStore), modules.NewProviderPieceStore), + Override(new(*sectorblocks.SectorBlocks), sectorblocks.NewSectorBlocks), + + // Markets (retrieval deps) + Override(new(*sectorstorage.PieceProvider), sectorstorage.NewPieceProvider), + + // Markets (retrieval) + + Override(new(retrievalmarket.RetrievalProviderNode), retrievaladapter.NewRetrievalProviderNode), + Override(new(rmnet.RetrievalMarketNetwork), modules.RetrievalNetwork), + Override(new(retrievalmarket.RetrievalProvider), modules.RetrievalProvider), + Override(new(dtypes.RetrievalDealFilter), modules.RetrievalDealFilter(nil)), + Override(HandleRetrievalKey, modules.HandleRetrieval), + + // Markets (storage) + Override(new(dtypes.ProviderDataTransfer), modules.NewProviderDAGServiceDataTransfer), + Override(new(*storedask.StoredAsk), modules.NewStorageAsk), + Override(new(dtypes.StorageDealFilter), modules.BasicDealFilter(nil)), + Override(new(storagemarket.StorageProvider), modules.StorageProvider), + Override(new(*storageadapter.DealPublisher), storageadapter.NewDealPublisher(nil, storageadapter.PublishMsgConfig{})), + Override(HandleMigrateProviderFundsKey, modules.HandleMigrateProviderFunds), + Override(HandleDealsKey, modules.HandleDeals), + + // Config (todo: get a real property system) + Override(new(dtypes.ConsiderOnlineStorageDealsConfigFunc), modules.NewConsiderOnlineStorageDealsConfigFunc), + Override(new(dtypes.SetConsiderOnlineStorageDealsConfigFunc), modules.NewSetConsideringOnlineStorageDealsFunc), + Override(new(dtypes.ConsiderOnlineRetrievalDealsConfigFunc), modules.NewConsiderOnlineRetrievalDealsConfigFunc), + Override(new(dtypes.SetConsiderOnlineRetrievalDealsConfigFunc), modules.NewSetConsiderOnlineRetrievalDealsConfigFunc), + Override(new(dtypes.StorageDealPieceCidBlocklistConfigFunc), modules.NewStorageDealPieceCidBlocklistConfigFunc), + Override(new(dtypes.SetStorageDealPieceCidBlocklistConfigFunc), modules.NewSetStorageDealPieceCidBlocklistConfigFunc), + Override(new(dtypes.ConsiderOfflineStorageDealsConfigFunc), modules.NewConsiderOfflineStorageDealsConfigFunc), + Override(new(dtypes.SetConsiderOfflineStorageDealsConfigFunc), modules.NewSetConsideringOfflineStorageDealsFunc), + Override(new(dtypes.ConsiderOfflineRetrievalDealsConfigFunc), modules.NewConsiderOfflineRetrievalDealsConfigFunc), + Override(new(dtypes.SetConsiderOfflineRetrievalDealsConfigFunc), modules.NewSetConsiderOfflineRetrievalDealsConfigFunc), + Override(new(dtypes.ConsiderVerifiedStorageDealsConfigFunc), modules.NewConsiderVerifiedStorageDealsConfigFunc), + Override(new(dtypes.SetConsiderVerifiedStorageDealsConfigFunc), modules.NewSetConsideringVerifiedStorageDealsFunc), + Override(new(dtypes.ConsiderUnverifiedStorageDealsConfigFunc), modules.NewConsiderUnverifiedStorageDealsConfigFunc), + Override(new(dtypes.SetConsiderUnverifiedStorageDealsConfigFunc), modules.NewSetConsideringUnverifiedStorageDealsFunc), + Override(new(dtypes.SetExpectedSealDurationFunc), modules.NewSetExpectedSealDurationFunc), + Override(new(dtypes.GetExpectedSealDurationFunc), modules.NewGetExpectedSealDurationFunc), + + If(cfg.Dealmaking.Filter != "", + Override(new(dtypes.StorageDealFilter), modules.BasicDealFilter(dealfilter.CliStorageDealFilter(cfg.Dealmaking.Filter))), + ), + + If(cfg.Dealmaking.RetrievalFilter != "", + Override(new(dtypes.RetrievalDealFilter), modules.RetrievalDealFilter(dealfilter.CliRetrievalDealFilter(cfg.Dealmaking.RetrievalFilter))), + ), + Override(new(*storageadapter.DealPublisher), storageadapter.NewDealPublisher(&cfg.Fees, storageadapter.PublishMsgConfig{ + Period: time.Duration(cfg.Dealmaking.PublishMsgPeriod), + MaxDealsPerMsg: cfg.Dealmaking.MaxDealsPerPublishMsg, + })), + Override(new(storagemarket.StorageProviderNode), storageadapter.NewProviderNodeAdapter(&cfg.Fees, &cfg.Dealmaking)), + ), + + Override(new(sectorstorage.SealerConfig), cfg.Storage), + Override(new(*storage.AddressSelector), modules.AddressSelector(&cfg.Addresses)), + ) +} + +func StorageMiner(out *api.StorageMiner) Option { + return Options( + ApplyIf(func(s *Settings) bool { return s.Config }, + Error(errors.New("the StorageMiner option must be set before Config option")), + ), + ApplyIf(func(s *Settings) bool { return s.Online }, + Error(errors.New("the StorageMiner option must be set before Online option")), + ), + + func(s *Settings) error { + s.nodeType = repo.StorageMiner + return nil + }, + + func(s *Settings) error { + resAPI := &impl.StorageMinerAPI{} + s.invokes[ExtractApiKey] = fx.Populate(resAPI) + *out = resAPI + return nil + }, + ) +} diff --git a/node/config/def.go b/node/config/def.go index b4cf5e2fa..4013bde6e 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -38,6 +38,7 @@ type Backup struct { type StorageMiner struct { Common + Subsystems MinerSubsystemConfig Dealmaking DealmakingConfig Sealing SealingConfig Storage sectorstorage.SealerConfig @@ -45,6 +46,16 @@ type StorageMiner struct { Addresses MinerAddressConfig } +type MinerSubsystemConfig struct { + EnableMining bool + EnableSealing bool + EnableSectorStorage bool + EnableStorageMarket bool + + SealerApiInfo string // if EnableSealing == false + SectorIndexApiInfo string // if EnableSectorStorage == false +} + type DealmakingConfig struct { ConsiderOnlineStorageDeals bool ConsiderOfflineStorageDeals bool diff --git a/node/modules/storageminer_svc.go b/node/modules/storageminer_svc.go new file mode 100644 index 000000000..8718d236c --- /dev/null +++ b/node/modules/storageminer_svc.go @@ -0,0 +1,71 @@ +package modules + +import ( + "context" + + "github.com/filecoin-project/lotus/storage/sectorblocks" + + "go.uber.org/fx" + "golang.org/x/xerrors" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/api/client" + cliutil "github.com/filecoin-project/lotus/cli/util" + "github.com/filecoin-project/lotus/node/modules/helpers" +) + +type MinerSealingService api.StorageMiner +type MinerStorageService api.StorageMiner + +var _ sectorblocks.SectorBuilder = *new(MinerSealingService) + +func connectMinerService(apiInfo string) func(mctx helpers.MetricsCtx, lc fx.Lifecycle) (api.StorageMiner, error) { + return func(mctx helpers.MetricsCtx, lc fx.Lifecycle) (api.StorageMiner, error) { + ctx := helpers.LifecycleCtx(mctx, lc) + info := cliutil.ParseApiInfo(apiInfo) + addr, err := info.DialArgs("v0") + if err != nil { + return nil, xerrors.Errorf("could not get DialArgs: %w", err) + } + + log.Infof("Checking api version of %s", addr) + + mapi, closer, err := client.NewStorageMinerRPCV0(ctx, addr, info.AuthHeader()) + if err != nil { + return nil, err + } + lc.Append(fx.Hook{ + OnStart: func(ctx context.Context) error { + v, err := mapi.Version(ctx) + if err != nil { + return xerrors.Errorf("checking version: %w", err) + } + + if !v.APIVersion.EqMajorMinor(api.MinerAPIVersion0) { + // TODO(anteva): Is MinerAPIVersion0 correct??? + // we should probably bump it up + return xerrors.Errorf("remote service API version didn't match (expected %s, remote %s)", api.MinerAPIVersion0, v.APIVersion) + } + + return nil + }, + OnStop: func(context.Context) error { + closer() + return nil + }}) + + return mapi, nil + } +} + +func ConnectSealingService(apiInfo string) func(mctx helpers.MetricsCtx, lc fx.Lifecycle) (MinerSealingService, error) { + return func(mctx helpers.MetricsCtx, lc fx.Lifecycle) (MinerSealingService, error) { + return connectMinerService(apiInfo)(mctx, lc) + } +} + +func ConnectStorageService(apiInfo string) func(mctx helpers.MetricsCtx, lc fx.Lifecycle) (MinerStorageService, error) { + return func(mctx helpers.MetricsCtx, lc fx.Lifecycle) (MinerStorageService, error) { + return connectMinerService(apiInfo)(mctx, lc) + } +} From 90928991b3d299ec2185167c56c7bcc1d3d9dd9a Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Thu, 20 May 2021 13:11:10 +0200 Subject: [PATCH 13/90] remove DealInfo and DealSchedule from storage-sealing gen/main.go --- extern/storage-sealing/gen/main.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/extern/storage-sealing/gen/main.go b/extern/storage-sealing/gen/main.go index 97c2bacd5..825ce8d28 100644 --- a/extern/storage-sealing/gen/main.go +++ b/extern/storage-sealing/gen/main.go @@ -12,8 +12,6 @@ import ( func main() { err := gen.WriteMapEncodersToFile("./cbor_gen.go", "sealing", sealing.Piece{}, - sealing.DealInfo{}, - sealing.DealSchedule{}, sealing.SectorInfo{}, sealing.Log{}, ) From 1a9b5760a70ff0ab004d7e05ec0f1bae87235f47 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Thu, 20 May 2021 13:14:25 +0200 Subject: [PATCH 14/90] add remoteGetAllocated http handler --- extern/sector-storage/stores/http_handler.go | 102 +++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/extern/sector-storage/stores/http_handler.go b/extern/sector-storage/stores/http_handler.go index 3e3468470..2978c0f2f 100644 --- a/extern/sector-storage/stores/http_handler.go +++ b/extern/sector-storage/stores/http_handler.go @@ -5,11 +5,14 @@ import ( "io" "net/http" "os" + "strconv" "github.com/gorilla/mux" logging "github.com/ipfs/go-log/v2" "golang.org/x/xerrors" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/extern/sector-storage/storiface" "github.com/filecoin-project/lotus/extern/sector-storage/tarutil" @@ -26,6 +29,7 @@ func (handler *FetchHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { mux := mux.NewRouter() mux.HandleFunc("/remote/stat/{id}", handler.remoteStatFs).Methods("GET") + mux.HandleFunc("/remote/{type}/{id}/{spt}/allocated/{offset}/{size}", handler.remoteGetAllocated).Methods("GET") mux.HandleFunc("/remote/{type}/{id}", handler.remoteGetSector).Methods("GET") mux.HandleFunc("/remote/{type}/{id}", handler.remoteDeleteSector).Methods("DELETE") @@ -54,6 +58,104 @@ func (handler *FetchHandler) remoteStatFs(w http.ResponseWriter, r *http.Request } } +func (handler *FetchHandler) remoteGetAllocated(w http.ResponseWriter, r *http.Request) { + log.Infof("SERVE Alloc check %s", r.URL) + vars := mux.Vars(r) + + id, err := storiface.ParseSectorID(vars["id"]) + if err != nil { + log.Errorf("%+v", err) + w.WriteHeader(500) + return + } + + ft, err := ftFromString(vars["type"]) + if err != nil { + log.Errorf("%+v", err) + w.WriteHeader(500) + return + } + if ft != storiface.FTUnsealed { + log.Errorf("/allocated only supports unsealed sector files") + w.WriteHeader(500) + return + } + + spti, err := strconv.ParseInt(vars["spt"], 10, 64) + if err != nil { + log.Errorf("parsing spt: %+v", err) + w.WriteHeader(500) + return + } + spt := abi.RegisteredSealProof(spti) + ssize, err := spt.SectorSize() + if err != nil { + log.Errorf("%+v", err) + w.WriteHeader(500) + return + } + + offi, err := strconv.ParseInt(vars["offset"], 10, 64) + if err != nil { + log.Errorf("parsing offset: %+v", err) + w.WriteHeader(500) + return + } + szi, err := strconv.ParseInt(vars["size"], 10, 64) + if err != nil { + log.Errorf("parsing spt: %+v", err) + w.WriteHeader(500) + return + } + + // The caller has a lock on this sector already, no need to get one here + + // passing 0 spt because we don't allocate anything + si := storage.SectorRef{ + ID: id, + ProofType: 0, + } + + paths, _, err := handler.Local.AcquireSector(r.Context(), si, ft, storiface.FTNone, storiface.PathStorage, storiface.AcquireMove) + if err != nil { + log.Errorf("%+v", err) + w.WriteHeader(500) + return + } + + path := storiface.PathByType(paths, ft) + if path == "" { + log.Error("acquired path was empty") + w.WriteHeader(500) + return + } + + pf, err := ffiwrapper.OpenPartialFile(abi.PaddedPieceSize(ssize), path) + if err != nil { + log.Error("opening partial file: ", err) + w.WriteHeader(500) + return + } + defer func() { + if err := pf.Close(); err != nil { + log.Error("close partial file: ", err) + } + }() + + has, err := pf.HasAllocated(storiface.UnpaddedByteIndex(offi), abi.UnpaddedPieceSize(szi)) + if err != nil { + log.Error("has allocated: ", err) + w.WriteHeader(500) + return + } + + if has { + w.WriteHeader(http.StatusOK) + return + } + w.WriteHeader(http.StatusRequestedRangeNotSatisfiable) +} + func (handler *FetchHandler) remoteGetSector(w http.ResponseWriter, r *http.Request) { log.Infof("SERVE GET %s", r.URL) vars := mux.Vars(r) From 19bd5beb965eca7b51d39667995e50166cd79f3e Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Thu, 20 May 2021 13:15:54 +0200 Subject: [PATCH 15/90] update remoteGetSector --- extern/sector-storage/stores/http_handler.go | 44 ++++++++++---------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/extern/sector-storage/stores/http_handler.go b/extern/sector-storage/stores/http_handler.go index 2978c0f2f..6d01fd2f8 100644 --- a/extern/sector-storage/stores/http_handler.go +++ b/extern/sector-storage/stores/http_handler.go @@ -205,31 +205,29 @@ func (handler *FetchHandler) remoteGetSector(w http.ResponseWriter, r *http.Requ return } - var rd io.Reader if stat.IsDir() { - rd, err = tarutil.TarDirectory(path) - w.Header().Set("Content-Type", "application/x-tar") - } else { - rd, err = os.OpenFile(path, os.O_RDONLY, 0644) // nolint - w.Header().Set("Content-Type", "application/octet-stream") - } - if err != nil { - log.Errorf("%+v", err) - w.WriteHeader(500) - return - } - if !stat.IsDir() { - defer func() { - if err := rd.(*os.File).Close(); err != nil { - log.Errorf("closing source file: %+v", err) - } - }() - } + if _, has := r.Header["Range"]; has { + log.Error("Range not supported on directories") + w.WriteHeader(500) + return + } - w.WriteHeader(200) - if _, err := io.CopyBuffer(w, rd, make([]byte, CopyBuf)); err != nil { - log.Errorf("%+v", err) - return + rd, err := tarutil.TarDirectory(path) + if err != nil { + log.Errorf("%+v", err) + w.WriteHeader(500) + return + } + + w.Header().Set("Content-Type", "application/x-tar") + w.WriteHeader(200) + if _, err := io.CopyBuffer(w, rd, make([]byte, CopyBuf)); err != nil { + log.Errorf("%+v", err) + return + } + } else { + w.Header().Set("Content-Type", "application/octet-stream") + http.ServeFile(w, r, path) } } From 9a7b0b657eaca7c8c4fa1a7a4552e6c11b0ac12d Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Thu, 20 May 2021 13:18:56 +0200 Subject: [PATCH 16/90] comment --- extern/sector-storage/stores/remote.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/extern/sector-storage/stores/remote.go b/extern/sector-storage/stores/remote.go index 280b3760b..bb4741ab6 100644 --- a/extern/sector-storage/stores/remote.go +++ b/extern/sector-storage/stores/remote.go @@ -406,6 +406,9 @@ func (r *Remote) Reader(ctx context.Context, s storage.SectorRef, offset, size a } } } else { + // + // magik(start): This part technically should live on the Local store + // log.Infof("Read local %s (+%d,%d)", path, offset, size) ssize, err := s.ProofType.SectorSize() if err != nil { @@ -431,6 +434,9 @@ func (r *Remote) Reader(ctx context.Context, s storage.SectorRef, offset, size a } return pf.Reader(storiface.PaddedByteIndex(offset), size) + // + // magik(end) + // } // note: rd can be nil From 8d3d3c88d48a882e2cfd440847fd3025583df512 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Thu, 20 May 2021 13:25:54 +0200 Subject: [PATCH 17/90] introduce init service, refactor restore service --- cmd/lotus-storage-miner/init.go | 1 + cmd/lotus-storage-miner/init_restore.go | 448 ++++++++++++------------ cmd/lotus-storage-miner/init_service.go | 137 ++++++++ 3 files changed, 368 insertions(+), 218 deletions(-) create mode 100644 cmd/lotus-storage-miner/init_service.go diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index 71098e725..f67160d65 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -121,6 +121,7 @@ var initCmd = &cli.Command{ }, Subcommands: []*cli.Command{ restoreCmd, + serviceCmd, }, Action: func(cctx *cli.Context) error { log.Info("Initializing lotus miner") diff --git a/cmd/lotus-storage-miner/init_restore.go b/cmd/lotus-storage-miner/init_restore.go index c609fd2a4..d31c3788d 100644 --- a/cmd/lotus-storage-miner/init_restore.go +++ b/cmd/lotus-storage-miner/init_restore.go @@ -22,6 +22,7 @@ import ( lapi "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/extern/sector-storage/stores" @@ -30,6 +31,224 @@ import ( "github.com/filecoin-project/lotus/node/repo" ) +// ctx context.Context, api lapi.FullNode, addr address.Address, peerid peer.ID +func restore(ctx context.Context, cctx *cli.Context, manageConfig func(*config.StorageMiner) error, after func(api lapi.FullNode, addr address.Address, peerid peer.ID, mi miner.MinerInfo) error) error { + if cctx.Args().Len() != 1 { + return xerrors.Errorf("expected 1 argument") + } + + log.Info("Trying to connect to full node RPC") + + api, closer, err := lcli.GetFullNodeAPIV1(cctx) // TODO: consider storing full node address in config + if err != nil { + return err + } + defer closer() + + log.Info("Checking full node version") + + v, err := api.Version(ctx) + if err != nil { + return err + } + + //TODO(anteva): bump api version? + if !v.APIVersion.EqMajorMinor(lapi.FullAPIVersion0) { + return xerrors.Errorf("Remote API version didn't match (expected %s, remote %s)", lapi.FullAPIVersion0, v.APIVersion) + } + + if !cctx.Bool("nosync") { + if err := lcli.SyncWait(ctx, &v0api.WrapperV1Full{FullNode: api}, false); err != nil { + return xerrors.Errorf("sync wait: %w", err) + } + } + + bf, err := homedir.Expand(cctx.Args().First()) + if err != nil { + return xerrors.Errorf("expand backup file path: %w", err) + } + + st, err := os.Stat(bf) + if err != nil { + return xerrors.Errorf("stat backup file (%s): %w", bf, err) + } + + f, err := os.Open(bf) + if err != nil { + return xerrors.Errorf("opening backup file: %w", err) + } + defer f.Close() // nolint:errcheck + + log.Info("Checking if repo exists") + + repoPath := cctx.String(FlagMinerRepo) + r, err := repo.NewFS(repoPath) + if err != nil { + return err + } + + ok, err := r.Exists() + if err != nil { + return err + } + if ok { + return xerrors.Errorf("repo at '%s' is already initialized", cctx.String(FlagMinerRepo)) + } + + log.Info("Initializing repo") + + if err := r.Init(repo.StorageMiner); err != nil { + return err + } + + lr, err := r.Lock(repo.StorageMiner) + if err != nil { + return err + } + defer lr.Close() //nolint:errcheck + + if cctx.IsSet("config") { + log.Info("Restoring config") + + cf, err := homedir.Expand(cctx.String("config")) + if err != nil { + return xerrors.Errorf("expanding config path: %w", err) + } + + _, err = os.Stat(cf) + if err != nil { + return xerrors.Errorf("stat config file (%s): %w", cf, err) + } + + var cerr error + err = lr.SetConfig(func(raw interface{}) { + rcfg, ok := raw.(*config.StorageMiner) + if !ok { + cerr = xerrors.New("expected miner config") + return + } + + ff, err := config.FromFile(cf, rcfg) + if err != nil { + cerr = xerrors.Errorf("loading config: %w", err) + return + } + + *rcfg = *ff.(*config.StorageMiner) + if manageConfig != nil { + cerr = manageConfig(rcfg) + } + }) + if cerr != nil { + return cerr + } + if err != nil { + return xerrors.Errorf("setting config: %w", err) + } + + } else { + log.Warn("--config NOT SET, WILL USE DEFAULT VALUES") + } + + if cctx.IsSet("storage-config") { + log.Info("Restoring storage path config") + + cf, err := homedir.Expand(cctx.String("storage-config")) + if err != nil { + return xerrors.Errorf("expanding storage config path: %w", err) + } + + cfb, err := ioutil.ReadFile(cf) + if err != nil { + return xerrors.Errorf("reading storage config: %w", err) + } + + var cerr error + err = lr.SetStorage(func(scfg *stores.StorageConfig) { + cerr = json.Unmarshal(cfb, scfg) + }) + if cerr != nil { + return xerrors.Errorf("unmarshalling storage config: %w", cerr) + } + if err != nil { + return xerrors.Errorf("setting storage config: %w", err) + } + } else { + log.Warn("--storage-config NOT SET. NO SECTOR PATHS WILL BE CONFIGURED") + } + + log.Info("Restoring metadata backup") + + mds, err := lr.Datastore(context.TODO(), "/metadata") + if err != nil { + return err + } + + bar := pb.New64(st.Size()) + br := bar.NewProxyReader(f) + bar.ShowTimeLeft = true + bar.ShowPercent = true + bar.ShowSpeed = true + bar.Units = pb.U_BYTES + + bar.Start() + err = backupds.RestoreInto(br, mds) + bar.Finish() + + if err != nil { + return xerrors.Errorf("restoring metadata: %w", err) + } + + log.Info("Checking actor metadata") + + abytes, err := mds.Get(datastore.NewKey("miner-address")) + if err != nil { + return xerrors.Errorf("getting actor address from metadata datastore: %w", err) + } + + maddr, err := address.NewFromBytes(abytes) + if err != nil { + return xerrors.Errorf("parsing actor address: %w", err) + } + + log.Info("ACTOR ADDRESS: ", maddr.String()) + + mi, err := api.StateMinerInfo(ctx, maddr, types.EmptyTSK) + if err != nil { + return xerrors.Errorf("getting miner info: %w", err) + } + + log.Info("SECTOR SIZE: ", units.BytesSize(float64(mi.SectorSize))) + + wk, err := api.StateAccountKey(ctx, mi.Worker, types.EmptyTSK) + if err != nil { + return xerrors.Errorf("resolving worker key: %w", err) + } + + has, err := api.WalletHas(ctx, wk) + if err != nil { + return xerrors.Errorf("checking worker address: %w", err) + } + + if !has { + return xerrors.Errorf("worker address %s for miner actor %s not present in full node wallet", mi.Worker, maddr) + } + + log.Info("Initializing libp2p identity") + + p2pSk, err := makeHostKey(lr) + if err != nil { + return xerrors.Errorf("make host key: %w", err) + } + + peerid, err := peer.IDFromPrivateKey(p2pSk) + if err != nil { + return xerrors.Errorf("peer ID from private key: %w", err) + } + + return after(api, maddr, peerid, mi) +} + var restoreCmd = &cli.Command{ Name: "restore", Usage: "Initialize a lotus miner repo from a backup", @@ -49,231 +268,24 @@ var restoreCmd = &cli.Command{ }, ArgsUsage: "[backupFile]", Action: func(cctx *cli.Context) error { - log.Info("Initializing lotus miner using a backup") - if cctx.Args().Len() != 1 { - return xerrors.Errorf("expected 1 argument") - } - ctx := lcli.ReqContext(cctx) + log.Info("Initializing lotus miner using a backup") - log.Info("Trying to connect to full node RPC") + if err := restore(ctx, cctx, nil, func(api lapi.FullNode, maddr address.Address, peerid peer.ID, mi miner.MinerInfo) error { + log.Info("Checking proof parameters") - if err := checkV1ApiSupport(ctx, cctx); err != nil { - return err - } - - api, closer, err := lcli.GetFullNodeAPIV1(cctx) // TODO: consider storing full node address in config - if err != nil { - return err - } - defer closer() - - log.Info("Checking full node version") - - v, err := api.Version(ctx) - if err != nil { - return err - } - - if !v.APIVersion.EqMajorMinor(lapi.FullAPIVersion1) { - return xerrors.Errorf("Remote API version didn't match (expected %s, remote %s)", lapi.FullAPIVersion1, v.APIVersion) - } - - if !cctx.Bool("nosync") { - if err := lcli.SyncWait(ctx, &v0api.WrapperV1Full{FullNode: api}, false); err != nil { - return xerrors.Errorf("sync wait: %w", err) - } - } - - bf, err := homedir.Expand(cctx.Args().First()) - if err != nil { - return xerrors.Errorf("expand backup file path: %w", err) - } - - st, err := os.Stat(bf) - if err != nil { - return xerrors.Errorf("stat backup file (%s): %w", bf, err) - } - - f, err := os.Open(bf) - if err != nil { - return xerrors.Errorf("opening backup file: %w", err) - } - defer f.Close() // nolint:errcheck - - log.Info("Checking if repo exists") - - repoPath := cctx.String(FlagMinerRepo) - r, err := repo.NewFS(repoPath) - if err != nil { - return err - } - - ok, err := r.Exists() - if err != nil { - return err - } - if ok { - return xerrors.Errorf("repo at '%s' is already initialized", cctx.String(FlagMinerRepo)) - } - - log.Info("Initializing repo") - - if err := r.Init(repo.StorageMiner); err != nil { - return err - } - - lr, err := r.Lock(repo.StorageMiner) - if err != nil { - return err - } - defer lr.Close() //nolint:errcheck - - if cctx.IsSet("config") { - log.Info("Restoring config") - - cf, err := homedir.Expand(cctx.String("config")) - if err != nil { - return xerrors.Errorf("expanding config path: %w", err) + if err := paramfetch.GetParams(ctx, build.ParametersJSON(), uint64(mi.SectorSize)); err != nil { + return xerrors.Errorf("fetching proof parameters: %w", err) } - _, err = os.Stat(cf) - if err != nil { - return xerrors.Errorf("stat config file (%s): %w", cf, err) + log.Info("Configuring miner actor") + + if err := configureStorageMiner(ctx, api, maddr, peerid, big.Zero()); err != nil { + return err } - var cerr error - err = lr.SetConfig(func(raw interface{}) { - rcfg, ok := raw.(*config.StorageMiner) - if !ok { - cerr = xerrors.New("expected miner config") - return - } - - ff, err := config.FromFile(cf, rcfg) - if err != nil { - cerr = xerrors.Errorf("loading config: %w", err) - return - } - - *rcfg = *ff.(*config.StorageMiner) - }) - if cerr != nil { - return cerr - } - if err != nil { - return xerrors.Errorf("setting config: %w", err) - } - - } else { - log.Warn("--config NOT SET, WILL USE DEFAULT VALUES") - } - - if cctx.IsSet("storage-config") { - log.Info("Restoring storage path config") - - cf, err := homedir.Expand(cctx.String("storage-config")) - if err != nil { - return xerrors.Errorf("expanding storage config path: %w", err) - } - - cfb, err := ioutil.ReadFile(cf) - if err != nil { - return xerrors.Errorf("reading storage config: %w", err) - } - - var cerr error - err = lr.SetStorage(func(scfg *stores.StorageConfig) { - cerr = json.Unmarshal(cfb, scfg) - }) - if cerr != nil { - return xerrors.Errorf("unmarshalling storage config: %w", cerr) - } - if err != nil { - return xerrors.Errorf("setting storage config: %w", err) - } - } else { - log.Warn("--storage-config NOT SET. NO SECTOR PATHS WILL BE CONFIGURED") - } - - log.Info("Restoring metadata backup") - - mds, err := lr.Datastore(context.TODO(), "/metadata") - if err != nil { - return err - } - - bar := pb.New64(st.Size()) - br := bar.NewProxyReader(f) - bar.ShowTimeLeft = true - bar.ShowPercent = true - bar.ShowSpeed = true - bar.Units = pb.U_BYTES - - bar.Start() - err = backupds.RestoreInto(br, mds) - bar.Finish() - - if err != nil { - return xerrors.Errorf("restoring metadata: %w", err) - } - - log.Info("Checking actor metadata") - - abytes, err := mds.Get(datastore.NewKey("miner-address")) - if err != nil { - return xerrors.Errorf("getting actor address from metadata datastore: %w", err) - } - - maddr, err := address.NewFromBytes(abytes) - if err != nil { - return xerrors.Errorf("parsing actor address: %w", err) - } - - log.Info("ACTOR ADDRESS: ", maddr.String()) - - mi, err := api.StateMinerInfo(ctx, maddr, types.EmptyTSK) - if err != nil { - return xerrors.Errorf("getting miner info: %w", err) - } - - log.Info("SECTOR SIZE: ", units.BytesSize(float64(mi.SectorSize))) - - wk, err := api.StateAccountKey(ctx, mi.Worker, types.EmptyTSK) - if err != nil { - return xerrors.Errorf("resolving worker key: %w", err) - } - - has, err := api.WalletHas(ctx, wk) - if err != nil { - return xerrors.Errorf("checking worker address: %w", err) - } - - if !has { - return xerrors.Errorf("worker address %s for miner actor %s not present in full node wallet", mi.Worker, maddr) - } - - log.Info("Checking proof parameters") - - if err := paramfetch.GetParams(ctx, build.ParametersJSON(), uint64(mi.SectorSize)); err != nil { - return xerrors.Errorf("fetching proof parameters: %w", err) - } - - log.Info("Initializing libp2p identity") - - p2pSk, err := makeHostKey(lr) - if err != nil { - return xerrors.Errorf("make host key: %w", err) - } - - peerid, err := peer.IDFromPrivateKey(p2pSk) - if err != nil { - return xerrors.Errorf("peer ID from private key: %w", err) - } - - log.Info("Configuring miner actor") - - if err := configureStorageMiner(ctx, api, maddr, peerid, big.Zero()); err != nil { + return nil + }); err != nil { return err } diff --git a/cmd/lotus-storage-miner/init_service.go b/cmd/lotus-storage-miner/init_service.go new file mode 100644 index 000000000..99bd7ef84 --- /dev/null +++ b/cmd/lotus-storage-miner/init_service.go @@ -0,0 +1,137 @@ +package main + +import ( + "context" + "strings" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/big" + lapi "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/api/client" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + lcli "github.com/filecoin-project/lotus/cli" + cliutil "github.com/filecoin-project/lotus/cli/util" + "github.com/filecoin-project/lotus/node/config" + "github.com/libp2p/go-libp2p-core/peer" + "github.com/urfave/cli/v2" + "golang.org/x/xerrors" +) + +func checkApiInfo(ctx context.Context, ai string) (string, error) { + ai = strings.TrimPrefix(strings.TrimSpace(ai), "MINER_API_INFO=") + info := cliutil.ParseApiInfo(ai) + addr, err := info.DialArgs("v0") + if err != nil { + return "", xerrors.Errorf("could not get DialArgs: %w", err) + } + + log.Infof("Checking api version of %s", addr) + + api, closer, err := client.NewStorageMinerRPCV0(ctx, addr, info.AuthHeader()) + if err != nil { + return "", err + } + defer closer() + + v, err := api.Version(ctx) + if err != nil { + return "", xerrors.Errorf("checking version: %w", err) + } + + //TODO(anteva): bump api version? + if !v.APIVersion.EqMajorMinor(lapi.MinerAPIVersion0) { + return "", xerrors.Errorf("remote service API version didn't match (expected %s, remote %s)", lapi.MinerAPIVersion0, v.APIVersion) + } + + return ai, nil +} + +var serviceCmd = &cli.Command{ + Name: "service", + Usage: "Initialize a lotus miner sub-service", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "config", + Usage: "config file (config.toml)", + Required: true, + }, + &cli.StringFlag{ + Name: "storage-config", + Usage: "storage paths config (storage.json)", + Required: true, + }, + &cli.BoolFlag{ + Name: "nosync", + Usage: "don't check full-node sync status", + }, + + &cli.BoolFlag{ + Name: "enable-market", + Usage: "enable market module", + }, + + &cli.StringFlag{ + Name: "api-sealer", + Usage: "sealer API info (lotus-miner auth api-info --perm=admin)", + }, + &cli.StringFlag{ + Name: "api-sector-index", + Usage: "sector Index API info (lotus-miner auth api-info --perm=admin)", + }, + }, + ArgsUsage: "[backupFile]", + Action: func(cctx *cli.Context) error { + ctx := lcli.ReqContext(cctx) + log.Info("Initializing lotus miner service") + + if !cctx.Bool("enable-market") { + return xerrors.Errorf("at least one module must be enabled") + } + + if !cctx.IsSet("api-sealer") { + return xerrors.Errorf("--api-sealer is required without the sealer module enabled") + } + if !cctx.IsSet("api-sector-index") { + return xerrors.Errorf("--api-sector-index is required without the sector storage module enabled") + } + + if err := restore(ctx, cctx, func(cfg *config.StorageMiner) error { + cfg.Subsystems.EnableStorageMarket = cctx.Bool("enable-market") + cfg.Subsystems.EnableMining = false + cfg.Subsystems.EnableSealing = false + cfg.Subsystems.EnableSectorStorage = false + + if !cfg.Subsystems.EnableSealing { + ai, err := checkApiInfo(ctx, cctx.String("api-sealer")) + if err != nil { + return xerrors.Errorf("checking sealer API: %w", err) + } + cfg.Subsystems.SealerApiInfo = ai + } + + if !cfg.Subsystems.EnableSectorStorage { + ai, err := checkApiInfo(ctx, cctx.String("api-sector-index")) + if err != nil { + return xerrors.Errorf("checking sector index API: %w", err) + } + cfg.Subsystems.SectorIndexApiInfo = ai + } + + return nil + }, func(api lapi.FullNode, maddr address.Address, peerid peer.ID, mi miner.MinerInfo) error { + if cctx.Bool("enable-market") { + log.Info("Configuring miner actor") + + if err := configureStorageMiner(ctx, api, maddr, peerid, big.Zero()); err != nil { + return err + } + } + + return nil + }); err != nil { + return err + } + + return nil + }, +} From 9979de24db7693f36662be96b0e83b634ef5d324 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 26 May 2021 12:47:21 +0200 Subject: [PATCH 18/90] logs and defaults --- node/config/def.go | 7 +++++++ node/modules/storageminer_svc.go | 4 +++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/node/config/def.go b/node/config/def.go index 4013bde6e..8ca588f6e 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -277,6 +277,13 @@ func DefaultStorageMiner() *StorageMiner { MaxProviderCollateralMultiplier: 2, }, + Subsystems: MinerSubsystemConfig{ + EnableMining: true, + EnableSealing: true, + EnableSectorStorage: true, + EnableStorageMarket: true, + }, + Fees: MinerFeeConfig{ MaxPreCommitGasFee: types.MustParseFIL("0.025"), MaxCommitGasFee: types.MustParseFIL("0.05"), diff --git a/node/modules/storageminer_svc.go b/node/modules/storageminer_svc.go index 8718d236c..c0616cc14 100644 --- a/node/modules/storageminer_svc.go +++ b/node/modules/storageminer_svc.go @@ -28,7 +28,7 @@ func connectMinerService(apiInfo string) func(mctx helpers.MetricsCtx, lc fx.Lif return nil, xerrors.Errorf("could not get DialArgs: %w", err) } - log.Infof("Checking api version of %s", addr) + log.Infof("Checking (svc) api version of %s", addr) mapi, closer, err := client.NewStorageMinerRPCV0(ctx, addr, info.AuthHeader()) if err != nil { @@ -60,12 +60,14 @@ func connectMinerService(apiInfo string) func(mctx helpers.MetricsCtx, lc fx.Lif func ConnectSealingService(apiInfo string) func(mctx helpers.MetricsCtx, lc fx.Lifecycle) (MinerSealingService, error) { return func(mctx helpers.MetricsCtx, lc fx.Lifecycle) (MinerSealingService, error) { + log.Info("Connecting sealing service to miner") return connectMinerService(apiInfo)(mctx, lc) } } func ConnectStorageService(apiInfo string) func(mctx helpers.MetricsCtx, lc fx.Lifecycle) (MinerStorageService, error) { return func(mctx helpers.MetricsCtx, lc fx.Lifecycle) (MinerStorageService, error) { + log.Info("Connecting storage service to miner") return connectMinerService(apiInfo)(mctx, lc) } } From c311520968fdef9084e9a774d358039e6a69a591 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Thu, 27 May 2021 10:28:41 +0200 Subject: [PATCH 19/90] add missing messagepool override --- node/builder_chain.go | 1 + 1 file changed, 1 insertion(+) diff --git a/node/builder_chain.go b/node/builder_chain.go index 6fb5b540e..270534edb 100644 --- a/node/builder_chain.go +++ b/node/builder_chain.go @@ -136,6 +136,7 @@ var ChainNode = Options( // Full node API / service startup ApplyIf(isFullNode, + Override(new(messagepool.Provider), messagepool.NewProvider), Override(new(messagesigner.MpoolNonceAPI), From(new(*messagepool.MessagePool))), Override(new(full.ChainModuleAPI), From(new(full.ChainModule))), Override(new(full.GasModuleAPI), From(new(full.GasModule))), From 46a368d2b9bcd23534f26a8c65e22fef8916518e Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Thu, 27 May 2021 13:35:54 +0200 Subject: [PATCH 20/90] bump api ver --- cmd/lotus-storage-miner/init_restore.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/lotus-storage-miner/init_restore.go b/cmd/lotus-storage-miner/init_restore.go index d31c3788d..c114e9388 100644 --- a/cmd/lotus-storage-miner/init_restore.go +++ b/cmd/lotus-storage-miner/init_restore.go @@ -53,8 +53,8 @@ func restore(ctx context.Context, cctx *cli.Context, manageConfig func(*config.S } //TODO(anteva): bump api version? - if !v.APIVersion.EqMajorMinor(lapi.FullAPIVersion0) { - return xerrors.Errorf("Remote API version didn't match (expected %s, remote %s)", lapi.FullAPIVersion0, v.APIVersion) + if !v.APIVersion.EqMajorMinor(lapi.FullAPIVersion1) { + return xerrors.Errorf("Remote API version didn't match (expected %s, remote %s)", lapi.FullAPIVersion1, v.APIVersion) } if !cctx.Bool("nosync") { From e4136b0d42f8ee82518d2c9e8c787966e19776bb Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Tue, 1 Jun 2021 11:42:28 +0200 Subject: [PATCH 21/90] add modules.StorageAuthWithURL to set correct token --- dev.gen | Bin 0 -> 16860 bytes go.sum | 3 - localnet.json | 300 +++++++++++++++++++++++++++++++++++ node/builder_miner.go | 1 + node/modules/storageminer.go | 14 ++ 5 files changed, 315 insertions(+), 3 deletions(-) create mode 100644 dev.gen create mode 100644 localnet.json diff --git a/dev.gen b/dev.gen new file mode 100644 index 0000000000000000000000000000000000000000..44bbf6e36484be489bed7bce35472bcdf639d1fb GIT binary patch literal 16860 zcmeI4c|26#|HtnbTe4=?Vkk>V_I+vW`;w5QFc>>y%aUSdC`Fc#Bnc7Gf~1lp*|#WL zQK%?Fn}`+WZT{_&Z|xQ**|pZA>iIq%oK_slt|=3xVIxZohdzz!P` z01=S$nhLe&RrKAIu{b1EPu-qFckVtKpD_yyVZP(}k=lkcphLx} z0$(Omt_v#zF3HK3>&Y;(?L5rn9NyGI%V}>Yo3Yy=;XEmCB=r@ zN)cuf5NU>F)U}f5sFUmhHO5<-&F-GwIiP90ZIH<>y%{M{-pM-8=X7(HWggH0Xy!(N zMlfXX#V+vB#sp)7J$JtA4k}=Ne5+(k z_2l087ITPHr}R-Bs&mTz@FwT_9$q!Y*YENZk8=iQy>ZN9D1H5)_60rp{BTI3cD~8_ zZ6^stO~5W@V8>nnVFfU|dHP7mO86f12?pcm4mm3}-6Q*gVqN_Fsdq~Ddv+gV>avrt z;1c&dzTs0rHrh)LFafCU^WiP?eSs7(M@UpGNVOnQnaEU(iwmjQA<*u4gFN5FTTQ(} zw4!fx!v-3UhV;qdlz1|K5Voe8K64`_ohCpFa%?t|KwM-HnB$&qo>uu-=rh<#3T@>4sUcc3kYGI5+BwYH!StX2G93aDz3k;j{wj_uy{xiW=4WBkYO%?I6nvk z+x@KHHKo0`i~AXQ8YcuTi-&3sAO1SwQFaX-B$$H}{$rJeV8-jlbG zHr_LGZTE{{sh_vR$}>LL0D1s(jg;ECU{d@6khzCcLB{21g$D&=eIYcs9!ZW94T!Pf zA#vDOMBNt?=#7Q;im0~O^VY>)C;3FJXYz(mp`T_|s~pf}fAe)nKREZ3cgC-XnS(T# zE*oL+pyt?H+|kX49%;N+kaAb&J3CP!a$j=r_0w(>Wp80}UqFB$X!Cjcc|r?wtMgSZ zl{wluIApAxeqc&HMs;4-Z}-+WB@ybb#>iF?GQ<^x{P(V&v-cRkO|9O2pSIfo-^;J2v zf46cYe-L9N0Pmwn+A zvEpdH>0FS)mbeKpwUG(qEQd1^SlYqVMu9E3$|uO}NpTGivj4z2Ml1oe2*fJi0H7B| zo*!QAn2Q&^85L9wl@l)+5yU>Vp0mK4CufEA&|AeZZeKP!7#R_b-vX9-r3;aZylhEo zWDuMk5v&kw$Qs^q?wo6hk+>P0lXdg9)MHbUApHMvCh}QU`xPERJ|CP}MQZ|3<7aVn zHM%uJZ*PRFl*_Vz-r>afSa+90nd2Llmxh^@%1n?Zg^QdFxd3oPqBe@;gBAs= zxrSe^$YrvFE>{oEK9jTnbAa@t4IokEjUGVb-L${FR^rW?NqR4F9Q_GX3cztWeC2I?<<4&WF=g!j{Xn6$fZ#WGFlZr;UsASnw zzKAveI45_Se$)_H_kHuU(L)rzd)}TLHaI3xi|SX@G@zF2FI2OM<-TR+g|hJvn3vko z35i=X_)qelvBYY;s_B{zR423HWHGb=A{xB(kY9MR>1z@Y%Uu|*3PdgNP_RKsmx^eb z{{){0Y+K0Xi`=H93z;;zRB#tD0_j3XO#^Od*6ko40(Zdzv)|?SF7@j|wW3S?+B3d< zHxGB+N`kcf!NZmxzp`0*vD1eLqSB7b3i~{~VEN&>ctU@o?Mdl<#%MG|%*Aua@ot7o3kwQ~?Cgd2nB!fsl6AM(BxsGbRbP{fB>G@*6 zk(+~i`;^zLz2xkfh&RPv5UHa?yr%2oXh)>=OBbCl2hy!>_dPF6>26m=pRvynLP2Yo z+`3)dAZOdtl>M7kk^2`2*G-X*&F6}AHi;!7mqEftEo;Op+aR&JgL?}tkX zU);a{0O84~i$W`}L73}im-Wy22>`$ca2K+nb%1+{#{pd+u1RnN{3q!AI+;veU?yLZ z^AmthK&+%%9^na$X|FVQV100E@>mevUZL0|g+YPi9UDrLIR`Idzgf4FUN_RO*$~0^~Pa*X#F<3YwN$?8L(YnHeuRhJ^Wr3eRDi z6tl7d3Dkc?rCCXB4U?;Pr!oF6@f~^VM!f@ZjAgTdkfvO@4Z94NAw#waf##2>v@5A+ zsI|po6)%k22w%3Rib?<2<15v8$h73p0NdsflpvQCf%cE6bStSjK1ngmtg7!K*=nWQ z6|)W`(Hi<*WwlxCQ?2d_j^)4-=>CXGzmi(d@+r3G==<;HCB3uw8uv*bvnK@kCe`Vz zwAy^XS&ann@=gy zY^@5=jvlC+!;eU*0X(D94bhV@aev?DrasUXtZ#i;qH zL8MjWl&)atoYn$%p z801@GkJ#r}WOdDHFvPw6X8t#bv}nU;rhp=L+l)QUTvJG<^zz}-*{Ql-wR`@EX)dO4 z12me7in!uw)q!&x0c^6%BlbeXTY)ZbapU*7QP=A$W1h!l>Ry*#c1SG5=xwwou~?fLUKtlqEn+^JTT-3&kgD>K&jYPkYc`fp z57lUAy!P_bIE)UA0}4+#3ic~K0Ku84Qwu=5fdm->*K@%B2}ElEc>Dpf9xnf4Xs8j< z+Tf7hiYh%&kg=(omw8cn|3;^8ij~021Bi-+G9Is&zP*#-O|QS;ZOL>vU9>i^YUm;r zok|}zG1|g&T17_fzKwe?-=#_*-}Y^iI(?7SXsADh45tIhq;W_{PU+1`r#2c7jWCLS zNvZJcd3%)0DYq=|D0H90?(4FfKPHoGHbCRs7YM6qAN~41=2V*JGSUG)L`` zbgd6P=Sm;@BxacN5=5^K;{?r11kOzhKtji8-uc*u?Y^pzw8gY`7A}WTnlV=*=bml? zAKxfiOiki{#iTDRn>1Y2{xWGzbBWx(E zS$&8Ws9Hxg617v?x~!GLh0}rMML_LAz2(M`F87l$jY;QCc^cvF&zsCe3j3ELBA)Ot zBs?!G!t0Qe7)}RrsX&(|b*!~)df9g2GHPwxHPKk`EtX~SQV~MSg>oLWy?knF2Q3dB z{}vau!Bm=@=M)Qr)BSF{Ete10`@ATl3fVoa6xe^A<-k;kt$w)zM#VY!o1xJq844Lr z_q**HXOCP<>8n;$-)es>bCzRMp?RK>>h-z=F^SNZ<%^Cg6f&Idm+i7bTkNhHMs$&T z-2NqJD37ePt@c+2K?AONM$y!pEn7`W3|J_(3#TKOnPj^Z8-*L9g4*cVGrI5}^VYN1 z9EUn0=rqqhl_`_lKd0UxRE0mRcCASd;ur0cF-^wj3HHeKSueQ;nQJ(Rr#pAi59>QO;ZCm#r8NE3$%N z#w@%hio%7{f#qG1lruM-arPNcW!juHK7GIEEVRd|F!F#o<5kSaHx;_%qI`;t!s((l zQQ)1>QBAO8{TCaB%c!+&RAQ&b~CkQ;bYjK^osjqcI)`Y8h#2HPWQWwq8{xxGed8V zF}nhsG-@w5iHSBSQSL5YANt2#$>Qt3!X4yG~Av|H5i>a&9m z#N$qUVP5@pBJ^R>5WkQS%S~)jcFSQaHEszCh~;qTT(4~UeV_UfBa@3)3u@E;*K8E7 zY7iUkT5Y4{0tNSHl4dwGa`_|tQ(t1Mc?Uy&=$CUQnWnei6y*+pgykRGDBL;LveEU4 z%Em41DNDt-{eH}RJhF33cC^-k8&kvSkroo(Q;qc$`+(D}wxy2N)9dTXqLK{!l~a&s~wrR{L`!Fy4q zbYa^!QJ5#&GD7iUi!cgaBAa)NGF#JWYKN{wQpa5~ah56MQsj`d$` z6fUFIw$X^I*PBJ#d@bdJDzW0CbxYSA9aXNgX}g|8qY7xluXs@`2~PLBjUJA>yO+jx zA+_VwZowYQ$=optxcMR1hvs-@sz$pGcPPeBkZHfuNd&F?n}>k!v^>l zFX&Up$ufCTY!psM9&CZyDBM?@P#evsHlW_xIM~zM-VpE^s9Z{Z9@kR4Ma^T%;vp*{ zCFOK=Pd#?LhrH+5g{oE(MmkaUcjNSm3MrU$zb0aRo_kZqg6s~Fz8)fBO^-xcD zcgu`xNAo)i8aAiwDCA|2#r2i9CZhuomK?UIsPfa0u>4~ig*(SuHp)92?w4FHdu(}M zVSc^*&l_3u9G59vINfR+)joRet(|(#qyX>n24vSI=erGj1e)G!4+y84n2{!&XbKlj zx7tQM!X^jb{O~`yfA+D@ketZH{tsVoJ#&jqE{roD3e>Ou&qF<&E?Sd*MN%eJI(+2L z^&Fo(S)egf74OxfPIcCb*Q}edg4r>fp>qqxM&Wc|c}YXBnqbHJFE$F7QES`i{jO`z z763LktcLK1U@Qs?|Mwj#4UV|{iTK7VJn))D^QOq-JDRbFz< zbz56e$Z)#fN0tNSRBD8`I6LS|6?6trY!prhmKmI_ z4Yg6YX&+DO!P{qa1M5<;kpY#RX^N0l#7genqeeoEW=e*~9YO)L%;=M99NoaLXU?J3z=< zYaUk4bN#gp;@a|8jc{uq;vo@=77MORzm9$_U1hiKv8f@4Yv)Cbjks{5>=|SE9-enL z5b0Z9#e2EB5s^T;d`XCDw`mi%lC=3*m4y7hp5nW>SYe2CZt9}=WMT`wTRP35r00q# z&dA1G&)p?UYBVngKF0M$LZp9VBJ8u7?{%jq3_7R%X9zhq;q5+D@4G{qO!2O1`}aAr z-o3#9!5-VlNoVfyL}fa8qu!8fLVuVpYe(6=>461~hRb`iu6d%sD_j>1xeefA0G(w? zik&EU5Bn0CfnGTqL0gskITJg<>3X zHSYW3=Ib~aH2g|G_`A7RAylx3n)&TyiXU=75z=cTxOYchtpNOR+j<`KhM&8@A8|@d zgWj??t!rJ@@q2a>@=%1#+K8uD1UUyHa_rh_cO_^f2Xx=2xh_j>5^miv!1%WMp(HgF zA^ZOzxC{C3JfAGl5Wg{{EiuTi@}l;qLmP2-z>!!++YjFrp$NIP5mvfJ!i`&{^-0r)NiTd{ zG=M_=$zPVq^etNLI1=8Pp2hO-6G zx|8o4v*!PjI&1sy2?%JsX+M&|&s4gXT_QG2Ilql((Jmm>BYWr2Q;0OvD6e(AHqo|z z)~UqauKG)88v8xgttlSybuVTbO2V@t(z+9SD*I13S>%bPYP?KdyP8x@u~ip&N|FuPBTE7M7sk+^5YAFPsjCw4%yErn~X!GX;(@vjSM94vRCryq@>j52=XJI2!+eG zh+FSZhe!p_4rtMNKYUGHwr-cM+xKrX{ex$x*MA9l_^|ZF+kM6RAW|p3WbJTUydYh* zQiG~ubbnj2Ov-)dVx8{dbauM4z3~9+5}A2u&UP>;b>U-a;Juq5w-d7(t;;II1;S3#yZjlLE6xgHUMcu zPukFtHdOOOq6YOmQG<{v?utF0>Lf4nE%>K37sc^lm%9_(!bieCDA6(+Zv7((ooiJahiu)}5W^Ev zuOh^FB7V9o1+;t Date: Tue, 1 Jun 2021 11:45:34 +0200 Subject: [PATCH 22/90] fix tests --- extern/storage-sealing/precommit_policy_test.go | 17 +++++++++-------- extern/storage-sealing/types_test.go | 5 +++-- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/extern/storage-sealing/precommit_policy_test.go b/extern/storage-sealing/precommit_policy_test.go index 52814167a..a6c17d3fd 100644 --- a/extern/storage-sealing/precommit_policy_test.go +++ b/extern/storage-sealing/precommit_policy_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/filecoin-project/go-state-types/network" + api "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/ipfs/go-cid" @@ -58,9 +59,9 @@ func TestBasicPolicyMostConstrictiveSchedule(t *testing.T) { Size: abi.PaddedPieceSize(1024), PieceCID: fakePieceCid(t), }, - DealInfo: &sealing.DealInfo{ + DealInfo: &api.PieceDealInfo{ DealID: abi.DealID(42), - DealSchedule: sealing.DealSchedule{ + DealSchedule: api.DealSchedule{ StartEpoch: abi.ChainEpoch(70), EndEpoch: abi.ChainEpoch(75), }, @@ -71,9 +72,9 @@ func TestBasicPolicyMostConstrictiveSchedule(t *testing.T) { Size: abi.PaddedPieceSize(1024), PieceCID: fakePieceCid(t), }, - DealInfo: &sealing.DealInfo{ + DealInfo: &api.PieceDealInfo{ DealID: abi.DealID(43), - DealSchedule: sealing.DealSchedule{ + DealSchedule: api.DealSchedule{ StartEpoch: abi.ChainEpoch(80), EndEpoch: abi.ChainEpoch(100), }, @@ -98,9 +99,9 @@ func TestBasicPolicyIgnoresExistingScheduleIfExpired(t *testing.T) { Size: abi.PaddedPieceSize(1024), PieceCID: fakePieceCid(t), }, - DealInfo: &sealing.DealInfo{ + DealInfo: &api.PieceDealInfo{ DealID: abi.DealID(44), - DealSchedule: sealing.DealSchedule{ + DealSchedule: api.DealSchedule{ StartEpoch: abi.ChainEpoch(1), EndEpoch: abi.ChainEpoch(10), }, @@ -125,9 +126,9 @@ func TestMissingDealIsIgnored(t *testing.T) { Size: abi.PaddedPieceSize(1024), PieceCID: fakePieceCid(t), }, - DealInfo: &sealing.DealInfo{ + DealInfo: &api.PieceDealInfo{ DealID: abi.DealID(44), - DealSchedule: sealing.DealSchedule{ + DealSchedule: api.DealSchedule{ StartEpoch: abi.ChainEpoch(1), EndEpoch: abi.ChainEpoch(10), }, diff --git a/extern/storage-sealing/types_test.go b/extern/storage-sealing/types_test.go index aa314c37a..68e2b1111 100644 --- a/extern/storage-sealing/types_test.go +++ b/extern/storage-sealing/types_test.go @@ -10,6 +10,7 @@ import ( cborutil "github.com/filecoin-project/go-cbor-util" "github.com/filecoin-project/go-state-types/abi" + api "github.com/filecoin-project/lotus/api" market2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/market" tutils "github.com/filecoin-project/specs-actors/v2/support/testing" ) @@ -22,9 +23,9 @@ func TestSectorInfoSerialization(t *testing.T) { t.Fatal(err) } - dealInfo := DealInfo{ + dealInfo := api.PieceDealInfo{ DealID: d, - DealSchedule: DealSchedule{ + DealSchedule: api.DealSchedule{ StartEpoch: 0, EndEpoch: 100, }, From 88756f3ebf2689ddfdc899d141a15030192614e7 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Fri, 4 Jun 2021 15:41:38 +0200 Subject: [PATCH 23/90] fix TestAPIDeal tests --- dev.gen | Bin 16860 -> 0 bytes extern/sector-storage/mock/mock.go | 12 +- extern/sector-storage/piece_provider.go | 14 +- extern/sector-storage/stores/remote.go | 2 +- localnet.json | 300 ------------------------ markets/retrievaladapter/provider.go | 4 +- node/builder_miner.go | 2 +- node/test/builder.go | 11 +- 8 files changed, 31 insertions(+), 314 deletions(-) delete mode 100644 dev.gen delete mode 100644 localnet.json diff --git a/dev.gen b/dev.gen deleted file mode 100644 index 44bbf6e36484be489bed7bce35472bcdf639d1fb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16860 zcmeI4c|26#|HtnbTe4=?Vkk>V_I+vW`;w5QFc>>y%aUSdC`Fc#Bnc7Gf~1lp*|#WL zQK%?Fn}`+WZT{_&Z|xQ**|pZA>iIq%oK_slt|=3xVIxZohdzz!P` z01=S$nhLe&RrKAIu{b1EPu-qFckVtKpD_yyVZP(}k=lkcphLx} z0$(Omt_v#zF3HK3>&Y;(?L5rn9NyGI%V}>Yo3Yy=;XEmCB=r@ zN)cuf5NU>F)U}f5sFUmhHO5<-&F-GwIiP90ZIH<>y%{M{-pM-8=X7(HWggH0Xy!(N zMlfXX#V+vB#sp)7J$JtA4k}=Ne5+(k z_2l087ITPHr}R-Bs&mTz@FwT_9$q!Y*YENZk8=iQy>ZN9D1H5)_60rp{BTI3cD~8_ zZ6^stO~5W@V8>nnVFfU|dHP7mO86f12?pcm4mm3}-6Q*gVqN_Fsdq~Ddv+gV>avrt z;1c&dzTs0rHrh)LFafCU^WiP?eSs7(M@UpGNVOnQnaEU(iwmjQA<*u4gFN5FTTQ(} zw4!fx!v-3UhV;qdlz1|K5Voe8K64`_ohCpFa%?t|KwM-HnB$&qo>uu-=rh<#3T@>4sUcc3kYGI5+BwYH!StX2G93aDz3k;j{wj_uy{xiW=4WBkYO%?I6nvk z+x@KHHKo0`i~AXQ8YcuTi-&3sAO1SwQFaX-B$$H}{$rJeV8-jlbG zHr_LGZTE{{sh_vR$}>LL0D1s(jg;ECU{d@6khzCcLB{21g$D&=eIYcs9!ZW94T!Pf zA#vDOMBNt?=#7Q;im0~O^VY>)C;3FJXYz(mp`T_|s~pf}fAe)nKREZ3cgC-XnS(T# zE*oL+pyt?H+|kX49%;N+kaAb&J3CP!a$j=r_0w(>Wp80}UqFB$X!Cjcc|r?wtMgSZ zl{wluIApAxeqc&HMs;4-Z}-+WB@ybb#>iF?GQ<^x{P(V&v-cRkO|9O2pSIfo-^;J2v zf46cYe-L9N0Pmwn+A zvEpdH>0FS)mbeKpwUG(qEQd1^SlYqVMu9E3$|uO}NpTGivj4z2Ml1oe2*fJi0H7B| zo*!QAn2Q&^85L9wl@l)+5yU>Vp0mK4CufEA&|AeZZeKP!7#R_b-vX9-r3;aZylhEo zWDuMk5v&kw$Qs^q?wo6hk+>P0lXdg9)MHbUApHMvCh}QU`xPERJ|CP}MQZ|3<7aVn zHM%uJZ*PRFl*_Vz-r>afSa+90nd2Llmxh^@%1n?Zg^QdFxd3oPqBe@;gBAs= zxrSe^$YrvFE>{oEK9jTnbAa@t4IokEjUGVb-L${FR^rW?NqR4F9Q_GX3cztWeC2I?<<4&WF=g!j{Xn6$fZ#WGFlZr;UsASnw zzKAveI45_Se$)_H_kHuU(L)rzd)}TLHaI3xi|SX@G@zF2FI2OM<-TR+g|hJvn3vko z35i=X_)qelvBYY;s_B{zR423HWHGb=A{xB(kY9MR>1z@Y%Uu|*3PdgNP_RKsmx^eb z{{){0Y+K0Xi`=H93z;;zRB#tD0_j3XO#^Od*6ko40(Zdzv)|?SF7@j|wW3S?+B3d< zHxGB+N`kcf!NZmxzp`0*vD1eLqSB7b3i~{~VEN&>ctU@o?Mdl<#%MG|%*Aua@ot7o3kwQ~?Cgd2nB!fsl6AM(BxsGbRbP{fB>G@*6 zk(+~i`;^zLz2xkfh&RPv5UHa?yr%2oXh)>=OBbCl2hy!>_dPF6>26m=pRvynLP2Yo z+`3)dAZOdtl>M7kk^2`2*G-X*&F6}AHi;!7mqEftEo;Op+aR&JgL?}tkX zU);a{0O84~i$W`}L73}im-Wy22>`$ca2K+nb%1+{#{pd+u1RnN{3q!AI+;veU?yLZ z^AmthK&+%%9^na$X|FVQV100E@>mevUZL0|g+YPi9UDrLIR`Idzgf4FUN_RO*$~0^~Pa*X#F<3YwN$?8L(YnHeuRhJ^Wr3eRDi z6tl7d3Dkc?rCCXB4U?;Pr!oF6@f~^VM!f@ZjAgTdkfvO@4Z94NAw#waf##2>v@5A+ zsI|po6)%k22w%3Rib?<2<15v8$h73p0NdsflpvQCf%cE6bStSjK1ngmtg7!K*=nWQ z6|)W`(Hi<*WwlxCQ?2d_j^)4-=>CXGzmi(d@+r3G==<;HCB3uw8uv*bvnK@kCe`Vz zwAy^XS&ann@=gy zY^@5=jvlC+!;eU*0X(D94bhV@aev?DrasUXtZ#i;qH zL8MjWl&)atoYn$%p z801@GkJ#r}WOdDHFvPw6X8t#bv}nU;rhp=L+l)QUTvJG<^zz}-*{Ql-wR`@EX)dO4 z12me7in!uw)q!&x0c^6%BlbeXTY)ZbapU*7QP=A$W1h!l>Ry*#c1SG5=xwwou~?fLUKtlqEn+^JTT-3&kgD>K&jYPkYc`fp z57lUAy!P_bIE)UA0}4+#3ic~K0Ku84Qwu=5fdm->*K@%B2}ElEc>Dpf9xnf4Xs8j< z+Tf7hiYh%&kg=(omw8cn|3;^8ij~021Bi-+G9Is&zP*#-O|QS;ZOL>vU9>i^YUm;r zok|}zG1|g&T17_fzKwe?-=#_*-}Y^iI(?7SXsADh45tIhq;W_{PU+1`r#2c7jWCLS zNvZJcd3%)0DYq=|D0H90?(4FfKPHoGHbCRs7YM6qAN~41=2V*JGSUG)L`` zbgd6P=Sm;@BxacN5=5^K;{?r11kOzhKtji8-uc*u?Y^pzw8gY`7A}WTnlV=*=bml? zAKxfiOiki{#iTDRn>1Y2{xWGzbBWx(E zS$&8Ws9Hxg617v?x~!GLh0}rMML_LAz2(M`F87l$jY;QCc^cvF&zsCe3j3ELBA)Ot zBs?!G!t0Qe7)}RrsX&(|b*!~)df9g2GHPwxHPKk`EtX~SQV~MSg>oLWy?knF2Q3dB z{}vau!Bm=@=M)Qr)BSF{Ete10`@ATl3fVoa6xe^A<-k;kt$w)zM#VY!o1xJq844Lr z_q**HXOCP<>8n;$-)es>bCzRMp?RK>>h-z=F^SNZ<%^Cg6f&Idm+i7bTkNhHMs$&T z-2NqJD37ePt@c+2K?AONM$y!pEn7`W3|J_(3#TKOnPj^Z8-*L9g4*cVGrI5}^VYN1 z9EUn0=rqqhl_`_lKd0UxRE0mRcCASd;ur0cF-^wj3HHeKSueQ;nQJ(Rr#pAi59>QO;ZCm#r8NE3$%N z#w@%hio%7{f#qG1lruM-arPNcW!juHK7GIEEVRd|F!F#o<5kSaHx;_%qI`;t!s((l zQQ)1>QBAO8{TCaB%c!+&RAQ&b~CkQ;bYjK^osjqcI)`Y8h#2HPWQWwq8{xxGed8V zF}nhsG-@w5iHSBSQSL5YANt2#$>Qt3!X4yG~Av|H5i>a&9m z#N$qUVP5@pBJ^R>5WkQS%S~)jcFSQaHEszCh~;qTT(4~UeV_UfBa@3)3u@E;*K8E7 zY7iUkT5Y4{0tNSHl4dwGa`_|tQ(t1Mc?Uy&=$CUQnWnei6y*+pgykRGDBL;LveEU4 z%Em41DNDt-{eH}RJhF33cC^-k8&kvSkroo(Q;qc$`+(D}wxy2N)9dTXqLK{!l~a&s~wrR{L`!Fy4q zbYa^!QJ5#&GD7iUi!cgaBAa)NGF#JWYKN{wQpa5~ah56MQsj`d$` z6fUFIw$X^I*PBJ#d@bdJDzW0CbxYSA9aXNgX}g|8qY7xluXs@`2~PLBjUJA>yO+jx zA+_VwZowYQ$=optxcMR1hvs-@sz$pGcPPeBkZHfuNd&F?n}>k!v^>l zFX&Up$ufCTY!psM9&CZyDBM?@P#evsHlW_xIM~zM-VpE^s9Z{Z9@kR4Ma^T%;vp*{ zCFOK=Pd#?LhrH+5g{oE(MmkaUcjNSm3MrU$zb0aRo_kZqg6s~Fz8)fBO^-xcD zcgu`xNAo)i8aAiwDCA|2#r2i9CZhuomK?UIsPfa0u>4~ig*(SuHp)92?w4FHdu(}M zVSc^*&l_3u9G59vINfR+)joRet(|(#qyX>n24vSI=erGj1e)G!4+y84n2{!&XbKlj zx7tQM!X^jb{O~`yfA+D@ketZH{tsVoJ#&jqE{roD3e>Ou&qF<&E?Sd*MN%eJI(+2L z^&Fo(S)egf74OxfPIcCb*Q}edg4r>fp>qqxM&Wc|c}YXBnqbHJFE$F7QES`i{jO`z z763LktcLK1U@Qs?|Mwj#4UV|{iTK7VJn))D^QOq-JDRbFz< zbz56e$Z)#fN0tNSRBD8`I6LS|6?6trY!prhmKmI_ z4Yg6YX&+DO!P{qa1M5<;kpY#RX^N0l#7genqeeoEW=e*~9YO)L%;=M99NoaLXU?J3z=< zYaUk4bN#gp;@a|8jc{uq;vo@=77MORzm9$_U1hiKv8f@4Yv)Cbjks{5>=|SE9-enL z5b0Z9#e2EB5s^T;d`XCDw`mi%lC=3*m4y7hp5nW>SYe2CZt9}=WMT`wTRP35r00q# z&dA1G&)p?UYBVngKF0M$LZp9VBJ8u7?{%jq3_7R%X9zhq;q5+D@4G{qO!2O1`}aAr z-o3#9!5-VlNoVfyL}fa8qu!8fLVuVpYe(6=>461~hRb`iu6d%sD_j>1xeefA0G(w? zik&EU5Bn0CfnGTqL0gskITJg<>3X zHSYW3=Ib~aH2g|G_`A7RAylx3n)&TyiXU=75z=cTxOYchtpNOR+j<`KhM&8@A8|@d zgWj??t!rJ@@q2a>@=%1#+K8uD1UUyHa_rh_cO_^f2Xx=2xh_j>5^miv!1%WMp(HgF zA^ZOzxC{C3JfAGl5Wg{{EiuTi@}l;qLmP2-z>!!++YjFrp$NIP5mvfJ!i`&{^-0r)NiTd{ zG=M_=$zPVq^etNLI1=8Pp2hO-6G zx|8o4v*!PjI&1sy2?%JsX+M&|&s4gXT_QG2Ilql((Jmm>BYWr2Q;0OvD6e(AHqo|z z)~UqauKG)88v8xgttlSybuVTbO2V@t(z+9SD*I13S>%bPYP?KdyP8x@u~ip&N|FuPBTE7M7sk+^5YAFPsjCw4%yErn~X!GX;(@vjSM94vRCryq@>j52=XJI2!+eG zh+FSZhe!p_4rtMNKYUGHwr-cM+xKrX{ex$x*MA9l_^|ZF+kM6RAW|p3WbJTUydYh* zQiG~ubbnj2Ov-)dVx8{dbauM4z3~9+5}A2u&UP>;b>U-a;Juq5w-d7(t;;II1;S3#yZjlLE6xgHUMcu zPukFtHdOOOq6YOmQG<{v?utF0>Lf4nE%>K37sc^lm%9_(!bieCDA6(+Zv7((ooiJahiu)}5W^Ev zuOh^FB7V9o1+;t Date: Fri, 4 Jun 2021 15:44:13 +0200 Subject: [PATCH 24/90] make linter happy --- markets/retrievaladapter/provider.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/markets/retrievaladapter/provider.go b/markets/retrievaladapter/provider.go index 887c6ebff..fbddc910d 100644 --- a/markets/retrievaladapter/provider.go +++ b/markets/retrievaladapter/provider.go @@ -10,7 +10,6 @@ import ( "golang.org/x/xerrors" "github.com/ipfs/go-cid" - logging "github.com/ipfs/go-log/v2" "github.com/filecoin-project/lotus/chain/actors/builtin/paych" "github.com/filecoin-project/lotus/chain/types" @@ -24,8 +23,6 @@ import ( specstorage "github.com/filecoin-project/specs-storage/storage" ) -var log = logging.Logger("retrievaladapter") - type retrievalProviderNode struct { maddr address.Address secb sectorblocks.SectorBuilder From c877166b4a295ff4912d047fb7c6f0d3871a605c Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Fri, 4 Jun 2021 15:53:33 +0200 Subject: [PATCH 25/90] add messagepool to test node constructor --- node/test/builder.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/node/test/builder.go b/node/test/builder.go index 6f3dc7cfc..38458ae97 100644 --- a/node/test/builder.go +++ b/node/test/builder.go @@ -462,13 +462,16 @@ func mockSbBuilderOpts(t *testing.T, fullOpts []test.FullNodeOpts, storage []tes genesis = node.Override(new(modules.Genesis), modules.LoadGenesis(genbuf.Bytes())) } + r := repo.NewMemory(nil) + stop, err := node.New(ctx, node.FullAPI(&fulls[i].FullNode, node.Lite(fullOpts[i].Lite)), node.Online(), - node.Repo(repo.NewMemory(nil)), + node.Repo(r), node.MockHost(mn), node.Test(), + node.Override(new(messagepool.Provider), messagepool.NewProvider), node.Override(new(ffiwrapper.Verifier), mock.MockVerifier), // so that we subscribe to pubsub topics immediately From d9a7348ae1a3befca3776734d9bdcd47fdf3dc6f Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Fri, 4 Jun 2021 16:18:59 +0200 Subject: [PATCH 26/90] use masters filecoin-ffi --- extern/filecoin-ffi | 2 +- node/builder.go | 197 -------------------------------------------- 2 files changed, 1 insertion(+), 198 deletions(-) diff --git a/extern/filecoin-ffi b/extern/filecoin-ffi index dc4e4e8dc..8b97bd823 160000 --- a/extern/filecoin-ffi +++ b/extern/filecoin-ffi @@ -1 +1 @@ -Subproject commit dc4e4e8dc9554dedb6f48304f7f0c6328331f9ec +Subproject commit 8b97bd8230b77bd32f4f27e4766a6d8a03b4e801 diff --git a/node/builder.go b/node/builder.go index a3c3bdd3a..aaaa1247e 100644 --- a/node/builder.go +++ b/node/builder.go @@ -207,203 +207,6 @@ func isFullOrLiteNode(s *Settings) bool { return s.nodeType == repo.FullNode } func isFullNode(s *Settings) bool { return s.nodeType == repo.FullNode && !s.Lite } func isLiteNode(s *Settings) bool { return s.nodeType == repo.FullNode && s.Lite } -<<<<<<< HEAD -======= -// Chain node provides access to the Filecoin blockchain, by setting up a full -// validator node, or by delegating some actions to other nodes (lite mode) -var ChainNode = Options( - // Full node or lite node - // TODO: Fix offline mode - - // Consensus settings - Override(new(dtypes.DrandSchedule), modules.BuiltinDrandConfig), - Override(new(stmgr.UpgradeSchedule), stmgr.DefaultUpgradeSchedule()), - Override(new(dtypes.NetworkName), modules.NetworkName), - Override(new(modules.Genesis), modules.ErrorGenesis), - Override(new(dtypes.AfterGenesisSet), modules.SetGenesis), - Override(SetGenesisKey, modules.DoSetGenesis), - Override(new(beacon.Schedule), modules.RandomSchedule), - - // Network bootstrap - Override(new(dtypes.BootstrapPeers), modules.BuiltinBootstrap), - Override(new(dtypes.DrandBootstrap), modules.DrandBootstrap), - - // Consensus: crypto dependencies - Override(new(ffiwrapper.Verifier), ffiwrapper.ProofVerifier), - - // Consensus: VM - Override(new(vm.SyscallBuilder), vm.Syscalls), - - // Consensus: Chain storage/access - Override(new(*store.ChainStore), modules.ChainStore), - Override(new(*stmgr.StateManager), modules.StateManager), - Override(new(dtypes.ChainBitswap), modules.ChainBitswap), - Override(new(dtypes.ChainBlockService), modules.ChainBlockService), // todo: unused - - // Consensus: Chain sync - - // We don't want the SyncManagerCtor to be used as an fx constructor, but rather as a value. - // It will be called implicitly by the Syncer constructor. - Override(new(chain.SyncManagerCtor), func() chain.SyncManagerCtor { return chain.NewSyncManager }), - Override(new(*chain.Syncer), modules.NewSyncer), - Override(new(exchange.Client), exchange.NewClient), - - // Chain networking - Override(new(*hello.Service), hello.NewHelloService), - Override(new(exchange.Server), exchange.NewServer), - Override(new(*peermgr.PeerMgr), peermgr.NewPeerMgr), - - // Chain mining API dependencies - Override(new(*slashfilter.SlashFilter), modules.NewSlashFilter), - - // Service: Message Pool - Override(new(dtypes.DefaultMaxFeeFunc), modules.NewDefaultMaxFeeFunc), - Override(new(*messagepool.MessagePool), modules.MessagePool), - Override(new(*dtypes.MpoolLocker), new(dtypes.MpoolLocker)), - - // Shared graphsync (markets, serving chain) - Override(new(dtypes.Graphsync), modules.Graphsync(config.DefaultFullNode().Client.SimultaneousTransfers)), - - // Service: Wallet - Override(new(*messagesigner.MessageSigner), messagesigner.NewMessageSigner), - Override(new(*wallet.LocalWallet), wallet.NewWallet), - Override(new(wallet.Default), From(new(*wallet.LocalWallet))), - Override(new(api.Wallet), From(new(wallet.MultiWallet))), - - // Service: Payment channels - Override(new(paychmgr.PaychAPI), From(new(modules.PaychAPI))), - Override(new(*paychmgr.Store), modules.NewPaychStore), - Override(new(*paychmgr.Manager), modules.NewManager), - Override(HandlePaymentChannelManagerKey, modules.HandlePaychManager), - Override(SettlePaymentChannelsKey, settler.SettlePaymentChannels), - - // Markets (common) - Override(new(*discoveryimpl.Local), modules.NewLocalDiscovery), - - // Markets (retrieval) - Override(new(discovery.PeerResolver), modules.RetrievalResolver), - Override(new(retrievalmarket.RetrievalClient), modules.RetrievalClient), - Override(new(dtypes.ClientDataTransfer), modules.NewClientGraphsyncDataTransfer), - - // Markets (storage) - Override(new(*market.FundManager), market.NewFundManager), - Override(new(dtypes.ClientDatastore), modules.NewClientDatastore), - Override(new(storagemarket.StorageClient), modules.StorageClient), - Override(new(storagemarket.StorageClientNode), storageadapter.NewClientNodeAdapter), - Override(HandleMigrateClientFundsKey, modules.HandleMigrateClientFunds), - - Override(new(*full.GasPriceCache), full.NewGasPriceCache), - - // Lite node API - ApplyIf(isLiteNode, - Override(new(messagepool.Provider), messagepool.NewProviderLite), - Override(new(messagesigner.MpoolNonceAPI), From(new(modules.MpoolNonceAPI))), - Override(new(full.ChainModuleAPI), From(new(api.Gateway))), - Override(new(full.GasModuleAPI), From(new(api.Gateway))), - Override(new(full.MpoolModuleAPI), From(new(api.Gateway))), - Override(new(full.StateModuleAPI), From(new(api.Gateway))), - Override(new(stmgr.StateManagerAPI), rpcstmgr.NewRPCStateManager), - ), - - // Full node API / service startup - ApplyIf(isFullNode, - Override(new(messagepool.Provider), messagepool.NewProvider), - Override(new(messagesigner.MpoolNonceAPI), From(new(*messagepool.MessagePool))), - Override(new(full.ChainModuleAPI), From(new(full.ChainModule))), - Override(new(full.GasModuleAPI), From(new(full.GasModule))), - Override(new(full.MpoolModuleAPI), From(new(full.MpoolModule))), - Override(new(full.StateModuleAPI), From(new(full.StateModule))), - Override(new(stmgr.StateManagerAPI), From(new(*stmgr.StateManager))), - - Override(RunHelloKey, modules.RunHello), - Override(RunChainExchangeKey, modules.RunChainExchange), - Override(RunPeerMgrKey, modules.RunPeerMgr), - Override(HandleIncomingMessagesKey, modules.HandleIncomingMessages), - Override(HandleIncomingBlocksKey, modules.HandleIncomingBlocks), - ), -) - -var MinerNode = Options( - // API dependencies - Override(new(api.Common), From(new(common.CommonAPI))), - Override(new(sectorstorage.StorageAuth), modules.StorageAuth), - - // Actor config - Override(new(dtypes.MinerAddress), modules.MinerAddress), - Override(new(dtypes.MinerID), modules.MinerID), - Override(new(abi.RegisteredSealProof), modules.SealProofType), - Override(new(dtypes.NetworkName), modules.StorageNetworkName), - - // Sector storage - Override(new(*stores.Index), stores.NewIndex), - Override(new(stores.SectorIndex), From(new(*stores.Index))), - Override(new(stores.LocalStorage), From(new(repo.LockedRepo))), - Override(new(*sectorstorage.Manager), modules.SectorStorage), - Override(new(sectorstorage.SectorManager), From(new(*sectorstorage.Manager))), - Override(new(storiface.WorkerReturn), From(new(sectorstorage.SectorManager))), - - // Sector storage: Proofs - Override(new(ffiwrapper.Verifier), ffiwrapper.ProofVerifier), - Override(new(ffiwrapper.Prover), ffiwrapper.ProofProver), - Override(new(storage2.Prover), From(new(sectorstorage.SectorManager))), - - // Sealing - Override(new(sealing.SectorIDCounter), modules.SectorIDCounter), - Override(GetParamsKey, modules.GetParams), - - // Mining / proving - Override(new(*slashfilter.SlashFilter), modules.NewSlashFilter), - Override(new(*storage.Miner), modules.StorageMiner(config.DefaultStorageMiner().Fees)), - Override(new(*miner.Miner), modules.SetupBlockProducer), - Override(new(gen.WinningPoStProver), storage.NewWinningPoStProver), - - Override(new(*storage.AddressSelector), modules.AddressSelector(nil)), - - // Markets - Override(new(dtypes.StagingMultiDstore), modules.StagingMultiDatastore), - Override(new(dtypes.StagingBlockstore), modules.StagingBlockstore), - Override(new(dtypes.StagingDAG), modules.StagingDAG), - Override(new(dtypes.StagingGraphsync), modules.StagingGraphsync), - Override(new(dtypes.ProviderPieceStore), modules.NewProviderPieceStore), - Override(new(*sectorblocks.SectorBlocks), sectorblocks.NewSectorBlocks), - - // Markets (retrieval) - Override(new(retrievalmarket.RetrievalProvider), modules.RetrievalProvider), - Override(new(dtypes.RetrievalDealFilter), modules.RetrievalDealFilter(nil)), - Override(HandleRetrievalKey, modules.HandleRetrieval), - - // Markets (storage) - Override(new(dtypes.ProviderDataTransfer), modules.NewProviderDAGServiceDataTransfer), - Override(new(*storedask.StoredAsk), modules.NewStorageAsk), - Override(new(dtypes.StorageDealFilter), modules.BasicDealFilter(nil)), - Override(new(storagemarket.StorageProvider), modules.StorageProvider), - Override(new(*storageadapter.DealPublisher), storageadapter.NewDealPublisher(nil, storageadapter.PublishMsgConfig{})), - Override(new(storagemarket.StorageProviderNode), storageadapter.NewProviderNodeAdapter(nil, nil)), - Override(HandleMigrateProviderFundsKey, modules.HandleMigrateProviderFunds), - Override(HandleDealsKey, modules.HandleDeals), - - // Config (todo: get a real property system) - Override(new(dtypes.ConsiderOnlineStorageDealsConfigFunc), modules.NewConsiderOnlineStorageDealsConfigFunc), - Override(new(dtypes.SetConsiderOnlineStorageDealsConfigFunc), modules.NewSetConsideringOnlineStorageDealsFunc), - Override(new(dtypes.ConsiderOnlineRetrievalDealsConfigFunc), modules.NewConsiderOnlineRetrievalDealsConfigFunc), - Override(new(dtypes.SetConsiderOnlineRetrievalDealsConfigFunc), modules.NewSetConsiderOnlineRetrievalDealsConfigFunc), - Override(new(dtypes.StorageDealPieceCidBlocklistConfigFunc), modules.NewStorageDealPieceCidBlocklistConfigFunc), - Override(new(dtypes.SetStorageDealPieceCidBlocklistConfigFunc), modules.NewSetStorageDealPieceCidBlocklistConfigFunc), - Override(new(dtypes.ConsiderOfflineStorageDealsConfigFunc), modules.NewConsiderOfflineStorageDealsConfigFunc), - Override(new(dtypes.SetConsiderOfflineStorageDealsConfigFunc), modules.NewSetConsideringOfflineStorageDealsFunc), - Override(new(dtypes.ConsiderOfflineRetrievalDealsConfigFunc), modules.NewConsiderOfflineRetrievalDealsConfigFunc), - Override(new(dtypes.SetConsiderOfflineRetrievalDealsConfigFunc), modules.NewSetConsiderOfflineRetrievalDealsConfigFunc), - Override(new(dtypes.ConsiderVerifiedStorageDealsConfigFunc), modules.NewConsiderVerifiedStorageDealsConfigFunc), - Override(new(dtypes.SetConsiderVerifiedStorageDealsConfigFunc), modules.NewSetConsideringVerifiedStorageDealsFunc), - Override(new(dtypes.ConsiderUnverifiedStorageDealsConfigFunc), modules.NewConsiderUnverifiedStorageDealsConfigFunc), - Override(new(dtypes.SetConsiderUnverifiedStorageDealsConfigFunc), modules.NewSetConsideringUnverifiedStorageDealsFunc), - Override(new(dtypes.SetSealingConfigFunc), modules.NewSetSealConfigFunc), - Override(new(dtypes.GetSealingConfigFunc), modules.NewGetSealConfigFunc), - Override(new(dtypes.SetExpectedSealDurationFunc), modules.NewSetExpectedSealDurationFunc), - Override(new(dtypes.GetExpectedSealDurationFunc), modules.NewGetExpectedSealDurationFunc), -) - ->>>>>>> master // Online sets up basic libp2p node func Online() Option { From 6f125cc12911eef5ee5a998a65c59d3eb87dc593 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Fri, 4 Jun 2021 16:26:32 +0200 Subject: [PATCH 27/90] add prover to fullnode and miner builders --- node/builder_chain.go | 1 + node/builder_miner.go | 1 + 2 files changed, 2 insertions(+) diff --git a/node/builder_chain.go b/node/builder_chain.go index 270534edb..f6d19579a 100644 --- a/node/builder_chain.go +++ b/node/builder_chain.go @@ -60,6 +60,7 @@ var ChainNode = Options( // Consensus: crypto dependencies Override(new(ffiwrapper.Verifier), ffiwrapper.ProofVerifier), + Override(new(ffiwrapper.Prover), ffiwrapper.ProofProver), // Consensus: VM Override(new(vm.SyscallBuilder), vm.Syscalls), diff --git a/node/builder_miner.go b/node/builder_miner.go index bcca25915..3a0c86814 100644 --- a/node/builder_miner.go +++ b/node/builder_miner.go @@ -74,6 +74,7 @@ func ConfigStorageMiner(c interface{}) Option { // Sector storage: Proofs Override(new(ffiwrapper.Verifier), ffiwrapper.ProofVerifier), + Override(new(ffiwrapper.Prover), ffiwrapper.ProofProver), Override(new(storage2.Prover), From(new(sectorstorage.SectorManager))), // Sealing (todo should be under EnableSealing, but storagefsm is currently bundled with storage.Miner) From c4d10bc44e11d75a05c625559b62e912d6d0a18e Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Fri, 4 Jun 2021 16:49:32 +0200 Subject: [PATCH 28/90] apply messagepool.NewProviderLite to lite nodes --- chain/messagepool/messagepool.go | 3 +++ node/builder_chain.go | 1 + node/test/builder.go | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/chain/messagepool/messagepool.go b/chain/messagepool/messagepool.go index 0180d1abf..2201b5d83 100644 --- a/chain/messagepool/messagepool.go +++ b/chain/messagepool/messagepool.go @@ -11,6 +11,7 @@ import ( "sync" "time" + "github.com/davecgh/go-spew/spew" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" @@ -969,6 +970,8 @@ func (mp *MessagePool) getNonceLocked(ctx context.Context, addr address.Address, } func (mp *MessagePool) getStateNonce(addr address.Address, ts *types.TipSet) (uint64, error) { + spew.Dump(addr) + act, err := mp.api.GetActorAfter(addr, ts) if err != nil { return 0, err diff --git a/node/builder_chain.go b/node/builder_chain.go index f6d19579a..17a1666c4 100644 --- a/node/builder_chain.go +++ b/node/builder_chain.go @@ -127,6 +127,7 @@ var ChainNode = Options( // Lite node API ApplyIf(isLiteNode, + Override(new(messagepool.Provider), messagepool.NewProviderLite), Override(new(messagesigner.MpoolNonceAPI), From(new(modules.MpoolNonceAPI))), Override(new(full.ChainModuleAPI), From(new(api.Gateway))), Override(new(full.GasModuleAPI), From(new(api.Gateway))), diff --git a/node/test/builder.go b/node/test/builder.go index 4dc23c0e9..0c704bad3 100644 --- a/node/test/builder.go +++ b/node/test/builder.go @@ -579,7 +579,7 @@ func mockSbBuilderOpts(t *testing.T, fullOpts []test.FullNodeOpts, storage []tes }), node.Override(new(ffiwrapper.Verifier), mock.MockVerifier), node.Override(new(ffiwrapper.Prover), mock.MockProver), - // node.Unset(new(*sectorstorage.Manager)), + node.Unset(new(*sectorstorage.Manager)), opts, )) From ed634bc3a45e8d76e2d16fc1347f2e48326e13c6 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Fri, 4 Jun 2021 17:06:55 +0200 Subject: [PATCH 29/90] rebuild docs --- build/openrpc/full.json.gz | Bin 23314 -> 23305 bytes build/openrpc/miner.json.gz | Bin 8091 -> 8648 bytes build/openrpc/worker.json.gz | Bin 2577 -> 2580 bytes chain/messagepool/messagepool.go | 3 - documentation/en/api-v0-methods-miner.md | 74 +++++++++++++++++++++++ documentation/en/cli-lotus-miner.md | 20 ++++++ node/builder_miner.go | 1 - 7 files changed, 94 insertions(+), 4 deletions(-) diff --git a/build/openrpc/full.json.gz b/build/openrpc/full.json.gz index 055c137545abd0a856f6ebb956e9f572d9e31cc3..49354faee516ce0e4131b349971ca2990293f681 100644 GIT binary patch literal 23305 zcmV*8KykkxiwFP!00000|LnbKbKAIga~J?`~;*NAaUiK?-&v9|uwN7uby{oVwLG4*+G z_xsNtffHHv2nWccn5^w1=uz?>iFZV>X|Lb&5%(Alg>LB2KmQ!jOEQdz2cB{8;wTD& zPt-@}f<>Nq4)WEXh3mkhM4(%-cA)?5gKz`D0bV;8LrmU<)Ehqs&t8aWh`8xoRqX@B z7-i4F&r5PC7!na9fUg}Bff#|oTB#%1@bgRZ>yrHW=bycPk4KXUWYgYmFJBbswm1-D z1P1CAKeHYVytN z+_Hrmr4`E0ZEhmu4{wYnFuQ3;G#MaPYUY!A3@`@}Bt_6IoP+_AL)ci~Sl{^3@5zPh z?S7A&`SGjYW9aV)GvxPn1&feuIAq9!LOvP7fTLeOWFZ~}LJq`yN_h*E##C9P=JPT` z{n%V$(Owjk$RKwtu(GW3#Wqmc| zelN^C`e~Z^*_$9At3L*qKsJ3xJnCaIl1m3U+S!)JO;1se3nt(E>c_*WSOZhk{QmVT zonJcm3wgqFek-2I#nVUUkb3G9@^kwhi*Ymn3{9v&Ai(6R*Y8Cv=ME}2&jrHwU@|(&; zNA1@`6d-{<5B_RRbn?~fh>nN-wFB%w*My~xu9VuUrz`oQz*m1svfv}Y&=4_1JmiDH zG(%Mmjt>`h(#;3WKH_Q?hycevA_5O_#c@BtK45asa72VW^Be`km%7XVlaZcLA^`M# zhB!w)z)5Rs;8;jXF{EswKNy0qn)mbp1P9bd9(Z;$#@-lU;$yilWgn9$5ZIG%Uxv`T zLVnVV-1W}^BtC%PqbxOse7u&YKtFrGt7A5tZVvu|9)epp+#l~ux8T-}7;Nz?GztHr zw||Gxlup9G29qJ04!5?4cq^LjL=oEFyxvyFK0adRfOaQgpQ0eBAf)PHW>pf383d2~ zJipp%IM45q*q6+&+i{L-&pgcuc95k%Am}4D3p*4$m~q78l6mByp70QP@(7eq`Iiz54(J7;JSczWxy>|ifWbdZT z&)zpo4roL~Z})9~R^=2@DSRVIQ(rc4O3#I8X&4h3Ia^QPeiwzz+CzBd{%G=c)jRxi|kY=We6 z^(Tl6I0-d)Rr~22k8G>wQNG(}Q{Xu|Jv^z9*kX87g-WDHK#=VZ3QW0?LDX=*X(lyG$%cuEGl4Zd)OW;c?OK%25z zI@8F=OCM%f#f;f3eNTY^bgN!LcH0fUf-Q7u;9D}&UA`hu3LYm?YBy^|N|K_B`#X|V zhb+VAYz|ExnRFxPxdvu;sfzATg|57y&(1H$s%d+6-dae93H_T(N{m~vkQQ_0TSZF3BbNLNgHxz$pkR=Xekxr35cG4+I5c zc#STJkB3ULCDLyhGCI+p0VfD>M0~^m6zL;EiHFv}w=p7mXk!?Li17X;p(N0P^AIzh zvk{d|P;OC2XyWrx(@j3YT z?&8DeLva4#^OuuD@cGlpzrp@H@a6p7AvpSc2HwBBxH$gw9(;Uve!lnq-8ne^1opuB z>AQpDqvHdxfAaa@582}3XYlFsg{+rr_TsZ_AzS<4DUs7l6`DLT|xxNJjnPt?pPw0&cS4dWDw z@f;LgjO*xlR=@uTn)X4sS$w8x{09X6gE1s+0JG{=b|ageMbpJNq2!XhC=|@lHDcm- z7>2~1eo3_%Zt<~TWP9uC^=K<14Eg`O9^R7e z(O<8&w)`!+y(Ru8o8cx7Z*Rx!+8bY^-mf2bV-`v$ZPQKKeEch=0##fq_x5giRbn77 zN(oCNs}O0U55jHm{l{P`&~xxizDv8eQ#Q}du(>gGM`b~Fd@kTbg#u~|d$kw9_mXi5^?FvRB53`;1qGW1aq`a0VCY2MN zd)@vD>eK|`{X!w<=B;DSB_md zc9mmS+cWIjHJ-J${7F~0WJp~dl9Ntm8((zMoJ+FNy43^@N(SdR#zQD=oPYP-Q-B$U zSv|Y^Q!_pDQG&P`yEvpZ`usVFM5cc`&3?Wb2q?YcU;w=<1t^o-*zx7+gox-ErI#dk zU7XPPCU&Qi0QYNB)()F8)_FvlO7q4ahh;PRF|5lzG||Fet^7lP*q)b zwqMJD(!h!|CJoL2Pr4lN#z;k?7y_7vRs`r4^VIFcU4nTsP9buem1fpFoJ}}nPnprk zdI*w7e)M;mj8z{H^gAAW1pP%_;xuhFopuEMoRs86Ou3VaE?TGj%F%bD+gz12}oOp{#jDlbaI1)VJ zavH%(UbhCmaN{iPK^4f%DFPotW_@ar{5ReBr2MzN+491)sb_OPM0fV&!We9>H>o-_uK6~H)hOG7y$M-A*~g<}V#lru-OPZF z+0wwHlQ5FU@J9cR6V|2DW|u%$4l^zWGiG-}%|s?IEkwQb-c!;UZEn=*i~@{^IDmn7 zKqr&a_D4Hke|Dr2RqV_{wOMMo;V9?J0uKgW6ex}=&#!){ExhSjmJ_+KgS;G3PU5*O zt6ej1>=HUIGL?{Ks2l5>+bx-Px6~C+m49#6Ie(ecK|B;3LIE!rBz#x_cx}&mv7x#{ z??VBFDYF(AK0Z{kXowJ7Q-5!P0j0q#g;0IhSy$s`@=&Y{JoQMzZaGvCk2SpwuWh2q z^W#H>ztwF{^#u!~A2m~TXRaGtcjC5T7S)anZfqWI{Xyxs_y64f^T&VweT_c+j|kuI z>~Z$xzxTaww;%S8&Nn~scl7c=*?Of3&cCyfcTV9npwQQn!`psO?a^3l z+1fM@Dg)f#Q_TQeb$&}YwXx^Z60fq62O_EU*X}?nUi8zCC{Fh2V z-j}1g#=gX>X&O2yVM~Uy`?D82AzWlheA4n_9fUMym9wMob>_55o7?@~8N5+9KVv`O zUyIks_cKn$3h138ro(6GI%f66hz0z*ONvUKv0==aVL4=K?r+p!5bmaSiUKP2u# znN6YI>pd88m2hw$^2M&kG0#kep0Hznj`Vlw-(P9ujn!usc4zH@_$}jr}U4zg|A)i)=Fqs3}M&Fv8w>z~C70<0zjZTtWo>itA@*9E3MN==bJo#_i)K`UH)EtO%A{@?2giqUF9xz;Ff}QMoeFhj zRj4kWV37uF)h5nWr|5VRQr6*qeWvmZTj_J~?1e6PhGu8^$hSL-2IztStADsQkwy|W z6^x`I6ms7Ss){T)sOVF%nO2%Cl9esnEV8fe{|d0~U1`i`2Wu=$X;SsB^oJ`3$+v0~ zxT;fng1N{lKfA*6Qajc0Ouzrg7BX_29pS0!p^5&;@=4j8E!RMFIs`BXk{fm+DNBdS z`F-x-rTW&doYwlGwRcW2_k^_8E9>sJRhz#xCE5rEbFmd!zm}b}@&(E^ zH;Z?uatYrkJN`&c;;S+f=%Y9mv65zir&^gme0KVMRcxb;-^o0@!WpBTG1{?p$JQNN zcgAREjCRIoXN-P2#^@E3R%$bNC$L{D80V2!Obc}qi+*h{ucXb*l2}Yh5a@(m>X*5N z#Fe#3j1gvlQ7XW}@gZM;cv54c$h>%%3!idfYgFw0-XCasmx%wIM$Bu(`m1}sYEurY zO{#9iHry4qJ7a}bYSWEsQ|y|#Qk%_rpyEts;uW-wR!|;+ZP>{|%|yKbGa(6z7YRKu zk+swbUhIv+!W7v#Wti(B50F^i8T6MTph7uhrVFUB*nkQRB(h~N1;Zil;lfsYTPwXc zl?GiQ6g`#-W(6iB(O3}?de$^3k#?|Y%rJGhzY z+$KgvTs#9VIUW(@gS2CvB{T1crBLSLz~!R||6N9zONWUoUAyWwR2iBzl(f68p7QHh zca(i5*;z3>>PL86epck%7kGlWfRj*`*jV3C8e}X-V}72azdz9qxt-!>tdyj=ky_Qr zeXnJWE^NCaNHDWzrNB&Az{(twU;U@eY+f}w)t#XUy+(7n?`E5*V>(J`(GIinTqDak z^@XNq`SE6pt69B9#+z4b7#GGc%Aa$0ap%`v;Q2Xre{C9kvthV^Oe`R|OhrrgmR+X0 zZ(BWIMQs3uK$B2#ASht^y~{P@a?QA0GxtKXUN>6)O#YRw!QA8Ob<`RKQ??`bxbVC57eV&|t=cZ!3aiC-n{mB&XM0VZEj~h;5hXA=IIbk;b3$ z{lg~*G2S#fKg3M=f=!Kda7=hK9AZypG>C}b)#=EhWlJZCR$!#LyV+`mnXObxOkBVq zKz<5lIG`kn!WwE?2t4hScS*FV7cvC6h`cM{M@Y7YGkt-f>}++ND%{zWXS2~dbgqQ; z?g=K7`gP~*MN;Ba?V;!3*^9&($;Jkh?P1o@O3H#z`P(WF2bI%FsTbg)Ej`7Lv*dOv zFU=!}E&ELpoMsTO+@T9*5!DRO2a1hdm=920lEi8WJwX5RwMk~VlDGwp};cMVxj4sIp zi7}PCQZ1%Q)T$$Cpq6sB(+p&lcxkY1Fh)Z(v`Qpp@|LyM`{PE@V|MvIJ-~Hd| z-o*!i$Ten^D1F~GWEc(t1UM2Bqi{MwDzaeTTzrJ$lE~gvw83 zS-B_acNaK3N8%4OeGZ=KC;NDGOlDpht{TH^kEK#AfXq)0Mv#LcLcoLJ;vju)0xSW& zVL%${UeGK+K4Nr|{r!m&&)DC;suQDO@O#Kl@B|B$q5M7MW}TClrwmPSG>JjBB3CMz zkr6z0d;ccE`hU*%#~%6bG5Xhk{a3Hwt4wk<7=%6=U9&B`?NN?y#CXa_uWqky!@q}Q z^JeJ%vopEg98c(#xEc7jSMYY5_u5+L^;frrNSW{MP}R(z4GhdqXlu*9nR}al%wpTw z$@Mz^>`jo5)fO9I0@?H(@u)8esa!;kcD6I|I5Mc7c0i3 zfCpgNm99WJt;ouP?$q{61R&hvW5LMw*469LRzw)`|9L&UCEKIFUTtmpTXcI%{7p8) zO&s3dj@h+0zDB)YKkm)?m*pun6PdiUfG6vBbQyeEF+`0iF~dg_S~&-bh}&p^Bg*K( zAPZ~Qkm;%*exWaeFQvqqN6?-IkP@Z`EAn%k#IGpv=e4O)JewtE#S$E7d^!c6)J3 zg4~$0iUQe<4c|AlGo1fs425FqQkFG#vs5~ zD8OS%eL#l*6EBD)DFT>)KgmE@BILtA0Sc)%=8HoGOPf{eaotmIl+KowDy3sCQ&Pu^ zoJ4s)OlZ5t2PH#njm+{Mws#V*Y^8-%`NI4-N5nr5kyoRrF4nZ>Lb*Y;Qdg%n3$mxO zlqd@(hzLg_ku8-z8>3)>{@zZgbV8*QDwkQPED|uwGx`zU9wB|)T5jgJ=DpuQb87Lq zd~=GF+!1<5=pCV7E<(Sg5dKw-tJg;rcWux2_hA6#lD9?g#Rl4hyRK3rWreZl;F+b5 zdBrGZBlTh>IUcTWTLM9~VMrAL&=YiVA#IjfHS*XMIZGqUS26H9-`XecTKm9}!7ku>Zaa_w^SO8kkD~~;Dkm*06L6m^B4+s zW!{SzLqr5qz~V}AZUQs()uq`^%en9aSK~LMcjg%i5ag&^9E@@kZD3L!5;gkFS9a9w zDO0WjnEnxCq2zE4rpcG6Cr7Itn8LynUV& zoNTrDJjcgd8NJo**(Z9GmwB{K!f_JLs!BL-YRt--WPQV8>@#@NkzmKwACg#S#$ZdR z<0|TMpZtS^hy+yaI76=E%ot8LwbY|^l-p76Ri)f->#PZwAZD&9OQO}Db{86`bpujw zFVG|mpg^g2k9z!(k7m;6azFK|sYN;B1kIfYCfo`r4AMOnf7y$kGJ>EIP# z&yG$RrNb)Qt_lpIU;gZU!{mTQMB9#M)Y?bTqeNN$cpSpU*t(P7`3N$Z1Tx#3ec2ho zWOPc;h3Q8{oo;6y&X1y7ruaApO(%2d8I6dQ4c8`le5+_K@uIc((sW5Xer>uzx_xlE zLAotfCQH5ZA#gqf9eoILss2X*<=k1r{2ADv#spoZK?+ZBWz)1bw|nEAsr=qLu)8auBE8 z!OJ@$7*5g3H%tm(By%dVKffV78Vln=_rd%yAV-%_(GnLNE>JaW7~mQ55DxWeT}<;? z&b*iw_fO9EMk9ts5?$~9G*oZ1Z)j)bx)dOy3BNb8t$sw=m7?f@Iz&3(ik!B6$Wh{) zE8FEosCtgP@GAeXEfmc*nxQ3+Q|4|L!m!AL(gWUR0t*{v`Pp%akUVw)BzxcNW0ksG7Yq{<__@M!5Ctqw48OffAR z_o?PU(%(@2vOwy-!HoP z|H)Ff1uD7^7$#K-MXU9Nf^ru+WGDErUxh#=pH4$r}}?>}t4@QVF3TtCVc z1q1mP2qQjL;sIA90GYU86Eu-E78m8K`fIINpGi$&&gpk&;C2RXXW(vY;LcF&jC`P3 zLU*K_kgAIFKGU4>kO*Ja5=vg0}|D>L1wyQ7q2J$2Dt+VND=%1qYU6h}>Q+A5~h zj_syHnQ;%Xs#cK^N8~8kb4@gGUsL3iQ9<=$Jp2CR`w#P2wQ???h`1FW zF7`Tu1?P%`5HY#2I0)zndoTc=e9ZwR022WeCfO_w%oA|r!CZ~WwKv(C9#Yrj6{Ysw843gFp#_k-LQA#=-38FPa?duju28E< zxG0gi8!X0BNR|yu5E0Ix>m-KNkQf~PcT91`nc}M^@1}zQ!Lm-&jtk1Fb!<=0`K*ZO z*+yn@Fj)+E^8a}a3Ri(QW!Z%>C;1Ilhf(f`%%+L=n~M@|y+?PNjIi&KIG|*R%Q0uH z_GwbRVy|YhkE(5^%2^??Y^x0$Hz@{mI~+WNz@y<5(BX5nXHBXMy?5pKhIrI!3pXXC zOFybMDP69JjknP$!<1n$m11ccGzI^I6wdj8N8{e^#^zS$*$KTdj|>RipB^D}1_h?_ z`Px>$cgiAy4k!(LdP91`%I+uJrJR%9L| zS6B*PYQsk>X02ktg(a}M-IH6PCzpHe)$1mk(|aT|Ozui`RBd||Se(V*j`+ll&tRN! zNu9HYpj!cm`W>g^;UV%Q)bjyH#7AuHl6;agu2T7AEQ{LM80WKRvUF~w*zS^*PTaE3P3Q2 zi+$SHjN@qvCg322wi*t^5O;B>Nx0ka&Nta9z?=fi`KCPrZrEhc=zhVw>27z8_i-EF zW4kNe>NfX%-sA2}q;sV938ZB&Py(B9s}GkI-f=<41sxZ3nTA}Zp?k(AZ=1|YbKjS@ zxGHaVQNFtj-}Z)lRtcsxWH1P(0CI_Klm$tC$FE2XV*BqYLG@0YcAH#i8uC_I+gnO`x^spdJYl{F6`PdFi5Y@qPOUEr8w{+an zaZ9I?bt>7pVa-O9JL5SD@{Ak=)Vp$?R(I@a)u>l8`^IXrHa6F`y81slcQ1#k9IATk zsA{#`y>>R6oS)CBP++FJ>TC8Lq@MVxQgVew@N`-dbS1P3s~WH z#p_oU&vwi;=VzsOc{9bE;=r@HD&+`dkFtjvdYY@QJvFYCSRFESjcLS#DUhe1N@0D^ z_Z3uSshi2M00LeiO@)Uv2q#1((tK`j)3ebdEl3WQ^-5o(UT2amN&u_*^jFG;}rjM!|N_55cM z^?9Lc`Pufo_%t0^5ui4LQ&TMlqFfG5qDIJW zQy>^59C~tw!13Wia9h$}p>0(t#&#Mu=ITFDob?zKW{&`GIk4lvjsrUm?5rf%Y4UW` zJpbaBe1KhIcb1x14D_iruxc;qeLOlQLb*GZ3%yR;eFMJ#7)%A46G^3?R_7BbyOsUT z1X-r3wpgp54W1QoiKzxuj0+ga+)y0L4q|Mc7QSyGHKxl(^YhSnAf4@>rAVB@pN zkcUH@xjCADeJ#n6{9N@V!$cPtD+ybAn3`!76nHQQVl`*+6AI&3lOgM<8-YG*uI6B` zgS~g8W?bgcmaW=Z&R{px^#^bgUFJD`{U}e_n~K}8lIPn^ZAtX$dI8@FkWPSf0;Cfl zS5bg`T^CoGINf z=M?PZBqt|5mN_oVNy{jCJod|Uk73`I#47=Qr`q|jh>9g3^}0gD`<58q_r^TBDGVJB4?)3>)hk>l=4+YtYFJiMaiM?xx12_oKzQ%5yAbLeRZ7c;MAL;rzc$P=#vw?Q? z0A_^!uCAAa09Wd1%9O&L4l@Y+Hy#6BAjVL{&Q~{BjC}@g2%rq`5E1c2)!4!XF1JP* z1qigH_-ILK+w1j)n?di9IMt#1yOK0heLhO|{l^R0+(l}=v!rKf?^a~Y6i=vEoVFli zHq@yLBv~8bYeX_PxV1`Q4Sbd~t+FO$2n?ej^(FQDvZbj`_(~`UC>bGkNnjWv$iNsW zA5usE9Q~V>Gof%is`4e2i`;Fl1mpj$F!q8%S8IuCHVSV8?jH+cU;WVguh$!{ZM;X~ zE9#aEFk5>LKBN@k3qje*Q0#82CN*-X8ct|LRC1l8;}eacuXQO*vYKx7%R@{+OhAYj zrv75?C2#?m04Sv1IR63>Kl|7*Fvq}bHQa8}oYZqayk?`p!_yF4Oh)J482M4qS)o&C zWD65frw{hfHym*W^ZS9%xGa48}0>Rd5? z3=Usqml)@i?%ZFT`-^jbaqce*Lf~5s?k{I(La&j|veF5LXB$|y2;haW*~b9SR!s4> zM3{1=u3?}RCnl`pkkLsl9=O)fL(fBDoTWEL+=7A$yh7Uldko140ZRH=@R~wrK`)6! z)4Bx{K1g<^%csS_zzhueeG^m3Z01XXiH|*~ls<7}aLNFoLT3lYBccr5V}|&c27V?k zctCFuLp~Tx0VU{?z<|m|uo&mMH#727)ke!a+Q)uT)x`~W(9c0X2mKuM>k9q08@vP0 zZ(ul2?C?ze2S^*fpxER2k7%!y&D5WjLtGZskn%Am4_DZ*BHc_e4UL0=OE+4Vwa_I4 ze{^vk9V&OI+@W%Z%2yJVH+V`*lsHF>!nFGRV)A+LcgHN^Y1J8N#8cqC)Oo}m!gdJT zA?(MDupMS~nAKrchgnwCl`*a}LcpG`EUqZl~r_mPTe!M7%YSORLslIOEEh7Mz8; za!tx6szKJp7%`;6^AIY)6bS&y6nJzpK}5t6f0Cq;YYRdy59gfLut>RAHf-m7N<33x z_h-*sY?SO|AB{2b<=>JDAjHyA@)y6Lr_mt5eEgA*cyD)ObE{u?u^&+HO27Km@2SYQ zX&d_GDPk8mN7A zKhG^jSv~Vs1ioT1yHU{mR%$cFW~R z)Pki{{v14utwzm+y{dzH`o5~7W)*Eu>;{j;oUo*a<4uj@quzOOywjtI`0bHSMH}`9 zC6iaqXjw3?D`o?eOx@mY&jiVugkq{yVHHw&US>a_WQf@W z`Fj&#ByJO_vh?5^#Nr$gfmL553W9#`UlEG3b-&Wc8zXjxkdIKN)%J);e^FaGEB`rd zYj=~tv-8!lZhmT0im6U35#ZRDFNZjbB~m#yvj}bb?<4|Ot?~Q;2Ol*-uko3wpp5%d zxyvr-A_#lj)*N<;MH2nYaGGO5ttta}T;ls`SxW9y({D= zy~w$HZlVZ$l%>XykJs}0g2jYA>>8_QNzeS7^%|p!I$7^g$MgOk*`9T)7WH<049~%{5{b4W!Py?}9Gy9U^MC-SirAI;#NGmN zx=_E3YVAcR>#nIdW4>v^=+_2Qfm9%hf*oG8$UR*inOy*iRXy>I+P12-Xs0kJp7I>Z zHt8sHP~AK$iY-<5=o9(Z2(@W1nTz;Z04Y~bB+Kp1-0lBZlGhjdi^7>1T`Xy8PtBZc zq-B=PhK^ejH>X-~ss*Q7c#^7x#hM}>AQ@m|oSLO}i4rcW{2Z`yr^z+I;E01U@~+N9 zM0_~ts{5&IYSCabNmM+bG-v}aOT%(V(;-cVG#%1(NYf!rhcxdGX)aA7Y!mA0A>v;x zEmUlB?#lux6<2P4n5deW7a~5_D9{=N#+|yW<_`Ed;OBs!1AY$pJxbtL8AUS+%Am$P z%(#GIRi~YvK8R9t|dVKDMfxxHFfeHka9oBXZtI&M4)KQVw-E)ZtLa z(xMLMjdXY3NbWkfyUuNkm*1|}xb&Y3DA1ngNoM^AY9IEqP`|yo^8$zGNLFnh5VYKP zvx8QlJkTxbwBlhj2(SmP(6pE&e1xwN0mp|)V)3V$^1HBFR-;|eS?rgREr(ltEEw6| zx_Uj@iU>pgKd*0Vq;RBc0D9l%fn5 zQal+$GD3Xql3bFsWf+D5M!NC1SwYoLO>KYUAcRcHHC$F&0~cd-N#dfItL!1V6(3OY zQqb_FA=;)82bgI13Z?Az5|c}!+g@83DbKHxt#_6d^R#XA_2USk0~n?+PZ^rvXcB8YQ_bnA`KT(@>%sA1uN3NZFbI7#x@KE=+oK%a zi1CzM~IiAofaWn95ui))AZ>PF8SN^{%c(=-t70HBXd|~u4rf(c2XvF`Ik0??js)^V82XMy2elb z$ValVCqO^dx1M=wao*ixaX@|4+g9CZzi_$2Z#B_6pjdcSPpd`AfJ_8(mbw zxGa}fb1$31F&peO@l_v~vb&TBmJU@_#SHXnF*P|D)#2F0Gu8GKB-pTB_V7HEYu#;v z`r6j4!j^5@h(TuiwC4#P?WiKnpC3nz9Wh=QF*cHR%Tv7CM0~|=OTFE!xhPpe> zE0@iV7T;VGGjh}_lSG=Yb^^oidLe#jm78kB&L~mn)y6uJt&+3$iZKFEKfW1LjJjN^@OX%tmi5);Bh{c6uuNhQC0)=?4TCs1463GWTHs z)sNn~Hok>WND6zJ`72Va}tFEq+^JPW1&=bH*%K)wm(IJQ@ z3_wxx{#v^v7ZMB@&x1%(XGo?uW5iJ6XuznT9t{8%lC<{~am5@^5=tZIDy2yP@nG6XDS^cwrf2V-?xO!;7a8zTZd7zD`g=Vlw+;2_w&Brm}Ug?_%i z9z!2cCMlnK=S57pXL)iVz!*Uv;Gv?8aV^%T&> zwd{6KGbC63c$n`vw=!cq8Y5W|l5*|7TDYlP2H7t`NN$kaD)D$t&DMw~Am3v$9r?og zBqhU9ycG$=07rt$Bb02EWIV(N9wcCZG~*W(a6FL!BIA>o*1`n%csNArlL!(wlwE;q zEv)gi`kA>Ts=ulU(l2PSW_&K?j?GWdC5ec@dbyHMl_y+&gdVDLoJ=PsK0#{orK4p%h#6ehK801g7r$_+QGh_TVhePCvq`6B{u^2BkY^rkT{8J6_ zPq|)_4$5(uIaPFxfg30VY`scyTX-2Nqz2Vb=Y(rb&j6|B1id6P=UaXG$C^Q)kAuW-GmMvz|;oPcTu+(+Ru< zUQ!T1zy)P+l-QKmCGj8unDYoJ?xTtE?V0C}p!<%`wH z7MA1=6|-BN6V^29W$!7Wr&FCRMs-$orb`oAQ^@LuH%_kh!rLxMbWFU>*1)ziI9w4H z^r|T$%^~urj|?vB$a2ya+Jr5XG)}|JPtl|p*}Kegu;e2{=%Js#EICbXSawmqOEm!$ z0!>0~A@tRYyf1<#=Y1eSP+}ux$ipF4?zzhF9Y;b`tVn&bp~MU17)uuluZcC}*8c7| z+XC3yV~C{dxSTzg%aD=MJm%k7@U(gcmQ~@e2pidHx={2UiE8J1D;5&ho-gX{i?)}E zGTmC-BZ^Ds1dRClgzct@35n5~=cr@Ux?E9EiXMQJAttmu7{ z72y@Hn{Jn!U@qpld79R@zfbc0N0*rBxt?ZRB}vQX7({$OPlX9a*N9DxOn^z&&(vM( zcz|fTv%-s4l~}ZA;lez~mXs2_X}b6!&rIwa+npA~E!2j`H3dFGQ5|$#DIfA~u!>|@ zW)-U%h@~PsTT8_i=Ridk&qmFKL6CYi-(WEY5KQE!DCc_$C<8-ACo!ez11@M7%9pvT zY00&YMTN7?&Z;FWX^OUYFL7Yo8xUQf(R(D*je3{I;HO^AXX+9RW#4q*D=QdIprjjH zHE!5s4OYwm6EqJl*7ffL7biVg;9^~N7A~I1Qu>0>7;@kR5X=1*(i@b(Tm#T*55 zImPagNNujKH`#+eLUx5jtfcKksd7!sN3L>BSrMyZ=N1$zSH%ePQK)FtJMJ-hqX6#- zRY=&_hN`Jp40g_qlbUYX(KbihX3)0gQ&)Uzqp{ID*Z6Jez1ziF6F1XVUlaHElpVM> z*EgE1DSA~85ezztjhP1Q<;G0RytIA`${Ke!LwL*-PP2%SA!Yiu34j9{2_rr#N8P)n zoQ@WC3ci@GVpHtJTq(QIi<>)JJ6rO({1D1vp#H|lLzUT7x>~GLAMD+Y^{q{HIYfDZ ztHGLCIS4Q!BCFqyJ=C+$PT&9qQ`K~&TiUvJ_ec1X>|iD*_3TRwQ7^KZ6mwUY(`tX=R4fx${g2Z_Tzg7 z)t61<6VhTEmd}jUH#34xe}_k zpof`enl7En_s55I-rSw!%6!k(TDwfO201hD@Ga%dg>Dlfo$d=lf&y}22 zP#oa8WpQcT-6cpGXxu$maF^f^+@T5X9)i1s;Lx}`jYFfs5E=?Sf6A#xyC5=65neq$N(w=}%8UUIlu@@UrIE!1hC}u{9`x_p zH)fTbJ}F6^&^^~R$KJM%6BDAEj-LHBlp0;sq7O(qz@+W4VnZ}4;-4ZOpzogY8}rB! z*n&I!+VFpIHm{XS==RrTCsGi%6NEnj{kD$&Qk7Ib z)9zjb=UKZcwni9Rd7dy?AFsqyYsaP-{}w->*y`F_0hv)*q5v{ip;$n*BpEtI(W=|5QTsqobFf4eFvUA_;kH2QUarccS>xYA1+ z?Y8$cNw&Gcm6CuK9}&&?Q(P$l-DGZMfh)~GzK@5gmv|Iei1&&yf)1K;N^Y{_c-2`5 zid`dYNQo1WP9i~U5Do*~QVT*QDw{-M9(RJf+uLt56QZn-EtZaUq*@dd#z-K~1rQeD zFUFdX*F@ESYgwO~1Kxkzrnyb!J6KPzIs~w)AiM3=FLAu({;9}iG1M%3;b$yWMRojy zElu^~@UHAi`V)JQmtiX7T2o!6#r;g$1j`co$^36)hgTY)>iUo5jb-Mg4n_dxI6b?n zs~XgAvO@UQJ0$5F&DP#^vwjq{VbcPPE^DuHOFA!O?>w~vS=6YC`b_+Ro+VmepDw-!nHJ%^GW6ex_~iyXoCTb%oJ2hrT}|RV zQ3V9Kjf>}hdcBaa_wDD9DSFRMFzMZ5M5B@#$AG+$ppd7m3P} z6WKXqX~|n+uAZCPSNXj>>jKt}_%02`_&wn>_ZD4Bti|dQiKrS33HjM4p$q&no#rjw ztKue0Nc8z5Zyo1^tGHZU z!isnK0Rg~jF^%6^o$&9oz{C3YrM5GEMqJn&!vX62J}7ag{xOOMS6fwM^QnR|=5S)3 za5DSmp!mIU(M1L|Lj73OJrkzEw!g-geL$hB?xnsJ!^DpsI=66;ymsP~*4EcWU`+Lo zjuK;$#_9A4!k*y!5RU5j#&t2mPrf{JALxWuuUN9@oXEPxxkB4`cG`D@XKi%_ zwzx9E^{j#L3e8O|sO^&7kzMZdCAXDW5~x8VAHk&_u>e6~9GjGzy$OdD{`t_9tUxC$ z9N{|q^$JySP^pHEpX+?@iOR_1R!WdfmMC{k-M>a}6qnmFgshE2*hVissTpYP>Q(9c zFj>FI{6Dw{dF2U)hb15%O8cXZw9iYJVbdpaSBIX&obZRM(!H|#i@>p1yAFXxn@3b8 z-$9w;{ad`e?`sRZ?fxC7kx3keOF-|aL%@9pXW2Gb$@zazC2`qy;Fn_Fxz}^;4OqEX zg$rV$J8-gsOx>xf`AXq^0@P4Gu@+sF^ovG8!?OYx0ecfXVelL}f&xB!gqN9nEt4#! z)@!mAPp&n4uN8~BZ)BUBGP<|GITd*iBHD{qlKra2szH`7$l} zwpq(ya-2Y^7G=!KiIW8Fq~K$XG4t=AUN%DAz~^2A}+`#R-HZuNKOK|HCE+inL{4*#W@ce}ex z9WtdIQ=EwL`+DzoKVxi#Ha}w=wKaYlVH~>;117lYV)v(vH8YwN!ln;*-3G;HfL;tA&HQF1cpUzO(4CbFAhG5p;X?$l|D^vH~}tTsSt< z*~hGn;q}Y+Zd$*pvwGMc9rp_`t~7NQV7UhOvx~>vk-DBPag;pG?)J<9zD!(SLG;X7 znHrHT?u^>x;rez%#)K2ST9+vGE>5=aYjV_w2Zrl(%d6$071(GOvYQy)m_HV!gLz;y zVu-#R1{A(e%CvoDgqMmNiR&VCRQ^S)nJV=zgzCbimw{}7&YZ;V_Xl!tQlxS5W)zf7 zg&O;NV%v}7!xaBt=#^IF<&tdEeLSWF_sT--(Mc|?@H$q2-4d|}C@Fk2N6~AkcyemE zT1=Ebtu6ak^$%&|rux}17(FqXbhRsIBi*E(&#K6!PRh>s#817!FAF>M{5i8t=HJus zA_ZI3OGEf6_)k3Kq=)?@FPwkf2Ahne1fqfYJqh-Z-QQ|VsA zU;^~to_#i4B-vhH+JCo|OUS%Oee)CHOed4l+NtrF$S$kFnG^rm4yyW|c7FwyqZ(9H5w$FvfQ?H4k6MI9bZW-tm4Mq4Z_b4goYMMEOHJ zI9HxPSq=`v715_KcD|+b&8h=&7sQP}_>Vk8h#J8*FpQlyO<7`|4Dj63@KI{5_1!n^xh^VVAnl>rI9lwH{*5O9`r#nS3Z|ns9|s**CPBP3;^yX z9?yg24Dec(i_iF(imv9g-$oqi&9Q+$`fE-cnWi__)V3FEO?s$}3-!(FgVNB*co(+% z4)wQbC*wvPsY9ONZfOZT(N;g{_Mm2@d?u8ZYz9enCz*B>ZaX7fK0iv3Mox-L1tnYA zlVB)Hap%W?(JJWCjJvBC7%KX(Eq!BTe$$w`37;PNNR|?E=`AAKs$c8I_87>Xg;PS5+$J{PGSEfLL zDuw1WwV-^9C{1z<_2K+~Qbsnt; zB#ZnD*6&rfdLCDPD&!VO&Su*wins!W6>#D2U53p{pf$zb~azfeU6 zQoD|K6l&Ove33&-M5`F=XFRFe!~Vj_g46w7qT@PdcMzEQSI!oCH@*Pk>n&j2*#j9- z;__!!8S&nm0)jlG=FZP1HcbcTkJz@Uw)TE&?|cFxOb3yM57}f@n31aDU@3}{ims`4 zgz39~kK8d8iT=CM>8=@pA>0sH2yK98$kv#I%s})Y5VDW=$nLov?;R^Efj^5#iSNXpXrD$93h-ziPP9RSY$0(Iq!{~SG1J-ogP7=1BdSU z=BQMSv$KgWRg6I-{@XDoV~{@HgJp2YgEgyH?R5 zh{~YeWbS3@SoIU7)NVYJ!GV>AVx>1%B@3s!Ak+aCqm692ALdGd$%3VwXeciH4|1+{^feg-U*<14X8ORqUG>b)miungd)0mj2}7L9Nti2NiGxDK;U1z=bmPm~1K0aaB?!qD;-Vgf|NTX3pS>vqu6H_-+v1>XU>PLs(T+c^^3oh1vC~#%+u= zVumcAN=Tu=l=>DX-+xbB8)eE~yZ4@CTTgvYeX!r@3XZ2rr!b1o1OuCNqK%0GSqHI` zIY_j%v&NJ}R5oAN&^ENSuDgpxbMY*ItChP$mgryN>V76r-*eH!O4owvL-CWYoef>I z_rjB&J$1ACqGyoYh5_u;*da^Kj!}CU8h`sru$@{uf|1>qbiKehK7!4UQK%991%@ZG zHuPL9Qb=RMYJ3vu7|vHSxkoRn4UlPTvc~!JMx_;1j&nvV$w%x4v*mc!oJw6hPL(n> zPSL@YQVeHb^bH0LAD2$#UM`hYzD8RL%0j_5-Ku`=absV_+)$mWXoCZuMQB#V$3j}T z%;La)w`<-d%qd?Xhh~$%a_=#!tTPvpRMRjn;NbUt@WK$rz&g1OOM+)%(M9&U*ht6> zjLESXNF`^4B%Y#E7j5qbx3WWZCx{yU+IY=oII3nwlm)Xg`i;r;L>o8>y?*3347ud5 z0$3K95K0S_+Yp7YxKCsJWb!@g=BO|HbpLIc`R(jrZ(Kx&Z0-8ab?;f!{17UzoE81K()?ov(6SNg^EBOq>&Z8N4^w!8tkE68hl(D$b#KW#W1EjRY9LD0Pa8= z(V^5caH`=x8eJ~&apF9L6n1hPInW(B^y(=)IW{4VZC%zmY2#7Nj)BwWx6-_p9E@+5 zJN2Zsg_Khh)K|nli8`s0%_1*5^ZN`0J%0(#QpH!2+tVOLnIlC7hqYm0Eu$vB=QBq( z$mz_-lW+Lpo-G)^Pb=2CYp8SFcT&O!ucxz<^Q92-SGC4CIS#a&oD9>0>5y^}YBDzs zlwSIMi-r;))lrV%`l6v#A}+eUG1noRhm_ilQT8QHnwy@1ft+G}Q7?mEla|${&+cS%^pLQ+_LC!d2{a{@5Q^bf_-z0>jpsy!I+L z$Zy7uAA~#%KB%Q^nNHN;y!kT{6AHhLiB}qC^YB*SL*uhz*J(eGL{yHL;}O(WeirW& zAjJK`)qPlOp!CF-!RWRGg@?upfV>msPT zz6ipE&tvh4A-#d}7`?^wZjiANU@4l7ZFsxc9FNlEaBi3+F}VZYVz&aHk@|ppS>yTr zdkb4R0?YF@(P+QNctp91*yaP*OBq=o&ZHM>8?28?bS*i#WD_km7?P^TeISCP;PlJ^ zMeKJ#5Sgc-9Z36JCne;CVzAId7Duc7+i6L?2)|T#%3iJAFfHoiVKu~HBG6odm=q2!<3gQFD1Sz?0UB`WUXl&Eiyu%;-Z z^#@il&a$SGhUO*@9k^0P8a9XmOGEyyGA962-?yh?v8PLX@hF?(qM>O_w9uS#wN#|` zEucc0%$z)j%dTh#v?^y3(a^{7hU|IwA z_OR=lEfZ1{{nLy^5?7B&uW`I)9G{GRQel^eR{7f(;;tN6Npg$|W0Hw6?t9JtBOuJ$!QziXu6IN|Bj{|_N187EctzHkew6*=d0 zj%;s@s3B5dvA+J*Ce!^W=ckY6O0_R?n1jGQ0A;l#q=pU~A9_A0BpJJPdZwUD)qq)8 zC&cT6`8z&$kGu}ejVbO_4f!zp*p=VBRyipfRaDKl6Py;Cm#rTA2aC9q(+Xi};Vm}d z!?B_$*CwALY{LM~e_UA#rTzLFjWs1WsX}Z&UV}#YPul90esH4uH`I{WpaSivPX#@FgMfDwXqrJYu6H)5qI6 zAkF;y_7xoF5a4b$G2GkKv%ft^H27cC#b6-X-LbV;yAj{wUl%U4#NScyj3$0@ZZ$1C zz&~#gX}!{)pCG)(DX03lM}{2stVVX62_8%JDdUrP8K;A9q7_NbM+nr3I6cT!8uJ-Ou9e%h9c1oc=*cZ9M6jEZ zd~=asFDKr;`|G+%Pu%R_;E8mhiCP{epwZI6=U|(<3jj|2Me3kEUyu;SFIvii3A7%m z`d<^ij41+Z?dJ)7bGkaQSRFWPt=Lx>XtokBP_vh17dWp$4g*J_LWY#w{#eKedJ1bo zQ@J5RwiiVb@DDoWSJ{fDD}s}sGWTcU^+3G~O&^7-VMK*_d(tiSKJjgQr7=XCmLhzI z=mDbQ(3->}#3n2^tBS9q{31u%LBU;eH;EaR=X2t7s=J(2QNQB3UoWh%d&=0S&pUaM znjoUd*Px^D{m@`KoQeV-T*fE9MJYe2ZHNF_q-~9FC5*Cskej)1tPm&Bl7)lx&p=X$ zK)i4a!9*uMyg{?!>-n>M8G#GAqwrA!8X6bLrw@Z;`xG;uo@(t;(|Zbe)pVFXpGmW7 zKJSlEQ*BiHXL9?22FjYk1{pT;<6%*1`=wi_PvOqlK!u=!g9vUBu%%kZvq9GZK?msc z^^+-TIWcthI+J|fO*GlksFTs}7cMGWYH{l3{HS*y?VkW>5{Yxqa{Kb=(}U(trFF)( zD1babBk^Wfd)ZvPp z3e2aF`1Fg_913v-*F>}MQCDfiB+)h?Ea~xk?2WyWW&?DP?PDuh9`=QmM<<6^(-Pm8 zd2v=x!)BGI?K24!ExN2lj06=Ksb5obW+D8JA{D33M)llBgQgzr`U$~h8q1QlujLg7PvJwyG@h~uwFWiGu#EmYA>vhP6tPu zW+Iu%o+B%=vW2WXMgw$Aa1B$==czx%ZpYa=C z4fWy#{;gePq6}Dlb6(6_xA>9XXZ#)Nc6z6oXzBCed9z&cvmGUtJ8wAy7Hpk%;`F%$ zc^5*i=QQHSjUUe5Vd|{6&n!8X0^6S6zw4+a*vWzi+nSo<+$>Cp)bBU^uCzBd9Pnz#pk5lH*%2R{tvwZFc|;< literal 23314 zcmb5VQ>-Xm*G9Q*`)u2`&$eybwr$(CZQHhO+wAjxeTwecNK@tL; zUBASnhjDQ7aPWDOiFW(kLJ`VfD6_D#uB_;O*e!pS?q>R7Fy+3w^!{GSbH^+1CIWDa zoNe&x#m>Xy@5m!cdw%D@Z$*lcI|=%J-H(Q8&*@9?JVh0v-yY(>O?j~+%U^|`mZ^QK zx5aRafNq5~1Ap7%1!FRLU7^eZnJn zTNvc5U1sJS7<@{WM|B>NAfl)RLc2gReE*6c#?ekFu=2R(E7>0}eh^Ipf2N zqNj207swHgpZ;O?+TsEQ z51je?uwECwuhWdWrTJt9&1a`#HI(-YI&m~FK13J-pbP(4utjWT4t7qh4RJO+=HIYN z^*kV)cRtWR2@!`rd|oKyJcIsF?k5}|gQ*=;esl3T@{|ZK-w^EJc=1imEKbD4#TX4{ z1%ELLcUf-FiK5`r;Bh@%cu03J5HFD1>5TCb*%5{kZFNJ|Izj$k3*UU4LMYn<~3$*_U@8p7YLu)L0PEz)!~6M<=dA(8IqUY*p^=QN1Fo$4_Lo!eP@ zinSb^%U}<_oGiC+zKkO>fj4nMeqw|3gAypiJ60wrOyRqooxWQSy8i$e=rZ=&jB=Fo zyfT-r@u&Pitc9B4x};s}%^=E3pv4Dz3<%yGA(H1lu((d)T{6(wwrLb8kQ)Rl@ghRW z+qSZ4?XGQY+&p`p4t0Vof8Lhf$`X!*B%c-UdZPQ0d9oS%y!0S?^chfkblQGCQkup| z>|=>%d^GqYyO2upd|!cxI-H{T`T#RxM()_ycCX%gGyg^5lak1Sq~1$ncqW)sn=RUx z?c>1hr*PvF=uchkGItB7xxvf5jut_TssmWRXy1sHj#H}n0cqYx(d* z8$Un+hf16veQLTkgu4Y?N@y19Y0R}(%~SWag`UddJ9&8dStkM_{h}4!#eFypvBljZ zA5eVeXt%yYDCrf-(1772(!v4HiSM@w%;gY##2OdNlV0b}Ha9vdhG(iWn6XOfF%1E5 zs?_0k*@W#0pm8$a)8t&#?kNz1!X;I(V6$vYA{In45g_xVt5OAoLnAm_RhZN0XP2sINW7g-QPif zd>w18P%(S{2)D%5dfRh}cNT)lY@kzU(x zfluuMWAD`|E*mluq|ko@pO7I+iEZh}S^Fk`wyRFs#JC3f#vq>S?dfXZ}88kYD3 zW2dNJ%!s>@lAv+TQ(4xl?=c=jiFCxfmjtJ|khX(O#{T$m7Rr9CfMpoRO_n(?>V_lP zd()zMDq%fICq|Beiy4*7U@a}RvQlL}v~;pq`t(HfB?%+y>FS3)y;j|dmEf_nb4ndw zLtl)iM77bXfy|tJxg8G-xKtJLtXpCoBr<+pOT#iUHa3p9t=j^-81V;EC)KkFn%F`^ z(7C4rF+Ah*t782jq|BuG><%m(Y*==thw#SG7S=S{G{mkMcAF5bwV|~g=nZjt^f>$O zW@_KSmR&1u!+m2*eKW~?{V}H)1&0{)H6 zkkpj1*r+xwlss@UVcL+BQbf}($7xf#$eET+Z$jTu512S^ulD-$wZ8eio|2G3(6nfO zR%6jIzo<^u$CHrw@#6d_PO|~anY(Vl^jLrQibtbwS04Co)}~fGZh%&i5N;d+%YsI` zIfsqnlO@N#9bJ4r-J32;UYzw^m`g6x+p3`ETwuGsR1NL2T=X&_KfMKHzPuVpT7-4! z=mpQ*EdHH}MpPuu#2VB@UY4G5J!VnErP+|7BtCAD#EsEFURaMhq^#?aNm6s>1*48N z5=nrsl?F$=eGn$^MzJO-!jE;2nw^6};UzG(6wT3Zp{Asv%Y{Zk5Z)fqoMQ+RHWrQjg0g8~(^2Ss>K1vjV{aipT0BY-UrfafKy zDT<%;@D?DEkmc^8KQV++z`H=Jbm4VXaYIMG@C%vp6aMoWK#9sMU|=iD`?GsgzqYFw z{Q5#i{;``%r`zN9{@^Pu{8-D*!$RY0Xup>^%jfI0I_OLK@|yD+{}E|V! zmpXc9mO2x)T@5MuDpe_S9}O=u%*Ix`2iJO-8R++Rbr7qA=i_mCC8c+`rFBTxiD`>? ze|nm|<6%T~?(=l?IxV<+3j1P5{va4HLCmD0hv^Z$jeoiI+1LSS;1RME? zqJ%e7Ztv?~&3@R(oBAj&t6}{d3NOmlH4C!#2j%z^`#6A{x-E(BuSgX|^Fp=Gzv2a! z`<TEA-Cs*^}+ayQ3A-WiAl<%u7OkJ#R=VqL%M)KM|c|2Uc8kJbr6{MS-dmF<9Hdxh)*W zv4Nq68-&@RX(0B)?x$~tcY-_`WeFUyQcYL6v*+54iMdtvP)}9m*Ujv#wB#IE7+JBz6*H1T{zy{*zpGOtQRHmf?m0vh?X=&7ar?TUUk_v zu%6x#E6#1L66^0(s_kyw43up2pfevTMtqd~q`hH|Bhtw$U$w19*w-q(w*zs`E0f^Z zW-I}l;&(VQE3RvBu>Y0hTRFfU(1)4;-s`7_3!pf_cbhn74oL2@f%QHfQQsJ(j7wJp z*$JKcSR#D4?Xph5p9Mk7R&Oci4mGyCw43PZh51Q(u6lD_@ehufMv-}^MOQ0J;B2P#Xbo5?-vjE2+~Ct7Dr(&oB}P)QRvv88 zxMVjdPn#l)3)jUihpn8W`z@#G*4Ri*=3ox)&Qg#??5L^Id9t%c;-qClNiZ$Pw!+q` z$#`v3PL)IXyHn=)!&n;Vfq-q`KT9O_UI%oyk@LJBask&J3yM}UrJ?Y4CO2L{z}Hmr zvm*dJ65tM%|55W;GctXo-@^E?Ld%!5?(SJ3y1on(#o-nP95#DB(+@K zZez6_u^FPS#KqWF@80%|Ww^`x<^A|-`}_Wi>h*<(=(DzdIKTS4^V+lRwdL#E@*dx7 zh&N+fAUmqs`yGqnvO3Dz{=E|&ym;Op!^X;X`83;3udb9${1Xngt#;@ZC(c*7 zrUsci8vr%JLHAtEgq4B{DmkVM9q_~&!_zbW*acSjdq;JGzeXYy_W2FC!+M3ZKqc!F zcn)}Ld98?Q)gN(vkpc!4L!5dF&0@1j9&0Zjx>CtBK7Sb8vlL2Si9Kv=bP^cl5To#}-+aY#+Z ztnJ+HA;^7h*@MND)Jyy#hRQ4)@7l&Z>5Z+MSDE|fj`#F8z~4r%Xs^GgMpb@0kV2eZ zAu=_t7?))x9YOU8H$59W8&|JzgmZ|qVH+HmmzP*{%fB>$Pq!HNmGjxLxp->p}38dYOO{|kJMUz=06FX(I>2-o^@{f(<*}kKhx-3 zSz_F5D4&9G?wRIxi+z7*3~Z}-9Zg(UaKq>fvLMX0OG(p#gk_IZ$~g3@nLJQm=4NwF zI5$`zbfDgw1cJtK)>uDQp1QUxoXuXgG?veZ$5F$Td1$A{TN*Vxho6pLJUaE!UYt;C zW?bBsH?@2U6&BmGweMLs3u$l^ zyl1L6gcEa&8#6F9vu!Ai;tq%&Mfhda8E!HZ-QrEAz`tp8?cRl`y)l;vBx3Wgy(LCd z_TnhjnL%hqO`nMKNss=^YWbAep&96Y9HGwvT^`|t~AX+@%dG#;rnYz5X&r#A< zpzmL2J;jMGGLs>3;{kJ)Cv}|s*Q+OaGQ@!fmWy*-DV4>l1YaCL&Mf#;Z=0LGa&h;ZEb5O~M zd?kn!sV3eyrdUml%ggoIL)kJva%+UBvmZ4-gK@pqVR#u+&1KlcB=W!IFOK1pVWJ9V zv<=dWCFmjHLxg$0jKO^E^>K1lyE{@OML1A)o#Z!TYg}6cS~x=-gm4t3lbzF8Dt^vV z8o2}A&FMg?)*2?NXYVVPlr8doZaw=0X8q40k8U$X=rXBts{z&x)Mj>lx{AeOw1whT zM!Q1htb5*p)X4#AQ?!a4uD|u#czQ;;2jHmy(v%MI)11(X0-0Mo87JPf? z(8Ek#FA^Z?1ObyaU+^)Qk{NzHO&|pZoI5yrOTwncdUrYXSwIJ)avMu+P7)AQ*`|s! zAh1;8FqK6xJGA*-&{a3cy7ujhkwQYxz~YSDwSYXe_?c5a`3)_~uwY0|bNdbIor05N ztgg#YmiRMgT{>wYE~5a8~LT0 z={Pl~0Yy?fx7|DbfN!e&$%Oua#A;0kta6A6EXlP^rRf*vJkD{hxC<*5?lW=^#ox^1 zty3c)tvzvaNvkSba#+UrLWaMyd$qGwf;Oc^f2c^z^@~(kpmr_Q3!yh}Adsf2)G6p2 zQR^5(QB;4;nKD(QWt>at^;PTQK5cr!IZFzI(5~sJW?9rm%5%*+A@}@=y$p_ z2HTiJ5v{pc z-5DFO8Lh(XkkHakh4=~Q9pYkO0%$XF;PoWs`aYFpMmTSJ5#iW^ij2gSj$Cu{3Qls|8IZ)NXV17^`(%XLO24WR`F} zVsg60vg56+_}+0(aH^ZAp^|&6AME5fvXeXj={6UbKJ8PsWR8p70qTu=#RC>$4wAW^ zqSS?01|$6qkkL&GYvep&a<2fG2q0xBNGiqdNf^OdZuIMVccuUxZN*o{Wvs z>nquGl9wpi%nzX+1mOB|{Cd3jetLrUdh@*Zes?#z{<=1}{vZN&6CxH$?79XM1%LzM z!wDFm%#h|0LcdH8i&E2;T}caLO^LYkfJJa)gEWi|xFXev=usB{PdRTF17f9)UGd5s zi9|>I0H55p;UiP;@$>=O_)<&KED?HhWP43-&8qeQP*ycM$S&OeOM$=+*wT1KXP*+i z8956a>0ni(^?ODY%F%!x6AKYf%pm-@v1#EK&`JA{=CHtu@^3gMfsbAhGmm_IO_rEm zUU!$n=jq>oPezRdf+pmC0391=&L;HtZsSyByf^y$ze8GEud_MYF2<@`adEZljOKPjz6n{^1Lm`5^c3Pg zk`N?Ej_I3np~n)1UZ{f$RlMqg<4@86zE-DI+pY?jZTjPunF1v9TSVO1y;PK5KTs^b}wiG>#;t9TrklYXJxoTQ~I~8b9g91+!R*OTl(U)`;@VOWczDi$XpZ4M! z=CAo;Ai^WyHL~B$12AJ_G(hY_XV`i4&td2SVkXXGfF2)uS82ds=6Do3wNxlkU^%8IaOxZpPju=FY_jl@a~N0}rNFm! zC%PF^(G*RfpN#&I`=th{6_MxBT+Dz*1zVQT&R))n=SeOWQm2=f;s{e}QTng~>PPzS z&Xzk5k|0y6X^`WP?4zDNBlCL1eX_S~PfyHn^#PtPA4Y$ePA@)+nd1_;h4{8W{7??Q zW2w=}RoZIi+1(ulOp)9{>z-wuBi(XN#-yN&;t`#t_MV~=VZbY&RY^G9|7amWlEMm6 zLJZ>RPeETWNmeF;*N$&wZ16$_u6?ZbKB#Ga#pdu5rg*8uROORy=JjI-g_}oBYE8r$ zpJ~?f{_II6?5)W3Alp$eX9mhY1los;Ea$UIn>%4ZyO`)g7e$mql#)+Ul5jvy%m1sB zaovu(gBdQz-U< zGIm9rU1q?&3TY-2v$26xeK!o_6S3z;tS<55RlLg4u#RL&9M!p9_NQN73b@22i(pmU z1g11o<9wcT;+p)muz1UP1w?>=aW#o#-a@w16tm13SiyEy9|=s2?_|3W*Y}{X{i%IR zw%fu&J%&A)VH5M+bg(*WPYgGw*m~Mnz5lb@@j&}x54AV9=WjR$mdd(|vL3?sgXt|+ zJ~(yBh6{J;%0DV9Z~T5J4uNHODdUSyGS(xpE-cNOJwwybgB$(*JL1QEI1to(b)HM{s%~ATRpB zldp0g+n$eNo>RO8+(d}4K}<#xS!ppCF};aFQl1u^{4z3rS&Sw|#jImjZ#er=;#*y0 z@r9;l`B6d~UFK*%-r3A6i}myF7NvUlS}#Ze;N}QRyxBX(K}8&&n_v?#?Gw9EI{F_W zfMRFyvU;3JZ_YGhUvO%c&np9nP$rI8i-8N) zqS@ulr7LktXyCnq2^&Uv88>v6=qFZz%H)2`o$`4y3KCMYpT%zuiHT$N#klYXl`n;%&3JLMgt*NHd#uo8rh8zJ%&10`e%J9o>TNfC1lB( z&0VV9uAXPT+v@p0v7)`INX% z;Ie)a7h$V;!uLr*kml`tHYw2+I0ASSoH)lx)H1JhrRvqP){A`ev=Rpuaf)rT{$a^* zoK0wb%`_3Pz-#F(piDi!6tC{)#j8v~+5n)T|7?x9t*a@|H(j8d zZVf);y2diuSO(2^_C>~ry1!>cuDR!C$xy^p;1ks3RO-gve1v)>#@!T>+V%}a)m-9a z_lrg8_{kGsP{_D;c}#_MEs$ix+a_Xe(LF~L6Gge>i(DIGSjDb5CoXOPS$<9|yBuCY zGxUf0Oq;}UV&_*Bsau7OI6Q!UKk1OBpf1N*?DQun@&u@!uX_z1yZ|g3nfpbgVXQ5i z_xFp*Db^yD2ca$-MP+nH%cehb^5sk+IViqb-!Si`<;*=ZsG?mQza}g=Qdno#BJq~B z+iB(%ik=MMiPxe^E>@3ugQ-*R-nLU~=g4PU^lklivW;`*C+)4hE9nBL!Y9Z=!>k-o zrd{Q0D(a3s|0pO9x>I_HR!KhB2&rx6lmzr{YHJ&%t*d-bC(Q0kh?fpz35yXsL`MmqV){&-j zjiLlH7Yqg(*Jfn9eDSu1nXdD{LV+suv-4l8J7@ujM99}cnB<9oP_~JiU~)j9zg=+G z*ls9QF4P&n!m1qKSkv0PxcEKW>c*~cD7(lntQbAMX>Oc9`toX7qbX|)(Ub#qv~a;2 zJNeP+S?AaYZ6g1NJX~~Vx7-B+l5a%r*h(7*mLPrxi&= zaXb^a7H8)99WP653I@&?{iym3@Fik7KBU3VJiDrLEE8K&F(629NX)3X4fuMi3L*t&eWC$~+sM1lqjKehctQoT?sS zwQIi$;0m}h&b$jh3)4J})8&HxweS&k?f8oF9&#m`zneNJxh6!Jpk>i}OG=V{RXYW+Di6q}tDB z=C@+B0c5>1m_;(_7g*4WQKN4<#T8Opve^xoA~B&9$IB5+e&Jy8UQ1JNdW&*C$fo(4 zTG_9V{feMm8V^9%abFBD7u*$l-6KZ-j7(x_;mP|c+Y`|ZQfK4RkVBqqwZF2b_-*1m zBI?9;z1ErY$_>`j_oyDF9zSuiw??*X(Ur;z8@f??$%a9-53ifb6iEptJwr~v*h$EN znoHbS=TSTuAmVC$M^njlz(tN-UC4|h;Pd(%3=EYbBc@*s2?o%_A;etLFy}++yB{J3Pbj1=(bzmQ-*>o(;&`tmZ_oHn6+3$VBUSKBaoYeMXOM>Dki`N&A?? z`x{?&Ju_mnF5z@j+FJbS+yihIGET=Xa*6(G-&YR z)7~7b799l+E87`4zfK3=>Taz%71&mH(5Es$AzQI7F)=IaN$#YLnyk9u=zflFktfJo zGbJ_aXrCCK%$fqRE}3xYcgAkxN$v%@0$C|&|0^XxNLV)g=1oeRTIU^jRQ zYcmq}T_C$S#^~&N(DAHdO!2n}kghn*E;R|^rzg8|b?#;cX;YAG6rO)#mT z6B+OMbtA~+nyqq8d7{4Dj)+P>pm>&8Ks%ZW;dJ&Vwj^=@Wp)kJtVCIvLlA zqlaVERp_PD%(w`uP0IJ1M-)Ow2_@Krh~|Hf1+wIDNWEw4`)D>tSyFQVi|Jj8y zq@!?ZPANMnykp(_$;W>g#q;=m_84rAhR62_wHXI^Y)0P`ltAo3Ny>&BzG<=0DDhV3VDayp6bj}4rG%1$>yOAIi|Ea^v!Dp~7J#$Z;g?)z8tTt|U@ z#Csa$iYUQZV+UcY+y}{Vah8BF1_O~p>C>CB(qwG%%+W>dE( z_L7nlR=U>xzCLQK593#9wshh%6$uy9V5w=sK=8B_7kg1eQqMFuOM@n+{CHmsja5zo`|n^XG(c}tYgdMTr&K0xnuJpA3aadL5y;3Z_89fz1h{Q0bB42l^y3Se z_%{7>Ai4oBzqp2+KtpDK;dPkKsMkP~%`{r|2EfN>oZhQ2Fw^|~ci9+{r%ZbCfPx1+XUC(^tpn*;}Ba0tKb?rr!MF(4Tq{~F>SQO{F=k4 zY-@h3+YqP`D_q3Mb1GBck?t(9Ly+ukO8&H~5|pqGW#3F{UVFp-FNq^Vm<$D&_>(l;iDM5WMVKxdhs$J@86zy#G=_8JU+IIFH8 z^w2r_uyDs+yle55;dzrNJA(BJek?MOp#&wsz`GNeX|tG%QG&EKYra8GM|BE}G`*vH zI|nj>1#(!aa)7evpax)Z0NfOho&WQE zJR+_xtfSkt=AZth?GMOY_hgFeItO?E>Be4EHi;}ZM$jBKOs>WKrn9FIN6DwnYb${F z^2DNp$sL$PaQ%11v`qL;MX4OnqoBNm=9ZRcVq)z+W5m&dffNH{Y62e+g}^6)p*h;I z;cOzLD12xfQ~#evJhSbHEWDRU%`)QErLu%hO?IWfs>Va*ak5*rilMwT{sDk9`w3ph zbE(2`<6!m_2A7ViJSUijck|%`!$2h}hKnZRjUm6D5!bXe+WfZOWfhvCNY4#Dn8N>Z z1NBY|M1c1P+p}@Ui0U0cU}l*@&vzqTq?Bd}(GvJiPN69YlC@~SBhMBmrz5~AE}#r* zJ=J5h@+lRS1Qcrqwcu!wYmj+^$?&}Dd2I)_+v3hZXcy<jZlzg2j;bVfbexE=Ma$Vi+}U$RP{r}htSy{2<6$4zZY3jdui^exp&f&P(_X| z0WnUU3PqEMxea2fUhM*e+7&>j8!!&4ggPsg)(Jhw*xf^{$DcTGi;L{76ZbQW+xpA_ zGm>VzvCXT#%cp7J=Cq?p)ZQSBn3_AbX7;yxFkKKs;vhx_HSSItgG`JCYJpKIjU96% z9IEgg(~sHnw;|2W5rr^Awk+%SForiEd(PXN^#{e)Ttqoc2i0XUmdfy7E{TY>hs$MJ z+h{QZ?HVqnK0ngsrA`;R%++gCl2q~m@`wz9?o9RvaYA`a4!14S!m4KVEKSoN-VPkc zJ63Gzj!|+=r>(NAJc_;17T3b$mo??=hCSv3XRBa}fRqTsQ8+qHV?5fp>DmAhqaFuX z&DP46^D_GMo)ggIn?;4xe9n(nDputkqPY{jB4yJa3Kw|A4l>VpWOEv5&g2~D?)M_XWa1#JoN6OXBQ_1%mCg=dD zbXnTLFtF|E!EPU@K`svYRkvl06(_+m%?l0y4f(O$)G<>8GvLfQi_YUp{%_;%D&}a6 zq@O9L`il5#p`3ppljx|6D^}j-5IHEcJXL^j-!9_6f=_?Gq|M^mHw*X|{0sZevG0~2 zsCR-q!(I-IuHHx;9fQ9jOHm5DnU*Sqa%XPhuV}qa^1)m`2{@UXfCl zOiA@8vNJ_4+(1Q?=nLPGLb)Sb$!eX^ql(rj5Er?s{nF0}IPU6oW`r6^5F13h0QQ*D z)oI)|5&m93(NoC$!;!Wn4`qqP zqE4O1#ngJ;dyC4Md-+~zR1uF^*1LY3d(lMW5NKlx4G2C$!IE zqsjlj)@JrOuy)kKv0QcQCCg6n2Qhd*nE-)D>>*zhyomg~2+cEQ97{5Xvhj8mfzGaP zmt6IleD%7IHMwfOj6}B`0{+CAWE_9>klA$o#);(0fdj78)}dF6`L ze-^e5#nQPR!UugvM9WyqD7#kVeNwE>w$@INFYNir^W3MWsY5eIPMxGZ&z&vp{WPm> zHQaRyAI<-Z>+&Q1;kwPlZpOys3|jTknv4e8_syptOKqCXSoGcyhs1ybkX zRMwyyWIP;%UMZ!H5!s(1(S~|5zQ4<@IUW}FJKG5MeO_nRHLsqf5!Wj4d%!2U?OOY- z8D=>2HJD;;iwi3R*J9GMyeD_Kdb+WayzdU6^IWuJAd}sRp80pjkaE~z=0Ecs;-tSQ zAq>(Qyrw|lA5yeS`dQL3FmOdXi#s&H96I(|kTD9NO-qTZ8xyitE!GZ|aC=5hdO6+k1N@J)+Zu z5?Ne&I;;BV6{x*>I6AdcS_6aZe}R2HCSUoj#=fta1R&aC>ISs9){HujYtxWh%DH+N8qs` z`Wm$s@7Sztr7Uz7&vJ4?a%YVWrkyVJqi6Rai%<*Bf@X-TIMNnEnN@`_?I=Z}LSUQ()E4Z3xS zhr?ag{#vX~8s5d;7ce)kjvc};>bj!9GG5a`UeGP6!1=lwklho$7C>Ra2_T zYEwrRmzvD4g>E32B!w|haP-~m!aqxbL7xL&Uo?{@Q#*>le_@8e(9eyAPe^QjOLZW@ zZor%-F%X@=Lr4}iip+99=_q1XGJ~d5Hr%atUa0DIOSnIqkXcRI`VPzrc?c&z@Q|$8 zmgVy+?~t}p;gH5iQ>oK4L>NUgF2|Yo&Oh*g+T?md?}x<*0pM=M7|+cAo_nI52lb;{H$WxJ5U? z`gfjhtvioyo?+0Zz6VzjuU<@n_jx9g=s4$iJebe0kC#wT7%Neup}gpNs(uneb41m$ z1pN7!T59)%yN{M!fNq{?Ldo(17*bS=uWWk>jNS+_uQL2ch2?RCXNoW`V)Ra+gSAtPpBRhW6Zi!RNU5^dmBD$p3Lf*MIq zRrxyRI0%ZTRB54_T5gdOdqiu1AA~)E7H+;%VbNhe?*qLs@LZHG{m{4&P*|3%fl8#@ zQQ|(r5}7>0Ds_jDs)pL3DS+WOGOK zCIbc?i#4U=x%W`a13ZUrVw|KyddG@0ji8!(l)7iiaqIo#vN%koznRE2EIhz+(~vIw zr!%b|53($Q6FquJj{YNW6LXSi!h4WH#a#o>IhguB{K=ajqLR;&-gl}nChzanp5t7H z5ll7I9Gp^%%hqI-&32MBfiDfN7RGii(UKdDQ6L~FZB&@!cr$ktx_{}Sx=|9RIOMVNEXp|Un?$Xq|Ejxux{F*Rqf zv?$i=&qb(6Dn<4R>}tXv2!lr3m$y)ipe+lfAj}%g>na8LC;YPF+4-@*Ylbqm1&CR? zw)YX9Ks-u?Po`ETW~(P?Doxoq%cw1urFItv<;XaTn`ecj=N^t*8Qn5Y5+uaJ)Pw*1 ztv_N8)?XKXr(G8M&-&UBIIBb#)&-HC`4kJtTN1(a7ZoVja!lUm3J)q%i~leZ3{M?n zP@zT9LAE5X`o6YrA*pK)MG#-L=kTOWB_jG?xUy&d3`1Qz;7e>}RC+Ej500=r^Q=Nm zT;8p=w25v_hkV@8+$NfoL<@m2{tS8e`2@Ynixsf|-z52^SbQHt*-Wp=wk_*E|D!S? z*cG`TgW*>6Dvb+WYu2IdAl0UJbYiZu`@=i_il=Q*fbNmum`qHisfRHB&5cTmQG=g> zneI2&m0RAqVh=y>avF?YOIdFjE(vuQFH4`5}ofDf88HoRA(L-twMV~PyD?jrJQx5@4#41iM zjc4CKAlE~!B`sO&Oih(#?W01YFRs|s%@q#3x#M>R6um=`ZnM$wBkNmd; zSl2yhA4H;~UUUR@msvd!l?fjOt$(=VM|j+swV-n(_;@J(9%Bs(|fl^C(qKh8j(a=9+BnURztL_}z3+{MuF zq(9umMO&Iys7G76hJ#c-{rpVoIY)-A9Uzv}lufRjA3i*$oKHbzz_Qs6I!vj=FyMtv ziTNDkiP)*)_nJmQY-J_5tV9jSWjZ1*-Fjt~S!nHOrukT&>1|n9KhL%LW_{khIjoV$ zyr`m^`1F>9)5f~Gnps8Rp$rcQjx#)>1^nV*NNb{_(hF)V-H}PeeF9~wK6Dl*{LunA z0E`Ar#7il1yb0si+Mnud`=SXQBcJ1f+EIycr(q@w3i=%O@uj=s;>mEcGBNg0 zwQVw)tLFTQ>=mYhE|Zk3Xqc5c_fMrQH4cBJM!R8~9QUGVWX8W~d^ThKpI3qq8JS}az4|2eCh$?QzjQomZ)uuQF7!8b+2=_&4NUxLo{;Ix^p z-(P;A*?x3N>Ig1$4W#}mXa>0&+WAEB+$`tIHoFwkB-_|2H~SQ`+f+$p3v;O6cm!z& z;@Q*&U02|WOxNoDb<6*1UvaY3EpyoBE)X;cJ*fyW1U{8`hH{$$;IPp-U;7Xj@;>$< z-A}B*m3k(z<#a%Q!>-Bve>yqKuDG@Zi=x3oaJe{y;O-U(mO_IS?(XgyAW%qfC>(-2 z1r+YVp>PWXcXtimr{^-ki+UPepK1bp#Q2-BxW>g?)fX~&h3M5?A0@}!uKkw`^MhoA zO7gVYi7u*%~E>XWs3y zS0klm$a~K(!*xzEAZ|GTstD?nIS34=4*F@hNWank*pm${T3$?0*Qd@eW!M+tT>Nl% z0D`XZ%pwwGo{@*T3+7IPFVS{s^zBbckvHo z7;C%qyH<6|(&Jp|c*4Oe;Q&zPY54&s1qg6#Z2bQ~LN~9`r>=ys>381p0E(_>cJ zVvCU)1j1jm`viS$;?el$499tkh$^@QW8c`kS_aFH+T-8r==kPU7keOTaUEyYoQ+n9pB_g!I|-hA zgAS>Q9FC==C-I7nWhf{{sG%Yrb-}-@0rc{uv0rM%il`egQs{YsaW^A9_fZO>o(FhfDV#9j+aDrIo1B)ZlJW(@s0U-8anNd>Jf!rIkD$i zjgeR*Mw&?Ujd@@q+&{m!pP_&w=%0LZX-0rS*v{db+3WkoSkYwNH!tur`%2}q$8Upv zeIkH25x2NSRHxmykC-u5N38-y8gTyU zw{YLwgBtu!W6Q;D6)}pG!tJA2ZLuuq?A^7f$#aDArA&Z_e6)CO09e`JF^7;f<;_w$ zJ3gK9dE0&;V?dMP#XQfl^bb9%Y2ZTOE!Cbv2@MFK1G#uoCxDN5nXER@Rb#lPp4VL)jEGLc zOjX8X)hD}k=4TibAnu`&5!J1IRUUzgD(#XL5V7sXbg-Jys9a-Jh8^``dLg;3VQo|X zk4*lJ{(e2j-rmUCiR1Z-c;D7xb>d*wwLvEBrTs`%-;~z5?ZA-MIrojRy+)A<#xo>d{l}*DPyH+!gN50KZ`d>OU{jBr9 z3DIfx-xPl&zFu7@RDNDo6;I?bsl-ZK0Do+-yRI2J(RF=dCd3+Cq{ct}j!xa)FCK7? z+P*#CcC**O#dR9hl$~JK0g9>P5TgwWWGF^n3s1z{z&`fgNpVDn>WVQB@6Nf6Zx`mGD;RAzEm z`+GxP?T!($r*}A2Eh4`x)YAEmYPmMz@Cji&6Ba!d{$sL(S}cZ3Ef%c?X`TYH(a2~i zr3p@F!xRCtcCpZEW!)1G;0&p5mir8|9JNP!1)w<4KWm@_&snCNeeY|d){?0x%eg`5kjgK< zVFRI}Uw0QHXS$Khqop%f3nC}J7No+89t`f%CJSJh;g;kqKei~nDM@$X2Aror zaya=ZJE7XVVJJ)FjF^Z#)tlFZ?+Rx}BpX{uLMq30L`}V*ffo~DM?bkI$V#E{uS8=K z+xU`HzZ=N)q-@D>^#EfFbzjXs>bYXVz|>ao?`62?Pn~#127gre6qObWk&V-nlT1DM zN1RNl{x(TGMOJq%b=ok*Vf<}4bOC25T_9c%_Ss1utxq&h3vpM(U_~Cj+CeS z(HzNGs;z+Wvtik}W#85WiqQ_K-xV$nldh__z5wwl`j+Uw%^V=RUoQ>oVCkOJKdI5FT$cjdL)<9OoaL$Zu|+ zUdt7E1ObUuLuCqI3MbL1Z$|0wc(V3HTPW}xE0~4|B;lKpLgL3a#NpVzQD-zCT&jE) z*O%lXYAR$vlHs)1BCp7ryck&G56X!9h9<hyNe97U%ur2?w3*&-SfSqbM?KY54se*i07krBntCrgV7 zcV9gaBticMt6=qigH?6lUgJv%B}!EJ8&PZYwl;SeVb$r{Siv@w`MJ#mr4Bvus~C(t zysce>y$q92H7-Vp{V%mXv2`O$}n_HE=e8MC3?iK3CyI4uZ>bi#I;>PsvYz+`V)MFhxW zK~Pupec#%(uPeh!I~l?u^mr_c(j?g~+ehB(?B108Og_#xw2AxOVDK)Mje=wp#}p8iA{!N5uzZ0p z=0Nj2Kf+j*u>>231FUEzfWHue>wHl1BQIzM5TQkxGU^1aX9>GEYT>MazS_r|uwE^x9H@SEO( zq$#J&8ie2O3?i{C`S;N7-%h0&Z~utWeI0!r{b9*qY<=X$TtSgfYaS z<%FH8mvNy`e#KkMtE|E$a77GMy9TTqZlu+}|6zs4Y@)v6;2INAOCNrZUat;vwW zCq|S6BC_c+sJ+=OD}Tu^hnol0lTD|umAX(*ZjkL6WJlfL%T0lP6*`~fbZkgkHt z)^*NxLxvAHhFh#cUa;G*S1u>g)LuciQC8-csNkl)H1yiJiPGt*s@};pqvxYa5TYGd zmp*2jG7TbO$8QpwV8s~;8Ic1hE>nmhW8%ENB^nEK1Rbs0xO_%OAJGCd15VyZwb8$J zk(Xk(o8gg1yPTJ7msPIhmRL$bl5WUhAy^v>Xn;xFa5MeJ59_u02g+i1uy^XCSMDX+ z+4b+DY0K~G?#3+=Hx0WXSg+a6Ai#ORj8dJIl{g)Uw$`uE2cd>;O3ZFnhev3t*QTzK zVnm5#>bp;|e7H-;NN=%=gTv{|pmDbid#AEyTS}>=Jg^tbR=;0-FT*<|VL1 zeFb1bXOXys<(&cY7@c`D&gDY`thuPg95i}T(`x-A{d$3)#&~|wR6MZE+KFQhXN~o2 zUWVuXqZr(kc)_s0PZUGb53EoOOFVrK^NqNw60+4&L9bcf8Zm0{uS43xUlz8A6v`O%Nc-O(c1UAmsg=5-#`5CD zEj69o;@}9C2mv?=XcdMhE#gd*S$zR2FEwBA>&0Q4ZiQsPQ*>iB5a^ z9-PiaHAGgzF!XCka+zUT^0k?-Z{*)AZ3L^7L{2@(OHT0odOtCVoHn0ni!|K>DAf+g zWy~64pKfr~AQxm2`+`12DV`8wpk1vvXC`w=*9U-E>dvGvm(6H_|*DC#qMb}HXSNYs`p~ubioVF!*#OW>s zmdc#FpnU$+^xg|wqAO;87*P`rPKg$^+>afB+x+b%rIW$V(pm?F(m76mr)up+lNey! zpy1Sg!h1^jNrioP{$@kwP^{kn7enW)TS;y7pCpXbI^jMrdg8q-QyWwzwIqj}%C^um zF5sn(2m~MhPP6;I#wV0^W?)LiHrz`-hvk~lujeB7T@@HYm1~}*RbFD^2>o|tG>~* z^H^w@i=M|LD=_jcyj6EGj}K!cfs6d$Nch(I8r@iGT_}}?9l?BWMhL~8Pq#G(zqdY# zs8_(`+^Pc8DCutad%GYvd@}y=dn3cub=9Ge3r33Z2+vZb*;i@csCpd~K~psm4k*-e z)v~}Zfu*J3PBRV>d2>21*qfAI)-Fda~mJ*TheP?U>H1VH>qnY!{|P^2|4%o zUeqnM7)u^R<$>9&yr<7_b(h3M7fGXOh|_@gyOAe$H@;z7Yw7NS^muCsN)rBf8usPf zT+qtrY-y-KUy$l?x-!A1P}vFa7R@r2*G)T}IGxwJ`MI%thmKXIXZFTfLA-dKqAB{0nmfEZS2s| zTt<_h4qI&#fzsj3+HPN&a%`bJcOmUu(l$7MNwNiXCJ&qL&C$y1H{X{vg}NrU4Y1vK zv9?G{fFxrCO_n+(DGis%D#U>I%5?DW)qGz^Zg-e%GIdRTk-ULN{%>jl5TPqpI zbfZF0zk74Arau$?ViIj`ub(t>pE|6c@7IakW?Tin9e8B(&XUXRH&-e!K_KrCQCKtzTN-JJ+e;YGM%;+HVLqCG3ZF ze~wRbn}VtG-!tuGE&gUZh_%Z#>w<`Ij7Nz#JJs`})r04=2LhjXWahj|AAPqSjx@5e z9v5b?bFT_pU4(o`8f@NX`#4f`V-EzCU^p6iE<{EHTW#vzwyI1!O>;66LZYDYXqp{t znM*Nk8mufQee2!QEY@=lb9<->h3|5{@vC$|d2Owdb}V!<2hiNax5~e|(SEMw20J>; zW&K%Wd`wK#F_*xim%VOQa`8(8 zuJB@GPBlaon%h%1)fU7zj2*}MqhVp=PRq`o0iZ(*j44NskD3+(4o9+$VzK=GAluF7 z@oV=$8+aD^*ivzF`uBnj^%yOlp|Uv=vU2}?iS{GHL|6fkwK-kRj%|NF8Bbx-w09F9 z-%#f4qyCN^O^=O(g(7>6dBtVui`OpoFkp!*ZlP;71Y+j??E%!eXft21CTG^3()M30 zkrkAm?Iw@4`}LEH02HdRn>ANa-5jiE%%S?CR%4(j`#B|5%Y2(Jf0~-&^>Y@6@wI}Y z7U)CA${HemJpQF1Xs3PF{}syBvS{UCE^d1FR`nd$A;TnD?PQ(u6?Xr`(N5W z4fs{lxcW+e2`iN{5p7a4P|WC=Vht|r_kV!@RFoe;mfaOdM}2v64inJK9Q3=3>dM%S zz&_qS*13srlaaE$rMV65g=A{q1Yb8eLZXMNVVP3 zDyh!>H94Mm`gJs<#&yg0l-k#zKom=_e>%CRICmpI@}K2U;7)rW9&cj|*d}lH&l;zpZGv|dPC#g@J){o_!KBq~h zg-A%kF$M6DkYhLF@BSYEBt?pV2wo)13foQ-k-!23c7MBAEEeA_>LKELwsmH8+r9RY z)v_>|Q0wfw#V~V`b!J^rwg|w@^$?t2Us^5e9{B;LMAYbZ+noak-CIvBYmV5AI)Qce z`MX8noL7C1U1U>C+UGW-{<-7$Ccc7e zhzasX%wO2-HAoV8<#m8f2}2L8J(PdG2Ctxe*$xY}0P=n6zXspGkT1+fY~d3Sv@Z!m zK7np~h2ZNi#x78mK7}KbLc->XO`8nJm|x@?8U<8-!y(a@MwTV-^17Zo4>X! z9|a4S@u3Spf2B*A56`|&H67S9G>4oXdVq7!MZ6tr(&_g41M6#xR~OxY?zK-nU^ZKffsf{tA>d+iXSJ+_?^}bd-G;0inMJye=`Xg;=uDM@7EibGuNB3 zOUG~KFkr}k13mo4^XbFmpHoEIQ~Lj|dPgUR*3(mKJzEVanc!(<=8|O8ES4%#gu!B> z8)38c1F-rwpT)jg*FGB3`5ZI;?LKv_GqP~qmZf}ni~KnzkfD|57dU`p7yb5)Apbpd z-7&Q9BGwDv{Z9S6)t66sob4qZeP8X;cMCH#7jF#2Ba)7%_yP2N_{e{&d;dKuH>T7@ zkR(d3ev7JTc3f-gZ*~@HBK@>ySaRB5gC!#*{*V7g<$+{LLxCbK;B~mBo;SAY#*FL@HKv+t0QWUguGsf*3G{wAs$hoE?cI{I- zX#uYXK7S1)*>H+)5P#kW9DQGdS99onUe3Rq!mk1IG09crt6k$=;Z1V#o0U2B=WdK~ z%d_k9D|(q$6%%8G9^8od+KP18=Nu)LHVQ5;araQrDf@a3dq6mbSEMR-za>mK5FC0H$tKiotS|c zFOHm{#LK9-m>I&rrGp5=6P&CXu|Kc66vO8Ov#h?!vP`)_FQZH|+Oy;bs-RNA^=nwp zz*?w@Tv!#v&Jw$M}x$2bRF=P<)a*KBL4zK43Eh0uR|7 zZ5+Ud4e*a|gWDPULZoLIgcBM7HiJx55yN>f=by+W(O(dP$rOCUY{uVzAy>cMej5Gx z_I&uur|Uo8o{!%Bd3}ESGeG1X`;^Q%%H2aBbITqChyg}HK<5Z}kj+{E5=XT4h49As zvqUGq%@6@LbY0$J>N9a5dVq=OBWGk$_Y0{>u@ta*!m{8Vx(j3_#3ZCq9*%xpxUK>e z#lyskBnCv+ec?jc)A*n-5OjkfB>{Gj|J!8ZVuEfE^D(-Iu6S8WRP)KIebn40&$EV% zXm7+-9j-K0#pjBb@7y% zbi-6zVk(V)Kru%(qf~}wtqj#*)U&YDp=ODbnxRkR9{IxBu9v0MbZcTOE%Ao6w!>OS zCWiXIHo{mcsx`5dhIGSJ+hMBXbyoW2+@n6bfDD#rp(PESW5&i#dx#z37dLX!Wa)0j z?;Z9tf16V2XHL@^Vh8w~Ap%5L03ZmEJ2{YL#+XdyKoJ&&R3Hc7m|7cLGJpdxq5d2> zVEhOm_?HY`Yyk*{)Im0Q^=*dj8NkHBws1~CFkiR~+x+c;2kkrL#I10iel6TW5PamN zW-yqw)4QnOtvAAbIeF}j|Asb#{cnLYJ9_NH{t+AZgF7_${zD)B>n$GX-22aXK0%L@ z{$PUpi^rqI0u6fi1M8`*t7ffhym1^o2XvgQTaI1ffRzUgUR$kSK7aiP{X4{XF(W+_ zlG=zqNp?oDXOVij;mr!4dNhD;!SH~gixFYPBk5$lA$48IkZ;gzhQd4Iyz{6%GuvFb zf@9<=Hmh6KHAXfnwa+JNihwE8{hfPmuN42(sAiSn8Zqdz0s|IG*0qmqbdCJDX0cWy z+f1;qBJfic2r^=jp@M991q1fUb2z{i`9>M?R&-=s7|e{zXTR_V*rq#K=Ms?uhn=@B zweQ?YD4!vvgZ$V+G{Acm(G7wB(jZ(#{#;YhWP^T>%*c$9wPzA@y3Xq}LTp3ZXnB-% zD=ry3n!m;}puh0|?dOZ(`_T;glg>H^QLV^xGV&Zdq3Zd3ldMl`Q_ zCx@D4s@yIdLSiHL0y22(LlR6BTw@}cI$}D(bRdH)EYpq(UtWly3=bi{E&kp&(Kc(E z&Oc(Vs|;-Mi1RP+Dv0_TpNRoee2>WT`Q?RRWXfj97hh1@F%`{FZ7%P^MvkwClWG<; z(d}LIf1&jE4}U-W{mUQ!zDGa*AM-vOod^D>f4;SUfB5 z*rarD#f~27{^4-yNaKr;BKiTb^Wd%mHc0~NEx(__)CzM^vOU665?ib1!Cl@);7S7< z+5XGvVrF3S%bT3^=OsnB2ex@9I@WTNKKbk;CW-KdZl~XCSs&p;_L)o%5hb)~S+{tO z0tV+EFVXFETh_Y=5A#liKS$`lzf$XL(0giG{2QPgx$lCU&F^aB20|HJ#;w$XC7V>+ea`!0-cLt zZHBy2M4Ya9vv9iPR~N;))Bu&lqdgI3UD`h>BVm+0f^O+3eifnGLishsn|S^vp1+Cb z-!77z>XINz8NI>*OFCrAZFFd8=>k8Kg@!Xo2+9rm*SD~tC{RTo6&_`Esm&e=cngDa zq4Af&CBY0sMtxP-H!m5x3d7f=&Sp*ENEnM=r}G@b*s_KT-!Hs7ADV|Y!lleZ(=WY3Q)oY4`;<|eYAn@t zrsg$#bSW%%Q^zwzEmSk(F2fsWi>!47_>|m2D6-(l9Nq|@pPmNhGO!gMf&H)3S0bSyIn+4m-4cOkIKD}BQS-8J$tb;KgpIR zPWM3kciZWliU0nvR?1puN1gbef9C&;{e(P2au7akrs5L!B^nOcWXlZ zIb`zW7&>5hj@Fnb`w^CUs=KQrR2$Ja)uY?8)RZdDAQDwgY9ngxhl?-N-LxS`{sskH zUTQk{s-U*DldtNwO^?1RY6Hj^ntLn&i~?W2Z_dFs=U|(2uv@_`-IXp82#zyuQ21L=vNk8K8P zG;w(QyV0$?`b(nZ6%N=CJDlmu?QmzwRJJS}x!iw=L$~?8imVa(7EQ9`O(GwGLZTm-IZoH zg1gQgxer>3xzi+c%U!* z-F!cLgT4*=ZV7$2a}>LAE;kCmL;>h-ZH_?HMKD0}v>3`po)l37DN5V1zq5@tHh7~2 zc0vm@5Zpj;1i@?f_eKc1Lpq;hmIWGf_)SnL1ij&8b!08isoePw(dyeAVT#lZtj zI|bEMS365Wr8JO4q9JyI8r4s;@M+Y~PO6_Jw;OUvya`!bdY4hH^a?WGvhiZ4(vril zl|(9yo^AAOzMk!N)&>%U`DQD|k|n5YCx9T`U=uvh=+K?ep$+Udup7ayCU}6G7nn@X z$JEbeST;f2Bmq;f&B4?ak{izf*+6Lnr43FJ>V1UEL`M+fI)_8%ck>umuw|QG}Ei} zI|BgJLze7PIeTVD<)WwApprlsEq*Oi1_Ys~u4iMRS0iysCKs`i2Ac(cWO|?SN1-khF3H^XpIX*02zg5%q+p2Rsj*G7Td&S)a|d{H2l#W{ z0iKnY0^C@PCyEV;s;g-!&3;}h1*qHENl+QCNy8&J)fai1M zV91dB2m(Z$K;*CH@4;O9Ii(F8nW%6&_0g11pvZD8i?+WIxyuj$ZqWcdSyqMt`5(qf zv&yNE^&_N6B)+C-iTI!*04aRfDmC~XVi!4b4n$E8GCqaZsrv1o)_Il_$oMgs7FVc= z*6a1nrTuRzgxIgl<=ug1n=mP3*N8ZL;H$z$NJj@8MKYPI$>zgFSQhx9phMu|fB`xI zUic=2jR+|iSXC8(HuUGi!Pa0 z@GP4_CVi&v<3Z@^3|aUK`35l)_o@ibC~y#T129Gt{)6ilzMKrA?T?1jcbn67m&i*N z`3U{H_tgL7O?~K;i=|Q#TDK=5%gY(b-Qib~-uO=cYBx8=cc*2!t=WXW>UVXwpUbI> zW7exW4bp`2{RVOQVy=TsGwYx|U0CTrh>NIw2xH#4(18#aeFh&wX0+D25mcq0%ahP7 z{hw9(byw1_2uStS|0=2a8~_T+Mna$O23QJzjwe9CK;s2yyZ}kbTI&V4L34Uv#Z;68 zYBdpKlIxgy^c@)7||X^Hm+5 zzR`M5UwLs!QZ*&BGPKxq_4^9)|-% z*F_GXB(~Nt0q^D>dj!-6zeCI}gxrWD0Kpf+C}2LGPLUs<6F|^+9~&F)#}XXf{nIPU zgbGp<)2LNsl2r5LB`&*ME2WlJ(V_{w;>#K`BrA;BzkJeGbHu&z&5|GGn2=LmJR=d=v!80XU{6 zygXP~zH{V&@gsoXU&s%{Ry+hl>L44ul71dc9Bd0*gJ8aJ8MgV`0}tAF$cbCwQubQ- zc_8@6OU+;~Yo~=jbZABb8SV_cY?0L6cDlmK)m`zcr0R3UD<~U@XTB>w@{uyMvW z&UjW3qN7{$eEu3QQ{SLTB?;v9)koaB_Hj)M;7K!Ef>V zN$}sAb4YQd&_)j(nx3LIJqfHjE|njp^rFRXt!pCSfW4?Bc#?vtqc6VtHcshrNhZ`# z7~`harz!k^m~{1t;(Xl$!4%)exjgu?`tsrj^3mb$upL4aVid% zGv0tWyDt{^Lpq;dL}@knkDKU+yq)CC`ZsYqe~jMFZ1T?;`rm*2!;**LgQ(msw(lf% z{Y&Js2p_Mcju;_%7=2xtI1Ww(Yc9K#dT@i7FHm#PYgxB63Wg8I_X-B=o$qJ!Yh2No zg3OUEKJKJto!boG%YDr_@x_CO{q!UL#lIswov|>lMly#$>cu6&3`6(YMUKd@$e;a2 zZt)x~=|kUrj6S>A>GWi8P2fMA-lBo8AtRKj7EV^ts>yLIn$IC*$IS-m#?xjp>`4wX zDZj!rMF$|}dPU%~VEl4c$uF2V>R@VZK2Ht@a}+Q*_sqWh)`#}m#GqPxWGPl=c}`FT z?hN7LyCod^{4Zlnp#S)e*i?8)&#W;F(9s~#&1LNvvdql87@am?4TdOOB&6OvbxDZr zTMCv>r;%YgZj6-Eo_%sc^QkhtsMJ$2RM|IkUw7?cWI>0fR26$)lF+ZU6P}>2m>4GO zOOVwY_2Dmo$%JyVzAi_tqSwk*c(#mDUboN+J?r*5Ig9EVq-x(7T6eu#$Tn58?q|uY z%LKi=)kAoY!n~F87+91?@$Dl+m&8F2B8{akeP?Q!QLIq)^C* zo!ZMaQq@r#Te)NBg*&B-?k2}Zb|Yuy<*cu};3|}ybH}-&Hq4HRk))+*=&fu%X?YTr zp|ypJR(!dWV8DwhpM1lRRTxsYc^NVQW6&-cP@SZ(Zpnv&s5QA!>Gi-QSTO9i-;4c7 zLspI2Sr7hEgW>L%gC1(zrObXyfwh`-d^G8vo}7#i(4^lv7&yo1VBAOc!6`iI9Zv>h zcnD9dmi6i8D&8=Q{BeU`%lZf(!iMGVArx1*#wq0?;jdAtWa}|4m4k3eS@kybCujWr ziQby53*}cQoTC|vt1kVeuku+If++?zy;Ti*(8lahY_Oi+L+rva@A+xVdOu44zCi)< z?|ENN#bAm*KLz4J(T%_*VWWM(xRWNOQ~fr&;EjKaNx|g^81YYFR7^Ved$xnm zaAkJ-qc`WN*~pS4Jv|%ie1FhCJU#B8^p8$9;xx;$l%Y`@9;L3+o>g(OD%Dy!)1}JE zkRQ*e*so~gzV^~HIR_WWcA<}?JXeAI6rr4TM0izm`XE>ext4w63t-T__kEdK2qwb$f?z~ob} z2BeLt)C{wR=$dSPtG06ks|~Df6;>NeyXyC~xA;gnc#nxw9n+Rn^7kG?SX*N(_OmX8 z$bL*X4Z@wZqOOBfSGQ_fHqc@o*t6@U$c zI8v(V!S!3PO6Dq{0eSsgd;T?eRodkpHn4>Nf!(pxRoGzQX>GQM_r{z1k`YIH51JhC z^4W!!_17he2*vnotRXLu?HYp3+B8G8WQ$PtPLnc1SJp(Njigy?YKsgLE4JuaZzDM+ zwids?*J&d9+AG;Y#;zLorwjVah@C`4Q+W%iZb9JU0Kj#URv=2yRK%<|4rK6+2ieTZ z5g*;z;yqze?bT{AFWgn+(7SrPFx4E*VL!I!Vs6|kKZtwf<$OltfWGSYwRg!$n7Bb6 z_1Vt|x~x(JbdrwQ##;DGR*}`skZq-@t(=3|rDb_8$gGU2Y3d^enOx=86QSF(6sd&@ zg1atFOVrmU1h`@daODIR4aRbeUGLls!{bY1E3a?j+)Tye8!ml9+CRKsbM-7kbJAg- zU~W&AuG(7@r7Yk2k5yN%A&E^dU|)?XHFbtAyVR^@o3(7SmR;(sFhbxt!UlE}AqToE zF@LB29rA~i5H0{^05V0P(y{YZN_yKXRT41dy7KZdLLCGctV@W}R8RPOo#&?H+V{x1H0oUhk}P{D;-Dex?BnhkZr^W^H8Kb=io* z%TqWBo2*c8a?v?_xReCDM=fiPN#fUPu*GtIcE;$-I(QO$(J?oe=g!^DkBgc4yEUQy z9CEgHAVUWX&ygxcu@sULd6f5TNjcD6;wrIm#2~ApZbW?(eX|P6qN16;iQ3qL!cq1z zN8#w|)TeO1TO!6Tg{Bg+gyCnqNz@p$Q(>2(LkgOmQz;G|^@uRjIW+0me7eGq^9Y{3uLpFp?YKeoPd zA$dpO*j)<5M>PKwAm<|{LiI#H;e+`3!COd|PAC^c@!LAxZzUc}QW3jf}j2NiH(1Ai1$FMVI6^!w$sky3O5` aoz)Vj*3;A1r~e-S0RR7QCmb^Up8)^?M)I)$ literal 8091 zcmV;MA7tPkiwFP!00000|LlEhbKADI_Fv)f{cw_utmqbBG_yZAc9Obv8Xe1N&Tisq zArg|XO#<8`3q(~7E!HX1GVcTgU5?BC;wVrjs0`O=O4`Sc5%%RzB_u7YM z%S3oe%;BR+5p|$BG%pEV1jg0q%s3rgm@V@T`T-)C*XVZJodX-*nNKa#g>+8rz#M*k zG%;{l)%VDO7D2duYEj~!+O`jaz-*Z|3@jgcRFL`m@4si{8=fw(WxNQCS7Su-|1Ydr`Pv7w0fB$W^%wXZVz<)4@rr9z*-~(6mV&StfO&$(B5+L69@HHFrP0RFQ zuy82*bm8zcxKqUOyL>)(GD$cMH$r2Yc3&Uikw0A){jttZK|mI?30Cp#xrz&ev2 z!v{p?i;3Ywmr!Up2;Z75bKyJY(41511;f{`XS_(;BJS&H$PfpfPguWRJB+zrPaHCN z?Sgf;=oVeG4$U9$C&`@ zHe$W--528DuD*QA<7_YS=yA16k0zqf~^Ba)7%_yO>J@W6h{d;b`fn-JnafDfn-TTf+8)$>l!P#&y?2$wU_m@pe5A| zBU>3;;&0-0cVAUW^=;}aX2-DlS}}yk@WBgr0{sk`-}z$$1qLwUB8g%52mLoENB!ge z;qlFKaAC>kEjuSia@jeXL+f_rlRE~4r6easK`A#A)V_i<#QF#A5gF5wPsp@ocs=9m zO(4jIQ+x&4^S;5*_oeZ|1>V=?{EI2P38;^7t|DLU8tV#cl9Au6%*ltlF~%*=uFFyM zGObD`MhiWd5pmOsblBG%C6+b{F3xkcUb+J3Md>FDm0|e%sTF=jow837-O6Y1nm;W- zAc2jEIT8xG@D%kkQ^aIMYfc#TLn$jZkf#K)CkUv3{p-zSfgFl( zIJp;iN;s;s`DVa(1+YzAER3aP>4XZ%!SHk^A_(6oe9H#4apW6AUWiQfz>F1#_L;!T zs5qY)OpOa0Vv43HSv6vRR&^zyy)8pDk{6sM!YivrLl!{P)16&8!{{89j_5>L~F)b3}ppkeVJfq{t)jfr?hunm2K&Y=-_&|+v~8|4v-D~Pt&P`Fua1)NAM0f{ADRo%_l35Q8Sx7&l=LAy^&CL z=xO{2_hV=IN)eBgO;X4x^>S5`XuUkOLb>6n?Qm3I5v5B{5#9(;NvPJvQ*zP`Q*DQ- z28#HAWR7Y^sSM3p8LGjkAHq&=6fRC`hCbsv=yPkkUY3&6t%4vGc#8m67^b6M`K0OB%lxLwO4OL*qLU#KM**q?;6{N}1-Add$>}UQm zr8Lf*rgesF!)FW;KuiMz7y)#q2cpab;hFeQm_{iT&^Ay^t!-RT0|myE_%5`K$%6rm zKSc6k%K*k1v7u$W_&!J0+(6hy77tDV!(BKOS?ukB2drCY$E`4)e#yf^!1%~Y%|S44 zr&m!Q&DT79Ieq9&{s9&Q{qKQ2KYZwe{vn<8gInl&{~`DP^%f7r_5L$)r|@CgA52ky z@o>0Uz(MbBU_O-%)vOJTH-V!c0Ubx{+_6g((DIu77^%hqy!2~*z z#p;$hLePSx*7{WSzf3o#I&An{T#ePz2Vf@uQ^XjrBl6}~v^>9-4k|w&n^Zsvy zeEIOt{Xf6``R_aU%l}dD!{KS*fByBY_2vGTx9`WjUxIh!!}Y!Y@$&As|D)5=$rU@g zrTeGDsW*yPgb>jWke&v&6|hMVkT3bY3XfKBdCB$|&2Vh3o(8vh3xO*QY-IT_r;C{n zlV4tEr#~yn(>;*II?*wgi}cB79}!NZH*`DwUd#Ll?z7Lt@et90HZAiSxiFx>^;n5+ zr`s~$-Ft|2D*QQy|NWhq!$I$QvQF_84lC9%X^u#e zaSBYZV69TSt6OH_!%5brd^~4hKT?D@V%UtA7%C%tl9B@I{isNYj79K$h5TlKUeduV zy2}gcok=KiwBA0@2);k#fFF0U1oRPja3QAr3)}&R0dVHw1-5($0?5(17}jRUYh}c# zn%7IGOOCoI)}=mBNj%yUVOAacCuAhdl83-4oy9LBR9h;)f_Rh9-{kW*`TSc(64fDy zLPjr9K$8KPatj>_TB^j)WTCS;z!>Hx{i|EpP!_19kCKS8ywqkl1-u18xzzZJ-~uCx z042UG?VA-CLmVc%h@ny>$f&Y?eosj7*yw>75I_vIT;l zaAPx0_-tzCjmCxKp+g0r(SOl6L?J^8o zs5}R1EgzUHthtnH_kTmciRui+1lE>b`d*T1em@de+vmvdZ=&Abna7oF{>Wfp`e!swWaE9#S z$a#7PkOL+Td{6v1`YLw=mI@|AMZ9`u%K$CmD$4nrJ7Y)uQ62`Q)M?ym538df#7H{aZoTXRrd-`3vz#y z1G#bDxdPvLo@Id=gwW#Mik$BliWLrZYH_>Rq3lI%y@c(m$F;C%q#+FYUF1Wx?8=Y) zc4QH|B-aHVS&7)GzxPjI=c-OUURsPb>=ek2z$!^7 zp>QcyvY^8UUU0^)!1t1#ZPgW&Vnz+2z!^v0eQxoZ({=XE6j__WGlG~+eN`hQNk(pd zBua>Ds&q_^6|x&KGFCd4`3TwfI%#(yu!<{vLx*kzeMD^T#F|61)9G{%_7Xm<9mPT68wLZQCM;^^7@m)a0&M|mEkqg(D zC;O3>dZN0kBUBsFcdAFXWvNM3oJJ<9oYY3t+Rv6>c%r&#Lwxxw7%+LMY2&Me+SYcy z%G=i6`YNey07{|j(ZHa@@WuOPA8fM^w%G@}72Hx?=_2HHTy-DpShWw9qZ9j%LHjM5 zU0})~Y2`tmUnP7t2MBZHsbzQral0_EZD_~FW6SW~$ewElmX-cqE<@=X>mq-C^F zLoBe=G3apNg=5bpv3TBhh(QHSiSnyHtBsdiDlC}OS$?JJ3Of3kAqDn??g@gcu2qV0 zahW4j3BzMD#ZWx1rj6sth8iUQnUP%0xvElHeqaKL>Vfn`(#JNFHJUuU{axr*UHv6c z@)8B~4B3q7%dK!{$yB@~PfD=4Aa73&*xpSk>339Di!HfP{H>ylh`-e)Au+i|J{tMh ztb8cKh$9EuGbnU-IoU`Mkjn*_pmt;&Sc9Y@Y$g2{XI+GVIdh_4>xEbh&pz5Y{fPW!leA@>(|Xh4{D77bI%S$yyz& zrZkM>y($X15F5hG&XM@{K=QxnU$dYP2^S)Et29=)^QFxN_ZNa-_poKU2q%86Ix^!j z`*b!d`PC1rU6-wrVtri(NgNtqw_9FxWwvyDQa1)#!&lE{-^k!aH>yo+XbwAxjg;wA zPm9K*IXSHZ?3Eign>FJK8c}smqR<8JA?4XwWz`|CUq~00)_hS@TRE<|QEiQC%TsL) z$TlDwf~@Ke&k(r~1`hhCif1h&v?bVGZL`7O27ep;ZSZ%O@K<#Kt$?tP`AZZVk6JSn0EQk1r1e`gzQZ16@4?1UC*Ah?0x5P}ca zuJ4T@aL$P9BANvnefUjKDFnUYr6cXI?xzKN%gbjc1l%0ViPo^AB3hMqlH zn@A9zH(N26EJ0;E2?Xf|o8*B;hwg+9ZD6;7T>-n@jv{%0nHLDpPAA0Ap0KQgxJd%K zV4H)fD@v4d*OP4fkI+NAl@5ng6)L^eIuZN!rD&L`Ei$QCBA5;SLt^e0H}s6(WP?s z%#O-MPq9E1h9`Io?Y*-I${n$hp`to#kh7*xRlM+9K&|-|4D#h8+ki?qR!SPk<@fng z=sXw!#CE_B;bZR&R~T#JmcW453=A-v`EbT2kRR11inhOD;VK0NLrDWomz6<)|A#Wt zwDNgw>MtZY4RKA;5{n!K6rnK0W%v2M2grf8m;+wa1C&kSC{@4xYMptUI9>N+Fe@H8 zO0-_Dfsr;M_Wla?epht`ds4>2J7FusMo32m97T>}l#|Vei?FQWhk_0a2L;q1Q^O12 zgs>4I1trD=a=|wc=C_Pucu$uA5n411NSyR+9W{?J4hG6*CU|r%W7=#kX1db`a z%JP&NZ1&#DmA9L@h48Qw|5N%##T{jI)s@yU1MrE54tS`>-7%RuSvpBHhmYf;OXlS~ zOXq;HS%=@9hXc!p+?Et9{0)Bxi1Nds4VU=Pupw{)V*;n_2h%MS_%w)oxQ5ero6}X7 zAPW}x2>jdk#Q*Hgd|;Pb5Rws6g&megfS8foFr}3A#>17X-P{=8oivxcR$y(_C9h)Y z;>4S(L4!1*{J23}zF6oW)66<(PZyR35aJ^87{Y{gE({>VMSnmHA@jA?h7n|?e^ex) zas7X+^sjaOV@9d&{jZX$F90B+tR?jMVSuIZ=R^WH3^Y-ICJK;{EY*#*V&bl#OYW+e zih@9`CSpu--ICO@Apd@MQ20Zk<39C?8z%r87u2u`G05kOCxDjTG&P6!lcsK!%*hSR~B(sUk%QGPRnAD}-|e zDN&qRc+7FKfa5^h;3`L0YX}?fT#r5cYGMEOO-B9AHfhAAz=_{pNtH^0GUNE-?&zM%B!~jcAh~O{~F-3kr zU$$6n{0)NzB`%}>u)*6ZWxg<(;E#(R?R*m|n3CUxAuQV?vR#4QEDRqE< z8JNfo@FxUqOCkPOswCr0NUBODJoJ zXMQL?ib5^yYk6#r_)JnT)sHxJCHbSzd1VWwBl>o`R^yre{1$3#hF3{?5(Ix9fNDxU9HkZZ(~nvm&(sldd^~}y22&o zZwA#WN40jp5ldqmc<*t!+E3oSieiTah_?<_rIYZ5Er^``GoV~TKOq>VD zlQoxJNhvpF?~X2)!qe?T8i5wtTeE-<19I3owz{FoT_>=V95E?m+FzVi?b zyVyY&$=>R~e>lBG6JJ9{C{rz*tfEzu-!VsZX7p6Zj+;%=ji*iLvnM%7r~C@j6#W1( z*Gm$gIpdeJN`6DwmM2py4<%x7;KG0c*V7yMwGXVdryD8lk)>Gaw3Eh`qTCt6#iPjs z`|K|hgn|F?4qJprNr&bH1n_W>=;pF^3|V^SU5ri}um(dEE|O9&Fj|Lti0xZSmQOH7 zsMe(17%8Vc8*)PP#lwyak0-f$DuyciX6~!53yUo1(3GlT?+X%YUpo;AxV;>^=$NLF6*}zpIyp{t4N|pl46VA1C}f+@o?R_fC8SWu ze@(@4H4;~rZEWR`&}Z(D7L6pw0#H{O$ zq~%FehSnBNiu6SkL4y}l{&>TXRTxsWUJx<>rNAngOqHat?(|X^M6HP+rPnj2!Ga>E z{hr4nHCZ)kXFd2w4Tg~~8{XFzq0Ej&({$k{nI0WZyC=uTlLI*IcMb;j5j>dmp>=Qq z4tqz_!34Yk$7aj?e03QI%OYER&}*3=!F|}U_qAY7jJ@64r_V#+}qDo$9yN3Ds;YIt7h^(9(%-^>o;nV(s%Vp#Nd9ry0^(UOt{*(e)ymRjAESRDzu{+Degx?Jnx}ET z_ui{R?3+!RkA{VKBHp4GL8DmTMxt>oI}e2J`4}DuK@WeLOFg!c-Kri2|R`J z8Lcn~^CWb9jMc6r0iKcCILPxM>Y<62KT&!+f*y=(U-zK6 z%4zx8Z0o-A>WC1SeB#xBv@w;MVOA4elg)3{c5Yy`fz_?TYJ+K){l4`75w@#i z+LB8C+GFA_V1=>R&xQ~p`!NwT2v^pMh7M9)-KuHXK#R_exz7Vy)*|GyjIivI;i(q# zHBl8w(dQOh8Tm-`S5~J9q*T*`>z81q%vC@GviiC9{7d6S>5zBWz*er3+!ae*h7B5# z)@F%#Z=$)+8F7@C{fG}<{$zn={dJo}LovRIHRL(898IuUd&EF3J4L0vQ*0MimDLex zEooMo+I)BEiY+?qZDc>`madlF^DUx%<;6oGV^_`lQziXn#7-chuDpd*w;*tF0^mAH zE086qD`Mtr8&L4t19WcYh>vPt(VnoV^2VT;7j7$Z=xsewm}-IMupe6sF*o6rpCr8U ze1_2kpfCG<G7qpmDSe?Zl?0_ zHC>;O_D>(zT;0pi>~z>CnA;QARe5unkmYOtp{n;9lGwBZ`(i?fZZLG&rN)Tr*E$ zr<&NLz@QuAZx-5s{H3)8* z@Iv6czU#bZEY6Q{d(E`?)dGJ3h?>I=52EW z->iZ%uc)VQqBgdmaHPG=Q8=nP^(mZ3ljqo_&{RM+kE@V{Z-v4L-O>CZKo)S!Vee>g zd~|fuIXoE*TI|*MJMhe5w|9IzIO%kckB>Y2@iH>g>kf_v$Nj^>amzd#eGbgw;h<%H z;D7sSf)At5Mz`NTGH;lWyu)DPECu2tbUz2s{)jMFJ<(5a&wqaK79yk*%EcM~Z4Nt6 zIVz_}@?J<&>oD4=^RiUFxU7O>D>`HS5SG+WDNM1ERunMFMVb`^ pH`b--lKiIGftX#lxtg-GTH?ffdb)Y~{{a91|NroJ@1MSm0RTUIznlO7 diff --git a/build/openrpc/worker.json.gz b/build/openrpc/worker.json.gz index fa6f28aff51de3c425348f18f3df7cfe1096b7a1..13b0c1e156cd46a923264efbeca28a822c9fbafd 100644 GIT binary patch delta 2521 zcmV;~2`2WD6qFQ@8xG62K4fniVHG<#wFN_iwhWA$nT)58#1De zZ_Foujsph4CH)B&^1CSSo8A)F8Byd2z0n`(nAp%55Z66Fvk(Zu#SwbCX4gu93Is#4 zan2M7M%+e>R85*DL=UJ@A6>KgHM_mN#TJ%<=Mgc(eY_gL%vld3uNk(mPXzJI5(Gjo z<7js5b4kM`i@r`Qv)@jwYieZ@W9tw06g;AT_Zc3E=euAqo}cNa&y<0{{dTL>vhbI{ z8|hE)=N29c&c}bve8By8^BW5j#|gRY+!_C~n0{#xD00;M#2taR!FTf^Bw}g|MTdUoF z>ZEonhhq!v84FzZ#==xV#1yF+B*!vqg{VXXtxfW30du4GcF{(JAhYtb?50L8ImasV zh^kdl=k}4 zf>em>ums{#Ho-BXq!MAvc6_*hlo%JI%j5S@nW);0tKGN*bK_20?{lm;WyGaFA=-v$|d65QE%K>Z*c&ubZTyc69g3qV9vE`*QM7wGUeRpoeC3x;ckrL!zd#q`3)U zvb|6f&4O%1ra*VyWH}b-cH)L+5tI{yr#kI65FZeB36aNTcyDnUH@CtsHA&gu8o^EZ zb)UFR+>rLHZ@&YbObZ46TMXy-RUCRbM}|YeZIJR3q%_P`N-h$Az}W&41k`DK`?lGi zN+HV@4+E%`v>?pk9@4lGAESqmG$akYg(kp2=xvBPx-;}J1s3(YX1*B@NkxPzS3s5} zcguOE)-e7}fpd~VhHoc@OPfvEfyhEm;{5_hrE1G1TwW|fdE>jF3dAx7S&vNO=jMx2 zum7JZIA;z#h6m_>nMGg~!j?F(zx5D(xd^|l_G_beUdK}V8B0BP7@m@j9Tr0NTHSZM zsfBzwC7pF_SjUEk78{=AoKIYU3q-;E)k##rpJj2c|`g^R}9oHKz-Ant&B9#v`Xd0swZ zkmdY6!<59m2y>GRq+DLJ4P}Pdirz!2M=!dh8^(;L;>AB%8)85mS;2oyPGvncq{pfK z$h|OF``|}^rrGxZ@7(4J;)|YZIvKU5Qg$8`gd+{2Gree0LXF83I(L96I;L?>ULEQE z4?=|4%AIUbT3YcI)LP3FL;ndkwZbIhS!eW<*g(1u(wUylx}owjm{UaH+|+RlZhO*ff7 z(K}eXLS>;*pwRM)6N_qUV)vFb$2wGoFK*U%mR`-BrPFq9>58iM?CT0J&*2(z?rx~+ zN+5ZrvTNB5c|_bvq^7XoXCe`p@0ts~ZlEuJTp=|j`Y$Sx1f?jOTr-c%kRo@02pYo! zl3;V@NTfK@N|a290{uY|d51KkI)S^zI$ve|B9@q!R39tT^W%(p@ zexM;Jn{49z6s~;s-&(K`CNqjX0vdlIXq%62zCKpW$!h;a}>_kG=N)W%q7a2 zFJA>;_Hu%;#h#RXeq{-l9gum%ze;cXjSQim=9|K_IQDGjT=YQ4*Fme*1!P_p`Q`9y z1L3Jv+T6cLN@l_9nsXNH-hT=!{(Pc;^r8$C8HvUp8&enuo?JM|iM>qqVwD#oVvtqe zugiKR;<=yc$~+7|K{j<71L{=Bm?!5@-xsJ;=Y`(Fdx`Hi9PCeW9aw=vxzYd{?&^zagx(o>P(j*m^Bk&d?Q2nTT-0aAj4=n`84oh!WnV=g>!&}fE#2>Q4A zK}ZV`G~f={=*`2F+EYZCL-iE(7+&BiYU^*0ePZ8%vm8Z*&NpEWBIvX3H6?P|NHWfL zE(6p*+EE`g)dx++7d*|$Oh&R?(hrQFvg4YidU)OsTjAKrdy;t*Tea`V=5^cM&b!K1 zZ(rFbC!6@4Q1!2Xf-1>COSnux7J4=_69>)aoMjI*a=CpcQNR>+k(z=4LIf~s-uNwKbEe=?H3sHi3;As%6&GXzjVrJf7nd-$kl#VyZ^(!? zzA~SGI1U&DSM(=X$nT=OZ+c5y=R}bs^hSTAV`4*NKwS6Y+(IA(7f0ynnq4aaDi93G z#syO#7;zghQZ;Fs5IvwqeRR#{*X;K87F$>bo=3zC_wi}~GiN=Fyk^+KJ`uz-OArXX zjHB7H&m|3)Ec!aJ%zityuBnwtjIBS|Q}Bp?-e-6up6`OecyX?qK2Zh&_uH*j%fg=n zZ=^rHn_GA&I3NEt^8xqI<~J55juVna+@at;2t_rXF>xhW*pD?WFqZ&`h*&WqGlW=J zg#;`<&sQXF?mJ($@X-&j6;4j71cL7i3^#KN3-~*r0vy~|A^^P^z5tsj-4af2Zmo8I ztCQNT9F8rtXDo2t8w*nf5mTgQkQ~db6`~Rmv^L4_3z!?dw~IC+1eukeWj8f)$vIY; zM{Hf%txoSIV`e3%(aRMrttg5~RlcNX0t!)dqdUpKRT_6h+q#O%q;#!A5glN3#)75mf z%g`LTeAM)aR3Mtfr%hk*$Ju`-z#0?&e;>QOw_QD_w6pLJMn*0;C4GemD#KNOI6`km zoV%MW_zFR`!7gE$v!qp%keX7|xLGRo-o~_e@Pmx8>GlHAwR77}x|H-9V4y9dZ zbWhws>(5J&JP>xovfvG;Gf8dY;(o7V>4vKZ;x9j3f&_7|8Adk@hF_)m--qyjTGP%; ziuQ`Krv+z6&i#CixDrSWVMQeGL)3GrTT(Qv>w?4@6&WJ2Yy|<#qSxx6d(t||RtKN> zJzN=IprlraL2f5?Rl(PPP0~<1x_f0&_rcM9IeDns2d#b3Lo+(XWPTj7_Qr0j2v;HLb# zkK871NPE_|-+)f0g#!OAhV%O>4xOAM!=d0dNO=WP8s;h`7YX2hYyk-Z>NLK7-Rw`L zkY$U90n|!b5aw_XY21j9(L+cYk_O&F6JQ|pHbfoW8G4uki~3zN-;9T(B0`lbAj^`w zdWgPUgx^;Cwb47TW2ya&rJg$sPf5oP3n6>0?%8f? zAzw~OXB`{XvEiY`hTWX=iA!*SNchSmkn)9;YtX;@>^(hvos--vDcXmeWPSKv8@JlH z9kg-l<(#%%LRf)W)~0EoJX5Bj=91M0{M{$p|~>!~3SW+6PN@@L7^Nb5Ps(+Qz>k8~-+^awcIBtHhft z;A$a%=gJ&ck2(`H)Pd(-3DQ28^GsfQc^194OSN4(G`n<`Q}j~pUek6iG;O-c^pW1d z+7&7bjRJ+1SDaW>Qxm(lq&e21GJJ8fzO(dt?kv5_EnQL7o_$>b<~dv=&fN`FT?r)5 zRCX=9A&-bViPRJp{7fVQ^Idbn*A4WUE2O4>ME^x4lAsi2lWXRY8B*j95J6*jKoV@u z9ElW1T8WYgQJ_C4BJYr9R3~t^SV#7I-khxqwLW%FTX|<%#shorIN{abvMis3&JQ#M zWs^;upTd>T{#y$c!emCVM?m8*1a0%t&DY0@Ia%$$h`c*D3aca=#vpT95QwX_m<)%1 zF%Q%{%Z^E$yJy7|eZ}9VZQi$VEnFesHjJZTv^X04H z%T7)(w%C)h&#x@uvI8=Y_}|kTeiczB zuS7ieGhLa7;U~zZPGdlw3K{d{9P0Z5b?Us(TX-+={f2}6Nv;DcP)HZ(N9t|!&SX{* zBykO>gM=Pl0#kY_65R2z=`+&NwjJRhjw3)ya1dQ$YoH6I7hueV2M!v|5JCTcHa`ey zA%X_n0UNz}m{NO+D08Tuq8`HwTt#jD?XgenJ8+hxsL=T)%s~Ww(!HicP8&(a+0JEv z`bRtJgQohR$@qf3oXlh-%O(B52r4_SS*nNU{je2|oxCTRH?dXwj%;4H-R+!JwtD-@ zJ~`RMZ-lCU1r$_C23o>pve2_VnVC3fHs>sRppnb%JBb3OsEgDT1Rx>+F>}@;kq8(I gPiTIT* Date: Fri, 4 Jun 2021 17:22:36 +0200 Subject: [PATCH 30/90] fixup test node builder --- node/test/builder.go | 1 - 1 file changed, 1 deletion(-) diff --git a/node/test/builder.go b/node/test/builder.go index 0c704bad3..8aa257cc7 100644 --- a/node/test/builder.go +++ b/node/test/builder.go @@ -502,7 +502,6 @@ func mockSbBuilderOpts(t *testing.T, fullOpts []test.FullNodeOpts, storage []tes node.MockHost(mn), node.Test(), - node.Override(new(messagepool.Provider), messagepool.NewProvider), node.Override(new(ffiwrapper.Verifier), mock.MockVerifier), node.Override(new(ffiwrapper.Prover), mock.MockProver), From 042ac8240f8759848ead07a4838817d71e48391a Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Mon, 7 Jun 2021 11:27:29 +0200 Subject: [PATCH 31/90] testplans: lotus-soup: new images with filecoin-ffi ; use default WPoStChallengeWindow --- testplans/Makefile | 18 +++++++++--------- .../docker-images/Dockerfile.oni-buildbase | 2 +- .../docker-images/Dockerfile.oni-runtime-debug | 2 +- testplans/lotus-soup/init.go | 2 +- testplans/lotus-soup/manifest.toml | 4 ++-- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/testplans/Makefile b/testplans/Makefile index 0bf685005..38f46baa8 100644 --- a/testplans/Makefile +++ b/testplans/Makefile @@ -6,18 +6,18 @@ download-proofs: go run github.com/filecoin-project/go-paramfetch/paramfetch 2048 ./docker-images/proof-parameters.json build-images: - docker build -t "iptestground/oni-buildbase:v14-lotus" -f "docker-images/Dockerfile.oni-buildbase" "docker-images" - docker build -t "iptestground/oni-runtime:v9" -f "docker-images/Dockerfile.oni-runtime" "docker-images" - docker build -t "iptestground/oni-runtime:v9-debug" -f "docker-images/Dockerfile.oni-runtime-debug" "docker-images" + docker build -t "iptestground/oni-buildbase:v15-lotus" -f "docker-images/Dockerfile.oni-buildbase" "docker-images" + docker build -t "iptestground/oni-runtime:v10" -f "docker-images/Dockerfile.oni-runtime" "docker-images" + docker build -t "iptestground/oni-runtime:v10-debug" -f "docker-images/Dockerfile.oni-runtime-debug" "docker-images" push-images: - docker push iptestground/oni-buildbase:v14-lotus - docker push iptestground/oni-runtime:v9 - docker push iptestground/oni-runtime:v9-debug + docker push iptestground/oni-buildbase:v15-lotus + docker push iptestground/oni-runtime:v10 + docker push iptestground/oni-runtime:v10-debug pull-images: - docker pull iptestground/oni-buildbase:v14-lotus - docker pull iptestground/oni-runtime:v9 - docker pull iptestground/oni-runtime:v9-debug + docker pull iptestground/oni-buildbase:v15-lotus + docker pull iptestground/oni-runtime:v10 + docker pull iptestground/oni-runtime:v10-debug .PHONY: download-proofs build-images push-images pull-images diff --git a/testplans/docker-images/Dockerfile.oni-buildbase b/testplans/docker-images/Dockerfile.oni-buildbase index 306d40f9a..265066537 100644 --- a/testplans/docker-images/Dockerfile.oni-buildbase +++ b/testplans/docker-images/Dockerfile.oni-buildbase @@ -4,7 +4,7 @@ FROM golang:${GO_VERSION}-buster RUN apt-get update && apt-get install -y ca-certificates llvm clang mesa-opencl-icd ocl-icd-opencl-dev jq gcc git pkg-config bzr libhwloc-dev -ARG FILECOIN_FFI_COMMIT=d82899449741ce190e950a3582ebe33806f018a9 +ARG FILECOIN_FFI_COMMIT=8b97bd8230b77bd32f4f27e4766a6d8a03b4e801 ARG FFI_DIR=/extern/filecoin-ffi RUN mkdir -p ${FFI_DIR} \ diff --git a/testplans/docker-images/Dockerfile.oni-runtime-debug b/testplans/docker-images/Dockerfile.oni-runtime-debug index 126ae8de7..856fcc1fc 100644 --- a/testplans/docker-images/Dockerfile.oni-runtime-debug +++ b/testplans/docker-images/Dockerfile.oni-runtime-debug @@ -12,7 +12,7 @@ RUN go get github.com/filecoin-project/go-paramfetch/paramfetch@master COPY /proof-parameters.json / RUN paramfetch 8388608 /proof-parameters.json -ARG LOTUS_COMMIT=7e25a811c3d80ea3e007a54aa1da089985110c2c +ARG LOTUS_COMMIT=b8deee048eaf850113e8626a73f64b17ba69a9f6 ## for debug purposes RUN apt update && apt install -y mesa-opencl-icd ocl-icd-opencl-dev gcc git bzr jq pkg-config libhwloc-dev curl && git clone https://github.com/filecoin-project/lotus.git && cd lotus/ && git checkout ${LOTUS_COMMIT} && make clean && make all && make install diff --git a/testplans/lotus-soup/init.go b/testplans/lotus-soup/init.go index 7eada2ed6..c20f5f2b8 100644 --- a/testplans/lotus-soup/init.go +++ b/testplans/lotus-soup/init.go @@ -42,7 +42,7 @@ func init() { // deadline when the challenge is available. // // This will auto-scale the proving period. - policy.SetWPoStChallengeWindow(abi.ChainEpoch(5)) + // policy.SetWPoStChallengeWindow(abi.ChainEpoch(5)) // commented-out until we enable PoSt faults tests // Number of epochs between publishing the precommit and when the challenge for interactive PoRep is drawn // used to ensure it is not predictable by miner. diff --git a/testplans/lotus-soup/manifest.toml b/testplans/lotus-soup/manifest.toml index fc58fbd5b..9f5a57444 100644 --- a/testplans/lotus-soup/manifest.toml +++ b/testplans/lotus-soup/manifest.toml @@ -9,8 +9,8 @@ enabled = true [builders."docker:go"] enabled = true -build_base_image = "iptestground/oni-buildbase:v14-lotus" -runtime_image = "iptestground/oni-runtime:v9-debug" +build_base_image = "iptestground/oni-buildbase:v15-lotus" +runtime_image = "iptestground/oni-runtime:v10-debug" [runners."local:exec"] enabled = true From b0cb0c1a4a285a3a42e02923d7cdb11b8e7095a1 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Mon, 7 Jun 2021 11:51:25 +0200 Subject: [PATCH 32/90] do not depend on filecoin-ffi in api package --- .../sector-storage/ffiwrapper/sealer_cgo.go | 19 +++++++++--------- .../ffiwrapper/unseal_ranges.go | 3 ++- .../partialfile.go | 20 +++++++++++-------- extern/sector-storage/stores/http_handler.go | 4 ++-- extern/sector-storage/stores/remote.go | 4 ++-- 5 files changed, 28 insertions(+), 22 deletions(-) rename extern/sector-storage/{ffiwrapper => partialfile}/partialfile.go (93%) diff --git a/extern/sector-storage/ffiwrapper/sealer_cgo.go b/extern/sector-storage/ffiwrapper/sealer_cgo.go index f7848de6e..820c53c4b 100644 --- a/extern/sector-storage/ffiwrapper/sealer_cgo.go +++ b/extern/sector-storage/ffiwrapper/sealer_cgo.go @@ -23,6 +23,7 @@ import ( commpffi "github.com/filecoin-project/go-commp-utils/ffiwrapper" "github.com/filecoin-project/go-commp-utils/zerocomm" "github.com/filecoin-project/lotus/extern/sector-storage/fr32" + "github.com/filecoin-project/lotus/extern/sector-storage/partialfile" "github.com/filecoin-project/lotus/extern/sector-storage/storiface" ) @@ -66,7 +67,7 @@ func (sb *Sealer) AddPiece(ctx context.Context, sector storage.SectorRef, existi } var done func() - var stagedFile *PartialFile + var stagedFile *partialfile.PartialFile defer func() { if done != nil { @@ -87,7 +88,7 @@ func (sb *Sealer) AddPiece(ctx context.Context, sector storage.SectorRef, existi return abi.PieceInfo{}, xerrors.Errorf("acquire unsealed sector: %w", err) } - stagedFile, err = createPartialFile(maxPieceSize, stagedPath.Unsealed) + stagedFile, err = partialfile.CreatePartialFile(maxPieceSize, stagedPath.Unsealed) if err != nil { return abi.PieceInfo{}, xerrors.Errorf("creating unsealed sector file: %w", err) } @@ -97,7 +98,7 @@ func (sb *Sealer) AddPiece(ctx context.Context, sector storage.SectorRef, existi return abi.PieceInfo{}, xerrors.Errorf("acquire unsealed sector: %w", err) } - stagedFile, err = OpenPartialFile(maxPieceSize, stagedPath.Unsealed) + stagedFile, err = partialfile.OpenPartialFile(maxPieceSize, stagedPath.Unsealed) if err != nil { return abi.PieceInfo{}, xerrors.Errorf("opening unsealed sector file: %w", err) } @@ -257,7 +258,7 @@ func (sb *Sealer) UnsealPiece(ctx context.Context, sector storage.SectorRef, off // try finding existing unsealedPath, done, err := sb.sectors.AcquireSector(ctx, sector, storiface.FTUnsealed, storiface.FTNone, storiface.PathStorage) - var pf *PartialFile + var pf *partialfile.PartialFile switch { case xerrors.Is(err, storiface.ErrSectorNotFound): @@ -267,7 +268,7 @@ func (sb *Sealer) UnsealPiece(ctx context.Context, sector storage.SectorRef, off } defer done() - pf, err = createPartialFile(maxPieceSize, unsealedPath.Unsealed) + pf, err = partialfile.CreatePartialFile(maxPieceSize, unsealedPath.Unsealed) if err != nil { return xerrors.Errorf("create unsealed file: %w", err) } @@ -275,7 +276,7 @@ func (sb *Sealer) UnsealPiece(ctx context.Context, sector storage.SectorRef, off case err == nil: defer done() - pf, err = OpenPartialFile(maxPieceSize, unsealedPath.Unsealed) + pf, err = partialfile.OpenPartialFile(maxPieceSize, unsealedPath.Unsealed) if err != nil { return xerrors.Errorf("opening partial file: %w", err) } @@ -427,7 +428,7 @@ func (sb *Sealer) ReadPiece(ctx context.Context, writer io.Writer, sector storag } maxPieceSize := abi.PaddedPieceSize(ssize) - pf, err := OpenPartialFile(maxPieceSize, path.Unsealed) + pf, err := partialfile.OpenPartialFile(maxPieceSize, path.Unsealed) if err != nil { if xerrors.Is(err, os.ErrNotExist) { return false, nil @@ -589,7 +590,7 @@ func (sb *Sealer) FinalizeSector(ctx context.Context, sector storage.SectorRef, if len(keepUnsealed) > 0 { - sr := pieceRun(0, maxPieceSize) + sr := partialfile.PieceRun(0, maxPieceSize) for _, s := range keepUnsealed { si := &rlepluslazy.RunSliceIterator{} @@ -611,7 +612,7 @@ func (sb *Sealer) FinalizeSector(ctx context.Context, sector storage.SectorRef, } defer done() - pf, err := OpenPartialFile(maxPieceSize, paths.Unsealed) + pf, err := partialfile.OpenPartialFile(maxPieceSize, paths.Unsealed) if err == nil { var at uint64 for sr.HasNext() { diff --git a/extern/sector-storage/ffiwrapper/unseal_ranges.go b/extern/sector-storage/ffiwrapper/unseal_ranges.go index 4519fc21e..3a13c73a7 100644 --- a/extern/sector-storage/ffiwrapper/unseal_ranges.go +++ b/extern/sector-storage/ffiwrapper/unseal_ranges.go @@ -7,6 +7,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/extern/sector-storage/partialfile" "github.com/filecoin-project/lotus/extern/sector-storage/storiface" ) @@ -17,7 +18,7 @@ const mergeGaps = 32 << 20 // TODO const expandRuns = 16 << 20 // unseal more than requested for future requests func computeUnsealRanges(unsealed rlepluslazy.RunIterator, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (rlepluslazy.RunIterator, error) { - todo := pieceRun(offset.Padded(), size.Padded()) + todo := partialfile.PieceRun(offset.Padded(), size.Padded()) todo, err := rlepluslazy.Subtract(todo, unsealed) if err != nil { return nil, xerrors.Errorf("compute todo-unsealed: %w", err) diff --git a/extern/sector-storage/ffiwrapper/partialfile.go b/extern/sector-storage/partialfile/partialfile.go similarity index 93% rename from extern/sector-storage/ffiwrapper/partialfile.go rename to extern/sector-storage/partialfile/partialfile.go index 0e8827dd3..529e889ea 100644 --- a/extern/sector-storage/ffiwrapper/partialfile.go +++ b/extern/sector-storage/partialfile/partialfile.go @@ -1,4 +1,4 @@ -package ffiwrapper +package partialfile import ( "encoding/binary" @@ -14,8 +14,12 @@ import ( "github.com/filecoin-project/lotus/extern/sector-storage/fsutil" "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + + logging "github.com/ipfs/go-log/v2" ) +var log = logging.Logger("partialfile") + const veryLargeRle = 1 << 20 // Sectors can be partially unsealed. We support this by appending a small @@ -57,7 +61,7 @@ func writeTrailer(maxPieceSize int64, w *os.File, r rlepluslazy.RunIterator) err return w.Truncate(maxPieceSize + int64(rb) + 4) } -func createPartialFile(maxPieceSize abi.PaddedPieceSize, path string) (*PartialFile, error) { +func CreatePartialFile(maxPieceSize abi.PaddedPieceSize, path string) (*PartialFile, error) { f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0644) // nolint if err != nil { return nil, xerrors.Errorf("openning partial file '%s': %w", path, err) @@ -188,7 +192,7 @@ func (pf *PartialFile) Writer(offset storiface.PaddedByteIndex, size abi.PaddedP return nil, err } - and, err := rlepluslazy.And(have, pieceRun(offset, size)) + and, err := rlepluslazy.And(have, PieceRun(offset, size)) if err != nil { return nil, err } @@ -212,7 +216,7 @@ func (pf *PartialFile) MarkAllocated(offset storiface.PaddedByteIndex, size abi. return err } - ored, err := rlepluslazy.Or(have, pieceRun(offset, size)) + ored, err := rlepluslazy.Or(have, PieceRun(offset, size)) if err != nil { return err } @@ -234,7 +238,7 @@ func (pf *PartialFile) Free(offset storiface.PaddedByteIndex, size abi.PaddedPie return xerrors.Errorf("deallocating: %w", err) } - s, err := rlepluslazy.Subtract(have, pieceRun(offset, size)) + s, err := rlepluslazy.Subtract(have, PieceRun(offset, size)) if err != nil { return err } @@ -257,7 +261,7 @@ func (pf *PartialFile) Reader(offset storiface.PaddedByteIndex, size abi.PaddedP return nil, err } - and, err := rlepluslazy.And(have, pieceRun(offset, size)) + and, err := rlepluslazy.And(have, PieceRun(offset, size)) if err != nil { return nil, err } @@ -285,7 +289,7 @@ func (pf *PartialFile) HasAllocated(offset storiface.UnpaddedByteIndex, size abi return false, err } - u, err := rlepluslazy.And(have, pieceRun(offset.Padded(), size.Padded())) + u, err := rlepluslazy.And(have, PieceRun(offset.Padded(), size.Padded())) if err != nil { return false, err } @@ -298,7 +302,7 @@ func (pf *PartialFile) HasAllocated(offset storiface.UnpaddedByteIndex, size abi return abi.PaddedPieceSize(uc) == size.Padded(), nil } -func pieceRun(offset storiface.PaddedByteIndex, size abi.PaddedPieceSize) rlepluslazy.RunIterator { +func PieceRun(offset storiface.PaddedByteIndex, size abi.PaddedPieceSize) rlepluslazy.RunIterator { var runs []rlepluslazy.Run if offset > 0 { runs = append(runs, rlepluslazy.Run{ diff --git a/extern/sector-storage/stores/http_handler.go b/extern/sector-storage/stores/http_handler.go index 6d01fd2f8..e68484cfc 100644 --- a/extern/sector-storage/stores/http_handler.go +++ b/extern/sector-storage/stores/http_handler.go @@ -12,7 +12,7 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" + "github.com/filecoin-project/lotus/extern/sector-storage/partialfile" "github.com/filecoin-project/lotus/extern/sector-storage/storiface" "github.com/filecoin-project/lotus/extern/sector-storage/tarutil" @@ -130,7 +130,7 @@ func (handler *FetchHandler) remoteGetAllocated(w http.ResponseWriter, r *http.R return } - pf, err := ffiwrapper.OpenPartialFile(abi.PaddedPieceSize(ssize), path) + pf, err := partialfile.OpenPartialFile(abi.PaddedPieceSize(ssize), path) if err != nil { log.Error("opening partial file: ", err) w.WriteHeader(500) diff --git a/extern/sector-storage/stores/remote.go b/extern/sector-storage/stores/remote.go index 8f5043c73..2025418f0 100644 --- a/extern/sector-storage/stores/remote.go +++ b/extern/sector-storage/stores/remote.go @@ -16,8 +16,8 @@ import ( "sort" "sync" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/extern/sector-storage/fsutil" + "github.com/filecoin-project/lotus/extern/sector-storage/partialfile" "github.com/filecoin-project/lotus/extern/sector-storage/storiface" "github.com/filecoin-project/lotus/extern/sector-storage/tarutil" @@ -415,7 +415,7 @@ func (r *Remote) Reader(ctx context.Context, s storage.SectorRef, offset, size a return nil, err } - pf, err := ffiwrapper.OpenPartialFile(abi.PaddedPieceSize(ssize), path) + pf, err := partialfile.OpenPartialFile(abi.PaddedPieceSize(ssize), path) if err != nil { return nil, xerrors.Errorf("opening partial file: %w", err) } From de646a5b9340ce180d268800eac9ec51ec95e8fd Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Mon, 7 Jun 2021 12:16:10 +0200 Subject: [PATCH 33/90] fix node builder --- node/test/builder.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/node/test/builder.go b/node/test/builder.go index 8aa257cc7..a8b8ac57b 100644 --- a/node/test/builder.go +++ b/node/test/builder.go @@ -522,9 +522,17 @@ func mockSbBuilderOpts(t *testing.T, fullOpts []test.FullNodeOpts, storage []tes fulls[i] = fullRpc(t, fulls[i]) } + mgr := mock.NewMockSectorMgr(nil) + fulls[i].Stb = storageBuilder(fulls[i], mn, node.Options( node.Override(new(sectorstorage.SectorManager), func() (sectorstorage.SectorManager, error) { - return mock.NewMockSectorMgr(nil), nil + return mgr, nil + }), + node.Override(new(sectorstorage.PieceProvider), func() (sectorstorage.PieceProvider, error) { + return mgr, nil + }), + node.Override(new(sectorstorage.Unsealer), func() (sectorstorage.Unsealer, error) { + return mgr, nil }), node.Override(new(ffiwrapper.Verifier), mock.MockVerifier), node.Override(new(ffiwrapper.Prover), mock.MockProver), From 4dd093b160ec7b9c4229378751290668c8f8ea14 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Tue, 8 Jun 2021 14:57:45 +0200 Subject: [PATCH 34/90] fix piece provider test --- extern/sector-storage/piece_provider_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/sector-storage/piece_provider_test.go b/extern/sector-storage/piece_provider_test.go index 6a58ad945..81dceb730 100644 --- a/extern/sector-storage/piece_provider_test.go +++ b/extern/sector-storage/piece_provider_test.go @@ -193,7 +193,7 @@ func newPieceProviderTestHarness(t *testing.T, mgrConfig SealerConfig, sectorPro wsts := statestore.New(namespace.Wrap(dstore, datastore.NewKey("/worker/calls"))) smsts := statestore.New(namespace.Wrap(dstore, datastore.NewKey("/stmgr/calls"))) - mgr, err := New(ctx, localStore, remoteStore, storage, index, mgrConfig, wsts, smsts) + mgr, err := New(ctx, localStore, remoteStore, storage, index, mgrConfig, nil, wsts, smsts) require.NoError(t, err) // start a http server on the manager to serve sector file requests. From 3f5f96dcbdff0cc8538becbdc190e9892aefbcba Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Tue, 8 Jun 2021 14:58:20 +0200 Subject: [PATCH 35/90] go mod tidy --- go.mod | 1 - go.sum | 1 - 2 files changed, 2 deletions(-) diff --git a/go.mod b/go.mod index 4d5ab9a33..e22be541a 100644 --- a/go.mod +++ b/go.mod @@ -155,7 +155,6 @@ require ( golang.org/x/time v0.0.0-20191024005414-555d28b269f0 golang.org/x/tools v0.0.0-20210106214847-113979e3529a golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 - google.golang.org/appengine v1.6.5 gopkg.in/cheggaaa/pb.v1 v1.0.28 gotest.tools v2.2.0+incompatible honnef.co/go/tools v0.0.1-2020.1.3 // indirect diff --git a/go.sum b/go.sum index f8f77a153..9ad58a702 100644 --- a/go.sum +++ b/go.sum @@ -1970,7 +1970,6 @@ google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= From 30ba9a27518e266f1935e44807c1012ec855e7ed Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Tue, 8 Jun 2021 15:33:10 +0200 Subject: [PATCH 36/90] remove double http handler decl --- extern/sector-storage/stores/http_handler.go | 80 +++++++------------- extern/sector-storage/stores/mocks/index.go | 17 ++++- extern/sector-storage/stores/mocks/stores.go | 2 +- 3 files changed, 46 insertions(+), 53 deletions(-) diff --git a/extern/sector-storage/stores/http_handler.go b/extern/sector-storage/stores/http_handler.go index 1acbfe841..8cfa8a7e9 100644 --- a/extern/sector-storage/stores/http_handler.go +++ b/extern/sector-storage/stores/http_handler.go @@ -2,6 +2,7 @@ package stores import ( "encoding/json" + "io" "net/http" "os" "strconv" @@ -13,6 +14,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/extern/sector-storage/partialfile" "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/extern/sector-storage/tarutil" "github.com/filecoin-project/specs-storage/storage" ) @@ -55,8 +57,6 @@ func (handler *FetchHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { mux.HandleFunc("/remote/{type}/{id}", handler.remoteGetSector).Methods("GET") mux.HandleFunc("/remote/{type}/{id}", handler.remoteDeleteSector).Methods("DELETE") - mux.HandleFunc("/remote/{type}/{id}/{spt}/allocated/{offset}/{size}", handler.remoteGetAllocated).Methods("GET") - mux.ServeHTTP(w, r) } @@ -101,40 +101,9 @@ func (handler *FetchHandler) remoteGetSector(w http.ResponseWriter, r *http.Requ w.WriteHeader(500) return } - if ft != storiface.FTUnsealed { - log.Errorf("/allocated only supports unsealed sector files") - w.WriteHeader(500) - return - } - - spti, err := strconv.ParseInt(vars["spt"], 10, 64) - if err != nil { - log.Errorf("parsing spt: %+v", err) - w.WriteHeader(500) - return - } - spt := abi.RegisteredSealProof(spti) - ssize, err := spt.SectorSize() - if err != nil { - log.Errorf("%+v", err) - w.WriteHeader(500) - return - } - - offi, err := strconv.ParseInt(vars["offset"], 10, 64) - if err != nil { - log.Errorf("parsing offset: %+v", err) - w.WriteHeader(500) - return - } - szi, err := strconv.ParseInt(vars["size"], 10, 64) - if err != nil { - log.Errorf("parsing spt: %+v", err) - w.WriteHeader(500) - return - } // The caller has a lock on this sector already, no need to get one here + // passing 0 spt because we don't allocate anything si := storage.SectorRef{ ID: id, @@ -143,11 +112,13 @@ func (handler *FetchHandler) remoteGetSector(w http.ResponseWriter, r *http.Requ paths, _, err := handler.Local.AcquireSector(r.Context(), si, ft, storiface.FTNone, storiface.PathStorage, storiface.AcquireMove) if err != nil { - log.Errorf("AcquireSector: %+v", err) + log.Errorf("%+v", err) w.WriteHeader(500) return } + // TODO: reserve local storage here + path := storiface.PathByType(paths, ft) if path == "" { log.Error("acquired path was empty") @@ -155,30 +126,37 @@ func (handler *FetchHandler) remoteGetSector(w http.ResponseWriter, r *http.Requ return } - pf, err := partialfile.OpenPartialFile(abi.PaddedPieceSize(ssize), path) + stat, err := os.Stat(path) if err != nil { - log.Error("opening partial file: ", err) + log.Errorf("%+v", err) w.WriteHeader(500) return } - defer func() { - if err := pf.Close(); err != nil { - log.Error("close partial file: ", err) + + if stat.IsDir() { + if _, has := r.Header["Range"]; has { + log.Error("Range not supported on directories") + w.WriteHeader(500) + return } - }() - has, err := pf.HasAllocated(storiface.UnpaddedByteIndex(offi), abi.UnpaddedPieceSize(szi)) - if err != nil { - log.Error("has allocated: ", err) - w.WriteHeader(500) - return - } + rd, err := tarutil.TarDirectory(path) + if err != nil { + log.Errorf("%+v", err) + w.WriteHeader(500) + return + } - if has { - w.WriteHeader(http.StatusOK) - return + w.Header().Set("Content-Type", "application/x-tar") + w.WriteHeader(200) + if _, err := io.CopyBuffer(w, rd, make([]byte, CopyBuf)); err != nil { + log.Errorf("%+v", err) + return + } + } else { + w.Header().Set("Content-Type", "application/octet-stream") + http.ServeFile(w, r, path) } - w.WriteHeader(http.StatusRequestedRangeNotSatisfiable) } func (handler *FetchHandler) remoteDeleteSector(w http.ResponseWriter, r *http.Request) { diff --git a/extern/sector-storage/stores/mocks/index.go b/extern/sector-storage/stores/mocks/index.go index e06fa70cc..59a6017b5 100644 --- a/extern/sector-storage/stores/mocks/index.go +++ b/extern/sector-storage/stores/mocks/index.go @@ -1,7 +1,7 @@ // Code generated by MockGen. DO NOT EDIT. // Source: index.go -// Package mock_stores is a generated GoMock package. +// Package mocks is a generated GoMock package. package mocks import ( @@ -125,6 +125,21 @@ func (mr *MockSectorIndexMockRecorder) StorageInfo(arg0, arg1 interface{}) *gomo return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageInfo", reflect.TypeOf((*MockSectorIndex)(nil).StorageInfo), arg0, arg1) } +// StorageList mocks base method. +func (m *MockSectorIndex) StorageList(ctx context.Context) (map[stores.ID][]stores.Decl, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "StorageList", ctx) + ret0, _ := ret[0].(map[stores.ID][]stores.Decl) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// StorageList indicates an expected call of StorageList. +func (mr *MockSectorIndexMockRecorder) StorageList(ctx interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageList", reflect.TypeOf((*MockSectorIndex)(nil).StorageList), ctx) +} + // StorageLock mocks base method. func (m *MockSectorIndex) StorageLock(ctx context.Context, sector abi.SectorID, read, write storiface.SectorFileType) error { m.ctrl.T.Helper() diff --git a/extern/sector-storage/stores/mocks/stores.go b/extern/sector-storage/stores/mocks/stores.go index a408419a9..fdfd73a07 100644 --- a/extern/sector-storage/stores/mocks/stores.go +++ b/extern/sector-storage/stores/mocks/stores.go @@ -1,7 +1,7 @@ // Code generated by MockGen. DO NOT EDIT. // Source: interface.go -// Package mock_stores is a generated GoMock package. +// Package mocks is a generated GoMock package. package mocks import ( From 9ab84bdc0af52cd3bffc9804906ac5e26d9f502c Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Tue, 8 Jun 2021 15:59:37 +0200 Subject: [PATCH 37/90] upgrade docsgen --- build/openrpc/full.json.gz | Bin 23305 -> 23434 bytes build/openrpc/miner.json.gz | Bin 8648 -> 8642 bytes build/openrpc/worker.json.gz | Bin 2580 -> 2496 bytes 3 files changed, 0 insertions(+), 0 deletions(-) diff --git a/build/openrpc/full.json.gz b/build/openrpc/full.json.gz index 49354faee516ce0e4131b349971ca2990293f681..14679ffb479cb22a6a787bd800d4cea2edd11bf1 100644 GIT binary patch literal 23434 zcmV)*K#9K}iwFP!00000|LnbKbKAJGKm1itdR|PDQas+Wx$4$aUgE@eoW#d=Cik2; z_YOq1B#bG5Lx8q5srl~z!op2l1X6M$%a}TKVv*R0rTfGvie#?0rv zy|3SU6i;N;V-kSJ2;DdU>@oTd#9Jz&X|Lb&fqM}Ng>LANKmHi8D>{s+hn{fs>^KgB zkIaV)5yhT(ii*{rgqz4?RKT6sIMo04QMiThfNUI&F`;im=8d1CC(pz*1a3N4Rr?5{ zh(%A)_bYlOA`o#z5#KnW0wRiojZ#Ol;ip&hpDX&uAAj`vJswXcIGXnMdc~qhx5bed z0~)AT2-J%qi`2srxdDoXW=0UmF>auf;T4r{b=fiIh%$kuAkYASqA+59?1A4`O?Ws6 z&@Bl9Gzge?jbbWDfYRz$bO0Wk07v)+lK>9_xT3v&FT@d^=&6h&xh#X}^N?|(Cf~d+ zEL*r$TA||H7A8Xe@WwE~`AtLP$pE5KGauDsL^#4IEdqCV5(XfLu(i3hx%I8zlMC0| z`ZufgfSm54X@82}o{LjU~*rWeFhJXFnfA!>;u#=%*{LWl&B_0^F3 zy)gIayJ_ZoZvsA1e+&r4(ey3#m`~_PE*<9Za#tQVJw-h(BKhW@elnbjH84fZ?|=Tu z=9dlr1W#DbZ^bjYc>3rZF;9I$ac-97`s+v) z+3?up*W_?K40ubL58H_xE~T9MRLz@phBK`4#DubE;Ar>q-T^0gnd5( z&cR0{ZEXXc2uUf1ESl&K#^`s=d-@2YL*|2rp4^U!H%5f|MD9!3$0QB}@#Nd*A@;7p zPkWKO{wc!LM;LvOrN)?#H;NSKd+%3u%!bqL!Qa>eymQO_@yqEB-gzkoJNz0Z;h*g8 zuP~mnN%-erGKA@HXLm?;;_1schTZL(U3Kh}BW4b0cM|q74uT3osvhQ6C8d}_@W9XW zi`|Cv{0_u{WPaU_b5eWmX-=_&Ed3s^579jAFx$bLBc7BjA_w(^hv3O0P(I}w?wGIZ zF)wepTBK4aVuMsZ5DHOmuQ%M>+TPji^?R4Zy9U+!@8`#75o5zj^wRIW4ZxGVn=;>f zUkE*9F%`YNH~o2)Gf1WIjU-Kd*}xgQ5T>4fc7`LlbBaH|02z%%Z?Cu6>-XN99||BQ zB`yROaL$O&7i45x{g_3vIK_d&q_PP;Z3mdcF@RdKmdJPZGkz3k*5BI#GbfO_yuH8o zh=2+Ik)v-9`E$cJ4{#LWsrsDUHh2irUm{H{2AvVEjswrgV7I}~oT2%RWF*k0Y@W_E zGV;=g8CEf8HcQ_#Ljbx}uOPea24BGzx-{@5o#`%LQ6vQq6DhTuwOLA%p^G1PB&!Zt zj?dW~nj$jkM$QWj%_n6DT!i%IGex~O?%YZXh41(28!VLl1lx;5e_&*A>*730#Hiua`Q+qG{!e@ zMSU_0r|HLdjB?^b1pXD3BZoWiV!46T41T(NyN52{pIo31Z!h0} zIzkukKYczuLZ3dK{u>>~Q-d>=Sk7yrVoV`6f zIX*c=2dAG7f0Hd9eL^2UUCMfeW-mX<7V@+WB2NXEzE{&P`6>?Z+skWn_= zTN`MWK`x`#O@Ady#Lc-_Y^^rv>n0bGcaX?Toj0hdZ*5jKF-(JkxV~ucyg4=+gH?+Eg5lLun3&dd41)=!gj2qJ=tp;;*&D_-E9prGPASBc-e0Q zc4R?=ER?Y~LuPLZ2rqe=0x|U`%J)W>@aXawuuJPS2@BPzKj&;b^A=@={yL>apO-u60&J0yP ztK+76p(Qgt2?PI=noCYmL{nY0()S%S*txvhK} z{Mp<4-jSgi^}c-};H-^lK}HRVVt^P2IX+S50cK z9M#M%kXC&?Y(ly<$M--}o78`4F+!R&V?#)bf|{w>j;!Xovn8uZm6WVT$%7$_G8$V! zT=k{5S>lQ=$B?yQychz)h3P=jEshw4=Yl zTUB`qaYO-p@Fi^>fl$ihD=JBH=GlcQ6Jv<_NV`y}^RcJ8zoJP0Cs_I?`rj+6RL}^C z{eAiPf0M`JU|$v9Ke(dkf1f{p{_Oel|1}lOzvBGdwETs|fp0zo{qa=~|A{!*f7(at zBc6!;gWTWyPrv=KGkb^R z%OaF!e9d^(&K%QTkZk3IA5q4!-so_`JC&#fX$F3Z^a_q{ns{*^) zonzmw@vODwPrAA#L+a{~oOUwb_@avzT#}8}ttN0#IylEMeumP<`FGzt1$Z&XFsoLp`?U-h z3#>?E+Ta}Uq{|_13@Q@E5aBGeB7!@@Gq)3W3Fgr_g~)AInpyL3KH-o(^n&$+w9`2hCht0txNIbgpt zKQfhV94!Y^Vds65wK<0;nS$oNCJ6&y#ro!U7dF-Qy`mYPQq(u~$Hcctj(CZxV#mE! zjC-ZZ&lS*#t}5wx)1)4_0MVVJnabA6!TDJNvKHqM4pUGb01t+75KIvV!BZ}$5v=5O z8|X7P&e9%Mfy|sD@Mp-Zk1dk_)|(~?@Gb6B>v4yWw<)8J28&x(I%9a!kt zOyQL8a)AZmf_NO^0gJVsY!u;eJj2z0VS_(^(Hgd22AIPyO6?SImhpO(`bW#8>D>m| zej2S1BcBY1S9Ad47EDQT%aGo#j`gnx>u)wiyOmijP-ZAiX>zmYHX`5!`P)RtI3-ld zf4kc)FHD}sZ)InXg*8hLCI z#_|~6>fcGix=h;a66nfd#^qqf{7$Hu$n>R!sJGdBOgf|Otva1iK!A!v9C(LpGC6C1 zw2SrUM=DXp&Mj1%rG^`haWA9Go1JAjkxM(ss}bcSncK43HTT9Y zq2n@B32BD9wYk09l4*BKUGZ4?_jaB0mpL8OgWw1Yd>LWNhZTU=_N>n~RCnloEU++T zHj=_8M@kkA0iq4{_YN8`7R*x!)pwnDHEt#kvz3vj9!c0Ohl-N1X1C$BO*DOea-{IL zy3MJ+U}^NDW~%PZbz|pV+&0Xj+L6Jn?W3LF82j?>@4LT$`_I2`;QjxI@ZHOO9)14r z1Mkb-`-9_)?f3jGdv|#keK@`O?SI6uyFRwb?f1u6IGrBsbZdL3PXC-cqURtY0yj7~ z0$iS*Hbkn>QXQ3wl0p+4U4z(=WlxS;(4*O2=Hrx7t&9>;#OR1ge2|U{`Le_SbMO(P zNRMx6l&AXh?~U-hSY}S^$9H_{)W~XM$)!6Z;K89EFk0xTv&bE0_ z892AwRJ0U?3gfei8=D(%%fx@#r$0Ussy5+vO1&yKoIP!o)$QnNJECgZokviUqIr56 zK>{&STI$H=I+M*>nz=lJ`9sLtV6qjEX)>a+q6*|;GqkvRA-T)#fSUn3Sm?@ zh{(sw0~F@_tTDM5r~BKOm;1dFezC8yNP$NglYZJ zX_{*j8Ja_(R0$_^$T}bFv;~V?aY<1-(W2P)Je^3cbm5Mi(vAp3sDx@NppkK5POi%O zfCDEytcCDU=}l9+!}D*f2y?_j#&NLJyg?TN3*!@;CZm|8{Yu>@k_^VlUn&84UykaA z_!6&XY3QVcEg8<<_g>-R*E@=JR5M+|SV{L@?Cxy$dxsI|h-XCYoXyS6t>@~0mz$e=>VN;~ zsSIffBqZA~nc>4RrW6AGDJPT&f(4Vy(#b=j-#Z5$GOAa$V-s>LJOAkYkh%-yHidex z_h8Ib!odT~m%AD#JU113!cO=F=b=49L1t=(e! za(lhqFbjLS3Rh=4yy|T(zfq~_+|I6eo9)pSpP5zalKWomXYiBl+)lS~l;^h<-*d`F zodt>C9U3Y6RuUs)KXCYOQZlg|F6cZ|5?mk6N z^t!ZC6s0X&s)}^^!X5XR8>J9N9`l#yK%+nF-n?;lJIB$k(wL;0+MlezUDcyfniEb4H-K392;t@J5+@=O;zhxu7PitWy$0lFX}>L0F6q_KoeGe*)7 z3%TzFQ$>~>RP?dfOe;+m>B^RE7Wr2{{tB?}U1`kc2Wy;}(zNP*=?~WolJC?e*i)zU zlyH$(es+cBrFN?0nSTA2FJ$C6JHk`dLlgax<03Wq&MtDT9yr!^ZUZV zPb)vR!a=UO)ctzOIj!~2*4{bA+#}LjudTb^PHj%SjA#Q67Gf*1el0s`X*5N#Fe#3 zjDbWbVoadJlOw(a@ubE?k$drTE_}*`tx>V}d%wZ-J`w)~izBZQ>#y(ms!ceWZqhW; z*&;_QYozX!r6t-a2MP_(J8xOx1~oyH6{^z{Ea4m$KDXjH&c45AVjKf}a7He74& zNmDh>!2@z*3(uXD{D7qeCkQ-FDcN=yjna#b57a%L*+P*pHz0KI^6c_$pj80 zX>C7-hbgeQU<-|StI>f-MImX3&0+_*W?HFD@2*|nHcSYUe2ys<*E<&96%Nh|6=*%+eS1+ZFf0ILQP z*)n*Q(MbG&3-ZbB`0So~7UW7@Q7ToA%5*!&Ko|OJNl@2?{`zsDzaBF5tj&e5fS011`~12i=1i*Bfz2#o%=Rqse2Sht zDxa4F+!ADugIxBh6@m-E*OIhrR9F6&T z0e^jD9dcwct+@J+o7uc-^fEn%3A=%X zu1n>csAD=xXwesx2_ zp=C5CL%X0RyA;L)-3Z4@I2~`D| z9Od#P!Q$8dlpDS^DoPa;LV=Cm-qyCUW-d`*sh6XquzVOA%w+IwMUecY-lKrz^qOm| z*E9#Q?GpVAb!cN|m&Y9M;qgI?M(2lwDPKlYBORPj9uJ4aQyHXU>UZ@(wP@MWNum`P zY3^>mT48Q0l@b#dI0(Sc`>`^bM!gR;EkvI7DY~NC)QclPT*Tfr@?(&#kz8MBC_7tU zrwVsAPq%hA8?8eZN?7ln&^xVPcg~)rCC=0ydWxPrOP!HyY(V)Q<{hoHEDF{ACgtIv za%d{`0$#SIXZGVPxt+=b_yA)0e~|>I8N{pWXESCI)eJ8N=1<%%4_Odk0Z}S!m>Zap zFc;a34dfW-y!z9?@Mc;CrQYHFIZkes0V1b6Ztlu-$0OQ1@>G;CwM~khO820HM%}1xU!s zqalglIi~*eA&W*Jo@c;;A~qZbgzDfj5<46Gpuhx+1>*65!(Xx5L=pC`2^~3X;jo3n z77klDY~irQBgPgxbv8$JmGuC7*Kyb$ROBkxp$WTmv&DuQ07I@I^3)HGDDwdg^fOiw zMp=0X$q(N^mt(l16A)u2ccofPlNev$JQp!XK86XSa@$2QrE-fVt`JxB^wZ_%i~l>= zKm6_U+5a8vU%dUlv;E8W2n;u0xWutPsd1yE@;lzWnXcS*tv5Wm6nDSD!x9FWloog1X3Y7Fx|mP)k{n4cVtFh@fG zh*h-NhJq^(1+1Yv_p11#^F|s zr+oC{?)onLYe=_mhu+^WCpX*U3A+}z1OM(C-|g~VTkE|3>b4Lm_uU<;n)$PVf%yq- zZP_<-Z_|%iY&$!-UdQjf3HU^9u>qktn!cqT^Ccmbi^$>SZZ7OW4%M>`h*a;t4nUS$ z0`7Y;Ti;%E-fUaeFSFptOg9-By&Dzo@y$SLAOAL0OEQn^u~W zgjQW4SE_;J?e6lF1cfo>6$P>z8@_L9=Q#iE7z@Sz!-#l*_?X3kj|QOOz>q}w{2GE#{#wZ}y5RfrrK4L>es29YN6d^*rjm+{6w|5e+Y^8-%`ON&d0P0_a;MFLqXKPw> zp~9eAsjIV^GqR_$lqd@(iYW(?%9cu>O(0mJzqb=Aolxn7%2gIBX9<|)8U29ojzJ%{ zmYX@L`NMCZIkosgzB#j$+!1<5=pCV7EkeJd5dKw-tJen=cWvMI4{(6xlD9?gvkkNf zcU`4M$_ryp(GyD_^NLZ-N9wbc}L(&VfU81Lf^xN3wG*Sl_;HrO2=9rrwFSR2q#y=(s?5 z!eS~A8z!`QjD@-~@5NCBR0LBLC6$ug1ZL>#OS7GpbJ+*(b&cPQ-kIkR0Orsw4o10& zHZZ9Oi5h+8D?4iTlqpw1k^Yfjq4aPKrs@FCNFQ- z&Sy7k;%ACDGvlxoa`M%Z^PC)SW%O3JXCLWNUggm`3CBq|YbxQq*sL)tYm)Vah_N5x z+l~Y~ss1yGb#4r{ggU9BF89elIErb&)Q&UcI?0UTbW-aYE^2R@ohpUeH!tEQIah*KD|t&x8=+{zh_l2mBG;$+F92U7uEPuKh{u+;cdMI2Y>(W(~JGlD1wnh*L&X$)!XhH+F7|SGZ4{)-y7LhKW5Rj zqUeD-L^|J!oVEkZA$87`?ebZudI4T|U3}OUisl>5(UONLbGHj&ILm{w1Kwr=XEx00 zv*QvWdFTR2{=hwi=`S&Xha?QmP%9hskT4so$eKA~)b2DdgdkH=+pT&9gKHV&wakFK zH1H*z>CQ0vQFDNYD;2D>L{P~5TmU?edb_UK?SdCub-t>Su)hOEqfh9Nb+ZTL8fazB z9gvN~F(y=LIqPl(e;`sSPujP#8x+Sg-?Do-h6t3yDrEvqK_E=0$YYa<+Tq$|m2jj( z&~7|VOKxGY?ky675}I9nzn^`Pv3TezF2v5hw>FiXv0J6z+3fXuA3@x*=sN%C0zAQ@ zOasHBYl!4>{56#WFv8Govznk%`|~y+BQgl! zm__O<*ExEHBDe_19XlK9icloYSw) z!0im&&cNN)z@4MmIr%`dgziW;B~=yYeXeI^(-B`ougj8$WXE+_Rc5+VcSotfdg`LP zwBxC!mAS07DUO=rv{g*09ox-@GUpy*RjpY@9Md1kihmRncI?!#)0&twU1{JWjyG1# zhilNa1L2NptO?a{Abf?v-)`Mq#{%m4OhpuP(Iuu!bvr*wx03JBR>_jqvDUWeyN_{N zkFz-PV2PpD9sPIo-_ievO#ht{#wlT%`0$Imo2!z)9FD=uT$DPo;6f7(+}9L2i*I!61j813&kAbvJ6GmZ-P&ey64M#Vv^Dc0evbvkvDzqYpPOQvSG+1^G`Zt}Ah z-V^OAA7UYCg&c>>s&xnhI0l~9{nZo*O`{=!hdYe)nYG3BXR@uaYiz*PJQ7YVvM6 z3;>sPqIO(RR;^=ua?a;PM9()ekAul#$dmuiYf!ifyeZ2rjXBA0usV!#N8~n5vfo^k zaO*w#y2%Lp4#Xj&LsE`8W3^Ay>J@u6mwi-iGgZzCiRD{u*tlsiq}$=>2}T|Zr-%)o zsy%B`W$3*t$2TOSR$I6!Azk)SwMpp;MQpr{&LYAhBBoL-&4Q+ozd_-g_hdBg?QL!E z$q8{oYv=Q#fQS@YyZx?Y-Pg|H~lMryGb&wpR5CvSv$v zr@E%TSKyxFXWHJjDX}8+Aicse^QAU?v|`pO20XI_R=0Z!EA;4cuf2HFWOI54Lc`>) zR7cgeSAiv2{OyQO()b+4nUvHydnnuqMAh#k9S;w|lTgn`9HOe=;x6wr33nUb`6fFBm{Wi`-?WwI zhA%gp>>1rJcsJecuJJx@<9lp(rCZ(R{*m{%I}_;wseJ@#*=Hz$O}N#cmlfV|LB|Ch z7j&71T&AHPj7_$h%t{O2m$$eoZ+B6?yA0pgiEWeyM^f9qf;zmf z_FM{o4DPw21>iGLw{zu&XEd0iSRRJkF;J8;qF*ykfrbPEpCcRr{Y7MYy(@Y}zsO2i zYaE0`SeyN__U@Jh0UE`?Il`eNa1p@*&=BB?O8Vs^MpKi8H^zj@wb4X!d7f__oI)N2 zGHVBo9LRAXXAz@~i&Ri$Q*G7-Iw8{unU4NC`s;+uRhC`0o7|0-5bnV$(!-K8tt|pP z6k|KAKvW0EEgiRX+|qGN$1R;o)~RF{hBZ4)?u-`@$gS83&*-|Au2V`AI}8{WFLxJn zqe{EA(7x)jsUsCe0pS9%Va06$79m2tAeMU+5vtrEm1pE2VBWR!w7O?ct46(&**8{` zwY9ym)7AgcxqCTOY5Q8xBPclXKNeY)?vPwsx_>DJ5LI!9JcM*F_MABGXT z>EJ3-sNO0CW$D=tN-ki9yA`irR6KiWt~oy`#mk#3-pmd>pQ}=iK=vqmsG+BY>e^G| zMv2v71UH1mJeVSR`k55g_kCYM)hKf_IT1+UYtU49$bx{~N*PHL&pmV%Z*J~*N<`;m zlm<4EPoZz#>g!_3J1J?&w#-n_gpZ2F)6pbq=942+;EEPXMAh9svO-H5w&P2{P6RsnVEk#qsw4puu@?O^pi!fFRp9Z+>Z)d5up zRJ#JzS9M-38jPK3jMEjVn(B)})$+65Me%7ivRQ!I2u@A41c-7uG>I~epKp|Syd*gD z9$~2&LJpTY*y&*BN{_Y-Hywa;0L}q82jCol>kQ!P+?pg{S;X?+-t3=h6ibM$Gf=BE z;Wr?(CY$~9OktP6tweJ-m?9Bj%CRSB2%Q`)1-GUB&9towCD=~G#zOr^inAVq!t4Ry zEeCcS*l}RTft_^(JFlBO9W~Ftyrb`Nm)M=9CYA$zY7MN~OZtF}PN-1sj^#qHvUXpi zuipk!0Sh9j?9=LeB4xL-zqugGG}RVs_4C2AVlFY&fQj)8Mlv@P$FhS&-zASU2#C0j z)ulCUS5)_egAH_<3#iYAW?)#ElF0xT7RWRh?AV}1l zv!5_Cel;1gj=B-(Lvu9;dmZe(CpF_LkG5>p&T|I4p>F;JC(%`&(^n7jl)bLF4J&!R z-PD#upRO11odD?sNGCu#0dgG$$TxLyl{pzL;FR0mBX^F#I3pZZ-;Yb%c(tH#*yyaA z2!-_mVS~qRy&QP|8uoh?lm-jp6o_fSyldpgkqY~h z;B@_xP$CEp&;SQgXt26+sK2iyU5qeCVMM%TO9Br8qm(6r(w4r5Fo7i+)HNzvqmWQI zh4;1$Tbo;(TlaEn(8&$Umm5}QP8}+DsNA7)hsxIxm2Wj%e|!gG{|1u)4+1#Q*sT*p zuV|-@r9c!TT|Y%niU@r^(5@c9oUq^3_0kaFNZu?X=HsCcAmY~cczTceB_2(+U3Xk}?z!_A;~AkK8?{;nj=R9}peef{=KHg}m> z?=0zA*1HuMGsRQtm831mnGJR70!h|J`t|$&ez!4gQ@*#Ef&(XhmITL1%M^(Ora*_MZm0fjI_dtKoK&=A@n?3nK_NzSU0>fHvHm_-5)E-C8Zj3Z3-E$-CA=H-Gu`1Jt^=exs{qY~bayZya* z(wE~HI#+rZ(QqXp@A_OZeh3aGD}IG%y1Lzi(nHnazAf3H6DGmC`4P49+4% znb6sR$%raL_c#JRW`UoJ3m&jrh`>jKDPk0^C=QrxgotsWdov@CRc*A&qkZTXRbAY0 z2mKuMbI{L0zs}IF!8`Ec7Ke+(4$sy932DQZ6nnh*5$%<-x%%^Rh|8iHQa;4w;R+j8 zq??&dL*roJ%8k}#Ep*AiA6%SAhsqr)cc|Q<@^wVzFB?3iB}!Z%MqyU{$71qD@OQ^7 z;#t)>X~Z+&z0!Ha9l~}9+ac_SjIbSMb(qy*R)<;F5wpH(xN#u?)*`OFx%&6vLa;=r z+9jcQu3qL#z;AIFO2oOHX5?Olq~Z>Pt`P=Z<