From 695f6cfe4553ba1ebca9dd6365ac46256effe47a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Fri, 17 Jul 2020 14:22:37 +0100 Subject: [PATCH 001/885] wip in-memory journal. --- chain/events/events_height.go | 6 +- chain/store/store.go | 17 ++- cmd/lotus-seal-worker/main.go | 1 + cmd/lotus-storage-miner/main.go | 1 + journal/events.go | 16 +++ journal/filesystem.go | 131 +++++++++++++++++ journal/journal.go | 148 -------------------- journal/memory.go | 239 ++++++++++++++++++++++++++++++++ journal/memory_test.go | 178 ++++++++++++++++++++++++ journal/types.go | 60 ++++++++ node/builder.go | 6 +- node/modules/core.go | 6 - storage/wdpost_run.go | 2 +- 13 files changed, 643 insertions(+), 168 deletions(-) create mode 100644 journal/events.go create mode 100644 journal/filesystem.go delete mode 100644 journal/journal.go create mode 100644 journal/memory.go create mode 100644 journal/memory_test.go create mode 100644 journal/types.go diff --git a/chain/events/events_height.go b/chain/events/events_height.go index 24d758a31..b419f06d2 100644 --- a/chain/events/events_height.go +++ b/chain/events/events_height.go @@ -26,7 +26,6 @@ type heightEvents struct { } func (e *heightEvents) headChangeAt(rev, app []*types.TipSet) error { - ctx, span := trace.StartSpan(e.ctx, "events.HeightHeadChange") defer span.End() span.AddAttributes(trace.Int64Attribute("endHeight", int64(app[0].Height()))) @@ -144,12 +143,11 @@ func (e *heightEvents) headChangeAt(rev, app []*types.TipSet) error { } // ChainAt invokes the specified `HeightHandler` when the chain reaches the -// specified height+confidence threshold. If the chain is rolled-back under the -// specified height, `RevertHandler` will be called. +// specified height+confidence threshold. If the chain is rolled-back under the +// specified height, `RevertHandler` will be called. // // ts passed to handlers is the tipset at the specified, or above, if lower tipsets were null func (e *heightEvents) ChainAt(hnd HeightHandler, rev RevertHandler, confidence int, h abi.ChainEpoch) error { - e.lk.Lock() // Tricky locking, check your locks if you modify this function! bestH := e.tsc.best().Height() diff --git a/chain/store/store.go b/chain/store/store.go index fa31f6b08..ec134df1a 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -85,9 +85,10 @@ type ChainStore struct { tsCache *lru.ARCCache vmcalls runtime.Syscalls + journal journal.Journal } -func NewChainStore(bs bstore.Blockstore, ds dstore.Batching, vmcalls runtime.Syscalls) *ChainStore { +func NewChainStore(bs bstore.Blockstore, ds dstore.Batching, vmcalls runtime.Syscalls, journal journal.Journal) *ChainStore { c, _ := lru.NewARC(2048) tsc, _ := lru.NewARC(4096) cs := &ChainStore{ @@ -98,6 +99,7 @@ func NewChainStore(bs bstore.Blockstore, ds dstore.Batching, vmcalls runtime.Sys mmCache: c, tsCache: tsc, vmcalls: vmcalls, + journal: journal, } ci := NewChainIndex(cs.LoadTipSet) @@ -326,12 +328,13 @@ func (cs *ChainStore) reorgWorker(ctx context.Context, initialNotifees []ReorgNo continue } - journal.Add("sync", map[string]interface{}{ - "op": "headChange", - "from": r.old.Key(), - "to": r.new.Key(), - "rev": len(revert), - "apply": len(apply), + cs.journal.AddEntry(journal.EventType{"sync", "head_change"}, map[string]interface{}{ + "from": r.old.Key(), + "from_height": r.old.Height(), + "to": r.new.Key(), + "to_height": r.new.Height(), + "rev_cnt": len(revert), + "apply_cnt": len(apply), }) // reverse the apply array diff --git a/cmd/lotus-seal-worker/main.go b/cmd/lotus-seal-worker/main.go index bf0b420f0..805777e7a 100644 --- a/cmd/lotus-seal-worker/main.go +++ b/cmd/lotus-seal-worker/main.go @@ -37,6 +37,7 @@ import ( var log = logging.Logger("main") const FlagWorkerRepo = "worker-repo" + // TODO remove after deprecation period const FlagWorkerRepoDeprecation = "workerrepo" diff --git a/cmd/lotus-storage-miner/main.go b/cmd/lotus-storage-miner/main.go index 9576aae0b..5fec6ad22 100644 --- a/cmd/lotus-storage-miner/main.go +++ b/cmd/lotus-storage-miner/main.go @@ -22,6 +22,7 @@ import ( var log = logging.Logger("main") const FlagMinerRepo = "miner-repo" + // TODO remove after deprecation period const FlagMinerRepoDeprecation = "storagerepo" diff --git a/journal/events.go b/journal/events.go new file mode 100644 index 000000000..b6b5a15d4 --- /dev/null +++ b/journal/events.go @@ -0,0 +1,16 @@ +package journal + +import ( + "github.com/filecoin-project/specs-actors/actors/abi" + + "github.com/filecoin-project/lotus/chain/types" +) + +type HeadChangeEvt struct { + From types.TipSetKey `json:"from"` + FromHeight abi.ChainEpoch `json:"from_height"` + To types.TipSetKey `json:"to"` + ToHeight abi.ChainEpoch `json:"to_height"` + RevertCount int `json:"rev_cnt"` + ApplyCount int `json:"apply_cnt"` +} diff --git a/journal/filesystem.go b/journal/filesystem.go new file mode 100644 index 000000000..d49f77ff9 --- /dev/null +++ b/journal/filesystem.go @@ -0,0 +1,131 @@ +package journal + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sync" + "time" + + logging "github.com/ipfs/go-log" + "go.uber.org/fx" + "golang.org/x/xerrors" + + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/node/repo" +) + +var log = logging.Logger("journal") + +// fsJournal is a basic journal backed by files on a filesystem. +type fsJournal struct { + disabledTracker + + dir string + sizeLimit int64 + + lk sync.Mutex + fi *os.File + fSize int64 + + incoming chan *Entry + + closing chan struct{} +} + +// OpenFSJournal constructs a rolling filesystem journal, with a default +// per-file size limit of 1GiB. +func OpenFSJournal(lr repo.LockedRepo, lc fx.Lifecycle, disabled []EventType) (Journal, error) { + dir := filepath.Join(lr.Path(), "journal") + if err := os.MkdirAll(dir, 0755); err != nil { + return nil, fmt.Errorf("failed to mk directory %s for file journal: %w", dir, err) + } + + f := &fsJournal{ + disabledTracker: newDisabledTracker(disabled), + dir: dir, + sizeLimit: 1 << 30, + incoming: make(chan *Entry, 32), + closing: make(chan struct{}), + } + + if err := f.rollJournalFile(); err != nil { + return nil, err + } + + lc.Append(fx.Hook{ + OnStop: func(_ context.Context) error { return f.Close() }, + }) + + go f.runLoop() + + return f, nil +} + +func (f *fsJournal) AddEntry(evtType EventType, obj interface{}) { + je := &Entry{ + EventType: evtType, + Timestamp: build.Clock.Now(), + Data: obj, + } + select { + case f.incoming <- je: + case <-f.closing: + log.Warnw("journal closed but tried to log event", "entry", je) + } +} + +func (f *fsJournal) Close() error { + close(f.closing) + return nil +} + +func (f *fsJournal) putEntry(je *Entry) error { + b, err := json.Marshal(je) + if err != nil { + return err + } + n, err := f.fi.Write(append(b, '\n')) + if err != nil { + return err + } + + f.fSize += int64(n) + + if f.fSize >= f.sizeLimit { + _ = f.rollJournalFile() + } + + return nil +} + +func (f *fsJournal) rollJournalFile() error { + if f.fi != nil { + _ = f.fi.Close() + } + + nfi, err := os.Create(filepath.Join(f.dir, fmt.Sprintf("lotus-journal-%s.ndjson", build.Clock.Now().Format(time.RFC3339)))) + if err != nil { + return xerrors.Errorf("failed to open journal file: %w", err) + } + + f.fi = nfi + f.fSize = 0 + return nil +} + +func (f *fsJournal) runLoop() { + for { + select { + case je := <-f.incoming: + if err := f.putEntry(je); err != nil { + log.Errorw("failed to write out journal entry", "entry", je, "err", err) + } + case <-f.closing: + _ = f.fi.Close() + return + } + } +} diff --git a/journal/journal.go b/journal/journal.go deleted file mode 100644 index 8d509d51c..000000000 --- a/journal/journal.go +++ /dev/null @@ -1,148 +0,0 @@ -package journal - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" - "sync" - "time" - - logging "github.com/ipfs/go-log" - "golang.org/x/xerrors" - - "github.com/filecoin-project/lotus/build" -) - -func InitializeSystemJournal(dir string) error { - if err := os.MkdirAll(dir, 0755); err != nil { - return err - } - j, err := OpenFSJournal(dir) - if err != nil { - return err - } - currentJournal = j - return nil -} - -func Add(sys string, val interface{}) { - if currentJournal == nil { - log.Warn("no journal configured") - return - } - currentJournal.AddEntry(sys, val) -} - -var log = logging.Logger("journal") - -var currentJournal Journal - -type Journal interface { - AddEntry(system string, obj interface{}) - Close() error -} - -// fsJournal is a basic journal backed by files on a filesystem -type fsJournal struct { - fi *os.File - fSize int64 - - lk sync.Mutex - - journalDir string - - incoming chan *JournalEntry - journalSizeLimit int64 - - closing chan struct{} -} - -func OpenFSJournal(dir string) (*fsJournal, error) { - fsj := &fsJournal{ - journalDir: dir, - incoming: make(chan *JournalEntry, 32), - journalSizeLimit: 1 << 30, - closing: make(chan struct{}), - } - - if err := fsj.rollJournalFile(); err != nil { - return nil, err - } - - go fsj.runLoop() - - return fsj, nil -} - -type JournalEntry struct { - System string - Timestamp time.Time - Val interface{} -} - -func (fsj *fsJournal) putEntry(je *JournalEntry) error { - b, err := json.Marshal(je) - if err != nil { - return err - } - n, err := fsj.fi.Write(append(b, '\n')) - if err != nil { - return err - } - - fsj.fSize += int64(n) - - if fsj.fSize >= fsj.journalSizeLimit { - fsj.rollJournalFile() - } - - return nil -} - -func (fsj *fsJournal) rollJournalFile() error { - if fsj.fi != nil { - fsj.fi.Close() - } - - nfi, err := os.Create(filepath.Join(fsj.journalDir, fmt.Sprintf("lotus-journal-%s.ndjson", build.Clock.Now().Format(time.RFC3339)))) - if err != nil { - return xerrors.Errorf("failed to open journal file: %w", err) - } - - fsj.fi = nfi - fsj.fSize = 0 - return nil -} - -func (fsj *fsJournal) runLoop() { - for { - select { - case je := <-fsj.incoming: - if err := fsj.putEntry(je); err != nil { - log.Errorw("failed to write out journal entry", "entry", je, "err", err) - } - case <-fsj.closing: - fsj.fi.Close() - return - } - } -} - -func (fsj *fsJournal) AddEntry(system string, obj interface{}) { - je := &JournalEntry{ - System: system, - Timestamp: build.Clock.Now(), - Val: obj, - } - select { - case fsj.incoming <- je: - case <-fsj.closing: - log.Warnw("journal closed but tried to log event", "entry", je) - } -} - -func (fsj *fsJournal) Close() error { - close(fsj.closing) - return nil -} diff --git a/journal/memory.go b/journal/memory.go new file mode 100644 index 000000000..483722635 --- /dev/null +++ b/journal/memory.go @@ -0,0 +1,239 @@ +package journal + +import ( + "context" + "sync/atomic" + + "go.uber.org/fx" + + "github.com/filecoin-project/lotus/build" +) + +// Control messages. +type ( + clearCtrl struct{} + addObserverCtrl struct { + observer *observer + replay bool + } + rmObserverCtrl *observer + getEntriesCtrl chan []*Entry +) + +type MemJournal struct { + disabledTracker + + entries []*Entry + index map[string]map[string][]*Entry + observers []observer + + incomingCh chan *Entry + controlCh chan interface{} + + state int32 // guarded by atomic; 0=closed, 1=running. + closed chan struct{} +} + +var _ Journal = (*MemJournal)(nil) + +type observer struct { + accept map[EventType]struct{} + ch chan *Entry +} + +func (o *observer) dispatch(entry *Entry) { + if o.accept == nil { + o.ch <- entry + } + if _, ok := o.accept[entry.EventType]; ok { + o.ch <- entry + } +} + +func NewMemoryJournal(lc fx.Lifecycle, disabled []EventType) *MemJournal { + m := &MemJournal{ + disabledTracker: newDisabledTracker(disabled), + + index: make(map[string]map[string][]*Entry, 16), + observers: make([]observer, 0, 16), + incomingCh: make(chan *Entry, 256), + controlCh: make(chan interface{}, 16), + state: 1, + closed: make(chan struct{}), + } + + lc.Append(fx.Hook{ + OnStop: func(_ context.Context) error { return m.Close() }, + }) + + go m.process() + + return m +} + +func (m *MemJournal) AddEntry(evtType EventType, obj interface{}) { + entry := &Entry{ + EventType: evtType, + Timestamp: build.Clock.Now(), + Data: obj, + } + + select { + case m.incomingCh <- entry: + case <-m.closed: + } +} + +func (m *MemJournal) Close() error { + if !atomic.CompareAndSwapInt32(&m.state, 1, 0) { + // already closed. + return nil + } + close(m.closed) + return nil +} + +func (m *MemJournal) Clear() { + select { + case m.controlCh <- clearCtrl{}: + case <-m.closed: + } +} + +// Observe starts observing events that are recorded in the MemJournal, and +// returns a channel where new events will be sent. When replay is true, all +// entries that have been recorded prior to the observer being registered will +// be replayed. To restrict the event types this observer will sent, use the +// include argument. If no include set is passed, the observer will receive all +// events types. +func (m *MemJournal) Observe(ctx context.Context, replay bool, include ...EventType) <-chan *Entry { + var acc map[EventType]struct{} + if include != nil { + acc = make(map[EventType]struct{}, 16) + for _, et := range include { + acc[et] = struct{}{} + } + } + + ch := make(chan *Entry, 256) + o := &observer{ + accept: acc, + ch: ch, + } + + // watch the context, and fire the "remove observer" control message upon + // cancellation. + go func() { + <-ctx.Done() + select { + case m.controlCh <- rmObserverCtrl(o): + case <-m.closed: + } + }() + + select { + case m.controlCh <- addObserverCtrl{o, replay}: + case <-m.closed: + // we are already stopped. + close(ch) + } + + return ch +} + +// Entries gets a snapshot of stored entries. +func (m *MemJournal) Entries() []*Entry { + ch := make(chan []*Entry) + m.controlCh <- getEntriesCtrl(ch) + return <-ch +} + +func (m *MemJournal) process() { + processCtrlMsg := func(message interface{}) { + switch msg := message.(type) { + case addObserverCtrl: + // adding an observer. + m.observers = append(m.observers, *msg.observer) + + if msg.replay { + // replay all existing entries. + for _, e := range m.entries { + msg.observer.dispatch(e) + } + } + case rmObserverCtrl: + // removing an observer; find the observer, close its channel. + // then discard it from our list by replacing it with the last + // observer and reslicing. + for i, o := range m.observers { + if o.ch == msg.ch { + close(o.ch) + m.observers[i] = m.observers[len(m.observers)-1] + m.observers = m.observers[:len(m.observers)-1] + } + } + case clearCtrl: + m.entries = m.entries[0:0] + // carry over system and event names; there are unlikely to change; + // just reslice the entry slices, so we are not thrashing memory. + for _, events := range m.index { + for ev := range events { + events[ev] = events[ev][0:0] + } + } + case getEntriesCtrl: + cpy := make([]*Entry, len(m.entries)) + copy(cpy, m.entries) + msg <- cpy + close(msg) + } + } + + processClose := func() { + m.entries = nil + m.index = make(map[string]map[string][]*Entry, 16) + for _, o := range m.observers { + close(o.ch) + } + m.observers = nil + } + + for { + // Drain all control messages first! + select { + case msg := <-m.controlCh: + processCtrlMsg(msg) + continue + case <-m.closed: + processClose() + return + default: + } + + // Now consume and pipe messages. + select { + case entry := <-m.incomingCh: + m.entries = append(m.entries, entry) + events := m.index[entry.System] + if events == nil { + events = make(map[string][]*Entry, 16) + m.index[entry.System] = events + } + + entries := events[entry.Event] + events[entry.Event] = append(entries, entry) + + for _, o := range m.observers { + o.dispatch(entry) + } + + case msg := <-m.controlCh: + processCtrlMsg(msg) + continue + + case <-m.closed: + processClose() + return + } + } +} diff --git a/journal/memory_test.go b/journal/memory_test.go new file mode 100644 index 000000000..10b3a90cb --- /dev/null +++ b/journal/memory_test.go @@ -0,0 +1,178 @@ +package journal + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/specs-actors/actors/abi" + + "github.com/raulk/clock" + "github.com/stretchr/testify/require" + "go.uber.org/fx/fxtest" +) + +func TestMemJournal_AddEntry(t *testing.T) { + lc := fxtest.NewLifecycle(t) + defer lc.RequireStop() + + clk := clock.NewMock() + build.Clock = clk + + journal := NewMemoryJournal(lc, nil) + addEntries(journal, 100) + + require.Eventually(t, func() bool { return len(journal.Entries()) == 100 }, 1*time.Second, 100*time.Millisecond) + + entries := journal.Entries() + cnt := make(map[string]int, 10) + for i, e := range entries { + require.EqualValues(t, "spaceship", e.System) + require.Equal(t, HeadChangeEvt{ + From: types.TipSetKey{}, + FromHeight: abi.ChainEpoch(i), + To: types.TipSetKey{}, + ToHeight: abi.ChainEpoch(i), + RevertCount: i, + ApplyCount: i, + }, e.Data) + require.Equal(t, build.Clock.Now(), e.Timestamp) + cnt[e.Event]++ + } + + // we received 10 entries of each event type. + for _, c := range cnt { + require.Equal(t, 10, c) + } +} + +func TestMemJournal_Close(t *testing.T) { + lc := fxtest.NewLifecycle(t) + defer lc.RequireStop() + + journal := NewMemoryJournal(lc, nil) + addEntries(journal, 100) + + require.Eventually(t, func() bool { return len(journal.Entries()) == 100 }, 1*time.Second, 100*time.Millisecond) + + o1 := journal.Observe(context.TODO(), false) + o2 := journal.Observe(context.TODO(), false) + o3 := journal.Observe(context.TODO(), false) + + time.Sleep(500 * time.Millisecond) + + // Close the journal. + require.NoError(t, journal.Close()) + + time.Sleep(500 * time.Millisecond) + +NextChannel: + for _, ch := range []<-chan *Entry{o1, o2, o3} { + for { + select { + case _, more := <-ch: + if more { + // keep consuming + } else { + continue NextChannel + } + default: + t.Fatal("nothing more to consume, and channel is not closed") + } + } + } +} + +func TestMemJournal_Clear(t *testing.T) { + lc := fxtest.NewLifecycle(t) + defer lc.RequireStop() + + journal := NewMemoryJournal(lc, nil) + addEntries(journal, 100) + + require.Eventually(t, func() bool { return len(journal.Entries()) == 100 }, 1*time.Second, 100*time.Millisecond) + + journal.Clear() + require.Empty(t, journal.Entries()) + require.Empty(t, journal.Entries()) + require.Empty(t, journal.Entries()) +} + +func TestMemJournal_Observe(t *testing.T) { + lc := fxtest.NewLifecycle(t) + defer lc.RequireStop() + + journal := NewMemoryJournal(lc, nil) + addEntries(journal, 100) + + require.Eventually(t, func() bool { return len(journal.Entries()) == 100 }, 1*time.Second, 100*time.Millisecond) + + o1 := journal.Observe(context.TODO(), false, EventType{"spaceship", "wheezing-1"}) + o2 := journal.Observe(context.TODO(), true, EventType{"spaceship", "wheezing-1"}, EventType{"spaceship", "wheezing-2"}) + o3 := journal.Observe(context.TODO(), true) + + time.Sleep(1 * time.Second) + + require.Len(t, o1, 0) // no replay + require.Len(t, o2, 20) // replay with include set + require.Len(t, o3, 100) // replay with no include set (all entries) + + // add another 100 entries and assert what the observers have seen. + addEntries(journal, 100) + + require.Eventually(t, func() bool { return len(journal.Entries()) == 200 }, 1*time.Second, 100*time.Millisecond) + + // note: we're able to queue items because the observer channel buffer size is 256. + require.Len(t, o1, 10) // should have 0 old entries + 10 new entries + require.Len(t, o2, 40) // should have 20 old entries + 20 new entries + require.Len(t, o3, 200) // should have 100 old entries + 100 new entries +} + +func TestMemJournal_ObserverCancellation(t *testing.T) { + lc := fxtest.NewLifecycle(t) + defer lc.RequireStop() + + journal := NewMemoryJournal(lc, nil) + + ctx, cancel := context.WithCancel(context.TODO()) + o1 := journal.Observe(ctx, false) + o2 := journal.Observe(context.TODO(), false) + addEntries(journal, 100) + + require.Eventually(t, func() bool { return len(journal.Entries()) == 100 }, 1*time.Second, 100*time.Millisecond) + + // all observers have received the 100 entries. + require.Len(t, o1, 100) + require.Len(t, o2, 100) + + // cancel o1's context. + cancel() + time.Sleep(500 * time.Millisecond) + + // add 50 new entries + addEntries(journal, 50) + + require.Eventually(t, func() bool { return len(journal.Entries()) == 150 }, 1*time.Second, 100*time.Millisecond) + + require.Len(t, o1, 100) // has not moved. + require.Len(t, o2, 150) // should have 100 old entries + 50 new entries + +} + +func addEntries(journal *MemJournal, count int) { + for i := 0; i < count; i++ { + eventIdx := i % 10 + journal.AddEntry(EventType{"spaceship", fmt.Sprintf("wheezing-%d", eventIdx)}, HeadChangeEvt{ + From: types.TipSetKey{}, + FromHeight: abi.ChainEpoch(i), + To: types.TipSetKey{}, + ToHeight: abi.ChainEpoch(i), + RevertCount: i, + ApplyCount: i, + }) + } +} diff --git a/journal/types.go b/journal/types.go new file mode 100644 index 000000000..d8c897cfa --- /dev/null +++ b/journal/types.go @@ -0,0 +1,60 @@ +package journal + +import "time" + +// EventType represents the signature of an event. +type EventType struct { + System string + Event string +} + +// Journal represents an audit trail of system actions. +// +// Every entry is tagged with a timestamp, a system name, and an event name. +// The supplied data can be any type, as long as it is JSON serializable, +// including structs, map[string]interface{}, or primitive types. +// +// For cleanliness and type safety, we recommend to use typed events. See the +// *Evt struct types in this package for more info. +type Journal interface { + // IsEnabled allows components to check if a given event type is enabled. + // All event types are enabled by default, and specific event types can only + // be disabled at construction type. Components are advised to check if the + // journal event types they record are enabled as soon as possible, and hold + // on to the answer all throughout the lifetime of the process. + IsEnabled(evtType EventType) bool + + // AddEntry adds an entry to this journal. See godocs on the Journal type + // for more info. + AddEntry(evtType EventType, data interface{}) + + // Close closes this journal for further writing. + Close() error +} + +// Entry represents a journal entry. +// +// See godocs on Journal for more information. +type Entry struct { + EventType + + Timestamp time.Time + Data interface{} +} + +// disabledTracker is an embeddable mixin that takes care of tracking disabled +// event types. +type disabledTracker map[EventType]struct{} + +func newDisabledTracker(disabled []EventType) disabledTracker { + dis := make(map[EventType]struct{}, len(disabled)) + for _, et := range disabled { + dis[et] = struct{}{} + } + return dis +} + +func (d disabledTracker) IsEnabled(evtType EventType) bool { + _, ok := d[evtType] + return !ok +} diff --git a/node/builder.go b/node/builder.go index 780497dba..90b150a8a 100644 --- a/node/builder.go +++ b/node/builder.go @@ -41,6 +41,7 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" "github.com/filecoin-project/lotus/chain/wallet" + "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/lib/peermgr" _ "github.com/filecoin-project/lotus/lib/sigs/bls" _ "github.com/filecoin-project/lotus/lib/sigs/secp" @@ -119,7 +120,7 @@ const ( ExtractApiKey HeadMetricsKey RunPeerTaggerKey - JournalKey + InitJournalKey SetApiEndpointKey @@ -151,8 +152,9 @@ func defaults() []Option { Override(new(record.Validator), modules.RecordValidator), Override(new(dtypes.Bootstrapper), dtypes.Bootstrapper(false)), Override(new(dtypes.ShutdownChan), make(chan struct{})), - Override(JournalKey, modules.SetupJournal), + Override(new(journal.Journal), journal.OpenFSJournal), + Override(InitJournalKey, func(j *journal.Journal) { /* forces the creation of the journal at startup */ }), // Filecoin modules } diff --git a/node/modules/core.go b/node/modules/core.go index 84179bd63..ca9872d90 100644 --- a/node/modules/core.go +++ b/node/modules/core.go @@ -6,7 +6,6 @@ import ( "errors" "io" "io/ioutil" - "path/filepath" "github.com/gbrlsnchs/jwt/v3" logging "github.com/ipfs/go-log/v2" @@ -19,7 +18,6 @@ import ( "github.com/filecoin-project/lotus/api/apistruct" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/lib/addrutil" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/repo" @@ -95,7 +93,3 @@ func BuiltinBootstrap() (dtypes.BootstrapPeers, error) { func DrandBootstrap() (dtypes.DrandBootstrap, error) { return build.DrandBootstrap() } - -func SetupJournal(lr repo.LockedRepo) error { - return journal.InitializeSystemJournal(filepath.Join(lr.Path(), "journal")) -} diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 8e404610c..66bb0d53e 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -256,7 +256,7 @@ func (s *WindowPoStScheduler) checkNextFaults(ctx context.Context, deadline uint Method: builtin.MethodsMiner.DeclareFaults, Params: enc, Value: types.NewInt(0), // TODO: Is there a fee? - GasLimit: 100_000_000, // i dont know help + GasLimit: 100_000_000, // i dont know help GasPrice: types.NewInt(2), } From bef7d64e666242c19e4f76cb88422c64947e8f00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Fri, 17 Jul 2020 18:35:15 +0100 Subject: [PATCH 002/885] refactor EventType construction. --- journal/filesystem.go | 12 +++---- journal/memory.go | 15 ++++++-- journal/memory_test.go | 13 ++++--- journal/types.go | 80 ++++++++++++++++++++++++++++++++---------- 4 files changed, 89 insertions(+), 31 deletions(-) diff --git a/journal/filesystem.go b/journal/filesystem.go index d49f77ff9..dbc6d2c27 100644 --- a/journal/filesystem.go +++ b/journal/filesystem.go @@ -21,7 +21,7 @@ var log = logging.Logger("journal") // fsJournal is a basic journal backed by files on a filesystem. type fsJournal struct { - disabledTracker + *eventTypeFactory dir string sizeLimit int64 @@ -44,11 +44,11 @@ func OpenFSJournal(lr repo.LockedRepo, lc fx.Lifecycle, disabled []EventType) (J } f := &fsJournal{ - disabledTracker: newDisabledTracker(disabled), - dir: dir, - sizeLimit: 1 << 30, - incoming: make(chan *Entry, 32), - closing: make(chan struct{}), + eventTypeFactory: newEventTypeFactory(disabled), + dir: dir, + sizeLimit: 1 << 30, + incoming: make(chan *Entry, 32), + closing: make(chan struct{}), } if err := f.rollJournalFile(); err != nil { diff --git a/journal/memory.go b/journal/memory.go index 483722635..c1472ccca 100644 --- a/journal/memory.go +++ b/journal/memory.go @@ -21,7 +21,7 @@ type ( ) type MemJournal struct { - disabledTracker + *eventTypeFactory entries []*Entry index map[string]map[string][]*Entry @@ -52,7 +52,7 @@ func (o *observer) dispatch(entry *Entry) { func NewMemoryJournal(lc fx.Lifecycle, disabled []EventType) *MemJournal { m := &MemJournal{ - disabledTracker: newDisabledTracker(disabled), + eventTypeFactory: newEventTypeFactory(disabled), index: make(map[string]map[string][]*Entry, 16), observers: make([]observer, 0, 16), @@ -72,6 +72,11 @@ func NewMemoryJournal(lc fx.Lifecycle, disabled []EventType) *MemJournal { } func (m *MemJournal) AddEntry(evtType EventType, obj interface{}) { + if !evtType.enabled || !evtType.safe { + // tried to record a disabled event type, or used an invalid EventType. + return + } + entry := &Entry{ EventType: evtType, Timestamp: build.Clock.Now(), @@ -109,8 +114,12 @@ func (m *MemJournal) Clear() { func (m *MemJournal) Observe(ctx context.Context, replay bool, include ...EventType) <-chan *Entry { var acc map[EventType]struct{} if include != nil { - acc = make(map[EventType]struct{}, 16) + acc = make(map[EventType]struct{}, len(include)) for _, et := range include { + if !et.enabled { + // skip over disabled event type. + continue + } acc[et] = struct{}{} } } diff --git a/journal/memory_test.go b/journal/memory_test.go index 10b3a90cb..442a7f46d 100644 --- a/journal/memory_test.go +++ b/journal/memory_test.go @@ -111,8 +111,11 @@ func TestMemJournal_Observe(t *testing.T) { require.Eventually(t, func() bool { return len(journal.Entries()) == 100 }, 1*time.Second, 100*time.Millisecond) - o1 := journal.Observe(context.TODO(), false, EventType{"spaceship", "wheezing-1"}) - o2 := journal.Observe(context.TODO(), true, EventType{"spaceship", "wheezing-1"}, EventType{"spaceship", "wheezing-2"}) + et1 := journal.RegisterEventType("spaceship", "wheezing-1") + et2 := journal.RegisterEventType("spaceship", "wheezing-2") + + o1 := journal.Observe(context.TODO(), false, et1) + o2 := journal.Observe(context.TODO(), true, et1, et2) o3 := journal.Observe(context.TODO(), true) time.Sleep(1 * time.Second) @@ -160,13 +163,15 @@ func TestMemJournal_ObserverCancellation(t *testing.T) { require.Len(t, o1, 100) // has not moved. require.Len(t, o2, 150) // should have 100 old entries + 50 new entries - } func addEntries(journal *MemJournal, count int) { for i := 0; i < count; i++ { eventIdx := i % 10 - journal.AddEntry(EventType{"spaceship", fmt.Sprintf("wheezing-%d", eventIdx)}, HeadChangeEvt{ + + // RegisterEventType is not _really_ intended to be used this way (on every write). + et := journal.RegisterEventType("spaceship", fmt.Sprintf("wheezing-%d", eventIdx)) + journal.AddEntry(et, HeadChangeEvt{ From: types.TipSetKey{}, FromHeight: abi.ChainEpoch(i), To: types.TipSetKey{}, diff --git a/journal/types.go b/journal/types.go index d8c897cfa..7a166d58a 100644 --- a/journal/types.go +++ b/journal/types.go @@ -1,11 +1,32 @@ package journal -import "time" +import ( + "sync" + "time" +) // EventType represents the signature of an event. type EventType struct { System string Event string + + // enabled stores whether this event type is enabled. + enabled bool + + // safe is a sentinel marker that's set to true if this EventType was + // constructed correctly (via Journal#RegisterEventType). + safe bool +} + +// Enabled returns whether this event type is enabled in the journaling +// subsystem. Users are advised to check this before actually attempting to +// add a journal entry, as it helps bypass object construction for events that +// would be discarded anyway. +// +// All event types are enabled by default, and specific event types can only +// be disabled at Journal construction time. +func (et EventType) Enabled() bool { + return et.enabled } // Journal represents an audit trail of system actions. @@ -17,12 +38,11 @@ type EventType struct { // For cleanliness and type safety, we recommend to use typed events. See the // *Evt struct types in this package for more info. type Journal interface { - // IsEnabled allows components to check if a given event type is enabled. - // All event types are enabled by default, and specific event types can only - // be disabled at construction type. Components are advised to check if the - // journal event types they record are enabled as soon as possible, and hold - // on to the answer all throughout the lifetime of the process. - IsEnabled(evtType EventType) bool + // RegisterEventType introduces a new event type to this journal, and + // returns an EventType token that components can later use to check whether + // journalling for that type is enabled/suppressed, and to tag journal + // entries appropriately. + RegisterEventType(system, event string) EventType // AddEntry adds an entry to this journal. See godocs on the Journal type // for more info. @@ -42,19 +62,43 @@ type Entry struct { Data interface{} } -// disabledTracker is an embeddable mixin that takes care of tracking disabled -// event types. -type disabledTracker map[EventType]struct{} +// eventTypeFactory is an embeddable mixin that takes care of tracking disabled +// event types, and returning initialized/safe EventTypes when requested. +type eventTypeFactory struct { + sync.Mutex -func newDisabledTracker(disabled []EventType) disabledTracker { - dis := make(map[EventType]struct{}, len(disabled)) - for _, et := range disabled { - dis[et] = struct{}{} + m map[string]EventType +} + +func newEventTypeFactory(disabled []EventType) *eventTypeFactory { + ret := &eventTypeFactory{ + m: make(map[string]EventType, len(disabled)+32), // + extra capacity. } - return dis + + for _, et := range disabled { + et.enabled, et.safe = false, true + ret.m[et.System+":"+et.Event] = et + } + + return ret } -func (d disabledTracker) IsEnabled(evtType EventType) bool { - _, ok := d[evtType] - return !ok +func (d *eventTypeFactory) RegisterEventType(system, event string) EventType { + d.Lock() + defer d.Unlock() + + key := system + ":" + event + if et, ok := d.m[key]; ok { + return et + } + + et := EventType{ + System: system, + Event: event, + enabled: true, + safe: true, + } + + d.m[key] = et + return et } From 4bcf01938c151cf4a0d1c4cbf99fe18b36f885f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Fri, 17 Jul 2020 18:54:26 +0100 Subject: [PATCH 003/885] use new journal from chain/store pkg. --- chain/store/store.go | 33 +++++++++++++++++++++++---------- journal/sugar.go | 11 +++++++++++ 2 files changed, 34 insertions(+), 10 deletions(-) create mode 100644 journal/sugar.go diff --git a/chain/store/store.go b/chain/store/store.go index ec134df1a..fd670eee9 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -54,6 +54,11 @@ var blockValidationCacheKeyPrefix = dstore.NewKey("blockValidation") // ReorgNotifee represents a callback that gets called upon reorgs. type ReorgNotifee func(rev, app []*types.TipSet) error +// Journal event types. +const ( + evtTypeHeadChange = iota +) + // ChainStore is the main point of access to chain data. // // Raw chain data is stored in the Blockstore, with relevant markers (genesis, @@ -85,10 +90,12 @@ type ChainStore struct { tsCache *lru.ARCCache vmcalls runtime.Syscalls - journal journal.Journal + + journal journal.Journal + evtTypes [1]journal.EventType } -func NewChainStore(bs bstore.Blockstore, ds dstore.Batching, vmcalls runtime.Syscalls, journal journal.Journal) *ChainStore { +func NewChainStore(bs bstore.Blockstore, ds dstore.Batching, vmcalls runtime.Syscalls, jrnl journal.Journal) *ChainStore { c, _ := lru.NewARC(2048) tsc, _ := lru.NewARC(4096) cs := &ChainStore{ @@ -99,7 +106,11 @@ func NewChainStore(bs bstore.Blockstore, ds dstore.Batching, vmcalls runtime.Sys mmCache: c, tsCache: tsc, vmcalls: vmcalls, - journal: journal, + journal: jrnl, + } + + cs.evtTypes = [1]journal.EventType{ + evtTypeHeadChange: jrnl.RegisterEventType("sync", "head_change"), } ci := NewChainIndex(cs.LoadTipSet) @@ -328,13 +339,15 @@ func (cs *ChainStore) reorgWorker(ctx context.Context, initialNotifees []ReorgNo continue } - cs.journal.AddEntry(journal.EventType{"sync", "head_change"}, map[string]interface{}{ - "from": r.old.Key(), - "from_height": r.old.Height(), - "to": r.new.Key(), - "to_height": r.new.Height(), - "rev_cnt": len(revert), - "apply_cnt": len(apply), + journal.MaybeAddEntry(cs.journal, cs.evtTypes[evtTypeHeadChange], func() interface{} { + return journal.HeadChangeEvt{ + From: r.old.Key(), + FromHeight: r.old.Height(), + To: r.new.Key(), + ToHeight: r.new.Height(), + RevertCount: len(revert), + ApplyCount: len(apply), + } }) // reverse the apply array diff --git a/journal/sugar.go b/journal/sugar.go new file mode 100644 index 000000000..6ed1bfc32 --- /dev/null +++ b/journal/sugar.go @@ -0,0 +1,11 @@ +package journal + +// MaybeAddEntry is a convenience function that evaluates if the EventType is +// enabled, and if so, it calls the supplier to create the entry and +// subsequently journal.AddEntry on the provided journal to record it. +func MaybeAddEntry(journal Journal, evtType EventType, supplier func() interface{}) { + if !evtType.Enabled() { + return + } + journal.AddEntry(evtType, supplier()) +} From becbff0b134e84e37e4cd03349964ef13ac67f4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Fri, 17 Jul 2020 19:09:18 +0100 Subject: [PATCH 004/885] add NilJournal; fix call sites to ChainStore constructor. --- chain/blocksync/graphsync_client.go | 3 ++- chain/gen/gen.go | 3 ++- chain/gen/genesis/genesis.go | 3 ++- chain/validation/applier.go | 3 ++- cmd/lotus-bench/import.go | 3 ++- cmd/lotus/daemon.go | 3 ++- journal/nil.go | 16 ++++++++++++++++ journal/sugar.go | 6 ++++++ node/modules/chain.go | 5 +++-- 9 files changed, 37 insertions(+), 8 deletions(-) create mode 100644 journal/nil.go diff --git a/chain/blocksync/graphsync_client.go b/chain/blocksync/graphsync_client.go index 03e4a30e5..338ee61bf 100644 --- a/chain/blocksync/graphsync_client.go +++ b/chain/blocksync/graphsync_client.go @@ -12,6 +12,7 @@ import ( store "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/journal" cidlink "github.com/ipld/go-ipld-prime/linking/cid" basicnode "github.com/ipld/go-ipld-prime/node/basic" @@ -138,7 +139,7 @@ func (bs *BlockSync) fetchBlocksGraphSync(ctx context.Context, p peer.ID, req *B } // Now pull the data we fetched out of the chainstore (where it should now be persisted) - tempcs := store.NewChainStore(bs.bserv.Blockstore(), datastore.NewMapDatastore(), nil) + tempcs := store.NewChainStore(bs.bserv.Blockstore(), datastore.NewMapDatastore(), nil, journal.NilJournal()) opts := ParseBSOptions(req.Options) tsk := types.NewTipSetKey(req.Start...) diff --git a/chain/gen/gen.go b/chain/gen/gen.go index 990383ad6..2658772ff 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -36,6 +36,7 @@ import ( "github.com/filecoin-project/lotus/chain/wallet" "github.com/filecoin-project/lotus/cmd/lotus-seed/seed" "github.com/filecoin-project/lotus/genesis" + "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/lib/sigs" "github.com/filecoin-project/lotus/node/repo" "github.com/filecoin-project/sector-storage/ffiwrapper" @@ -204,7 +205,7 @@ func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) { return nil, xerrors.Errorf("make genesis block failed: %w", err) } - cs := store.NewChainStore(bs, ds, sys) + cs := store.NewChainStore(bs, ds, sys, journal.NilJournal()) genfb := &types.FullBlock{Header: genb.Genesis} gents := store.NewFullTipSet([]*types.FullBlock{genfb}) diff --git a/chain/gen/genesis/genesis.go b/chain/gen/genesis/genesis.go index b5b9dfe28..0f99f22a5 100644 --- a/chain/gen/genesis/genesis.go +++ b/chain/gen/genesis/genesis.go @@ -24,6 +24,7 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" "github.com/filecoin-project/lotus/genesis" + "github.com/filecoin-project/lotus/journal" ) const AccountStart = 100 @@ -302,7 +303,7 @@ func MakeGenesisBlock(ctx context.Context, bs bstore.Blockstore, sys runtime.Sys } // temp chainstore - cs := store.NewChainStore(bs, datastore.NewMapDatastore(), sys) + cs := store.NewChainStore(bs, datastore.NewMapDatastore(), sys, journal.NilJournal()) // Verify PreSealed Data stateroot, err = VerifyPreSealedData(ctx, cs, stateroot, template) diff --git a/chain/validation/applier.go b/chain/validation/applier.go index 153c8ca71..7bde26495 100644 --- a/chain/validation/applier.go +++ b/chain/validation/applier.go @@ -20,6 +20,7 @@ import ( "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" + "github.com/filecoin-project/lotus/journal" ) // Applier applies messages to state trees and storage. @@ -67,7 +68,7 @@ func (a *Applier) ApplySignedMessage(epoch abi.ChainEpoch, msg *vtypes.SignedMes } func (a *Applier) ApplyTipSetMessages(epoch abi.ChainEpoch, blocks []vtypes.BlockMessagesInfo, rnd vstate.RandomnessSource) (vtypes.ApplyTipSetResult, error) { - cs := store.NewChainStore(a.stateWrapper.bs, a.stateWrapper.ds, a.syscalls) + cs := store.NewChainStore(a.stateWrapper.bs, a.stateWrapper.ds, a.syscalls, journal.NilJournal()) sm := stmgr.NewStateManager(cs) var bms []stmgr.BlockMessages diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index 3c56a5707..9bd520ff9 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -21,6 +21,7 @@ import ( "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" + "github.com/filecoin-project/lotus/journal" _ "github.com/filecoin-project/lotus/lib/sigs/bls" _ "github.com/filecoin-project/lotus/lib/sigs/secp" "github.com/filecoin-project/sector-storage/ffiwrapper" @@ -85,7 +86,7 @@ var importBenchCmd = &cli.Command{ } bs = cbs ds := datastore.NewMapDatastore() - cs := store.NewChainStore(bs, ds, vm.Syscalls(ffiwrapper.ProofVerifier)) + cs := store.NewChainStore(bs, ds, vm.Syscalls(ffiwrapper.ProofVerifier), journal.NilJournal()) stm := stmgr.NewStateManager(cs) prof, err := os.Create("import-bench.prof") diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index bb17c0d87..49c052d0f 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -13,6 +13,7 @@ import ( "strings" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/journal" paramfetch "github.com/filecoin-project/go-paramfetch" blockstore "github.com/ipfs/go-ipfs-blockstore" @@ -336,7 +337,7 @@ func ImportChain(r repo.Repo, fname string) error { bs := blockstore.NewBlockstore(ds) - cst := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier)) + cst := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier), journal.NilJournal()) log.Info("importing chain from file...") ts, err := cst.Import(fi) diff --git a/journal/nil.go b/journal/nil.go new file mode 100644 index 000000000..0123fe585 --- /dev/null +++ b/journal/nil.go @@ -0,0 +1,16 @@ +package journal + +type nilJournal struct{} + +// nilj is a singleton nil journal. +var nilj Journal = &nilJournal{} + +func NilJournal() Journal { + return nilj +} + +func (n *nilJournal) RegisterEventType(_, _ string) EventType { return EventType{} } + +func (n *nilJournal) AddEntry(_ EventType, _ interface{}) {} + +func (n *nilJournal) Close() error { return nil } diff --git a/journal/sugar.go b/journal/sugar.go index 6ed1bfc32..9ee59f2c7 100644 --- a/journal/sugar.go +++ b/journal/sugar.go @@ -3,7 +3,13 @@ package journal // MaybeAddEntry is a convenience function that evaluates if the EventType is // enabled, and if so, it calls the supplier to create the entry and // subsequently journal.AddEntry on the provided journal to record it. +// +// This is safe to call with a nil Journal, either because the value is nil, +// or because a journal obtained through NilJournal() is in use. func MaybeAddEntry(journal Journal, evtType EventType, supplier func() interface{}) { + if journal == nil || journal == nilj { + return + } if !evtType.Enabled() { return } diff --git a/node/modules/chain.go b/node/modules/chain.go index 229a97cc4..9f5ac05a3 100644 --- a/node/modules/chain.go +++ b/node/modules/chain.go @@ -26,6 +26,7 @@ import ( "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/helpers" "github.com/filecoin-project/lotus/node/repo" @@ -83,8 +84,8 @@ func ChainBlockservice(bs dtypes.ChainBlockstore, rem dtypes.ChainExchange) dtyp return blockservice.New(bs, rem) } -func ChainStore(lc fx.Lifecycle, bs dtypes.ChainBlockstore, ds dtypes.MetadataDS, syscalls runtime.Syscalls) *store.ChainStore { - chain := store.NewChainStore(bs, ds, syscalls) +func ChainStore(lc fx.Lifecycle, bs dtypes.ChainBlockstore, ds dtypes.MetadataDS, syscalls runtime.Syscalls, journal journal.Journal) *store.ChainStore { + chain := store.NewChainStore(bs, ds, syscalls, journal) if err := chain.Load(); err != nil { log.Warnf("loading chain state from disk: %s", err) From 226786c1daeae33a5d4ef9b11ec572c97c300d40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Mon, 20 Jul 2020 14:45:17 +0100 Subject: [PATCH 005/885] wip --- api/api_full.go | 2 +- api/apistruct/struct.go | 2 +- chain/gen/genesis/genesis.go | 14 ++--- chain/store/store.go | 11 +++- journal/events.go | 16 ------ journal/filesystem.go | 2 +- journal/memory.go | 2 +- journal/types.go | 5 +- node/builder.go | 1 + node/modules/storageminer.go | 32 ++++++++++- storage/journal_events.go | 34 ++++++++++++ storage/wdpost_run.go | 104 ++++++++++++++++++++++++++++++++--- storage/wdpost_sched.go | 27 ++++++++- 13 files changed, 210 insertions(+), 42 deletions(-) delete mode 100644 journal/events.go create mode 100644 storage/journal_events.go diff --git a/api/api_full.go b/api/api_full.go index 9821b4557..895c1e65b 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -375,7 +375,7 @@ type FileRef struct { type MinerSectors struct { Sectors uint64 - Active uint64 + Active uint64 } type SectorExpiration struct { diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 326e98453..d5efd5266 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -127,7 +127,7 @@ type FullNodeStruct struct { StateNetworkName func(context.Context) (dtypes.NetworkName, error) `perm:"read"` StateMinerSectors func(context.Context, address.Address, *abi.BitField, bool, types.TipSetKey) ([]*api.ChainSectorInfo, error) `perm:"read"` - StateMinerActiveSectors func(context.Context, address.Address, types.TipSetKey) ([]*api.ChainSectorInfo, error) `perm:"read"` + StateMinerActiveSectors func(context.Context, address.Address, types.TipSetKey) ([]*api.ChainSectorInfo, error) `perm:"read"` StateMinerProvingDeadline func(context.Context, address.Address, types.TipSetKey) (*miner.DeadlineInfo, error) `perm:"read"` StateMinerPower func(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error) `perm:"read"` StateMinerInfo func(context.Context, address.Address, types.TipSetKey) (api.MinerInfo, error) `perm:"read"` diff --git a/chain/gen/genesis/genesis.go b/chain/gen/genesis/genesis.go index f72ab7bdd..22376a088 100644 --- a/chain/gen/genesis/genesis.go +++ b/chain/gen/genesis/genesis.go @@ -209,7 +209,7 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge // var newAddress address.Address - if (info.Type == genesis.TAccount) { + if info.Type == genesis.TAccount { var ainfo genesis.AccountMeta if err := json.Unmarshal(info.Meta, &ainfo); err != nil { return nil, xerrors.Errorf("unmarshaling account meta: %w", err) @@ -226,7 +226,7 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge if err != nil { return nil, xerrors.Errorf("setting account from actmap: %w", err) } - } else if (info.Type == genesis.TMultisig) { + } else if info.Type == genesis.TMultisig { var ainfo genesis.MultisigMeta if err := json.Unmarshal(info.Meta, &ainfo); err != nil { return nil, xerrors.Errorf("unmarshaling account meta: %w", err) @@ -238,12 +238,12 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge } st, err := cst.Put(ctx, &multisig.State{ - Signers: ainfo.Signers, + Signers: ainfo.Signers, NumApprovalsThreshold: uint64(ainfo.Threshold), - StartEpoch: abi.ChainEpoch(ainfo.VestingStart), - UnlockDuration: abi.ChainEpoch(ainfo.VestingDuration), - PendingTxns: pending, - InitialBalance: info.Balance, + StartEpoch: abi.ChainEpoch(ainfo.VestingStart), + UnlockDuration: abi.ChainEpoch(ainfo.VestingDuration), + PendingTxns: pending, + InitialBalance: info.Balance, }) if err != nil { return nil, err diff --git a/chain/store/store.go b/chain/store/store.go index 8ab599aa3..ba7a54ed3 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -59,6 +59,15 @@ const ( evtTypeHeadChange = iota ) +type HeadChangeEvt struct { + From types.TipSetKey + FromHeight abi.ChainEpoch + To types.TipSetKey + ToHeight abi.ChainEpoch + RevertCount int + ApplyCount int +} + // ChainStore is the main point of access to chain data. // // Raw chain data is stored in the Blockstore, with relevant markers (genesis, @@ -340,7 +349,7 @@ func (cs *ChainStore) reorgWorker(ctx context.Context, initialNotifees []ReorgNo } journal.MaybeAddEntry(cs.journal, cs.evtTypes[evtTypeHeadChange], func() interface{} { - return journal.HeadChangeEvt{ + return HeadChangeEvt{ From: r.old.Key(), FromHeight: r.old.Height(), To: r.new.Key(), diff --git a/journal/events.go b/journal/events.go deleted file mode 100644 index b6b5a15d4..000000000 --- a/journal/events.go +++ /dev/null @@ -1,16 +0,0 @@ -package journal - -import ( - "github.com/filecoin-project/specs-actors/actors/abi" - - "github.com/filecoin-project/lotus/chain/types" -) - -type HeadChangeEvt struct { - From types.TipSetKey `json:"from"` - FromHeight abi.ChainEpoch `json:"from_height"` - To types.TipSetKey `json:"to"` - ToHeight abi.ChainEpoch `json:"to_height"` - RevertCount int `json:"rev_cnt"` - ApplyCount int `json:"apply_cnt"` -} diff --git a/journal/filesystem.go b/journal/filesystem.go index dbc6d2c27..7aa470a2e 100644 --- a/journal/filesystem.go +++ b/journal/filesystem.go @@ -37,7 +37,7 @@ type fsJournal struct { // OpenFSJournal constructs a rolling filesystem journal, with a default // per-file size limit of 1GiB. -func OpenFSJournal(lr repo.LockedRepo, lc fx.Lifecycle, disabled []EventType) (Journal, error) { +func OpenFSJournal(lr repo.LockedRepo, lc fx.Lifecycle, disabled DisabledEvents) (Journal, error) { dir := filepath.Join(lr.Path(), "journal") if err := os.MkdirAll(dir, 0755); err != nil { return nil, fmt.Errorf("failed to mk directory %s for file journal: %w", dir, err) diff --git a/journal/memory.go b/journal/memory.go index c1472ccca..5471d3e11 100644 --- a/journal/memory.go +++ b/journal/memory.go @@ -50,7 +50,7 @@ func (o *observer) dispatch(entry *Entry) { } } -func NewMemoryJournal(lc fx.Lifecycle, disabled []EventType) *MemJournal { +func NewMemoryJournal(lc fx.Lifecycle, disabled DisabledEvents) *MemJournal { m := &MemJournal{ eventTypeFactory: newEventTypeFactory(disabled), diff --git a/journal/types.go b/journal/types.go index 7a166d58a..95349b24e 100644 --- a/journal/types.go +++ b/journal/types.go @@ -5,6 +5,9 @@ import ( "time" ) +// DisabledEvents is the set of event types whose journaling is suppressed. +type DisabledEvents []EventType + // EventType represents the signature of an event. type EventType struct { System string @@ -70,7 +73,7 @@ type eventTypeFactory struct { m map[string]EventType } -func newEventTypeFactory(disabled []EventType) *eventTypeFactory { +func newEventTypeFactory(disabled DisabledEvents) *eventTypeFactory { ret := &eventTypeFactory{ m: make(map[string]EventType, len(disabled)+32), // + extra capacity. } diff --git a/node/builder.go b/node/builder.go index 90b150a8a..eab575d20 100644 --- a/node/builder.go +++ b/node/builder.go @@ -153,6 +153,7 @@ func defaults() []Option { Override(new(dtypes.Bootstrapper), dtypes.Bootstrapper(false)), Override(new(dtypes.ShutdownChan), make(chan struct{})), Override(new(journal.Journal), journal.OpenFSJournal), + Override(new(journal.DisabledEvents), journal.DisabledEvents{}), Override(InitJournalKey, func(j *journal.Journal) { /* forces the creation of the journal at startup */ }), // Filecoin modules diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 1a22fdd95..d97dbd3fb 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -42,6 +42,7 @@ import ( paramfetch "github.com/filecoin-project/go-paramfetch" "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/go-storedcounter" + "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/node/config" sectorstorage "github.com/filecoin-project/sector-storage" "github.com/filecoin-project/sector-storage/ffiwrapper" @@ -137,7 +138,34 @@ func SectorIDCounter(ds dtypes.MetadataDS) sealing.SectorIDCounter { return &sidsc{sc} } -func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api lapi.FullNode, h host.Host, ds dtypes.MetadataDS, sealer sectorstorage.SectorManager, sc sealing.SectorIDCounter, verif ffiwrapper.Verifier, gsd dtypes.GetSealingDelayFunc) (*storage.Miner, error) { +type StorageMinerParams struct { + fx.In + + Lifecycle fx.Lifecycle + MetricsCtx helpers.MetricsCtx + API lapi.FullNode + Host host.Host + MetadataDS dtypes.MetadataDS + Sealer sectorstorage.SectorManager + SectorIDCounter sealing.SectorIDCounter + Verifier ffiwrapper.Verifier + GetSealingDelayFn dtypes.GetSealingDelayFunc + Journal journal.Journal +} + +func StorageMiner(params StorageMinerParams) (*storage.Miner, error) { + var ( + ds = params.MetadataDS + mctx = params.MetricsCtx + lc = params.Lifecycle + api = params.API + sealer = params.Sealer + h = params.Host + sc = params.SectorIDCounter + verif = params.Verifier + gsd = params.GetSealingDelayFn + jrnl = params.Journal + ) maddr, err := minerAddrFromDS(ds) if err != nil { return nil, err @@ -155,7 +183,7 @@ func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api lapi.FullNode, h return nil, err } - fps, err := storage.NewWindowedPoStScheduler(api, sealer, sealer, maddr, worker) + fps, err := storage.NewWindowedPoStScheduler(api, sealer, sealer, maddr, worker, jrnl) if err != nil { return nil, err } diff --git a/storage/journal_events.go b/storage/journal_events.go new file mode 100644 index 000000000..d87c1ce43 --- /dev/null +++ b/storage/journal_events.go @@ -0,0 +1,34 @@ +package storage + +import ( + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-actors/actors/builtin/miner" + "github.com/ipfs/go-cid" +) + +type WindowPoStEvt struct { + State string + Deadline *miner.DeadlineInfo + Height abi.ChainEpoch + TipSet []cid.Cid + Error error `json:",omitempty"` + + Proofs *WindowPoStEvt_Proofs `json:",omitempty"` + Recoveries *WindowPoStEvt_Recoveries `json:",omitempty"` + Faults *WindowPoStEvt_Faults `json:",omitempty"` +} + +type WindowPoStEvt_Proofs struct { + Partitions []miner.PoStPartition + MessageCID cid.Cid +} + +type WindowPoStEvt_Recoveries struct { + Declarations []miner.RecoveryDeclaration + MessageCID cid.Cid +} + +type WindowPoStEvt_Faults struct { + Declarations []miner.FaultDeclaration + MessageCID cid.Cid +} diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index c7d99b2fc..681e9e3d5 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -14,17 +14,29 @@ import ( "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/specs-actors/actors/crypto" + "github.com/ipfs/go-cid" "go.opencensus.io/trace" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/journal" ) var errNoPartitions = errors.New("no partitions") -func (s *WindowPoStScheduler) failPost(deadline *miner.DeadlineInfo) { +func (s *WindowPoStScheduler) failPost(err error, deadline *miner.DeadlineInfo) { + journal.MaybeAddEntry(s.jrnl, s.evtTypes[evtTypeWindowPoSt], func() interface{} { + return WindowPoStEvt{ + State: "failed", + Deadline: s.activeDeadline, + Height: s.cur.Height(), + TipSet: s.cur.Cids(), + Error: err, + } + }) + log.Errorf("TODO") /*s.failLk.Lock() if eps > s.failed { @@ -39,6 +51,15 @@ func (s *WindowPoStScheduler) doPost(ctx context.Context, deadline *miner.Deadli s.abort = abort s.activeDeadline = deadline + journal.MaybeAddEntry(s.jrnl, s.evtTypes[evtTypeWindowPoSt], func() interface{} { + return WindowPoStEvt{ + State: "started", + Deadline: s.activeDeadline, + Height: s.cur.Height(), + TipSet: s.cur.Cids(), + } + }) + go func() { defer abort() @@ -52,14 +73,23 @@ func (s *WindowPoStScheduler) doPost(ctx context.Context, deadline *miner.Deadli case nil: if err := s.submitPost(ctx, proof); err != nil { log.Errorf("submitPost failed: %+v", err) - s.failPost(deadline) + s.failPost(err, deadline) return } default: log.Errorf("runPost failed: %+v", err) - s.failPost(deadline) + s.failPost(err, deadline) return } + + journal.MaybeAddEntry(s.jrnl, s.evtTypes[evtTypeWindowPoSt], func() interface{} { + return WindowPoStEvt{ + State: "succeeded", + Deadline: s.activeDeadline, + Height: s.cur.Height(), + TipSet: s.cur.Cids(), + } + }) }() } @@ -112,11 +142,29 @@ func (s *WindowPoStScheduler) checkNextRecoveries(ctx context.Context, dlIdx uin ctx, span := trace.StartSpan(ctx, "storage.checkNextRecoveries") defer span.End() + var sm *types.SignedMessage + faulty := uint64(0) params := &miner.DeclareFaultsRecoveredParams{ Recoveries: []miner.RecoveryDeclaration{}, } - faulty := uint64(0) + defer journal.MaybeAddEntry(s.jrnl, s.evtTypes[evtTypeWindowPoSt], func() interface{} { + var mcid cid.Cid + if sm != nil { + mcid = sm.Cid() + } + + return WindowPoStEvt{ + State: "recoveries_processed", + Deadline: s.activeDeadline, + Height: s.cur.Height(), + TipSet: s.cur.Cids(), + Recoveries: &WindowPoStEvt_Recoveries{ + Declarations: params.Recoveries, + MessageCID: mcid, + }, + } + }) for partIdx, partition := range partitions { unrecovered, err := bitfield.SubtractBitField(partition.Faults, partition.Recoveries) @@ -180,7 +228,8 @@ func (s *WindowPoStScheduler) checkNextRecoveries(ctx context.Context, dlIdx uin GasPrice: types.NewInt(2), } - sm, err := s.api.MpoolPushMessage(ctx, msg) + var err error + sm, err = s.api.MpoolPushMessage(ctx, msg) if err != nil { return xerrors.Errorf("pushing message to mpool: %w", err) } @@ -203,11 +252,28 @@ func (s *WindowPoStScheduler) checkNextFaults(ctx context.Context, dlIdx uint64, ctx, span := trace.StartSpan(ctx, "storage.checkNextFaults") defer span.End() + var sm *types.SignedMessage + bad := uint64(0) params := &miner.DeclareFaultsParams{ Faults: []miner.FaultDeclaration{}, } - bad := uint64(0) + defer journal.MaybeAddEntry(s.jrnl, s.evtTypes[evtTypeWindowPoSt], func() interface{} { + var mcid cid.Cid + if sm != nil { + mcid = sm.Cid() + } + return WindowPoStEvt{ + State: "faults_processed", + Deadline: s.activeDeadline, + Height: s.cur.Height(), + TipSet: s.cur.Cids(), + Faults: &WindowPoStEvt_Faults{ + Declarations: params.Faults, + MessageCID: mcid, + }, + } + }) for partIdx, partition := range partitions { toCheck, err := partition.ActiveSectors() @@ -264,7 +330,8 @@ func (s *WindowPoStScheduler) checkNextFaults(ctx context.Context, dlIdx uint64, GasPrice: types.NewInt(2), } - sm, err := s.api.MpoolPushMessage(ctx, msg) + var err error + sm, err = s.api.MpoolPushMessage(ctx, msg) if err != nil { return xerrors.Errorf("pushing message to mpool: %w", err) } @@ -450,6 +517,26 @@ func (s *WindowPoStScheduler) submitPost(ctx context.Context, proof *miner.Submi ctx, span := trace.StartSpan(ctx, "storage.commitPost") defer span.End() + var sm *types.SignedMessage + + defer journal.MaybeAddEntry(s.jrnl, s.evtTypes[evtTypeWindowPoSt], func() interface{} { + var mcid cid.Cid + if sm != nil { + mcid = sm.Cid() + } + + return WindowPoStEvt{ + State: "proofs_processed", + Deadline: s.activeDeadline, + Height: s.cur.Height(), + TipSet: s.cur.Cids(), + Proofs: &WindowPoStEvt_Proofs{ + Partitions: proof.Partitions, + MessageCID: mcid, + }, + } + }) + enc, aerr := actors.SerializeParams(proof) if aerr != nil { return xerrors.Errorf("could not serialize submit post parameters: %w", aerr) @@ -467,7 +554,8 @@ func (s *WindowPoStScheduler) submitPost(ctx context.Context, proof *miner.Submi } // TODO: consider maybe caring about the output - sm, err := s.api.MpoolPushMessage(ctx, msg) + var err error + sm, err = s.api.MpoolPushMessage(ctx, msg) if err != nil { return xerrors.Errorf("pushing message to mpool: %w", err) } diff --git a/storage/wdpost_sched.go b/storage/wdpost_sched.go index 8e3221ef9..83d37d24b 100644 --- a/storage/wdpost_sched.go +++ b/storage/wdpost_sched.go @@ -17,10 +17,16 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/journal" ) const StartConfidence = 4 // TODO: config +// Journal event types. +const ( + evtTypeWindowPoSt = iota +) + type WindowPoStScheduler struct { api storageMinerApi prover storage.Prover @@ -36,12 +42,14 @@ type WindowPoStScheduler struct { // if a post is in progress, this indicates for which ElectionPeriodStart activeDeadline *miner.DeadlineInfo abort context.CancelFunc + jrnl journal.Journal + evtTypes [1]journal.EventType - //failed abi.ChainEpoch // eps - //failLk sync.Mutex + // failed abi.ChainEpoch // eps + // failLk sync.Mutex } -func NewWindowedPoStScheduler(api storageMinerApi, sb storage.Prover, ft sectorstorage.FaultTracker, actor address.Address, worker address.Address) (*WindowPoStScheduler, error) { +func NewWindowedPoStScheduler(api storageMinerApi, sb storage.Prover, ft sectorstorage.FaultTracker, actor address.Address, worker address.Address, jrnl journal.Journal) (*WindowPoStScheduler, error) { mi, err := api.StateMinerInfo(context.TODO(), actor, types.EmptyTSK) if err != nil { return nil, xerrors.Errorf("getting sector size: %w", err) @@ -61,6 +69,10 @@ func NewWindowedPoStScheduler(api storageMinerApi, sb storage.Prover, ft sectors actor: actor, worker: worker, + jrnl: jrnl, + evtTypes: [...]journal.EventType{ + evtTypeWindowPoSt: jrnl.RegisterEventType("storage", "wdpost"), + }, }, nil } @@ -214,6 +226,15 @@ func (s *WindowPoStScheduler) abortActivePoSt() { s.abort() } + journal.MaybeAddEntry(s.jrnl, s.evtTypes[evtTypeWindowPoSt], func() interface{} { + return WindowPoStEvt{ + State: "abort", + Deadline: s.activeDeadline, + Height: s.cur.Height(), + TipSet: s.cur.Cids(), + } + }) + log.Warnf("Aborting Window PoSt (Deadline: %+v)", s.activeDeadline) s.activeDeadline = nil From 7459ec6bbafa709a34fa27bacd2bda51110023a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Mon, 20 Jul 2020 16:37:24 +0100 Subject: [PATCH 006/885] add mpool journal events; fix dependency injection. --- chain/messagepool/messagepool.go | 54 ++++++++++++++++++++++++++- chain/messagepool/messagepool_test.go | 4 +- node/builder.go | 2 +- node/modules/chain.go | 4 +- 4 files changed, 58 insertions(+), 6 deletions(-) diff --git a/chain/messagepool/messagepool.go b/chain/messagepool/messagepool.go index 7b2f19bbd..2f420b308 100644 --- a/chain/messagepool/messagepool.go +++ b/chain/messagepool/messagepool.go @@ -28,6 +28,7 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/lib/sigs" "github.com/filecoin-project/lotus/node/modules/dtypes" @@ -65,6 +66,20 @@ const ( localUpdates = "update" ) +// Journal event types. +const ( + evtTypeMpoolAdd = iota + evtTypeMpoolRemove + evtTypeMpoolRepub +) + +// MessagePoolEvt is the journal event type emitted by the MessagePool. +type MessagePoolEvt struct { + Action string + MessageCIDs []cid.Cid + Error error `json:",omitempty"` +} + type MessagePool struct { lk sync.Mutex @@ -93,6 +108,9 @@ type MessagePool struct { netName dtypes.NetworkName sigValCache *lru.TwoQueueCache + + jrnl journal.Journal + evtTypes [3]journal.EventType } type msgSet struct { @@ -185,7 +203,7 @@ func (mpp *mpoolProvider) LoadTipSet(tsk types.TipSetKey) (*types.TipSet, error) return mpp.sm.ChainStore().LoadTipSet(tsk) } -func New(api Provider, ds dtypes.MetadataDS, netName dtypes.NetworkName) (*MessagePool, error) { +func New(api Provider, ds dtypes.MetadataDS, netName dtypes.NetworkName, jrnl journal.Journal) (*MessagePool, error) { cache, _ := lru.New2Q(build.BlsSignatureCacheSize) verifcache, _ := lru.New2Q(build.VerifSigCacheSize) @@ -202,6 +220,12 @@ func New(api Provider, ds dtypes.MetadataDS, netName dtypes.NetworkName) (*Messa localMsgs: namespace.Wrap(ds, datastore.NewKey(localMsgsDs)), api: api, netName: netName, + jrnl: jrnl, + evtTypes: [...]journal.EventType{ + evtTypeMpoolAdd: jrnl.RegisterEventType("mpool", "add"), + evtTypeMpoolRemove: jrnl.RegisterEventType("mpool", "remove"), + evtTypeMpoolRepub: jrnl.RegisterEventType("mpool", "repub"), + }, } if err := mp.loadLocal(); err != nil { @@ -284,6 +308,19 @@ func (mp *MessagePool) repubLocal() { if errout != nil { log.Errorf("errors while republishing: %+v", errout) } + + journal.MaybeAddEntry(mp.jrnl, mp.evtTypes[evtTypeMpoolRepub], func() interface{} { + cids := make([]cid.Cid, 0, len(outputMsgs)) + for _, m := range outputMsgs { + cids = append(cids, m.Cid()) + } + return MessagePoolEvt{ + Action: "repub", + MessageCIDs: cids, + Error: errout, + } + }) + case <-mp.closer: mp.repubTk.Stop() return @@ -448,6 +485,14 @@ func (mp *MessagePool) addLocked(m *types.SignedMessage) error { Type: api.MpoolAdd, Message: m, }, localUpdates) + + journal.MaybeAddEntry(mp.jrnl, mp.evtTypes[evtTypeMpoolAdd], func() interface{} { + return MessagePoolEvt{ + Action: "add", + MessageCIDs: []cid.Cid{m.Cid()}, + } + }) + return nil } @@ -580,6 +625,13 @@ func (mp *MessagePool) Remove(from address.Address, nonce uint64) { Type: api.MpoolRemove, Message: m, }, localUpdates) + + journal.MaybeAddEntry(mp.jrnl, mp.evtTypes[evtTypeMpoolRemove], func() interface{} { + return MessagePoolEvt{ + Action: "remove", + MessageCIDs: []cid.Cid{m.Cid()}, + } + }) } // NB: This deletes any message with the given nonce. This makes sense diff --git a/chain/messagepool/messagepool_test.go b/chain/messagepool/messagepool_test.go index d1dedd534..64af26e9a 100644 --- a/chain/messagepool/messagepool_test.go +++ b/chain/messagepool/messagepool_test.go @@ -147,7 +147,7 @@ func TestMessagePool(t *testing.T) { ds := datastore.NewMapDatastore() - mp, err := New(tma, ds, "mptest") + mp, err := New(tma, ds, "mptest", nil) if err != nil { t.Fatal(err) } @@ -188,7 +188,7 @@ func TestRevertMessages(t *testing.T) { ds := datastore.NewMapDatastore() - mp, err := New(tma, ds, "mptest") + mp, err := New(tma, ds, "mptest", nil) if err != nil { t.Fatal(err) } diff --git a/node/builder.go b/node/builder.go index eab575d20..98ae568da 100644 --- a/node/builder.go +++ b/node/builder.go @@ -155,7 +155,7 @@ func defaults() []Option { Override(new(journal.Journal), journal.OpenFSJournal), Override(new(journal.DisabledEvents), journal.DisabledEvents{}), - Override(InitJournalKey, func(j *journal.Journal) { /* forces the creation of the journal at startup */ }), + Override(InitJournalKey, func(j journal.Journal) { /* forces the creation of the journal at startup */ }), // Filecoin modules } diff --git a/node/modules/chain.go b/node/modules/chain.go index 9f5ac05a3..b647cfb98 100644 --- a/node/modules/chain.go +++ b/node/modules/chain.go @@ -47,9 +47,9 @@ func ChainExchange(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, rt return exch } -func MessagePool(lc fx.Lifecycle, sm *stmgr.StateManager, ps *pubsub.PubSub, ds dtypes.MetadataDS, nn dtypes.NetworkName) (*messagepool.MessagePool, error) { +func MessagePool(lc fx.Lifecycle, sm *stmgr.StateManager, ps *pubsub.PubSub, ds dtypes.MetadataDS, nn dtypes.NetworkName, jrnl journal.Journal) (*messagepool.MessagePool, error) { mpp := messagepool.NewProvider(sm, ps) - mp, err := messagepool.New(mpp, ds, nn) + mp, err := messagepool.New(mpp, ds, nn, jrnl) if err != nil { return nil, xerrors.Errorf("constructing mpool: %w", err) } From d6e6eedd581565fa69dccae8ecbb648b9f2272e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Tue, 21 Jul 2020 13:02:51 +0100 Subject: [PATCH 007/885] pipe sealing events to the journal. --- go.mod | 2 +- go.sum | 4 +-- node/modules/storageminer.go | 2 +- storage/journal_events.go | 34 ------------------ storage/miner.go | 32 +++++++++++++++-- storage/wdpost_run.go | 12 +++---- storage/wdpost_sched.go | 68 ++++++++++++++++++++++++------------ 7 files changed, 85 insertions(+), 69 deletions(-) delete mode 100644 storage/journal_events.go diff --git a/go.mod b/go.mod index d4170d4cd..6b905b9d9 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/filecoin-project/sector-storage v0.0.0-20200717213554-a109ef9cbeab github.com/filecoin-project/specs-actors v0.7.3-0.20200717200758-365408676dbb github.com/filecoin-project/specs-storage v0.1.1-0.20200622113353-88a9704877ea - github.com/filecoin-project/storage-fsm v0.0.0-20200717125541-d575c3a5f7f2 + github.com/filecoin-project/storage-fsm v0.0.0-20200721113842-ab98dc7ab341 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 github.com/go-kit/kit v0.10.0 github.com/go-ole/go-ole v1.2.4 // indirect diff --git a/go.sum b/go.sum index 92dba878a..0ffae068c 100644 --- a/go.sum +++ b/go.sum @@ -275,8 +275,8 @@ github.com/filecoin-project/specs-storage v0.1.0 h1:PkDgTOT5W5Ao7752onjDl4QSv+sg github.com/filecoin-project/specs-storage v0.1.0/go.mod h1:Pr5ntAaxsh+sLG/LYiL4tKzvA83Vk5vLODYhfNwOg7k= github.com/filecoin-project/specs-storage v0.1.1-0.20200622113353-88a9704877ea h1:iixjULRQFPn7Q9KlIqfwLJnlAXO10bbkI+xy5GKGdLY= github.com/filecoin-project/specs-storage v0.1.1-0.20200622113353-88a9704877ea/go.mod h1:Pr5ntAaxsh+sLG/LYiL4tKzvA83Vk5vLODYhfNwOg7k= -github.com/filecoin-project/storage-fsm v0.0.0-20200717125541-d575c3a5f7f2 h1:A9zUXOMuVnSTp9a0i0KtHkB05hA8mRWVLls6Op9Czuo= -github.com/filecoin-project/storage-fsm v0.0.0-20200717125541-d575c3a5f7f2/go.mod h1:1CGbd11KkHuyWPT+xwwCol1zl/jnlpiKD2L4fzKxaiI= +github.com/filecoin-project/storage-fsm v0.0.0-20200721113842-ab98dc7ab341 h1:4catrjuFXo/VcQcmvAMyeBYEp35IHFi+Tqz0dNAJp54= +github.com/filecoin-project/storage-fsm v0.0.0-20200721113842-ab98dc7ab341/go.mod h1:1CGbd11KkHuyWPT+xwwCol1zl/jnlpiKD2L4fzKxaiI= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6 h1:u/UEqS66A5ckRmS4yNpjmVH56sVtS/RfclBAYocb4as= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ= diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index d97dbd3fb..96f5beba1 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -188,7 +188,7 @@ func StorageMiner(params StorageMinerParams) (*storage.Miner, error) { return nil, err } - sm, err := storage.NewMiner(api, maddr, worker, h, ds, sealer, sc, verif, gsd) + sm, err := storage.NewMiner(api, maddr, worker, h, ds, sealer, sc, verif, gsd, jrnl) if err != nil { return nil, err } diff --git a/storage/journal_events.go b/storage/journal_events.go deleted file mode 100644 index d87c1ce43..000000000 --- a/storage/journal_events.go +++ /dev/null @@ -1,34 +0,0 @@ -package storage - -import ( - "github.com/filecoin-project/specs-actors/actors/abi" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" - "github.com/ipfs/go-cid" -) - -type WindowPoStEvt struct { - State string - Deadline *miner.DeadlineInfo - Height abi.ChainEpoch - TipSet []cid.Cid - Error error `json:",omitempty"` - - Proofs *WindowPoStEvt_Proofs `json:",omitempty"` - Recoveries *WindowPoStEvt_Recoveries `json:",omitempty"` - Faults *WindowPoStEvt_Faults `json:",omitempty"` -} - -type WindowPoStEvt_Proofs struct { - Partitions []miner.PoStPartition - MessageCID cid.Cid -} - -type WindowPoStEvt_Recoveries struct { - Declarations []miner.RecoveryDeclaration - MessageCID cid.Cid -} - -type WindowPoStEvt_Faults struct { - Declarations []miner.FaultDeclaration - MessageCID cid.Cid -} diff --git a/storage/miner.go b/storage/miner.go index c4a3cccc3..6f16b9cfe 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -24,6 +24,7 @@ import ( "github.com/filecoin-project/lotus/chain/events" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/node/modules/dtypes" sealing "github.com/filecoin-project/storage-fsm" ) @@ -43,6 +44,18 @@ type Miner struct { getSealDelay dtypes.GetSealingDelayFunc sealing *sealing.Sealing + + jrnl journal.Journal + sealingEvtType journal.EventType +} + +// SealingStateEvt is a journal event that records a sector state transition. +type SealingStateEvt struct { + SectorNumber abi.SectorNumber + SectorType abi.RegisteredSealProof + From sealing.SectorState + After sealing.SectorState + Error string } type storageMinerApi interface { @@ -80,7 +93,7 @@ type storageMinerApi interface { WalletHas(context.Context, address.Address) (bool, error) } -func NewMiner(api storageMinerApi, maddr, worker address.Address, h host.Host, ds datastore.Batching, sealer sectorstorage.SectorManager, sc sealing.SectorIDCounter, verif ffiwrapper.Verifier, gsd dtypes.GetSealingDelayFunc) (*Miner, error) { +func NewMiner(api storageMinerApi, maddr, worker address.Address, h host.Host, ds datastore.Batching, sealer sectorstorage.SectorManager, sc sealing.SectorIDCounter, verif ffiwrapper.Verifier, gsd dtypes.GetSealingDelayFunc, jrnl journal.Journal) (*Miner, error) { m := &Miner{ api: api, h: h, @@ -92,6 +105,9 @@ func NewMiner(api storageMinerApi, maddr, worker address.Address, h host.Host, d maddr: maddr, worker: worker, getSealDelay: gsd, + + jrnl: jrnl, + sealingEvtType: jrnl.RegisterEventType("storage", "sealing_states"), } return m, nil @@ -110,13 +126,25 @@ func (m *Miner) Run(ctx context.Context) error { evts := events.NewEvents(ctx, m.api) adaptedAPI := NewSealingAPIAdapter(m.api) pcp := sealing.NewBasicPreCommitPolicy(adaptedAPI, miner.MaxSectorExpirationExtension-(miner.WPoStProvingPeriod*2), md.PeriodStart%miner.WPoStProvingPeriod) - m.sealing = sealing.New(adaptedAPI, NewEventsAdapter(evts), m.maddr, m.ds, m.sealer, m.sc, m.verif, &pcp, sealing.GetSealingDelayFunc(m.getSealDelay)) + m.sealing = sealing.New(adaptedAPI, NewEventsAdapter(evts), m.maddr, m.ds, m.sealer, m.sc, m.verif, &pcp, sealing.GetSealingDelayFunc(m.getSealDelay), m.handleSealingNotifications) go m.sealing.Run(ctx) //nolint:errcheck // logged intside the function return nil } +func (m *Miner) handleSealingNotifications(before, after sealing.SectorInfo) { + journal.MaybeAddEntry(m.jrnl, m.sealingEvtType, func() interface{} { + return SealingStateEvt{ + SectorNumber: before.SectorNumber, + SectorType: before.SectorType, + From: before.State, + After: after.State, + Error: after.LastErr, + } + }) +} + func (m *Miner) Stop(ctx context.Context) error { return m.sealing.Stop(ctx) } diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 681e9e3d5..191ac5319 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -27,7 +27,7 @@ import ( var errNoPartitions = errors.New("no partitions") func (s *WindowPoStScheduler) failPost(err error, deadline *miner.DeadlineInfo) { - journal.MaybeAddEntry(s.jrnl, s.evtTypes[evtTypeWindowPoSt], func() interface{} { + journal.MaybeAddEntry(s.jrnl, s.wdPoStEvtType, func() interface{} { return WindowPoStEvt{ State: "failed", Deadline: s.activeDeadline, @@ -51,7 +51,7 @@ func (s *WindowPoStScheduler) doPost(ctx context.Context, deadline *miner.Deadli s.abort = abort s.activeDeadline = deadline - journal.MaybeAddEntry(s.jrnl, s.evtTypes[evtTypeWindowPoSt], func() interface{} { + journal.MaybeAddEntry(s.jrnl, s.wdPoStEvtType, func() interface{} { return WindowPoStEvt{ State: "started", Deadline: s.activeDeadline, @@ -82,7 +82,7 @@ func (s *WindowPoStScheduler) doPost(ctx context.Context, deadline *miner.Deadli return } - journal.MaybeAddEntry(s.jrnl, s.evtTypes[evtTypeWindowPoSt], func() interface{} { + journal.MaybeAddEntry(s.jrnl, s.wdPoStEvtType, func() interface{} { return WindowPoStEvt{ State: "succeeded", Deadline: s.activeDeadline, @@ -148,7 +148,7 @@ func (s *WindowPoStScheduler) checkNextRecoveries(ctx context.Context, dlIdx uin Recoveries: []miner.RecoveryDeclaration{}, } - defer journal.MaybeAddEntry(s.jrnl, s.evtTypes[evtTypeWindowPoSt], func() interface{} { + defer journal.MaybeAddEntry(s.jrnl, s.wdPoStEvtType, func() interface{} { var mcid cid.Cid if sm != nil { mcid = sm.Cid() @@ -258,7 +258,7 @@ func (s *WindowPoStScheduler) checkNextFaults(ctx context.Context, dlIdx uint64, Faults: []miner.FaultDeclaration{}, } - defer journal.MaybeAddEntry(s.jrnl, s.evtTypes[evtTypeWindowPoSt], func() interface{} { + defer journal.MaybeAddEntry(s.jrnl, s.wdPoStEvtType, func() interface{} { var mcid cid.Cid if sm != nil { mcid = sm.Cid() @@ -519,7 +519,7 @@ func (s *WindowPoStScheduler) submitPost(ctx context.Context, proof *miner.Submi var sm *types.SignedMessage - defer journal.MaybeAddEntry(s.jrnl, s.evtTypes[evtTypeWindowPoSt], func() interface{} { + defer journal.MaybeAddEntry(s.jrnl, s.wdPoStEvtType, func() interface{} { var mcid cid.Cid if sm != nil { mcid = sm.Cid() diff --git a/storage/wdpost_sched.go b/storage/wdpost_sched.go index 83d37d24b..3bc591bb7 100644 --- a/storage/wdpost_sched.go +++ b/storage/wdpost_sched.go @@ -4,6 +4,7 @@ import ( "context" "time" + "github.com/ipfs/go-cid" "go.opencensus.io/trace" "golang.org/x/xerrors" @@ -22,10 +23,32 @@ import ( const StartConfidence = 4 // TODO: config -// Journal event types. -const ( - evtTypeWindowPoSt = iota -) +type WindowPoStEvt struct { + State string + Deadline *miner.DeadlineInfo + Height abi.ChainEpoch + TipSet []cid.Cid + Error error `json:",omitempty"` + + Proofs *WindowPoStEvt_Proofs `json:",omitempty"` + Recoveries *WindowPoStEvt_Recoveries `json:",omitempty"` + Faults *WindowPoStEvt_Faults `json:",omitempty"` +} + +type WindowPoStEvt_Proofs struct { + Partitions []miner.PoStPartition + MessageCID cid.Cid `json:",omitempty"` +} + +type WindowPoStEvt_Recoveries struct { + Declarations []miner.RecoveryDeclaration + MessageCID cid.Cid `json:",omitempty"` +} + +type WindowPoStEvt_Faults struct { + Declarations []miner.FaultDeclaration + MessageCID cid.Cid `json:",omitempty"` +} type WindowPoStScheduler struct { api storageMinerApi @@ -42,8 +65,9 @@ type WindowPoStScheduler struct { // if a post is in progress, this indicates for which ElectionPeriodStart activeDeadline *miner.DeadlineInfo abort context.CancelFunc - jrnl journal.Journal - evtTypes [1]journal.EventType + + jrnl journal.Journal + wdPoStEvtType journal.EventType // failed abi.ChainEpoch // eps // failLk sync.Mutex @@ -67,12 +91,10 @@ func NewWindowedPoStScheduler(api storageMinerApi, sb storage.Prover, ft sectors proofType: rt, partitionSectors: mi.WindowPoStPartitionSectors, - actor: actor, - worker: worker, - jrnl: jrnl, - evtTypes: [...]journal.EventType{ - evtTypeWindowPoSt: jrnl.RegisterEventType("storage", "wdpost"), - }, + actor: actor, + worker: worker, + jrnl: jrnl, + wdPoStEvtType: jrnl.RegisterEventType("storage", "wdpost"), }, nil } @@ -224,19 +246,19 @@ func (s *WindowPoStScheduler) abortActivePoSt() { if s.abort != nil { s.abort() + + journal.MaybeAddEntry(s.jrnl, s.wdPoStEvtType, func() interface{} { + return WindowPoStEvt{ + State: "abort", + Deadline: s.activeDeadline, + Height: s.cur.Height(), + TipSet: s.cur.Cids(), + } + }) + + log.Warnf("Aborting Window PoSt (Deadline: %+v)", s.activeDeadline) } - journal.MaybeAddEntry(s.jrnl, s.evtTypes[evtTypeWindowPoSt], func() interface{} { - return WindowPoStEvt{ - State: "abort", - Deadline: s.activeDeadline, - Height: s.cur.Height(), - TipSet: s.cur.Cids(), - } - }) - - log.Warnf("Aborting Window PoSt (Deadline: %+v)", s.activeDeadline) - s.activeDeadline = nil s.abort = nil } From 3bd9d55a156c5f43ecf44be24a7eb53435745edd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Tue, 21 Jul 2020 13:10:29 +0100 Subject: [PATCH 008/885] message pool: write message(s) in journal entries. --- chain/messagepool/messagepool.go | 33 ++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/chain/messagepool/messagepool.go b/chain/messagepool/messagepool.go index 2f420b308..dcd7acf50 100644 --- a/chain/messagepool/messagepool.go +++ b/chain/messagepool/messagepool.go @@ -73,11 +73,17 @@ const ( evtTypeMpoolRepub ) -// MessagePoolEvt is the journal event type emitted by the MessagePool. +// MessagePoolEvt is the journal entry for message pool events. type MessagePoolEvt struct { - Action string - MessageCIDs []cid.Cid - Error error `json:",omitempty"` + Action string + Messages []MessagePoolEvt_Message + Error error `json:",omitempty"` +} + +type MessagePoolEvt_Message struct { + types.Message + + CID cid.Cid } type MessagePool struct { @@ -310,14 +316,14 @@ func (mp *MessagePool) repubLocal() { } journal.MaybeAddEntry(mp.jrnl, mp.evtTypes[evtTypeMpoolRepub], func() interface{} { - cids := make([]cid.Cid, 0, len(outputMsgs)) + msgs := make([]MessagePoolEvt_Message, 0, len(outputMsgs)) for _, m := range outputMsgs { - cids = append(cids, m.Cid()) + msgs = append(msgs, MessagePoolEvt_Message{Message: m.Message, CID: m.Cid()}) } return MessagePoolEvt{ - Action: "repub", - MessageCIDs: cids, - Error: errout, + Action: "repub", + Messages: msgs, + Error: errout, } }) @@ -488,8 +494,8 @@ func (mp *MessagePool) addLocked(m *types.SignedMessage) error { journal.MaybeAddEntry(mp.jrnl, mp.evtTypes[evtTypeMpoolAdd], func() interface{} { return MessagePoolEvt{ - Action: "add", - MessageCIDs: []cid.Cid{m.Cid()}, + Action: "add", + Messages: []MessagePoolEvt_Message{{Message: m.Message, CID: m.Cid()}}, } }) @@ -628,9 +634,8 @@ func (mp *MessagePool) Remove(from address.Address, nonce uint64) { journal.MaybeAddEntry(mp.jrnl, mp.evtTypes[evtTypeMpoolRemove], func() interface{} { return MessagePoolEvt{ - Action: "remove", - MessageCIDs: []cid.Cid{m.Cid()}, - } + Action: "remove", + Messages: []MessagePoolEvt_Message{{Message: m.Message, CID: m.Cid()}}} }) } From cb8e209f7806170fb48a8528934878a8815e7c62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Tue, 21 Jul 2020 13:13:09 +0100 Subject: [PATCH 009/885] journal: disable noisy message pool events. --- node/builder.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/node/builder.go b/node/builder.go index 98ae568da..a5933a272 100644 --- a/node/builder.go +++ b/node/builder.go @@ -153,7 +153,12 @@ func defaults() []Option { Override(new(dtypes.Bootstrapper), dtypes.Bootstrapper(false)), Override(new(dtypes.ShutdownChan), make(chan struct{})), Override(new(journal.Journal), journal.OpenFSJournal), - Override(new(journal.DisabledEvents), journal.DisabledEvents{}), + + // By default, disable noisy mpool events; keep only mpool:repub on. + Override(new(journal.DisabledEvents), journal.DisabledEvents{ + journal.EventType{System: "mpool", Event: "add"}, + journal.EventType{System: "mpool", Event: "remove"}, + }), Override(InitJournalKey, func(j journal.Journal) { /* forces the creation of the journal at startup */ }), // Filecoin modules From d547c2588c0b84f0660f1332aa9e0ae4f0509483 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Tue, 21 Jul 2020 17:26:44 +0100 Subject: [PATCH 010/885] record deals events in journal. --- markets/storageadapter/client.go | 26 +++++++++- markets/storageadapter/journal_events.go | 64 ++++++++++++++++++++++++ markets/storageadapter/provider.go | 49 ++++++++++++++++-- 3 files changed, 132 insertions(+), 7 deletions(-) create mode 100644 markets/storageadapter/journal_events.go diff --git a/markets/storageadapter/client.go b/markets/storageadapter/client.go index 2bb762e28..1da85fa69 100644 --- a/markets/storageadapter/client.go +++ b/markets/storageadapter/client.go @@ -7,6 +7,7 @@ import ( "context" "github.com/filecoin-project/lotus/chain/events/state" + "github.com/filecoin-project/lotus/journal" "golang.org/x/xerrors" @@ -43,6 +44,9 @@ type ClientNodeAdapter struct { cs *store.ChainStore fm *market.FundMgr ev *events.Events + + jrnl journal.Journal + evtTypes [4]journal.EventType } type clientApi struct { @@ -50,7 +54,7 @@ type clientApi struct { full.StateAPI } -func NewClientNodeAdapter(state full.StateAPI, chain full.ChainAPI, mpool full.MpoolAPI, sm *stmgr.StateManager, cs *store.ChainStore, fm *market.FundMgr) storagemarket.StorageClientNode { +func NewClientNodeAdapter(state full.StateAPI, chain full.ChainAPI, mpool full.MpoolAPI, sm *stmgr.StateManager, cs *store.ChainStore, fm *market.FundMgr, jrnl journal.Journal) storagemarket.StorageClientNode { return &ClientNodeAdapter{ StateAPI: state, ChainAPI: chain, @@ -60,6 +64,13 @@ func NewClientNodeAdapter(state full.StateAPI, chain full.ChainAPI, mpool full.M cs: cs, fm: fm, ev: events.NewEvents(context.TODO(), &clientApi{chain, state}), + + jrnl: jrnl, + evtTypes: [...]journal.EventType{ + evtTypeDealSectorCommitted: jrnl.RegisterEventType("markets:storage:client", "deal_sector_committed"), + evtTypeDealExpired: jrnl.RegisterEventType("markets:storage:client", "deal_expired"), + evtTypeDealSlashed: jrnl.RegisterEventType("markets:storage:client", "deal_slashed"), + }, } } @@ -235,7 +246,14 @@ func (c *ClientNodeAdapter) ValidatePublishedDeal(ctx context.Context, deal stor return 0, err } - return res.IDs[dealIdx], nil + dealID := res.IDs[dealIdx] + journal.MaybeAddEntry(c.jrnl, c.evtTypes[evtTypeDealAccepted], func() interface{} { + deal := deal // copy and strip fields we don't want to log to the journal + deal.ClientSignature = crypto.Signature{} + return ClientDealAcceptedEvt{ID: dealID, Deal: deal, Height: c.cs.GetHeaviestTipSet().Height()} + }) + + return dealID, nil } func (c *ClientNodeAdapter) OnDealSectorCommitted(ctx context.Context, provider address.Address, dealId abi.DealID, cb storagemarket.DealSectorCommittedCallback) error { @@ -278,6 +296,10 @@ func (c *ClientNodeAdapter) OnDealSectorCommitted(ctx context.Context, provider log.Infof("Storage deal %d activated at epoch %d", dealId, sd.State.SectorStartEpoch) + journal.MaybeAddEntry(c.jrnl, c.evtTypes[evtTypeDealSectorCommitted], func() interface{} { + return ClientDealSectorCommittedEvt{ID: dealId, State: sd.State, Height: curH} + }) + cb(nil) return false, nil diff --git a/markets/storageadapter/journal_events.go b/markets/storageadapter/journal_events.go new file mode 100644 index 000000000..6192738df --- /dev/null +++ b/markets/storageadapter/journal_events.go @@ -0,0 +1,64 @@ +package storageadapter + +import ( + "github.com/filecoin-project/go-fil-markets/storagemarket" + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-actors/actors/builtin/market" +) + +// Journal entry types emitted from this module. +const ( + evtTypeDealAccepted = iota + evtTypeDealSectorCommitted + evtTypeDealExpired + evtTypeDealSlashed +) + +type ClientDealAcceptedEvt struct { + ID abi.DealID + Deal storagemarket.ClientDeal + Height abi.ChainEpoch +} + +type ClientDealSectorCommittedEvt struct { + ID abi.DealID + State market.DealState + Height abi.ChainEpoch +} + +type ClientDealExpiredEvt struct { + ID abi.DealID + State market.DealState + Height abi.ChainEpoch +} + +type ClientDealSlashedEvt struct { + ID abi.DealID + State market.DealState + Height abi.ChainEpoch +} + +type MinerDealAcceptedEvt struct { + ID abi.DealID + Deal storagemarket.MinerDeal + State market.DealState + Height abi.ChainEpoch +} + +type MinerDealSectorCommittedEvt struct { + ID abi.DealID + State market.DealState + Height abi.ChainEpoch +} + +type MinerDealExpiredEvt struct { + ID abi.DealID + State market.DealState + Height abi.ChainEpoch +} + +type MinerDealSlashedEvt struct { + ID abi.DealID + State market.DealState + Height abi.ChainEpoch +} diff --git a/markets/storageadapter/provider.go b/markets/storageadapter/provider.go index 9e1a101cc..2aa9c45df 100644 --- a/markets/storageadapter/provider.go +++ b/markets/storageadapter/provider.go @@ -28,6 +28,7 @@ import ( "github.com/filecoin-project/lotus/chain/events" "github.com/filecoin-project/lotus/chain/events/state" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/lib/sigs" "github.com/filecoin-project/lotus/markets/utils" "github.com/filecoin-project/lotus/node/modules/dtypes" @@ -45,14 +46,25 @@ type ProviderNodeAdapter struct { secb *sectorblocks.SectorBlocks ev *events.Events + + jrnl journal.Journal + evtTypes [4]journal.EventType } -func NewProviderNodeAdapter(dag dtypes.StagingDAG, secb *sectorblocks.SectorBlocks, full api.FullNode) storagemarket.StorageProviderNode { +func NewProviderNodeAdapter(dag dtypes.StagingDAG, secb *sectorblocks.SectorBlocks, full api.FullNode, jrnl journal.Journal) storagemarket.StorageProviderNode { return &ProviderNodeAdapter{ FullNode: full, dag: dag, secb: secb, ev: events.NewEvents(context.TODO(), full), + + jrnl: jrnl, + evtTypes: [...]journal.EventType{ + evtTypeDealAccepted: jrnl.RegisterEventType("markets:storage:provider", "deal_complete"), + evtTypeDealSectorCommitted: jrnl.RegisterEventType("markets:storage:provider", "deal_sector_committed"), + evtTypeDealExpired: jrnl.RegisterEventType("markets:storage:provider", "deal_expired"), + evtTypeDealSlashed: jrnl.RegisterEventType("markets:storage:provider", "deal_slashed"), + }, } } @@ -100,8 +112,15 @@ func (n *ProviderNodeAdapter) OnDealComplete(ctx context.Context, deal storagema if err != nil { return xerrors.Errorf("AddPiece failed: %s", err) } + log.Warnf("New Deal: deal %d", deal.DealID) + journal.MaybeAddEntry(n.jrnl, n.evtTypes[evtTypeDealAccepted], func() interface{} { + deal := deal // copy and strip fields we don't want to log to the journal + deal.ClientSignature = crypto.Signature{} + return MinerDealAcceptedEvt{ID: deal.DealID, Deal: deal} + }) + return nil } @@ -268,6 +287,10 @@ func (n *ProviderNodeAdapter) OnDealSectorCommitted(ctx context.Context, provide log.Infof("Storage deal %d activated at epoch %d", dealID, sd.State.SectorStartEpoch) + journal.MaybeAddEntry(n.jrnl, n.evtTypes[evtTypeDealSectorCommitted], func() interface{} { + return MinerDealSectorCommittedEvt{ID: dealID, State: sd.State, Height: curH} + }) + cb(nil) return false, nil @@ -369,15 +392,23 @@ func (n *ProviderNodeAdapter) OnDealExpiredOrSlashed(ctx context.Context, dealID // Called immediately to check if the deal has already expired or been slashed checkFunc := func(ts *types.TipSet) (done bool, more bool, err error) { + height := ts.Height() + // Check if the deal has already expired - if sd.Proposal.EndEpoch <= ts.Height() { + if sd.Proposal.EndEpoch <= height { onDealExpired(nil) + journal.MaybeAddEntry(n.jrnl, n.evtTypes[evtTypeDealExpired], func() interface{} { + return MinerDealExpiredEvt{ID: dealID, State: sd.State, Height: height} + }) return true, false, nil } // If there is no deal assume it's already been slashed if sd.State.SectorStartEpoch < 0 { - onDealSlashed(ts.Height(), nil) + onDealSlashed(height, nil) + journal.MaybeAddEntry(n.jrnl, n.evtTypes[evtTypeDealSlashed], func() interface{} { + return MinerDealSlashedEvt{ID: dealID, State: sd.State, Height: height} + }) return true, false, nil } @@ -389,9 +420,14 @@ func (n *ProviderNodeAdapter) OnDealExpiredOrSlashed(ctx context.Context, dealID // Called when there was a match against the state change we're looking for // and the chain has advanced to the confidence height stateChanged := func(ts *types.TipSet, ts2 *types.TipSet, states events.StateChange, h abi.ChainEpoch) (more bool, err error) { + height := ts2.Height() + // Check if the deal has already expired - if sd.Proposal.EndEpoch <= ts2.Height() { + if sd.Proposal.EndEpoch <= height { onDealExpired(nil) + journal.MaybeAddEntry(n.jrnl, n.evtTypes[evtTypeDealExpired], func() interface{} { + return MinerDealExpiredEvt{ID: dealID, State: sd.State, Height: height} + }) return false, nil } @@ -414,7 +450,10 @@ func (n *ProviderNodeAdapter) OnDealExpiredOrSlashed(ctx context.Context, dealID // Deal was slashed if deal.To == nil { - onDealSlashed(ts2.Height(), nil) + onDealSlashed(height, nil) + journal.MaybeAddEntry(n.jrnl, n.evtTypes[evtTypeDealSlashed], func() interface{} { + return MinerDealSlashedEvt{ID: dealID, State: sd.State, Height: height} + }) return false, nil } From 4e82cf369c92d2f611cb1c62832aac448ac32280 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Tue, 21 Jul 2020 17:32:01 +0100 Subject: [PATCH 011/885] rename journal entries to journal events. --- chain/messagepool/messagepool.go | 6 +++--- chain/store/store.go | 2 +- go.sum | 2 ++ journal/filesystem.go | 18 ++++++++-------- journal/memory.go | 34 +++++++++++++++--------------- journal/memory_test.go | 4 ++-- journal/nil.go | 2 +- journal/sugar.go | 10 ++++----- journal/types.go | 8 +++---- markets/storageadapter/client.go | 4 ++-- markets/storageadapter/provider.go | 12 +++++------ storage/miner.go | 2 +- storage/wdpost_run.go | 12 +++++------ storage/wdpost_sched.go | 2 +- 14 files changed, 60 insertions(+), 58 deletions(-) diff --git a/chain/messagepool/messagepool.go b/chain/messagepool/messagepool.go index dcd7acf50..28cf1b263 100644 --- a/chain/messagepool/messagepool.go +++ b/chain/messagepool/messagepool.go @@ -315,7 +315,7 @@ func (mp *MessagePool) repubLocal() { log.Errorf("errors while republishing: %+v", errout) } - journal.MaybeAddEntry(mp.jrnl, mp.evtTypes[evtTypeMpoolRepub], func() interface{} { + journal.MaybeRecordEvent(mp.jrnl, mp.evtTypes[evtTypeMpoolRepub], func() interface{} { msgs := make([]MessagePoolEvt_Message, 0, len(outputMsgs)) for _, m := range outputMsgs { msgs = append(msgs, MessagePoolEvt_Message{Message: m.Message, CID: m.Cid()}) @@ -492,7 +492,7 @@ func (mp *MessagePool) addLocked(m *types.SignedMessage) error { Message: m, }, localUpdates) - journal.MaybeAddEntry(mp.jrnl, mp.evtTypes[evtTypeMpoolAdd], func() interface{} { + journal.MaybeRecordEvent(mp.jrnl, mp.evtTypes[evtTypeMpoolAdd], func() interface{} { return MessagePoolEvt{ Action: "add", Messages: []MessagePoolEvt_Message{{Message: m.Message, CID: m.Cid()}}, @@ -632,7 +632,7 @@ func (mp *MessagePool) Remove(from address.Address, nonce uint64) { Message: m, }, localUpdates) - journal.MaybeAddEntry(mp.jrnl, mp.evtTypes[evtTypeMpoolRemove], func() interface{} { + journal.MaybeRecordEvent(mp.jrnl, mp.evtTypes[evtTypeMpoolRemove], func() interface{} { return MessagePoolEvt{ Action: "remove", Messages: []MessagePoolEvt_Message{{Message: m.Message, CID: m.Cid()}}} diff --git a/chain/store/store.go b/chain/store/store.go index ba7a54ed3..07ac57ac4 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -348,7 +348,7 @@ func (cs *ChainStore) reorgWorker(ctx context.Context, initialNotifees []ReorgNo continue } - journal.MaybeAddEntry(cs.journal, cs.evtTypes[evtTypeHeadChange], func() interface{} { + journal.MaybeRecordEvent(cs.journal, cs.evtTypes[evtTypeHeadChange], func() interface{} { return HeadChangeEvt{ From: r.old.Key(), FromHeight: r.old.Height(), diff --git a/go.sum b/go.sum index 0ffae068c..4d99d3d46 100644 --- a/go.sum +++ b/go.sum @@ -82,6 +82,7 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkY github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.32.11 h1:1nYF+Tfccn/hnAZsuwPPMSCVUVnx3j6LKOpx/WhgH0A= github.com/aws/aws-sdk-go v1.32.11/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/beevik/ntp v0.2.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NRpg= @@ -656,6 +657,7 @@ github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= diff --git a/journal/filesystem.go b/journal/filesystem.go index 7aa470a2e..0f953b4e2 100644 --- a/journal/filesystem.go +++ b/journal/filesystem.go @@ -30,7 +30,7 @@ type fsJournal struct { fi *os.File fSize int64 - incoming chan *Entry + incoming chan *Event closing chan struct{} } @@ -47,7 +47,7 @@ func OpenFSJournal(lr repo.LockedRepo, lc fx.Lifecycle, disabled DisabledEvents) eventTypeFactory: newEventTypeFactory(disabled), dir: dir, sizeLimit: 1 << 30, - incoming: make(chan *Entry, 32), + incoming: make(chan *Event, 32), closing: make(chan struct{}), } @@ -64,8 +64,8 @@ func OpenFSJournal(lr repo.LockedRepo, lc fx.Lifecycle, disabled DisabledEvents) return f, nil } -func (f *fsJournal) AddEntry(evtType EventType, obj interface{}) { - je := &Entry{ +func (f *fsJournal) RecordEvent(evtType EventType, obj interface{}) { + je := &Event{ EventType: evtType, Timestamp: build.Clock.Now(), Data: obj, @@ -73,7 +73,7 @@ func (f *fsJournal) AddEntry(evtType EventType, obj interface{}) { select { case f.incoming <- je: case <-f.closing: - log.Warnw("journal closed but tried to log event", "entry", je) + log.Warnw("journal closed but tried to log event", "event", je) } } @@ -82,8 +82,8 @@ func (f *fsJournal) Close() error { return nil } -func (f *fsJournal) putEntry(je *Entry) error { - b, err := json.Marshal(je) +func (f *fsJournal) putEvent(evt *Event) error { + b, err := json.Marshal(evt) if err != nil { return err } @@ -120,8 +120,8 @@ func (f *fsJournal) runLoop() { for { select { case je := <-f.incoming: - if err := f.putEntry(je); err != nil { - log.Errorw("failed to write out journal entry", "entry", je, "err", err) + if err := f.putEvent(je); err != nil { + log.Errorw("failed to write out journal event", "event", je, "err", err) } case <-f.closing: _ = f.fi.Close() diff --git a/journal/memory.go b/journal/memory.go index 5471d3e11..6676b9696 100644 --- a/journal/memory.go +++ b/journal/memory.go @@ -17,17 +17,17 @@ type ( replay bool } rmObserverCtrl *observer - getEntriesCtrl chan []*Entry + getEntriesCtrl chan []*Event ) type MemJournal struct { *eventTypeFactory - entries []*Entry - index map[string]map[string][]*Entry + entries []*Event + index map[string]map[string][]*Event observers []observer - incomingCh chan *Entry + incomingCh chan *Event controlCh chan interface{} state int32 // guarded by atomic; 0=closed, 1=running. @@ -38,10 +38,10 @@ var _ Journal = (*MemJournal)(nil) type observer struct { accept map[EventType]struct{} - ch chan *Entry + ch chan *Event } -func (o *observer) dispatch(entry *Entry) { +func (o *observer) dispatch(entry *Event) { if o.accept == nil { o.ch <- entry } @@ -54,9 +54,9 @@ func NewMemoryJournal(lc fx.Lifecycle, disabled DisabledEvents) *MemJournal { m := &MemJournal{ eventTypeFactory: newEventTypeFactory(disabled), - index: make(map[string]map[string][]*Entry, 16), + index: make(map[string]map[string][]*Event, 16), observers: make([]observer, 0, 16), - incomingCh: make(chan *Entry, 256), + incomingCh: make(chan *Event, 256), controlCh: make(chan interface{}, 16), state: 1, closed: make(chan struct{}), @@ -71,13 +71,13 @@ func NewMemoryJournal(lc fx.Lifecycle, disabled DisabledEvents) *MemJournal { return m } -func (m *MemJournal) AddEntry(evtType EventType, obj interface{}) { +func (m *MemJournal) RecordEvent(evtType EventType, obj interface{}) { if !evtType.enabled || !evtType.safe { // tried to record a disabled event type, or used an invalid EventType. return } - entry := &Entry{ + entry := &Event{ EventType: evtType, Timestamp: build.Clock.Now(), Data: obj, @@ -111,7 +111,7 @@ func (m *MemJournal) Clear() { // be replayed. To restrict the event types this observer will sent, use the // include argument. If no include set is passed, the observer will receive all // events types. -func (m *MemJournal) Observe(ctx context.Context, replay bool, include ...EventType) <-chan *Entry { +func (m *MemJournal) Observe(ctx context.Context, replay bool, include ...EventType) <-chan *Event { var acc map[EventType]struct{} if include != nil { acc = make(map[EventType]struct{}, len(include)) @@ -124,7 +124,7 @@ func (m *MemJournal) Observe(ctx context.Context, replay bool, include ...EventT } } - ch := make(chan *Entry, 256) + ch := make(chan *Event, 256) o := &observer{ accept: acc, ch: ch, @@ -151,8 +151,8 @@ func (m *MemJournal) Observe(ctx context.Context, replay bool, include ...EventT } // Entries gets a snapshot of stored entries. -func (m *MemJournal) Entries() []*Entry { - ch := make(chan []*Entry) +func (m *MemJournal) Entries() []*Event { + ch := make(chan []*Event) m.controlCh <- getEntriesCtrl(ch) return <-ch } @@ -191,7 +191,7 @@ func (m *MemJournal) process() { } } case getEntriesCtrl: - cpy := make([]*Entry, len(m.entries)) + cpy := make([]*Event, len(m.entries)) copy(cpy, m.entries) msg <- cpy close(msg) @@ -200,7 +200,7 @@ func (m *MemJournal) process() { processClose := func() { m.entries = nil - m.index = make(map[string]map[string][]*Entry, 16) + m.index = make(map[string]map[string][]*Event, 16) for _, o := range m.observers { close(o.ch) } @@ -225,7 +225,7 @@ func (m *MemJournal) process() { m.entries = append(m.entries, entry) events := m.index[entry.System] if events == nil { - events = make(map[string][]*Entry, 16) + events = make(map[string][]*Event, 16) m.index[entry.System] = events } diff --git a/journal/memory_test.go b/journal/memory_test.go index 442a7f46d..db38085e3 100644 --- a/journal/memory_test.go +++ b/journal/memory_test.go @@ -71,7 +71,7 @@ func TestMemJournal_Close(t *testing.T) { time.Sleep(500 * time.Millisecond) NextChannel: - for _, ch := range []<-chan *Entry{o1, o2, o3} { + for _, ch := range []<-chan *Event{o1, o2, o3} { for { select { case _, more := <-ch: @@ -171,7 +171,7 @@ func addEntries(journal *MemJournal, count int) { // RegisterEventType is not _really_ intended to be used this way (on every write). et := journal.RegisterEventType("spaceship", fmt.Sprintf("wheezing-%d", eventIdx)) - journal.AddEntry(et, HeadChangeEvt{ + journal.RecordEvent(et, HeadChangeEvt{ From: types.TipSetKey{}, FromHeight: abi.ChainEpoch(i), To: types.TipSetKey{}, diff --git a/journal/nil.go b/journal/nil.go index 0123fe585..5d0c78b05 100644 --- a/journal/nil.go +++ b/journal/nil.go @@ -11,6 +11,6 @@ func NilJournal() Journal { func (n *nilJournal) RegisterEventType(_, _ string) EventType { return EventType{} } -func (n *nilJournal) AddEntry(_ EventType, _ interface{}) {} +func (n *nilJournal) RecordEvent(_ EventType, _ interface{}) {} func (n *nilJournal) Close() error { return nil } diff --git a/journal/sugar.go b/journal/sugar.go index 9ee59f2c7..069434916 100644 --- a/journal/sugar.go +++ b/journal/sugar.go @@ -1,17 +1,17 @@ package journal -// MaybeAddEntry is a convenience function that evaluates if the EventType is -// enabled, and if so, it calls the supplier to create the entry and -// subsequently journal.AddEntry on the provided journal to record it. +// MaybeRecordEvent is a convenience function that evaluates if the EventType is +// enabled, and if so, it calls the supplier to create the event and +// subsequently journal.RecordEvent on the provided journal to record it. // // This is safe to call with a nil Journal, either because the value is nil, // or because a journal obtained through NilJournal() is in use. -func MaybeAddEntry(journal Journal, evtType EventType, supplier func() interface{}) { +func MaybeRecordEvent(journal Journal, evtType EventType, supplier func() interface{}) { if journal == nil || journal == nilj { return } if !evtType.Enabled() { return } - journal.AddEntry(evtType, supplier()) + journal.RecordEvent(evtType, supplier()) } diff --git a/journal/types.go b/journal/types.go index 95349b24e..b1c36b515 100644 --- a/journal/types.go +++ b/journal/types.go @@ -47,18 +47,18 @@ type Journal interface { // entries appropriately. RegisterEventType(system, event string) EventType - // AddEntry adds an entry to this journal. See godocs on the Journal type + // RecordEvent records this event to the journal. See godocs on the Journal type // for more info. - AddEntry(evtType EventType, data interface{}) + RecordEvent(evtType EventType, data interface{}) // Close closes this journal for further writing. Close() error } -// Entry represents a journal entry. +// Event represents a journal entry. // // See godocs on Journal for more information. -type Entry struct { +type Event struct { EventType Timestamp time.Time diff --git a/markets/storageadapter/client.go b/markets/storageadapter/client.go index 1da85fa69..925a0a271 100644 --- a/markets/storageadapter/client.go +++ b/markets/storageadapter/client.go @@ -247,7 +247,7 @@ func (c *ClientNodeAdapter) ValidatePublishedDeal(ctx context.Context, deal stor } dealID := res.IDs[dealIdx] - journal.MaybeAddEntry(c.jrnl, c.evtTypes[evtTypeDealAccepted], func() interface{} { + journal.MaybeRecordEvent(c.jrnl, c.evtTypes[evtTypeDealAccepted], func() interface{} { deal := deal // copy and strip fields we don't want to log to the journal deal.ClientSignature = crypto.Signature{} return ClientDealAcceptedEvt{ID: dealID, Deal: deal, Height: c.cs.GetHeaviestTipSet().Height()} @@ -296,7 +296,7 @@ func (c *ClientNodeAdapter) OnDealSectorCommitted(ctx context.Context, provider log.Infof("Storage deal %d activated at epoch %d", dealId, sd.State.SectorStartEpoch) - journal.MaybeAddEntry(c.jrnl, c.evtTypes[evtTypeDealSectorCommitted], func() interface{} { + journal.MaybeRecordEvent(c.jrnl, c.evtTypes[evtTypeDealSectorCommitted], func() interface{} { return ClientDealSectorCommittedEvt{ID: dealId, State: sd.State, Height: curH} }) diff --git a/markets/storageadapter/provider.go b/markets/storageadapter/provider.go index 2aa9c45df..0b877cf6e 100644 --- a/markets/storageadapter/provider.go +++ b/markets/storageadapter/provider.go @@ -115,7 +115,7 @@ func (n *ProviderNodeAdapter) OnDealComplete(ctx context.Context, deal storagema log.Warnf("New Deal: deal %d", deal.DealID) - journal.MaybeAddEntry(n.jrnl, n.evtTypes[evtTypeDealAccepted], func() interface{} { + journal.MaybeRecordEvent(n.jrnl, n.evtTypes[evtTypeDealAccepted], func() interface{} { deal := deal // copy and strip fields we don't want to log to the journal deal.ClientSignature = crypto.Signature{} return MinerDealAcceptedEvt{ID: deal.DealID, Deal: deal} @@ -287,7 +287,7 @@ func (n *ProviderNodeAdapter) OnDealSectorCommitted(ctx context.Context, provide log.Infof("Storage deal %d activated at epoch %d", dealID, sd.State.SectorStartEpoch) - journal.MaybeAddEntry(n.jrnl, n.evtTypes[evtTypeDealSectorCommitted], func() interface{} { + journal.MaybeRecordEvent(n.jrnl, n.evtTypes[evtTypeDealSectorCommitted], func() interface{} { return MinerDealSectorCommittedEvt{ID: dealID, State: sd.State, Height: curH} }) @@ -397,7 +397,7 @@ func (n *ProviderNodeAdapter) OnDealExpiredOrSlashed(ctx context.Context, dealID // Check if the deal has already expired if sd.Proposal.EndEpoch <= height { onDealExpired(nil) - journal.MaybeAddEntry(n.jrnl, n.evtTypes[evtTypeDealExpired], func() interface{} { + journal.MaybeRecordEvent(n.jrnl, n.evtTypes[evtTypeDealExpired], func() interface{} { return MinerDealExpiredEvt{ID: dealID, State: sd.State, Height: height} }) return true, false, nil @@ -406,7 +406,7 @@ func (n *ProviderNodeAdapter) OnDealExpiredOrSlashed(ctx context.Context, dealID // If there is no deal assume it's already been slashed if sd.State.SectorStartEpoch < 0 { onDealSlashed(height, nil) - journal.MaybeAddEntry(n.jrnl, n.evtTypes[evtTypeDealSlashed], func() interface{} { + journal.MaybeRecordEvent(n.jrnl, n.evtTypes[evtTypeDealSlashed], func() interface{} { return MinerDealSlashedEvt{ID: dealID, State: sd.State, Height: height} }) return true, false, nil @@ -425,7 +425,7 @@ func (n *ProviderNodeAdapter) OnDealExpiredOrSlashed(ctx context.Context, dealID // Check if the deal has already expired if sd.Proposal.EndEpoch <= height { onDealExpired(nil) - journal.MaybeAddEntry(n.jrnl, n.evtTypes[evtTypeDealExpired], func() interface{} { + journal.MaybeRecordEvent(n.jrnl, n.evtTypes[evtTypeDealExpired], func() interface{} { return MinerDealExpiredEvt{ID: dealID, State: sd.State, Height: height} }) return false, nil @@ -451,7 +451,7 @@ func (n *ProviderNodeAdapter) OnDealExpiredOrSlashed(ctx context.Context, dealID // Deal was slashed if deal.To == nil { onDealSlashed(height, nil) - journal.MaybeAddEntry(n.jrnl, n.evtTypes[evtTypeDealSlashed], func() interface{} { + journal.MaybeRecordEvent(n.jrnl, n.evtTypes[evtTypeDealSlashed], func() interface{} { return MinerDealSlashedEvt{ID: dealID, State: sd.State, Height: height} }) return false, nil diff --git a/storage/miner.go b/storage/miner.go index 6f16b9cfe..499509b25 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -134,7 +134,7 @@ func (m *Miner) Run(ctx context.Context) error { } func (m *Miner) handleSealingNotifications(before, after sealing.SectorInfo) { - journal.MaybeAddEntry(m.jrnl, m.sealingEvtType, func() interface{} { + journal.MaybeRecordEvent(m.jrnl, m.sealingEvtType, func() interface{} { return SealingStateEvt{ SectorNumber: before.SectorNumber, SectorType: before.SectorType, diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 191ac5319..c6aedc068 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -27,7 +27,7 @@ import ( var errNoPartitions = errors.New("no partitions") func (s *WindowPoStScheduler) failPost(err error, deadline *miner.DeadlineInfo) { - journal.MaybeAddEntry(s.jrnl, s.wdPoStEvtType, func() interface{} { + journal.MaybeRecordEvent(s.jrnl, s.wdPoStEvtType, func() interface{} { return WindowPoStEvt{ State: "failed", Deadline: s.activeDeadline, @@ -51,7 +51,7 @@ func (s *WindowPoStScheduler) doPost(ctx context.Context, deadline *miner.Deadli s.abort = abort s.activeDeadline = deadline - journal.MaybeAddEntry(s.jrnl, s.wdPoStEvtType, func() interface{} { + journal.MaybeRecordEvent(s.jrnl, s.wdPoStEvtType, func() interface{} { return WindowPoStEvt{ State: "started", Deadline: s.activeDeadline, @@ -82,7 +82,7 @@ func (s *WindowPoStScheduler) doPost(ctx context.Context, deadline *miner.Deadli return } - journal.MaybeAddEntry(s.jrnl, s.wdPoStEvtType, func() interface{} { + journal.MaybeRecordEvent(s.jrnl, s.wdPoStEvtType, func() interface{} { return WindowPoStEvt{ State: "succeeded", Deadline: s.activeDeadline, @@ -148,7 +148,7 @@ func (s *WindowPoStScheduler) checkNextRecoveries(ctx context.Context, dlIdx uin Recoveries: []miner.RecoveryDeclaration{}, } - defer journal.MaybeAddEntry(s.jrnl, s.wdPoStEvtType, func() interface{} { + defer journal.MaybeRecordEvent(s.jrnl, s.wdPoStEvtType, func() interface{} { var mcid cid.Cid if sm != nil { mcid = sm.Cid() @@ -258,7 +258,7 @@ func (s *WindowPoStScheduler) checkNextFaults(ctx context.Context, dlIdx uint64, Faults: []miner.FaultDeclaration{}, } - defer journal.MaybeAddEntry(s.jrnl, s.wdPoStEvtType, func() interface{} { + defer journal.MaybeRecordEvent(s.jrnl, s.wdPoStEvtType, func() interface{} { var mcid cid.Cid if sm != nil { mcid = sm.Cid() @@ -519,7 +519,7 @@ func (s *WindowPoStScheduler) submitPost(ctx context.Context, proof *miner.Submi var sm *types.SignedMessage - defer journal.MaybeAddEntry(s.jrnl, s.wdPoStEvtType, func() interface{} { + defer journal.MaybeRecordEvent(s.jrnl, s.wdPoStEvtType, func() interface{} { var mcid cid.Cid if sm != nil { mcid = sm.Cid() diff --git a/storage/wdpost_sched.go b/storage/wdpost_sched.go index 3bc591bb7..8344097dc 100644 --- a/storage/wdpost_sched.go +++ b/storage/wdpost_sched.go @@ -247,7 +247,7 @@ func (s *WindowPoStScheduler) abortActivePoSt() { if s.abort != nil { s.abort() - journal.MaybeAddEntry(s.jrnl, s.wdPoStEvtType, func() interface{} { + journal.MaybeRecordEvent(s.jrnl, s.wdPoStEvtType, func() interface{} { return WindowPoStEvt{ State: "abort", Deadline: s.activeDeadline, From 443473f8338effeaa3e1f8c0f7005012b558083c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Tue, 21 Jul 2020 17:37:06 +0100 Subject: [PATCH 012/885] fix tests. --- chain/messagepool/messagepool_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/chain/messagepool/messagepool_test.go b/chain/messagepool/messagepool_test.go index 64af26e9a..b1787f8d6 100644 --- a/chain/messagepool/messagepool_test.go +++ b/chain/messagepool/messagepool_test.go @@ -9,6 +9,7 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/types/mock" "github.com/filecoin-project/lotus/chain/wallet" + "github.com/filecoin-project/lotus/journal" _ "github.com/filecoin-project/lotus/lib/sigs/bls" _ "github.com/filecoin-project/lotus/lib/sigs/secp" "github.com/filecoin-project/specs-actors/actors/crypto" @@ -147,7 +148,7 @@ func TestMessagePool(t *testing.T) { ds := datastore.NewMapDatastore() - mp, err := New(tma, ds, "mptest", nil) + mp, err := New(tma, ds, "mptest", journal.NilJournal()) if err != nil { t.Fatal(err) } @@ -188,7 +189,7 @@ func TestRevertMessages(t *testing.T) { ds := datastore.NewMapDatastore() - mp, err := New(tma, ds, "mptest", nil) + mp, err := New(tma, ds, "mptest", journal.NilJournal()) if err != nil { t.Fatal(err) } From 5074ce5a38986ac517c00fc2c022aa2dcf01dd4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Mon, 10 Aug 2020 15:02:15 +0100 Subject: [PATCH 013/885] move in-mem journal to Project Oni. --- journal/{filesystem.go => fs.go} | 4 +- journal/memory.go | 248 ------------------------------- journal/memory_test.go | 183 ----------------------- journal/sugar.go | 17 --- journal/types.go | 38 ++++- 5 files changed, 33 insertions(+), 457 deletions(-) rename journal/{filesystem.go => fs.go} (97%) delete mode 100644 journal/memory.go delete mode 100644 journal/memory_test.go delete mode 100644 journal/sugar.go diff --git a/journal/filesystem.go b/journal/fs.go similarity index 97% rename from journal/filesystem.go rename to journal/fs.go index 0f953b4e2..096f5f1fd 100644 --- a/journal/filesystem.go +++ b/journal/fs.go @@ -21,7 +21,7 @@ var log = logging.Logger("journal") // fsJournal is a basic journal backed by files on a filesystem. type fsJournal struct { - *eventTypeFactory + EventTypeFactory dir string sizeLimit int64 @@ -44,7 +44,7 @@ func OpenFSJournal(lr repo.LockedRepo, lc fx.Lifecycle, disabled DisabledEvents) } f := &fsJournal{ - eventTypeFactory: newEventTypeFactory(disabled), + EventTypeFactory: NewEventTypeFactory(disabled), dir: dir, sizeLimit: 1 << 30, incoming: make(chan *Event, 32), diff --git a/journal/memory.go b/journal/memory.go deleted file mode 100644 index 6676b9696..000000000 --- a/journal/memory.go +++ /dev/null @@ -1,248 +0,0 @@ -package journal - -import ( - "context" - "sync/atomic" - - "go.uber.org/fx" - - "github.com/filecoin-project/lotus/build" -) - -// Control messages. -type ( - clearCtrl struct{} - addObserverCtrl struct { - observer *observer - replay bool - } - rmObserverCtrl *observer - getEntriesCtrl chan []*Event -) - -type MemJournal struct { - *eventTypeFactory - - entries []*Event - index map[string]map[string][]*Event - observers []observer - - incomingCh chan *Event - controlCh chan interface{} - - state int32 // guarded by atomic; 0=closed, 1=running. - closed chan struct{} -} - -var _ Journal = (*MemJournal)(nil) - -type observer struct { - accept map[EventType]struct{} - ch chan *Event -} - -func (o *observer) dispatch(entry *Event) { - if o.accept == nil { - o.ch <- entry - } - if _, ok := o.accept[entry.EventType]; ok { - o.ch <- entry - } -} - -func NewMemoryJournal(lc fx.Lifecycle, disabled DisabledEvents) *MemJournal { - m := &MemJournal{ - eventTypeFactory: newEventTypeFactory(disabled), - - index: make(map[string]map[string][]*Event, 16), - observers: make([]observer, 0, 16), - incomingCh: make(chan *Event, 256), - controlCh: make(chan interface{}, 16), - state: 1, - closed: make(chan struct{}), - } - - lc.Append(fx.Hook{ - OnStop: func(_ context.Context) error { return m.Close() }, - }) - - go m.process() - - return m -} - -func (m *MemJournal) RecordEvent(evtType EventType, obj interface{}) { - if !evtType.enabled || !evtType.safe { - // tried to record a disabled event type, or used an invalid EventType. - return - } - - entry := &Event{ - EventType: evtType, - Timestamp: build.Clock.Now(), - Data: obj, - } - - select { - case m.incomingCh <- entry: - case <-m.closed: - } -} - -func (m *MemJournal) Close() error { - if !atomic.CompareAndSwapInt32(&m.state, 1, 0) { - // already closed. - return nil - } - close(m.closed) - return nil -} - -func (m *MemJournal) Clear() { - select { - case m.controlCh <- clearCtrl{}: - case <-m.closed: - } -} - -// Observe starts observing events that are recorded in the MemJournal, and -// returns a channel where new events will be sent. When replay is true, all -// entries that have been recorded prior to the observer being registered will -// be replayed. To restrict the event types this observer will sent, use the -// include argument. If no include set is passed, the observer will receive all -// events types. -func (m *MemJournal) Observe(ctx context.Context, replay bool, include ...EventType) <-chan *Event { - var acc map[EventType]struct{} - if include != nil { - acc = make(map[EventType]struct{}, len(include)) - for _, et := range include { - if !et.enabled { - // skip over disabled event type. - continue - } - acc[et] = struct{}{} - } - } - - ch := make(chan *Event, 256) - o := &observer{ - accept: acc, - ch: ch, - } - - // watch the context, and fire the "remove observer" control message upon - // cancellation. - go func() { - <-ctx.Done() - select { - case m.controlCh <- rmObserverCtrl(o): - case <-m.closed: - } - }() - - select { - case m.controlCh <- addObserverCtrl{o, replay}: - case <-m.closed: - // we are already stopped. - close(ch) - } - - return ch -} - -// Entries gets a snapshot of stored entries. -func (m *MemJournal) Entries() []*Event { - ch := make(chan []*Event) - m.controlCh <- getEntriesCtrl(ch) - return <-ch -} - -func (m *MemJournal) process() { - processCtrlMsg := func(message interface{}) { - switch msg := message.(type) { - case addObserverCtrl: - // adding an observer. - m.observers = append(m.observers, *msg.observer) - - if msg.replay { - // replay all existing entries. - for _, e := range m.entries { - msg.observer.dispatch(e) - } - } - case rmObserverCtrl: - // removing an observer; find the observer, close its channel. - // then discard it from our list by replacing it with the last - // observer and reslicing. - for i, o := range m.observers { - if o.ch == msg.ch { - close(o.ch) - m.observers[i] = m.observers[len(m.observers)-1] - m.observers = m.observers[:len(m.observers)-1] - } - } - case clearCtrl: - m.entries = m.entries[0:0] - // carry over system and event names; there are unlikely to change; - // just reslice the entry slices, so we are not thrashing memory. - for _, events := range m.index { - for ev := range events { - events[ev] = events[ev][0:0] - } - } - case getEntriesCtrl: - cpy := make([]*Event, len(m.entries)) - copy(cpy, m.entries) - msg <- cpy - close(msg) - } - } - - processClose := func() { - m.entries = nil - m.index = make(map[string]map[string][]*Event, 16) - for _, o := range m.observers { - close(o.ch) - } - m.observers = nil - } - - for { - // Drain all control messages first! - select { - case msg := <-m.controlCh: - processCtrlMsg(msg) - continue - case <-m.closed: - processClose() - return - default: - } - - // Now consume and pipe messages. - select { - case entry := <-m.incomingCh: - m.entries = append(m.entries, entry) - events := m.index[entry.System] - if events == nil { - events = make(map[string][]*Event, 16) - m.index[entry.System] = events - } - - entries := events[entry.Event] - events[entry.Event] = append(entries, entry) - - for _, o := range m.observers { - o.dispatch(entry) - } - - case msg := <-m.controlCh: - processCtrlMsg(msg) - continue - - case <-m.closed: - processClose() - return - } - } -} diff --git a/journal/memory_test.go b/journal/memory_test.go deleted file mode 100644 index db38085e3..000000000 --- a/journal/memory_test.go +++ /dev/null @@ -1,183 +0,0 @@ -package journal - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/types" - - "github.com/filecoin-project/specs-actors/actors/abi" - - "github.com/raulk/clock" - "github.com/stretchr/testify/require" - "go.uber.org/fx/fxtest" -) - -func TestMemJournal_AddEntry(t *testing.T) { - lc := fxtest.NewLifecycle(t) - defer lc.RequireStop() - - clk := clock.NewMock() - build.Clock = clk - - journal := NewMemoryJournal(lc, nil) - addEntries(journal, 100) - - require.Eventually(t, func() bool { return len(journal.Entries()) == 100 }, 1*time.Second, 100*time.Millisecond) - - entries := journal.Entries() - cnt := make(map[string]int, 10) - for i, e := range entries { - require.EqualValues(t, "spaceship", e.System) - require.Equal(t, HeadChangeEvt{ - From: types.TipSetKey{}, - FromHeight: abi.ChainEpoch(i), - To: types.TipSetKey{}, - ToHeight: abi.ChainEpoch(i), - RevertCount: i, - ApplyCount: i, - }, e.Data) - require.Equal(t, build.Clock.Now(), e.Timestamp) - cnt[e.Event]++ - } - - // we received 10 entries of each event type. - for _, c := range cnt { - require.Equal(t, 10, c) - } -} - -func TestMemJournal_Close(t *testing.T) { - lc := fxtest.NewLifecycle(t) - defer lc.RequireStop() - - journal := NewMemoryJournal(lc, nil) - addEntries(journal, 100) - - require.Eventually(t, func() bool { return len(journal.Entries()) == 100 }, 1*time.Second, 100*time.Millisecond) - - o1 := journal.Observe(context.TODO(), false) - o2 := journal.Observe(context.TODO(), false) - o3 := journal.Observe(context.TODO(), false) - - time.Sleep(500 * time.Millisecond) - - // Close the journal. - require.NoError(t, journal.Close()) - - time.Sleep(500 * time.Millisecond) - -NextChannel: - for _, ch := range []<-chan *Event{o1, o2, o3} { - for { - select { - case _, more := <-ch: - if more { - // keep consuming - } else { - continue NextChannel - } - default: - t.Fatal("nothing more to consume, and channel is not closed") - } - } - } -} - -func TestMemJournal_Clear(t *testing.T) { - lc := fxtest.NewLifecycle(t) - defer lc.RequireStop() - - journal := NewMemoryJournal(lc, nil) - addEntries(journal, 100) - - require.Eventually(t, func() bool { return len(journal.Entries()) == 100 }, 1*time.Second, 100*time.Millisecond) - - journal.Clear() - require.Empty(t, journal.Entries()) - require.Empty(t, journal.Entries()) - require.Empty(t, journal.Entries()) -} - -func TestMemJournal_Observe(t *testing.T) { - lc := fxtest.NewLifecycle(t) - defer lc.RequireStop() - - journal := NewMemoryJournal(lc, nil) - addEntries(journal, 100) - - require.Eventually(t, func() bool { return len(journal.Entries()) == 100 }, 1*time.Second, 100*time.Millisecond) - - et1 := journal.RegisterEventType("spaceship", "wheezing-1") - et2 := journal.RegisterEventType("spaceship", "wheezing-2") - - o1 := journal.Observe(context.TODO(), false, et1) - o2 := journal.Observe(context.TODO(), true, et1, et2) - o3 := journal.Observe(context.TODO(), true) - - time.Sleep(1 * time.Second) - - require.Len(t, o1, 0) // no replay - require.Len(t, o2, 20) // replay with include set - require.Len(t, o3, 100) // replay with no include set (all entries) - - // add another 100 entries and assert what the observers have seen. - addEntries(journal, 100) - - require.Eventually(t, func() bool { return len(journal.Entries()) == 200 }, 1*time.Second, 100*time.Millisecond) - - // note: we're able to queue items because the observer channel buffer size is 256. - require.Len(t, o1, 10) // should have 0 old entries + 10 new entries - require.Len(t, o2, 40) // should have 20 old entries + 20 new entries - require.Len(t, o3, 200) // should have 100 old entries + 100 new entries -} - -func TestMemJournal_ObserverCancellation(t *testing.T) { - lc := fxtest.NewLifecycle(t) - defer lc.RequireStop() - - journal := NewMemoryJournal(lc, nil) - - ctx, cancel := context.WithCancel(context.TODO()) - o1 := journal.Observe(ctx, false) - o2 := journal.Observe(context.TODO(), false) - addEntries(journal, 100) - - require.Eventually(t, func() bool { return len(journal.Entries()) == 100 }, 1*time.Second, 100*time.Millisecond) - - // all observers have received the 100 entries. - require.Len(t, o1, 100) - require.Len(t, o2, 100) - - // cancel o1's context. - cancel() - time.Sleep(500 * time.Millisecond) - - // add 50 new entries - addEntries(journal, 50) - - require.Eventually(t, func() bool { return len(journal.Entries()) == 150 }, 1*time.Second, 100*time.Millisecond) - - require.Len(t, o1, 100) // has not moved. - require.Len(t, o2, 150) // should have 100 old entries + 50 new entries -} - -func addEntries(journal *MemJournal, count int) { - for i := 0; i < count; i++ { - eventIdx := i % 10 - - // RegisterEventType is not _really_ intended to be used this way (on every write). - et := journal.RegisterEventType("spaceship", fmt.Sprintf("wheezing-%d", eventIdx)) - journal.RecordEvent(et, HeadChangeEvt{ - From: types.TipSetKey{}, - FromHeight: abi.ChainEpoch(i), - To: types.TipSetKey{}, - ToHeight: abi.ChainEpoch(i), - RevertCount: i, - ApplyCount: i, - }) - } -} diff --git a/journal/sugar.go b/journal/sugar.go deleted file mode 100644 index 069434916..000000000 --- a/journal/sugar.go +++ /dev/null @@ -1,17 +0,0 @@ -package journal - -// MaybeRecordEvent is a convenience function that evaluates if the EventType is -// enabled, and if so, it calls the supplier to create the event and -// subsequently journal.RecordEvent on the provided journal to record it. -// -// This is safe to call with a nil Journal, either because the value is nil, -// or because a journal obtained through NilJournal() is in use. -func MaybeRecordEvent(journal Journal, evtType EventType, supplier func() interface{}) { - if journal == nil || journal == nilj { - return - } - if !evtType.Enabled() { - return - } - journal.RecordEvent(evtType, supplier()) -} diff --git a/journal/types.go b/journal/types.go index b1c36b515..29954d9cf 100644 --- a/journal/types.go +++ b/journal/types.go @@ -29,7 +29,17 @@ type EventType struct { // All event types are enabled by default, and specific event types can only // be disabled at Journal construction time. func (et EventType) Enabled() bool { - return et.enabled + return et.safe && et.enabled +} + +// EventTypeFactory is a component that constructs tracked EventType tokens, +// for usage with a Journal. +type EventTypeFactory interface { + // RegisterEventType introduces a new event type to a journal, and + // returns an EventType token that components can later use to check whether + // journalling for that type is enabled/suppressed, and to tag journal + // entries appropriately. + RegisterEventType(system, event string) EventType } // Journal represents an audit trail of system actions. @@ -41,11 +51,7 @@ func (et EventType) Enabled() bool { // For cleanliness and type safety, we recommend to use typed events. See the // *Evt struct types in this package for more info. type Journal interface { - // RegisterEventType introduces a new event type to this journal, and - // returns an EventType token that components can later use to check whether - // journalling for that type is enabled/suppressed, and to tag journal - // entries appropriately. - RegisterEventType(system, event string) EventType + EventTypeFactory // RecordEvent records this event to the journal. See godocs on the Journal type // for more info. @@ -65,6 +71,22 @@ type Event struct { Data interface{} } +// MaybeRecordEvent is a convenience function that evaluates if the EventType is +// enabled, and if so, it calls the supplier to create the event and +// subsequently journal.RecordEvent on the provided journal to record it. +// +// This is safe to call with a nil Journal, either because the value is nil, +// or because a journal obtained through NilJournal() is in use. +func MaybeRecordEvent(journal Journal, evtType EventType, supplier func() interface{}) { + if journal == nil || journal == nilj { + return + } + if !evtType.Enabled() { + return + } + journal.RecordEvent(evtType, supplier()) +} + // eventTypeFactory is an embeddable mixin that takes care of tracking disabled // event types, and returning initialized/safe EventTypes when requested. type eventTypeFactory struct { @@ -73,7 +95,9 @@ type eventTypeFactory struct { m map[string]EventType } -func newEventTypeFactory(disabled DisabledEvents) *eventTypeFactory { +var _ EventTypeFactory = (*eventTypeFactory)(nil) + +func NewEventTypeFactory(disabled DisabledEvents) EventTypeFactory { ret := &eventTypeFactory{ m: make(map[string]EventType, len(disabled)+32), // + extra capacity. } From 5e2c28d7e9e84c77ffde8b0b63178f42e2af0119 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Tue, 11 Aug 2020 13:10:22 +0100 Subject: [PATCH 014/885] add comment. --- markets/storageadapter/client.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/markets/storageadapter/client.go b/markets/storageadapter/client.go index db43c9a56..9186b94c0 100644 --- a/markets/storageadapter/client.go +++ b/markets/storageadapter/client.go @@ -43,7 +43,9 @@ type ClientNodeAdapter struct { fm *market.FundMgr ev *events.Events - jrnl journal.Journal + jrnl journal.Journal + + // index 0 is unused, as it corresponds to evtTypeDealAccepted, a provider-only event. evtTypes [4]journal.EventType } From d53c6f25991630fb085508298c4e3e6d7e9eebef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Tue, 11 Aug 2020 13:22:35 +0100 Subject: [PATCH 015/885] storage-fsm: emit sealing state transition notifications. --- extern/storage-fsm/fsm.go | 8 +++++++- extern/storage-fsm/fsm_test.go | 17 ++++++++++++++++- extern/storage-fsm/sealing.go | 7 ++++++- go.mod | 2 +- go.sum | 6 ++++++ 5 files changed, 36 insertions(+), 4 deletions(-) diff --git a/extern/storage-fsm/fsm.go b/extern/storage-fsm/fsm.go index 25b8f364a..4d6c4b7e0 100644 --- a/extern/storage-fsm/fsm.go +++ b/extern/storage-fsm/fsm.go @@ -165,6 +165,12 @@ func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(sta state.Log = append(state.Log, l) } + if m.notifee != nil { + defer func(before SectorInfo) { + m.notifee(before, *state) + }(*state) // take safe-ish copy of the before state (except for nested pointers) + } + p := fsmPlanners[state.State] if p == nil { return nil, xerrors.Errorf("planner for state %s not found", state.State) @@ -351,7 +357,7 @@ func on(mut mutator, next SectorState) func() (mutator, SectorState) { } } -func planOne(ts ...func() (mut mutator, next SectorState)) func(events []statemachine.Event, state *SectorInfo) error { +func planOne(ts ...func() (mutator, SectorState)) func(events []statemachine.Event, state *SectorInfo) error { return func(events []statemachine.Event, state *SectorInfo) error { if len(events) != 1 { for _, event := range events { diff --git a/extern/storage-fsm/fsm_test.go b/extern/storage-fsm/fsm_test.go index b1e53133c..4625bd26c 100644 --- a/extern/storage-fsm/fsm_test.go +++ b/extern/storage-fsm/fsm_test.go @@ -25,8 +25,13 @@ type test struct { } func TestHappyPath(t *testing.T) { + var notif []struct{ before, after SectorInfo } m := test{ - s: &Sealing{}, + s: &Sealing{ + notifee: func(before, after SectorInfo) { + notif = append(notif, struct{ before, after SectorInfo }{before, after}) + }, + }, t: t, state: &SectorInfo{State: Packing}, } @@ -57,6 +62,16 @@ func TestHappyPath(t *testing.T) { m.planSingle(SectorFinalized{}) require.Equal(m.t, m.state.State, Proving) + + expected := []SectorState{Packing, PreCommit1, PreCommit2, PreCommitting, PreCommitWait, WaitSeed, Committing, CommitWait, FinalizeSector, Proving} + for i, n := range notif { + if n.before.State != expected[i] { + t.Fatalf("expected before state: %s, got: %s", expected[i], n.before.State) + } + if n.after.State != expected[i+1] { + t.Fatalf("expected after state: %s, got: %s", expected[i+1], n.after.State) + } + } } func TestSeedRevert(t *testing.T) { diff --git a/extern/storage-fsm/sealing.go b/extern/storage-fsm/sealing.go index d15255696..76040764f 100644 --- a/extern/storage-fsm/sealing.go +++ b/extern/storage-fsm/sealing.go @@ -51,6 +51,8 @@ type SealingAPI interface { ChainReadObj(context.Context, cid.Cid) ([]byte, error) } +type SectorStateNotifee func(before, after SectorInfo) + type Sealing struct { api SealingAPI events Events @@ -69,6 +71,8 @@ type Sealing struct { toUpgrade map[abi.SectorNumber]struct{} getSealDelay GetSealingDelayFunc + + notifee SectorStateNotifee } type UnsealedSectorMap struct { @@ -83,7 +87,7 @@ type UnsealedSectorInfo struct { pieceSizes []abi.UnpaddedPieceSize } -func New(api SealingAPI, events Events, maddr address.Address, ds datastore.Batching, sealer sectorstorage.SectorManager, sc SectorIDCounter, verif ffiwrapper.Verifier, pcp PreCommitPolicy, gsd GetSealingDelayFunc) *Sealing { +func New(api SealingAPI, events Events, maddr address.Address, ds datastore.Batching, sealer sectorstorage.SectorManager, sc SectorIDCounter, verif ffiwrapper.Verifier, pcp PreCommitPolicy, gsd GetSealingDelayFunc, notifee SectorStateNotifee) *Sealing { s := &Sealing{ api: api, events: events, @@ -100,6 +104,7 @@ func New(api SealingAPI, events Events, maddr address.Address, ds datastore.Batc toUpgrade: map[abi.SectorNumber]struct{}{}, getSealDelay: gsd, + notifee: notifee, } s.sectors = statemachine.New(namespace.Wrap(ds, datastore.NewKey(SectorStorePrefix)), s, SectorInfo{}) diff --git a/go.mod b/go.mod index 90bc30a80..51ca85830 100644 --- a/go.mod +++ b/go.mod @@ -37,7 +37,7 @@ require ( github.com/filecoin-project/sector-storage v0.0.0-20200805173933-deec7a2658d4 github.com/filecoin-project/specs-actors v0.8.7-0.20200805174427-9d42fb163883 github.com/filecoin-project/specs-storage v0.1.1-0.20200622113353-88a9704877ea - github.com/filecoin-project/storage-fsm v0.0.0-20200810142921-b9bc6718b0b2 + github.com/filecoin-project/storage-fsm v0.0.0-20200805013058-9d9ea4e6331f github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 github.com/go-kit/kit v0.10.0 github.com/go-ole/go-ole v1.2.4 // indirect diff --git a/go.sum b/go.sum index 3fb0bfc5d..ca9f0df33 100644 --- a/go.sum +++ b/go.sum @@ -40,6 +40,7 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/GeertJohan/go.incremental v1.0.0 h1:7AH+pY1XUgQE4Y1HcXYaMqAI0m9yrFqo/jt0CW30vsg= github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0= github.com/GeertJohan/go.rice v1.0.0 h1:KkI6O9uMaQU3VEKaj01ulavtF7o1fWT7+pk/4voiMLQ= github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0= @@ -61,6 +62,7 @@ github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpH github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= +github.com/akavel/rsrc v0.8.0 h1:zjWn7ukO9Kc5Q62DOJCcxGpXC18RawVtYAGdz2aLlfw= github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= @@ -644,6 +646,7 @@ github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0 github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= @@ -1128,6 +1131,7 @@ github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJE github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= github.com/nikkolasg/hexjson v0.0.0-20181101101858-78e39397e00c h1:5bFTChQxSKNwy8ALwOebjekYExl9HTT9urdawqC95tA= github.com/nikkolasg/hexjson v0.0.0-20181101101858-78e39397e00c/go.mod h1:7qN3Y0BvzRUf4LofcoJplQL10lsFDb4PYlePTVwrP28= +github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229 h1:E2B8qYyeSgv5MXpmzZXRNp8IAQ4vjxIjhpAf5hv/tAg= github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= @@ -1337,7 +1341,9 @@ github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX github.com/urfave/cli/v2 v2.0.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4= github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= From 2ea5abdfb5cbba97d5c935a3ad3f292287e760cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Tue, 11 Aug 2020 13:48:32 +0100 Subject: [PATCH 016/885] wire journal into miner. --- cmd/lotus-storage-miner/init.go | 15 +++++++++++---- journal/fs.go | 13 ++++++------- journal/types.go | 9 +++++++++ miner/journal_events.go | 1 + miner/miner.go | 28 +++++++++++++++++++++------- node/builder.go | 9 ++------- node/modules/services.go | 17 +++++++++++++++++ node/modules/storageminer.go | 4 ++-- 8 files changed, 69 insertions(+), 27 deletions(-) create mode 100644 miner/journal_events.go diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index 5007a7120..1714bd901 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -24,9 +24,6 @@ import ( "github.com/filecoin-project/go-address" cborutil "github.com/filecoin-project/go-cbor-util" paramfetch "github.com/filecoin-project/go-paramfetch" - sectorstorage "github.com/filecoin-project/sector-storage" - "github.com/filecoin-project/sector-storage/ffiwrapper" - "github.com/filecoin-project/sector-storage/stores" "github.com/filecoin-project/specs-actors/actors/abi" "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/builtin/market" @@ -34,6 +31,11 @@ import ( "github.com/filecoin-project/specs-actors/actors/builtin/power" crypto2 "github.com/filecoin-project/specs-actors/actors/crypto" + "github.com/filecoin-project/lotus/journal" + sectorstorage "github.com/filecoin-project/sector-storage" + "github.com/filecoin-project/sector-storage/ffiwrapper" + "github.com/filecoin-project/sector-storage/stores" + lapi "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" @@ -446,7 +448,12 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api lapi.FullNode, return err } - m := miner.NewMiner(api, epp, a, slashfilter.New(mds)) + jrnl, err := journal.OpenFSJournal(lr, journal.DefaultDisabledEvents) + if err != nil { + return fmt.Errorf("failed to open filesystem journal: %w", err) + } + + m := miner.NewMiner(api, epp, a, slashfilter.New(mds), jrnl) { if err := m.Start(ctx); err != nil { return xerrors.Errorf("failed to start up genesis miner: %w", err) diff --git a/journal/fs.go b/journal/fs.go index 096f5f1fd..91b0d1e6d 100644 --- a/journal/fs.go +++ b/journal/fs.go @@ -1,7 +1,6 @@ package journal import ( - "context" "encoding/json" "fmt" "os" @@ -10,7 +9,6 @@ import ( "time" logging "github.com/ipfs/go-log" - "go.uber.org/fx" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/build" @@ -33,11 +31,12 @@ type fsJournal struct { incoming chan *Event closing chan struct{} + closed chan struct{} } // OpenFSJournal constructs a rolling filesystem journal, with a default // per-file size limit of 1GiB. -func OpenFSJournal(lr repo.LockedRepo, lc fx.Lifecycle, disabled DisabledEvents) (Journal, error) { +func OpenFSJournal(lr repo.LockedRepo, disabled DisabledEvents) (Journal, error) { dir := filepath.Join(lr.Path(), "journal") if err := os.MkdirAll(dir, 0755); err != nil { return nil, fmt.Errorf("failed to mk directory %s for file journal: %w", dir, err) @@ -49,16 +48,13 @@ func OpenFSJournal(lr repo.LockedRepo, lc fx.Lifecycle, disabled DisabledEvents) sizeLimit: 1 << 30, incoming: make(chan *Event, 32), closing: make(chan struct{}), + closed: make(chan struct{}), } if err := f.rollJournalFile(); err != nil { return nil, err } - lc.Append(fx.Hook{ - OnStop: func(_ context.Context) error { return f.Close() }, - }) - go f.runLoop() return f, nil @@ -79,6 +75,7 @@ func (f *fsJournal) RecordEvent(evtType EventType, obj interface{}) { func (f *fsJournal) Close() error { close(f.closing) + <-f.closed return nil } @@ -117,6 +114,8 @@ func (f *fsJournal) rollJournalFile() error { } func (f *fsJournal) runLoop() { + defer close(f.closed) + for { select { case je := <-f.incoming: diff --git a/journal/types.go b/journal/types.go index 29954d9cf..6b6d53c08 100644 --- a/journal/types.go +++ b/journal/types.go @@ -5,6 +5,15 @@ import ( "time" ) +var ( + // DefaultDisabledEvents lists the journal events disabled by + // default, usually because they are considered noisy. + DefaultDisabledEvents = DisabledEvents{ + EventType{System: "mpool", Event: "add"}, + EventType{System: "mpool", Event: "remove"}, + } +) + // DisabledEvents is the set of event types whose journaling is suppressed. type DisabledEvents []EventType diff --git a/miner/journal_events.go b/miner/journal_events.go new file mode 100644 index 000000000..ab865910b --- /dev/null +++ b/miner/journal_events.go @@ -0,0 +1 @@ +package miner diff --git a/miner/miner.go b/miner/miner.go index b8bb9e562..091919b6c 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -31,6 +31,11 @@ import ( var log = logging.Logger("miner") +// Journal event types. +const ( + evtTypeBlockMined = iota +) + // returns a callback reporting whether we mined a blocks in this round type waitFunc func(ctx context.Context, baseTime uint64) (func(bool, error), abi.ChainEpoch, error) @@ -42,7 +47,7 @@ func randTimeOffset(width time.Duration) time.Duration { return val - (width / 2) } -func NewMiner(api api.FullNode, epp gen.WinningPoStProver, addr address.Address, sf *slashfilter.SlashFilter) *Miner { +func NewMiner(api api.FullNode, epp gen.WinningPoStProver, addr address.Address, sf *slashfilter.SlashFilter, jrnl journal.Journal) *Miner { arc, err := lru.NewARC(10000) if err != nil { panic(err) @@ -66,6 +71,10 @@ func NewMiner(api api.FullNode, epp gen.WinningPoStProver, addr address.Address, sf: sf, minedBlockHeights: arc, + jrnl: jrnl, + evtTypes: [...]journal.EventType{ + evtTypeBlockMined: jrnl.RegisterEventType("miner", "block_mined"), + }, } } @@ -85,6 +94,9 @@ type Miner struct { sf *slashfilter.SlashFilter minedBlockHeights *lru.ARCCache + + jrnl journal.Journal + evtTypes [1]journal.EventType } func (m *Miner) Address() address.Address { @@ -213,12 +225,14 @@ func (m *Miner) mine(ctx context.Context) { onDone(b != nil, nil) if b != nil { - journal.Add("blockMined", map[string]interface{}{ - "parents": base.TipSet.Cids(), - "nulls": base.NullRounds, - "epoch": b.Header.Height, - "timestamp": b.Header.Timestamp, - "cid": b.Header.Cid(), + journal.MaybeRecordEvent(m.jrnl, m.evtTypes[evtTypeBlockMined], func() interface{} { + return map[string]interface{}{ + "parents": base.TipSet.Cids(), + "nulls": base.NullRounds, + "epoch": b.Header.Height, + "timestamp": b.Header.Timestamp, + "cid": b.Header.Cid(), + } }) btime := time.Unix(int64(b.Header.Timestamp), 0) diff --git a/node/builder.go b/node/builder.go index 86ef23192..580aec3df 100644 --- a/node/builder.go +++ b/node/builder.go @@ -156,13 +156,8 @@ func defaults() []Option { Override(new(record.Validator), modules.RecordValidator), Override(new(dtypes.Bootstrapper), dtypes.Bootstrapper(false)), Override(new(dtypes.ShutdownChan), make(chan struct{})), - Override(new(journal.Journal), journal.OpenFSJournal), - - // By default, disable noisy mpool events; keep only mpool:repub on. - Override(new(journal.DisabledEvents), journal.DisabledEvents{ - journal.EventType{System: "mpool", Event: "add"}, - journal.EventType{System: "mpool", Event: "remove"}, - }), + Override(new(journal.Journal), modules.OpenFilesystemJournal), + Override(new(journal.DisabledEvents), journal.DefaultDisabledEvents), Override(InitJournalKey, func(j journal.Journal) { /* forces the creation of the journal at startup */ }), // Filecoin modules diff --git a/node/modules/services.go b/node/modules/services.go index 0d148ffb4..def286913 100644 --- a/node/modules/services.go +++ b/node/modules/services.go @@ -1,6 +1,8 @@ package modules import ( + "context" + "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" eventbus "github.com/libp2p/go-eventbus" @@ -22,10 +24,12 @@ import ( "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/sub" + "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/lib/peermgr" "github.com/filecoin-project/lotus/node/hello" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/helpers" + "github.com/filecoin-project/lotus/node/repo" ) func RunHello(mctx helpers.MetricsCtx, lc fx.Lifecycle, h host.Host, svc *hello.Service) error { @@ -141,3 +145,16 @@ func RandomBeacon(p RandomBeaconParams, _ dtypes.AfterGenesisSet) (beacon.Random //return beacon.NewMockBeacon(build.BlockDelaySecs * time.Second) return drand.NewDrandBeacon(gen.Timestamp, build.BlockDelaySecs, p.PubSub, p.DrandConfig) } + +func OpenFilesystemJournal(lr repo.LockedRepo, lc fx.Lifecycle, disabled journal.DisabledEvents) (journal.Journal, error) { + jrnl, err := journal.OpenFSJournal(lr, disabled) + if err != nil { + return nil, err + } + + lc.Append(fx.Hook{ + OnStop: func(_ context.Context) error { return jrnl.Close() }, + }) + + return jrnl, err +} diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index a06e44198..fc7458f8f 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -340,13 +340,13 @@ func StagingGraphsync(mctx helpers.MetricsCtx, lc fx.Lifecycle, ibs dtypes.Stagi return gs } -func SetupBlockProducer(lc fx.Lifecycle, ds dtypes.MetadataDS, api lapi.FullNode, epp gen.WinningPoStProver, sf *slashfilter.SlashFilter) (*miner.Miner, error) { +func SetupBlockProducer(lc fx.Lifecycle, ds dtypes.MetadataDS, api lapi.FullNode, epp gen.WinningPoStProver, sf *slashfilter.SlashFilter, jrnl journal.Journal) (*miner.Miner, error) { minerAddr, err := minerAddrFromDS(ds) if err != nil { return nil, err } - m := miner.NewMiner(api, epp, minerAddr, sf) + m := miner.NewMiner(api, epp, minerAddr, sf, jrnl) lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { From b534ab9d3c8aa8f4a4f86e91cd089465701118ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Tue, 11 Aug 2020 14:28:00 +0100 Subject: [PATCH 017/885] panic recovery in MaybeRecordEvent; handle nil current tipsets in wdpost. --- journal/fs.go | 3 --- journal/types.go | 16 ++++++++++++++++ miner/journal_events.go | 1 - storage/wdpost_run.go | 36 ++++++++++++------------------------ storage/wdpost_sched.go | 21 +++++++++++++++------ 5 files changed, 43 insertions(+), 34 deletions(-) delete mode 100644 miner/journal_events.go diff --git a/journal/fs.go b/journal/fs.go index 91b0d1e6d..73a2fff43 100644 --- a/journal/fs.go +++ b/journal/fs.go @@ -8,15 +8,12 @@ import ( "sync" "time" - logging "github.com/ipfs/go-log" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/node/repo" ) -var log = logging.Logger("journal") - // fsJournal is a basic journal backed by files on a filesystem. type fsJournal struct { EventTypeFactory diff --git a/journal/types.go b/journal/types.go index 6b6d53c08..6183e6cfa 100644 --- a/journal/types.go +++ b/journal/types.go @@ -3,8 +3,12 @@ package journal import ( "sync" "time" + + logging "github.com/ipfs/go-log" ) +var log = logging.Logger("journal") + var ( // DefaultDisabledEvents lists the journal events disabled by // default, usually because they are considered noisy. @@ -30,6 +34,10 @@ type EventType struct { safe bool } +func (et EventType) String() string { + return et.System + ":" + et.Event +} + // Enabled returns whether this event type is enabled in the journaling // subsystem. Users are advised to check this before actually attempting to // add a journal entry, as it helps bypass object construction for events that @@ -84,9 +92,17 @@ type Event struct { // enabled, and if so, it calls the supplier to create the event and // subsequently journal.RecordEvent on the provided journal to record it. // +// It also recovers from panics raised when calling the supplier function. +// // This is safe to call with a nil Journal, either because the value is nil, // or because a journal obtained through NilJournal() is in use. func MaybeRecordEvent(journal Journal, evtType EventType, supplier func() interface{}) { + defer func() { + if r := recover(); r != nil { + log.Warnf("recovered from panic while recording journal event; type=%s, err=%v", evtType, r) + } + }() + if journal == nil || journal == nilj { return } diff --git a/miner/journal_events.go b/miner/journal_events.go deleted file mode 100644 index ab865910b..000000000 --- a/miner/journal_events.go +++ /dev/null @@ -1 +0,0 @@ -package miner diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 6750042cf..0ec4ac1f6 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -27,13 +27,11 @@ var errNoPartitions = errors.New("no partitions") func (s *WindowPoStScheduler) failPost(err error, deadline *miner.DeadlineInfo) { journal.MaybeRecordEvent(s.jrnl, s.wdPoStEvtType, func() interface{} { - return WindowPoStEvt{ + return s.enrichWithTipset(WindowPoStEvt{ State: "failed", Deadline: s.activeDeadline, - Height: s.cur.Height(), - TipSet: s.cur.Cids(), Error: err, - } + }) }) log.Errorf("TODO") @@ -51,12 +49,10 @@ func (s *WindowPoStScheduler) doPost(ctx context.Context, deadline *miner.Deadli s.activeDeadline = deadline journal.MaybeRecordEvent(s.jrnl, s.wdPoStEvtType, func() interface{} { - return WindowPoStEvt{ + return s.enrichWithTipset(WindowPoStEvt{ State: "started", Deadline: s.activeDeadline, - Height: s.cur.Height(), - TipSet: s.cur.Cids(), - } + }) }) go func() { @@ -82,12 +78,10 @@ func (s *WindowPoStScheduler) doPost(ctx context.Context, deadline *miner.Deadli } journal.MaybeRecordEvent(s.jrnl, s.wdPoStEvtType, func() interface{} { - return WindowPoStEvt{ + return s.enrichWithTipset(WindowPoStEvt{ State: "succeeded", Deadline: s.activeDeadline, - Height: s.cur.Height(), - TipSet: s.cur.Cids(), - } + }) }) }() } @@ -153,16 +147,14 @@ func (s *WindowPoStScheduler) checkNextRecoveries(ctx context.Context, dlIdx uin mcid = sm.Cid() } - return WindowPoStEvt{ + return s.enrichWithTipset(WindowPoStEvt{ State: "recoveries_processed", Deadline: s.activeDeadline, - Height: s.cur.Height(), - TipSet: s.cur.Cids(), Recoveries: &WindowPoStEvt_Recoveries{ Declarations: params.Recoveries, MessageCID: mcid, }, - } + }) }) for partIdx, partition := range partitions { @@ -260,16 +252,14 @@ func (s *WindowPoStScheduler) checkNextFaults(ctx context.Context, dlIdx uint64, if sm != nil { mcid = sm.Cid() } - return WindowPoStEvt{ + return s.enrichWithTipset(WindowPoStEvt{ State: "faults_processed", Deadline: s.activeDeadline, - Height: s.cur.Height(), - TipSet: s.cur.Cids(), Faults: &WindowPoStEvt_Faults{ Declarations: params.Faults, MessageCID: mcid, }, - } + }) }) for partIdx, partition := range partitions { @@ -517,16 +507,14 @@ func (s *WindowPoStScheduler) submitPost(ctx context.Context, proof *miner.Submi mcid = sm.Cid() } - return WindowPoStEvt{ + return s.enrichWithTipset(WindowPoStEvt{ State: "proofs_processed", Deadline: s.activeDeadline, - Height: s.cur.Height(), - TipSet: s.cur.Cids(), Proofs: &WindowPoStEvt_Proofs{ Partitions: proof.Partitions, MessageCID: mcid, }, - } + }) }) enc, aerr := actors.SerializeParams(proof) diff --git a/storage/wdpost_sched.go b/storage/wdpost_sched.go index 8344097dc..2c55b93ef 100644 --- a/storage/wdpost_sched.go +++ b/storage/wdpost_sched.go @@ -140,12 +140,13 @@ func (s *WindowPoStScheduler) Run(ctx context.Context) { log.Errorf("expected first notif to have len = 1") continue } - if changes[0].Type != store.HCCurrent { + chg := changes[0] + if chg.Type != store.HCCurrent { log.Errorf("expected first notif to tell current ts") continue } - if err := s.update(ctx, changes[0].Val); err != nil { + if err := s.update(ctx, chg.Val); err != nil { log.Errorf("%+v", err) } @@ -248,12 +249,10 @@ func (s *WindowPoStScheduler) abortActivePoSt() { s.abort() journal.MaybeRecordEvent(s.jrnl, s.wdPoStEvtType, func() interface{} { - return WindowPoStEvt{ + return s.enrichWithTipset(WindowPoStEvt{ State: "abort", Deadline: s.activeDeadline, - Height: s.cur.Height(), - TipSet: s.cur.Cids(), - } + }) }) log.Warnf("Aborting Window PoSt (Deadline: %+v)", s.activeDeadline) @@ -262,3 +261,13 @@ func (s *WindowPoStScheduler) abortActivePoSt() { s.activeDeadline = nil s.abort = nil } + +// enrichWithTipset enriches a WindowPoStEvt with tipset information, +// if available. +func (s *WindowPoStScheduler) enrichWithTipset(evt WindowPoStEvt) WindowPoStEvt { + if s.cur != nil { + evt.Height = s.cur.Height() + evt.TipSet = s.cur.Cids() + } + return evt +} From 8f19fff1ecb770d4108a692be47829b91e4ba8e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Tue, 11 Aug 2020 14:47:02 +0100 Subject: [PATCH 018/885] fix tests. --- chain/messagepool/messagepool_test.go | 2 +- chain/messagepool/selection_test.go | 4 +++- chain/store/index_test.go | 3 ++- chain/store/store_test.go | 5 +++-- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/chain/messagepool/messagepool_test.go b/chain/messagepool/messagepool_test.go index 1dcc6a233..06541d7e5 100644 --- a/chain/messagepool/messagepool_test.go +++ b/chain/messagepool/messagepool_test.go @@ -271,7 +271,7 @@ func TestPruningSimple(t *testing.T) { ds := datastore.NewMapDatastore() - mp, err := New(tma, ds, "mptest") + mp, err := New(tma, ds, "mptest", journal.NilJournal()) if err != nil { t.Fatal(err) } diff --git a/chain/messagepool/selection_test.go b/chain/messagepool/selection_test.go index 177c62dc2..91337d63c 100644 --- a/chain/messagepool/selection_test.go +++ b/chain/messagepool/selection_test.go @@ -10,6 +10,8 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/types/mock" "github.com/filecoin-project/lotus/chain/wallet" + "github.com/filecoin-project/lotus/journal" + "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/crypto" "github.com/ipfs/go-datastore" @@ -42,7 +44,7 @@ func makeTestMessage(w *wallet.Wallet, from, to address.Address, nonce uint64, g func makeTestMpool() (*MessagePool, *testMpoolAPI) { tma := newTestMpoolAPI() ds := datastore.NewMapDatastore() - mp, err := New(tma, ds, "test") + mp, err := New(tma, ds, "test", journal.NilJournal()) if err != nil { panic(err) } diff --git a/chain/store/index_test.go b/chain/store/index_test.go index 9443b4c07..2a0fdc2d1 100644 --- a/chain/store/index_test.go +++ b/chain/store/index_test.go @@ -8,6 +8,7 @@ import ( "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types/mock" + "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/lib/blockstore" "github.com/filecoin-project/specs-actors/actors/abi" datastore "github.com/ipfs/go-datastore" @@ -31,7 +32,7 @@ func TestIndexSeeks(t *testing.T) { ctx := context.TODO() nbs := blockstore.NewTemporarySync() - cs := store.NewChainStore(nbs, syncds.MutexWrap(datastore.NewMapDatastore()), nil) + cs := store.NewChainStore(nbs, syncds.MutexWrap(datastore.NewMapDatastore()), nil, journal.NilJournal()) _, err = cs.Import(bytes.NewReader(gencar)) if err != nil { diff --git a/chain/store/store_test.go b/chain/store/store_test.go index 3267feccd..56a78330a 100644 --- a/chain/store/store_test.go +++ b/chain/store/store_test.go @@ -17,6 +17,7 @@ import ( "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/lib/blockstore" "github.com/filecoin-project/lotus/node/repo" ) @@ -67,7 +68,7 @@ func BenchmarkGetRandomness(b *testing.B) { bs := blockstore.NewBlockstore(bds) - cs := store.NewChainStore(bs, mds, nil) + cs := store.NewChainStore(bs, mds, nil, journal.NilJournal()) b.ResetTimer() @@ -101,7 +102,7 @@ func TestChainExportImport(t *testing.T) { } nbs := blockstore.NewTemporary() - cs := store.NewChainStore(nbs, datastore.NewMapDatastore(), nil) + cs := store.NewChainStore(nbs, datastore.NewMapDatastore(), nil, journal.NilJournal()) root, err := cs.Import(buf) if err != nil { From 200b0f72cc4a9b5d78085b24e9b69f139296764d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Tue, 11 Aug 2020 15:06:13 +0100 Subject: [PATCH 019/885] mpool/repub: only record in journal if actually repubbing. --- chain/messagepool/messagepool.go | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/chain/messagepool/messagepool.go b/chain/messagepool/messagepool.go index c87a02b95..19856b9f7 100644 --- a/chain/messagepool/messagepool.go +++ b/chain/messagepool/messagepool.go @@ -371,17 +371,19 @@ func (mp *MessagePool) runLoop() { log.Errorf("errors while republishing: %+v", errout) } - journal.MaybeRecordEvent(mp.jrnl, mp.evtTypes[evtTypeMpoolRepub], func() interface{} { - msgs := make([]MessagePoolEvt_Message, 0, len(outputMsgs)) - for _, m := range outputMsgs { - msgs = append(msgs, MessagePoolEvt_Message{Message: m.Message, CID: m.Cid()}) - } - return MessagePoolEvt{ - Action: "repub", - Messages: msgs, - Error: errout, - } - }) + if len(outputMsgs) > 0 { + journal.MaybeRecordEvent(mp.jrnl, mp.evtTypes[evtTypeMpoolRepub], func() interface{} { + msgs := make([]MessagePoolEvt_Message, 0, len(outputMsgs)) + for _, m := range outputMsgs { + msgs = append(msgs, MessagePoolEvt_Message{Message: m.Message, CID: m.Cid()}) + } + return MessagePoolEvt{ + Action: "repub", + Messages: msgs, + Error: errout, + } + }) + } case <-mp.pruneTrigger: if err := mp.pruneExcessMessages(); err != nil { From cc859d3948c0f4a1c01e4a3cd6ea2284afd66927 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Tue, 11 Aug 2020 16:30:54 +0100 Subject: [PATCH 020/885] journal: wdpost: record proofs_processed even if no eligible partitions. --- storage/wdpost_run.go | 44 ++++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 0ec4ac1f6..c448fb223 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -61,16 +61,34 @@ func (s *WindowPoStScheduler) doPost(ctx context.Context, deadline *miner.Deadli ctx, span := trace.StartSpan(ctx, "WindowPoStScheduler.doPost") defer span.End() + // recordEvent records a successful proofs_processed event in the + // journal, even if it was a noop (no partitions). + recordEvent := func(partitions []miner.PoStPartition, mcid cid.Cid) { + journal.MaybeRecordEvent(s.jrnl, s.wdPoStEvtType, func() interface{} { + return s.enrichWithTipset(WindowPoStEvt{ + State: "proofs_processed", + Deadline: s.activeDeadline, + Proofs: &WindowPoStEvt_Proofs{ + Partitions: partitions, + MessageCID: mcid, + }, + }) + }) + } + proof, err := s.runPost(ctx, *deadline, ts) switch err { case errNoPartitions: + recordEvent(nil, cid.Undef) return case nil: - if err := s.submitPost(ctx, proof); err != nil { + sm, err := s.submitPost(ctx, proof) + if err != nil { log.Errorf("submitPost failed: %+v", err) s.failPost(err, deadline) return } + recordEvent(proof.Partitions, sm.Cid()) default: log.Errorf("runPost failed: %+v", err) s.failPost(err, deadline) @@ -495,31 +513,15 @@ func (s *WindowPoStScheduler) sectorInfo(ctx context.Context, deadlineSectors *a return sbsi, nil } -func (s *WindowPoStScheduler) submitPost(ctx context.Context, proof *miner.SubmitWindowedPoStParams) error { +func (s *WindowPoStScheduler) submitPost(ctx context.Context, proof *miner.SubmitWindowedPoStParams) (*types.SignedMessage, error) { ctx, span := trace.StartSpan(ctx, "storage.commitPost") defer span.End() var sm *types.SignedMessage - defer journal.MaybeRecordEvent(s.jrnl, s.wdPoStEvtType, func() interface{} { - var mcid cid.Cid - if sm != nil { - mcid = sm.Cid() - } - - return s.enrichWithTipset(WindowPoStEvt{ - State: "proofs_processed", - Deadline: s.activeDeadline, - Proofs: &WindowPoStEvt_Proofs{ - Partitions: proof.Partitions, - MessageCID: mcid, - }, - }) - }) - enc, aerr := actors.SerializeParams(proof) if aerr != nil { - return xerrors.Errorf("could not serialize submit post parameters: %w", aerr) + return nil, xerrors.Errorf("could not serialize submit post parameters: %w", aerr) } msg := &types.Message{ @@ -534,7 +536,7 @@ func (s *WindowPoStScheduler) submitPost(ctx context.Context, proof *miner.Submi var err error sm, err = s.api.MpoolPushMessage(ctx, msg) if err != nil { - return xerrors.Errorf("pushing message to mpool: %w", err) + return nil, xerrors.Errorf("pushing message to mpool: %w", err) } log.Infof("Submitted window post: %s", sm.Cid()) @@ -553,5 +555,5 @@ func (s *WindowPoStScheduler) submitPost(ctx context.Context, proof *miner.Submi log.Errorf("Submitting window post %s failed: exit %d", sm.Cid(), rec.Receipt.ExitCode) }() - return nil + return sm, nil } From e4daadbfcf70c30af00d254bec1b09037a49a187 Mon Sep 17 00:00:00 2001 From: austinabell Date: Thu, 20 Aug 2020 17:54:59 -0400 Subject: [PATCH 021/885] Remove extra tipset load from checking beacon entries --- chain/store/store.go | 12 +++++++----- chain/sync.go | 12 +++++++----- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/chain/store/store.go b/chain/store/store.go index 9c424dc3b..6ecc30b66 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -1258,11 +1258,13 @@ func (cs *ChainStore) GetLatestBeaconEntry(ts *types.TipSet) (*types.BeaconEntry return nil, xerrors.Errorf("made it back to genesis block without finding beacon entry") } - next, err := cs.LoadTipSet(cur.Parents()) - if err != nil { - return nil, xerrors.Errorf("failed to load parents when searching back for latest beacon entry: %w", err) + if i != 19 { + next, err := cs.LoadTipSet(cur.Parents()) + if err != nil { + return nil, xerrors.Errorf("failed to load parents when searching back for latest beacon entry: %w", err) + } + cur = next } - cur = next } if os.Getenv("LOTUS_IGNORE_DRAND") == "_yes_" { @@ -1271,7 +1273,7 @@ func (cs *ChainStore) GetLatestBeaconEntry(ts *types.TipSet) (*types.BeaconEntry }, nil } - return nil, xerrors.Errorf("found NO beacon entries in the 20 blocks prior to given tipset") + return nil, xerrors.Errorf("found NO beacon entries in the 20 latest tipsets") } type chainRand struct { diff --git a/chain/sync.go b/chain/sync.go index b0ae185b0..31f57b402 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -1591,14 +1591,16 @@ func (syncer *Syncer) getLatestBeaconEntry(_ context.Context, ts *types.TipSet) return nil, xerrors.Errorf("made it back to genesis block without finding beacon entry") } - next, err := syncer.store.LoadTipSet(cur.Parents()) - if err != nil { - return nil, xerrors.Errorf("failed to load parents when searching back for latest beacon entry: %w", err) + if i != 19 { + next, err := syncer.store.LoadTipSet(cur.Parents()) + if err != nil { + return nil, xerrors.Errorf("failed to load parents when searching back for latest beacon entry: %w", err) + } + cur = next } - cur = next } - return nil, xerrors.Errorf("found NO beacon entries in the 20 blocks prior to given tipset") + return nil, xerrors.Errorf("found NO beacon entries in the 20 latest tipsets") } func (syncer *Syncer) IsEpochBeyondCurrMax(epoch abi.ChainEpoch) bool { From 4e1ef09751ab8d937c16d7012a977cad93d3c492 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Wed, 26 Aug 2020 16:09:37 +0100 Subject: [PATCH 022/885] make journal a global var. --- chain/gen/gen.go | 3 +- chain/gen/genesis/genesis.go | 3 +- chain/messagepool/messagepool.go | 15 +++-- chain/messagepool/messagepool_test.go | 18 +++--- chain/messagepool/selection_test.go | 11 ++-- chain/store/index_test.go | 12 ++-- chain/store/store.go | 8 +-- chain/store/store_test.go | 4 +- chain/validation/applier.go | 2 +- cmd/lotus-bench/import.go | 2 +- cmd/lotus-storage-miner/init.go | 7 ++- cmd/lotus/daemon.go | 7 +-- go.sum | 17 +++++ journal/fs.go | 28 ++++++--- journal/global.go | 9 +++ journal/nil.go | 2 +- journal/registry.go | 57 +++++++++++++++++ journal/types.go | 89 +++------------------------ markets/storageadapter/client.go | 15 ++--- markets/storageadapter/provider.go | 24 ++++---- miner/miner.go | 8 +-- node/builder.go | 10 ++- node/modules/chain.go | 9 ++- node/modules/storageminer.go | 14 ++--- storage/miner.go | 8 +-- storage/wdpost_run.go | 12 ++-- storage/wdpost_sched.go | 8 +-- 27 files changed, 202 insertions(+), 200 deletions(-) create mode 100644 journal/global.go create mode 100644 journal/registry.go diff --git a/chain/gen/gen.go b/chain/gen/gen.go index a58c34764..d4851a933 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -35,7 +35,6 @@ import ( "github.com/filecoin-project/lotus/chain/wallet" "github.com/filecoin-project/lotus/cmd/lotus-seed/seed" "github.com/filecoin-project/lotus/genesis" - "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/lib/blockstore" "github.com/filecoin-project/lotus/lib/sigs" "github.com/filecoin-project/lotus/node/repo" @@ -221,7 +220,7 @@ func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) { return nil, xerrors.Errorf("make genesis block failed: %w", err) } - cs := store.NewChainStore(bs, ds, sys, journal.NilJournal()) + cs := store.NewChainStore(bs, ds, sys) genfb := &types.FullBlock{Header: genb.Genesis} gents := store.NewFullTipSet([]*types.FullBlock{genfb}) diff --git a/chain/gen/genesis/genesis.go b/chain/gen/genesis/genesis.go index 16f8d5307..1731d780a 100644 --- a/chain/gen/genesis/genesis.go +++ b/chain/gen/genesis/genesis.go @@ -27,7 +27,6 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" "github.com/filecoin-project/lotus/genesis" - "github.com/filecoin-project/lotus/journal" bstore "github.com/filecoin-project/lotus/lib/blockstore" "github.com/filecoin-project/lotus/lib/sigs" ) @@ -398,7 +397,7 @@ func MakeGenesisBlock(ctx context.Context, bs bstore.Blockstore, sys vm.SyscallB } // temp chainstore - cs := store.NewChainStore(bs, datastore.NewMapDatastore(), sys, journal.NilJournal()) + cs := store.NewChainStore(bs, datastore.NewMapDatastore(), sys) // Verify PreSealed Data stateroot, err = VerifyPreSealedData(ctx, cs, stateroot, template, keyIDs) diff --git a/chain/messagepool/messagepool.go b/chain/messagepool/messagepool.go index 19856b9f7..ab23af856 100644 --- a/chain/messagepool/messagepool.go +++ b/chain/messagepool/messagepool.go @@ -234,7 +234,7 @@ func (mpp *mpoolProvider) ChainComputeBaseFee(ctx context.Context, ts *types.Tip return baseFee, nil } -func New(api Provider, ds dtypes.MetadataDS, netName dtypes.NetworkName, jrnl journal.Journal) (*MessagePool, error) { +func New(api Provider, ds dtypes.MetadataDS, netName dtypes.NetworkName) (*MessagePool, error) { cache, _ := lru.New2Q(build.BlsSignatureCacheSize) verifcache, _ := lru.New2Q(build.VerifSigCacheSize) @@ -263,11 +263,10 @@ func New(api Provider, ds dtypes.MetadataDS, netName dtypes.NetworkName, jrnl jo cfg: cfg, rbfNum: types.NewInt(uint64((cfg.ReplaceByFeeRatio - 1) * RbfDenom)), rbfDenom: types.NewInt(RbfDenom), - jrnl: jrnl, evtTypes: [...]journal.EventType{ - evtTypeMpoolAdd: jrnl.RegisterEventType("mpool", "add"), - evtTypeMpoolRemove: jrnl.RegisterEventType("mpool", "remove"), - evtTypeMpoolRepub: jrnl.RegisterEventType("mpool", "repub"), + evtTypeMpoolAdd: journal.J.RegisterEventType("mpool", "add"), + evtTypeMpoolRemove: journal.J.RegisterEventType("mpool", "remove"), + evtTypeMpoolRepub: journal.J.RegisterEventType("mpool", "repub"), }, } @@ -372,7 +371,7 @@ func (mp *MessagePool) runLoop() { } if len(outputMsgs) > 0 { - journal.MaybeRecordEvent(mp.jrnl, mp.evtTypes[evtTypeMpoolRepub], func() interface{} { + journal.J.RecordEvent(mp.evtTypes[evtTypeMpoolRepub], func() interface{} { msgs := make([]MessagePoolEvt_Message, 0, len(outputMsgs)) for _, m := range outputMsgs { msgs = append(msgs, MessagePoolEvt_Message{Message: m.Message, CID: m.Cid()}) @@ -584,7 +583,7 @@ func (mp *MessagePool) addLocked(m *types.SignedMessage) error { Message: m, }, localUpdates) - journal.MaybeRecordEvent(mp.jrnl, mp.evtTypes[evtTypeMpoolAdd], func() interface{} { + journal.J.RecordEvent(mp.evtTypes[evtTypeMpoolAdd], func() interface{} { return MessagePoolEvt{ Action: "add", Messages: []MessagePoolEvt_Message{{Message: m.Message, CID: m.Cid()}}, @@ -732,7 +731,7 @@ func (mp *MessagePool) remove(from address.Address, nonce uint64) { Message: m, }, localUpdates) - journal.MaybeRecordEvent(mp.jrnl, mp.evtTypes[evtTypeMpoolRemove], func() interface{} { + journal.J.RecordEvent(mp.evtTypes[evtTypeMpoolRemove], func() interface{} { return MessagePoolEvt{ Action: "remove", Messages: []MessagePoolEvt_Message{{Message: m.Message, CID: m.Cid()}}} diff --git a/chain/messagepool/messagepool_test.go b/chain/messagepool/messagepool_test.go index 06541d7e5..a16f61c48 100644 --- a/chain/messagepool/messagepool_test.go +++ b/chain/messagepool/messagepool_test.go @@ -6,17 +6,17 @@ import ( "testing" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/chain/types/mock" - "github.com/filecoin-project/lotus/chain/wallet" - "github.com/filecoin-project/lotus/journal" - _ "github.com/filecoin-project/lotus/lib/sigs/bls" - _ "github.com/filecoin-project/lotus/lib/sigs/secp" "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/crypto" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" logging "github.com/ipfs/go-log/v2" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/types/mock" + "github.com/filecoin-project/lotus/chain/wallet" + _ "github.com/filecoin-project/lotus/lib/sigs/bls" + _ "github.com/filecoin-project/lotus/lib/sigs/secp" ) func init() { @@ -174,7 +174,7 @@ func TestMessagePool(t *testing.T) { ds := datastore.NewMapDatastore() - mp, err := New(tma, ds, "mptest", journal.NilJournal()) + mp, err := New(tma, ds, "mptest") if err != nil { t.Fatal(err) } @@ -215,7 +215,7 @@ func TestRevertMessages(t *testing.T) { ds := datastore.NewMapDatastore() - mp, err := New(tma, ds, "mptest", journal.NilJournal()) + mp, err := New(tma, ds, "mptest") if err != nil { t.Fatal(err) } @@ -271,7 +271,7 @@ func TestPruningSimple(t *testing.T) { ds := datastore.NewMapDatastore() - mp, err := New(tma, ds, "mptest", journal.NilJournal()) + mp, err := New(tma, ds, "mptest") if err != nil { t.Fatal(err) } diff --git a/chain/messagepool/selection_test.go b/chain/messagepool/selection_test.go index 91337d63c..af3f88807 100644 --- a/chain/messagepool/selection_test.go +++ b/chain/messagepool/selection_test.go @@ -5,16 +5,15 @@ import ( "testing" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/specs-actors/actors/builtin" + "github.com/filecoin-project/specs-actors/actors/crypto" + "github.com/ipfs/go-datastore" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/messagepool/gasguess" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/types/mock" "github.com/filecoin-project/lotus/chain/wallet" - "github.com/filecoin-project/lotus/journal" - - "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/crypto" - "github.com/ipfs/go-datastore" _ "github.com/filecoin-project/lotus/lib/sigs/bls" _ "github.com/filecoin-project/lotus/lib/sigs/secp" @@ -44,7 +43,7 @@ func makeTestMessage(w *wallet.Wallet, from, to address.Address, nonce uint64, g func makeTestMpool() (*MessagePool, *testMpoolAPI) { tma := newTestMpoolAPI() ds := datastore.NewMapDatastore() - mp, err := New(tma, ds, "test", journal.NilJournal()) + mp, err := New(tma, ds, "test") if err != nil { panic(err) } diff --git a/chain/store/index_test.go b/chain/store/index_test.go index 2a0fdc2d1..38cad96dd 100644 --- a/chain/store/index_test.go +++ b/chain/store/index_test.go @@ -5,15 +5,15 @@ import ( "context" "testing" + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/ipfs/go-datastore" + syncds "github.com/ipfs/go-datastore/sync" + "github.com/stretchr/testify/assert" + "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types/mock" - "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/lib/blockstore" - "github.com/filecoin-project/specs-actors/actors/abi" - datastore "github.com/ipfs/go-datastore" - syncds "github.com/ipfs/go-datastore/sync" - "github.com/stretchr/testify/assert" ) func TestIndexSeeks(t *testing.T) { @@ -32,7 +32,7 @@ func TestIndexSeeks(t *testing.T) { ctx := context.TODO() nbs := blockstore.NewTemporarySync() - cs := store.NewChainStore(nbs, syncds.MutexWrap(datastore.NewMapDatastore()), nil, journal.NilJournal()) + cs := store.NewChainStore(nbs, syncds.MutexWrap(datastore.NewMapDatastore()), nil) _, err = cs.Import(bytes.NewReader(gencar)) if err != nil { diff --git a/chain/store/store.go b/chain/store/store.go index 198f18756..053da258b 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -96,11 +96,10 @@ type ChainStore struct { vmcalls vm.SyscallBuilder - journal journal.Journal evtTypes [1]journal.EventType } -func NewChainStore(bs bstore.Blockstore, ds dstore.Batching, vmcalls vm.SyscallBuilder, jrnl journal.Journal) *ChainStore { +func NewChainStore(bs bstore.Blockstore, ds dstore.Batching, vmcalls vm.SyscallBuilder) *ChainStore { c, _ := lru.NewARC(2048) tsc, _ := lru.NewARC(4096) cs := &ChainStore{ @@ -111,11 +110,10 @@ func NewChainStore(bs bstore.Blockstore, ds dstore.Batching, vmcalls vm.SyscallB mmCache: c, tsCache: tsc, vmcalls: vmcalls, - journal: jrnl, } cs.evtTypes = [1]journal.EventType{ - evtTypeHeadChange: jrnl.RegisterEventType("sync", "head_change"), + evtTypeHeadChange: journal.J.RegisterEventType("sync", "head_change"), } ci := NewChainIndex(cs.LoadTipSet) @@ -344,7 +342,7 @@ func (cs *ChainStore) reorgWorker(ctx context.Context, initialNotifees []ReorgNo continue } - journal.MaybeRecordEvent(cs.journal, cs.evtTypes[evtTypeHeadChange], func() interface{} { + journal.J.RecordEvent(cs.evtTypes[evtTypeHeadChange], func() interface{} { return HeadChangeEvt{ From: r.old.Key(), FromHeight: r.old.Height(), diff --git a/chain/store/store_test.go b/chain/store/store_test.go index 56a78330a..e52a77570 100644 --- a/chain/store/store_test.go +++ b/chain/store/store_test.go @@ -68,7 +68,7 @@ func BenchmarkGetRandomness(b *testing.B) { bs := blockstore.NewBlockstore(bds) - cs := store.NewChainStore(bs, mds, nil, journal.NilJournal()) + cs := store.NewChainStore(bs, mds, nil) b.ResetTimer() @@ -102,7 +102,7 @@ func TestChainExportImport(t *testing.T) { } nbs := blockstore.NewTemporary() - cs := store.NewChainStore(nbs, datastore.NewMapDatastore(), nil, journal.NilJournal()) + cs := store.NewChainStore(nbs, datastore.NewMapDatastore(), nil) root, err := cs.Import(buf) if err != nil { diff --git a/chain/validation/applier.go b/chain/validation/applier.go index 9b688b8f0..c67a91c1a 100644 --- a/chain/validation/applier.go +++ b/chain/validation/applier.go @@ -69,7 +69,7 @@ func (a *Applier) ApplySignedMessage(epoch abi.ChainEpoch, msg *vtypes.SignedMes } func (a *Applier) ApplyTipSetMessages(epoch abi.ChainEpoch, blocks []vtypes.BlockMessagesInfo, rnd vstate.RandomnessSource) (vtypes.ApplyTipSetResult, error) { - cs := store.NewChainStore(a.stateWrapper.bs, a.stateWrapper.ds, a.syscalls, journal.NilJournal()) + cs := store.NewChainStore(a.stateWrapper.bs, a.stateWrapper.ds, a.syscalls) sm := stmgr.NewStateManager(cs) var bms []stmgr.BlockMessages diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index 03035880d..c7f687eb7 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -87,7 +87,7 @@ var importBenchCmd = &cli.Command{ } bs = cbs ds := datastore.NewMapDatastore() - cs := store.NewChainStore(bs, ds, vm.Syscalls(ffiwrapper.ProofVerifier), journal.NilJournal()) + cs := store.NewChainStore(bs, ds, vm.Syscalls(ffiwrapper.ProofVerifier)) stm := stmgr.NewStateManager(cs) prof, err := os.Create("import-bench.prof") diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index 1714bd901..fc1e417ce 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -448,12 +448,13 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api lapi.FullNode, return err } - jrnl, err := journal.OpenFSJournal(lr, journal.DefaultDisabledEvents) - if err != nil { + if jrnl, err := journal.OpenFSJournal(lr, journal.DefaultDisabledEvents); err == nil { + journal.J = jrnl + } else { return fmt.Errorf("failed to open filesystem journal: %w", err) } - m := miner.NewMiner(api, epp, a, slashfilter.New(mds), jrnl) + m := miner.NewMiner(api, epp, a, slashfilter.New(mds)) { if err := m.Start(ctx); err != nil { return xerrors.Errorf("failed to start up genesis miner: %w", err) diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index dc5168d69..52e8976c7 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -12,9 +12,6 @@ import ( "runtime/pprof" "strings" - "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/journal" - paramfetch "github.com/filecoin-project/go-paramfetch" "github.com/mitchellh/go-homedir" "github.com/multiformats/go-multiaddr" @@ -25,6 +22,8 @@ import ( "go.opencensus.io/tag" "golang.org/x/xerrors" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/stmgr" @@ -338,7 +337,7 @@ func ImportChain(r repo.Repo, fname string) error { bs := blockstore.NewBlockstore(ds) - cst := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier), journal.NilJournal()) + cst := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier)) log.Info("importing chain from file...") ts, err := cst.Import(fi) diff --git a/go.sum b/go.sum index ca9f0df33..37703fd02 100644 --- a/go.sum +++ b/go.sum @@ -1230,6 +1230,7 @@ github.com/prometheus/procfs v0.1.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/raulk/clock v1.1.0 h1:dpb29+UKMbLqiU/jqIJptgLR1nn23HLgMY0sTCDza5Y= github.com/raulk/clock v1.1.0/go.mod h1:3MpVxdZ/ODBQDxbN+kzshf5OSZwPjtMDx6BBXBmOeY0= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -1327,6 +1328,7 @@ github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFd github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e/go.mod h1:XDKHRm5ThF8YJjx001LtgelzsoaEcvnA7lVWz9EeX3g= +github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-client-go v2.23.1+incompatible h1:uArBYHQR0HqLFFAypI7RsWTzPSj/bDpmZZuQjMLSg1A= @@ -1395,11 +1397,14 @@ github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee/go.mod h1: github.com/whyrusleeping/yamux v1.1.5/go.mod h1:E8LnQQ8HKx5KD29HZFUwM1PxCOdPRzGwur1mcYhXcD8= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xlab/c-for-go v0.0.0-20200718154222-87b0065af829/go.mod h1:h/1PEBwj7Ym/8kOuMWvO2ujZ6Lt+TMbySEXNhjjR87I= +github.com/xlab/pkgconfig v0.0.0-20170226114623-cea12a0fd245/go.mod h1:C+diUUz7pxhNY6KAoLgrTYARGWnt82zWTylZlxT92vk= github.com/xorcare/golden v0.6.0/go.mod h1:7T39/ZMvaSEZlBPoYfVFmsBLmUl3uz9IuzWj/U6FtvQ= github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542 h1:oWgZJmC1DorFZDpfMfWg7xk29yEOZiXmo/wZl+utTI8= github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542/go.mod h1:7T39/ZMvaSEZlBPoYfVFmsBLmUl3uz9IuzWj/U6FtvQ= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.dedis.ch/fixbuf v1.0.3 h1:hGcV9Cd/znUxlusJ64eAlExS+5cJDIyTyEG+otu5wQs= go.dedis.ch/fixbuf v1.0.3/go.mod h1:yzJMt34Wa5xD37V5RTdmp38cz3QhMagdGoem9anUalw= go.dedis.ch/kyber/v3 v3.0.4/go.mod h1:OzvaEnPvKlyrWyp3kGXlFdp7ap1VC6RkZDTaPikqhsQ= @@ -1482,6 +1487,8 @@ golang.org/x/crypto v0.0.0-20200427165652-729f1e841bcc/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9 h1:vEg9joUBmeBcK9iSJftGNf3coIG4HqZElCPehJsfAYM= golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1515,6 +1522,7 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180524181706-dfa909b99c79/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1557,6 +1565,8 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200519113804-d87ec0cfa476/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM= golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1695,6 +1705,7 @@ golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200318150045-ba25ddc85566/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4 h1:kDtqNkeBrZb8B+atrj50B5XLHpzXXqcCdZPP/ApQ5NY= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200711155855-7342f9734a7d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= @@ -1808,6 +1819,7 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= @@ -1825,6 +1837,11 @@ howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqp howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= launchpad.net/gocheck v0.0.0-20140225173054-000000000087 h1:Izowp2XBH6Ya6rv+hqbceQyw/gSGoXfH/UPoTGduL54= launchpad.net/gocheck v0.0.0-20140225173054-000000000087/go.mod h1:hj7XX3B/0A+80Vse0e+BUHsHMTEhd0O4cpUHr/e/BUM= +modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= +modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= +modernc.org/mathutil v1.1.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/strutil v1.1.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= +modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/journal/fs.go b/journal/fs.go index 73a2fff43..7d5203556 100644 --- a/journal/fs.go +++ b/journal/fs.go @@ -16,7 +16,7 @@ import ( // fsJournal is a basic journal backed by files on a filesystem. type fsJournal struct { - EventTypeFactory + EventTypeRegistry dir string sizeLimit int64 @@ -40,12 +40,12 @@ func OpenFSJournal(lr repo.LockedRepo, disabled DisabledEvents) (Journal, error) } f := &fsJournal{ - EventTypeFactory: NewEventTypeFactory(disabled), - dir: dir, - sizeLimit: 1 << 30, - incoming: make(chan *Event, 32), - closing: make(chan struct{}), - closed: make(chan struct{}), + EventTypeRegistry: NewEventTypeRegistry(disabled), + dir: dir, + sizeLimit: 1 << 30, + incoming: make(chan *Event, 32), + closing: make(chan struct{}), + closed: make(chan struct{}), } if err := f.rollJournalFile(); err != nil { @@ -57,11 +57,21 @@ func OpenFSJournal(lr repo.LockedRepo, disabled DisabledEvents) (Journal, error) return f, nil } -func (f *fsJournal) RecordEvent(evtType EventType, obj interface{}) { +func (f *fsJournal) RecordEvent(evtType EventType, supplier func() interface{}) { + defer func() { + if r := recover(); r != nil { + log.Warnf("recovered from panic while recording journal event; type=%s, err=%v", evtType, r) + } + }() + + if !evtType.Enabled() { + return + } + je := &Event{ EventType: evtType, Timestamp: build.Clock.Now(), - Data: obj, + Data: supplier(), } select { case f.incoming <- je: diff --git a/journal/global.go b/journal/global.go new file mode 100644 index 000000000..b4d0e0a1b --- /dev/null +++ b/journal/global.go @@ -0,0 +1,9 @@ +package journal + +var ( + // J is a globally accessible Journal. It starts being NilJournal, and early + // during the Lotus initialization routine, it is reset to whichever Journal + // is configured (by default, the filesystem journal). Components can safely + // record in the journal by calling: journal.J.RecordEvent(...). + J Journal = NilJournal() // nolint +) diff --git a/journal/nil.go b/journal/nil.go index 5d0c78b05..fa72fa373 100644 --- a/journal/nil.go +++ b/journal/nil.go @@ -11,6 +11,6 @@ func NilJournal() Journal { func (n *nilJournal) RegisterEventType(_, _ string) EventType { return EventType{} } -func (n *nilJournal) RecordEvent(_ EventType, _ interface{}) {} +func (n *nilJournal) RecordEvent(_ EventType, _ func() interface{}) {} func (n *nilJournal) Close() error { return nil } diff --git a/journal/registry.go b/journal/registry.go new file mode 100644 index 000000000..6ab5b5fb1 --- /dev/null +++ b/journal/registry.go @@ -0,0 +1,57 @@ +package journal + +import "sync" + +// EventTypeRegistry is a component that constructs tracked EventType tokens, +// for usage with a Journal. +type EventTypeRegistry interface { + + // RegisterEventType introduces a new event type to a journal, and + // returns an EventType token that components can later use to check whether + // journalling for that type is enabled/suppressed, and to tag journal + // entries appropriately. + RegisterEventType(system, event string) EventType +} + +// eventTypeRegistry is an embeddable mixin that takes care of tracking disabled +// event types, and returning initialized/safe EventTypes when requested. +type eventTypeRegistry struct { + sync.Mutex + + m map[string]EventType +} + +var _ EventTypeRegistry = (*eventTypeRegistry)(nil) + +func NewEventTypeRegistry(disabled DisabledEvents) EventTypeRegistry { + ret := &eventTypeRegistry{ + m: make(map[string]EventType, len(disabled)+32), // + extra capacity. + } + + for _, et := range disabled { + et.enabled, et.safe = false, true + ret.m[et.System+":"+et.Event] = et + } + + return ret +} + +func (d *eventTypeRegistry) RegisterEventType(system, event string) EventType { + d.Lock() + defer d.Unlock() + + key := system + ":" + event + if et, ok := d.m[key]; ok { + return et + } + + et := EventType{ + System: system, + Event: event, + enabled: true, + safe: true, + } + + d.m[key] = et + return et +} diff --git a/journal/types.go b/journal/types.go index 6183e6cfa..c81fb8c89 100644 --- a/journal/types.go +++ b/journal/types.go @@ -1,7 +1,6 @@ package journal import ( - "sync" "time" logging "github.com/ipfs/go-log" @@ -49,16 +48,6 @@ func (et EventType) Enabled() bool { return et.safe && et.enabled } -// EventTypeFactory is a component that constructs tracked EventType tokens, -// for usage with a Journal. -type EventTypeFactory interface { - // RegisterEventType introduces a new event type to a journal, and - // returns an EventType token that components can later use to check whether - // journalling for that type is enabled/suppressed, and to tag journal - // entries appropriately. - RegisterEventType(system, event string) EventType -} - // Journal represents an audit trail of system actions. // // Every entry is tagged with a timestamp, a system name, and an event name. @@ -68,11 +57,14 @@ type EventTypeFactory interface { // For cleanliness and type safety, we recommend to use typed events. See the // *Evt struct types in this package for more info. type Journal interface { - EventTypeFactory + EventTypeRegistry - // RecordEvent records this event to the journal. See godocs on the Journal type - // for more info. - RecordEvent(evtType EventType, data interface{}) + // RecordEvent records this event to the journal, if and only if the + // EventType is enabled. If so, it calls the supplier function to obtain + // the payload to record. + // + // Implementations MUST recover from panics raised by the supplier function. + RecordEvent(evtType EventType, supplier func() interface{}) // Close closes this journal for further writing. Close() error @@ -87,70 +79,3 @@ type Event struct { Timestamp time.Time Data interface{} } - -// MaybeRecordEvent is a convenience function that evaluates if the EventType is -// enabled, and if so, it calls the supplier to create the event and -// subsequently journal.RecordEvent on the provided journal to record it. -// -// It also recovers from panics raised when calling the supplier function. -// -// This is safe to call with a nil Journal, either because the value is nil, -// or because a journal obtained through NilJournal() is in use. -func MaybeRecordEvent(journal Journal, evtType EventType, supplier func() interface{}) { - defer func() { - if r := recover(); r != nil { - log.Warnf("recovered from panic while recording journal event; type=%s, err=%v", evtType, r) - } - }() - - if journal == nil || journal == nilj { - return - } - if !evtType.Enabled() { - return - } - journal.RecordEvent(evtType, supplier()) -} - -// eventTypeFactory is an embeddable mixin that takes care of tracking disabled -// event types, and returning initialized/safe EventTypes when requested. -type eventTypeFactory struct { - sync.Mutex - - m map[string]EventType -} - -var _ EventTypeFactory = (*eventTypeFactory)(nil) - -func NewEventTypeFactory(disabled DisabledEvents) EventTypeFactory { - ret := &eventTypeFactory{ - m: make(map[string]EventType, len(disabled)+32), // + extra capacity. - } - - for _, et := range disabled { - et.enabled, et.safe = false, true - ret.m[et.System+":"+et.Event] = et - } - - return ret -} - -func (d *eventTypeFactory) RegisterEventType(system, event string) EventType { - d.Lock() - defer d.Unlock() - - key := system + ":" + event - if et, ok := d.m[key]; ok { - return et - } - - et := EventType{ - System: system, - Event: event, - enabled: true, - safe: true, - } - - d.m[key] = et - return et -} diff --git a/markets/storageadapter/client.go b/markets/storageadapter/client.go index 9186b94c0..761a4aa74 100644 --- a/markets/storageadapter/client.go +++ b/markets/storageadapter/client.go @@ -43,8 +43,6 @@ type ClientNodeAdapter struct { fm *market.FundMgr ev *events.Events - jrnl journal.Journal - // index 0 is unused, as it corresponds to evtTypeDealAccepted, a provider-only event. evtTypes [4]journal.EventType } @@ -54,7 +52,7 @@ type clientApi struct { full.StateAPI } -func NewClientNodeAdapter(state full.StateAPI, chain full.ChainAPI, mpool full.MpoolAPI, sm *stmgr.StateManager, cs *store.ChainStore, fm *market.FundMgr, jrnl journal.Journal) storagemarket.StorageClientNode { +func NewClientNodeAdapter(state full.StateAPI, chain full.ChainAPI, mpool full.MpoolAPI, sm *stmgr.StateManager, cs *store.ChainStore, fm *market.FundMgr) storagemarket.StorageClientNode { return &ClientNodeAdapter{ StateAPI: state, ChainAPI: chain, @@ -65,11 +63,10 @@ func NewClientNodeAdapter(state full.StateAPI, chain full.ChainAPI, mpool full.M fm: fm, ev: events.NewEvents(context.TODO(), &clientApi{chain, state}), - jrnl: jrnl, evtTypes: [...]journal.EventType{ - evtTypeDealSectorCommitted: jrnl.RegisterEventType("markets:storage:client", "deal_sector_committed"), - evtTypeDealExpired: jrnl.RegisterEventType("markets:storage:client", "deal_expired"), - evtTypeDealSlashed: jrnl.RegisterEventType("markets:storage:client", "deal_slashed"), + evtTypeDealSectorCommitted: journal.J.RegisterEventType("markets:storage:client", "deal_sector_committed"), + evtTypeDealExpired: journal.J.RegisterEventType("markets:storage:client", "deal_expired"), + evtTypeDealSlashed: journal.J.RegisterEventType("markets:storage:client", "deal_slashed"), }, } } @@ -236,7 +233,7 @@ func (c *ClientNodeAdapter) ValidatePublishedDeal(ctx context.Context, deal stor } dealID := res.IDs[dealIdx] - journal.MaybeRecordEvent(c.jrnl, c.evtTypes[evtTypeDealAccepted], func() interface{} { + journal.J.RecordEvent(c.evtTypes[evtTypeDealAccepted], func() interface{} { deal := deal // copy and strip fields we don't want to log to the journal deal.ClientSignature = crypto.Signature{} return ClientDealAcceptedEvt{ID: dealID, Deal: deal, Height: c.cs.GetHeaviestTipSet().Height()} @@ -294,7 +291,7 @@ func (c *ClientNodeAdapter) OnDealSectorCommitted(ctx context.Context, provider log.Infof("Storage deal %d activated at epoch %d", dealId, sd.State.SectorStartEpoch) - journal.MaybeRecordEvent(c.jrnl, c.evtTypes[evtTypeDealSectorCommitted], func() interface{} { + journal.J.RecordEvent(c.evtTypes[evtTypeDealSectorCommitted], func() interface{} { return ClientDealSectorCommittedEvt{ID: dealId, State: sd.State, Height: curH} }) diff --git a/markets/storageadapter/provider.go b/markets/storageadapter/provider.go index 8e31939df..d772a36a8 100644 --- a/markets/storageadapter/provider.go +++ b/markets/storageadapter/provider.go @@ -47,23 +47,21 @@ type ProviderNodeAdapter struct { secb *sectorblocks.SectorBlocks ev *events.Events - jrnl journal.Journal evtTypes [4]journal.EventType } -func NewProviderNodeAdapter(dag dtypes.StagingDAG, secb *sectorblocks.SectorBlocks, full api.FullNode, jrnl journal.Journal) storagemarket.StorageProviderNode { +func NewProviderNodeAdapter(dag dtypes.StagingDAG, secb *sectorblocks.SectorBlocks, full api.FullNode) storagemarket.StorageProviderNode { return &ProviderNodeAdapter{ FullNode: full, dag: dag, secb: secb, ev: events.NewEvents(context.TODO(), full), - jrnl: jrnl, evtTypes: [...]journal.EventType{ - evtTypeDealAccepted: jrnl.RegisterEventType("markets:storage:provider", "deal_complete"), - evtTypeDealSectorCommitted: jrnl.RegisterEventType("markets:storage:provider", "deal_sector_committed"), - evtTypeDealExpired: jrnl.RegisterEventType("markets:storage:provider", "deal_expired"), - evtTypeDealSlashed: jrnl.RegisterEventType("markets:storage:provider", "deal_slashed"), + evtTypeDealAccepted: journal.J.RegisterEventType("markets:storage:provider", "deal_complete"), + evtTypeDealSectorCommitted: journal.J.RegisterEventType("markets:storage:provider", "deal_sector_committed"), + evtTypeDealExpired: journal.J.RegisterEventType("markets:storage:provider", "deal_expired"), + evtTypeDealSlashed: journal.J.RegisterEventType("markets:storage:provider", "deal_slashed"), }, } } @@ -113,7 +111,7 @@ func (n *ProviderNodeAdapter) OnDealComplete(ctx context.Context, deal storagema log.Warnf("New Deal: deal %d", deal.DealID) - journal.MaybeRecordEvent(n.jrnl, n.evtTypes[evtTypeDealAccepted], func() interface{} { + journal.J.RecordEvent(n.evtTypes[evtTypeDealAccepted], func() interface{} { deal := deal // copy and strip fields we don't want to log to the journal deal.ClientSignature = crypto.Signature{} return MinerDealAcceptedEvt{ID: deal.DealID, Deal: deal} @@ -297,7 +295,7 @@ func (n *ProviderNodeAdapter) OnDealSectorCommitted(ctx context.Context, provide log.Infof("Storage deal %d activated at epoch %d", dealID, sd.State.SectorStartEpoch) - journal.MaybeRecordEvent(n.jrnl, n.evtTypes[evtTypeDealSectorCommitted], func() interface{} { + journal.J.RecordEvent(n.evtTypes[evtTypeDealSectorCommitted], func() interface{} { return MinerDealSectorCommittedEvt{ID: dealID, State: sd.State, Height: curH} }) @@ -407,7 +405,7 @@ func (n *ProviderNodeAdapter) OnDealExpiredOrSlashed(ctx context.Context, dealID // Check if the deal has already expired if sd.Proposal.EndEpoch <= height { onDealExpired(nil) - journal.MaybeRecordEvent(n.jrnl, n.evtTypes[evtTypeDealExpired], func() interface{} { + journal.J.RecordEvent(n.evtTypes[evtTypeDealExpired], func() interface{} { return MinerDealExpiredEvt{ID: dealID, State: sd.State, Height: height} }) return true, false, nil @@ -416,7 +414,7 @@ func (n *ProviderNodeAdapter) OnDealExpiredOrSlashed(ctx context.Context, dealID // If there is no deal assume it's already been slashed if sd.State.SectorStartEpoch < 0 { onDealSlashed(height, nil) - journal.MaybeRecordEvent(n.jrnl, n.evtTypes[evtTypeDealSlashed], func() interface{} { + journal.J.RecordEvent(n.evtTypes[evtTypeDealSlashed], func() interface{} { return MinerDealSlashedEvt{ID: dealID, State: sd.State, Height: height} }) return true, false, nil @@ -435,7 +433,7 @@ func (n *ProviderNodeAdapter) OnDealExpiredOrSlashed(ctx context.Context, dealID // Check if the deal has already expired if sd.Proposal.EndEpoch <= height { onDealExpired(nil) - journal.MaybeRecordEvent(n.jrnl, n.evtTypes[evtTypeDealExpired], func() interface{} { + journal.J.RecordEvent(n.evtTypes[evtTypeDealExpired], func() interface{} { return MinerDealExpiredEvt{ID: dealID, State: sd.State, Height: height} }) return false, nil @@ -461,7 +459,7 @@ func (n *ProviderNodeAdapter) OnDealExpiredOrSlashed(ctx context.Context, dealID // Deal was slashed if deal.To == nil { onDealSlashed(height, nil) - journal.MaybeRecordEvent(n.jrnl, n.evtTypes[evtTypeDealSlashed], func() interface{} { + journal.J.RecordEvent(n.evtTypes[evtTypeDealSlashed], func() interface{} { return MinerDealSlashedEvt{ID: dealID, State: sd.State, Height: height} }) return false, nil diff --git a/miner/miner.go b/miner/miner.go index 091919b6c..bf1874f53 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -47,7 +47,7 @@ func randTimeOffset(width time.Duration) time.Duration { return val - (width / 2) } -func NewMiner(api api.FullNode, epp gen.WinningPoStProver, addr address.Address, sf *slashfilter.SlashFilter, jrnl journal.Journal) *Miner { +func NewMiner(api api.FullNode, epp gen.WinningPoStProver, addr address.Address, sf *slashfilter.SlashFilter) *Miner { arc, err := lru.NewARC(10000) if err != nil { panic(err) @@ -71,9 +71,8 @@ func NewMiner(api api.FullNode, epp gen.WinningPoStProver, addr address.Address, sf: sf, minedBlockHeights: arc, - jrnl: jrnl, evtTypes: [...]journal.EventType{ - evtTypeBlockMined: jrnl.RegisterEventType("miner", "block_mined"), + evtTypeBlockMined: journal.J.RegisterEventType("miner", "block_mined"), }, } } @@ -95,7 +94,6 @@ type Miner struct { sf *slashfilter.SlashFilter minedBlockHeights *lru.ARCCache - jrnl journal.Journal evtTypes [1]journal.EventType } @@ -225,7 +223,7 @@ func (m *Miner) mine(ctx context.Context) { onDone(b != nil, nil) if b != nil { - journal.MaybeRecordEvent(m.jrnl, m.evtTypes[evtTypeBlockMined], func() interface{} { + journal.J.RecordEvent(m.evtTypes[evtTypeBlockMined], func() interface{} { return map[string]interface{}{ "parents": base.TipSet.Cids(), "nulls": base.NullRounds, diff --git a/node/builder.go b/node/builder.go index 580aec3df..000099a43 100644 --- a/node/builder.go +++ b/node/builder.go @@ -152,14 +152,18 @@ type Settings struct { func defaults() []Option { return []Option{ + // global system journal. + Override(new(journal.DisabledEvents), journal.DefaultDisabledEvents), + Override(new(journal.Journal), modules.OpenFilesystemJournal), + Override(InitJournalKey, func(j journal.Journal) { + journal.J = j // eagerly sets the global journal through fx.Invoke. + }), + Override(new(helpers.MetricsCtx), context.Background), Override(new(record.Validator), modules.RecordValidator), Override(new(dtypes.Bootstrapper), dtypes.Bootstrapper(false)), Override(new(dtypes.ShutdownChan), make(chan struct{})), - Override(new(journal.Journal), modules.OpenFilesystemJournal), - Override(new(journal.DisabledEvents), journal.DefaultDisabledEvents), - Override(InitJournalKey, func(j journal.Journal) { /* forces the creation of the journal at startup */ }), // Filecoin modules } diff --git a/node/modules/chain.go b/node/modules/chain.go index 7ad4786ca..904c9f23b 100644 --- a/node/modules/chain.go +++ b/node/modules/chain.go @@ -26,7 +26,6 @@ import ( "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" - "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/lib/blockstore" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/helpers" @@ -48,9 +47,9 @@ func ChainExchange(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, rt return exch } -func MessagePool(lc fx.Lifecycle, sm *stmgr.StateManager, ps *pubsub.PubSub, ds dtypes.MetadataDS, nn dtypes.NetworkName, jrnl journal.Journal) (*messagepool.MessagePool, error) { +func MessagePool(lc fx.Lifecycle, sm *stmgr.StateManager, ps *pubsub.PubSub, ds dtypes.MetadataDS, nn dtypes.NetworkName) (*messagepool.MessagePool, error) { mpp := messagepool.NewProvider(sm, ps) - mp, err := messagepool.New(mpp, ds, nn, jrnl) + mp, err := messagepool.New(mpp, ds, nn) if err != nil { return nil, xerrors.Errorf("constructing mpool: %w", err) } @@ -85,8 +84,8 @@ func ChainBlockservice(bs dtypes.ChainBlockstore, rem dtypes.ChainExchange) dtyp return blockservice.New(bs, rem) } -func ChainStore(lc fx.Lifecycle, bs dtypes.ChainBlockstore, ds dtypes.MetadataDS, syscalls vm.SyscallBuilder, journal journal.Journal) *store.ChainStore { - chain := store.NewChainStore(bs, ds, syscalls, journal) +func ChainStore(lc fx.Lifecycle, bs dtypes.ChainBlockstore, ds dtypes.MetadataDS, syscalls vm.SyscallBuilder) *store.ChainStore { + chain := store.NewChainStore(bs, ds, syscalls) if err := chain.Load(); err != nil { log.Warnf("loading chain state from disk: %s", err) diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index fc7458f8f..85ffeb9cc 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -44,11 +44,11 @@ import ( paramfetch "github.com/filecoin-project/go-paramfetch" "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/go-storedcounter" - "github.com/filecoin-project/lotus/journal" + "github.com/filecoin-project/specs-actors/actors/abi" + sectorstorage "github.com/filecoin-project/sector-storage" "github.com/filecoin-project/sector-storage/ffiwrapper" "github.com/filecoin-project/sector-storage/stores" - "github.com/filecoin-project/specs-actors/actors/abi" sealing "github.com/filecoin-project/storage-fsm" lapi "github.com/filecoin-project/lotus/api" @@ -155,7 +155,6 @@ type StorageMinerParams struct { SectorIDCounter sealing.SectorIDCounter Verifier ffiwrapper.Verifier GetSealingDelayFn dtypes.GetSealingDelayFunc - Journal journal.Journal } func StorageMiner(params StorageMinerParams) (*storage.Miner, error) { @@ -169,7 +168,6 @@ func StorageMiner(params StorageMinerParams) (*storage.Miner, error) { sc = params.SectorIDCounter verif = params.Verifier gsd = params.GetSealingDelayFn - jrnl = params.Journal ) maddr, err := minerAddrFromDS(ds) if err != nil { @@ -188,12 +186,12 @@ func StorageMiner(params StorageMinerParams) (*storage.Miner, error) { return nil, err } - fps, err := storage.NewWindowedPoStScheduler(api, sealer, sealer, maddr, worker, jrnl) + fps, err := storage.NewWindowedPoStScheduler(api, sealer, sealer, maddr, worker) if err != nil { return nil, err } - sm, err := storage.NewMiner(api, maddr, worker, h, ds, sealer, sc, verif, gsd, jrnl) + sm, err := storage.NewMiner(api, maddr, worker, h, ds, sealer, sc, verif, gsd) if err != nil { return nil, err } @@ -340,13 +338,13 @@ func StagingGraphsync(mctx helpers.MetricsCtx, lc fx.Lifecycle, ibs dtypes.Stagi return gs } -func SetupBlockProducer(lc fx.Lifecycle, ds dtypes.MetadataDS, api lapi.FullNode, epp gen.WinningPoStProver, sf *slashfilter.SlashFilter, jrnl journal.Journal) (*miner.Miner, error) { +func SetupBlockProducer(lc fx.Lifecycle, ds dtypes.MetadataDS, api lapi.FullNode, epp gen.WinningPoStProver, sf *slashfilter.SlashFilter) (*miner.Miner, error) { minerAddr, err := minerAddrFromDS(ds) if err != nil { return nil, err } - m := miner.NewMiner(api, epp, minerAddr, sf, jrnl) + m := miner.NewMiner(api, epp, minerAddr, sf) lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { diff --git a/storage/miner.go b/storage/miner.go index 2c37e8c4a..b09ee2fc4 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -45,7 +45,6 @@ type Miner struct { getSealDelay dtypes.GetSealingDelayFunc sealing *sealing.Sealing - jrnl journal.Journal sealingEvtType journal.EventType } @@ -94,7 +93,7 @@ type storageMinerApi interface { WalletHas(context.Context, address.Address) (bool, error) } -func NewMiner(api storageMinerApi, maddr, worker address.Address, h host.Host, ds datastore.Batching, sealer sectorstorage.SectorManager, sc sealing.SectorIDCounter, verif ffiwrapper.Verifier, gsd dtypes.GetSealingDelayFunc, jrnl journal.Journal) (*Miner, error) { +func NewMiner(api storageMinerApi, maddr, worker address.Address, h host.Host, ds datastore.Batching, sealer sectorstorage.SectorManager, sc sealing.SectorIDCounter, verif ffiwrapper.Verifier, gsd dtypes.GetSealingDelayFunc) (*Miner, error) { m := &Miner{ api: api, h: h, @@ -107,8 +106,7 @@ func NewMiner(api storageMinerApi, maddr, worker address.Address, h host.Host, d worker: worker, getSealDelay: gsd, - jrnl: jrnl, - sealingEvtType: jrnl.RegisterEventType("storage", "sealing_states"), + sealingEvtType: journal.J.RegisterEventType("storage", "sealing_states"), } return m, nil @@ -135,7 +133,7 @@ func (m *Miner) Run(ctx context.Context) error { } func (m *Miner) handleSealingNotifications(before, after sealing.SectorInfo) { - journal.MaybeRecordEvent(m.jrnl, m.sealingEvtType, func() interface{} { + journal.J.RecordEvent(m.sealingEvtType, func() interface{} { return SealingStateEvt{ SectorNumber: before.SectorNumber, SectorType: before.SectorType, diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index c448fb223..7fff0b916 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -26,7 +26,7 @@ import ( var errNoPartitions = errors.New("no partitions") func (s *WindowPoStScheduler) failPost(err error, deadline *miner.DeadlineInfo) { - journal.MaybeRecordEvent(s.jrnl, s.wdPoStEvtType, func() interface{} { + journal.J.RecordEvent(s.wdPoStEvtType, func() interface{} { return s.enrichWithTipset(WindowPoStEvt{ State: "failed", Deadline: s.activeDeadline, @@ -48,7 +48,7 @@ func (s *WindowPoStScheduler) doPost(ctx context.Context, deadline *miner.Deadli s.abort = abort s.activeDeadline = deadline - journal.MaybeRecordEvent(s.jrnl, s.wdPoStEvtType, func() interface{} { + journal.J.RecordEvent(s.wdPoStEvtType, func() interface{} { return s.enrichWithTipset(WindowPoStEvt{ State: "started", Deadline: s.activeDeadline, @@ -64,7 +64,7 @@ func (s *WindowPoStScheduler) doPost(ctx context.Context, deadline *miner.Deadli // recordEvent records a successful proofs_processed event in the // journal, even if it was a noop (no partitions). recordEvent := func(partitions []miner.PoStPartition, mcid cid.Cid) { - journal.MaybeRecordEvent(s.jrnl, s.wdPoStEvtType, func() interface{} { + journal.J.RecordEvent(s.wdPoStEvtType, func() interface{} { return s.enrichWithTipset(WindowPoStEvt{ State: "proofs_processed", Deadline: s.activeDeadline, @@ -95,7 +95,7 @@ func (s *WindowPoStScheduler) doPost(ctx context.Context, deadline *miner.Deadli return } - journal.MaybeRecordEvent(s.jrnl, s.wdPoStEvtType, func() interface{} { + journal.J.RecordEvent(s.wdPoStEvtType, func() interface{} { return s.enrichWithTipset(WindowPoStEvt{ State: "succeeded", Deadline: s.activeDeadline, @@ -159,7 +159,7 @@ func (s *WindowPoStScheduler) checkNextRecoveries(ctx context.Context, dlIdx uin Recoveries: []miner.RecoveryDeclaration{}, } - defer journal.MaybeRecordEvent(s.jrnl, s.wdPoStEvtType, func() interface{} { + defer journal.J.RecordEvent(s.wdPoStEvtType, func() interface{} { var mcid cid.Cid if sm != nil { mcid = sm.Cid() @@ -265,7 +265,7 @@ func (s *WindowPoStScheduler) checkNextFaults(ctx context.Context, dlIdx uint64, Faults: []miner.FaultDeclaration{}, } - defer journal.MaybeRecordEvent(s.jrnl, s.wdPoStEvtType, func() interface{} { + defer journal.J.RecordEvent(s.wdPoStEvtType, func() interface{} { var mcid cid.Cid if sm != nil { mcid = sm.Cid() diff --git a/storage/wdpost_sched.go b/storage/wdpost_sched.go index 2c55b93ef..c73d320f3 100644 --- a/storage/wdpost_sched.go +++ b/storage/wdpost_sched.go @@ -66,14 +66,13 @@ type WindowPoStScheduler struct { activeDeadline *miner.DeadlineInfo abort context.CancelFunc - jrnl journal.Journal wdPoStEvtType journal.EventType // failed abi.ChainEpoch // eps // failLk sync.Mutex } -func NewWindowedPoStScheduler(api storageMinerApi, sb storage.Prover, ft sectorstorage.FaultTracker, actor address.Address, worker address.Address, jrnl journal.Journal) (*WindowPoStScheduler, error) { +func NewWindowedPoStScheduler(api storageMinerApi, sb storage.Prover, ft sectorstorage.FaultTracker, actor address.Address, worker address.Address) (*WindowPoStScheduler, error) { mi, err := api.StateMinerInfo(context.TODO(), actor, types.EmptyTSK) if err != nil { return nil, xerrors.Errorf("getting sector size: %w", err) @@ -93,8 +92,7 @@ func NewWindowedPoStScheduler(api storageMinerApi, sb storage.Prover, ft sectors actor: actor, worker: worker, - jrnl: jrnl, - wdPoStEvtType: jrnl.RegisterEventType("storage", "wdpost"), + wdPoStEvtType: journal.J.RegisterEventType("storage", "wdpost"), }, nil } @@ -248,7 +246,7 @@ func (s *WindowPoStScheduler) abortActivePoSt() { if s.abort != nil { s.abort() - journal.MaybeRecordEvent(s.jrnl, s.wdPoStEvtType, func() interface{} { + journal.J.RecordEvent(s.wdPoStEvtType, func() interface{} { return s.enrichWithTipset(WindowPoStEvt{ State: "abort", Deadline: s.activeDeadline, From cb8e105a948aaeea1bdd2e989821a1403b573cea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Wed, 26 Aug 2020 16:44:06 +0100 Subject: [PATCH 023/885] reduce diff noise. --- chain/messagepool/messagepool_test.go | 5 ----- chain/messagepool/repub.go | 1 - chain/messagepool/selection_test.go | 2 -- chain/store/index_test.go | 9 ++++----- cmd/lotus-storage-miner/init.go | 2 -- cmd/lotus/daemon.go | 4 ++-- 6 files changed, 6 insertions(+), 17 deletions(-) diff --git a/chain/messagepool/messagepool_test.go b/chain/messagepool/messagepool_test.go index d5caf7428..316c86d87 100644 --- a/chain/messagepool/messagepool_test.go +++ b/chain/messagepool/messagepool_test.go @@ -7,22 +7,17 @@ import ( "testing" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/messagepool/gasguess" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/types/mock" "github.com/filecoin-project/lotus/chain/wallet" _ "github.com/filecoin-project/lotus/lib/sigs/bls" _ "github.com/filecoin-project/lotus/lib/sigs/secp" - "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/crypto" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" logging "github.com/ipfs/go-log/v2" - - _ "github.com/filecoin-project/lotus/lib/sigs/bls" - _ "github.com/filecoin-project/lotus/lib/sigs/secp" ) func init() { diff --git a/chain/messagepool/repub.go b/chain/messagepool/repub.go index 1fbd66d86..5095821fb 100644 --- a/chain/messagepool/repub.go +++ b/chain/messagepool/repub.go @@ -11,7 +11,6 @@ import ( "github.com/filecoin-project/lotus/chain/messagepool/gasguess" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/journal" - "github.com/ipfs/go-cid" ) diff --git a/chain/messagepool/selection_test.go b/chain/messagepool/selection_test.go index 98c82e324..2d17f7115 100644 --- a/chain/messagepool/selection_test.go +++ b/chain/messagepool/selection_test.go @@ -8,13 +8,11 @@ import ( "testing" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/messagepool/gasguess" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/types/mock" "github.com/filecoin-project/lotus/chain/wallet" - "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/crypto" "github.com/ipfs/go-cid" diff --git a/chain/store/index_test.go b/chain/store/index_test.go index 0a7fc3559..5c49c6791 100644 --- a/chain/store/index_test.go +++ b/chain/store/index_test.go @@ -5,15 +5,14 @@ import ( "context" "testing" - "github.com/filecoin-project/specs-actors/actors/abi" - "github.com/ipfs/go-datastore" - syncds "github.com/ipfs/go-datastore/sync" - "github.com/stretchr/testify/assert" - "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types/mock" "github.com/filecoin-project/lotus/lib/blockstore" + "github.com/filecoin-project/specs-actors/actors/abi" + datastore "github.com/ipfs/go-datastore" + syncds "github.com/ipfs/go-datastore/sync" + "github.com/stretchr/testify/assert" ) func TestIndexSeeks(t *testing.T) { diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index 57f9c072f..e33030e4e 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -26,11 +26,9 @@ import ( "github.com/filecoin-project/go-address" cborutil "github.com/filecoin-project/go-cbor-util" paramfetch "github.com/filecoin-project/go-paramfetch" - sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/extern/sector-storage/stores" - "github.com/filecoin-project/specs-actors/actors/abi" "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/builtin/market" diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index b697cda26..848596944 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -12,6 +12,8 @@ import ( "runtime/pprof" "strings" + "github.com/filecoin-project/lotus/chain/types" + paramfetch "github.com/filecoin-project/go-paramfetch" "github.com/mitchellh/go-homedir" "github.com/multiformats/go-multiaddr" @@ -22,8 +24,6 @@ import ( "go.opencensus.io/tag" "golang.org/x/xerrors" - "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/stmgr" From cac848c106fb1608944958d4105c9186f957e42e Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Mon, 31 Aug 2020 14:24:23 -0700 Subject: [PATCH 024/885] add a command to import an ipld object into the chainstore --- cmd/lotus-shed/import-car.go | 57 ++++++++++++++++++++++++++++++++++++ cmd/lotus-shed/main.go | 1 + 2 files changed, 58 insertions(+) diff --git a/cmd/lotus-shed/import-car.go b/cmd/lotus-shed/import-car.go index 01343c4a3..9cbff953b 100644 --- a/cmd/lotus-shed/import-car.go +++ b/cmd/lotus-shed/import-car.go @@ -1,10 +1,13 @@ package main import ( + "encoding/hex" "fmt" "io" "os" + block "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" "github.com/ipld/go-car" "github.com/urfave/cli/v2" "golang.org/x/xerrors" @@ -81,3 +84,57 @@ var importCarCmd = &cli.Command{ } }, } + +var importObjectCmd = &cli.Command{ + Name: "import-obj", + Usage: "import a raw ipld object into your datastore", + Action: func(cctx *cli.Context) error { + r, err := repo.NewFS(cctx.String("repo")) + if err != nil { + return xerrors.Errorf("opening fs repo: %w", err) + } + + exists, err := r.Exists() + if err != nil { + return err + } + if !exists { + return xerrors.Errorf("lotus repo doesn't exist") + } + + lr, err := r.Lock(repo.FullNode) + if err != nil { + return err + } + defer lr.Close() //nolint:errcheck + + ds, err := lr.Datastore("/chain") + if err != nil { + return err + } + + bs := blockstore.NewBlockstore(ds) + + c, err := cid.Decode(cctx.Args().Get(0)) + if err != nil { + return err + } + + data, err := hex.DecodeString(cctx.Args().Get(1)) + if err != nil { + return err + } + + blk, err := block.NewBlockWithCid(data, c) + if err != nil { + return err + } + + if err := bs.Put(blk); err != nil { + return err + } + + return nil + + }, +} diff --git a/cmd/lotus-shed/main.go b/cmd/lotus-shed/main.go index 5438a31ef..11b98a3ac 100644 --- a/cmd/lotus-shed/main.go +++ b/cmd/lotus-shed/main.go @@ -24,6 +24,7 @@ func main() { bigIntParseCmd, staterootCmd, importCarCmd, + importObjectCmd, commpToCidCmd, fetchParamCmd, proofsCmd, From f58e8bc9a393c7fb3aca463fad961b45c8a0b8f4 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Tue, 1 Sep 2020 02:18:02 -0400 Subject: [PATCH 025/885] Fix some failed precommit handling --- extern/storage-sealing/states_failed.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/extern/storage-sealing/states_failed.go b/extern/storage-sealing/states_failed.go index e313fd712..25f60ff2f 100644 --- a/extern/storage-sealing/states_failed.go +++ b/extern/storage-sealing/states_failed.go @@ -37,16 +37,16 @@ func (m *Sealing) checkPreCommitted(ctx statemachine.Context, sector SectorInfo) tok, _, err := m.api.ChainHead(ctx.Context()) if err != nil { log.Errorf("handleSealPrecommit1Failed(%d): temp error: %+v", sector.SectorNumber, err) - return nil, true + return nil, false } info, err := m.api.StateSectorPreCommitInfo(ctx.Context(), m.maddr, sector.SectorNumber, tok) if err != nil { log.Errorf("handleSealPrecommit1Failed(%d): temp error: %+v", sector.SectorNumber, err) - return nil, true + return nil, false } - return info, false + return info, true } func (m *Sealing) handleSealPrecommit1Failed(ctx statemachine.Context, sector SectorInfo) error { @@ -107,7 +107,7 @@ func (m *Sealing) handlePreCommitFailed(ctx statemachine.Context, sector SectorI } if pci, is := m.checkPreCommitted(ctx, sector); is && pci != nil { - if sector.PreCommitMessage != nil { + if sector.PreCommitMessage == nil { log.Warn("sector %d is precommitted on chain, but we don't have precommit message", sector.SectorNumber) return ctx.Send(SectorPreCommitLanded{TipSet: tok}) } From 0b6a182a44d8860ddb580f82f0e4fa7cd059ec18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Wed, 2 Sep 2020 19:15:07 +0100 Subject: [PATCH 026/885] move InitJournalKey to the top; add godcs on invoke order. --- node/builder.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/node/builder.go b/node/builder.go index 670702fff..bd9bc5005 100644 --- a/node/builder.go +++ b/node/builder.go @@ -91,11 +91,16 @@ var ( type invoke int +// Invokes are called in the order they are defined. //nolint:golint const ( + // InitJournal at position 0 initializes the journal global var as soon as + // the system starts, so that it's available for all other components. + InitJournalKey = invoke(iota) + // libp2p - PstoreAddSelfKeysKey = invoke(iota) + PstoreAddSelfKeysKey StartListeningKey BootstrapKey @@ -123,7 +128,6 @@ const ( HeadMetricsKey SettlePaymentChannelsKey RunPeerTaggerKey - InitJournalKey SetApiEndpointKey From ac152abc75419c785f1da1a4ddec61d7015d4c39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Wed, 2 Sep 2020 19:15:25 +0100 Subject: [PATCH 027/885] shuffle code to journal wdpost events. --- storage/wdpost_run.go | 125 ++++++++++++++++++++++-------------------- 1 file changed, 66 insertions(+), 59 deletions(-) diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index cde31d135..83861a6d8 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -151,41 +151,24 @@ func (s *WindowPoStScheduler) checkSectors(ctx context.Context, check abi.BitFie return sbf, nil } -func (s *WindowPoStScheduler) checkNextRecoveries(ctx context.Context, dlIdx uint64, partitions []*miner.Partition) error { +func (s *WindowPoStScheduler) checkNextRecoveries(ctx context.Context, dlIdx uint64, partitions []*miner.Partition) ([]miner.RecoveryDeclaration, *types.SignedMessage, error) { ctx, span := trace.StartSpan(ctx, "storage.checkNextRecoveries") defer span.End() - var sm *types.SignedMessage faulty := uint64(0) params := &miner.DeclareFaultsRecoveredParams{ Recoveries: []miner.RecoveryDeclaration{}, } - defer journal.J.RecordEvent(s.wdPoStEvtType, func() interface{} { - var mcid cid.Cid - if sm != nil { - mcid = sm.Cid() - } - - return s.enrichWithTipset(WindowPoStEvt{ - State: "recoveries_processed", - Deadline: s.activeDeadline, - Recoveries: &WindowPoStEvt_Recoveries{ - Declarations: params.Recoveries, - MessageCID: mcid, - }, - }) - }) - for partIdx, partition := range partitions { unrecovered, err := bitfield.SubtractBitField(partition.Faults, partition.Recoveries) if err != nil { - return xerrors.Errorf("subtracting recovered set from fault set: %w", err) + return nil, nil, xerrors.Errorf("subtracting recovered set from fault set: %w", err) } uc, err := unrecovered.Count() if err != nil { - return xerrors.Errorf("counting unrecovered sectors: %w", err) + return nil, nil, xerrors.Errorf("counting unrecovered sectors: %w", err) } if uc == 0 { @@ -196,13 +179,13 @@ func (s *WindowPoStScheduler) checkNextRecoveries(ctx context.Context, dlIdx uin recovered, err := s.checkSectors(ctx, unrecovered) if err != nil { - return xerrors.Errorf("checking unrecovered sectors: %w", err) + return nil, nil, xerrors.Errorf("checking unrecovered sectors: %w", err) } // if all sectors failed to recover, don't declare recoveries recoveredCount, err := recovered.Count() if err != nil { - return xerrors.Errorf("counting recovered sectors: %w", err) + return nil, nil, xerrors.Errorf("counting recovered sectors: %w", err) } if recoveredCount == 0 { @@ -216,17 +199,18 @@ func (s *WindowPoStScheduler) checkNextRecoveries(ctx context.Context, dlIdx uin }) } - if len(params.Recoveries) == 0 { + recoveries := params.Recoveries + if len(recoveries) == 0 { if faulty != 0 { log.Warnw("No recoveries to declare", "deadline", dlIdx, "faulty", faulty) } - return nil + return recoveries, nil, nil } enc, aerr := actors.SerializeParams(params) if aerr != nil { - return xerrors.Errorf("could not serialize declare recoveries parameters: %w", aerr) + return recoveries, nil, xerrors.Errorf("could not serialize declare recoveries parameters: %w", aerr) } msg := &types.Message{ @@ -241,67 +225,51 @@ func (s *WindowPoStScheduler) checkNextRecoveries(ctx context.Context, dlIdx uin sm, err := s.api.MpoolPushMessage(ctx, msg, &api.MessageSendSpec{MaxFee: abi.TokenAmount(s.feeCfg.MaxWindowPoStGasFee)}) if err != nil { - return xerrors.Errorf("pushing message to mpool: %w", err) + return recoveries, sm, xerrors.Errorf("pushing message to mpool: %w", err) } log.Warnw("declare faults recovered Message CID", "cid", sm.Cid()) rec, err := s.api.StateWaitMsg(context.TODO(), sm.Cid(), build.MessageConfidence) if err != nil { - return xerrors.Errorf("declare faults recovered wait error: %w", err) + return recoveries, sm, xerrors.Errorf("declare faults recovered wait error: %w", err) } if rec.Receipt.ExitCode != 0 { - return xerrors.Errorf("declare faults recovered wait non-0 exit code: %d", rec.Receipt.ExitCode) + return recoveries, sm, xerrors.Errorf("declare faults recovered wait non-0 exit code: %d", rec.Receipt.ExitCode) } - return nil + return recoveries, sm, nil } -func (s *WindowPoStScheduler) checkNextFaults(ctx context.Context, dlIdx uint64, partitions []*miner.Partition) error { +func (s *WindowPoStScheduler) checkNextFaults(ctx context.Context, dlIdx uint64, partitions []*miner.Partition) ([]miner.FaultDeclaration, *types.SignedMessage, error) { ctx, span := trace.StartSpan(ctx, "storage.checkNextFaults") defer span.End() - var sm *types.SignedMessage bad := uint64(0) params := &miner.DeclareFaultsParams{ Faults: []miner.FaultDeclaration{}, } - defer journal.J.RecordEvent(s.wdPoStEvtType, func() interface{} { - var mcid cid.Cid - if sm != nil { - mcid = sm.Cid() - } - return s.enrichWithTipset(WindowPoStEvt{ - State: "faults_processed", - Deadline: s.activeDeadline, - Faults: &WindowPoStEvt_Faults{ - Declarations: params.Faults, - MessageCID: mcid, - }, - }) - }) - for partIdx, partition := range partitions { toCheck, err := partition.ActiveSectors() if err != nil { - return xerrors.Errorf("getting active sectors: %w", err) + return nil, nil, xerrors.Errorf("getting active sectors: %w", err) } good, err := s.checkSectors(ctx, toCheck) if err != nil { - return xerrors.Errorf("checking sectors: %w", err) + return nil, nil, xerrors.Errorf("checking sectors: %w", err) } faulty, err := bitfield.SubtractBitField(toCheck, good) if err != nil { - return xerrors.Errorf("calculating faulty sector set: %w", err) + return nil, nil, xerrors.Errorf("calculating faulty sector set: %w", err) } c, err := faulty.Count() if err != nil { - return xerrors.Errorf("counting faulty sectors: %w", err) + return nil, nil, xerrors.Errorf("counting faulty sectors: %w", err) } if c == 0 { @@ -317,15 +285,16 @@ func (s *WindowPoStScheduler) checkNextFaults(ctx context.Context, dlIdx uint64, }) } - if len(params.Faults) == 0 { - return nil + faults := params.Faults + if len(faults) == 0 { + return faults, nil, nil } log.Errorw("DETECTED FAULTY SECTORS, declaring faults", "count", bad) enc, aerr := actors.SerializeParams(params) if aerr != nil { - return xerrors.Errorf("could not serialize declare faults parameters: %w", aerr) + return faults, nil, xerrors.Errorf("could not serialize declare faults parameters: %w", aerr) } msg := &types.Message{ @@ -340,21 +309,21 @@ func (s *WindowPoStScheduler) checkNextFaults(ctx context.Context, dlIdx uint64, sm, err := s.api.MpoolPushMessage(ctx, msg, spec) if err != nil { - return xerrors.Errorf("pushing message to mpool: %w", err) + return faults, sm, xerrors.Errorf("pushing message to mpool: %w", err) } log.Warnw("declare faults Message CID", "cid", sm.Cid()) rec, err := s.api.StateWaitMsg(context.TODO(), sm.Cid(), build.MessageConfidence) if err != nil { - return xerrors.Errorf("declare faults wait error: %w", err) + return faults, sm, xerrors.Errorf("declare faults wait error: %w", err) } if rec.Receipt.ExitCode != 0 { - return xerrors.Errorf("declare faults wait non-0 exit code: %d", rec.Receipt.ExitCode) + return faults, sm, xerrors.Errorf("declare faults wait non-0 exit code: %d", rec.Receipt.ExitCode) } - return nil + return faults, sm, nil } func (s *WindowPoStScheduler) runPost(ctx context.Context, di miner.DeadlineInfo, ts *types.TipSet) (*miner.SubmitWindowedPoStParams, error) { @@ -374,15 +343,53 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di miner.DeadlineInfo return } - if err := s.checkNextRecoveries(context.TODO(), declDeadline, partitions); err != nil { + var ( + sigmsg *types.SignedMessage + recoveries []miner.RecoveryDeclaration + faults []miner.FaultDeclaration + + // optionalCid returns the CID of the message, or cid.Undef is the + // message is nil. We don't need the argument (could capture the + // pointer), but it's clearer and purer like that. + optionalCid = func(sigmsg *types.SignedMessage) cid.Cid { + if sigmsg == nil { + return cid.Undef + } + return sigmsg.Cid() + } + ) + + if recoveries, sigmsg, err = s.checkNextRecoveries(context.TODO(), declDeadline, partitions); err != nil { // TODO: This is potentially quite bad, but not even trying to post when this fails is objectively worse log.Errorf("checking sector recoveries: %v", err) } - if err := s.checkNextFaults(context.TODO(), declDeadline, partitions); err != nil { + journal.J.RecordEvent(s.wdPoStEvtType, func() interface{} { + return s.enrichWithTipset(WindowPoStEvt{ + State: "recoveries_processed", + Deadline: s.activeDeadline, + Recoveries: &WindowPoStEvt_Recoveries{ + Declarations: recoveries, + MessageCID: optionalCid(sigmsg), + }, + }) + }) + + if faults, sigmsg, err = s.checkNextFaults(context.TODO(), declDeadline, partitions); err != nil { // TODO: This is also potentially really bad, but we try to post anyways log.Errorf("checking sector faults: %v", err) } + + journal.J.RecordEvent(s.wdPoStEvtType, func() interface{} { + return s.enrichWithTipset(WindowPoStEvt{ + State: "faults_processed", + Deadline: s.activeDeadline, + Faults: &WindowPoStEvt_Faults{ + Declarations: faults, + MessageCID: optionalCid(sigmsg), + }, + }) + }) }() buf := new(bytes.Buffer) From f046af337ae2f7391d6b0b288137d95a0620148f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Wed, 2 Sep 2020 19:45:25 +0100 Subject: [PATCH 028/885] split wdpost event into finer-grained ones. --- storage/wdpost_journal.go | 74 +++++++++++++++++++++++++++++++++ storage/wdpost_run.go | 86 ++++++++++++++++++--------------------- storage/wdpost_sched.go | 65 ++++++++++------------------- 3 files changed, 135 insertions(+), 90 deletions(-) create mode 100644 storage/wdpost_journal.go diff --git a/storage/wdpost_journal.go b/storage/wdpost_journal.go new file mode 100644 index 000000000..04bca895c --- /dev/null +++ b/storage/wdpost_journal.go @@ -0,0 +1,74 @@ +package storage + +import ( + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-actors/actors/builtin/miner" + + "github.com/ipfs/go-cid" +) + +// SchedulerState defines the possible states in which the scheduler could be, +// for the purposes of journalling. +type SchedulerState string + +const ( + // SchedulerStateStarted gets recorded when a WdPoSt cycle for an + // epoch begins. + SchedulerStateStarted = SchedulerState("started") + // SchedulerStateAborted gets recorded when a WdPoSt cycle for an + // epoch is aborted, normally because of a chain reorg or advancement. + SchedulerStateAborted = SchedulerState("aborted") + // SchedulerStateFaulted gets recorded when a WdPoSt cycle for an + // epoch terminates abnormally, in which case the error is also recorded. + SchedulerStateFaulted = SchedulerState("faulted") + // SchedulerStateSucceeded gets recorded when a WdPoSt cycle for an + // epoch ends successfully. + SchedulerStateSucceeded = SchedulerState("succeeded") +) + +// Journal event types. +const ( + evtTypeWdPoStScheduler = iota + evtTypeWdPoStProofs + evtTypeWdPoStRecoveries + evtTypeWdPoStFaults +) + +// evtCommon is a common set of attributes for Windowed PoSt journal events. +type evtCommon struct { + Deadline *miner.DeadlineInfo + Height abi.ChainEpoch + TipSet []cid.Cid + Error error `json:",omitempty"` +} + +// WdPoStSchedulerEvt is the journal event that gets recorded on scheduler +// actions. +type WdPoStSchedulerEvt struct { + evtCommon + State SchedulerState +} + +// WdPoStProofsProcessedEvt is the journal event that gets recorded when +// Windowed PoSt proofs have been processed. +type WdPoStProofsProcessedEvt struct { + evtCommon + Partitions []miner.PoStPartition + MessageCID cid.Cid `json:",omitempty"` +} + +// WdPoStRecoveriesProcessedEvt is the journal event that gets recorded when +// Windowed PoSt recoveries have been processed. +type WdPoStRecoveriesProcessedEvt struct { + evtCommon + Declarations []miner.RecoveryDeclaration + MessageCID cid.Cid `json:",omitempty"` +} + +// WdPoStFaultsProcessedEvt is the journal event that gets recorded when +// Windowed PoSt faults have been processed. +type WdPoStFaultsProcessedEvt struct { + evtCommon + Declarations []miner.FaultDeclaration + MessageCID cid.Cid `json:",omitempty"` +} diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 83861a6d8..1ae454295 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -28,12 +28,11 @@ import ( var errNoPartitions = errors.New("no partitions") func (s *WindowPoStScheduler) failPost(err error, deadline *miner.DeadlineInfo) { - journal.J.RecordEvent(s.wdPoStEvtType, func() interface{} { - return s.enrichWithTipset(WindowPoStEvt{ - State: "failed", - Deadline: s.activeDeadline, - Error: err, - }) + journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} { + return WdPoStSchedulerEvt{ + evtCommon: s.getEvtCommon(err), + State: SchedulerStateFaulted, + } }) log.Errorf("TODO") @@ -50,11 +49,11 @@ func (s *WindowPoStScheduler) doPost(ctx context.Context, deadline *miner.Deadli s.abort = abort s.activeDeadline = deadline - journal.J.RecordEvent(s.wdPoStEvtType, func() interface{} { - return s.enrichWithTipset(WindowPoStEvt{ - State: "started", - Deadline: s.activeDeadline, - }) + journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} { + return WdPoStSchedulerEvt{ + evtCommon: s.getEvtCommon(nil), + State: SchedulerStateStarted, + } }) go func() { @@ -63,25 +62,22 @@ func (s *WindowPoStScheduler) doPost(ctx context.Context, deadline *miner.Deadli ctx, span := trace.StartSpan(ctx, "WindowPoStScheduler.doPost") defer span.End() - // recordEvent records a successful proofs_processed event in the + // recordProofsEvent records a successful proofs_processed event in the // journal, even if it was a noop (no partitions). - recordEvent := func(partitions []miner.PoStPartition, mcid cid.Cid) { - journal.J.RecordEvent(s.wdPoStEvtType, func() interface{} { - return s.enrichWithTipset(WindowPoStEvt{ - State: "proofs_processed", - Deadline: s.activeDeadline, - Proofs: &WindowPoStEvt_Proofs{ - Partitions: partitions, - MessageCID: mcid, - }, - }) + recordProofsEvent := func(partitions []miner.PoStPartition, mcid cid.Cid) { + journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStProofs], func() interface{} { + return &WdPoStProofsProcessedEvt{ + evtCommon: s.getEvtCommon(nil), + Partitions: partitions, + MessageCID: mcid, + } }) } proof, err := s.runPost(ctx, *deadline, ts) switch err { case errNoPartitions: - recordEvent(nil, cid.Undef) + recordProofsEvent(nil, cid.Undef) return case nil: sm, err := s.submitPost(ctx, proof) @@ -90,18 +86,18 @@ func (s *WindowPoStScheduler) doPost(ctx context.Context, deadline *miner.Deadli s.failPost(err, deadline) return } - recordEvent(proof.Partitions, sm.Cid()) + recordProofsEvent(proof.Partitions, sm.Cid()) default: log.Errorf("runPost failed: %+v", err) s.failPost(err, deadline) return } - journal.J.RecordEvent(s.wdPoStEvtType, func() interface{} { - return s.enrichWithTipset(WindowPoStEvt{ - State: "succeeded", - Deadline: s.activeDeadline, - }) + journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} { + return WdPoStSchedulerEvt{ + evtCommon: s.getEvtCommon(nil), + State: SchedulerStateSucceeded, + } }) }() } @@ -364,15 +360,14 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di miner.DeadlineInfo log.Errorf("checking sector recoveries: %v", err) } - journal.J.RecordEvent(s.wdPoStEvtType, func() interface{} { - return s.enrichWithTipset(WindowPoStEvt{ - State: "recoveries_processed", - Deadline: s.activeDeadline, - Recoveries: &WindowPoStEvt_Recoveries{ - Declarations: recoveries, - MessageCID: optionalCid(sigmsg), - }, - }) + journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStRecoveries], func() interface{} { + j := WdPoStRecoveriesProcessedEvt{ + evtCommon: s.getEvtCommon(err), + Declarations: recoveries, + MessageCID: optionalCid(sigmsg), + } + j.Error = err + return j }) if faults, sigmsg, err = s.checkNextFaults(context.TODO(), declDeadline, partitions); err != nil { @@ -380,15 +375,12 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di miner.DeadlineInfo log.Errorf("checking sector faults: %v", err) } - journal.J.RecordEvent(s.wdPoStEvtType, func() interface{} { - return s.enrichWithTipset(WindowPoStEvt{ - State: "faults_processed", - Deadline: s.activeDeadline, - Faults: &WindowPoStEvt_Faults{ - Declarations: faults, - MessageCID: optionalCid(sigmsg), - }, - }) + journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStFaults], func() interface{} { + return WdPoStFaultsProcessedEvt{ + evtCommon: s.getEvtCommon(err), + Declarations: faults, + MessageCID: optionalCid(sigmsg), + } }) }() diff --git a/storage/wdpost_sched.go b/storage/wdpost_sched.go index 1b2303dc0..d2b8279a0 100644 --- a/storage/wdpost_sched.go +++ b/storage/wdpost_sched.go @@ -19,39 +19,11 @@ import ( "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/node/config" - "github.com/ipfs/go-cid" "go.opencensus.io/trace" ) const StartConfidence = 4 // TODO: config -type WindowPoStEvt struct { - State string - Deadline *miner.DeadlineInfo - Height abi.ChainEpoch - TipSet []cid.Cid - Error error `json:",omitempty"` - - Proofs *WindowPoStEvt_Proofs `json:",omitempty"` - Recoveries *WindowPoStEvt_Recoveries `json:",omitempty"` - Faults *WindowPoStEvt_Faults `json:",omitempty"` -} - -type WindowPoStEvt_Proofs struct { - Partitions []miner.PoStPartition - MessageCID cid.Cid `json:",omitempty"` -} - -type WindowPoStEvt_Recoveries struct { - Declarations []miner.RecoveryDeclaration - MessageCID cid.Cid `json:",omitempty"` -} - -type WindowPoStEvt_Faults struct { - Declarations []miner.FaultDeclaration - MessageCID cid.Cid `json:",omitempty"` -} - type WindowPoStScheduler struct { api storageMinerApi feeCfg config.MinerFeeConfig @@ -69,7 +41,7 @@ type WindowPoStScheduler struct { activeDeadline *miner.DeadlineInfo abort context.CancelFunc - wdPoStEvtType journal.EventType + evtTypes [4]journal.EventType // failed abi.ChainEpoch // eps // failLk sync.Mutex @@ -94,9 +66,14 @@ func NewWindowedPoStScheduler(api storageMinerApi, fc config.MinerFeeConfig, sb proofType: rt, partitionSectors: mi.WindowPoStPartitionSectors, - actor: actor, - worker: worker, - wdPoStEvtType: journal.J.RegisterEventType("storage", "wdpost"), + actor: actor, + worker: worker, + evtTypes: [...]journal.EventType{ + evtTypeWdPoStScheduler: journal.J.RegisterEventType("wdpost", "scheduler"), + evtTypeWdPoStProofs: journal.J.RegisterEventType("wdpost", "proofs_processed"), + evtTypeWdPoStRecoveries: journal.J.RegisterEventType("wdpost", "recoveries_processed"), + evtTypeWdPoStFaults: journal.J.RegisterEventType("wdpost", "faults_processed"), + }, }, nil } @@ -253,11 +230,11 @@ func (s *WindowPoStScheduler) abortActivePoSt() { if s.abort != nil { s.abort() - journal.J.RecordEvent(s.wdPoStEvtType, func() interface{} { - return s.enrichWithTipset(WindowPoStEvt{ - State: "abort", - Deadline: s.activeDeadline, - }) + journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} { + return WdPoStSchedulerEvt{ + evtCommon: s.getEvtCommon(nil), + State: SchedulerStateAborted, + } }) log.Warnf("Aborting Window PoSt (Deadline: %+v)", s.activeDeadline) @@ -267,12 +244,14 @@ func (s *WindowPoStScheduler) abortActivePoSt() { s.abort = nil } -// enrichWithTipset enriches a WindowPoStEvt with tipset information, -// if available. -func (s *WindowPoStScheduler) enrichWithTipset(evt WindowPoStEvt) WindowPoStEvt { +// getEvtCommon populates and returns common attributes from state, for a +// WdPoSt journal event. +func (s *WindowPoStScheduler) getEvtCommon(err error) evtCommon { + c := evtCommon{Error: err} if s.cur != nil { - evt.Height = s.cur.Height() - evt.TipSet = s.cur.Cids() + c.Deadline = s.activeDeadline + c.Height = s.cur.Height() + c.TipSet = s.cur.Cids() } - return evt + return c } From fd04a195449a2620555b60925eeeb94464a622d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Wed, 2 Sep 2020 19:46:57 +0100 Subject: [PATCH 029/885] go mod tidy. --- go.sum | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/go.sum b/go.sum index b35a642a5..24c3cedc7 100644 --- a/go.sum +++ b/go.sum @@ -1278,6 +1278,7 @@ github.com/prometheus/procfs v0.1.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/raulk/clock v1.1.0 h1:dpb29+UKMbLqiU/jqIJptgLR1nn23HLgMY0sTCDza5Y= github.com/raulk/clock v1.1.0/go.mod h1:3MpVxdZ/ODBQDxbN+kzshf5OSZwPjtMDx6BBXBmOeY0= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= @@ -1293,8 +1294,10 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sercand/kuberesolver v2.1.0+incompatible/go.mod h1:lWF3GL0xptCB/vCiJPl/ZshwPsX/n4Y7u0CW9E7aQIQ= +github.com/sercand/kuberesolver v2.4.0+incompatible h1:WE2OlRf6wjLxHwNkkFLQGaZcVLEXjMjBPjjEU5vksH8= github.com/sercand/kuberesolver v2.4.0+incompatible/go.mod h1:lWF3GL0xptCB/vCiJPl/ZshwPsX/n4Y7u0CW9E7aQIQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM= github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= @@ -1323,13 +1326,16 @@ github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5k github.com/siebenmann/go-kstat v0.0.0-20160321171754-d34789b79745/go.mod h1:G81aIFAMS9ECrwBYR9YxhlPjWgrItd+Kje78O6+uqm8= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= +github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w= github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2RVY1rIf+2J2o/IM9+vPq9RzmHDSseB7FoXiSNIUsoU= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= @@ -1338,6 +1344,7 @@ github.com/soundcloud/go-runit v0.0.0-20150630195641-06ad41a06c4a/go.mod h1:LeFC github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a/go.mod h1:7AyxJNCJ7SBZ1MfVQCWD6Uqo2oubI2Eq2y2eqf+A5r0= +github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 h1:RC6RW7j+1+HkWaX/Yh71Ee5ZHaHYt7ZP4sQgUrm6cDU= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= @@ -1356,6 +1363,7 @@ github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3 github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -1370,11 +1378,14 @@ github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFd github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e/go.mod h1:XDKHRm5ThF8YJjx001LtgelzsoaEcvnA7lVWz9EeX3g= +github.com/tj/go-spin v1.1.0 h1:lhdWZsvImxvZ3q1C5OIB7d72DuOwP4O2NdBg9PyzNds= github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= +github.com/uber/jaeger-client-go v2.23.1+incompatible h1:uArBYHQR0HqLFFAypI7RsWTzPSj/bDpmZZuQjMLSg1A= github.com/uber/jaeger-client-go v2.23.1+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v1.5.1-0.20181102163054-1fc5c315e03c/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= +github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw= github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= @@ -1392,10 +1403,14 @@ github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMI github.com/wangjia184/sortedset v0.0.0-20160527075905-f5d03557ba30/go.mod h1:YkocrP2K2tcw938x9gCOmT5G5eCD6jsTz0SZuyAqwIE= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= +github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a h1:G++j5e0OC488te356JvdhaM8YS6nMsjLAYF7JxCv07w= github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= +github.com/weaveworks/common v0.0.0-20200512154658-384f10054ec5 h1:EYxr08r8x6r/5fLEAMMkida1BVgxVXE4LfZv/XV+znU= github.com/weaveworks/common v0.0.0-20200512154658-384f10054ec5/go.mod h1:c98fKi5B9u8OsKGiWHLRKus6ToQ1Tubeow44ECO1uxY= +github.com/weaveworks/promrus v1.2.0 h1:jOLf6pe6/vss4qGHjXmGz4oDJQA+AOCqEL3FvvZGz7M= github.com/weaveworks/promrus v1.2.0/go.mod h1:SaE82+OJ91yqjrE1rsvBWVzNZKcHYFtMUyS1+Ogs/KA= github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc/go.mod h1:r45hJU7yEoA81k6MWNhpMj/kms0n14dkzkxYHoB96UM= +github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba h1:X4n8JG2e2biEZZXdBKt9HX7DN3bYGFUqljqqy0DqgnY= github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba/go.mod h1:CHQnYnQUEPydYCwuy8lmTHfGmdw9TKrhWV0xLx8l0oM= github.com/whyrusleeping/cbor-gen v0.0.0-20191212224538-d370462a7e8a/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/cbor-gen v0.0.0-20191216205031-b047b6acb3c0/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= @@ -1435,20 +1450,27 @@ github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee/go.mod h1: github.com/whyrusleeping/yamux v1.1.5/go.mod h1:E8LnQQ8HKx5KD29HZFUwM1PxCOdPRzGwur1mcYhXcD8= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xlab/c-for-go v0.0.0-20200718154222-87b0065af829 h1:wb7xrDzfkLgPHsSEBm+VSx6aDdi64VtV0xvP0E6j8bk= github.com/xlab/c-for-go v0.0.0-20200718154222-87b0065af829/go.mod h1:h/1PEBwj7Ym/8kOuMWvO2ujZ6Lt+TMbySEXNhjjR87I= +github.com/xlab/pkgconfig v0.0.0-20170226114623-cea12a0fd245 h1:Sw125DKxZhPUI4JLlWugkzsrlB50jR9v2khiD9FxuSo= github.com/xlab/pkgconfig v0.0.0-20170226114623-cea12a0fd245/go.mod h1:C+diUUz7pxhNY6KAoLgrTYARGWnt82zWTylZlxT92vk= github.com/xorcare/golden v0.6.0/go.mod h1:7T39/ZMvaSEZlBPoYfVFmsBLmUl3uz9IuzWj/U6FtvQ= +github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542 h1:oWgZJmC1DorFZDpfMfWg7xk29yEOZiXmo/wZl+utTI8= github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542/go.mod h1:7T39/ZMvaSEZlBPoYfVFmsBLmUl3uz9IuzWj/U6FtvQ= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.dedis.ch/fixbuf v1.0.3 h1:hGcV9Cd/znUxlusJ64eAlExS+5cJDIyTyEG+otu5wQs= go.dedis.ch/fixbuf v1.0.3/go.mod h1:yzJMt34Wa5xD37V5RTdmp38cz3QhMagdGoem9anUalw= go.dedis.ch/kyber/v3 v3.0.4/go.mod h1:OzvaEnPvKlyrWyp3kGXlFdp7ap1VC6RkZDTaPikqhsQ= +go.dedis.ch/kyber/v3 v3.0.9 h1:i0ZbOQocHUjfFasBiUql5zVeC7u/vahFd96DFA8UOWk= go.dedis.ch/kyber/v3 v3.0.9/go.mod h1:rhNjUUg6ahf8HEg5HUvVBYoWY4boAafX8tYxX+PS+qg= go.dedis.ch/protobuf v1.0.5/go.mod h1:eIV4wicvi6JK0q/QnfIEGeSFNG0ZeB24kzut5+HaRLo= go.dedis.ch/protobuf v1.0.7/go.mod h1:pv5ysfkDX/EawiPqcW3ikOxsL5t+BqnV6xHSmE79KI4= +go.dedis.ch/protobuf v1.0.11 h1:FTYVIEzY/bfl37lu3pR4lIj+F9Vp1jE8oh91VmxKgLo= go.dedis.ch/protobuf v1.0.11/go.mod h1:97QR256dnkimeNdfmURz0wAMNVbd1VmLXhG1CrTYrJ4= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg= go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= @@ -1471,12 +1493,14 @@ go.uber.org/dig v1.8.0 h1:1rR6hnL/bu1EVcjnRDN5kx1vbIjEJDTGhSQ2B3ddpcI= go.uber.org/dig v1.8.0/go.mod h1:X34SnWGr8Fyla9zQNO2GSO2D+TIuqB14OS8JhYocIyw= go.uber.org/fx v1.9.0 h1:7OAz8ucp35AU8eydejpYG7QrbE8rLKzGhHbZlJi5LYY= go.uber.org/fx v1.9.0/go.mod h1:mFdUyAUuJ3w4jAckiKSKbldsxy1ojpAMJ+dVZg5Y0Aw= +go.uber.org/goleak v1.0.0 h1:qsup4IcBdlmsnGfqyLl4Ntn3C2XCCuKAE7DwHpScyUo= go.uber.org/goleak v1.0.0/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= @@ -1528,6 +1552,7 @@ golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 h1:QE6XYQK6naiK1EPAe1g/ILLxN5RBoH5xkJk3CqlMI/Y= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -1541,6 +1566,7 @@ golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -1549,6 +1575,7 @@ golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180524181706-dfa909b99c79/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1733,6 +1760,7 @@ golang.org/x/tools v0.0.0-20200216192241-b320d3a0f5a2/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200318150045-ba25ddc85566/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200711155855-7342f9734a7d h1:F3OmlXCzYtG9YE6tXDnUOlJBzVzHF8EcmZ1yTJlcgIk= golang.org/x/tools v0.0.0-20200711155855-7342f9734a7d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1824,6 +1852,7 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/cheggaaa/pb.v1 v1.0.28 h1:n1tBJnnK2r7g9OW2btFH91V92STTUevLXYFb8gy9EMk= @@ -1835,6 +1864,7 @@ gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= @@ -1844,9 +1874,11 @@ gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -1855,14 +1887,21 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqpnJdvzuiHsl/dnxl11M= howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= +launchpad.net/gocheck v0.0.0-20140225173054-000000000087 h1:Izowp2XBH6Ya6rv+hqbceQyw/gSGoXfH/UPoTGduL54= launchpad.net/gocheck v0.0.0-20140225173054-000000000087/go.mod h1:hj7XX3B/0A+80Vse0e+BUHsHMTEhd0O4cpUHr/e/BUM= +modernc.org/cc v1.0.0 h1:nPibNuDEx6tvYrUAtvDTTw98rx5juGsa5zuDnKwEEQQ= modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= +modernc.org/golex v1.0.0 h1:wWpDlbK8ejRfSyi0frMyhilD3JBvtcx2AdGDnU+JtsE= modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= +modernc.org/mathutil v1.1.1 h1:FeylZSVX8S+58VsyJlkEj2bcpdytmp9MmDKZkKx8OIE= modernc.org/mathutil v1.1.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/strutil v1.1.0 h1:+1/yCzZxY2pZwwrsbH+4T7BQMoLQ9QiBshRC9eicYsc= modernc.org/strutil v1.1.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= +modernc.org/xc v1.0.0 h1:7ccXrupWZIS3twbUGrtKmHS2DXY6xegFua+6O3xgAFU= modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= From 4554c4bda93da2246d2af1b73427d98b5b0f283b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Wed, 2 Sep 2020 19:54:13 +0100 Subject: [PATCH 030/885] fix lotus-bench stale import. --- cmd/lotus-bench/import.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index 29a5cb91b..7400cd92e 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -21,7 +21,6 @@ import ( "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" - "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/lib/blockstore" _ "github.com/filecoin-project/lotus/lib/sigs/bls" _ "github.com/filecoin-project/lotus/lib/sigs/secp" From 485f13de1254ad1ef833ab4ebe663bd4f972e9cc Mon Sep 17 00:00:00 2001 From: austinabell Date: Wed, 2 Sep 2020 16:45:23 -0400 Subject: [PATCH 031/885] Remove load check --- chain/store/store.go | 11 +++++------ chain/sync.go | 10 ++++------ 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/chain/store/store.go b/chain/store/store.go index 0d38ba9b5..b64277302 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -1278,13 +1278,12 @@ func (cs *ChainStore) GetLatestBeaconEntry(ts *types.TipSet) (*types.BeaconEntry return nil, xerrors.Errorf("made it back to genesis block without finding beacon entry") } - if i != 19 { - next, err := cs.LoadTipSet(cur.Parents()) - if err != nil { - return nil, xerrors.Errorf("failed to load parents when searching back for latest beacon entry: %w", err) - } - cur = next + next, err := cs.LoadTipSet(cur.Parents()) + if err != nil { + return nil, xerrors.Errorf("failed to load parents when searching back for latest beacon entry: %w", err) } + cur = next + } if os.Getenv("LOTUS_IGNORE_DRAND") == "_yes_" { diff --git a/chain/sync.go b/chain/sync.go index 3e2bf9a0f..775c32843 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -1630,13 +1630,11 @@ func (syncer *Syncer) getLatestBeaconEntry(_ context.Context, ts *types.TipSet) return nil, xerrors.Errorf("made it back to genesis block without finding beacon entry") } - if i != 19 { - next, err := syncer.store.LoadTipSet(cur.Parents()) - if err != nil { - return nil, xerrors.Errorf("failed to load parents when searching back for latest beacon entry: %w", err) - } - cur = next + next, err := syncer.store.LoadTipSet(cur.Parents()) + if err != nil { + return nil, xerrors.Errorf("failed to load parents when searching back for latest beacon entry: %w", err) } + cur = next } return nil, xerrors.Errorf("found NO beacon entries in the 20 latest tipsets") From 6b63fa379f522cfe9996f25929640c3aede2e6cd Mon Sep 17 00:00:00 2001 From: austinabell Date: Wed, 2 Sep 2020 16:45:50 -0400 Subject: [PATCH 032/885] format --- chain/store/store.go | 1 - 1 file changed, 1 deletion(-) diff --git a/chain/store/store.go b/chain/store/store.go index b64277302..903bdc624 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -1283,7 +1283,6 @@ func (cs *ChainStore) GetLatestBeaconEntry(ts *types.TipSet) (*types.BeaconEntry return nil, xerrors.Errorf("failed to load parents when searching back for latest beacon entry: %w", err) } cur = next - } if os.Getenv("LOTUS_IGNORE_DRAND") == "_yes_" { From fe52c475703ad417178667477c003e6cd5d930bc Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Mon, 31 Aug 2020 17:11:01 +0200 Subject: [PATCH 033/885] Docs review and re-organization This: * Re-organizes the docs into sections that align with what docs.filecoin.io becoming: * An installation section * A "getting started" section (lotus client focused) * A "storing" section (lotus client focused) * A "mining" section (miner focused) * A "build" section (developer focused) * An legacy "architecture" section is left in the last place. A few high-value documentation pages have been reviewed and updated with the latest recommendations: * Installation section and lotus setup * Miner setup * etc. ... Other pages have been correctly merged into the new relevant sections. Some pages have not been touched. The filesystem layout of the documentation has been changed into folders corresponding to the sections (as requested by @cw). Some pages that were not linked at all and/or where hidden, have been moved to "unclassified". This should make the porting of the Lotus documentation to docs.filecoin.io much easier, while ensuring it is more up to date than it was before. For the moment, this breaks most links as link-aliasing is not supported in lotus-docs. --- Makefile | 2 +- documentation/en/.library.json | 353 +++++++++--------- documentation/en/about.md | 14 + documentation/en/api-scripting-support.md | 25 -- documentation/en/api.md | 85 ----- .../en/{ => architecture}/architecture.md | 2 +- documentation/en/{ => architecture}/mpool.md | 0 .../en/{ => building}/api-methods.md | 0 .../en/{ => building}/api-troubleshooting.md | 0 documentation/en/building/api.md | 38 ++ documentation/en/building/building.md | 5 + .../jaeger-tracing.md} | 0 .../local-devnet.md} | 0 .../en/{ => building}/payment-channels.md | 0 documentation/en/building/remote-api.md | 69 ++++ documentation/en/cli.md | 108 ------ documentation/en/dev-tools.md | 3 - documentation/en/environment-vars.md | 65 ---- documentation/en/faqs.md | 36 +- documentation/en/getting-started.md | 23 -- .../en/getting-started/getting-started.md | 3 + .../setup-troubleshooting.md | 13 +- documentation/en/getting-started/setup.md | 169 +++++++++ documentation/en/getting-started/wallet.md | 58 +++ documentation/en/hardware-mining.md | 54 --- documentation/en/hardware.md | 7 - documentation/en/install-lotus-arch.md | 51 --- documentation/en/install-lotus-fedora.md | 54 --- documentation/en/install-lotus-ubuntu.md | 54 --- documentation/en/install-systemd-services.md | 145 ------- .../en/installation/install-linux.md | 129 +++++++ .../install-macos.md} | 2 +- documentation/en/installation/installation.md | 39 ++ documentation/en/installation/update.md | 72 ++++ documentation/en/join-testnet.md | 93 ----- documentation/en/miner-deals.md | 39 -- documentation/en/mining.md | 149 -------- documentation/en/mining/gpus.md | 17 + .../lotus-seal-worker.md} | 58 +-- documentation/en/mining/managing-deals.md | 19 + documentation/en/mining/miner-setup.md | 241 ++++++++++++ .../en/{ => mining}/mining-troubleshooting.md | 9 +- documentation/en/mining/mining.md | 8 + documentation/en/retrieving-data.md | 27 -- documentation/en/setting-a-static-port.md | 54 --- .../adding-from-ipfs.md} | 6 +- documentation/en/store/making-deals.md | 71 ++++ documentation/en/store/retrieve.md | 27 ++ .../storage-troubleshooting.md} | 13 +- documentation/en/store/store.md | 11 + documentation/en/storing-data.md | 62 --- .../WIP-arch-complementary-notes.md | 0 .../en/{ => unclassified}/block-validation.md | 0 .../en/{dev => unclassified}/create-miner.md | 0 .../{ => unclassified}/dev-tools-pond-ui.md | 0 .../en/{ => unclassified}/sealing-procs.md | 0 documentation/en/updating-lotus.md | 14 - 57 files changed, 1239 insertions(+), 1357 deletions(-) create mode 100644 documentation/en/about.md delete mode 100644 documentation/en/api-scripting-support.md delete mode 100644 documentation/en/api.md rename documentation/en/{ => architecture}/architecture.md (99%) rename documentation/en/{ => architecture}/mpool.md (100%) rename documentation/en/{ => building}/api-methods.md (100%) rename documentation/en/{ => building}/api-troubleshooting.md (100%) create mode 100644 documentation/en/building/api.md create mode 100644 documentation/en/building/building.md rename documentation/en/{dev-tools-jaeger-tracing.md => building/jaeger-tracing.md} (100%) rename documentation/en/{local-dev-net.md => building/local-devnet.md} (100%) rename documentation/en/{ => building}/payment-channels.md (100%) create mode 100644 documentation/en/building/remote-api.md delete mode 100644 documentation/en/cli.md delete mode 100644 documentation/en/dev-tools.md delete mode 100644 documentation/en/environment-vars.md delete mode 100644 documentation/en/getting-started.md create mode 100644 documentation/en/getting-started/getting-started.md rename documentation/en/{ => getting-started}/setup-troubleshooting.md (60%) create mode 100644 documentation/en/getting-started/setup.md create mode 100644 documentation/en/getting-started/wallet.md delete mode 100644 documentation/en/hardware-mining.md delete mode 100644 documentation/en/hardware.md delete mode 100644 documentation/en/install-lotus-arch.md delete mode 100644 documentation/en/install-lotus-fedora.md delete mode 100644 documentation/en/install-lotus-ubuntu.md delete mode 100644 documentation/en/install-systemd-services.md create mode 100644 documentation/en/installation/install-linux.md rename documentation/en/{install-lotus-macos.md => installation/install-macos.md} (85%) create mode 100644 documentation/en/installation/installation.md create mode 100644 documentation/en/installation/update.md delete mode 100644 documentation/en/join-testnet.md delete mode 100644 documentation/en/miner-deals.md delete mode 100644 documentation/en/mining.md create mode 100644 documentation/en/mining/gpus.md rename documentation/en/{mining-lotus-worker.md => mining/lotus-seal-worker.md} (60%) create mode 100644 documentation/en/mining/managing-deals.md create mode 100644 documentation/en/mining/miner-setup.md rename documentation/en/{ => mining}/mining-troubleshooting.md (90%) create mode 100644 documentation/en/mining/mining.md delete mode 100644 documentation/en/retrieving-data.md delete mode 100644 documentation/en/setting-a-static-port.md rename documentation/en/{storing-ipfs-integration.md => store/adding-from-ipfs.md} (79%) create mode 100644 documentation/en/store/making-deals.md create mode 100644 documentation/en/store/retrieve.md rename documentation/en/{storing-data-troubleshooting.md => store/storage-troubleshooting.md} (51%) create mode 100644 documentation/en/store/store.md delete mode 100644 documentation/en/storing-data.md rename documentation/en/{dev => unclassified}/WIP-arch-complementary-notes.md (100%) rename documentation/en/{ => unclassified}/block-validation.md (100%) rename documentation/en/{dev => unclassified}/create-miner.md (100%) rename documentation/en/{ => unclassified}/dev-tools-pond-ui.md (100%) rename documentation/en/{ => unclassified}/sealing-procs.md (100%) delete mode 100644 documentation/en/updating-lotus.md diff --git a/Makefile b/Makefile index 4f6ece417..2e91cfa65 100644 --- a/Makefile +++ b/Makefile @@ -280,7 +280,7 @@ method-gen: gen: type-gen method-gen docsgen: - go run ./api/docgen > documentation/en/api-methods.md + go run ./api/docgen > documentation/en/building/api-methods.md print-%: @echo $*=$($*) diff --git a/documentation/en/.library.json b/documentation/en/.library.json index 3fab0df9b..87c7353c1 100644 --- a/documentation/en/.library.json +++ b/documentation/en/.library.json @@ -1,214 +1,207 @@ { "posts": [ { - "title": "Hardware Requirements", - "slug": "en+hardware", - "github": "en/hardware.md", + "title": "About Lotus", + "slug": "en+lotus", + "github": "en/about.md", + "value": null, + "posts": [] + }, + { + "title": "Installation", + "slug": "en+install", + "github": "en/installation/installation.md", "value": null, "posts": [ - { - "title": "Testing Configuration", - "slug": "en+hardware-mining", - "github": "en/hardware-mining.md", - "value": null - } + { + "title": "Linux installation", + "slug": "en+install-linux", + "github": "en/installation/install-linux.md", + "value": null + }, + { + "title": "MacOS installation", + "slug": "en+install-macos", + "github": "en/installation/install-macos.md", + "value": null + }, + { + "title": "Updating Lotus", + "slug": "en+update", + "github": "en/installation/update.md", + "value": null + } ] }, { - "title": "Setup", + "title": "Getting started", "slug": "en+getting-started", - "github": "en/getting-started.md", + "github": "en/getting-started/getting-started.md", "value": null, "posts": [ - { - "title": "Arch Linux Installation", - "slug": "en+install-lotus-arch", - "github": "en/install-lotus-arch.md", - "value": null - }, - { - "title": "Ubuntu Installation", - "slug": "en+install-lotus-ubuntu", - "github": "en/install-lotus-ubuntu.md", - "value": null - }, - { - "title": "Fedora Installation", - "slug": "en+install-lotus-fedora", - "github": "en/install-lotus-fedora.md", - "value": null - }, - { - "title": "MacOS Installation", - "slug": "en+install-lotus-macos", - "github": "en/install-lotus-macos.md", - "value": null - }, - { - "title": "Updating Lotus", - "slug": "en+updating-lotus", - "github": "en/updating-lotus.md", - "value": null - }, - { - "title": "Join Testnet", - "slug": "en+join-testnet", - "github": "en/join-testnet.md", - "value": null - }, - { - "title": "Use Lotus with systemd", - "slug": "en+install-systemd-services", - "github": "en/install-systemd-services.md", - "value": null - }, - { - "title": "Setup Troubleshooting", - "slug": "en+setup-troubleshooting", - "github": "en/setup-troubleshooting.md", - "value": null - }, - { - "title": "Environment Variables", - "slug": "en+env-vars", - "github": "en/environment-vars.md", - "value": null - } + { + "title": "Setting up Lotus", + "slug": "en+setup", + "github": "en/getting-started/setup.md", + "value": null + }, + { + + "title": "Obtaining and sending FIL", + "slug": "en+wallet", + "github": "en/getting-started/wallet.md", + "value": null + }, + { + "title": "Setup troubleshooting", + "slug": "en+setup-troubleshooting", + "github": "en/getting-started/setup-troubleshooting.md", + "value": null + } ] }, { - "title": "Architecture", - "slug": "en+arch", - "github": "en/architecture.md", + "title": "Storing and retrieving data", + "slug": "en+store", + "github": "en/store/store.md", "value": null, "posts": [ - { - "title": "The Message Pool", - "slug": "en+mpool", - "github": "en/mpool.md", - "value": null - } + { + "title": "Making storage deals", + "slug": "en+making-deals", + "github": "en/store/making-deals.md", + "value": null + }, + { + "title": "Adding data from IPFS", + "slug": "en+adding-from-ipfs", + "github": "en/store/adding-from-ipfs.md", + "value": null + }, + { + "title": "Retrieving data", + "slug": "en+retriving", + "github": "en/store/retrieve.md", + "value": null + }, + { + "title": "Storage Troubleshooting", + "slug": "en+storage-troubleshooting", + "github": "en/store/storage-troubleshooting.md", + "value": null + } ] }, { - "title": "Storage Mining", + "title": "Storage mining", "slug": "en+mining", - "github": "en/mining.md", + "github": "en/mining/mining.md", "value": null, "posts": [ - { - "title": "Lotus Worker", - "slug": "en+lotus-worker", - "github": "en/mining-lotus-worker.md", - "value": null - }, - { - "title": "Static Ports", - "slug": "en+setting-a-static-port", - "github": "en/setting-a-static-port.md", - "value": null - }, - { - "title": "Mining Troubleshooting", - "slug": "en+mining-troubleshooting", - "github": "en/mining-troubleshooting.md", - "value": null - } + { + "title": "Miner setup", + "slug": "en+miner-setup", + "github": "en/mining/miner-setup.md", + "value": null + }, + { + "title": "Managing deals", + "slug": "en+managing-deals", + "github": "en/mining/managing-deals.md", + "value": null + }, + { + "title": "Lotus Worker", + "slug": "en+lotus-worker", + "github": "en/mining/lotus-seal-worker.md", + "value": null + }, + { + "title": "Benchmarking GPUs", + "slug": "en+gpus", + "github": "en/mining/gpus.md", + "value": null + }, + { + "title": "Mining Troubleshooting", + "slug": "en+mining-troubleshooting", + "github": "en/mining/mining-troubleshooting.md", + "value": null + } ] }, { - "title": "Storing Data", - "slug": "en+storing-data", - "github": "en/storing-data.md", + "title": "Building", + "slug": "en+building", + "github": "en/building/building.md", "value": null, "posts": [ - { - "title": "Storage Troubleshooting", - "slug": "en+storing-data-troubleshooting", - "github": "en/storing-data-troubleshooting.md", - "value": null - }, - { - "title": "Information for Miners", - "slug": "en+info-for-miners", - "github": "en/miner-deals.md", - "value": null - }, - { - "title": "IPFS Integration", - "slug": "en+ipfs-client-integration", - "github": "en/storing-ipfs-integration.md", - "value": null - } + { + "title": "Setting up remote API access", + "slug": "en+remote-api", + "github": "en/building/remote-api.md", + "value": null, + "posts": [] + }, + { + "title": "API endpoints and methods", + "slug": "en+api", + "github": "en/building/api.md", + "value": null, + "posts": [] + }, + { + "title": "API Reference", + "slug": "en+api-methods", + "github": "en/building/api-methods.md", + "value": null, + "posts": [] + }, + + { + "title": "Payment Channels", + "slug": "en+payment-channels", + "github": "en/building/payment-channels.md", + "value": null, + "posts": [] + }, + + { + "title": "Running a local devnet", + "slug": "en+local-devnet", + "github": "en/building/local-devnet.md", + "value": null, + "posts": [] + }, + { + "title": "Jaeger Tracing", + "slug": "en+jaeger-tracing", + "github": "en/building/jaeger-tracing.md", + "value": null, + "posts": [] + }, + + { + "title": "API Troubleshooting", + "slug": "en+api-troubleshooting", + "github": "en/building/api-troubleshooting.md", + "value": null, + "posts": [] + } ] }, { - "title": "Retrieving Data", - "slug": "en+retrieving-data", - "github": "en/retrieving-data.md", - "value": null, - "posts": [] - }, - { - "title": "Payment Channels", - "slug": "en+payment-channels", - "github": "en/payment-channels.md", - "value": null, - "posts": [] - }, - { - "title": "Command Line Interface", - "slug": "en+cli", - "github": "en/cli.md", - "value": null, - "posts": [] - }, - { - "title": "API", - "slug": "en+api", - "github": "en/api.md", + "title": "Lotus Architecture (WIP)", + "slug": "en+arch", + "github": "en/architectiure/architecture.md", "value": null, "posts": [ - { - "title": "Remote API Support", - "slug": "en+api-scripting-support", - "github": "en/api-scripting-support.md", - "value": null - }, - { - "title": "API Methods", - "slug": "en+api-methods", - "github": "en/api-methods.md", - "value": null - }, - { - "title": "API Troubleshooting", - "slug": "en+api-troubleshooting", - "github": "en/api-troubleshooting.md", - "value": null - } - ] - }, - { - "title": "Developer Tools", - "slug": "en+dev-tools", - "github": "en/dev-tools.md", - "value": null, - "posts": [ - { - "title": "Setup Local Devnet", - "slug": "en+setup-local-dev-net", - "github": "en/local-dev-net.md", - "value": null, - "posts": [] - }, - { - "title": "Jaeger Tracing", - "slug": "en+dev-tools-jaeger-tracing", - "github": "en/dev-tools-jaeger-tracing.md", - "value": null, - "posts": [] - } + { + "title": "The Message Pool", + "slug": "en+mpool", + "github": "en/architecture/mpool.md", + "value": null + } ] }, { @@ -224,7 +217,7 @@ "github": "en/.glossary.json", "value": null, "custom": { - "glossary": true + "glossary": true }, "posts": [] } diff --git a/documentation/en/about.md b/documentation/en/about.md new file mode 100644 index 000000000..ee8536ac9 --- /dev/null +++ b/documentation/en/about.md @@ -0,0 +1,14 @@ +# Lotus + +Lotus is an implementation of the **Filecoin Distributed Storage Network**. + +The **Lotus Node** (and the mining applications) can be built to join any of the [Filecoin networks](https://docs.filecoin.io/how-to/networks/). + +For more details about Filecoin, check out the [Filecoin Docs](https://docs.filecoin.io) and [Filecoin Spec](https://filecoin-project.github.io/specs/). + +## What can I learn here? + +* How to [install](en+installation) and [setup](en+setup) the Lotus software +* How to [store data on the Filecoin network](en+store) +* How to [setup a high performance FIL miner](en+miner-setup) +* How to [configure and access Lotus APIs](en+remote-api) diff --git a/documentation/en/api-scripting-support.md b/documentation/en/api-scripting-support.md deleted file mode 100644 index 653f144ed..000000000 --- a/documentation/en/api-scripting-support.md +++ /dev/null @@ -1,25 +0,0 @@ -# Remote API Support - -You may want to delegate the work **Lotus Miner** or **Lotus Node** performs to other machines. -Here is how to setup the necessary authorization and environment variables. - -## Environment variables - -Environmental variables are variables that are defined for the current shell and are inherited by any child shells or processes. Environmental variables are used to pass information into processes that are spawned from the shell. - -Using the [JWT you generated](https://lotu.sh/en+api#how-do-i-generate-a-token-18865), you can assign it and the **multiaddr** to the appropriate environment variable. - -```sh -# Lotus Node -FULLNODE_API_INFO="JWT_TOKEN:/ip4/127.0.0.1/tcp/1234/http" - -# Lotus Miner -MINER_API_INFO="JWT_TOKEN:/ip4/127.0.0.1/tcp/2345/http" -``` - -You can also use `lotus auth api-info --perm admin` to quickly create _API_INFO env vars - -- The **Lotus Node**'s `mutliaddr` is in `~/.lotus/api`. -- The default token is in `~/.lotus/token`. -- The **Lotus Miner**'s `multiaddr` is in `~/.lotusminer/config`. -- The default token is in `~/.lotusminer/token`. diff --git a/documentation/en/api.md b/documentation/en/api.md deleted file mode 100644 index 9760e2f32..000000000 --- a/documentation/en/api.md +++ /dev/null @@ -1,85 +0,0 @@ -# API - -Here is an early overview of how to make API calls. - -Implementation details for the **JSON-RPC** package are [here](https://github.com/filecoin-project/go-jsonrpc). - -## Overview: How do you modify the config.toml to change the API endpoint? - -API requests are made against `127.0.0.1:1234` unless you modify `.lotus/config.toml`. - -Options: - -- `http://[api:port]/rpc/v0` - HTTP endpoint -- `ws://[api:port]/rpc/v0` - Websocket endpoint -- `PUT http://[api:port]/rest/v0/import` - File import, it requires write permissions. - -## What methods can I use? - -For now, you can look into different files to find methods available to you based on your needs: - -- [Both Lotus node + miner APIs](https://github.com/filecoin-project/lotus/blob/master/api/api_common.go) -- [Lotus node API](https://github.com/filecoin-project/lotus/blob/master/api/api_full.go) -- [Lotus miner API](https://github.com/filecoin-project/lotus/blob/master/api/api_storage.go) - -The necessary permissions for each are in [api/struct.go](https://github.com/filecoin-project/lotus/blob/master/api/struct.go). - -## How do I make an API request? - -To demonstrate making an API request, we will take the method `ChainHead` from [api/api_full.go](https://github.com/filecoin-project/lotus/blob/master/api/api_full.go). - -```go -ChainHead(context.Context) (*types.TipSet, error) -``` - -And create a CURL command. In this command, `ChainHead` is included as `{ "method": "Filecoin.ChainHead" }`: - -```sh -curl -X POST \ - -H "Content-Type: application/json" \ - --data '{ "jsonrpc": "2.0", "method": "Filecoin.ChainHead", "params": [], "id": 3 }' \ - 'http://127.0.0.1:1234/rpc/v0' -``` - -If the request requires authorization, add an authorization header: - -```sh -curl -X POST \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer $(cat ~/.lotusminer/token)" \ - --data '{ "jsonrpc": "2.0", "method": "Filecoin.ChainHead", "params": [], "id": 3 }' \ - 'http://127.0.0.1:1234/rpc/v0' -``` - -> In the future we will add a playground to make it easier to build and experiment with API requests. - -## CURL authorization - -To authorize your request, you will need to include the **JWT** in a HTTP header, for example: - -```sh --H "Authorization: Bearer $(cat ~/.lotusminer/token)" -``` - -Admin token is stored in `~/.lotus/token` for the **Lotus Node** or `~/.lotusminer/token` for the **Lotus Miner**. - -## How do I generate a token? - -To generate a JWT with custom permissions, use this command: - -```sh -# Lotus Node -lotus auth create-token --perm admin - -# Lotus Miner -lotus-miner auth create-token --perm admin -``` - -## What authorization level should I use? - -When viewing [api/apistruct/struct.go](https://github.com/filecoin-project/lotus/blob/master/api/apistruct/struct.go), you will encounter these types: - -- `read` - Read node state, no private data. -- `write` - Write to local store / chain, and `read` permissions. -- `sign` - Use private keys stored in wallet for signing, `read` and `write` permissions. -- `admin` - Manage permissions, `read`, `write`, and `sign` permissions. diff --git a/documentation/en/architecture.md b/documentation/en/architecture/architecture.md similarity index 99% rename from documentation/en/architecture.md rename to documentation/en/architecture/architecture.md index 619e04f05..8c4d7be5c 100644 --- a/documentation/en/architecture.md +++ b/documentation/en/architecture/architecture.md @@ -6,7 +6,7 @@ Filecoin protocol, validating the blocks and state transitions. The specification for the Filecoin protocol can be found [here](https://filecoin-project.github.io/specs/). For information on how to setup and operate a Lotus node, -please follow the instructions [here](https://lotu.sh/en+getting-started). +please follow the instructions [here](en+getting-started). # Components diff --git a/documentation/en/mpool.md b/documentation/en/architecture/mpool.md similarity index 100% rename from documentation/en/mpool.md rename to documentation/en/architecture/mpool.md diff --git a/documentation/en/api-methods.md b/documentation/en/building/api-methods.md similarity index 100% rename from documentation/en/api-methods.md rename to documentation/en/building/api-methods.md diff --git a/documentation/en/api-troubleshooting.md b/documentation/en/building/api-troubleshooting.md similarity index 100% rename from documentation/en/api-troubleshooting.md rename to documentation/en/building/api-troubleshooting.md diff --git a/documentation/en/building/api.md b/documentation/en/building/api.md new file mode 100644 index 000000000..626193ee2 --- /dev/null +++ b/documentation/en/building/api.md @@ -0,0 +1,38 @@ +# API endpoints and methods + +The API can be accessed on: + +- `http://[api:port]/rpc/v0` - HTTP RPC-API endpoint +- `ws://[api:port]/rpc/v0` - Websocket RPC-API endpoint +- `PUT http://[api:port]/rest/v0/import` - REST endpoint for file import (multipart upload). It requires write permissions. + +The RPC methods can be found in the [Reference](en+api-methods) and directly in the source code: + +- [Both Lotus node + miner APIs](https://github.com/filecoin-project/lotus/blob/master/api/api_common.go) +- [Lotus node API](https://github.com/filecoin-project/lotus/blob/master/api/api_full.go) +- [Lotus miner API](https://github.com/filecoin-project/lotus/blob/master/api/api_storage.go) + + +## JSON-RPC client + +Lotus uses its own Go library implementation of [JSON-RPC](https://github.com/filecoin-project/go-jsonrpc). + +## cURL example + +To demonstrate making an API request, we will take the method `ChainHead` from [api/api.go](https://github.com/filecoin-project/lotus/blob/master/api/api_full.go). + +```go +ChainHead(context.Context) (*types.TipSet, error) +``` + +And create a CURL command. In this command, `ChainHead` is included as `{ "method": "Filecoin.ChainHead" }`: + +```sh +curl -X POST \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $(cat ~/.lotusminer/token)" \ + --data '{ "jsonrpc": "2.0", "method": "Filecoin.ChainHead", "params": [], "id": 3 }' \ + 'http://127.0.0.1:1234/rpc/v0' +``` + +(See [this section](en+remote-api) to learn how to generate authorization tokens). diff --git a/documentation/en/building/building.md b/documentation/en/building/building.md new file mode 100644 index 000000000..5194f8314 --- /dev/null +++ b/documentation/en/building/building.md @@ -0,0 +1,5 @@ +# Building with Lotus + +Lotus applications provide HTTP (JSON-RPC) APIs that allow developers to control Lotus programatically. + +This section dives into how to setup and use these APIs, additionally providing information on advanced Lotus features and workflows, like Payment Channels or how to setup a fully local Lotus development network. diff --git a/documentation/en/dev-tools-jaeger-tracing.md b/documentation/en/building/jaeger-tracing.md similarity index 100% rename from documentation/en/dev-tools-jaeger-tracing.md rename to documentation/en/building/jaeger-tracing.md diff --git a/documentation/en/local-dev-net.md b/documentation/en/building/local-devnet.md similarity index 100% rename from documentation/en/local-dev-net.md rename to documentation/en/building/local-devnet.md diff --git a/documentation/en/payment-channels.md b/documentation/en/building/payment-channels.md similarity index 100% rename from documentation/en/payment-channels.md rename to documentation/en/building/payment-channels.md diff --git a/documentation/en/building/remote-api.md b/documentation/en/building/remote-api.md new file mode 100644 index 000000000..d0fedb51b --- /dev/null +++ b/documentation/en/building/remote-api.md @@ -0,0 +1,69 @@ +# Setting up remote API access + +The **Lotus Miner** and the **Lotus Node** applications come with their own local API endpoints setup by default when they are running. + +These endpoints are used by `lotus` and `lotus-miner` to interact with the running process. In this section we will explain how to enable remote access to the Lotus APIs. + +Note that instructions are the same for `lotus` and `lotus-miner`. For simplicity, we will just show how to do it with `lotus`. + +## Setting the listening interface for the API endpoint + +By default, the API listens on the local "loopback" interface (`127.0.0.1`). This is configured in the `config.toml` file: + +```toml +[API] +# ListenAddress = "/ip4/127.0.0.1/tcp/1234/http" +# RemoteListenAddress = "" +# Timeout = "30s" +``` + +To access the API remotely, Lotus needs to listen on the right IP/interface. The IP associated to each interface can be usually found with the command `ip a`. Once the right IP is known, it can be set in the configuration: + +```toml +[API] +ListenAddress = "/ip4//tcp/3453/http" # port is an example + +# Only relevant for lotus-miner +# This should be the IP:Port pair where the miner is reachable from anyone trying to dial to it. +# If you have placed a reverse proxy or a NAT'ing device in front of it, this may be different from +# the EXTERNAL_INTERFACE_IP. +RemoteListenAddress = "" +``` + +> `0.0.0.0` can be used too. This is a wildcard that means "all interfaces". Depending on the network setup, this may affect security (listening on the wrong, exposed interface). + +After making these changes, please restart the affected process. + +## Issuing tokens + +Any client wishing to talk to the API endpoints will need a token. Tokens can be generated with: + +```sh +lotus auth create-token --perm +``` + +(similarly for the Lotus Miner). + +The permissions work as follows: + +- `read` - Read node state, no private data. +- `write` - Write to local store / chain, and `read` permissions. +- `sign` - Use private keys stored in wallet for signing, `read` and `write` permissions. +- `admin` - Manage permissions, `read`, `write`, and `sign` permissions. + + +Tokens can then be used in applications by setting an Authorization header as: + +``` +Authorization: Bearer +``` + + +## Environment variables + +`lotus`, `lotus-miner` and `lotus-worker` can actually interact with their respective applications running on a different node. All is needed to configure them are the following the *environment variables*: + +```sh +FULLNODE_API_INFO="TOKEN:/ip4//tcp//http" +MINER_API_INFO="TOKEN:/ip4//tcp//http" +``` diff --git a/documentation/en/cli.md b/documentation/en/cli.md deleted file mode 100644 index fd26400d0..000000000 --- a/documentation/en/cli.md +++ /dev/null @@ -1,108 +0,0 @@ -# Lotus Command Line Interface - -The Command Line Interface (CLI) is a convenient way to interact with -a Lotus node. You can use the CLI to operate your node, -get information about the blockchain, -manage your accounts and transfer funds, -create storage deals, and much more! - -The CLI is intended to be self-documenting, so when in doubt, simply add `--help` -to whatever command you're trying to run! This will also display all of the -input parameters that can be provided to a command. - -We highlight some of the commonly -used features of the CLI below. -All CLI commands should be run from the home directory of the Lotus project. - -## Operating a Lotus node - -### Starting up a node - -```sh -lotus daemon -``` -This command will start up your Lotus node, with its API port open at 1234. -You can pass `--api=` to use a different port. - -### Checking your sync progress - -```sh -lotus sync status -``` -This command will print your current tipset height under `Height`, and the target tipset height -under `Taregt`. - -You can also run `lotus sync wait` to get constant updates on your sync progress. - -### Getting the head tipset - -```sh -lotus chain head -``` - -### Control the logging level - -```sh -lotus log set-level -``` -This command can be used to toggle the logging levels of the different -systems of a Lotus node. In decreasing order -of logging detail, the levels are `debug`, `info`, `warn`, and `error`. - -As an example, -to set the `chain` and `blocksync` to log at the `debug` level, run -`lotus log set-level --system chain --system blocksync debug`. - -To see the various logging system, run `lotus log list`. - -### Find out what version of Lotus you're running - -```sh -lotus version -``` - -## Managing your accounts - -### Listing accounts in your wallet - -```sh -lotus wallet list -``` - -### Creating a new account - -```sh -lotus wallet new bls -``` -This command will create a new BLS account in your wallet; these -addresses start with the prefix `t3`. Running `lotus wallet new secp256k1` -(or just `lotus wallet new`) will create -a new Secp256k1 account, which begins with the prefix `t1`. - -### Getting an account's balance - -```sh -lotus wallet balance
-``` - -### Transferring funds - -```sh -lotus send --source= -``` -This command will transfer `amount` (in attoFIL) from `source address` to `destination address`. - -### Importing an account into your wallet - -```sh -lotus wallet import -``` -This command will import an account whose private key is saved at the specified file. - -### Exporting an account from your wallet - -```sh -lotus wallet export
-``` -This command will print out the private key of the specified address -if it is in your wallet. Always be careful with your private key! diff --git a/documentation/en/dev-tools.md b/documentation/en/dev-tools.md deleted file mode 100644 index 60b9b26d4..000000000 --- a/documentation/en/dev-tools.md +++ /dev/null @@ -1,3 +0,0 @@ -# Developer Tools - -> Running a local network can be a great way to understand how Lotus works and test your setup. diff --git a/documentation/en/environment-vars.md b/documentation/en/environment-vars.md deleted file mode 100644 index 9d455a74d..000000000 --- a/documentation/en/environment-vars.md +++ /dev/null @@ -1,65 +0,0 @@ -# Lotus Environment Variables - -## Building - -## Common - -The environment variables are common across most lotus binaries. - -### `LOTUS_FD_MAX` - -Sets the file descriptor limit for the process. This should be set high (8192 -or higher) if you ever notice 'too many open file descriptor' errors. - -### `LOTUS_JAEGER` - -This can be set to enable jaeger trace reporting. The value should be the url -of the jaeger trace collector, the default for most jaeger setups should be -`localhost:6831`. - -### `LOTUS_DEV` - -If set to a non-empty value, certain parts of the application will print more -verbose information to aid in development of the software. Not recommended for -end users. - -## Lotus Daemon - -### `LOTUS_PATH` - -Sets the location for the lotus daemon on-disk repo. If left empty, this defaults to `~/.lotus`. - -### `LOTUS_SKIP_GENESIS_CHECK` - -Can be set to `_yes_` if you wish to run a lotus network with a different -genesis than the default one built into your lotus binary. - -### `LOTUS_CHAIN_TIPSET_CACHE` - -Sets the cache size for the chainstore tipset cache. The default value is 8192, -but if your usage of the lotus API involves frequent arbitrary tipset lookups, -you may want to increase this. - -### `LOTUS_CHAIN_INDEX_CACHE` - -Sets the cache size for the chainstore epoch index cache. The default value is 32768, -but if your usage of the lotus API involves frequent deep chain lookups for -block heights that are very far from the current chain height, you may want to -increase this. - - -### `LOTUS_BSYNC_MSG_WINDOW` - -Set the initial maximum window size for message fetching blocksync requests. If -you have a slower internet connection and are having trouble syncing, you might -try lowering this down to 10-20 for a 'poor' internet connection. - -## Lotus Miner - -A number of environment variables are respected for configuring the behavior of the filecoin proving subsystem. For more details on those [see here](https://github.com/filecoin-project/rust-fil-proofs/#settings). - -### `LOTUS_MINER_PATH` - -Sets the location for the lotus miners on-disk repo. If left empty, this defaults to `~/.lotusminer`. - - diff --git a/documentation/en/faqs.md b/documentation/en/faqs.md index c2d526830..74119a5b6 100644 --- a/documentation/en/faqs.md +++ b/documentation/en/faqs.md @@ -11,7 +11,6 @@ go [here](https://filecoin.io/faqs/). Lotus is an implementation of the **Filecoin Distributed Storage Network**, written in Go. It is designed to be modular and interoperable with any other implementation of the Filecoin Protocol. -More information about Lotus can be found [here](https://lotu.sh/). ### What are the components of Lotus? @@ -30,21 +29,19 @@ to a Lotus Node over the JSON-RPC API. ### How do I set up a Lotus Node? -Follow the instructions found [here](https://lotu.sh/en+getting-started). +Follow the instructions found [here](en+install) and [here](en+setup). ### Where can I get the latest version of Lotus? -Download the binary tagged as the `Latest Release` from the - [Lotus Github repo](https://github.com/filecoin-project/lotus/releases). +Download the binary tagged as the `Latest Release` from the [Lotus Github repo](https://github.com/filecoin-project/lotus/releases) or checkout the `master` branch of the source repository. ### What operating systems can Lotus run on? -Lotus can build and run on most Linux and MacOS systems with at least -8GB of RAM. Windows is not yet supported. +Lotus can build and run on most Linux and MacOS systems with [at least 8GB of RAM](en+install#hardware-requirements-1). Windows is not yet supported. ### How can I update to the latest version of Lotus? -To update Lotus, follow the instructions [here](https://lotu.sh/en+updating-lotus). +To update Lotus, follow the instructions [here](en+update). ### How do I prepare a fresh installation of Lotus? @@ -52,7 +49,7 @@ Stop the Lotus daemon, and delete all related files, including sealed and chain running `rm ~/.lotus ~/.lotusminer`. Then, install Lotus afresh by following the instructions -found [here](https://lotu.sh/en+getting-started). +found [here](en+install). ### Can I configure where the node's config and data goes? @@ -73,48 +70,45 @@ directory for more. ### How can I send a request over the JSON-RPC API? Information on how to send a `cURL` request to the JSON-RPC API can be found -[here](https://lotu.sh/en+api). A JavaScript client is under development. +[here](en+api). ### What are the requests I can send over the JSON-RPC API? -Please have a look at the -[source code](https://github.com/filecoin-project/lotus/blob/master/api/api_full.go) -for a list of methods supported by the JSON-RPC API. +Please have a look [here](en+api). + + ## The Test Network ### What is Testnet? Testnet is a live network of Lotus Nodes run by the community for testing purposes. - It has 2 PiB of storage (and growing!) dedicated to it. ### Is FIL on the Testnet worth anything? -Nothing at all! Real-world incentives may be provided in a future phase of Testnet, but this is -yet to be confirmed. +Nothing at all! ### How can I see the status of Testnet? The [dashboard](https://stats.testnet.filecoin.io/) displays the status of the network as -well as a ton -of other metrics you might find interesting. +well as a ton of other metrics you might find interesting. ## Mining with a Lotus Node on Testnet ### How do I get started mining with Lotus? -Follow the instructions found [here](https://lotu.sh/en+mining). +Follow the instructions found [here](en+mining). ### What are the minimum hardware requirements? An example test configuration, and minimum hardware requirements can be found -[here](https://lotu.sh/en+hardware-mining). +[here](en+install#hardware-requirements-8). Note that these might NOT be the minimum requirements for mining on Mainnet. ### What are some GPUs that have been tested? -A list of benchmarked GPUs can be found [here](https://lotu.sh/en+hardware-mining#benchmarked-gpus-7393). +See previous question. ### Why is my GPU not being used when sealing a sector? @@ -135,4 +129,4 @@ You can do so by changing the storage path variable for the second miner, e.g., ### How do I setup my own local devnet? -Follow the instructions found [here](https://lotu.sh/en+setup-local-dev-net). +Follow the instructions found [here](en+local-devnet). diff --git a/documentation/en/getting-started.md b/documentation/en/getting-started.md deleted file mode 100644 index e38a2ab97..000000000 --- a/documentation/en/getting-started.md +++ /dev/null @@ -1,23 +0,0 @@ -# Lotus - -Lotus is an implementation of the **Filecoin Distributed Storage Network**. You can run the Lotus software client to join the **Filecoin Testnet**. - -For more details about Filecoin, check out the [Filecoin Docs](https://docs.filecoin.io) and [Filecoin Spec](https://filecoin-project.github.io/specs/). - -## What can I learn here? - -- How to install Lotus on [Arch Linux](https://lotu.sh/en+install-lotus-arch), [Ubuntu](https://lotu.sh/en+install-lotus-ubuntu), or [MacOS](https://lotu.sh/en+install-lotus-macos). -- Joining the [Lotus Testnet](https://lotu.sh/en+join-testnet). -- [Storing](https://lotu.sh/en+storing-data) or [retrieving](https://lotu.sh/en+retrieving-data) data. -- Mining Filecoin using the **Lotus Miner** in your [CLI](https://lotu.sh/en+mining). - -## How is Lotus designed? - -Lotus is architected modularly to keep clean API boundaries while using the same process. Installing Lotus will include two separate programs: - -- The **Lotus Node** -- The **Lotus Miner** - -The **Lotus Miner** is intended to be run on the machine that manages a single miner instance, and is meant to communicate with the **Lotus Node** via the websocket **JSON-RPC** API for all of the chain interaction needs. - -This way, a mining operation may easily run a **Lotus Miner** or many of them, connected to one or many **Lotus Node** instances. diff --git a/documentation/en/getting-started/getting-started.md b/documentation/en/getting-started/getting-started.md new file mode 100644 index 000000000..99b4095d4 --- /dev/null +++ b/documentation/en/getting-started/getting-started.md @@ -0,0 +1,3 @@ +# Getting started + +This section will get you started with Lotus. We will setup the Lotus daemon (that should already be [installed](en+install)), start it, create a wallet and use it to send and receive some Filecoin. diff --git a/documentation/en/setup-troubleshooting.md b/documentation/en/getting-started/setup-troubleshooting.md similarity index 60% rename from documentation/en/setup-troubleshooting.md rename to documentation/en/getting-started/setup-troubleshooting.md index a1c78b51b..f27a3faa5 100644 --- a/documentation/en/setup-troubleshooting.md +++ b/documentation/en/getting-started/setup-troubleshooting.md @@ -1,5 +1,12 @@ # Setup Troubleshooting + +## Error: initializing node error: cbor input had wrong number of fields + +This happens when you are starting Lotus which has been compiled for one network, but it encounters data in the Lotus data folder which is for a different network, or for an older incompatible version. + +The solution is to clear the data folder (see below). + ## Config: Clearing data Here is a command that will delete your chain data, stored wallets, stored data and any miners you have set up: @@ -8,7 +15,7 @@ Here is a command that will delete your chain data, stored wallets, stored data rm -rf ~/.lotus ~/.lotusminer ``` -This command usually resolves any issues with running `lotus` but it is not always required for updates. We will share information about when resetting your chain data and miners is required for an update in the future. +Note you do not always need to clear your data for [updating](en+update). ## Error: Failed to connect bootstrap peer @@ -33,6 +40,8 @@ ERROR hello hello/hello.go:81 other peer has different genesis! ## Config: Open files limit +Lotus will attempt to set up the file descriptor (FD) limit automatically. If that does not work, you can still configure your system to allow higher than the default values. + On most systems you can check the open files limit with: ```sh @@ -44,3 +53,5 @@ You can also modify this number by using the `ulimit` command. It gives you the ```sh ulimit -n 10000 ``` + +Note that this is not persisted and that systemd manages its own FD limits for services. Please use your favourite search engine to find instructions on how to persist and configure FD limits for your system. diff --git a/documentation/en/getting-started/setup.md b/documentation/en/getting-started/setup.md new file mode 100644 index 000000000..e751da80b --- /dev/null +++ b/documentation/en/getting-started/setup.md @@ -0,0 +1,169 @@ +# Setting up Lotus + +Your Lotus binaries have been installed and you are ready to start participating in the Filecoin network. + +## Selecting the right network + +You should have built the Lotus binaries from the right Github branch and Lotus will be fully setup to join the matching [Filecoin network](https://docs.filecoin.io/how-to/networks/). For more information on switching networks, check the [updating Lotus section](en+update). + +## Starting the daemon + +To start the daemon simply run: + +```sh +lotus daemon +``` + +or if you are using the provided systemd service files, do: + +```sh +systemctl start lotus-daemon +``` + +__If you are using Lotus from China__, make sure you set the following environment variable before running Lotus: + +``` +export IPFS_GATEWAY="https://proof-parameters.s3.cn-south-1.jdcloud-oss.com/ipfs/" +``` + + +During the first start, Lotus: + +* Will setup its data folder at `~/.lotus` +* Will download the necessary parameters +* Start syncing the Lotus chain + +If you started lotus using systemd, the logs will appear in `/var/log/lotus/daemon.log` (not in journalctl as usual), otherwise you will see them in your screen. + +Do not be appalled by the amount of warnings and sometimes errors showing in the logs, there are usually part of the usual functioning of the daemon as part of a distributed network. + +## Waiting to sync + +After the first start, the chain will start syncing until it has reached the tip. You can check how far the syncing process is with: + +```sh +lotus sync status +``` + +You can also interactively wait for the chain to be fully synced with: + +```sh +lotus sync wait +``` + +## Interacting with the Lotus daemon + +As shown above, the `lotus` command allows to interact with the running daemon. You will see it getting used in many of the documentation examples. + +This command-line-interface is self-documenting: + +```sh +# Show general help +lotus --help +# Show specific help for the "client" subcommand +lotus client --help +``` + +For example, after your Lotus daemon has been running for a few minutes, use `lotus` to check the number of other peers that it is connected to in the Filecoin network: + +```sh +lotus net peers +``` + +## Controlling the logging level + +```sh +lotus log set-level +``` +This command can be used to toggle the logging levels of the different +systems of a Lotus node. In decreasing order +of logging detail, the levels are `debug`, `info`, `warn`, and `error`. + +As an example, +to set the `chain` and `blocksync` to log at the `debug` level, run +`lotus log set-level --system chain --system blocksync debug`. + +To see the various logging system, run `lotus log list`. + + +## Configuration + +### Configuration file + +The Lotus daemon stores a configuration file in `~/.lotus/config.toml`. Note that by default all settings are commented. Here is an example configuration: + +```toml +[API] + # Binding address for the Lotus API + ListenAddress = "/ip4/127.0.0.1/tcp/1234/http" + # Not used by lotus daemon + RemoteListenAddress = "" + # General network timeout value + Timeout = "30s" + +# Libp2p provides connectivity to other Filecoin network nodes +[Libp2p] + # Binding address swarm - 0 means random port. + ListenAddresses = ["/ip4/0.0.0.0/tcp/0", "/ip6/::/tcp/0"] + # Insert any addresses you want to explicitally + # announce to other peers here. Otherwise, they are + # guessed. + AnnounceAddresses = [] + # Insert any addresses to avoid announcing here. + NoAnnounceAddresses = [] + # Connection manager settings, decrease if your + # machine is overwhelmed by connections. + ConnMgrLow = 150 + ConnMgrHigh = 180 + ConnMgrGrace = "20s" + +# Pubsub is used to broadcast information in the network +[Pubsub] + Bootstrapper = false + RemoteTracer = "/dns4/pubsub-tracer.filecoin.io/tcp/4001/p2p/QmTd6UvR47vUidRNZ1ZKXHrAFhqTJAD27rKL9XYghEKgKX" + +# This section can be used to enable adding and retriving files from IPFS +[Client] + UseIpfs = false + IpfsMAddr = "" + IpfsUseForRetrieval = false + +# Metrics configuration +[Metrics] + Nickname = "" + HeadNotifs = false +``` + +### Ensuring connectivity to your Lotus daemon + +Usually your lotus daemon will establish connectivity with others in the network and try to make itself diallable using uPnP. If you wish to manually ensure that your daemon is reachable: + +* Set a fixed port of your choice in the `ListenAddresses` in the Libp2p section (i.e. 6665). +* Open a port in your router that is forwarded to this port. This is usually called featured as "Port forwarding" and the instructions differ from router model to model but there are many guides online. +* Add your public IP/port to `AnnounceAddresses`. i.e. `/ip4//tcp/6665/`. + +Note that it is not a requirement to use Lotus as a client to the network to be fully reachable, as your node already connects to others directly. + + +### Environment variables + +Common to most Lotus binaries: + +* `LOTUS_FD_MAX`: Sets the file descriptor limit for the process +* `LOTUS_JAEGER`: Sets the Jaeger URL to send traces. See TODO. +* `LOTUS_DEV`: Any non-empty value will enable more verbose logging, useful only for developers. + +Specific to the *Lotus daemon*: + +* `LOTUS_PATH`: Location to store Lotus data (defaults to `~/.lotus`). +* `LOTUS_SKIP_GENESIS_CHECK=_yes_`: Set only if you wish to run a lotus network with a different genesis block. +* `LOTUS_CHAIN_TIPSET_CACHE`: Sets the size for the chainstore tipset cache. Defaults to `8192`. Increase if you perform frequent arbitrary tipset lookups. +* `LOTUS_CHAIN_INDEX_CACHE`: Sets the size for the epoch index cache. Defaults to `32768`. Increase if you perform frequent deep chain lookups for block heights far from the latest height. +* `LOTUS_BSYNC_MSG_WINDOW`: Set the initial maximum window size for message fetching blocksync request. Set to 10-20 if you have an internet connection with low bandwidth. + +Specific to the *Lotus miner*: + +* `LOTUS_MINER_PATH`: Location for the miner's on-disk repo. Defaults to `./lotusminer`. +* A number of environment variables are respected for configuring the behaviour of the Filecoin proving subsystem. [See here](en+miner-setup). + + diff --git a/documentation/en/getting-started/wallet.md b/documentation/en/getting-started/wallet.md new file mode 100644 index 000000000..25a67fb09 --- /dev/null +++ b/documentation/en/getting-started/wallet.md @@ -0,0 +1,58 @@ +# Obtaining and sending FIL + +In order to receive and send FIL with Lotus you will need to have installed the program and be running the Lotus daemon. + +## Creating a wallet + + +```sh +lotus wallet new bls +``` + +This will print your Filecoin address. + +Your wallet information is stored in the `~/.lotus/keystore` (or `$LOTUS_PATH/keystore`). For instructions on export/import, see below. + +You can create multiple wallets and list them with: + +```sh +lotus wallet list +``` + +## Obtaining FIL + +FIL can be obtained either by using one of the Faucets (available for the test networks) or by buying it from an exchange supporting FIL trading (once mainnet has launched). + +Once you have received some FIL you can check your balance with: + +```sh +lotus wallet balance +``` + +Remember that your will only see the latest balance when your daemon is fully synced to the chain. + +## Sending FIL + +Sending some FIL can be achieved by running: + +```sh +lotus wallet send
+``` + +Make sure to check `lotus wallet send --help` for additional options. + +## Exporting and importing a wallet + +You can export and re-import a wallet with: + +```sh +lotus wallet export
> wallet.private +``` + +and: + +```sh +lotus wallet import wallet.private +``` + +Keep your wallet's private key safe! diff --git a/documentation/en/hardware-mining.md b/documentation/en/hardware-mining.md deleted file mode 100644 index d421f6fb1..000000000 --- a/documentation/en/hardware-mining.md +++ /dev/null @@ -1,54 +0,0 @@ -# Protocol Labs Standard Testing Configuration - -> This documentation page describes the standard testing configuration the Protocol Labs team has used to test **Lotus Miner**s on Lotus. There is no guarantee this testing configuration will be suitable for Filecoin storage mining at MainNet launch. If you need to buy new hardware to join the Filecoin Testnet, we recommend to buy no more hardware than you require for testing. To learn more please read this [Protocol Labs Standard Testing Configuration post](https://filecoin.io/blog/filecoin-testnet-mining/). - -**Sector sizes** and **minimum pledged storage** required to mine blocks are two very important Filecoin Testnet parameters that impact hardware decisions. We will continue to refine all parameters during Testnet. - -BECAUSE OF THIS, OUR STANDARD TESTING CONFIGURATION FOR FILECOIN MAINNET CAN AND WILL CHANGE. YOU HAVE BEEN WARNED. - -## Example configuration - -The setup below is a minimal example for sealing 32 GiB sectors on Lotus: - -- 2 TB of hard drive space. -- 8 core CPU -- 128 GiB of RAM - -Note that 1GB sectors don't require as high of specs, but are likely to be removed as we improve the performance of 32GB sector sealing. - -For the first part of the sealing process, AMD CPU's are __highly recommended__, because of the `Intel SHA Extensions` instruction set that is available there ever since the `Zen` microarchitecture. Hence, AMD CPU's seem to perform much better on the testnet than other CPU's. Contrary to what the name implies, this extended instruction set is not available on recent Intel desktop/server chips. - -## Testnet discoveries - -- If you only have 128GiB of ram, enabling 256GB of **NVMe** swap on an SSD will help you avoid out-of-memory issues while mining. - -## Benchmarked GPUs - -GPUs are a must for getting **block rewards**. Here are a few that have been confirmed to generate **SNARKs** quickly enough to successfully mine blocks on the Lotus Testnet. - -- GeForce RTX 2080 Ti -- GeForce RTX 2080 SUPER -- GeForce RTX 2080 -- GeForce GTX 1080 Ti -- GeForce GTX 1080 -- GeForce GTX 1060 - -## Testing other GPUs - -If you want to test a GPU that is not explicitly supported, use the following global **environment variable**: - -```sh -BELLMAN_CUSTOM_GPU=":" -``` - -Here is an example of trying a GeForce GTX 1660 Ti with 1536 cores. - -```sh -BELLMAN_CUSTOM_GPU="GeForce GTX 1660 Ti:1536" -``` - -To get the number of cores for your GPU, you will need to check your card’s specifications. - -## Benchmarking - -Here is a [benchmarking tool](https://github.com/filecoin-project/lotus/tree/master/cmd/lotus-bench) and a [GitHub issue thread](https://github.com/filecoin-project/lotus/issues/694) for those who wish to experiment with and contribute hardware setups for the **Filecoin Testnet**. diff --git a/documentation/en/hardware.md b/documentation/en/hardware.md deleted file mode 100644 index f6250548a..000000000 --- a/documentation/en/hardware.md +++ /dev/null @@ -1,7 +0,0 @@ -# Hardware - -> This page is a work in progress. Exact mining requirements are still in the works. - -Lotus can build and run on most [Linux](https://ubuntu.com/) and [MacOS](https://www.apple.com/macos) systems with at least 8GiB of RAM. - -Windows is not yet supported. diff --git a/documentation/en/install-lotus-arch.md b/documentation/en/install-lotus-arch.md deleted file mode 100644 index 8e06aae4e..000000000 --- a/documentation/en/install-lotus-arch.md +++ /dev/null @@ -1,51 +0,0 @@ -# Arch Linux Instructions - -These steps will install the following dependencies: - -- go (1.14 or higher) -- gcc (7.4.0 or higher) -- git (version 2 or higher) -- bzr (some go dependency needs this) -- jq -- pkg-config -- opencl-icd-loader -- opencl driver (like nvidia-opencl on arch) (for GPU acceleration) -- opencl-headers (build) -- rustup (proofs build) -- llvm (proofs build) -- clang (proofs build) - -### Install dependencies - -```sh -sudo pacman -Syu opencl-icd-loader gcc git bzr jq pkg-config opencl-icd-loader opencl-headers -``` - -### Install Go 1.14 - -Install the latest version of Go by following [the docs on their website](https://golang.org/doc/install). - -### Clone the Lotus repository - -```sh -git clone https://github.com/filecoin-project/lotus.git -cd lotus/ -``` - -### Build the Lotus binaries from source and install - -! **If you are running an AMD platform or if your CPU supports SHA extensions you will want to build the Filecoin proofs natively** - -```sh -make clean && make all -sudo make install -``` - -#### Native Filecoin FFI building - -```sh -env env RUSTFLAGS="-C target-cpu=native -g" FFI_BUILD_FROM_SOURCE=1 make clean deps all -sudo make install -``` - -After installing Lotus, you can run the `lotus` command directly from your CLI to see usage documentation. Next, you can join the [Lotus Testnet](https://lotu.sh/en+join-testnet). diff --git a/documentation/en/install-lotus-fedora.md b/documentation/en/install-lotus-fedora.md deleted file mode 100644 index c37161b7a..000000000 --- a/documentation/en/install-lotus-fedora.md +++ /dev/null @@ -1,54 +0,0 @@ -# Fedora Instructions - -> tested on 30 - -**NOTE:** If you have an AMD GPU the opencl instructions may be incorrect... - -These steps will install the following dependencies: - -- go (1.14 or higher) -- gcc (7.4.0 or higher) -- git (version 2 or higher) -- bzr (some go dependency needs this) -- jq -- pkg-config -- rustup (proofs build) -- llvm (proofs build) -- clang (proofs build) - -### Install dependencies - -```sh -$ sudo dnf -y update -$ sudo dnf -y install gcc git bzr jq pkgconfig mesa-libOpenCL mesa-libOpenCL-devel opencl-headers ocl-icd ocl-icd-devel clang llvm -$ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -``` - -### Install Go 1.14 - -Install the latest version of Go by following [the docs on their website](https://golang.org/doc/install). - -### Clone the Lotus repository - -```sh -git clone https://github.com/filecoin-project/lotus.git -cd lotus/ -``` - -### Build the Lotus binaries from source and install - -! **If you are running an AMD platform or if your CPU supports SHA extensions you will want to build the Filecoin proofs natively** - -```sh -$ make clean && make all -$ sudo make install -``` - -#### Native Filecoin FFI building - -```sh -env env RUSTFLAGS="-C target-cpu=native -g" FFI_BUILD_FROM_SOURCE=1 make clean deps all -sudo make install -``` - -After installing Lotus, you can run the `lotus` command directly from your CLI to see usage documentation. Next, you can join the [Lotus TestNet](https://lotu.sh/en+join-testnet). diff --git a/documentation/en/install-lotus-ubuntu.md b/documentation/en/install-lotus-ubuntu.md deleted file mode 100644 index 500650692..000000000 --- a/documentation/en/install-lotus-ubuntu.md +++ /dev/null @@ -1,54 +0,0 @@ -# Ubuntu Instructions - -These steps will install the following dependencies: - -- go (1.14 or higher) -- gcc (7.4.0 or higher) -- git (version 2 or higher) -- bzr (some go dependency needs this) -- jq -- pkg-config -- opencl-icd-loader -- opencl driver (like nvidia-opencl on arch) (for GPU acceleration) -- opencl-headers (build) -- rustup (proofs build) -- llvm (proofs build) -- clang (proofs build) - -### Install dependencies - -```sh -sudo apt update -sudo apt install mesa-opencl-icd ocl-icd-opencl-dev gcc git bzr jq pkg-config curl -sudo apt upgrade -``` - -### Install Go 1.14 - -Install the latest version of Go by following [the docs on their website](https://golang.org/doc/install). - -### Clone the Lotus repository - -```sh -git clone https://github.com/filecoin-project/lotus.git -cd lotus/ -``` - -### Build the Lotus binaries from source and install - -! **If you are running an AMD platform or if your CPU supports SHA extensions you will want to build the Filecoin proofs natively** - -```sh -make clean && make all -sudo make install -``` - -#### Native Filecoin FFI building - -```sh -env env RUSTFLAGS="-C target-cpu=native -g" FFI_BUILD_FROM_SOURCE=1 make clean deps all -sudo make install -``` - - -After installing Lotus, you can run the `lotus` command directly from your CLI to see usage documentation. Next, you can join the [Lotus Testnet](https://lotu.sh/en+join-testnet). diff --git a/documentation/en/install-systemd-services.md b/documentation/en/install-systemd-services.md deleted file mode 100644 index fbde1feec..000000000 --- a/documentation/en/install-systemd-services.md +++ /dev/null @@ -1,145 +0,0 @@ -# Use Lotus with systemd - -Lotus is capable of running as a systemd service daemon. You can find installable service files for systemd in the [lotus repo scripts directory](https://github.com/filecoin-project/lotus/tree/master/scripts) as files with `.service` extension. In order to install these service files, you can copy these `.service` files to the default systemd unit load path. - -The services expect their binaries to be present in `/usr/local/bin/`. You can use `make` to install them by running: - -```sh -$ sudo make install -``` - -for `lotus` and `lotus-storage-miner` and - -```sh -$ sudo make install-chainwatch -``` - -for the `chainwatch` tool. - -## Installing services via `make` - -If your host uses the default systemd unit load path, the `lotus-daemon` and `lotus-miner` services can be installed by running: - -```sh -$ sudo make install-services -``` - -To install the the `lotus-chainwatch` service run: - -```sh -$ sudo make install-chainwatch-service -``` - -You can install all services together by running: - -```sh -$ sudo make install-all-services -``` - -The `lotus-daemon` and the `lotus-miner` services can be installed individually too by running: - -```sh -$ sudo make install-daemon-service -``` - -and - -```sh -$ sudo make install-miner-service -``` - -### Notes - -When installing the `lotus-miner` and/or `lotus-chainwatch` service the `lotus-daemon` service gets automatically installed since the other two services depend on it being installed to run. - -All `install-*-service*` commands will install the latest binaries in the lotus build folders to `/usr/local/bin/`. If you do not want to use the latest build binaries please copy the `*.service` files by hand. - -## Removing via `make` - -All services can beremoved via `make`. To remove all services together run: - -```sh -$ sudo make clean-all-services -``` - -Individual services can be removed by running: - -```sh -$ sudo make clean-chainwatch-services -$ sudo make clean-miner-services -$ sudo make clean-daemon-services -``` - -### Notes - -The services will be stoppend and disabled when removed. - -Removing the `lotus-daemon` service will automatically remove the depending services `lotus-miner` and `lotus-chainwatch`. - - -## Controlling services - -All service can be controlled with the `systemctl`. A few basic control commands are listed below. To get detailed infos about the capabilities of the `systemctl` command please consult your distributions man pages by running: - -```sh -$ man systemctl -``` - -### Start/Stop services - -You can start the services by running: - -```sh -$ sudo systemctl start lotus-daemon -$ sudo systemctl start lotus-miner -$ sudo systemctl start lotus-chainwatch -``` - -and can be stopped by running: - -```sh -$ sudo systemctl stop lotus-daemon -$ sudo systemctl stop lotus-miner -$ sudo systemctl stop lotus-chainwatch -``` - -### Enabling services on startup - -To enable the services to run automatically on startup execute: - -```sh -$ sudo systemctl enable lotus-daemon -$ sudo systemctl enable lotus-miner -$ sudo systemctl enable lotus-chainwatch -``` - -To disable the services on startup run: - -```sh -$ sudo systemctl disable lotus-daemon -$ sudo systemctl disable lotus-miner -$ sudo systemctl disable lotus-chainwatch -``` -### Notes - -Systemd will not let services be enabled or started without their requirements. Starting the `lotus-chainwatch` and/or `lotus-miner` service with automatically start the `lotus-daemon` service (if installed!). Stopping the `lotus-daemon` service will stop the other two services. The same pattern is executed for enabling and disabling the services. - -## Interacting with service logs - -Logs from the services can be reviewed using `journalctl`. - -### Follow logs from a specific service unit - -```sh -$ sudo journalctl -u lotus-daemon -f -``` - -### View logs in reverse order - -```sh -$ sudo journalctl -u lotus-miner -r -``` - -### Log files - -Besides the systemd service logs all services save their own log files in `/var/log/lotus/`. diff --git a/documentation/en/installation/install-linux.md b/documentation/en/installation/install-linux.md new file mode 100644 index 000000000..6fe12996e --- /dev/null +++ b/documentation/en/installation/install-linux.md @@ -0,0 +1,129 @@ +# Linux installation + +This page will show you the steps to build and install Lotus in your Linux computer. + +## Dependencies + +### System dependencies + +First of all, building Lotus will require installing some system dependencies, usually provided by your distribution. + +For Arch Linux: + +```sh +sudo pacman -Syu opencl-icd-loader gcc git bzr jq pkg-config opencl-icd-loader opencl-headers +``` + +For Ubuntu: + +```sh +sudo apt update +sudo apt install mesa-opencl-icd ocl-icd-opencl-dev gcc git bzr jq pkg-config curl +sudo apt upgrade +``` + +For Fedora: + +```sh +sudo dnf -y update +sudo dnf -y install gcc git bzr jq pkgconfig mesa-libOpenCL mesa-libOpenCL-devel opencl-headers ocl-icd ocl-icd-devel clang llvm +``` + +For OpenSUSE: + +```sh +sudo zypper in gcc git jq make libOpenCL1 opencl-headers ocl-icd-devel clang llvm +sudo ln -s /usr/lib64/libOpenCL.so.1 /usr/lib64/libOpenCL.so +``` + +### Rustup + +Lotus needs [rustup](https://rustup.rs/): + +```sh +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh +``` + +Please make sure your `$PATH` variable is correctly configured after the rustup installation so that `cargo` and `rustc` are found in their rustup-configured locations. + +### Go + +To build lotus you will need a working installation of **[Go1.14](https://golang.org/dl/)**. Follow the [installation instructions](https://golang.org/doc/install), which generally amount to: + +```sh +# Example! Check the installation instructions. +wget -c https://dl.google.com/go/go1.14.7.linux-amd64.tar.gz -O - | sudo tar -xz -C /usr/local +``` + +## Build and install Lotus + +With all the above, you are ready to build and install the Lotus suite (`lotus`, `lotus-miner` and `lotus-worker`): + +```sh +git clone https://github.com/filecoin-project/lotus.git +cd lotus/ +``` + +__IF YOU ARE IN CHINA__, set `export GOPROXY=https://goproxy.cn` before building + +Now, choose the network that you will be joining: + +* For `testnet`: `git checkout master` +* For `nerpa`: `git checkout ntwk-nerpa` +* For `butterfly`: `git checkout ntwk-butterfly` + +Once on the right branch, do: + +```sh +make clean install +sudo make install +``` + +This will put `lotus`, `lotus-miner` and `lotus-worker` in `/usr/local/bin`. `lotus` will use the `$HOME/.lotus` folder by default for storage (configuration, chain data, wallets...). `lotus-miner` will use `$HOME/.lotusminer` respectively. See the *environment variables* section below for how to customize these. + +> Remeber to [move your Lotus folder](en+update) if you are switching between different networks, or there has been a network reset. + + +### Native Filecoin FFI + +Some newer processors (AMD Zen (and later), Intel Ice Lake) have support SHA extensions. To make full use of your processor's capabilities, make sure you set the following variables BEFORE building from source (as described above): + +```sh +export RUSTFLAGS="-C target-cpu=native -g" +export FFI_BUILD_FROM_SOURCE=1 +``` + +> __NOTE__: This method of building does not produce portable binaries! Make sure you run the binary in the same machine as you built it. + +### systemd service files + +Lotus provides Systemd service files. They can be installed with: + +```sh +make install-daemon-service +make install-miner-service +``` + +After that, you should be able to control Lotus using `systemctl`. + +## Troubleshooting + +This section mentions some of the common pitfalls for building Lotus. Check the [getting started](en+getting-started) section for more tips on issues when running the lotus daemon. + +### Build errors + +Please check the build logs closely. If you have a dirty state in your git branch make sure to: + +```sh +git checkout +git reset origin/ --hard +make clean +``` + +### Slow builds from China + +Users from China can speed up their builds by setting: + +```sh +export GOPROXY=https://goproxy.cn +``` diff --git a/documentation/en/install-lotus-macos.md b/documentation/en/installation/install-macos.md similarity index 85% rename from documentation/en/install-lotus-macos.md rename to documentation/en/installation/install-macos.md index 371832c96..ea9ecb8ca 100644 --- a/documentation/en/install-lotus-macos.md +++ b/documentation/en/installation/install-macos.md @@ -59,4 +59,4 @@ make clean && make all sudo make install ``` -After installing Lotus, you can run the `lotus` command directly from your CLI to see usage documentation. Next, you can join the [Lotus Testnet](https://lotu.sh/en+join-testnet). +After intalling Lotus you will be ready to [setup and run the daemon](en+setup.md). diff --git a/documentation/en/installation/installation.md b/documentation/en/installation/installation.md new file mode 100644 index 000000000..98534da92 --- /dev/null +++ b/documentation/en/installation/installation.md @@ -0,0 +1,39 @@ +# Installation + +Lotus can be installed in [Linux](en-install-linux) and [MacOS](en-install-macos) machines by building it from source. Windows is not supported yet. + +This section contains guides to install Lotus in the supported platforms. + +Lotus is made of 3 binaries: + +* `lotus`: the main [Lotus node](en+setup) (Lotus client) +* `lotus-miner`: an application specifically for [Filecoin mining](en+miner-setup) +* `lotus-worker`: an additional [application to offload some heavy-processing tasks](en+lotus-worker) from the Lotus Miner. + +These applications are written in Go, but also import several Rust libraries. Lotus does not distribute +pre-compiled builds. + +## Hardware requirements + +### For client nodes + +* 8GiB of RAM +* Recommended for syncing speed: CPU with support for *Intel SHA Extensions* (AMD since Zen microarchitecture, Intel since Ice Lake). +* Recommended for speed: SSD hard drive (the bigger the better) + +### For miners + +The following correspond to the latest testing configuration: + +* 2 TB of hard drive space +* 8 core CPU +* 128 GiB of RAM with 256 GiB of NVMe SSD storage for swap (or simply, more RAM). +* Recommended for speed: CPU with support for *Intel SHA Extensions* (AMD since Zen microarchitecture, Intel since Ice Lake). +* GPU for block mining. The following have been [confirmed to be fast enough](en+gpus): + +- GeForce RTX 2080 Ti +- GeForce RTX 2080 SUPER +- GeForce RTX 2080 +- GeForce GTX 1080 Ti +- GeForce GTX 1080 +- GeForce GTX 1060 diff --git a/documentation/en/installation/update.md b/documentation/en/installation/update.md new file mode 100644 index 000000000..5d76592c9 --- /dev/null +++ b/documentation/en/installation/update.md @@ -0,0 +1,72 @@ +# Updating and restarting Lotus + +Updating Lotus is as simple as rebuilding and re-installing the software as explained in the previous sections. + +You can verify which version of Lotus you are running with: + +```sh +lotus version +``` + +Make sure that you `git pull` the branch that corresponds to the network that your Lotus daemon is using: + +```sh +git pull origin +make clean +make all +sudo make install # if necessary +``` + +Finally, restart the Lotus Node and/or Lotus Miner(s). + +__CAVEAT__: If you are running miners: check if your miner is safe to shut down and restart: `lotus-miner proving info`. If any deadline shows a block height in the past, do not restart: + +In the following example, Deadline Open is 454 which is earlier than Current Epoch of 500. This miner should **not** be shut down or restarted. + +``` +$ sudo lotus-miner proving info +Miner: t01001 +Current Epoch: 500 +Proving Period Boundary: 154 +Proving Period Start: 154 (2h53m0s ago) +Next Period Start: 3034 (in 21h7m0s) +Faults: 768 (100.00%) +Recovering: 768 +Deadline Index: 5 +Deadline Sectors: 0 +Deadline Open: 454 (23m0s ago) +Deadline Close: 514 (in 7m0s) +Deadline Challenge: 434 (33m0s ago) +Deadline FaultCutoff: 384 (58m0s ago) +``` + +In this next example, the miner can be safely restarted because no Deadlines are earlier than Current Epoch of 497. You have ~45 minutes before the miner must be back online to declare faults (FaultCutoff). If the miner has no faults, you have about an hour. + +``` +$ sudo lotus-miner proving info +Miner: t01000 +Current Epoch: 497 +Proving Period Boundary: 658 +Proving Period Start: 658 (in 1h20m30s) +Next Period Start: 3538 (in 25h20m30s) +Faults: 0 (0.00%) +Recovering: 0 +Deadline Index: 0 +Deadline Sectors: 768 +Deadline Open: 658 (in 1h20m30s) +Deadline Close: 718 (in 1h50m30s) +Deadline Challenge: 638 (in 1h10m30s) +Deadline FaultCutoff: 588 (in 45m30s) +``` + +## Switching networks and network resets + +If you wish to switch to a different lotus network or there has been a network reset, you will need to: + +* Checkout the appropiate repository branch and rebuild +* Ensure you do not mix Lotus data (`LOTUS_PATH`, usually `~/.lotus`) from a previous or different network. For this, either: + * Rename the folder to something else or, + * Set a different `LOTUS_PATH` for the new network. +* Same for `~/.lotusminer` if you are running a miner. + +Note that deleting the Lotus data folder will wipe all the chain data, wallets and configuration, so think twice before taking any non-reversible action. diff --git a/documentation/en/join-testnet.md b/documentation/en/join-testnet.md deleted file mode 100644 index 6660d26d8..000000000 --- a/documentation/en/join-testnet.md +++ /dev/null @@ -1,93 +0,0 @@ -# Join Testnet - -## Introduction - -Anyone can set up a **Lotus Node** and connect to the **Lotus Testnet**. This is the best way to explore the current CLI and the **Filecoin Decentralized Storage Market**. - -## Note: Using the Lotus Node from China - -If you are trying to use `lotus` from China. You should set this **environment variable** on your machine: - -```sh -export IPFS_GATEWAY="https://proof-parameters.s3.cn-south-1.jdcloud-oss.com/ipfs/" -``` - -## Get started - -Start the **daemon** using the default configuration in `./build`: - -```sh -lotus daemon -``` - -In another terminal window, check your connection with peers: - -```sh -lotus net peers | wc -l -``` - -In order to connect to the network, you need to be connected to at least 1 peer. If you’re seeing 0 peers, read our [troubleshooting notes](https://lotu.sh/en+setup-troubleshooting). - -Make sure that you have a reasonable "open files limit" set on your machine, such as 10000. If you're seeing a lower value, such as 256 (default on macOS), read our [troubleshooting notes](https://lotu.sh/en+setup-troubleshooting) on how to update it prior to starting the Lotus daemon. - -## Chain sync - -While the daemon is running, the next requirement is to sync the chain. Run the command below to view the chain sync progress. To see current chain height, visit the [network stats page](https://stats.testnet.filecoin.io/). - -```sh -lotus sync wait -``` - -- This step will take anywhere between a few hours to a couple of days. -- You will be able to perform **Lotus Testnet** operations after it is finished. - -## Create your first address - -Initialize a new wallet: - -```sh -lotus wallet new -``` - -Sometimes your operating system may limit file name length to under 150 characters. You need to use a file system that supports long filenames. - -Here is an example of the response: - -```sh -t1aswwvjsae63tcrniz6x5ykvsuotlgkvlulnqpsi -``` - -- Visit the [faucet](http://spacerace.faucet.glif.io/) to add funds. -- Paste the address you created under REQUEST. -- Press the Request button. - -## Check wallet address balance - -Wallet balances in the Lotus Testnet are in **FIL**, the smallest denomination of FIL is an **attoFil**, where 1 attoFil = 10^-18 FIL. - -```sh -lotus wallet balance -``` - -You will not see any attoFIL in your wallet if your **chain** is not fully synced. - -## Send FIL to another wallet - -To send FIL to another wallet from your default account, use this command: - -``` -lotus send -``` - -## Configure your node's connectivity - -To effectively accept incoming storage & retrieval deals, your Lotus node needs to be accessible to other nodes on the network. To improve your connectivity, be sure to: - -- [Set the multiaddresses for you miner to listen on](https://docs.filecoin.io/mine/connectivity/#setting-multiaddresses) -- [Maintain a healthy peer count](https://docs.filecoin.io/mine/connectivity/#checking-peer-count) -- [Enable port forwarding](https://docs.filecoin.io/mine/connectivity/#port-forwarding) -- [Configure your public IP address and port](https://docs.filecoin.io/mine/connectivity/#setting-a-public-ip-address) - -## Monitor the dashboard - -To see the latest network activity, including **chain block height**, **block height**, **blocktime**, **total network power**, largest **block producer miner**, check out the [monitoring dashboard](https://stats.testnet.filecoin.io). diff --git a/documentation/en/miner-deals.md b/documentation/en/miner-deals.md deleted file mode 100644 index 0aee0e1af..000000000 --- a/documentation/en/miner-deals.md +++ /dev/null @@ -1,39 +0,0 @@ -# Information for Miners - -Here is how a miner can get set up to accept storage deals. The first step is -to install a Lotus node and sync to the top of the chain. - -## Set up an ask - -``` -lotus-miner set-price -``` - -This command will set up your miner to accept deal proposals that meet the input price. -The price is inputted in FIL per GiB per epoch, and the default is 0.0000000005. - -## Ensure you can be discovered - -Clients need to be able to find you in order to make storage deals with you. -While there isn't necessarily anything you need to do to become discoverable, here are some things you can -try to check that people can connect to you. - -To start off, make sure you are connected to at least some peers, and your port is -open and working. - -### Connect to your own node - -If you are in contact with someone else running Lotus, you can ask them to try connecting -to your node. To do so, provide them your peer ID, which you can get by running `lotus net id` on -your node. - -They can then try running `lotus net findpeer ` to get your address(es), and can then -run `lotus net connect
` to connect to you. If successful, your node will now -appear on their peers list (run `lotus net peers` to check). - -You can also check this by running a second instance of Lotus yourself. - -### Query your own ask - -A client should be able to find your ask by running `lotus client query-ask `. If -someone is not able to retrieve your ask by doing so, then there is an issue with your node. \ No newline at end of file diff --git a/documentation/en/mining.md b/documentation/en/mining.md deleted file mode 100644 index 32c3c51d2..000000000 --- a/documentation/en/mining.md +++ /dev/null @@ -1,149 +0,0 @@ -# Storage Mining - -Here are instructions to learn how to perform storage mining. For hardware specifications please read [this](https://lotu.sh/en+hardware-mining). - -It is useful to [join the Testnet](https://lotu.sh/en+join-testnet) prior to attempting storage mining for the first time. - -## Note: Using the Lotus Miner from China - -If you are trying to use `lotus-miner` from China. You should set this **environment variable** on your machine. - -```sh -export IPFS_GATEWAY="https://proof-parameters.s3.cn-south-1.jdcloud-oss.com/ipfs/" -``` - -## Get started - -Please ensure that at least one **BLS address** (starts with `t3`) in your wallet exists with the following command: - -```sh -lotus wallet list -``` - -If you do not have a bls address, create a new bls wallet: - -```sh -lotus wallet new bls -``` - -With your wallet address: - -- Visit the [faucet](http://spacerace.faucet.glif.io/) -- Paste the address you created under REQUEST. -- Press the Request button. -- Run `/lotus-miner init --owner= --worker=` - -You will have to wait some time for this operation to complete. - -## Mining - -To mine: - -```sh -lotus-miner run -``` - -If you are downloading **Filecoin Proof Parameters**, the download can take some time. - -Get information about your miner: - -```sh -lotus-miner info -# example: miner id `t0111` -``` - -**Seal** random data to start producing **PoSts**: - -```sh -lotus-miner sectors pledge -``` - -- Warning: On Linux configurations, this command will write data to `$TMPDIR` which is not usually the largest partition. You should point the value to a larger partition if possible. - -Get **miner power** and **sector usage**: - -```sh -lotus state power -# returns total power - -lotus state power - -lotus state sectors -``` - -## Performance tuning - -### `FIL_PROOFS_MAXIMIZE_CACHING=1` Environment variable - -This env var can be used with `lotus-miner`, `lotus-worker`, and `lotus-bench` to make the precommit1 step faster at the cost of some memory use (1x sector size) - -### `FIL_PROOFS_USE_GPU_COLUMN_BUILDER=1` Environment variable - -This env var can be used with `lotus-miner`, `lotus-worker`, and `lotus-bench` to enable experimental precommit2 GPU acceleration - -### Setting multiaddresses - -Set multiaddresses for the miner to listen on in a miner's `config.toml` file -(by default, it is located at `~/.lotusminer/config.toml`). The `ListenAddresses` in this file should be interface listen addresses (usually `/ip4/0.0.0.0/tcp/PORT`), and the `AnnounceAddresses` should match the addresses being passed to `set-addrs`. - -The addresses passed to `set-addrs` parameter in the commands below should be currently active and dialable; confirm they are before entering them. - -Once the config file has been updated, set the on-chain record of the miner's listen addresses: - -``` -lotus-miner actor set-addrs ... -``` - -This updates the `MinerInfo` object in the miner's actor, which will be looked up -when a client attempts to make a deal. Any number of addresses can be provided. - -Example: - -``` -lotus-miner actor set-addrs /ip4/123.123.73.123/tcp/12345 /ip4/223.223.83.223/tcp/23456 -``` - -# Separate address for windowPoSt messages - -WindowPoSt is the mechanism through which storage is verified in Filecoin. It requires miners to submit proofs for all sectors every 24h, which require sending messages to the chain. - -Because many other mining related actions require sending messages to the chain, and not all of those are "high value", it may be desirable to use a separate account to send PoSt messages from. This allows for setting lower GasFeeCaps on the lower value messages without creating head-of-line blocking problems for the PoSt messages in congested chain conditions - -To set this up, first create a new account, and send it some funds for gas fees: -```sh -lotus wallet new bls -t3defg... - -lotus send t3defg... 100 -``` - -Next add the control address -```sh -lotus-miner actor control set t3defg... -Add t3defg... -Pass --really-do-it to actually execute this action -``` - -Now actually set the addresses -```sh -lotus-miner actor control set --really-do-it t3defg... -Add t3defg... -Message CID: bafy2.. -``` - -Wait for the message to land on chain -```sh -lotus state wait-msg bafy2.. -... -Exit Code: 0 -... -``` - -Check miner control address list to make sure the address was correctly setup -```sh -lotus-miner actor control list -name ID key use balance -owner t01111 t3abcd... other 300 FIL -worker t01111 t3abcd... other 300 FIL -control-0 t02222 t3defg... post 100 FIL -``` diff --git a/documentation/en/mining/gpus.md b/documentation/en/mining/gpus.md new file mode 100644 index 000000000..ad0ed4f66 --- /dev/null +++ b/documentation/en/mining/gpus.md @@ -0,0 +1,17 @@ +# Benchmarking additional GPUs + +If you want to test a GPU that is not explicitly supported, set the following *environment variable*: + +```sh +BELLMAN_CUSTOM_GPU=":" +``` + +Here is an example of trying a GeForce GTX 1660 Ti with 1536 cores. + +```sh +BELLMAN_CUSTOM_GPU="GeForce GTX 1660 Ti:1536" +``` + +To get the number of cores for your GPU, you will need to check your card’s specifications. + +To perform the benchmark you can use Lotus' [benchmarking tool](https://github.com/filecoin-project/lotus/tree/master/cmd/lotus-bench). Results and discussion are tracked in a [GitHub issue thread](https://github.com/filecoin-project/lotus/issues/694). diff --git a/documentation/en/mining-lotus-worker.md b/documentation/en/mining/lotus-seal-worker.md similarity index 60% rename from documentation/en/mining-lotus-worker.md rename to documentation/en/mining/lotus-seal-worker.md index f93780c44..62dde9bfb 100644 --- a/documentation/en/mining-lotus-worker.md +++ b/documentation/en/mining/lotus-seal-worker.md @@ -2,24 +2,14 @@ The **Lotus Worker** is an extra process that can offload heavy processing tasks from your **Lotus Miner**. The sealing process automatically runs in the **Lotus Miner** process, but you can use the Worker on another machine communicating over a fast network to free up resources on the machine running the mining process. -## Note: Using the Lotus Worker from China +## Installation -If you are trying to use `lotus-worker` from China. You should set this **environment variable** on your machine: - -```sh -export IPFS_GATEWAY="https://proof-parameters.s3.cn-south-1.jdcloud-oss.com/ipfs/" -``` - -## Get Started - -Make sure that the `lotus-worker` is compiled and installed by running: - -```sh -make lotus-worker -``` +The `lotus-worker` application is installed along with the others when running `sudo make install` as shown in the [Installation section](en+install-linux). For simplicity, we recommend following the same procedure in the machines that will run the Lotus Workers (even if the Lotus miner and the Lotus daemon are not used there). ## Setting up the Miner +### Allow external connections to the miner API + First, you will need to ensure your `lotus-miner`'s API is accessible over the network. To do this, open up `~/.lotusminer/config.toml` (Or if you manually set `LOTUS_MINER_PATH`, look under that directory) and look for the API field. @@ -32,30 +22,49 @@ ListenAddress = "/ip4/127.0.0.1/tcp/2345/http" RemoteListenAddress = "127.0.0.1:2345" ``` -To make your node accessible over the local area network, you will need to determine your machines IP on the LAN, and change the `127.0.0.1` in the file to that address. +To make your node accessible over the local area network, you will need to determine your machine's IP on the LAN (`ip a`), and change the `127.0.0.1` in the file to that address. -A more permissive and less secure option is to change it to `0.0.0.0`. This will allow anyone who can connect to your computer on that port to access the [API](https://lotu.sh/en+api). They will still need an auth token. +A more permissive and less secure option is to change it to `0.0.0.0`. This will allow anyone who can connect to your computer on that port to access the miner's API, though they will still need an auth token. `RemoteListenAddress` must be set to an address which other nodes on your network will be able to reach. -Next, you will need to [create an authentication token](https://lotu.sh/en+api-scripting-support#generate-a-jwt-46). All Lotus APIs require authentication tokens to ensure your processes are as secure against attackers attempting to make unauthenticated requests to them. +### Create an authentication token -### Connect the Lotus Worker +Write down the output of: -On the machine that will run `lotus-worker`, set the `MINER_API_INFO` environment variable to `TOKEN:MINER_NODE_MULTIADDR`. Where `TOKEN` is the token we created above, and `NIMER_NODE_MULTIADDR` is the `multiaddr` of the **Lotus Miner** API that was set in `config.toml`. +```sh +lotus-miner auth api-info --perm admin +``` -Once this is set, run: +The Lotus Workers will need this token to connect to the miner. + +## Connecting the Lotus Workers + +On each machine that will run the `lotus-worker` application you will need to define the following *environment variable*: + +```sh +export MINER_API_INFO::/ip4//tcp/2345` +``` + +If you are trying to use `lotus-worker` from China. You should additionally set: + +```sh +export IPFS_GATEWAY="https://proof-parameters.s3.cn-south-1.jdcloud-oss.com/ipfs/" +``` + + +Once that is done, you can run the Worker with: ```sh lotus-worker run ``` -If you are running multiple workers on the same host, you will need to specify the `--listen` flag and ensure each worker is on a different port. +> If you are running multiple workers on the same host, you will need to specify the `--listen` flag and ensure each worker is on a different port. -To check that the **Lotus Worker** is connected to your **Lotus Miner**, run `lotus-miner sealing workers` and check that the remote worker count has increased. +On your Lotus miner, check that the workers are correctly connected: ```sh -why@computer ~/lotus> lotus-miner sealing workers +lotus-miner sealing workers Worker 0, host computer CPU: [ ] 0 core(s) in use RAM: [|||||||||||||||||| ] 28% 18.1 GiB/62.7 GiB @@ -69,9 +78,10 @@ Worker 1, host othercomputer GPU: GeForce RTX 2080, not used ``` -### Running locally for manually managing process priority +## Running locally for manually managing process priority You can also run the **Lotus Worker** on the same machine as your **Lotus Miner**, so you can manually manage the process priority. + To do so you have to first __disable all seal task types__ in the miner config. This is important to prevent conflicts between the two processes. You can then run the miner on your local-loopback interface; diff --git a/documentation/en/mining/managing-deals.md b/documentation/en/mining/managing-deals.md new file mode 100644 index 000000000..5f73a6a2d --- /dev/null +++ b/documentation/en/mining/managing-deals.md @@ -0,0 +1,19 @@ +# Managing deals + + +While the Lotus Miner is running as a daemon, the `lotus-miner` application can be used to manage and configure the miner: + + +```sh +lotus-miner storage-deals --help +``` + +Running the above command will show the different options related to deals. For example, `lotus-miner storage-deals set-ask` allows to set the price for storage that your miner uses to respond ask requests from clients. + +If deals are ongoing, you can check the data transfers with: + +```sh +lotus-miner data-transfers list +``` + +Make sure you explore the `lotus-miner` CLI. Every command is self-documented and takes a `--help` flag that offers specific information about it. diff --git a/documentation/en/mining/miner-setup.md b/documentation/en/mining/miner-setup.md new file mode 100644 index 000000000..cafa1e7b1 --- /dev/null +++ b/documentation/en/mining/miner-setup.md @@ -0,0 +1,241 @@ +# Miner setup + +This page will guide you through all you need to know to sucessfully run a **Lotus Miner**. Before proceeding, remember that you should be running the Lotus daemon on a fully synced chain. + +## Performance tweaks + +This is a list of performance tweaks to consider before starting the miner: + +### Building + +As [explained already](en+install-linux#native-filecoin-ffi-10) should have exported the following variables before building the Lotus applications: + +```sh +export RUSTFLAGS="-C target-cpu=native -g" +export FFI_BUILD_FROM_SOURCE=1 +``` + +### Environment + +For high performance mining, we recommend setting the following variables in your environment so that they are available when running any of the Lotus applications: + +```sh +# See https://github.com/filecoin-project/bellman +export BELLMAN_CPU_UTILIZATION=0.875 + +# See https://github.com/filecoin-project/rust-fil-proofs/ +export FIL_PROOFS_MAXIMIZE_CACHING=1 # More speed at RAM cost (1x sector-size of RAM - 32 GB). +export FIL_PROOFS_USE_GPU_COLUMN_BUILDER=1 # precommit2 GPU acceleration +export FIL_PROOFS_USE_GPU_TREE_BUILDER=1 +``` + +IF YOU ARE RUNNING FROM CHINA: + +```sh +export IPFS_GATEWAY="https://proof-parameters.s3.cn-south-1.jdcloud-oss.com/ipfs/" +``` + +IF YOUR MINER RUNS IN A DIFFERENT MACHINE AS THE LOTUS DAEMON: + +```sh +export FULLNODE_API_INFO=:/ip4//tcp//http +``` + +If you will be using systemd service files to run the Lotus daemon and miner, make sure you include these variables manually in the service files. + +### Adding swap + +If you have only 128GiB of RAM, you will need to make sure your system provides at least an extra 256GiB of fast swap (preferably NVMe SSD): + +```sh +sudo fallocate -l 256G /swapfile +sudo chmod 600 /swapfile +sudo mkswap /swapfile +sudo swapon /swapfile +# show current swap spaces and take note of the current highest priority +swapon --show +# append the following line to /etc/fstab (ensure highest priority) and then reboot +# /swapfile swap swap pri=50 0 0 +sudo reboot +# check a 256GB swap file is automatically mounted and has the highest priority +swapon --show +``` + +## Creating a new BLS wallet + +You will need a BLS wallet (`t3...`) for mining. To create it, if you don't have one already, run: + +```sh +lotus wallet new bls +``` + +Next make sure to [send some funds](en+wallet) to this address so that the miner setup can be completed. + +## Initializing the miner + +> SPACE RACE: +> To participate in the Space race, please register your miner: +> +> - Visit the [faucet](http://spacerace.faucet.glif.io/) +> - Paste the address you created under REQUEST. +> - Press the Request button. + +Now that you have a miner address you can initialize the Lotus Miner: + +```sh +lotus-miner init --owner= --no-local-storage +``` + +* The `--no-local-storage` flag is used so that we configure specific locations for storage later below. +* The init process will download over 100GiB of initialization parameters to /var/tmp/filecoin-proof-parameters. Make sure there is space or set `FIL_PROOFS_PARAMETER_CACHE` to somewhere else. +* The Lotus Miner configuration folder is created at `~/.lotusminer/` or `$LOTUS_MINER_PATH` if set. + +## Reachability + +Before you start your miner, it is __very important__ to configure it so that it is reachable from any peer in the Filecoin network. For this you will need a stable public IP and edit your `~/.lotusminer/config.toml` as follows: + +```toml +... +[Libp2p] + ListenAddresses = ["/ip4/0.0.0.0/tcp/24001"] # choose a fixed port + AnnounceAddresses = ["/ip4//tcp/24001"] # important! +... +``` + +Once you start your miner, make sure you can connect to its public IP/port (you can use `telnet`, `nc` for the task...). If you have an active firewall or some sort, you may need to additionally open ports in it. + + +## Starting the miner + +You are now ready to start your Lotus miner: + +```sh +lotus-miner run +``` + +or if you are using the systemd service file: + +```sh +systemctl start lotus-miner +``` + +> __Do not proceed__ from here until you have verified that your miner not only is running, but also __reachable on its public IP address__. + +## Publishing the miner addresses + +Once the miner is up and running, publish your miner address (which you configured above) on the chain (please ensure it is dialable): + +```sh +lotus-miner actor set-addrs /ip4//tcp/24001 +``` + +## Setting locations for sealing and long-term storage + +If you used the `--no-local-storage` flag during initialization, you can now specify the disk locations for sealing (SSD recommended) and long-term storage (otherwise you can skip this): + +``` +lotus-miner storage attach --init --seal +lotus-miner storage attach --init --store +lotus-miner storage list +``` + +## Pledging sectors + +If you would like to compete for block rewards by increasing your power in the network as soon as possible, you can optionally pledge one or several sectors, depending on your storage. It can also be used to test that the sealing process works correctly. Pledging is equivalent to storing random data instead of real data obtained through storage deals. + +> Note that pledging sectors to the mainnet network makes most sense when trying to obtain a reasonable amount of total power in the network, thus obtaining real chances to mine new blocks. Otherwise it is only useful for testing purposes. + +If you decide to go ahead, then do: + +```sh +lotus-miner sectors pledge +``` + +This will write data to `$TMPDIR` so make sure that there is enough space available. + +You shoud check that your sealing job has started with: + +```sh +lotus-miner sealing jobs +``` + +This will be accommpanied by a file in `/unsealed`. + +After some minutes, you can check the sealing progress with: + +```sh +lotus-miner sectors list +# and +lotus-miner sealing workers +``` + +When sealing for the new is complete, `pSet: NO` will become `pSet: YES`. + +Once the sealing is finished, you will want to configure how long it took your miner to seal this sector and configure the miner accordingly. To find out how long it took use: + +``` +lotus-miner sectors status --log 0 +``` + +Once you know, you can edit the Miner's `~/.lotusminer/config.toml` accordingly: + +``` +... +[Dealmaking] +... + ExpectedSealDuration = "12h0m0s" # The time it took your miner +``` + +You can also take the chance to edit other values, such as `WaitForDealsDelay` which specifies the delay between accepting the first deal and sealing, allowing to place multiple deals in the same sector. + +Once you are done editing the configuration, [restart your miner](en+update). + +If you wish to be able to re-use a pledged sector for real storage deals before the pledged period of 6 months ends, you will need to mark them for upgrade: + +```sh +lotus-miner sectors mark-for-upgrade +``` + +The sector should become inactive within 24 hours. From that point, the pledged storage can be re-used to store real data associated with real storage deals. + +## Separate address for windowPoSt messages + +WindowPoSt is the mechanism through which storage is verified in Filecoin. It requires miners to submit proofs for all sectors every 24h, which require sending messages to the chain. + +Because many other mining related actions require sending messages to the chain, and not all of those are "high value", it may be desirable to use a separate account to send PoSt messages from. This allows for setting lower GasFeeCaps on the lower value messages without creating head-of-line blocking problems for the PoSt messages in congested chain conditions + +To set this up, first create a new account, and send it some funds for gas fees: + +```sh +lotus wallet new bls +t3defg... + +lotus send t3defg... 100 +``` + +Next add the control address: + +```sh +lotus-miner actor control set --really-do-it t3defg... +Add t3defg... +Message CID: bafy2.. +``` + +Wait for the message to land on chain: + +```sh +lotus state wait-msg bafy2.. +... +Exit Code: 0 +... +``` + +Finally, check the miner control address list to make sure the address was correctly setup: + +```sh +lotus-miner actor control list +name ID key use balance +owner t01111 t3abcd... other 300 FIL +worker t01111 t3abcd... other 300 FIL +control-0 t02222 t3defg... post 100 FIL +``` diff --git a/documentation/en/mining-troubleshooting.md b/documentation/en/mining/mining-troubleshooting.md similarity index 90% rename from documentation/en/mining-troubleshooting.md rename to documentation/en/mining/mining-troubleshooting.md index 5aaf9f6ef..758929075 100644 --- a/documentation/en/mining-troubleshooting.md +++ b/documentation/en/mining/mining-troubleshooting.md @@ -25,7 +25,7 @@ lotus-miner info # WARN main lotus-storage-miner/main.go:73 failed to get api endpoint: (/Users/myrmidon/.lotusminer) %!w(*errors.errorString=&{API not running (no endpoint)}): ``` -If you see this, that means your **Lotus Miner** isn't ready yet. You need to finish [syncing the chain](https://lotu.sh/en+join-testnet). +If you see this, that means your **Lotus Miner** isn't ready yet. You need to finish [syncing the chain](en+setup#waiting-to-sync-370). ## Error: Your computer may not be fast enough @@ -57,10 +57,3 @@ make bench This process uses a fair amount of GPU, and generally takes ~4 minutes to complete. If you do not see any activity in nvtop from lotus during the entire process, it is likely something is misconfigured with your GPU. -## Checking Sync Progress - -You can use this command to check how far behind you are on syncing: - -```sh -date -d @$(./lotus chain getblock $(./lotus chain head) | jq .Timestamp) -``` diff --git a/documentation/en/mining/mining.md b/documentation/en/mining/mining.md new file mode 100644 index 000000000..b1b944c6e --- /dev/null +++ b/documentation/en/mining/mining.md @@ -0,0 +1,8 @@ +# Storage Mining + +This section of the documentation explains how to do storage mining with Lotus. Please note that not everyone can do storage mining, and that you should not attempt it on on networks where sector sizes are 32GB+ unless you meet the [hardware requirements](en+install#hardware-requirements-1). + +From this point we assume that you have setup and are running the [Lotus Node](en+setup), that it has fully synced the Filecoin chain and that you are familiar with how to interact with it using the `lotus` command-line interface. + +In order to perform storage mining, apart from the Lotus daemon, you will be additionally interacting with the `lotus-miner` and potentially the `lotus-worker` applications (which you should have [installed](en+install-linux) along the `lotus` application already). + diff --git a/documentation/en/retrieving-data.md b/documentation/en/retrieving-data.md deleted file mode 100644 index 7cb0e31be..000000000 --- a/documentation/en/retrieving-data.md +++ /dev/null @@ -1,27 +0,0 @@ -# Retrieving Data - -> There are recent bug reports with these instructions. If you happen to encounter any problems, please create a [GitHub issue](https://github.com/filecoin-project/lotus/issues/new) and a maintainer will address the problem as soon as they can. - -Here are the operations you can perform after you have stored and sealed a **Data CID** with the **Lotus Miner** in the network. - -If you would like to learn how to store a **Data CID** on a miner, read the instructions [here](https://lotu.sh/en+storing-data). - -## Find by Data CID - -```sh -lotus client find -# LOCAL -# RETRIEVAL @-- -``` - -## Retrieve by Data CID - -All fields are required. - -```sh -lotus client retrieve -``` - -If the outfile does not exist it will be created in the Lotus repository directory. - -This command will initiate a **retrieval deal** and write the data to your computer. This process may take 2 to 10 minutes. diff --git a/documentation/en/setting-a-static-port.md b/documentation/en/setting-a-static-port.md deleted file mode 100644 index 97ac6528e..000000000 --- a/documentation/en/setting-a-static-port.md +++ /dev/null @@ -1,54 +0,0 @@ -# Static Ports - -Depending on how your network is set up, you may need to set a static port to successfully connect to peers to perform storage deals with your **Lotus Miner**. - -## Setup - -To change the random **swarm port**, you may edit the `config.toml` file located under `$LOTUS_MINER_PATH`. The default location of this file is `$HOME/.lotusminer`. - -To change the port to `1347`: - -```sh -[Libp2p] - ListenAddresses = ["/ip4/0.0.0.0/tcp/1347", "/ip6/::/tcp/1347"] -``` - -After changing the port value, restart your **daemon**. - -## Announce Addresses - -If the **swarm port** is port-forwarded from another address, it is possible to control what addresses -are announced to the network. - -```sh -[Libp2p] - AnnounceAddresses = ["/ip4//tcp/1347"] -``` - -If non-empty, this array specifies the swarm addresses to announce to the network. If empty, the daemon will announce inferred swarm addresses. - -Similarly, it is possible to set `NoAnnounceAddresses` with an array of addresses to not announce to the network. - -## Ubuntu's Uncomplicated Firewall - -Open firewall manually: - -```sh -ufw allow 1347/tcp -``` - -Or open and modify the profile located at `/etc/ufw/applications.d/lotus-daemon`: - -```sh -[Lotus Daemon] -title=Lotus Daemon -description=Lotus Daemon firewall rules -ports=1347/tcp -``` - -Then run these commands: - -```sh -ufw update lotus-daemon -ufw allow lotus-daemon -``` diff --git a/documentation/en/storing-ipfs-integration.md b/documentation/en/store/adding-from-ipfs.md similarity index 79% rename from documentation/en/storing-ipfs-integration.md rename to documentation/en/store/adding-from-ipfs.md index 041364380..2f6b097cc 100644 --- a/documentation/en/storing-ipfs-integration.md +++ b/documentation/en/store/adding-from-ipfs.md @@ -1,10 +1,10 @@ -# IPFS Integration +# Adding data from IPFS Lotus supports making deals with data stored in IPFS, without having to re-import it into lotus. To enable this integration, you need to have an IPFS daemon running in the background. -Then, open up `~/.lotus/config.toml` (or if you manually set `LOTUS_PATH`, look under that directory) -and look for the Client field, and set `UseIpfs` to `true`. + +Then, open up `~/.lotus/config.toml` (or if you manually set `LOTUS_PATH`, look under that directory) and look for the Client field, and set `UseIpfs` to `true`. ```toml [Client] diff --git a/documentation/en/store/making-deals.md b/documentation/en/store/making-deals.md new file mode 100644 index 000000000..ca3a47182 --- /dev/null +++ b/documentation/en/store/making-deals.md @@ -0,0 +1,71 @@ +# Making storage deals + +## Adding a file to Lotus + +Before sending data to a Filecoin miner for storage, the data needs to be correctly formatted and packed. This can be achieved by locally importing the data into Lotus with: + +```sh +lotus client import ./your-example-file.txt +``` + +Upon success, this command will return a **Data CID**. This is a very important piece of information, as it will be used to make deals to both store and retrieve the data in the future. + +You can list the data CIDs of the files you locally imported with: + +```sh +lotus client local +``` + +## Storing data in the network + +To store data in the network you will need to: + +* Find a Filecoin miner willing to store it +* Make a deal with the miner agreeing on the price to pay and the duration for which the data should be stored. + +You can obtain a list of all miners in the network with: + +```sh +lotus state list-miners +t0xxxx +t0xxxy +t0xxxz +... +``` + +This will print a list of miner IDs. In order to ask for the terms offered by a particular miner, you can then run: + +```sh +lotus client query-ask +``` + +If you are satisfied with the terms, you can proceed to propose a deal to the miner, using the **Data CID** that you obtained during the import step: + + +```sh +lotus client deal +``` + +This command will interactively ask you for the CID, miner ID and duration in days for the deal. You can also call it with arguments: + +```sh +lotus client deal +``` + +where the `duration` is expressed in blocks (1 block is equivalent to 30s). + +## Checking the status of the deals + +You can list deals with: + +```sh +lotus client list-deals +``` + +Among other things, this will give you information about the current state on your deals, whether they have been published on chain (by the miners) and whether the miners have been slashed for not honoring them. + +For a deal to succeed, the miner needs to be correctly configured and running, accept the deal and *seal* the file correctly. Otherwise, the deal will appear in error state. + +You can make deals with multiple miners for the same data. + +Once a deal is sucessful and the data is *sealed*, it can be [retrieved](en+retrieving). diff --git a/documentation/en/store/retrieve.md b/documentation/en/store/retrieve.md new file mode 100644 index 000000000..1e8db65af --- /dev/null +++ b/documentation/en/store/retrieve.md @@ -0,0 +1,27 @@ +# Retrieving Data + +Once data has been succesfully [stored](en+making-deals) and sealed by a Filecoin miner, it can be retrieved. + +In order to do this we will need to create a **retrieval deal**. + +## Finding data by CID + +In order to retrieve some data you will need the **Data CID** that was used to create the storage deal. + +You can find who is storing the data by running: + +```sh +lotus client find +``` + +## Making a retrieval deal + +You can then make a retrieval deal with: + +```sh +lotus client retrieve +``` + +This commands take other optional flags (check `--help`). + +If the outfile does not exist it will be created in the Lotus repository directory. This process may take 2 to 10 minutes. diff --git a/documentation/en/storing-data-troubleshooting.md b/documentation/en/store/storage-troubleshooting.md similarity index 51% rename from documentation/en/storing-data-troubleshooting.md rename to documentation/en/store/storage-troubleshooting.md index c8a0254fa..7087ec3d0 100644 --- a/documentation/en/storing-data-troubleshooting.md +++ b/documentation/en/store/storage-troubleshooting.md @@ -2,11 +2,11 @@ ## Error: Routing: not found -```sh +``` WARN main lotus/main.go:72 routing: not found ``` -- This miner is offline. +This error means that the miner is offline. ## Error: Failed to start deal @@ -14,14 +14,17 @@ WARN main lotus/main.go:72 routing: not found WARN main lotus/main.go:72 failed to start deal: computing commP failed: generating CommP: Piece must be at least 127 bytes ``` -- There is a minimum file size of 127 bytes. +This error means that there is a minimum file size of 127 bytes. ## Error: 0kb file response during retrieval -In order to retrieve a file, it must be sealed. Miners can check sealing progress with this command: +This means that the file to be retrieved may have not yet been sealed and is thus, not retrievable yet. + +Miners can check sealing progress with this command: ```sh lotus-miner sectors list ``` -When sealing is complete, `pSet: NO` will become `pSet: YES`. From now on the **Data CID** is [retrievable](https://lotu.sh/en+retrieving-data) from the **Lotus Miner**. +When sealing is complete, `pSet: NO` will become `pSet: YES`. + diff --git a/documentation/en/store/store.md b/documentation/en/store/store.md new file mode 100644 index 000000000..205bd0e23 --- /dev/null +++ b/documentation/en/store/store.md @@ -0,0 +1,11 @@ +# Storing and retrieving data + +Lotus enables you to store any data on the Filecoin network and retrieve it later. This is achieved by making *deals* with miners. + +A *storage deal* specifies that a miner should store ceratin data for a previously agreed period and price. + +Once a deal is made, the data is then sent to the miners, which regularly proves that it is storing it. If they fail to do so, the miner is penalized (slashed). + +The data can be retrieved with a *retrieval deal*. + +This section explains how to use Lotus to [store](en+making-deals) and [retrieve](en+retrieving) data from the Filecoin network. diff --git a/documentation/en/storing-data.md b/documentation/en/storing-data.md deleted file mode 100644 index 67d2b1533..000000000 --- a/documentation/en/storing-data.md +++ /dev/null @@ -1,62 +0,0 @@ -# Storing Data - -> There are recent bug reports with these instructions. If you happen to encounter any problems, please create a [GitHub issue](https://github.com/filecoin-project/lotus/issues/new) and a maintainer will address the problem as soon as they can. - -Here are instructions for how to store data on the **Lotus Testnet**. - -## Adding a file locally - -Adding a file locally allows you to make miner deals on the **Lotus Testnet**. - -```sh -lotus client import ./your-example-file.txt -``` - -Upon success, this command will return a **Data CID**. - -## List your local files - -The command to see a list of files by `CID`, `name`, `size` in bytes, and `status`: - -```sh -lotus client local -``` - -An example of the output: - -```sh -bafkreierupr5ioxn4obwly4i2a5cd2rwxqi6kwmcyyylifxjsmos7hrgpe Development/sample-1.txt 2332 ok -bafkreieuk7h4zs5alzpdyhlph4lxkefowvwdho3a3pml6j7dam5mipzaii Development/sample-2.txt 30618 ok -``` - -## Make a Miner Deal on Lotus Testnet - -Get a list of all miners that can store data: - -```sh -lotus state list-miners -``` - -Get the requirements of a miner you wish to store data with: - -```sh -lotus client query-ask -``` - -Store a **Data CID** with a miner: - -```sh -lotus client deal -``` - -Check the status of a deal: - -```sh -lotus client list-deals -``` - -- The `duration`, which represents how long the miner will keep your file hosted, is represented in blocks. Each block represents 25 seconds. - -Upon success, this command will return a **Deal CID**. - -The miner will need to **seal** the file before it can be retrieved. If the **Lotus Miner** is not running on a machine designed for sealing, the process will take a very long time. diff --git a/documentation/en/dev/WIP-arch-complementary-notes.md b/documentation/en/unclassified/WIP-arch-complementary-notes.md similarity index 100% rename from documentation/en/dev/WIP-arch-complementary-notes.md rename to documentation/en/unclassified/WIP-arch-complementary-notes.md diff --git a/documentation/en/block-validation.md b/documentation/en/unclassified/block-validation.md similarity index 100% rename from documentation/en/block-validation.md rename to documentation/en/unclassified/block-validation.md diff --git a/documentation/en/dev/create-miner.md b/documentation/en/unclassified/create-miner.md similarity index 100% rename from documentation/en/dev/create-miner.md rename to documentation/en/unclassified/create-miner.md diff --git a/documentation/en/dev-tools-pond-ui.md b/documentation/en/unclassified/dev-tools-pond-ui.md similarity index 100% rename from documentation/en/dev-tools-pond-ui.md rename to documentation/en/unclassified/dev-tools-pond-ui.md diff --git a/documentation/en/sealing-procs.md b/documentation/en/unclassified/sealing-procs.md similarity index 100% rename from documentation/en/sealing-procs.md rename to documentation/en/unclassified/sealing-procs.md diff --git a/documentation/en/updating-lotus.md b/documentation/en/updating-lotus.md deleted file mode 100644 index 862cea136..000000000 --- a/documentation/en/updating-lotus.md +++ /dev/null @@ -1,14 +0,0 @@ -# Updating Lotus - -If you installed Lotus on your machine, you can upgrade to the latest version by doing the following: - -```sh -# get the latest -git pull origin master - -# clean and remake the binaries -make clean && make build - -# instal binaries in correct location -make install # or sudo make install if necessary -``` From a153e1d5862b67a031fe7ee7306a7e430c517187 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Mon, 31 Aug 2020 21:18:50 +0200 Subject: [PATCH 034/885] Fix #2334: Specify seal options to disable with co-located storage worker --- documentation/en/mining/lotus-seal-worker.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/documentation/en/mining/lotus-seal-worker.md b/documentation/en/mining/lotus-seal-worker.md index 62dde9bfb..47e201ca5 100644 --- a/documentation/en/mining/lotus-seal-worker.md +++ b/documentation/en/mining/lotus-seal-worker.md @@ -82,7 +82,15 @@ Worker 1, host othercomputer You can also run the **Lotus Worker** on the same machine as your **Lotus Miner**, so you can manually manage the process priority. -To do so you have to first __disable all seal task types__ in the miner config. This is important to prevent conflicts between the two processes. +To do so you have to first __disable all seal task types__ in the miner config. This is important to prevent conflicts between the two processes: + +```toml +[Storage] + AllowPreCommit1 = false + AllowPreCommit2 = false + AllowCommit = false + AllowUnseal = false +``` You can then run the miner on your local-loopback interface; From af38c902f88d5c7be13f24eca4e8d19b395e963f Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Tue, 1 Sep 2020 15:12:07 +0200 Subject: [PATCH 035/885] Fix architecture entry --- documentation/en/.library.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/en/.library.json b/documentation/en/.library.json index 87c7353c1..59cc01e29 100644 --- a/documentation/en/.library.json +++ b/documentation/en/.library.json @@ -193,7 +193,7 @@ { "title": "Lotus Architecture (WIP)", "slug": "en+arch", - "github": "en/architectiure/architecture.md", + "github": "en/architecture/architecture.md", "value": null, "posts": [ { From bd0c6a4cccd5a5e21ae3f4939cf9d8822ea9ca37 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 3 Sep 2020 16:35:55 +0200 Subject: [PATCH 036/885] Fix filename --- documentation/en/building/api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/en/building/api.md b/documentation/en/building/api.md index 626193ee2..3a2c2902b 100644 --- a/documentation/en/building/api.md +++ b/documentation/en/building/api.md @@ -19,7 +19,7 @@ Lotus uses its own Go library implementation of [JSON-RPC](https://github.com/fi ## cURL example -To demonstrate making an API request, we will take the method `ChainHead` from [api/api.go](https://github.com/filecoin-project/lotus/blob/master/api/api_full.go). +To demonstrate making an API request, we will take the method `ChainHead` from [api/api_full.go](https://github.com/filecoin-project/lotus/blob/master/api/api_full.go). ```go ChainHead(context.Context) (*types.TipSet, error) From 97ff90f8fa061ff408bdc1f884067f36f656c19f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Thu, 3 Sep 2020 15:51:26 +0100 Subject: [PATCH 037/885] fix imports. --- chain/store/store_test.go | 1 - chain/validation/applier.go | 1 - 2 files changed, 2 deletions(-) diff --git a/chain/store/store_test.go b/chain/store/store_test.go index c7be48b9c..42de4c19d 100644 --- a/chain/store/store_test.go +++ b/chain/store/store_test.go @@ -17,7 +17,6 @@ import ( "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/lib/blockstore" "github.com/filecoin-project/lotus/node/repo" ) diff --git a/chain/validation/applier.go b/chain/validation/applier.go index 9bf32962f..ac2fccf85 100644 --- a/chain/validation/applier.go +++ b/chain/validation/applier.go @@ -19,7 +19,6 @@ import ( "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" - "github.com/filecoin-project/lotus/journal" ) // Applier applies messages to state trees and storage. From d46f684bb8dffbe712fe66e363b7ceff50580805 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Fri, 4 Sep 2020 00:05:44 +0100 Subject: [PATCH 038/885] fix test following merge. --- extern/storage-sealing/fsm_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/storage-sealing/fsm_test.go b/extern/storage-sealing/fsm_test.go index 7fad643c6..35797c5c9 100644 --- a/extern/storage-sealing/fsm_test.go +++ b/extern/storage-sealing/fsm_test.go @@ -73,7 +73,7 @@ func TestHappyPath(t *testing.T) { m.planSingle(SectorFinalized{}) require.Equal(m.t, m.state.State, Proving) - expected := []SectorState{Packing, PreCommit1, PreCommit2, PreCommitting, PreCommitWait, WaitSeed, Committing, CommitWait, FinalizeSector, Proving} + expected := []SectorState{Packing, PreCommit1, PreCommit2, PreCommitting, PreCommitWait, WaitSeed, Committing, SubmitCommit, CommitWait, FinalizeSector, Proving} for i, n := range notif { if n.before.State != expected[i] { t.Fatalf("expected before state: %s, got: %s", expected[i], n.before.State) From 905168e72e2c194ca724071bd3482bd75e1727d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Fri, 4 Sep 2020 10:54:56 +0100 Subject: [PATCH 039/885] fix lint errors. --- chain/messagepool/messagepool.go | 9 ++++----- chain/messagepool/repub.go | 4 ++-- journal/fs.go | 1 - 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/chain/messagepool/messagepool.go b/chain/messagepool/messagepool.go index a7f134e12..75573539b 100644 --- a/chain/messagepool/messagepool.go +++ b/chain/messagepool/messagepool.go @@ -89,11 +89,11 @@ const ( // MessagePoolEvt is the journal entry for message pool events. type MessagePoolEvt struct { Action string - Messages []MessagePoolEvt_Message + Messages []MessagePoolEvtMessage Error error `json:",omitempty"` } -type MessagePoolEvt_Message struct { +type MessagePoolEvtMessage struct { types.Message CID cid.Cid @@ -145,7 +145,6 @@ type MessagePool struct { sigValCache *lru.TwoQueueCache - jrnl journal.Journal evtTypes [3]journal.EventType } @@ -561,7 +560,7 @@ func (mp *MessagePool) addLocked(m *types.SignedMessage, limit bool) error { journal.J.RecordEvent(mp.evtTypes[evtTypeMpoolAdd], func() interface{} { return MessagePoolEvt{ Action: "add", - Messages: []MessagePoolEvt_Message{{Message: m.Message, CID: m.Cid()}}, + Messages: []MessagePoolEvtMessage{{Message: m.Message, CID: m.Cid()}}, } }) @@ -725,7 +724,7 @@ func (mp *MessagePool) remove(from address.Address, nonce uint64) { journal.J.RecordEvent(mp.evtTypes[evtTypeMpoolRemove], func() interface{} { return MessagePoolEvt{ Action: "remove", - Messages: []MessagePoolEvt_Message{{Message: m.Message, CID: m.Cid()}}} + Messages: []MessagePoolEvtMessage{{Message: m.Message, CID: m.Cid()}}} }) mp.currentSize-- diff --git a/chain/messagepool/repub.go b/chain/messagepool/repub.go index 9b94ee61b..f23659eea 100644 --- a/chain/messagepool/repub.go +++ b/chain/messagepool/repub.go @@ -136,9 +136,9 @@ func (mp *MessagePool) republishPendingMessages() error { if len(msgs) > 0 { journal.J.RecordEvent(mp.evtTypes[evtTypeMpoolRepub], func() interface{} { - msgs := make([]MessagePoolEvt_Message, 0, len(msgs)) + msgs := make([]MessagePoolEvtMessage, 0, len(msgs)) for _, m := range msgs { - msgs = append(msgs, MessagePoolEvt_Message{Message: m.Message, CID: m.Cid()}) + msgs = append(msgs, MessagePoolEvtMessage{Message: m.Message, CID: m.Cid()}) } return MessagePoolEvt{ Action: "repub", diff --git a/journal/fs.go b/journal/fs.go index 83dd986bf..340705f07 100644 --- a/journal/fs.go +++ b/journal/fs.go @@ -22,7 +22,6 @@ type fsJournal struct { dir string sizeLimit int64 - lk sync.Mutex fi *os.File fSize int64 From 0a4c473a7cb0aad8d51525974ed591a5124f8e2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Fri, 4 Sep 2020 12:14:27 +0100 Subject: [PATCH 040/885] fix dangling import. --- journal/fs.go | 1 - 1 file changed, 1 deletion(-) diff --git a/journal/fs.go b/journal/fs.go index 340705f07..57774d3aa 100644 --- a/journal/fs.go +++ b/journal/fs.go @@ -5,7 +5,6 @@ import ( "fmt" "os" "path/filepath" - "sync" "golang.org/x/xerrors" From f90cfda2e63a90eb9ec6874b21c96b134177681f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 4 Sep 2020 22:17:28 +0200 Subject: [PATCH 041/885] wallet: Add interface layer --- chain/gen/gen.go | 20 ++--- chain/gen/mining.go | 4 +- chain/messagepool/messagepool_test.go | 24 +++--- chain/messagepool/repub_test.go | 5 +- chain/messagepool/selection_test.go | 44 +++++----- chain/stmgr/forks_test.go | 4 +- chain/sync_test.go | 2 +- chain/types/mock/chain.go | 4 +- chain/vectors/gen/main.go | 9 +- chain/wallet/interface.go | 27 ++++++ chain/wallet/key.go | 81 ++++++++++++++++++ chain/wallet/wallet.go | 114 +++++++------------------- cmd/lotus-keygen/main.go | 4 +- cmd/lotus-shed/keyinfo.go | 2 +- markets/storageadapter/client.go | 4 +- node/builder.go | 4 +- node/impl/full/state.go | 2 +- node/impl/full/wallet.go | 45 ++-------- 18 files changed, 214 insertions(+), 185 deletions(-) create mode 100644 chain/wallet/interface.go create mode 100644 chain/wallet/key.go diff --git a/chain/gen/gen.go b/chain/gen/gen.go index 551c3703f..4ba127ce9 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -68,7 +68,7 @@ type ChainGen struct { GetMessages func(*ChainGen) ([]*types.SignedMessage, error) - w *wallet.Wallet + w *wallet.LocalWallet eppProvs map[address.Address]WinningPoStProver Miners []address.Address @@ -151,14 +151,14 @@ func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) { return nil, xerrors.Errorf("creating memrepo wallet failed: %w", err) } - banker, err := w.GenerateKey(crypto.SigTypeSecp256k1) + banker, err := w.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { return nil, xerrors.Errorf("failed to generate banker key: %w", err) } receievers := make([]address.Address, msgsPerBlock) for r := range receievers { - receievers[r], err = w.GenerateKey(crypto.SigTypeBLS) + receievers[r], err = w.WalletNew(context.Background(), crypto.SigTypeBLS) if err != nil { return nil, xerrors.Errorf("failed to generate receiver key: %w", err) } @@ -188,11 +188,11 @@ func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) { return nil, err } - mk1, err := w.Import(k1) + mk1, err := w.WalletImport(context.Background(), k1) if err != nil { return nil, err } - mk2, err := w.Import(k2) + mk2, err := w.WalletImport(context.Background(), k2) if err != nil { return nil, err } @@ -372,7 +372,7 @@ func (cg *ChainGen) nextBlockProof(ctx context.Context, pts *types.TipSet, m add return nil, nil, nil, xerrors.Errorf("get miner worker: %w", err) } - vrfout, err := ComputeVRF(ctx, cg.w.Sign, worker, ticketRand) + vrfout, err := ComputeVRF(ctx, cg.w.WalletSign, worker, ticketRand) if err != nil { return nil, nil, nil, xerrors.Errorf("compute VRF: %w", err) } @@ -501,7 +501,7 @@ func (cg *ChainGen) Banker() address.Address { return cg.banker } -func (cg *ChainGen) Wallet() *wallet.Wallet { +func (cg *ChainGen) Wallet() *wallet.LocalWallet { return cg.w } @@ -523,7 +523,7 @@ func getRandomMessages(cg *ChainGen) ([]*types.SignedMessage, error) { GasPremium: types.NewInt(0), } - sig, err := cg.w.Sign(context.TODO(), cg.banker, msg.Cid().Bytes()) + sig, err := cg.w.WalletSign(context.TODO(), cg.banker, msg.Cid().Bytes()) if err != nil { return nil, err } @@ -554,7 +554,7 @@ type MiningCheckAPI interface { } type mca struct { - w *wallet.Wallet + w *wallet.LocalWallet sm *stmgr.StateManager pv ffiwrapper.Verifier bcn beacon.RandomBeacon @@ -583,7 +583,7 @@ func (mca mca) MinerGetBaseInfo(ctx context.Context, maddr address.Address, epoc } func (mca mca) WalletSign(ctx context.Context, a address.Address, v []byte) (*crypto.Signature, error) { - return mca.w.Sign(ctx, a, v) + return mca.w.WalletSign(ctx, a, v) } type WinningPoStProver interface { diff --git a/chain/gen/mining.go b/chain/gen/mining.go index 260c96808..a9ac47a60 100644 --- a/chain/gen/mining.go +++ b/chain/gen/mining.go @@ -19,7 +19,7 @@ import ( "github.com/filecoin-project/lotus/lib/sigs/bls" ) -func MinerCreateBlock(ctx context.Context, sm *stmgr.StateManager, w *wallet.Wallet, bt *api.BlockTemplate) (*types.FullBlock, error) { +func MinerCreateBlock(ctx context.Context, sm *stmgr.StateManager, w *wallet.LocalWallet, bt *api.BlockTemplate) (*types.FullBlock, error) { pts, err := sm.ChainStore().LoadTipSet(bt.Parents) if err != nil { @@ -131,7 +131,7 @@ func MinerCreateBlock(ctx context.Context, sm *stmgr.StateManager, w *wallet.Wal return nil, xerrors.Errorf("failed to get signing bytes for block: %w", err) } - sig, err := w.Sign(ctx, waddr, nosigbytes) + sig, err := w.WalletSign(ctx, waddr, nosigbytes) if err != nil { return nil, xerrors.Errorf("failed to sign new block: %w", err) } diff --git a/chain/messagepool/messagepool_test.go b/chain/messagepool/messagepool_test.go index 484c72746..cf77f3fb0 100644 --- a/chain/messagepool/messagepool_test.go +++ b/chain/messagepool/messagepool_test.go @@ -221,7 +221,7 @@ func TestMessagePool(t *testing.T) { a := tma.nextBlock() - sender, err := w.GenerateKey(crypto.SigTypeBLS) + sender, err := w.WalletNew(context.Background(), crypto.SigTypeBLS) if err != nil { t.Fatal(err) } @@ -262,7 +262,7 @@ func TestMessagePoolMessagesInEachBlock(t *testing.T) { a := tma.nextBlock() - sender, err := w.GenerateKey(crypto.SigTypeBLS) + sender, err := w.WalletNew(context.Background(), crypto.SigTypeBLS) if err != nil { t.Fatal(err) } @@ -312,7 +312,7 @@ func TestRevertMessages(t *testing.T) { a := tma.nextBlock() b := tma.nextBlock() - sender, err := w.GenerateKey(crypto.SigTypeBLS) + sender, err := w.WalletNew(context.Background(), crypto.SigTypeBLS) if err != nil { t.Fatal(err) } @@ -375,7 +375,7 @@ func TestPruningSimple(t *testing.T) { a := tma.nextBlock() tma.applyBlock(t, a) - sender, err := w.GenerateKey(crypto.SigTypeBLS) + sender, err := w.WalletNew(context.Background(), crypto.SigTypeBLS) if err != nil { t.Fatal(err) } @@ -422,7 +422,7 @@ func TestLoadLocal(t *testing.T) { t.Fatal(err) } - a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -432,7 +432,7 @@ func TestLoadLocal(t *testing.T) { t.Fatal(err) } - a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -494,7 +494,7 @@ func TestClearAll(t *testing.T) { t.Fatal(err) } - a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -504,7 +504,7 @@ func TestClearAll(t *testing.T) { t.Fatal(err) } - a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -548,7 +548,7 @@ func TestClearNonLocal(t *testing.T) { t.Fatal(err) } - a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -558,7 +558,7 @@ func TestClearNonLocal(t *testing.T) { t.Fatal(err) } - a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -609,7 +609,7 @@ func TestUpdates(t *testing.T) { t.Fatal(err) } - a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -619,7 +619,7 @@ func TestUpdates(t *testing.T) { t.Fatal(err) } - a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } diff --git a/chain/messagepool/repub_test.go b/chain/messagepool/repub_test.go index c89367f0e..2e3fa123f 100644 --- a/chain/messagepool/repub_test.go +++ b/chain/messagepool/repub_test.go @@ -1,6 +1,7 @@ package messagepool import ( + "context" "testing" "time" @@ -32,7 +33,7 @@ func TestRepubMessages(t *testing.T) { t.Fatal(err) } - a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -42,7 +43,7 @@ func TestRepubMessages(t *testing.T) { t.Fatal(err) } - a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } diff --git a/chain/messagepool/selection_test.go b/chain/messagepool/selection_test.go index a9ead3c01..ecbf08b13 100644 --- a/chain/messagepool/selection_test.go +++ b/chain/messagepool/selection_test.go @@ -29,7 +29,7 @@ func init() { MaxActorPendingMessages = 1000000 } -func makeTestMessage(w *wallet.Wallet, from, to address.Address, nonce uint64, gasLimit int64, gasPrice uint64) *types.SignedMessage { +func makeTestMessage(w *wallet.LocalWallet, from, to address.Address, nonce uint64, gasLimit int64, gasPrice uint64) *types.SignedMessage { msg := &types.Message{ From: from, To: to, @@ -40,7 +40,7 @@ func makeTestMessage(w *wallet.Wallet, from, to address.Address, nonce uint64, g GasFeeCap: types.NewInt(100 + gasPrice), GasPremium: types.NewInt(gasPrice), } - sig, err := w.Sign(context.TODO(), from, msg.Cid().Bytes()) + sig, err := w.WalletSign(context.TODO(), from, msg.Cid().Bytes()) if err != nil { panic(err) } @@ -70,7 +70,7 @@ func TestMessageChains(t *testing.T) { t.Fatal(err) } - a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -80,7 +80,7 @@ func TestMessageChains(t *testing.T) { t.Fatal(err) } - a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -308,7 +308,7 @@ func TestMessageChainSkipping(t *testing.T) { t.Fatal(err) } - a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -318,7 +318,7 @@ func TestMessageChainSkipping(t *testing.T) { t.Fatal(err) } - a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -384,7 +384,7 @@ func TestBasicMessageSelection(t *testing.T) { t.Fatal(err) } - a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -394,7 +394,7 @@ func TestBasicMessageSelection(t *testing.T) { t.Fatal(err) } - a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -528,7 +528,7 @@ func TestMessageSelectionTrimming(t *testing.T) { t.Fatal(err) } - a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -538,7 +538,7 @@ func TestMessageSelectionTrimming(t *testing.T) { t.Fatal(err) } - a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -591,7 +591,7 @@ func TestPriorityMessageSelection(t *testing.T) { t.Fatal(err) } - a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -601,7 +601,7 @@ func TestPriorityMessageSelection(t *testing.T) { t.Fatal(err) } - a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -670,7 +670,7 @@ func TestPriorityMessageSelection2(t *testing.T) { t.Fatal(err) } - a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -680,7 +680,7 @@ func TestPriorityMessageSelection2(t *testing.T) { t.Fatal(err) } - a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -740,7 +740,7 @@ func TestOptimalMessageSelection1(t *testing.T) { t.Fatal(err) } - a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -750,7 +750,7 @@ func TestOptimalMessageSelection1(t *testing.T) { t.Fatal(err) } - a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -807,7 +807,7 @@ func TestOptimalMessageSelection2(t *testing.T) { t.Fatal(err) } - a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -817,7 +817,7 @@ func TestOptimalMessageSelection2(t *testing.T) { t.Fatal(err) } - a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -883,7 +883,7 @@ func TestOptimalMessageSelection3(t *testing.T) { nActors := 10 // the actors var actors []address.Address - var wallets []*wallet.Wallet + var wallets []*wallet.LocalWallet for i := 0; i < nActors; i++ { w, err := wallet.NewWallet(wallet.NewMemKeyStore()) @@ -891,7 +891,7 @@ func TestOptimalMessageSelection3(t *testing.T) { t.Fatal(err) } - a, err := w.GenerateKey(crypto.SigTypeSecp256k1) + a, err := w.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -963,7 +963,7 @@ func testCompetitiveMessageSelection(t *testing.T, rng *rand.Rand, getPremium fu nActors := 300 // the actors var actors []address.Address - var wallets []*wallet.Wallet + var wallets []*wallet.LocalWallet for i := 0; i < nActors; i++ { w, err := wallet.NewWallet(wallet.NewMemKeyStore()) @@ -971,7 +971,7 @@ func testCompetitiveMessageSelection(t *testing.T, rng *rand.Rand, getPremium fu t.Fatal(err) } - a, err := w.GenerateKey(crypto.SigTypeSecp256k1) + a, err := w.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } diff --git a/chain/stmgr/forks_test.go b/chain/stmgr/forks_test.go index caa63c879..e5874d51d 100644 --- a/chain/stmgr/forks_test.go +++ b/chain/stmgr/forks_test.go @@ -174,7 +174,7 @@ func TestForkHeightTriggers(t *testing.T) { Params: enc, GasLimit: types.TestGasLimit, } - sig, err := cg.Wallet().Sign(ctx, cg.Banker(), m.Cid().Bytes()) + sig, err := cg.Wallet().WalletSign(ctx, cg.Banker(), m.Cid().Bytes()) if err != nil { t.Fatal(err) } @@ -202,7 +202,7 @@ func TestForkHeightTriggers(t *testing.T) { } nonce++ - sig, err := cg.Wallet().Sign(ctx, cg.Banker(), m.Cid().Bytes()) + sig, err := cg.Wallet().WalletSign(ctx, cg.Banker(), m.Cid().Bytes()) if err != nil { return nil, err } diff --git a/chain/sync_test.go b/chain/sync_test.go index cf1385baa..9a98b3b36 100644 --- a/chain/sync_test.go +++ b/chain/sync_test.go @@ -566,7 +566,7 @@ func TestDuplicateNonce(t *testing.T) { GasPremium: types.NewInt(0), } - sig, err := tu.g.Wallet().Sign(context.TODO(), tu.g.Banker(), msg.Cid().Bytes()) + sig, err := tu.g.Wallet().WalletSign(context.TODO(), tu.g.Banker(), msg.Cid().Bytes()) require.NoError(t, err) return &types.SignedMessage{ diff --git a/chain/types/mock/chain.go b/chain/types/mock/chain.go index 33b13d408..19b1352ad 100644 --- a/chain/types/mock/chain.go +++ b/chain/types/mock/chain.go @@ -21,7 +21,7 @@ func Address(i uint64) address.Address { return a } -func MkMessage(from, to address.Address, nonce uint64, w *wallet.Wallet) *types.SignedMessage { +func MkMessage(from, to address.Address, nonce uint64, w *wallet.LocalWallet) *types.SignedMessage { msg := &types.Message{ To: to, From: from, @@ -32,7 +32,7 @@ func MkMessage(from, to address.Address, nonce uint64, w *wallet.Wallet) *types. GasPremium: types.NewInt(1), } - sig, err := w.Sign(context.TODO(), from, msg.Cid().Bytes()) + sig, err := w.WalletSign(context.TODO(), from, msg.Cid().Bytes()) if err != nil { panic(err) } diff --git a/chain/vectors/gen/main.go b/chain/vectors/gen/main.go index ce8d137e8..631fb6ad1 100644 --- a/chain/vectors/gen/main.go +++ b/chain/vectors/gen/main.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/json" "fmt" "math/rand" @@ -63,11 +64,11 @@ func MakeMessageSigningVectors() []vectors.MessageSigningVector { panic(err) } - blsk, err := w.GenerateKey(crypto.SigTypeBLS) + blsk, err := w.WalletNew(context.Background(), crypto.SigTypeBLS) if err != nil { panic(err) } - bki, err := w.Export(blsk) + bki, err := w.WalletExport(context.Background(), blsk) if err != nil { panic(err) } @@ -87,11 +88,11 @@ func MakeMessageSigningVectors() []vectors.MessageSigningVector { Signature: &bmsg.Signature, } - secpk, err := w.GenerateKey(crypto.SigTypeBLS) + secpk, err := w.WalletNew(context.Background(), crypto.SigTypeBLS) if err != nil { panic(err) } - ski, err := w.Export(secpk) + ski, err := w.WalletExport(context.Background(), secpk) if err != nil { panic(err) } diff --git a/chain/wallet/interface.go b/chain/wallet/interface.go new file mode 100644 index 000000000..9dc95a32c --- /dev/null +++ b/chain/wallet/interface.go @@ -0,0 +1,27 @@ +package wallet + +import ( + "context" + + "github.com/filecoin-project/go-address" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/actors/crypto" +) + +type Wallet interface { + WalletNew(context.Context, crypto.SigType) (address.Address, error) + WalletHas(context.Context, address.Address) (bool, error) + WalletList(context.Context) ([]address.Address, error) + WalletSign(context.Context, address.Address, []byte) (*crypto.Signature, error) + // nonce keeping done by wallet app + WalletSignMessage(context.Context, address.Address, *types.Message) (*types.SignedMessage, error) + WalletExport(context.Context, address.Address) (*types.KeyInfo, error) + WalletImport(context.Context, *types.KeyInfo) (address.Address, error) + WalletDelete(context.Context, address.Address) error +} + +type Default interface { + GetDefault() (address.Address, error) + SetDefault(a address.Address) error +} diff --git a/chain/wallet/key.go b/chain/wallet/key.go new file mode 100644 index 000000000..2d4ecafd7 --- /dev/null +++ b/chain/wallet/key.go @@ -0,0 +1,81 @@ +package wallet + +import ( + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/specs-actors/actors/crypto" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/sigs" +) + +func GenerateKey(typ crypto.SigType) (*Key, error) { + pk, err := sigs.Generate(typ) + if err != nil { + return nil, err + } + ki := types.KeyInfo{ + Type: kstoreSigType(typ), + PrivateKey: pk, + } + return NewKey(ki) +} + +type Key struct { + types.KeyInfo + + PublicKey []byte + Address address.Address +} + +func NewKey(keyinfo types.KeyInfo) (*Key, error) { + k := &Key{ + KeyInfo: keyinfo, + } + + var err error + k.PublicKey, err = sigs.ToPublic(ActSigType(k.Type), k.PrivateKey) + if err != nil { + return nil, err + } + + switch k.Type { + case KTSecp256k1: + k.Address, err = address.NewSecp256k1Address(k.PublicKey) + if err != nil { + return nil, xerrors.Errorf("converting Secp256k1 to address: %w", err) + } + case KTBLS: + k.Address, err = address.NewBLSAddress(k.PublicKey) + if err != nil { + return nil, xerrors.Errorf("converting BLS to address: %w", err) + } + default: + return nil, xerrors.Errorf("unknown key type") + } + return k, nil + +} + +func kstoreSigType(typ crypto.SigType) string { + switch typ { + case crypto.SigTypeBLS: + return KTBLS + case crypto.SigTypeSecp256k1: + return KTSecp256k1 + default: + return "" + } +} + +func ActSigType(typ string) crypto.SigType { + switch typ { + case KTBLS: + return crypto.SigTypeBLS + case KTSecp256k1: + return crypto.SigTypeSecp256k1 + default: + return 0 + } +} diff --git a/chain/wallet/wallet.go b/chain/wallet/wallet.go index 9c069d819..7fdea8963 100644 --- a/chain/wallet/wallet.go +++ b/chain/wallet/wallet.go @@ -29,15 +29,15 @@ const ( KTSecp256k1 = "secp256k1" ) -type Wallet struct { +type LocalWallet struct { keys map[address.Address]*Key keystore types.KeyStore lk sync.Mutex } -func NewWallet(keystore types.KeyStore) (*Wallet, error) { - w := &Wallet{ +func NewWallet(keystore types.KeyStore) (*LocalWallet, error) { + w := &LocalWallet{ keys: make(map[address.Address]*Key), keystore: keystore, } @@ -45,18 +45,18 @@ func NewWallet(keystore types.KeyStore) (*Wallet, error) { return w, nil } -func KeyWallet(keys ...*Key) *Wallet { +func KeyWallet(keys ...*Key) *LocalWallet { m := make(map[address.Address]*Key) for _, key := range keys { m[key.Address] = key } - return &Wallet{ + return &LocalWallet{ keys: m, } } -func (w *Wallet) Sign(ctx context.Context, addr address.Address, msg []byte) (*crypto.Signature, error) { +func (w *LocalWallet) WalletSign(ctx context.Context, addr address.Address, msg []byte) (*crypto.Signature, error) { ki, err := w.findKey(addr) if err != nil { return nil, err @@ -68,7 +68,21 @@ func (w *Wallet) Sign(ctx context.Context, addr address.Address, msg []byte) (*c return sigs.Sign(ActSigType(ki.Type), ki.PrivateKey, msg) } -func (w *Wallet) findKey(addr address.Address) (*Key, error) { +func (w *LocalWallet) WalletSignMessage(ctx context.Context, k address.Address, msg *types.Message) (*types.SignedMessage, error) { + mcid := msg.Cid() + + sig, err := w.WalletSign(ctx, k, mcid.Bytes()) + if err != nil { + return nil, xerrors.Errorf("failed to sign message: %w", err) + } + + return &types.SignedMessage{ + Message: *msg, + Signature: *sig, + }, nil +} + +func (w *LocalWallet) findKey(addr address.Address) (*Key, error) { w.lk.Lock() defer w.lk.Unlock() @@ -96,7 +110,7 @@ func (w *Wallet) findKey(addr address.Address) (*Key, error) { return k, nil } -func (w *Wallet) Export(addr address.Address) (*types.KeyInfo, error) { +func (w *LocalWallet) WalletExport(ctx context.Context, addr address.Address) (*types.KeyInfo, error) { k, err := w.findKey(addr) if err != nil { return nil, xerrors.Errorf("failed to find key to export: %w", err) @@ -105,7 +119,7 @@ func (w *Wallet) Export(addr address.Address) (*types.KeyInfo, error) { return &k.KeyInfo, nil } -func (w *Wallet) Import(ki *types.KeyInfo) (address.Address, error) { +func (w *LocalWallet) WalletImport(ctx context.Context, ki *types.KeyInfo) (address.Address, error) { w.lk.Lock() defer w.lk.Unlock() @@ -121,7 +135,7 @@ func (w *Wallet) Import(ki *types.KeyInfo) (address.Address, error) { return k.Address, nil } -func (w *Wallet) ListAddrs() ([]address.Address, error) { +func (w *LocalWallet) WalletList(ctx context.Context) ([]address.Address, error) { all, err := w.keystore.List() if err != nil { return nil, xerrors.Errorf("listing keystore: %w", err) @@ -144,7 +158,7 @@ func (w *Wallet) ListAddrs() ([]address.Address, error) { return out, nil } -func (w *Wallet) GetDefault() (address.Address, error) { +func (w *LocalWallet) GetDefault() (address.Address, error) { w.lk.Lock() defer w.lk.Unlock() @@ -161,7 +175,7 @@ func (w *Wallet) GetDefault() (address.Address, error) { return k.Address, nil } -func (w *Wallet) SetDefault(a address.Address) error { +func (w *LocalWallet) SetDefault(a address.Address) error { w.lk.Lock() defer w.lk.Unlock() @@ -183,19 +197,7 @@ func (w *Wallet) SetDefault(a address.Address) error { return nil } -func GenerateKey(typ crypto.SigType) (*Key, error) { - pk, err := sigs.Generate(typ) - if err != nil { - return nil, err - } - ki := types.KeyInfo{ - Type: kstoreSigType(typ), - PrivateKey: pk, - } - return NewKey(ki) -} - -func (w *Wallet) GenerateKey(typ crypto.SigType) (address.Address, error) { +func (w *LocalWallet) WalletNew(ctx context.Context, typ crypto.SigType) (address.Address, error) { w.lk.Lock() defer w.lk.Unlock() @@ -223,7 +225,7 @@ func (w *Wallet) GenerateKey(typ crypto.SigType) (address.Address, error) { return k.Address, nil } -func (w *Wallet) HasKey(addr address.Address) (bool, error) { +func (w *LocalWallet) WalletHas(ctx context.Context, addr address.Address) (bool, error) { k, err := w.findKey(addr) if err != nil { return false, err @@ -231,7 +233,7 @@ func (w *Wallet) HasKey(addr address.Address) (bool, error) { return k != nil, nil } -func (w *Wallet) DeleteKey(addr address.Address) error { +func (w *LocalWallet) WalletDelete(ctx context.Context, addr address.Address) error { k, err := w.findKey(addr) if err != nil { return xerrors.Errorf("failed to delete key %s : %w", addr, err) @@ -248,60 +250,4 @@ func (w *Wallet) DeleteKey(addr address.Address) error { return nil } -type Key struct { - types.KeyInfo - - PublicKey []byte - Address address.Address -} - -func NewKey(keyinfo types.KeyInfo) (*Key, error) { - k := &Key{ - KeyInfo: keyinfo, - } - - var err error - k.PublicKey, err = sigs.ToPublic(ActSigType(k.Type), k.PrivateKey) - if err != nil { - return nil, err - } - - switch k.Type { - case KTSecp256k1: - k.Address, err = address.NewSecp256k1Address(k.PublicKey) - if err != nil { - return nil, xerrors.Errorf("converting Secp256k1 to address: %w", err) - } - case KTBLS: - k.Address, err = address.NewBLSAddress(k.PublicKey) - if err != nil { - return nil, xerrors.Errorf("converting BLS to address: %w", err) - } - default: - return nil, xerrors.Errorf("unknown key type") - } - return k, nil - -} - -func kstoreSigType(typ crypto.SigType) string { - switch typ { - case crypto.SigTypeBLS: - return KTBLS - case crypto.SigTypeSecp256k1: - return KTSecp256k1 - default: - return "" - } -} - -func ActSigType(typ string) crypto.SigType { - switch typ { - case KTBLS: - return crypto.SigTypeBLS - case KTSecp256k1: - return crypto.SigTypeSecp256k1 - default: - return 0 - } -} +var _ Wallet = &LocalWallet{} diff --git a/cmd/lotus-keygen/main.go b/cmd/lotus-keygen/main.go index 33124107e..8835a0928 100644 --- a/cmd/lotus-keygen/main.go +++ b/cmd/lotus-keygen/main.go @@ -40,12 +40,12 @@ func main() { return fmt.Errorf("unrecognized key type: %q", cctx.String("type")) } - kaddr, err := w.GenerateKey(kt) + kaddr, err := w.WalletNew(cctx.Context, kt) if err != nil { return err } - ki, err := w.Export(kaddr) + ki, err := w.WalletExport(cctx.Context, kaddr) if err != nil { return err } diff --git a/cmd/lotus-shed/keyinfo.go b/cmd/lotus-shed/keyinfo.go index f397bd4c7..b2ff97e4e 100644 --- a/cmd/lotus-shed/keyinfo.go +++ b/cmd/lotus-shed/keyinfo.go @@ -130,7 +130,7 @@ var keyinfoImportCmd = &cli.Command{ return err } - addr, err := w.Import(&keyInfo) + addr, err := w.WalletImport(cctx.Context, &keyInfo) if err != nil { return err } diff --git a/markets/storageadapter/client.go b/markets/storageadapter/client.go index 91fc6a054..03e5b5353 100644 --- a/markets/storageadapter/client.go +++ b/markets/storageadapter/client.go @@ -440,7 +440,7 @@ func (c *ClientNodeAdapter) SignProposal(ctx context.Context, signer address.Add return nil, err } - sig, err := c.Wallet.Sign(ctx, signer, buf) + sig, err := c.Wallet.WalletSign(ctx, signer, buf) if err != nil { return nil, err } @@ -524,7 +524,7 @@ func (c *ClientNodeAdapter) SignBytes(ctx context.Context, signer address.Addres return nil, err } - localSignature, err := c.Wallet.Sign(ctx, signer, b) + localSignature, err := c.Wallet.WalletSign(ctx, signer, b) if err != nil { return nil, err } diff --git a/node/builder.go b/node/builder.go index 5b6966cd4..7c3251df2 100644 --- a/node/builder.go +++ b/node/builder.go @@ -234,7 +234,9 @@ func Online() Option { Override(new(vm.SyscallBuilder), vm.Syscalls), Override(new(*store.ChainStore), modules.ChainStore), Override(new(*stmgr.StateManager), stmgr.NewStateManager), - Override(new(*wallet.Wallet), wallet.NewWallet), + Override(new(*wallet.LocalWallet), wallet.NewWallet), + Override(new(wallet.Wallet), From(new(*wallet.LocalWallet))), + Override(new(wallet.Default), From(new(*wallet.LocalWallet))), Override(new(dtypes.ChainGCLocker), blockstore.NewGCLocker), Override(new(dtypes.ChainGCBlockstore), modules.ChainGCBlockstore), diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 36721a93d..0e1aad27c 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -48,7 +48,7 @@ type StateAPI struct { // TODO: the wallet here is only needed because we have the MinerCreateBlock // API attached to the state API. It probably should live somewhere better - Wallet *wallet.Wallet + Wallet *wallet.LocalWallet ProofVerifier ffiwrapper.Verifier StateManager *stmgr.StateManager diff --git a/node/impl/full/wallet.go b/node/impl/full/wallet.go index 440f9642f..22b674737 100644 --- a/node/impl/full/wallet.go +++ b/node/impl/full/wallet.go @@ -20,19 +20,8 @@ type WalletAPI struct { fx.In StateManager *stmgr.StateManager - Wallet *wallet.Wallet -} - -func (a *WalletAPI) WalletNew(ctx context.Context, typ crypto.SigType) (address.Address, error) { - return a.Wallet.GenerateKey(typ) -} - -func (a *WalletAPI) WalletHas(ctx context.Context, addr address.Address) (bool, error) { - return a.Wallet.HasKey(addr) -} - -func (a *WalletAPI) WalletList(ctx context.Context) ([]address.Address, error) { - return a.Wallet.ListAddrs() + Default wallet.Default + wallet.Wallet } func (a *WalletAPI) WalletBalance(ctx context.Context, addr address.Address) (types.BigInt, error) { @@ -53,21 +42,15 @@ func (a *WalletAPI) WalletSign(ctx context.Context, k address.Address, msg []byt if err != nil { return nil, xerrors.Errorf("failed to resolve ID address: %w", keyAddr) } - return a.Wallet.Sign(ctx, keyAddr, msg) + return a.WalletSign(ctx, keyAddr, msg) } func (a *WalletAPI) WalletSignMessage(ctx context.Context, k address.Address, msg *types.Message) (*types.SignedMessage, error) { - mcid := msg.Cid() - - sig, err := a.WalletSign(ctx, k, mcid.Bytes()) + keyAddr, err := a.StateManager.ResolveToKeyAddress(ctx, k, nil) if err != nil { - return nil, xerrors.Errorf("failed to sign message: %w", err) + return nil, xerrors.Errorf("failed to resolve ID address: %w", keyAddr) } - - return &types.SignedMessage{ - Message: *msg, - Signature: *sig, - }, nil + return a.Wallet.WalletSignMessage(ctx, keyAddr, msg) } func (a *WalletAPI) WalletVerify(ctx context.Context, k address.Address, msg []byte, sig *crypto.Signature) bool { @@ -75,21 +58,9 @@ func (a *WalletAPI) WalletVerify(ctx context.Context, k address.Address, msg []b } func (a *WalletAPI) WalletDefaultAddress(ctx context.Context) (address.Address, error) { - return a.Wallet.GetDefault() + return a.Default.GetDefault() } func (a *WalletAPI) WalletSetDefault(ctx context.Context, addr address.Address) error { - return a.Wallet.SetDefault(addr) -} - -func (a *WalletAPI) WalletExport(ctx context.Context, addr address.Address) (*types.KeyInfo, error) { - return a.Wallet.Export(addr) -} - -func (a *WalletAPI) WalletImport(ctx context.Context, ki *types.KeyInfo) (address.Address, error) { - return a.Wallet.Import(ki) -} - -func (a *WalletAPI) WalletDelete(ctx context.Context, addr address.Address) error { - return a.Wallet.DeleteKey(addr) + return a.Default.SetDefault(addr) } From 780f6dba34afbb65f178d208fad8a3e636cf56a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 5 Sep 2020 21:36:32 +0200 Subject: [PATCH 042/885] Remote wallet backends --- .gitignore | 2 + .../wallet/interface.go => api/api_wallet.go | 10 +- api/apistruct/permissioned.go | 6 + api/apistruct/struct.go | 46 ++++++ api/client/client.go | 12 ++ chain/wallet/remotewallet/remote.go | 51 +++++++ chain/wallet/wallet.go | 8 +- cmd/lotus-wallet/logged.go | 68 +++++++++ cmd/lotus-wallet/main.go | 141 ++++++++++++++++++ go.sum | 3 + node/builder.go | 6 +- node/config/def.go | 5 + node/impl/full/wallet.go | 5 +- node/repo/fsrepo.go | 9 ++ 14 files changed, 360 insertions(+), 12 deletions(-) rename chain/wallet/interface.go => api/api_wallet.go (84%) create mode 100644 chain/wallet/remotewallet/remote.go create mode 100644 cmd/lotus-wallet/logged.go create mode 100644 cmd/lotus-wallet/main.go diff --git a/.gitignore b/.gitignore index 0424c1f24..a1d152294 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,8 @@ /lotus-fountain /lotus-stats /lotus-bench +/lotus-wallet +/lotus-pcr /bench.json /lotuspond/front/node_modules /lotuspond/front/build diff --git a/chain/wallet/interface.go b/api/api_wallet.go similarity index 84% rename from chain/wallet/interface.go rename to api/api_wallet.go index 9dc95a32c..06b00a75b 100644 --- a/chain/wallet/interface.go +++ b/api/api_wallet.go @@ -1,15 +1,14 @@ -package wallet +package api import ( "context" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/specs-actors/actors/crypto" ) -type Wallet interface { +type WalletAPI interface { WalletNew(context.Context, crypto.SigType) (address.Address, error) WalletHas(context.Context, address.Address) (bool, error) WalletList(context.Context) ([]address.Address, error) @@ -20,8 +19,3 @@ type Wallet interface { WalletImport(context.Context, *types.KeyInfo) (address.Address, error) WalletDelete(context.Context, address.Address) error } - -type Default interface { - GetDefault() (address.Address, error) - SetDefault(a address.Address) error -} diff --git a/api/apistruct/permissioned.go b/api/apistruct/permissioned.go index c93662733..86902d31b 100644 --- a/api/apistruct/permissioned.go +++ b/api/apistruct/permissioned.go @@ -36,3 +36,9 @@ func PermissionedWorkerAPI(a api.WorkerAPI) api.WorkerAPI { auth.PermissionedProxy(AllPermissions, DefaultPerms, a, &out.Internal) return &out } + +func PermissionedWalletAPI(a api.WalletAPI) api.WalletAPI { + var out WalletStruct + auth.PermissionedProxy(AllPermissions, DefaultPerms, a, &out.Internal) + return &out +} diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index dff614001..5eb757c7b 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -336,6 +336,19 @@ type WorkerStruct struct { } } +type WalletStruct struct { + Internal struct { + WalletNew func(context.Context, crypto.SigType) (address.Address, error) `perm:"write"` + WalletHas func(context.Context, address.Address) (bool, error) `perm:"write"` + WalletList func(context.Context) ([]address.Address, error) `perm:"write"` + WalletSign func(context.Context, address.Address, []byte) (*crypto.Signature, error) `perm:"sign"` + WalletSignMessage func(context.Context, address.Address, *types.Message) (*types.SignedMessage, error) `perm:"sign"` + WalletExport func(context.Context, address.Address) (*types.KeyInfo, error) `perm:"admin"` + WalletImport func(context.Context, *types.KeyInfo) (address.Address, error) `perm:"admin"` + WalletDelete func(context.Context, address.Address) error `perm:"write"` + } +} + // CommonStruct func (c *CommonStruct) AuthVerify(ctx context.Context, token string) ([]auth.Permission, error) { @@ -1271,7 +1284,40 @@ func (w *WorkerStruct) Closing(ctx context.Context) (<-chan struct{}, error) { return w.Internal.Closing(ctx) } +func (c *WalletStruct) WalletNew(ctx context.Context, typ crypto.SigType) (address.Address, error) { + return c.Internal.WalletNew(ctx, typ) +} + +func (c *WalletStruct) WalletHas(ctx context.Context, addr address.Address) (bool, error) { + return c.Internal.WalletHas(ctx, addr) +} + +func (c *WalletStruct) WalletList(ctx context.Context) ([]address.Address, error) { + return c.Internal.WalletList(ctx) +} + +func (c *WalletStruct) WalletSign(ctx context.Context, k address.Address, msg []byte) (*crypto.Signature, error) { + return c.Internal.WalletSign(ctx, k, msg) +} + +func (c *WalletStruct) WalletSignMessage(ctx context.Context, k address.Address, msg *types.Message) (*types.SignedMessage, error) { + return c.Internal.WalletSignMessage(ctx, k, msg) +} + +func (c *WalletStruct) WalletExport(ctx context.Context, a address.Address) (*types.KeyInfo, error) { + return c.Internal.WalletExport(ctx, a) +} + +func (c *WalletStruct) WalletImport(ctx context.Context, ki *types.KeyInfo) (address.Address, error) { + return c.Internal.WalletImport(ctx, ki) +} + +func (c *WalletStruct) WalletDelete(ctx context.Context, addr address.Address) error { + return c.Internal.WalletDelete(ctx, addr) +} + var _ api.Common = &CommonStruct{} var _ api.FullNode = &FullNodeStruct{} var _ api.StorageMiner = &StorageMinerStruct{} var _ api.WorkerAPI = &WorkerStruct{} +var _ api.WalletAPI = &WalletStruct{} diff --git a/api/client/client.go b/api/client/client.go index cd915acf0..6e624bdd3 100644 --- a/api/client/client.go +++ b/api/client/client.go @@ -82,3 +82,15 @@ func NewWorkerRPC(ctx context.Context, addr string, requestHeader http.Header) ( return &res, closer, err } + +func NewWalletRPC(ctx context.Context, addr string, requestHeader http.Header) (api.WalletAPI, jsonrpc.ClientCloser, error) { + var res apistruct.WalletStruct + closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin", + []interface{}{ + &res.Internal, + }, + requestHeader, + ) + + return &res, closer, err +} diff --git a/chain/wallet/remotewallet/remote.go b/chain/wallet/remotewallet/remote.go new file mode 100644 index 000000000..712f270b4 --- /dev/null +++ b/chain/wallet/remotewallet/remote.go @@ -0,0 +1,51 @@ +package remotewallet + +import ( + "context" + "go.uber.org/fx" + "golang.org/x/xerrors" + "net/http" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/api/client" + "github.com/filecoin-project/lotus/node/modules/helpers" +) + +type RemoteWallet struct { + api.WalletAPI +} + +func SetupRemoteWallet(url string) func(mctx helpers.MetricsCtx, lc fx.Lifecycle) (*RemoteWallet, error) { + return func(mctx helpers.MetricsCtx, lc fx.Lifecycle) (*RemoteWallet, error) { + /*sp := strings.SplitN(env, ":", 2) + if len(sp) != 2 { + log.Warnf("invalid env(%s) value, missing token or address", envKey) + } else { + ma, err := multiaddr.NewMultiaddr(sp[1]) + if err != nil { + return APIInfo{}, xerrors.Errorf("could not parse multiaddr from env(%s): %w", envKey, err) + } + return APIInfo{ + Addr: ma, + Token: []byte(sp[0]), + }, nil + }*/ + + headers := http.Header{} + /*headers.Add("Authorization", "Bearer "+token)*/ + + wapi, closer, err := client.NewWalletRPC(mctx, url, headers) + if err != nil { + return nil, xerrors.Errorf("creating jsonrpc client: %w", err) + } + + lc.Append(fx.Hook{ + OnStop: func(ctx context.Context) error { + closer() + return nil + }, + }) + + return &RemoteWallet{wapi}, nil + } +} \ No newline at end of file diff --git a/chain/wallet/wallet.go b/chain/wallet/wallet.go index 7fdea8963..bcc8d07b1 100644 --- a/chain/wallet/wallet.go +++ b/chain/wallet/wallet.go @@ -12,6 +12,7 @@ import ( "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/api" _ "github.com/filecoin-project/lotus/lib/sigs/bls" // enable bls signatures _ "github.com/filecoin-project/lotus/lib/sigs/secp" // enable secp signatures @@ -36,6 +37,11 @@ type LocalWallet struct { lk sync.Mutex } +type Default interface { + GetDefault() (address.Address, error) + SetDefault(a address.Address) error +} + func NewWallet(keystore types.KeyStore) (*LocalWallet, error) { w := &LocalWallet{ keys: make(map[address.Address]*Key), @@ -250,4 +256,4 @@ func (w *LocalWallet) WalletDelete(ctx context.Context, addr address.Address) er return nil } -var _ Wallet = &LocalWallet{} +var _ api.WalletAPI = &LocalWallet{} diff --git a/cmd/lotus-wallet/logged.go b/cmd/lotus-wallet/logged.go new file mode 100644 index 000000000..e73daae90 --- /dev/null +++ b/cmd/lotus-wallet/logged.go @@ -0,0 +1,68 @@ +package main + +import ( + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/specs-actors/actors/crypto" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/types" +) + +type LoggedWallet struct { + under api.WalletAPI +} + +func (c *LoggedWallet) WalletNew(ctx context.Context, typ crypto.SigType) (address.Address, error) { + n, err := typ.Name() + if err != nil { + return address.Address{}, err + } + + log.Infow("WalletNew", "type", n) + + return c.under.WalletNew(ctx, typ) +} + +func (c *LoggedWallet) WalletHas(ctx context.Context, addr address.Address) (bool, error) { + log.Infow("WalletHas", "address", addr) + + return c.under.WalletHas(ctx, addr) +} + +func (c *LoggedWallet) WalletList(ctx context.Context) ([]address.Address, error) { + log.Infow("WalletList") + + return c.under.WalletList(ctx) +} + +func (c *LoggedWallet) WalletSign(ctx context.Context, k address.Address, msg []byte) (*crypto.Signature, error) { + log.Infow("WalletSign", "address", k) + + return c.under.WalletSign(ctx, k, msg) +} + +func (c *LoggedWallet) WalletSignMessage(ctx context.Context, k address.Address, msg *types.Message) (*types.SignedMessage, error) { + log.Infow("WalletSignMessage", "address", k) + + return c.under.WalletSignMessage(ctx, k, msg) +} + +func (c *LoggedWallet) WalletExport(ctx context.Context, a address.Address) (*types.KeyInfo, error) { + log.Infow("WalletExport", "address", a) + + return c.under.WalletExport(ctx, a) +} + +func (c *LoggedWallet) WalletImport(ctx context.Context, ki *types.KeyInfo) (address.Address, error) { + log.Infow("WalletImport", "type", ki.Type) + + return c.under.WalletImport(ctx, ki) +} + +func (c *LoggedWallet) WalletDelete(ctx context.Context, addr address.Address) error { + log.Infow("WalletDelete", "address", addr) + + return c.under.WalletDelete(ctx, addr) +} \ No newline at end of file diff --git a/cmd/lotus-wallet/main.go b/cmd/lotus-wallet/main.go new file mode 100644 index 000000000..ae3580a59 --- /dev/null +++ b/cmd/lotus-wallet/main.go @@ -0,0 +1,141 @@ +package main + +import ( + "context" + "net" + "net/http" + "os" + + "github.com/gorilla/mux" + logging "github.com/ipfs/go-log/v2" + "github.com/urfave/cli/v2" + + "github.com/filecoin-project/go-jsonrpc" + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/wallet" + lcli "github.com/filecoin-project/lotus/cli" + "github.com/filecoin-project/lotus/lib/lotuslog" + "github.com/filecoin-project/lotus/node/repo" +) + +var log = logging.Logger("main") + +const FlagWalletRepo = "wallet-repo" + +func main() { + lotuslog.SetupLogLevels() + + local := []*cli.Command{ + runCmd, + } + + app := &cli.App{ + Name: "lotus-wallet", + Usage: "Basic external wallet", + Version: build.UserVersion(), + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: FlagWalletRepo, + EnvVars: []string{"WALLET_PATH"}, + Value: "~/.lotuswallet", // TODO: Consider XDG_DATA_HOME + }, + }, + + Commands: local, + } + app.Setup() + + if err := app.Run(os.Args); err != nil { + log.Warnf("%+v", err) + return + } +} + +var runCmd = &cli.Command{ + Name: "run", + Usage: "Start lotus wallet", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "listen", + Usage: "host address and port the wallet api will listen on", + Value: "0.0.0.0:1777", + }, + }, + Action: func(cctx *cli.Context) error { + log.Info("Starting lotus wallet") + + ctx := lcli.ReqContext(cctx) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + repoPath := cctx.String(FlagWalletRepo) + r, err := repo.NewFS(repoPath) + if err != nil { + return err + } + + ok, err := r.Exists() + if err != nil { + return err + } + if !ok { + if err := r.Init(repo.Worker); err != nil { + return err + } + } + + lr, err := r.Lock(repo.Wallet) + if err != nil { + return err + } + + ks, err := lr.KeyStore() + if err != nil { + return err + } + + w, err := wallet.NewWallet(ks) + if err != nil { + return err + } + + address := cctx.String("listen") + mux := mux.NewRouter() + + log.Info("Setting up API endpoint at " + address) + + rpcServer := jsonrpc.NewServer() + rpcServer.Register("Filecoin", &LoggedWallet{under: w}) + + mux.Handle("/rpc/v0", rpcServer) + mux.PathPrefix("/").Handler(http.DefaultServeMux) // pprof + + /*ah := &auth.Handler{ + Verify: nodeApi.AuthVerify, + Next: mux.ServeHTTP, + }*/ + + srv := &http.Server{ + Handler: mux, + BaseContext: func(listener net.Listener) context.Context { + return ctx + }, + } + + go func() { + <-ctx.Done() + log.Warn("Shutting down...") + if err := srv.Shutdown(context.TODO()); err != nil { + log.Errorf("shutting down RPC server failed: %s", err) + } + log.Warn("Graceful shutdown successful") + }() + + nl, err := net.Listen("tcp", address) + if err != nil { + return err + } + + return srv.Serve(nl) + }, +} diff --git a/go.sum b/go.sum index 0c525579e..ea7b3711e 100644 --- a/go.sum +++ b/go.sum @@ -65,9 +65,11 @@ github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia github.com/akavel/rsrc v0.8.0 h1:zjWn7ukO9Kc5Q62DOJCcxGpXC18RawVtYAGdz2aLlfw= github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= @@ -1867,6 +1869,7 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/node/builder.go b/node/builder.go index 7c3251df2..b653fa429 100644 --- a/node/builder.go +++ b/node/builder.go @@ -40,6 +40,7 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" "github.com/filecoin-project/lotus/chain/wallet" + "github.com/filecoin-project/lotus/chain/wallet/remotewallet" sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/extern/sector-storage/stores" @@ -235,7 +236,7 @@ func Online() Option { Override(new(*store.ChainStore), modules.ChainStore), Override(new(*stmgr.StateManager), stmgr.NewStateManager), Override(new(*wallet.LocalWallet), wallet.NewWallet), - Override(new(wallet.Wallet), From(new(*wallet.LocalWallet))), + Override(new(api.WalletAPI), From(new(*wallet.LocalWallet))), Override(new(wallet.Default), From(new(*wallet.LocalWallet))), Override(new(dtypes.ChainGCLocker), blockstore.NewGCLocker), @@ -419,6 +420,9 @@ func ConfigFullNode(c interface{}) Option { If(cfg.Metrics.HeadNotifs, Override(HeadMetricsKey, metrics.SendHeadNotifs(cfg.Metrics.Nickname)), ), + If(cfg.Wallet.RemoteBackend != "", + Override(new(api.WalletAPI), remotewallet.SetupRemoteWallet(cfg.Wallet.RemoteBackend)), + ), ) } diff --git a/node/config/def.go b/node/config/def.go index 9fee8895a..f5071ada5 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -22,6 +22,7 @@ type FullNode struct { Common Client Client Metrics Metrics + Wallet Wallet } // // Common @@ -105,6 +106,10 @@ type Client struct { IpfsUseForRetrieval bool } +type Wallet struct { + RemoteBackend string +} + func defCommon() Common { return Common{ API: API{ diff --git a/node/impl/full/wallet.go b/node/impl/full/wallet.go index 22b674737..263452cf3 100644 --- a/node/impl/full/wallet.go +++ b/node/impl/full/wallet.go @@ -10,6 +10,7 @@ import ( "github.com/filecoin-project/specs-actors/actors/abi/big" "github.com/filecoin-project/specs-actors/actors/crypto" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/wallet" @@ -21,7 +22,7 @@ type WalletAPI struct { StateManager *stmgr.StateManager Default wallet.Default - wallet.Wallet + api.WalletAPI } func (a *WalletAPI) WalletBalance(ctx context.Context, addr address.Address) (types.BigInt, error) { @@ -50,7 +51,7 @@ func (a *WalletAPI) WalletSignMessage(ctx context.Context, k address.Address, ms if err != nil { return nil, xerrors.Errorf("failed to resolve ID address: %w", keyAddr) } - return a.Wallet.WalletSignMessage(ctx, keyAddr, msg) + return a.WalletAPI.WalletSignMessage(ctx, keyAddr, msg) } func (a *WalletAPI) WalletVerify(ctx context.Context, k address.Address, msg []byte, sig *crypto.Signature) bool { diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index 14085d4ac..2b7fa9ed3 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -44,6 +44,7 @@ const ( FullNode RepoType = iota StorageMiner Worker + Wallet ) func defConfForType(t RepoType) interface{} { @@ -54,6 +55,8 @@ func defConfForType(t RepoType) interface{} { return config.DefaultStorageMiner() case Worker: return &struct{}{} + case Wallet: + return &struct{}{} default: panic(fmt.Sprintf("unknown RepoType(%d)", int(t))) } @@ -87,6 +90,12 @@ func (fsr *FsRepo) Exists() (bool, error) { notexist := os.IsNotExist(err) if notexist { err = nil + + _, err = os.Stat(filepath.Join(fsr.path, fsKeystore)) + notexist = os.IsNotExist(err) + if notexist { + err = nil + } } return !notexist, err } From 7a6ceebb34a95508acdef964519c75929d37cfc4 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Wed, 9 Sep 2020 14:00:15 -0700 Subject: [PATCH 043/885] windowed post generation now returns faulty sectors --- extern/filecoin-ffi | 2 +- .../sector-storage/ffiwrapper/verifier_cgo.go | 17 +++++++++++++---- go.mod | 8 ++++++-- storage/wdpost_run.go | 4 +++- 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/extern/filecoin-ffi b/extern/filecoin-ffi index 405691046..79b324966 160000 --- a/extern/filecoin-ffi +++ b/extern/filecoin-ffi @@ -1 +1 @@ -Subproject commit 40569104603407c999d6c9e4c3f1228cbd4d0e5c +Subproject commit 79b3249666a809871fad12fbf50f68e22218b01c diff --git a/extern/sector-storage/ffiwrapper/verifier_cgo.go b/extern/sector-storage/ffiwrapper/verifier_cgo.go index 0af12da0b..59d0856d1 100644 --- a/extern/sector-storage/ffiwrapper/verifier_cgo.go +++ b/extern/sector-storage/ffiwrapper/verifier_cgo.go @@ -32,16 +32,25 @@ func (sb *Sealer) GenerateWinningPoSt(ctx context.Context, minerID abi.ActorID, return ffi.GenerateWinningPoSt(minerID, privsectors, randomness) } -func (sb *Sealer) GenerateWindowPoSt(ctx context.Context, minerID abi.ActorID, sectorInfo []proof.SectorInfo, randomness abi.PoStRandomness) ([]proof.PoStProof, []abi.SectorID, error) { +func (sb *Sealer) GenerateWindowPoSt(ctx context.Context, minerID abi.ActorID, sectorInfo []proof.SectorInfo, randomness abi.PoStRandomness) ([]proof.PoStProof, []abi.SectorID, []abi.SectorID, error) { randomness[31] &= 0x3f privsectors, skipped, done, err := sb.pubSectorToPriv(ctx, minerID, sectorInfo, nil, abi.RegisteredSealProof.RegisteredWindowPoStProof) if err != nil { - return nil, nil, xerrors.Errorf("gathering sector info: %w", err) + return nil, nil, nil, xerrors.Errorf("gathering sector info: %w", err) } defer done() - proof, err := ffi.GenerateWindowPoSt(minerID, privsectors, randomness) - return proof, skipped, err + proof, faulty, err := ffi.GenerateWindowPoSt(minerID, privsectors, randomness) + + var faultyIDs []abi.SectorID + for _, f := range faulty { + faultyIDs = append(faultyIDs, abi.SectorID{ + Miner: minerID, + Number: f, + }) + } + + return proof, skipped, faultyIDs, err } func (sb *Sealer) pubSectorToPriv(ctx context.Context, mid abi.ActorID, sectorInfo []proof.SectorInfo, faults []abi.SectorNumber, rpt func(abi.RegisteredSealProof) (abi.RegisteredPoStProof, error)) (ffi.SortedPrivateSectorInfo, []abi.SectorID, func(), error) { diff --git a/go.mod b/go.mod index bc4067d67..56c1420d0 100644 --- a/go.mod +++ b/go.mod @@ -2,8 +2,6 @@ module github.com/filecoin-project/lotus go 1.14 -replace github.com/supranational/blst => github.com/supranational/blst v0.1.2-alpha.1 - require ( contrib.go.opencensus.io/exporter/jaeger v0.1.0 contrib.go.opencensus.io/exporter/prometheus v0.1.0 @@ -138,3 +136,9 @@ replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi replace github.com/dgraph-io/badger/v2 => github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200716180832-3ab515320794 replace github.com/filecoin-project/test-vectors => ./extern/test-vectors + +replace github.com/supranational/blst => ./extern/fil-blst/blst + +replace github.com/filecoin-project/fil-blst => ./extern/fil-blst + +replace github.com/filecoin-project/specs-storage => ../specs-storage diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 090648dbb..d3e91e450 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -406,11 +406,13 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di miner.DeadlineInfo return nil, err } - postOut, postSkipped, err := s.prover.GenerateWindowPoSt(ctx, abi.ActorID(mid), sinfos, abi.PoStRandomness(rand)) + postOut, postSkipped, faulty, err := s.prover.GenerateWindowPoSt(ctx, abi.ActorID(mid), sinfos, abi.PoStRandomness(rand)) if err != nil { return nil, xerrors.Errorf("running post failed: %w", err) } + _ = faulty // TODO(magik) + if len(postOut) == 0 { return nil, xerrors.Errorf("received proofs back from generate window post") } From 5e7737f55d5f7b7b0b658d45c481e5e2fb536041 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 10 Sep 2020 02:59:37 +0200 Subject: [PATCH 044/885] wdpost: Handle skipped sectors correctly --- api/test/window_post.go | 2 +- .../sector-storage/ffiwrapper/verifier_cgo.go | 10 +- extern/sector-storage/mock/mock.go | 22 ++- go.mod | 2 - go.sum | 2 + storage/wdpost_run.go | 166 ++++++++++-------- 6 files changed, 123 insertions(+), 81 deletions(-) diff --git a/api/test/window_post.go b/api/test/window_post.go index 59d7ac1d6..3f15c754b 100644 --- a/api/test/window_post.go +++ b/api/test/window_post.go @@ -192,7 +192,7 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector // Drop the partition err = parts[0].Sectors.ForEach(func(sid uint64) error { - return miner.StorageMiner.(*impl.StorageMinerAPI).IStorageMgr.(*mock.SectorMgr).MarkFailed(abi.SectorID{ + return miner.StorageMiner.(*impl.StorageMinerAPI).IStorageMgr.(*mock.SectorMgr).MarkCorrupted(abi.SectorID{ Miner: abi.ActorID(mid), Number: abi.SectorNumber(sid), }, true) diff --git a/extern/sector-storage/ffiwrapper/verifier_cgo.go b/extern/sector-storage/ffiwrapper/verifier_cgo.go index 59d0856d1..d6c0ae35f 100644 --- a/extern/sector-storage/ffiwrapper/verifier_cgo.go +++ b/extern/sector-storage/ffiwrapper/verifier_cgo.go @@ -32,14 +32,18 @@ func (sb *Sealer) GenerateWinningPoSt(ctx context.Context, minerID abi.ActorID, return ffi.GenerateWinningPoSt(minerID, privsectors, randomness) } -func (sb *Sealer) GenerateWindowPoSt(ctx context.Context, minerID abi.ActorID, sectorInfo []proof.SectorInfo, randomness abi.PoStRandomness) ([]proof.PoStProof, []abi.SectorID, []abi.SectorID, error) { +func (sb *Sealer) GenerateWindowPoSt(ctx context.Context, minerID abi.ActorID, sectorInfo []proof.SectorInfo, randomness abi.PoStRandomness) ([]proof.PoStProof, []abi.SectorID, error) { randomness[31] &= 0x3f privsectors, skipped, done, err := sb.pubSectorToPriv(ctx, minerID, sectorInfo, nil, abi.RegisteredSealProof.RegisteredWindowPoStProof) if err != nil { - return nil, nil, nil, xerrors.Errorf("gathering sector info: %w", err) + return nil, nil, xerrors.Errorf("gathering sector info: %w", err) } defer done() + if len(skipped) > 0 { + return nil, skipped, xerrors.Errorf("pubSectorToPriv skipped some sectors") + } + proof, faulty, err := ffi.GenerateWindowPoSt(minerID, privsectors, randomness) var faultyIDs []abi.SectorID @@ -50,7 +54,7 @@ func (sb *Sealer) GenerateWindowPoSt(ctx context.Context, minerID abi.ActorID, s }) } - return proof, skipped, faultyIDs, err + return proof, faultyIDs, err } func (sb *Sealer) pubSectorToPriv(ctx context.Context, mid abi.ActorID, sectorInfo []proof.SectorInfo, faults []abi.SectorNumber, rpt func(abi.RegisteredSealProof) (abi.RegisteredPoStProof, error)) (ffi.SortedPrivateSectorInfo, []abi.SectorID, func(), error) { diff --git a/extern/sector-storage/mock/mock.go b/extern/sector-storage/mock/mock.go index b70350ef8..7609f43b0 100644 --- a/extern/sector-storage/mock/mock.go +++ b/extern/sector-storage/mock/mock.go @@ -68,6 +68,7 @@ const ( type sectorState struct { pieces []cid.Cid failed bool + corrupted bool state int @@ -251,6 +252,18 @@ func (mgr *SectorMgr) MarkFailed(sid abi.SectorID, failed bool) error { return nil } +func (mgr *SectorMgr) MarkCorrupted(sid abi.SectorID, corrupted bool) error { + mgr.lk.Lock() + defer mgr.lk.Unlock() + ss, ok := mgr.sectors[sid] + if !ok { + return fmt.Errorf("no such sector in storage") + } + + ss.corrupted = corrupted + return nil +} + func opFinishWait(ctx context.Context) { val, ok := ctx.Value("opfinish").(chan struct{}) if !ok { @@ -275,6 +288,8 @@ func (mgr *SectorMgr) GenerateWindowPoSt(ctx context.Context, minerID abi.ActorI si := make([]proof.SectorInfo, 0, len(sectorInfo)) var skipped []abi.SectorID + var err error + for _, info := range sectorInfo { sid := abi.SectorID{ Miner: minerID, @@ -283,13 +298,18 @@ func (mgr *SectorMgr) GenerateWindowPoSt(ctx context.Context, minerID abi.ActorI _, found := mgr.sectors[sid] - if found && !mgr.sectors[sid].failed { + if found && !mgr.sectors[sid].failed && !mgr.sectors[sid].corrupted { si = append(si, info) } else { skipped = append(skipped, sid) + err = xerrors.Errorf("skipped some sectors") } } + if err != nil { + return nil, skipped, err + } + return generateFakePoSt(si, abi.RegisteredSealProof.RegisteredWindowPoStProof, randomness), skipped, nil } diff --git a/go.mod b/go.mod index 56c1420d0..4011577df 100644 --- a/go.mod +++ b/go.mod @@ -140,5 +140,3 @@ replace github.com/filecoin-project/test-vectors => ./extern/test-vectors replace github.com/supranational/blst => ./extern/fil-blst/blst replace github.com/filecoin-project/fil-blst => ./extern/fil-blst - -replace github.com/filecoin-project/specs-storage => ../specs-storage diff --git a/go.sum b/go.sum index 0fa13fe70..3f37bff04 100644 --- a/go.sum +++ b/go.sum @@ -250,6 +250,8 @@ github.com/filecoin-project/specs-actors v0.9.7 h1:7PAZ8kdqwBdmgf/23FCkQZLCXcVu0 github.com/filecoin-project/specs-actors v0.9.7/go.mod h1:wM2z+kwqYgXn5Z7scV1YHLyd1Q1cy0R8HfTIWQ0BFGU= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= +github.com/filecoin-project/specs-storage v0.1.1-0.20200909213410-c066548422be h1:UX457RrC0LwL7Bb5kd0WFyzJBxbHOCSw/64oYqeT+Zc= +github.com/filecoin-project/specs-storage v0.1.1-0.20200909213410-c066548422be/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= github.com/filecoin-project/test-vectors/schema v0.0.1 h1:5fNF76nl4qolEvcIsjc0kUADlTMVHO73tW4kXXPnsus= github.com/filecoin-project/test-vectors/schema v0.0.1/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index d3e91e450..4fd4d51e6 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -335,96 +335,114 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di miner.DeadlineInfo Proofs: nil, } - var sinfos []proof.SectorInfo - sidToPart := map[abi.SectorNumber]uint64{} skipCount := uint64(0) + postSkipped := bitfield.New() + var postOut []proof.PoStProof - for partIdx, partition := range partitions { - // TODO: Can do this in parallel - toProve, err := partition.ActiveSectors() + for retries := 0; retries < 5; retries++ { + var sinfos []proof.SectorInfo + sidToPart := map[abi.SectorNumber]int{} + + for partIdx, partition := range partitions { + // TODO: Can do this in parallel + toProve, err := partition.ActiveSectors() + if err != nil { + return nil, xerrors.Errorf("getting active sectors: %w", err) + } + + toProve, err = bitfield.MergeBitFields(toProve, partition.Recoveries) + if err != nil { + return nil, xerrors.Errorf("adding recoveries to set of sectors to prove: %w", err) + } + + toProve, err = bitfield.SubtractBitField(toProve, postSkipped) + if err != nil { + return nil, xerrors.Errorf("toProve - postSkipped: %w", err) + } + + good, err := s.checkSectors(ctx, toProve) + if err != nil { + return nil, xerrors.Errorf("checking sectors to skip: %w", err) + } + + skipped, err := bitfield.SubtractBitField(toProve, good) + if err != nil { + return nil, xerrors.Errorf("toProve - good: %w", err) + } + + sc, err := skipped.Count() + if err != nil { + return nil, xerrors.Errorf("getting skipped sector count: %w", err) + } + + skipCount += sc + + ssi, err := s.sectorsForProof(ctx, good, partition.Sectors, ts) + if err != nil { + return nil, xerrors.Errorf("getting sorted sector info: %w", err) + } + + if len(ssi) == 0 { + continue + } + + sinfos = append(sinfos, ssi...) + for _, si := range ssi { + sidToPart[si.SectorNumber] = partIdx + } + + params.Partitions = append(params.Partitions, miner.PoStPartition{ + Index: uint64(partIdx), + Skipped: skipped, + }) + } + + if len(sinfos) == 0 { + // nothing to prove.. + return nil, errNoPartitions + } + + log.Infow("running windowPost", + "chain-random", rand, + "deadline", di, + "height", ts.Height(), + "skipped", skipCount) + + tsStart := build.Clock.Now() + + mid, err := address.IDFromAddress(s.actor) if err != nil { - return nil, xerrors.Errorf("getting active sectors: %w", err) + return nil, err } - toProve, err = bitfield.MergeBitFields(toProve, partition.Recoveries) - if err != nil { - return nil, xerrors.Errorf("adding recoveries to set of sectors to prove: %w", err) + var ps []abi.SectorID + postOut, ps, err = s.prover.GenerateWindowPoSt(ctx, abi.ActorID(mid), sinfos, abi.PoStRandomness(rand)) + elapsed := time.Since(tsStart) + + log.Infow("computing window PoSt", "elapsed", elapsed) + + if err == nil { + break } - good, err := s.checkSectors(ctx, toProve) - if err != nil { - return nil, xerrors.Errorf("checking sectors to skip: %w", err) + if len(ps) == 0 { + return nil, xerrors.Errorf("running post failed: %w", err) } - skipped, err := bitfield.SubtractBitField(toProve, good) - if err != nil { - return nil, xerrors.Errorf("toProve - good: %w", err) + log.Warnw("generate window PoSt skipped sectors", "sectors", ps, "error", err, "try", retries) + + skipCount += uint64(len(ps)) + for _, sector := range ps { + postSkipped.Set(uint64(sector.Number)) } - - sc, err := skipped.Count() - if err != nil { - return nil, xerrors.Errorf("getting skipped sector count: %w", err) - } - - skipCount += sc - - ssi, err := s.sectorsForProof(ctx, good, partition.Sectors, ts) - if err != nil { - return nil, xerrors.Errorf("getting sorted sector info: %w", err) - } - - if len(ssi) == 0 { - continue - } - - sinfos = append(sinfos, ssi...) - for _, si := range ssi { - sidToPart[si.SectorNumber] = uint64(partIdx) - } - - params.Partitions = append(params.Partitions, miner.PoStPartition{ - Index: uint64(partIdx), - Skipped: skipped, - }) } - if len(sinfos) == 0 { - // nothing to prove.. - return nil, errNoPartitions - } - - log.Infow("running windowPost", - "chain-random", rand, - "deadline", di, - "height", ts.Height(), - "skipped", skipCount) - - tsStart := build.Clock.Now() - - mid, err := address.IDFromAddress(s.actor) - if err != nil { - return nil, err - } - - postOut, postSkipped, faulty, err := s.prover.GenerateWindowPoSt(ctx, abi.ActorID(mid), sinfos, abi.PoStRandomness(rand)) - if err != nil { - return nil, xerrors.Errorf("running post failed: %w", err) - } - - _ = faulty // TODO(magik) - if len(postOut) == 0 { - return nil, xerrors.Errorf("received proofs back from generate window post") + return nil, xerrors.Errorf("received no proofs back from generate window post") } params.Proofs = postOut - for _, sector := range postSkipped { - params.Partitions[sidToPart[sector.Number]].Skipped.Set(uint64(sector.Number)) - } - - elapsed := time.Since(tsStart) - commEpoch := di.Open commRand, err := s.api.ChainGetRandomnessFromTickets(ctx, ts.Key(), crypto.DomainSeparationTag_PoStChainCommit, commEpoch, nil) if err != nil { @@ -433,7 +451,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di miner.DeadlineInfo params.ChainCommitEpoch = commEpoch params.ChainCommitRand = commRand - log.Infow("submitting window PoSt", "elapsed", elapsed) + log.Infow("submitting window PoSt") return params, nil } From d4b185be5ebda7e48dd2cd18c8bae5480c9027c4 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 10 Sep 2020 14:22:25 +0100 Subject: [PATCH 045/885] test: add chaos actor tests --- conformance/chaos/actor_test.go | 152 ++++++++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 conformance/chaos/actor_test.go diff --git a/conformance/chaos/actor_test.go b/conformance/chaos/actor_test.go new file mode 100644 index 000000000..3503bf0b8 --- /dev/null +++ b/conformance/chaos/actor_test.go @@ -0,0 +1,152 @@ +package chaos + +import ( + "context" + "github.com/filecoin-project/go-state-types/exitcode" + "github.com/filecoin-project/specs-actors/actors/util/adt" + "github.com/filecoin-project/specs-actors/support/mock" + atesting "github.com/filecoin-project/specs-actors/support/testing" + "testing" +) + +func TestSingleton(t *testing.T) { + receiver := atesting.NewIDAddr(t, 100) + builder := mock.NewBuilder(context.Background(), receiver) + + rt := builder.Build(t) + a := Actor{} + + msg := "constructor should not be called; the Chaos actor is a singleton actor" + rt.ExpectAssertionFailure(msg, func() { + rt.Call(a.Constructor, adt.Empty) + }) + rt.Verify() +} + +func TestDeleteActor(t *testing.T) { + receiver := atesting.NewIDAddr(t, 100) + beneficiary := atesting.NewIDAddr(t, 101) + builder := mock.NewBuilder(context.Background(), receiver) + + rt := builder.Build(t) + a := Actor{} + + rt.ExpectValidateCallerAny() + rt.ExpectDeleteActor(beneficiary) + rt.Call(a.DeleteActor, &beneficiary) + rt.Verify() +} + +func TestMutateStateInTransaction(t *testing.T) { + receiver := atesting.NewIDAddr(t, 100) + builder := mock.NewBuilder(context.Background(), receiver) + + rt := builder.Build(t) + a := Actor{} + + rt.ExpectValidateCallerAny() + rt.Create(&State{}) + + val := "__mutstat test" + rt.Call(a.MutateState, &MutateStateArgs{ + Value: val, + Branch: MutateInTransaction, + }) + + var st State + rt.GetState(&st) + + if st.Value != val { + t.Fatal("state was not updated") + } + + rt.Verify() +} + +func TestMutateStateAfterTransaction(t *testing.T) { + receiver := atesting.NewIDAddr(t, 100) + builder := mock.NewBuilder(context.Background(), receiver) + + rt := builder.Build(t) + a := Actor{} + + rt.ExpectValidateCallerAny() + rt.Create(&State{}) + + val := "__mutstat test" + rt.Call(a.MutateState, &MutateStateArgs{ + Value: val, + Branch: MutateAfterTransaction, + }) + + var st State + rt.GetState(&st) + + // state should be updated successfully _in_ the transaction but not outside + if st.Value != val+"-in" { + t.Fatal("state was not updated") + } + + rt.Verify() +} + +func TestMutateStateReadonly(t *testing.T) { + receiver := atesting.NewIDAddr(t, 100) + builder := mock.NewBuilder(context.Background(), receiver) + + rt := builder.Build(t) + a := Actor{} + + rt.ExpectValidateCallerAny() + rt.Create(&State{}) + + val := "__mutstat test" + rt.Call(a.MutateState, &MutateStateArgs{ + Value: val, + Branch: MutateReadonly, + }) + + var st State + rt.GetState(&st) + + if st.Value != "" { + t.Fatal("state was not expected to be updated") + } + + rt.Verify() +} + +func TestAbortWith(t *testing.T) { + receiver := atesting.NewIDAddr(t, 100) + builder := mock.NewBuilder(context.Background(), receiver) + + rt := builder.Build(t) + a := Actor{} + + msg := "__test forbidden" + rt.ExpectAbortContainsMessage(exitcode.ErrForbidden, msg, func() { + rt.Call(a.AbortWith, &AbortWithArgs{ + Code: exitcode.ErrForbidden, + Message: msg, + Uncontrolled: false, + }) + }) + rt.Verify() +} + +func TestAbortWithUncontrolled(t *testing.T) { + receiver := atesting.NewIDAddr(t, 100) + builder := mock.NewBuilder(context.Background(), receiver) + + rt := builder.Build(t) + a := Actor{} + + msg := "__test uncontrolled panic" + rt.ExpectAssertionFailure(msg, func() { + rt.Call(a.AbortWith, &AbortWithArgs{ + Message: msg, + Uncontrolled: true, + }) + }) + rt.Verify() +} From a214069586f38d4b7c217df9bbd42079180a515e Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 10 Sep 2020 14:30:52 +0100 Subject: [PATCH 046/885] chore: appease linter --- conformance/chaos/actor_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conformance/chaos/actor_test.go b/conformance/chaos/actor_test.go index 3503bf0b8..ac49b8670 100644 --- a/conformance/chaos/actor_test.go +++ b/conformance/chaos/actor_test.go @@ -2,11 +2,12 @@ package chaos import ( "context" + "testing" + "github.com/filecoin-project/go-state-types/exitcode" "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/specs-actors/support/mock" atesting "github.com/filecoin-project/specs-actors/support/testing" - "testing" ) func TestSingleton(t *testing.T) { From c94ff3430ca4d2ba84ed635b07183a4c23e338a0 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 10 Sep 2020 14:51:30 +0100 Subject: [PATCH 047/885] refactor: use nil value --- conformance/chaos/actor_test.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/conformance/chaos/actor_test.go b/conformance/chaos/actor_test.go index ac49b8670..97c3b85cf 100644 --- a/conformance/chaos/actor_test.go +++ b/conformance/chaos/actor_test.go @@ -15,7 +15,7 @@ func TestSingleton(t *testing.T) { builder := mock.NewBuilder(context.Background(), receiver) rt := builder.Build(t) - a := Actor{} + var a Actor msg := "constructor should not be called; the Chaos actor is a singleton actor" rt.ExpectAssertionFailure(msg, func() { @@ -30,7 +30,7 @@ func TestDeleteActor(t *testing.T) { builder := mock.NewBuilder(context.Background(), receiver) rt := builder.Build(t) - a := Actor{} + var a Actor rt.ExpectValidateCallerAny() rt.ExpectDeleteActor(beneficiary) @@ -43,7 +43,7 @@ func TestMutateStateInTransaction(t *testing.T) { builder := mock.NewBuilder(context.Background(), receiver) rt := builder.Build(t) - a := Actor{} + var a Actor rt.ExpectValidateCallerAny() rt.Create(&State{}) @@ -69,7 +69,7 @@ func TestMutateStateAfterTransaction(t *testing.T) { builder := mock.NewBuilder(context.Background(), receiver) rt := builder.Build(t) - a := Actor{} + var a Actor rt.ExpectValidateCallerAny() rt.Create(&State{}) @@ -96,7 +96,7 @@ func TestMutateStateReadonly(t *testing.T) { builder := mock.NewBuilder(context.Background(), receiver) rt := builder.Build(t) - a := Actor{} + var a Actor rt.ExpectValidateCallerAny() rt.Create(&State{}) @@ -122,7 +122,7 @@ func TestAbortWith(t *testing.T) { builder := mock.NewBuilder(context.Background(), receiver) rt := builder.Build(t) - a := Actor{} + var a Actor msg := "__test forbidden" rt.ExpectAbortContainsMessage(exitcode.ErrForbidden, msg, func() { @@ -140,7 +140,7 @@ func TestAbortWithUncontrolled(t *testing.T) { builder := mock.NewBuilder(context.Background(), receiver) rt := builder.Build(t) - a := Actor{} + var a Actor msg := "__test uncontrolled panic" rt.ExpectAssertionFailure(msg, func() { From dc85a276cdca4064b1e4187e8e49095b63d814b4 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 10 Sep 2020 19:17:48 +0200 Subject: [PATCH 048/885] Upgrade badger Signed-off-by: Jakub Sztandera --- go.mod | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 903cfaae4..ac5106e1a 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/buger/goterm v0.0.0-20200322175922-2f3e71b85129 github.com/coreos/go-systemd/v22 v22.0.0 github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e - github.com/dgraph-io/badger/v2 v2.0.3 + github.com/dgraph-io/badger/v2 v2.2007.2 github.com/docker/go-units v0.4.0 github.com/drand/drand v1.0.3-0.20200714175734-29705eaf09d4 github.com/drand/kyber v1.1.1 @@ -135,6 +135,4 @@ replace github.com/golangci/golangci-lint => github.com/golangci/golangci-lint v replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi -replace github.com/dgraph-io/badger/v2 => github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200716180832-3ab515320794 - replace github.com/filecoin-project/test-vectors => ./extern/test-vectors From 64d150e215256cab04c0eb52819ca2675330b151 Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Thu, 10 Sep 2020 10:35:06 -0700 Subject: [PATCH 049/885] feat(cli): add chain delete cmd --- api/api_full.go | 3 +++ api/apistruct/struct.go | 5 +++++ cli/chain.go | 27 +++++++++++++++++++++++++++ node/impl/full/chain.go | 4 ++++ 4 files changed, 39 insertions(+) diff --git a/api/api_full.go b/api/api_full.go index 02417bf78..3dde0e517 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -75,6 +75,9 @@ type FullNode interface { // blockstore and returns raw bytes. ChainReadObj(context.Context, cid.Cid) ([]byte, error) + // ChainDeleteObj deletes node referenced by the given CID + ChainDeleteObj(context.Context, cid.Cid) error + // ChainHasObj checks if a given CID exists in the chain blockstore. ChainHasObj(context.Context, cid.Cid) (bool, error) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index ffb837785..172156c42 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -85,6 +85,7 @@ type FullNodeStruct struct { ChainGetParentMessages func(context.Context, cid.Cid) ([]api.Message, error) `perm:"read"` ChainGetTipSetByHeight func(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error) `perm:"read"` ChainReadObj func(context.Context, cid.Cid) ([]byte, error) `perm:"read"` + ChainDeleteObj func(context.Context, cid.Cid) error `perm:"admin"` ChainHasObj func(context.Context, cid.Cid) (bool, error) `perm:"read"` ChainStatObj func(context.Context, cid.Cid, cid.Cid) (api.ObjStat, error) `perm:"read"` ChainSetHead func(context.Context, types.TipSetKey) error `perm:"admin"` @@ -658,6 +659,10 @@ func (c *FullNodeStruct) ChainReadObj(ctx context.Context, obj cid.Cid) ([]byte, return c.Internal.ChainReadObj(ctx, obj) } +func (c *FullNodeStruct) ChainDeleteObj(ctx context.Context, obj cid.Cid) error { + return c.Internal.ChainDeleteObj(ctx, obj) +} + func (c *FullNodeStruct) ChainHasObj(ctx context.Context, o cid.Cid) (bool, error) { return c.Internal.ChainHasObj(ctx, o) } diff --git a/cli/chain.go b/cli/chain.go index ce1660641..1c63e37a8 100644 --- a/cli/chain.go +++ b/cli/chain.go @@ -193,6 +193,33 @@ var chainReadObjCmd = &cli.Command{ }, } +var chainDeleteObjCmd = &cli.Command{ + Name: "delete-obj", + Usage: "Delete an object", + ArgsUsage: "[objectCid]", + Action: func(cctx *cli.Context) error { + api, closer, err := GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + ctx := ReqContext(cctx) + + c, err := cid.Decode(cctx.Args().First()) + if err != nil { + return fmt.Errorf("failed to parse cid input: %s", err) + } + + err = api.ChainDeleteObj(ctx, c) + if err != nil { + return err + } + + fmt.Printf("Obj %s deleted\n", c.String()) + return nil + }, +} + var chainStatObjCmd = &cli.Command{ Name: "stat-obj", Usage: "Collect size and ipld link counts for objs", diff --git a/node/impl/full/chain.go b/node/impl/full/chain.go index c5dd5c9a9..84335280d 100644 --- a/node/impl/full/chain.go +++ b/node/impl/full/chain.go @@ -197,6 +197,10 @@ func (a *ChainAPI) ChainReadObj(ctx context.Context, obj cid.Cid) ([]byte, error return blk.RawData(), nil } +func (a *ChainAPI) ChainDeleteObj(ctx context.Context, obj cid.Cid) error { + return a.Chain.Blockstore().DeleteBlock(obj) +} + func (a *ChainAPI) ChainHasObj(ctx context.Context, obj cid.Cid) (bool, error) { return a.Chain.Blockstore().Has(obj) } From e86a74156e85c520b7ef63d885ee7b3874815340 Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Thu, 10 Sep 2020 10:39:17 -0700 Subject: [PATCH 050/885] feat(cli): add command to list --- cli/chain.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cli/chain.go b/cli/chain.go index 1c63e37a8..0f1e36518 100644 --- a/cli/chain.go +++ b/cli/chain.go @@ -40,6 +40,7 @@ var chainCmd = &cli.Command{ chainHeadCmd, chainGetBlock, chainReadObjCmd, + chainDeleteObjCmd, chainStatObjCmd, chainGetMsgCmd, chainSetHeadCmd, From 5e445f5a4803abfbffa977b9f52da5f72dd94bc9 Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Thu, 10 Sep 2020 10:47:19 -0700 Subject: [PATCH 051/885] docs(apidocs): run docs gen --- documentation/en/api-methods.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index 49582d6f8..203a5f141 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -9,6 +9,7 @@ * [Beacon](#Beacon) * [BeaconGetEntry](#BeaconGetEntry) * [Chain](#Chain) + * [ChainDeleteObj](#ChainDeleteObj) * [ChainExport](#ChainExport) * [ChainGetBlock](#ChainGetBlock) * [ChainGetBlockMessages](#ChainGetBlockMessages) @@ -279,6 +280,23 @@ The Chain method group contains methods for interacting with the blockchain, but that do not require any form of state computation. +### ChainDeleteObj +ChainDeleteObj deletes node referenced by the given CID + + +Perms: admin + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: `{}` + ### ChainExport ChainExport returns a stream of bytes with CAR dump of chain data. The exported chain data includes the header chain from the given tipset From c610bd818ea3b73414d4521c10ac5e8333b8e447 Mon Sep 17 00:00:00 2001 From: jennijuju Date: Thu, 10 Sep 2020 14:34:18 -0400 Subject: [PATCH 052/885] Increased ExpectedSealDuration and and WaitDealsDelay. --- node/config/def.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/node/config/def.go b/node/config/def.go index 9fee8895a..1e216238e 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -147,7 +147,7 @@ func DefaultStorageMiner() *StorageMiner { MaxWaitDealsSectors: 2, // 64G with 32G sectors MaxSealingSectors: 0, MaxSealingSectorsForDeals: 0, - WaitDealsDelay: Duration(time.Hour), + WaitDealsDelay: Duration(time.Hour * 6), }, Storage: sectorstorage.SealerConfig{ @@ -169,7 +169,7 @@ func DefaultStorageMiner() *StorageMiner { ConsiderOfflineRetrievalDeals: true, PieceCidBlocklist: []cid.Cid{}, // TODO: It'd be nice to set this based on sector size - ExpectedSealDuration: Duration(time.Hour * 12), + ExpectedSealDuration: Duration(time.Hour * 24), }, Fees: MinerFeeConfig{ From c7b0241a48b5f503cfc7a32e3ce2602e412a0fe1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 10 Sep 2020 21:19:26 +0200 Subject: [PATCH 053/885] ffiwrapper: Test skipping corrupted sectors in PoSt --- .../sector-storage/ffiwrapper/sealer_test.go | 87 ++++++++++--------- 1 file changed, 46 insertions(+), 41 deletions(-) diff --git a/extern/sector-storage/ffiwrapper/sealer_test.go b/extern/sector-storage/ffiwrapper/sealer_test.go index ae1ede7eb..acbd9ca21 100644 --- a/extern/sector-storage/ffiwrapper/sealer_test.go +++ b/extern/sector-storage/ffiwrapper/sealer_test.go @@ -168,50 +168,34 @@ func (s *seal) unseal(t *testing.T, sb *Sealer, sp *basicfs.Provider, si abi.Sec } } -func post(t *testing.T, sealer *Sealer, seals ...seal) time.Time { - /*randomness := abi.PoStRandomness{0, 9, 2, 7, 6, 5, 4, 3, 2, 1, 0, 9, 8, 7, 6, 45, 3, 2, 1, 0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9, 7} +func post(t *testing.T, sealer *Sealer, skipped []abi.SectorID, seals ...seal) { + randomness := abi.PoStRandomness{0, 9, 2, 7, 6, 5, 4, 3, 2, 1, 0, 9, 8, 7, 6, 45, 3, 2, 1, 0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9, 7} - sis := make([]abi.SectorInfo, len(seals)) + sis := make([]saproof.SectorInfo, len(seals)) for i, s := range seals { - sis[i] = abi.SectorInfo{ - RegisteredProof: sealProofType, + sis[i] = saproof.SectorInfo{ + SealProof: sealProofType, SectorNumber: s.id.Number, SealedCID: s.cids.Sealed, } } - candidates, err := sealer.GenerateEPostCandidates(context.TODO(), seals[0].id.Miner, sis, randomness, []abi.SectorNumber{}) - if err != nil { - t.Fatalf("%+v", err) - }*/ - - fmt.Println("skipping post") - - genCandidates := time.Now() - - /*if len(candidates) != 1 { - t.Fatal("expected 1 candidate") + proofs, skp, err := sealer.GenerateWindowPoSt(context.TODO(), seals[0].id.Miner, sis, randomness) + if len(skipped) > 0 { + require.Error(t, err) + require.EqualValues(t, skipped, skp) + return } - candidatesPrime := make([]abi.PoStCandidate, len(candidates)) - for idx := range candidatesPrime { - candidatesPrime[idx] = candidates[idx].Candidate - } - - proofs, err := sealer.ComputeElectionPoSt(context.TODO(), seals[0].id.Miner, sis, randomness, candidatesPrime) if err != nil { t.Fatalf("%+v", err) } - ePoStChallengeCount := ElectionPostChallengeCount(uint64(len(sis)), 0) - - ok, err := ProofVerifier.VerifyElectionPost(context.TODO(), abi.PoStVerifyInfo{ + ok, err := ProofVerifier.VerifyWindowPoSt(context.TODO(), saproof.WindowPoStVerifyInfo{ Randomness: randomness, - Candidates: candidatesPrime, Proofs: proofs, - EligibleSectors: sis, + ChallengedSectors: sis, Prover: seals[0].id.Miner, - ChallengeCount: ePoStChallengeCount, }) if err != nil { t.Fatalf("%+v", err) @@ -219,8 +203,21 @@ func post(t *testing.T, sealer *Sealer, seals ...seal) time.Time { if !ok { t.Fatal("bad post") } - */ - return genCandidates +} + +func corrupt(t *testing.T, sealer *Sealer, id abi.SectorID) { + paths, done, err := sealer.sectors.AcquireSector(context.Background(), id, stores.FTSealed, 0, stores.PathStorage) + require.NoError(t, err) + defer done() + + log.Infof("corrupt %s", paths.Sealed) + f, err := os.OpenFile(paths.Sealed, os.O_RDWR, 0664) + require.NoError(t, err) + + _, err = f.WriteAt(bytes.Repeat([]byte{'d'}, 2048), 0) + require.NoError(t, err) + + require.NoError(t, f.Close()) } func getGrothParamFileAndVerifyingKeys(s abi.SectorSize) { @@ -299,11 +296,11 @@ func TestSealAndVerify(t *testing.T) { commit := time.Now() - genCandidiates := post(t, sb, s) + post(t, sb, nil, s) epost := time.Now() - post(t, sb, s) + post(t, sb, nil, s) if err := sb.FinalizeSector(context.TODO(), si, nil); err != nil { t.Fatalf("%+v", err) @@ -313,8 +310,7 @@ func TestSealAndVerify(t *testing.T) { fmt.Printf("PreCommit: %s\n", precommit.Sub(start).String()) fmt.Printf("Commit: %s\n", commit.Sub(precommit).String()) - fmt.Printf("GenCandidates: %s\n", genCandidiates.Sub(commit).String()) - fmt.Printf("EPoSt: %s\n", epost.Sub(genCandidiates).String()) + fmt.Printf("EPoSt: %s\n", epost.Sub(commit).String()) } func TestSealPoStNoCommit(t *testing.T) { @@ -370,16 +366,15 @@ func TestSealPoStNoCommit(t *testing.T) { t.Fatal(err) } - genCandidiates := post(t, sb, s) + post(t, sb, nil, s) epost := time.Now() fmt.Printf("PreCommit: %s\n", precommit.Sub(start).String()) - fmt.Printf("GenCandidates: %s\n", genCandidiates.Sub(precommit).String()) - fmt.Printf("EPoSt: %s\n", epost.Sub(genCandidiates).String()) + fmt.Printf("EPoSt: %s\n", epost.Sub(precommit).String()) } -func TestSealAndVerify2(t *testing.T) { +func TestSealAndVerify3(t *testing.T) { defer requireFDsClosed(t, openFDs(t)) if runtime.NumCPU() < 10 && os.Getenv("CI") == "" { // don't bother on slow hardware @@ -419,22 +414,32 @@ func TestSealAndVerify2(t *testing.T) { si1 := abi.SectorID{Miner: miner, Number: 1} si2 := abi.SectorID{Miner: miner, Number: 2} + si3 := abi.SectorID{Miner: miner, Number: 3} s1 := seal{id: si1} s2 := seal{id: si2} + s3 := seal{id: si3} - wg.Add(2) + wg.Add(3) go s1.precommit(t, sb, si1, wg.Done) //nolint: staticcheck time.Sleep(100 * time.Millisecond) go s2.precommit(t, sb, si2, wg.Done) //nolint: staticcheck + time.Sleep(100 * time.Millisecond) + go s3.precommit(t, sb, si3, wg.Done) //nolint: staticcheck wg.Wait() - wg.Add(2) + wg.Add(3) go s1.commit(t, sb, wg.Done) //nolint: staticcheck go s2.commit(t, sb, wg.Done) //nolint: staticcheck + go s3.commit(t, sb, wg.Done) //nolint: staticcheck wg.Wait() - post(t, sb, s1, s2) + post(t, sb, nil, s1, s2, s3) + + corrupt(t, sb, si1) + corrupt(t, sb, si2) + + post(t, sb, []abi.SectorID{si1, si2}, s1, s2, s3) } func BenchmarkWriteWithAlignment(b *testing.B) { From 2374cb1bc599cfab8c0a62707fc4b6ec15f4568c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 10 Sep 2020 21:44:01 +0200 Subject: [PATCH 054/885] Update ffi --- extern/filecoin-ffi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/filecoin-ffi b/extern/filecoin-ffi index 79b324966..f640612a1 160000 --- a/extern/filecoin-ffi +++ b/extern/filecoin-ffi @@ -1 +1 @@ -Subproject commit 79b3249666a809871fad12fbf50f68e22218b01c +Subproject commit f640612a1a1f7a2dd8b3a49e1531db0aa0f63447 From 22556fb24c99c44390c914597e75d18357758647 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 10 Sep 2020 22:04:08 +0200 Subject: [PATCH 055/885] Add fil-blst submodule --- .gitmodules | 3 +++ extern/fil-blst | 1 + 2 files changed, 4 insertions(+) create mode 160000 extern/fil-blst diff --git a/.gitmodules b/.gitmodules index ad09aba35..4b450aaf3 100644 --- a/.gitmodules +++ b/.gitmodules @@ -8,3 +8,6 @@ [submodule "extern/test-vectors"] path = extern/test-vectors url = https://github.com/filecoin-project/test-vectors.git +[submodule "extern/fil-blst"] + path = extern/fil-blst + url = https://github.com/filecoin-project/fil-blst.git diff --git a/extern/fil-blst b/extern/fil-blst new file mode 160000 index 000000000..5f93488fc --- /dev/null +++ b/extern/fil-blst @@ -0,0 +1 @@ +Subproject commit 5f93488fc0dbfb450f2355269f18fc67010d59bb From bbac86f7454222adb1b4a89880b7625270ca4eb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 10 Sep 2020 22:07:20 +0200 Subject: [PATCH 056/885] gofmt, mod tidy --- extern/sector-storage/ffiwrapper/sealer_test.go | 12 ++++++------ extern/sector-storage/mock/mock.go | 4 ++-- go.sum | 4 ---- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/extern/sector-storage/ffiwrapper/sealer_test.go b/extern/sector-storage/ffiwrapper/sealer_test.go index acbd9ca21..d59de2cab 100644 --- a/extern/sector-storage/ffiwrapper/sealer_test.go +++ b/extern/sector-storage/ffiwrapper/sealer_test.go @@ -174,9 +174,9 @@ func post(t *testing.T, sealer *Sealer, skipped []abi.SectorID, seals ...seal) { sis := make([]saproof.SectorInfo, len(seals)) for i, s := range seals { sis[i] = saproof.SectorInfo{ - SealProof: sealProofType, - SectorNumber: s.id.Number, - SealedCID: s.cids.Sealed, + SealProof: sealProofType, + SectorNumber: s.id.Number, + SealedCID: s.cids.Sealed, } } @@ -192,10 +192,10 @@ func post(t *testing.T, sealer *Sealer, skipped []abi.SectorID, seals ...seal) { } ok, err := ProofVerifier.VerifyWindowPoSt(context.TODO(), saproof.WindowPoStVerifyInfo{ - Randomness: randomness, - Proofs: proofs, + Randomness: randomness, + Proofs: proofs, ChallengedSectors: sis, - Prover: seals[0].id.Miner, + Prover: seals[0].id.Miner, }) if err != nil { t.Fatalf("%+v", err) diff --git a/extern/sector-storage/mock/mock.go b/extern/sector-storage/mock/mock.go index 7609f43b0..64207e66d 100644 --- a/extern/sector-storage/mock/mock.go +++ b/extern/sector-storage/mock/mock.go @@ -66,8 +66,8 @@ const ( ) type sectorState struct { - pieces []cid.Cid - failed bool + pieces []cid.Cid + failed bool corrupted bool state int diff --git a/go.sum b/go.sum index 3f37bff04..953e3bba6 100644 --- a/go.sum +++ b/go.sum @@ -250,8 +250,6 @@ github.com/filecoin-project/specs-actors v0.9.7 h1:7PAZ8kdqwBdmgf/23FCkQZLCXcVu0 github.com/filecoin-project/specs-actors v0.9.7/go.mod h1:wM2z+kwqYgXn5Z7scV1YHLyd1Q1cy0R8HfTIWQ0BFGU= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= -github.com/filecoin-project/specs-storage v0.1.1-0.20200909213410-c066548422be h1:UX457RrC0LwL7Bb5kd0WFyzJBxbHOCSw/64oYqeT+Zc= -github.com/filecoin-project/specs-storage v0.1.1-0.20200909213410-c066548422be/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= github.com/filecoin-project/test-vectors/schema v0.0.1 h1:5fNF76nl4qolEvcIsjc0kUADlTMVHO73tW4kXXPnsus= github.com/filecoin-project/test-vectors/schema v0.0.1/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= @@ -1318,8 +1316,6 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/supranational/blst v0.1.2-alpha.1 h1:v0UqVlvbRNZIaSeMPr+T01kvTUq1h0EZuZ6gnDR1Mlg= -github.com/supranational/blst v0.1.2-alpha.1/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= From df5750a21c8f9487abe3284274a7fb040d2cdd9a Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 10 Sep 2020 23:15:56 +0200 Subject: [PATCH 057/885] go mod tidy Signed-off-by: Jakub Sztandera --- go.sum | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/go.sum b/go.sum index 39970d325..1c9723ba1 100644 --- a/go.sum +++ b/go.sum @@ -166,8 +166,10 @@ github.com/dgraph-io/badger v1.6.0-rc1/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhY github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgraph-io/badger v1.6.1 h1:w9pSFNSdq/JPM1N12Fz/F/bzo993Is1W+Q7HjPzi7yg= github.com/dgraph-io/badger v1.6.1/go.mod h1:FRmFw3uxvcpa8zG3Rxs0th+hCLIuaQg8HlNV5bjgnuU= -github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200716180832-3ab515320794 h1:PIPH4SLjYXMMlX/cQqV7nIRatv7556yqUfWY+KBjrtQ= -github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200716180832-3ab515320794/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= +github.com/dgraph-io/badger/v2 v2.0.3/go.mod h1:3KY8+bsP8wI0OEnQJAKpd4wIJW/Mm32yw2j/9FUVnIM= +github.com/dgraph-io/badger/v2 v2.2007.2 h1:EjjK0KqwaFMlPin1ajhP943VPENHJdEz1KLIegjaI3k= +github.com/dgraph-io/badger/v2 v2.2007.2/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= +github.com/dgraph-io/ristretto v0.0.2-0.20200115201040-8f368f2f2ab3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de h1:t0UHb5vdojIDUqktM6+xJAfScFBsVpXZmqC9dsgJmeA= github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= From 2865a0367bdbed5fad9d4e641d9dc7dc7398a98d Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 11 Sep 2020 00:59:57 +0200 Subject: [PATCH 058/885] Fix mpool replace --auto Signed-off-by: Jakub Sztandera --- chain/messagepool/messagepool.go | 17 +++++++++++++++++ cli/mpool.go | 23 ++++++++++++++++++++--- go.sum | 5 +++++ node/impl/full/gas.go | 18 +----------------- 4 files changed, 43 insertions(+), 20 deletions(-) diff --git a/chain/messagepool/messagepool.go b/chain/messagepool/messagepool.go index 06cbe65e7..653844c02 100644 --- a/chain/messagepool/messagepool.go +++ b/chain/messagepool/messagepool.go @@ -12,6 +12,7 @@ import ( "time" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" "github.com/hashicorp/go-multierror" lru "github.com/hashicorp/golang-lru" @@ -160,6 +161,22 @@ func ComputeMinRBF(curPrem abi.TokenAmount) abi.TokenAmount { return types.BigAdd(minPrice, types.NewInt(1)) } +func CapGasFee(msg *types.Message, maxFee abi.TokenAmount) { + if maxFee.Equals(big.Zero()) { + maxFee = types.NewInt(build.FilecoinPrecision / 10) + } + + gl := types.NewInt(uint64(msg.GasLimit)) + totalFee := types.BigMul(msg.GasFeeCap, gl) + + if totalFee.LessThanEqual(maxFee) { + return + } + + msg.GasFeeCap = big.Div(maxFee, gl) + msg.GasPremium = big.Min(msg.GasFeeCap, msg.GasPremium) // cap premium at FeeCap +} + func (ms *msgSet) add(m *types.SignedMessage, mp *MessagePool, strict bool) (bool, error) { nextNonce := ms.nextNonce nonceGap := false diff --git a/cli/mpool.go b/cli/mpool.go index 6ae94356a..65f4ef942 100644 --- a/cli/mpool.go +++ b/cli/mpool.go @@ -303,6 +303,10 @@ var mpoolReplaceCmd = &cli.Command{ Name: "auto", Usage: "automatically reprice the specified message", }, + &cli.StringFlag{ + Name: "max-fee", + Usage: "Spend up to X FIL for this message (applicable for auto mode)", + }, }, ArgsUsage: "[from] [nonce]", Action: func(cctx *cli.Context) error { @@ -353,17 +357,30 @@ var mpoolReplaceCmd = &cli.Command{ msg := found.Message if cctx.Bool("auto") { + minRBF := messagepool.ComputeMinRBF(msg.GasPremium) + + var mss *lapi.MessageSendSpec + if cctx.IsSet("max-fee") { + maxFee, err := types.BigFromString(cctx.String("max-fee")) + if err != nil { + return fmt.Errorf("parsing max-spend: %w", err) + } + mss = &lapi.MessageSendSpec{ + MaxFee: maxFee, + } + } + // msg.GasLimit = 0 // TODO: need to fix the way we estimate gas limits to account for the messages already being in the mempool msg.GasFeeCap = abi.NewTokenAmount(0) msg.GasPremium = abi.NewTokenAmount(0) - retm, err := api.GasEstimateMessageGas(ctx, &msg, &lapi.MessageSendSpec{}, types.EmptyTSK) + retm, err := api.GasEstimateMessageGas(ctx, &msg, mss, types.EmptyTSK) if err != nil { return fmt.Errorf("failed to estimate gas values: %w", err) } - msg.GasFeeCap = retm.GasFeeCap - minRBF := messagepool.ComputeMinRBF(msg.GasPremium) msg.GasPremium = big.Max(retm.GasPremium, minRBF) + msg.GasFeeCap = big.Max(retm.GasFeeCap, msg.GasPremium) + messagepool.CapGasFee(&msg, mss.Get().MaxFee) } else { msg.GasLimit = cctx.Int64("gas-limit") msg.GasPremium, err = types.BigFromString(cctx.String("gas-premium")) diff --git a/go.sum b/go.sum index 1d18f7b42..18ebc816f 100644 --- a/go.sum +++ b/go.sum @@ -211,10 +211,13 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.8.0 h1:5bzFgL+oy7JITMTxUPJ00n7VxmYd/PdMp5mHFX40/RY= github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGjnw8= github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E= +github.com/filecoin-project/go-address v0.0.2-0.20200218010043-eb9bb40ed5be/go.mod h1:SAOwJoakQ8EPjwNIsiakIQKsoKdkcbx8U3IapgCg9R0= github.com/filecoin-project/go-address v0.0.3 h1:eVfbdjEbpbzIrbiSa+PiGUY+oDK9HnUn+M1R/ggoHf8= github.com/filecoin-project/go-address v0.0.3/go.mod h1:jr8JxKsYx+lQlQZmF5i2U0Z+cGQ59wMIps/8YW/lDj8= +github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200424220931-6263827e49f2/go.mod h1:boRtQhzmxNocrMxOXo1NYn4oUc1NGvR8tEa79wApNXg= github.com/filecoin-project/go-amt-ipld/v2 v2.1.0 h1:t6qDiuGYYngDqaLc2ZUvdtAg4UNxPeOYaXhBWSNsVaM= github.com/filecoin-project/go-amt-ipld/v2 v2.1.0/go.mod h1:nfFPoGyX0CU9SkXX8EoCcSuHN1XcbN0c6KBh7yvP5fs= +github.com/filecoin-project/go-bitfield v0.0.1/go.mod h1:Ry9/iUlWSyjPUzlAvdnfy4Gtvrq4kWmWDztCU1yEgJY= github.com/filecoin-project/go-bitfield v0.2.0 h1:gCtLcjskIPtdg4NfN7gQZSQF9yrBQ7mkT0qCJxzGI2Q= github.com/filecoin-project/go-bitfield v0.2.0/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM= github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:av5fw6wmm58FYMgJeoB/lK9XXrgdugYiTqkdxjTy9k8= @@ -250,6 +253,7 @@ github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIi github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b h1:fkRZSPrYpk42PV3/lIXiL0LHetxde7vyYYvSsttQtfg= github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/go.mod h1:Q0GQOBtKf1oE10eSXSlhN45kDBdGvEcVOqMiffqX+N8= +github.com/filecoin-project/specs-actors v0.6.1/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY= github.com/filecoin-project/specs-actors v0.9.4/go.mod h1:BStZQzx5x7TmCkLv0Bpa07U6cPKol6fd3w9KjMPZ6Z4= github.com/filecoin-project/specs-actors v0.9.7 h1:7PAZ8kdqwBdmgf/23FCkQZLCXcVu02XJrkpkhBikiA8= github.com/filecoin-project/specs-actors v0.9.7/go.mod h1:wM2z+kwqYgXn5Z7scV1YHLyd1Q1cy0R8HfTIWQ0BFGU= @@ -504,6 +508,7 @@ github.com/ipfs/go-fs-lock v0.0.6/go.mod h1:OTR+Rj9sHiRubJh3dRhD15Juhd/+w6VPOY28 github.com/ipfs/go-graphsync v0.1.0/go.mod h1:jMXfqIEDFukLPZHqDPp8tJMbHO9Rmeb9CEGevngQbmE= github.com/ipfs/go-graphsync v0.1.2 h1:25Ll9kIXCE+DY0dicvfS3KMw+U5sd01b/FJbA7KAbhg= github.com/ipfs/go-graphsync v0.1.2/go.mod h1:sLXVXm1OxtE2XYPw62MuXCdAuNwkAdsbnfrmos5odbA= +github.com/ipfs/go-hamt-ipld v0.0.15-0.20200131012125-dd88a59d3f2e/go.mod h1:9aQJu/i/TaRDW6jqB5U217dLIDopn50wxLdHXM2CTfE= github.com/ipfs/go-hamt-ipld v0.1.1 h1:0IQdvwnAAUKmDE+PMJa5y1QiwOPHpI9+eAbQEEEYthk= github.com/ipfs/go-hamt-ipld v0.1.1/go.mod h1:1EZCr2v0jlCnhpa+aZ0JZYp8Tt2w16+JJOAVz17YcDk= github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= diff --git a/node/impl/full/gas.go b/node/impl/full/gas.go index 40ab88b6b..a597059e4 100644 --- a/node/impl/full/gas.go +++ b/node/impl/full/gas.go @@ -211,23 +211,7 @@ func (a *GasAPI) GasEstimateMessageGas(ctx context.Context, msg *types.Message, msg.GasFeeCap = feeCap } - capGasFee(msg, spec.Get().MaxFee) + messagepool.CapGasFee(msg, spec.Get().MaxFee) return msg, nil } - -func capGasFee(msg *types.Message, maxFee abi.TokenAmount) { - if maxFee.Equals(big.Zero()) { - maxFee = types.NewInt(build.FilecoinPrecision / 10) - } - - gl := types.NewInt(uint64(msg.GasLimit)) - totalFee := types.BigMul(msg.GasFeeCap, gl) - - if totalFee.LessThanEqual(maxFee) { - return - } - - msg.GasFeeCap = big.Div(maxFee, gl) - msg.GasPremium = big.Min(msg.GasFeeCap, msg.GasPremium) // cap premium at FeeCap -} From 56b237e7b8b73b6075b5c871a9fc6ad472ae7f29 Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Thu, 10 Sep 2020 16:00:23 -0700 Subject: [PATCH 059/885] fix(paych): add status command to list Add paych status cmd to list of payment channel sub commands --- cli/paych.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cli/paych.go b/cli/paych.go index bcd8e33f0..57fd1c142 100644 --- a/cli/paych.go +++ b/cli/paych.go @@ -28,6 +28,7 @@ var paychCmd = &cli.Command{ paychListCmd, paychVoucherCmd, paychSettleCmd, + paychStatusCmd, paychCloseCmd, }, } From 0e3dd3cb3b0f9e1a8f66f85bb2e1924577558081 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 10 Sep 2020 17:28:25 -0700 Subject: [PATCH 060/885] Add faster and slimmer option to chain export via lotus-shed --- chain/store/store.go | 12 ++-- chain/store/store_test.go | 2 +- cmd/lotus-shed/export.go | 123 ++++++++++++++++++++++++++++++++++++++ cmd/lotus-shed/main.go | 1 + node/impl/full/chain.go | 2 +- 5 files changed, 134 insertions(+), 6 deletions(-) create mode 100644 cmd/lotus-shed/export.go diff --git a/chain/store/store.go b/chain/store/store.go index c85f547a1..d5ecf95ef 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -1159,7 +1159,7 @@ func recurseLinks(bs bstore.Blockstore, walked *cid.Set, root cid.Cid, in []cid. return in, rerr } -func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, inclRecentRoots abi.ChainEpoch, w io.Writer) error { +func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, inclRecentRoots abi.ChainEpoch, skipOldMsgs bool, w io.Writer) error { if ts == nil { ts = cs.GetHeaviestTipSet() } @@ -1197,9 +1197,13 @@ func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, inclRecentRo return xerrors.Errorf("unmarshaling block header (cid=%s): %w", blk, err) } - cids, err := recurseLinks(cs.bs, walked, b.Messages, []cid.Cid{b.Messages}) - if err != nil { - return xerrors.Errorf("recursing messages failed: %w", err) + var cids []cid.Cid + if !skipOldMsgs || b.Height > ts.Height()-inclRecentRoots { + mcids, err := recurseLinks(cs.bs, walked, b.Messages, []cid.Cid{b.Messages}) + if err != nil { + return xerrors.Errorf("recursing messages failed: %w", err) + } + cids = mcids } if b.Height > 0 { diff --git a/chain/store/store_test.go b/chain/store/store_test.go index 1e3673f44..e56bab4c9 100644 --- a/chain/store/store_test.go +++ b/chain/store/store_test.go @@ -96,7 +96,7 @@ func TestChainExportImport(t *testing.T) { } buf := new(bytes.Buffer) - if err := cg.ChainStore().Export(context.TODO(), last, 0, buf); err != nil { + if err := cg.ChainStore().Export(context.TODO(), last, 0, false, buf); err != nil { t.Fatal(err) } diff --git a/cmd/lotus-shed/export.go b/cmd/lotus-shed/export.go new file mode 100644 index 000000000..6fa8d63f6 --- /dev/null +++ b/cmd/lotus-shed/export.go @@ -0,0 +1,123 @@ +package main + +import ( + "context" + "fmt" + "os" + + "github.com/urfave/cli/v2" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/store" + "github.com/filecoin-project/lotus/chain/types" + lcli "github.com/filecoin-project/lotus/cli" + "github.com/filecoin-project/lotus/lib/blockstore" + "github.com/filecoin-project/lotus/node/repo" +) + +var exportChainCmd = &cli.Command{ + Name: "export", + Description: "Export chain from repo (requires node to be offline)", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "repo", + Value: "~/.lotus", + }, + &cli.StringFlag{ + Name: "tipset", + Usage: "tipset to export from", + }, + &cli.Int64Flag{ + Name: "recent-stateroots", + }, + &cli.BoolFlag{ + Name: "full-state", + }, + &cli.BoolFlag{ + Name: "skip-old-msgs", + }, + }, + Action: func(cctx *cli.Context) error { + if !cctx.Args().Present() { + return lcli.ShowHelp(cctx, fmt.Errorf("must specifiy file name to write export to")) + } + + ctx := context.TODO() + + r, err := repo.NewFS(cctx.String("repo")) + if err != nil { + return xerrors.Errorf("opening fs repo: %w", err) + } + + exists, err := r.Exists() + if err != nil { + return err + } + if !exists { + return xerrors.Errorf("lotus repo doesn't exist") + } + + lr, err := r.Lock(repo.FullNode) + if err != nil { + return err + } + defer lr.Close() //nolint:errcheck + + fi, err := os.Create(cctx.Args().First()) + if err != nil { + return xerrors.Errorf("opening the output file: %w", err) + } + + defer fi.Close() + + ds, err := lr.Datastore("/chain") + if err != nil { + return err + } + + mds, err := lr.Datastore("/metadata") + if err != nil { + return err + } + + bs := blockstore.NewBlockstore(ds) + + cs := store.NewChainStore(bs, mds, nil) + if err := cs.Load(); err != nil { + return err + } + + nroots := abi.ChainEpoch(cctx.Int64("recent-stateroots")) + fullstate := cctx.Bool("full-state") + skipoldmsgs := cctx.Bool("skip-old-msgs") + + var ts *types.TipSet + if tss := cctx.String("tipset"); tss != "" { + cids, err := lcli.ParseTipSetString(tss) + if err != nil { + return xerrors.Errorf("failed to parse tipset (%q): %w", tss, err) + } + + tsk := types.NewTipSetKey(cids...) + + selts, err := cs.LoadTipSet(tsk) + if err != nil { + return xerrors.Errorf("loading tipset: %w", err) + } + ts = selts + } else { + ts = cs.GetHeaviestTipSet() + } + + if fullstate { + nroots = ts.Height() + 1 + } + + if err := cs.Export(ctx, ts, nroots, skipoldmsgs, fi); err != nil { + return xerrors.Errorf("export failed: %w", err) + } + + return nil + }, +} diff --git a/cmd/lotus-shed/main.go b/cmd/lotus-shed/main.go index 4944b67aa..cff3059b6 100644 --- a/cmd/lotus-shed/main.go +++ b/cmd/lotus-shed/main.go @@ -34,6 +34,7 @@ func main() { genesisVerifyCmd, mathCmd, mpoolStatsCmd, + exportChainCmd, } app := &cli.App{ diff --git a/node/impl/full/chain.go b/node/impl/full/chain.go index c5dd5c9a9..13b9dd00b 100644 --- a/node/impl/full/chain.go +++ b/node/impl/full/chain.go @@ -508,7 +508,7 @@ func (a *ChainAPI) ChainExport(ctx context.Context, nroots abi.ChainEpoch, tsk t bw := bufio.NewWriterSize(w, 1<<20) defer bw.Flush() //nolint:errcheck // it is a write to a pipe - if err := a.Chain.Export(ctx, ts, nroots, bw); err != nil { + if err := a.Chain.Export(ctx, ts, nroots, false, bw); err != nil { log.Errorf("chain export call failed: %s", err) return } From 0efe5ee7d5c2e23e188a8990eeafb158b947bcef Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 10 Sep 2020 17:32:30 -0700 Subject: [PATCH 061/885] include some other changes --- cli/state.go | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/cli/state.go b/cli/state.go index f84375782..1036e8fe5 100644 --- a/cli/state.go +++ b/cli/state.go @@ -27,6 +27,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/exitcode" "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/builtin/exported" @@ -122,7 +123,7 @@ var stateMinerInfo = &cli.Command{ }, } -func parseTipSetString(ts string) ([]cid.Cid, error) { +func ParseTipSetString(ts string) ([]cid.Cid, error) { strs := strings.Split(ts, ",") var cids []cid.Cid @@ -160,7 +161,7 @@ func ParseTipSetRef(ctx context.Context, api api.FullNode, tss string) (*types.T return api.ChainGetTipSetByHeight(ctx, abi.ChainEpoch(h), types.EmptyTSK) } - cids, err := parseTipSetString(tss) + cids, err := ParseTipSetString(tss) if err != nil { return nil, err } @@ -1384,7 +1385,7 @@ var stateCallCmd = &cli.Command{ } if ret.MsgRct.ExitCode != 0 { - return fmt.Errorf("invocation failed (exit: %d): %s", ret.MsgRct.ExitCode, ret.Error) + return fmt.Errorf("invocation failed (exit: %d, gasUsed: %d): %s", ret.MsgRct.ExitCode, ret.MsgRct.GasUsed, ret.Error) } s, err := formatOutput(cctx.String("ret"), ret.MsgRct.Return) @@ -1392,6 +1393,7 @@ var stateCallCmd = &cli.Command{ return fmt.Errorf("failed to format output: %s", err) } + fmt.Printf("gas used: %d\n", ret.MsgRct.GasUsed) fmt.Printf("return: %s\n", s) return nil @@ -1465,11 +1467,11 @@ func parseParamsForMethod(act cid.Cid, method uint64, args []string) ([]byte, er f := methods[method] rf := reflect.TypeOf(f) - if rf.NumIn() != 3 { + if rf.NumIn() != 2 { return nil, fmt.Errorf("expected referenced method to have three arguments") } - paramObj := rf.In(2).Elem() + paramObj := rf.In(1).Elem() if paramObj.NumField() != len(args) { return nil, fmt.Errorf("not enough arguments given to call that method (expecting %d)", paramObj.NumField()) } @@ -1489,6 +1491,18 @@ func parseParamsForMethod(act cid.Cid, method uint64, args []string) ([]byte, er return nil, err } p.Elem().Field(i).Set(reflect.ValueOf(val)) + case reflect.TypeOf(abi.ChainEpoch(0)): + val, err := strconv.ParseInt(args[i], 10, 64) + if err != nil { + return nil, err + } + p.Elem().Field(i).Set(reflect.ValueOf(abi.ChainEpoch(val))) + case reflect.TypeOf(big.Int{}): + val, err := big.FromString(args[i]) + if err != nil { + return nil, err + } + p.Elem().Field(i).Set(reflect.ValueOf(val)) case reflect.TypeOf(peer.ID("")): pid, err := peer.Decode(args[i]) if err != nil { From 72eb17d314e54886287e81f12aba4ee149c5479e Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 10 Sep 2020 17:40:47 -0700 Subject: [PATCH 062/885] wire skip old messages option through the api --- api/api_full.go | 3 ++- api/apistruct/struct.go | 6 +++--- cli/chain.go | 11 ++++++++++- node/impl/full/chain.go | 4 ++-- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index a604028fb..23226443a 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -118,7 +118,8 @@ type FullNode interface { // The exported chain data includes the header chain from the given tipset // back to genesis, the entire genesis state, and the most recent 'nroots' // state trees. - ChainExport(ctx context.Context, nroots abi.ChainEpoch, tsk types.TipSetKey) (<-chan []byte, error) + // If oldmsgskip is set, messages from before the requested roots are also not included. + ChainExport(ctx context.Context, nroots abi.ChainEpoch, oldmsgskip bool, tsk types.TipSetKey) (<-chan []byte, error) // MethodGroup: Beacon // The Beacon method group contains methods for interacting with the random beacon (DRAND) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 6f32d204b..fdb9843ff 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -95,7 +95,7 @@ type FullNodeStruct struct { ChainGetNode func(ctx context.Context, p string) (*api.IpldObject, error) `perm:"read"` ChainGetMessage func(context.Context, cid.Cid) (*types.Message, error) `perm:"read"` ChainGetPath func(context.Context, types.TipSetKey, types.TipSetKey) ([]*api.HeadChange, error) `perm:"read"` - ChainExport func(context.Context, abi.ChainEpoch, types.TipSetKey) (<-chan []byte, error) `perm:"read"` + ChainExport func(context.Context, abi.ChainEpoch, bool, types.TipSetKey) (<-chan []byte, error) `perm:"read"` BeaconGetEntry func(ctx context.Context, epoch abi.ChainEpoch) (*types.BeaconEntry, error) `perm:"read"` @@ -692,8 +692,8 @@ func (c *FullNodeStruct) ChainGetPath(ctx context.Context, from types.TipSetKey, return c.Internal.ChainGetPath(ctx, from, to) } -func (c *FullNodeStruct) ChainExport(ctx context.Context, nroots abi.ChainEpoch, tsk types.TipSetKey) (<-chan []byte, error) { - return c.Internal.ChainExport(ctx, nroots, tsk) +func (c *FullNodeStruct) ChainExport(ctx context.Context, nroots abi.ChainEpoch, iom bool, tsk types.TipSetKey) (<-chan []byte, error) { + return c.Internal.ChainExport(ctx, nroots, iom, tsk) } func (c *FullNodeStruct) BeaconGetEntry(ctx context.Context, epoch abi.ChainEpoch) (*types.BeaconEntry, error) { diff --git a/cli/chain.go b/cli/chain.go index ce1660641..36288c7d7 100644 --- a/cli/chain.go +++ b/cli/chain.go @@ -844,6 +844,9 @@ var chainExportCmd = &cli.Command{ Name: "recent-stateroots", Usage: "specify the number of recent state roots to include in the export", }, + &cli.BoolFlag{ + Name: "skip-old-msgs", + }, }, Action: func(cctx *cli.Context) error { api, closer, err := GetFullNodeAPI(cctx) @@ -878,7 +881,13 @@ var chainExportCmd = &cli.Command{ return err } - stream, err := api.ChainExport(ctx, rsrs, ts.Key()) + skipold := cctx.Bool("skip-old-msgs") + + if rsrs == 0 && skipold { + return fmt.Errorf("must pass recent stateroots along with skip-old-msgs") + } + + stream, err := api.ChainExport(ctx, rsrs, skipold, ts.Key()) if err != nil { return err } diff --git a/node/impl/full/chain.go b/node/impl/full/chain.go index 13b9dd00b..ad2c2944c 100644 --- a/node/impl/full/chain.go +++ b/node/impl/full/chain.go @@ -495,7 +495,7 @@ func (a *ChainAPI) ChainGetMessage(ctx context.Context, mc cid.Cid) (*types.Mess return cm.VMMessage(), nil } -func (a *ChainAPI) ChainExport(ctx context.Context, nroots abi.ChainEpoch, tsk types.TipSetKey) (<-chan []byte, error) { +func (a *ChainAPI) ChainExport(ctx context.Context, nroots abi.ChainEpoch, skipoldmsgs bool, tsk types.TipSetKey) (<-chan []byte, error) { ts, err := a.Chain.GetTipSetFromKey(tsk) if err != nil { return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) @@ -508,7 +508,7 @@ func (a *ChainAPI) ChainExport(ctx context.Context, nroots abi.ChainEpoch, tsk t bw := bufio.NewWriterSize(w, 1<<20) defer bw.Flush() //nolint:errcheck // it is a write to a pipe - if err := a.Chain.Export(ctx, ts, nroots, false, bw); err != nil { + if err := a.Chain.Export(ctx, ts, nroots, skipoldmsgs, bw); err != nil { log.Errorf("chain export call failed: %s", err) return } From c4b3970c70cd01537cfeb914448108c17833cdb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 11 Sep 2020 10:41:24 +0200 Subject: [PATCH 063/885] mod tidy --- go.sum | 5 ----- 1 file changed, 5 deletions(-) diff --git a/go.sum b/go.sum index 18ebc816f..1d18f7b42 100644 --- a/go.sum +++ b/go.sum @@ -211,13 +211,10 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.8.0 h1:5bzFgL+oy7JITMTxUPJ00n7VxmYd/PdMp5mHFX40/RY= github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGjnw8= github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E= -github.com/filecoin-project/go-address v0.0.2-0.20200218010043-eb9bb40ed5be/go.mod h1:SAOwJoakQ8EPjwNIsiakIQKsoKdkcbx8U3IapgCg9R0= github.com/filecoin-project/go-address v0.0.3 h1:eVfbdjEbpbzIrbiSa+PiGUY+oDK9HnUn+M1R/ggoHf8= github.com/filecoin-project/go-address v0.0.3/go.mod h1:jr8JxKsYx+lQlQZmF5i2U0Z+cGQ59wMIps/8YW/lDj8= -github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200424220931-6263827e49f2/go.mod h1:boRtQhzmxNocrMxOXo1NYn4oUc1NGvR8tEa79wApNXg= github.com/filecoin-project/go-amt-ipld/v2 v2.1.0 h1:t6qDiuGYYngDqaLc2ZUvdtAg4UNxPeOYaXhBWSNsVaM= github.com/filecoin-project/go-amt-ipld/v2 v2.1.0/go.mod h1:nfFPoGyX0CU9SkXX8EoCcSuHN1XcbN0c6KBh7yvP5fs= -github.com/filecoin-project/go-bitfield v0.0.1/go.mod h1:Ry9/iUlWSyjPUzlAvdnfy4Gtvrq4kWmWDztCU1yEgJY= github.com/filecoin-project/go-bitfield v0.2.0 h1:gCtLcjskIPtdg4NfN7gQZSQF9yrBQ7mkT0qCJxzGI2Q= github.com/filecoin-project/go-bitfield v0.2.0/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM= github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:av5fw6wmm58FYMgJeoB/lK9XXrgdugYiTqkdxjTy9k8= @@ -253,7 +250,6 @@ github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIi github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b h1:fkRZSPrYpk42PV3/lIXiL0LHetxde7vyYYvSsttQtfg= github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/go.mod h1:Q0GQOBtKf1oE10eSXSlhN45kDBdGvEcVOqMiffqX+N8= -github.com/filecoin-project/specs-actors v0.6.1/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY= github.com/filecoin-project/specs-actors v0.9.4/go.mod h1:BStZQzx5x7TmCkLv0Bpa07U6cPKol6fd3w9KjMPZ6Z4= github.com/filecoin-project/specs-actors v0.9.7 h1:7PAZ8kdqwBdmgf/23FCkQZLCXcVu02XJrkpkhBikiA8= github.com/filecoin-project/specs-actors v0.9.7/go.mod h1:wM2z+kwqYgXn5Z7scV1YHLyd1Q1cy0R8HfTIWQ0BFGU= @@ -508,7 +504,6 @@ github.com/ipfs/go-fs-lock v0.0.6/go.mod h1:OTR+Rj9sHiRubJh3dRhD15Juhd/+w6VPOY28 github.com/ipfs/go-graphsync v0.1.0/go.mod h1:jMXfqIEDFukLPZHqDPp8tJMbHO9Rmeb9CEGevngQbmE= github.com/ipfs/go-graphsync v0.1.2 h1:25Ll9kIXCE+DY0dicvfS3KMw+U5sd01b/FJbA7KAbhg= github.com/ipfs/go-graphsync v0.1.2/go.mod h1:sLXVXm1OxtE2XYPw62MuXCdAuNwkAdsbnfrmos5odbA= -github.com/ipfs/go-hamt-ipld v0.0.15-0.20200131012125-dd88a59d3f2e/go.mod h1:9aQJu/i/TaRDW6jqB5U217dLIDopn50wxLdHXM2CTfE= github.com/ipfs/go-hamt-ipld v0.1.1 h1:0IQdvwnAAUKmDE+PMJa5y1QiwOPHpI9+eAbQEEEYthk= github.com/ipfs/go-hamt-ipld v0.1.1/go.mod h1:1EZCr2v0jlCnhpa+aZ0JZYp8Tt2w16+JJOAVz17YcDk= github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= From 6e7de3c8d922b4b75db0ba1f80f2c676c4b36e4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Albert=C2=B7Gou?= Date: Fri, 11 Sep 2020 16:52:48 +0800 Subject: [PATCH 064/885] optimize definition of state_heights In the same parentstateroot, the one with the smallest height should be selected --- cmd/lotus-chainwatch/syncer/sync.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/lotus-chainwatch/syncer/sync.go b/cmd/lotus-chainwatch/syncer/sync.go index 52a36fc9e..306cf8b3e 100644 --- a/cmd/lotus-chainwatch/syncer/sync.go +++ b/cmd/lotus-chainwatch/syncer/sync.go @@ -136,7 +136,8 @@ create unique index if not exists block_cid_uindex on blocks (cid,height); create materialized view if not exists state_heights - as select distinct height, parentstateroot from blocks; + as select b.parentstateroot, min(b.height) height + from blocks b group by b.parentstateroot; create index if not exists state_heights_height_index on state_heights (height); From aeb3437a9fd7e9a9cb71bb77af678b46420561a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 11 Sep 2020 10:55:10 +0200 Subject: [PATCH 065/885] lint, docsgen --- cmd/lotus-shed/export.go | 4 ++-- documentation/en/api-methods.md | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/lotus-shed/export.go b/cmd/lotus-shed/export.go index 6fa8d63f6..c12cbd82d 100644 --- a/cmd/lotus-shed/export.go +++ b/cmd/lotus-shed/export.go @@ -40,7 +40,7 @@ var exportChainCmd = &cli.Command{ }, Action: func(cctx *cli.Context) error { if !cctx.Args().Present() { - return lcli.ShowHelp(cctx, fmt.Errorf("must specifiy file name to write export to")) + return lcli.ShowHelp(cctx, fmt.Errorf("must specify file name to write export to")) } ctx := context.TODO() @@ -69,7 +69,7 @@ var exportChainCmd = &cli.Command{ return xerrors.Errorf("opening the output file: %w", err) } - defer fi.Close() + defer fi.Close() //nolint:errcheck ds, err := lr.Datastore("/chain") if err != nil { diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index 7c2d91881..27875eca1 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -284,6 +284,7 @@ ChainExport returns a stream of bytes with CAR dump of chain data. The exported chain data includes the header chain from the given tipset back to genesis, the entire genesis state, and the most recent 'nroots' state trees. +If oldmsgskip is set, messages from before the requested roots are also not included. Perms: read @@ -292,6 +293,7 @@ Inputs: ```json [ 10101, + true, [ { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" From 3dfb3e641b5edad5a19d1df738f52c5f164f2558 Mon Sep 17 00:00:00 2001 From: vyzo Date: Fri, 11 Sep 2020 12:15:03 +0300 Subject: [PATCH 066/885] refactor getBaseFeeLowerBound, use it to prune less aggressively --- chain/messagepool/messagepool.go | 11 ++++++++++- chain/messagepool/pruning.go | 3 ++- chain/messagepool/repub.go | 6 +----- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/chain/messagepool/messagepool.go b/chain/messagepool/messagepool.go index 06cbe65e7..88fb1dcfc 100644 --- a/chain/messagepool/messagepool.go +++ b/chain/messagepool/messagepool.go @@ -399,7 +399,7 @@ func (mp *MessagePool) verifyMsgBeforeAdd(m *types.SignedMessage, curTs *types.T publish := local if strictBaseFeeValidation && len(curTs.Blocks()) > 0 { baseFee := curTs.Blocks()[0].ParentBaseFee - baseFeeLowerBound := types.BigDiv(baseFee, baseFeeLowerBoundFactor) + baseFeeLowerBound := getBaseFeeLowerBound(baseFee) if m.Message.GasFeeCap.LessThan(baseFeeLowerBound) { if local { log.Warnf("local message will not be immediately published because GasFeeCap doesn't meet the lower bound for inclusion in the next 20 blocks (GasFeeCap: %s, baseFeeLowerBound: %s)", @@ -1282,3 +1282,12 @@ func (mp *MessagePool) Clear(local bool) { delete(mp.pending, a) } } + +func getBaseFeeLowerBound(baseFee types.BigInt) types.BigInt { + baseFeeLowerBound := types.BigDiv(baseFee, baseFeeLowerBoundFactor) + if baseFeeLowerBound.LessThan(minimumBaseFee) { + baseFeeLowerBound = minimumBaseFee + } + + return baseFeeLowerBound +} diff --git a/chain/messagepool/pruning.go b/chain/messagepool/pruning.go index d1290e386..76b183808 100644 --- a/chain/messagepool/pruning.go +++ b/chain/messagepool/pruning.go @@ -46,6 +46,7 @@ func (mp *MessagePool) pruneMessages(ctx context.Context, ts *types.TipSet) erro if err != nil { return xerrors.Errorf("computing basefee: %w", err) } + baseFeeLowerBound := getBaseFeeLowerBound(baseFee) pending, _ := mp.getPendingMessages(ts, ts) @@ -72,7 +73,7 @@ func (mp *MessagePool) pruneMessages(ctx context.Context, ts *types.TipSet) erro for _, m := range mset { pruneMsgs[m.Message.Cid()] = m } - actorChains := mp.createMessageChains(actor, mset, baseFee, ts) + actorChains := mp.createMessageChains(actor, mset, baseFeeLowerBound, ts) chains = append(chains, actorChains...) } diff --git a/chain/messagepool/repub.go b/chain/messagepool/repub.go index fd0324d0a..7e651e426 100644 --- a/chain/messagepool/repub.go +++ b/chain/messagepool/repub.go @@ -27,11 +27,7 @@ func (mp *MessagePool) republishPendingMessages() error { mp.curTsLk.Unlock() return xerrors.Errorf("computing basefee: %w", err) } - - baseFeeLowerBound := types.BigDiv(baseFee, baseFeeLowerBoundFactor) - if baseFeeLowerBoundFactor.LessThan(minimumBaseFee) { - baseFeeLowerBound = minimumBaseFee - } + baseFeeLowerBound := getBaseFeeLowerBound(baseFee) pending := make(map[address.Address]map[uint64]*types.SignedMessage) mp.lk.Lock() From 62b2963781300ad9ac8bbdc4c6255f6886299a0c Mon Sep 17 00:00:00 2001 From: Aarsh Shah Date: Fri, 11 Sep 2020 16:58:09 +0530 Subject: [PATCH 067/885] retry add piece --- extern/storage-sealing/sealing.go | 4 +++- markets/storageadapter/provider.go | 17 +++++++++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/extern/storage-sealing/sealing.go b/extern/storage-sealing/sealing.go index 45b95e18c..28df664aa 100644 --- a/extern/storage-sealing/sealing.go +++ b/extern/storage-sealing/sealing.go @@ -28,6 +28,8 @@ import ( const SectorStorePrefix = "/sectors" +var ErrTooManySectorsSealing = xerrors.New("too many sectors sealing") + var log = logging.Logger("sectors") type SectorLocation struct { @@ -280,7 +282,7 @@ func (m *Sealing) newDealSector() (abi.SectorNumber, error) { if cfg.MaxSealingSectorsForDeals > 0 { if m.stats.curSealing() > cfg.MaxSealingSectorsForDeals { - return 0, xerrors.Errorf("too many sectors sealing") + return 0, ErrTooManySectorsSealing } } diff --git a/markets/storageadapter/provider.go b/markets/storageadapter/provider.go index 9c4a5946e..677e21455 100644 --- a/markets/storageadapter/provider.go +++ b/markets/storageadapter/provider.go @@ -6,6 +6,7 @@ import ( "bytes" "context" "io" + "time" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" @@ -35,6 +36,7 @@ import ( "github.com/filecoin-project/lotus/storage/sectorblocks" ) +var addPieceRetryWait = 5 * time.Minute var log = logging.Logger("storageadapter") type ProviderNodeAdapter struct { @@ -91,7 +93,7 @@ func (n *ProviderNodeAdapter) OnDealComplete(ctx context.Context, deal storagema return nil, xerrors.Errorf("deal.PublishCid can't be nil") } - p, offset, err := n.secb.AddPiece(ctx, pieceSize, pieceData, sealing.DealInfo{ + sdInfo := sealing.DealInfo{ DealID: deal.DealID, PublishCid: deal.PublishCid, DealSchedule: sealing.DealSchedule{ @@ -99,7 +101,18 @@ func (n *ProviderNodeAdapter) OnDealComplete(ctx context.Context, deal storagema EndEpoch: deal.ClientDealProposal.Proposal.EndEpoch, }, KeepUnsealed: deal.FastRetrieval, - }) + } + + p, offset, err := n.secb.AddPiece(ctx, pieceSize, pieceData, sdInfo) + if xerrors.Is(err, sealing.ErrTooManySectorsSealing) { + select { + case <-time.After(addPieceRetryWait): + p, offset, err = n.secb.AddPiece(ctx, pieceSize, pieceData, sdInfo) + case <-ctx.Done(): + return nil, xerrors.New("context expired while waiting to retry AddPiece") + } + } + if err != nil { return nil, xerrors.Errorf("AddPiece failed: %s", err) } From 16f7d5801b5d8c815a38cffe89d96a27928525f2 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Fri, 11 Sep 2020 15:25:59 +0100 Subject: [PATCH 068/885] fix: chaos tests --- conformance/chaos/actor_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conformance/chaos/actor_test.go b/conformance/chaos/actor_test.go index 97c3b85cf..a08267022 100644 --- a/conformance/chaos/actor_test.go +++ b/conformance/chaos/actor_test.go @@ -4,8 +4,8 @@ import ( "context" "testing" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/exitcode" - "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/specs-actors/support/mock" atesting "github.com/filecoin-project/specs-actors/support/testing" ) @@ -19,7 +19,7 @@ func TestSingleton(t *testing.T) { msg := "constructor should not be called; the Chaos actor is a singleton actor" rt.ExpectAssertionFailure(msg, func() { - rt.Call(a.Constructor, adt.Empty) + rt.Call(a.Constructor, abi.Empty) }) rt.Verify() } From cf9820137cbc1bea526256f4ac861530341a0d94 Mon Sep 17 00:00:00 2001 From: vyzo Date: Fri, 11 Sep 2020 20:32:52 +0300 Subject: [PATCH 069/885] don't prune locally published messages --- chain/messagepool/pruning.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/chain/messagepool/pruning.go b/chain/messagepool/pruning.go index 76b183808..cd20b462b 100644 --- a/chain/messagepool/pruning.go +++ b/chain/messagepool/pruning.go @@ -56,6 +56,13 @@ func (mp *MessagePool) pruneMessages(ctx context.Context, ts *types.TipSet) erro priority[actor] = struct{}{} } + // we also never prune locally published messages + mp.lk.Lock() + for actor := range mp.localAddrs { + priority[actor] = struct{}{} + } + mp.lk.Unlock() + // Collect all messages to track which ones to remove and create chains for block inclusion pruneMsgs := make(map[cid.Cid]*types.SignedMessage, mp.currentSize) keepCount := 0 From eabdf74946e7668334d11a2ed760012791f43dea Mon Sep 17 00:00:00 2001 From: Aarsh Shah Date: Fri, 11 Sep 2020 23:15:57 +0530 Subject: [PATCH 070/885] changes as per review --- markets/storageadapter/provider.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/markets/storageadapter/provider.go b/markets/storageadapter/provider.go index 677e21455..ed714ab61 100644 --- a/markets/storageadapter/provider.go +++ b/markets/storageadapter/provider.go @@ -37,6 +37,7 @@ import ( ) var addPieceRetryWait = 5 * time.Minute +var addPieceRetryTimeout = 6 * time.Hour var log = logging.Logger("storageadapter") type ProviderNodeAdapter struct { @@ -104,7 +105,11 @@ func (n *ProviderNodeAdapter) OnDealComplete(ctx context.Context, deal storagema } p, offset, err := n.secb.AddPiece(ctx, pieceSize, pieceData, sdInfo) - if xerrors.Is(err, sealing.ErrTooManySectorsSealing) { + curTime := time.Now() + for time.Since(curTime) < addPieceRetryTimeout { + if !xerrors.Is(err, sealing.ErrTooManySectorsSealing) { + break + } select { case <-time.After(addPieceRetryWait): p, offset, err = n.secb.AddPiece(ctx, pieceSize, pieceData, sdInfo) From 2e3ff9e4017eadad0a277d02ecbb26193279a2be Mon Sep 17 00:00:00 2001 From: Ingar Shu Date: Fri, 11 Sep 2020 11:00:42 -0700 Subject: [PATCH 071/885] Update market client/provider adapters to WaitForMessage API --- go.mod | 2 +- go.sum | 4 ++-- markets/storageadapter/client.go | 6 +++--- markets/storageadapter/provider.go | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 901f990b1..4cedea65f 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 github.com/filecoin-project/go-data-transfer v0.6.3 github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f - github.com/filecoin-project/go-fil-markets v0.6.0 + github.com/filecoin-project/go-fil-markets v0.6.1-0.20200911011457-2959ccca6a3c github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200822201400-474f4fdccc52 github.com/filecoin-project/go-multistore v0.0.3 github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 diff --git a/go.sum b/go.sum index c91d316e7..d5eb25358 100644 --- a/go.sum +++ b/go.sum @@ -227,8 +227,8 @@ github.com/filecoin-project/go-data-transfer v0.6.3 h1:7TLwm8nuodHYD/uiwJjKc/PGR github.com/filecoin-project/go-data-transfer v0.6.3/go.mod h1:PmBKVXkhh67/tnEdJXQwDHl5mT+7Tbcwe1NPninqhnM= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f h1:GxJzR3oRIMTPtpZ0b7QF8FKPK6/iPAc7trhlL5k/g+s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= -github.com/filecoin-project/go-fil-markets v0.6.0 h1:gfxMweUHo4u+2BZh2Q7/7+cV0/ttikuJfhkkxLRsE2Q= -github.com/filecoin-project/go-fil-markets v0.6.0/go.mod h1:LhSFYLkjaoe0vFRKABGYyw1Jz+9jCpF1sPA7yOftLTw= +github.com/filecoin-project/go-fil-markets v0.6.1-0.20200911011457-2959ccca6a3c h1:YGoyYmELQ0LHwDj/WcOvY3oYt+3iM0wdrAhqJQUAIy4= +github.com/filecoin-project/go-fil-markets v0.6.1-0.20200911011457-2959ccca6a3c/go.mod h1:PLr9svZxsnHkae1Ky7+66g7fP9AlneVxIVu+oSMq56A= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24= github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200822201400-474f4fdccc52 h1:FXtCp0ybqdQL9knb3OGDpkNTaBbPxgkqPeWKotUwkH0= diff --git a/markets/storageadapter/client.go b/markets/storageadapter/client.go index 4f7361d00..4168792da 100644 --- a/markets/storageadapter/client.go +++ b/markets/storageadapter/client.go @@ -501,12 +501,12 @@ func (c *ClientNodeAdapter) GetChainHead(ctx context.Context) (shared.TipSetToke return head.Key().Bytes(), head.Height(), nil } -func (c *ClientNodeAdapter) WaitForMessage(ctx context.Context, mcid cid.Cid, cb func(code exitcode.ExitCode, bytes []byte, err error) error) error { +func (c *ClientNodeAdapter) WaitForMessage(ctx context.Context, mcid cid.Cid, cb func(code exitcode.ExitCode, bytes []byte, finalCid cid.Cid, err error) error) error { receipt, err := c.StateWaitMsg(ctx, mcid, build.MessageConfidence) if err != nil { - return cb(0, nil, err) + return cb(0, nil, cid.Undef, err) } - return cb(receipt.Receipt.ExitCode, receipt.Receipt.Return, nil) + return cb(receipt.Receipt.ExitCode, receipt.Receipt.Return, receipt.Message, nil) } func (c *ClientNodeAdapter) GetMinerInfo(ctx context.Context, addr address.Address, encodedTs shared.TipSetToken) (*storagemarket.StorageProviderInfo, error) { diff --git a/markets/storageadapter/provider.go b/markets/storageadapter/provider.go index 9c4a5946e..49be6fe41 100644 --- a/markets/storageadapter/provider.go +++ b/markets/storageadapter/provider.go @@ -355,12 +355,12 @@ func (n *ProviderNodeAdapter) GetChainHead(ctx context.Context) (shared.TipSetTo return head.Key().Bytes(), head.Height(), nil } -func (n *ProviderNodeAdapter) WaitForMessage(ctx context.Context, mcid cid.Cid, cb func(code exitcode.ExitCode, bytes []byte, err error) error) error { +func (n *ProviderNodeAdapter) WaitForMessage(ctx context.Context, mcid cid.Cid, cb func(code exitcode.ExitCode, bytes []byte, finalCid cid.Cid, err error) error) error { receipt, err := n.StateWaitMsg(ctx, mcid, 2*build.MessageConfidence) if err != nil { - return cb(0, nil, err) + return cb(0, nil, cid.Undef, err) } - return cb(receipt.Receipt.ExitCode, receipt.Receipt.Return, nil) + return cb(receipt.Receipt.ExitCode, receipt.Receipt.Return, receipt.Message, nil) } func (n *ProviderNodeAdapter) GetDataCap(ctx context.Context, addr address.Address, encodedTs shared.TipSetToken) (*verifreg.DataCap, error) { From bf1036c78fa694f9fe8f4cad5cbce3d88d06c61c Mon Sep 17 00:00:00 2001 From: vyzo Date: Fri, 11 Sep 2020 21:12:11 +0300 Subject: [PATCH 072/885] rename priority variable to protected --- chain/messagepool/pruning.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/chain/messagepool/pruning.go b/chain/messagepool/pruning.go index cd20b462b..f40007f34 100644 --- a/chain/messagepool/pruning.go +++ b/chain/messagepool/pruning.go @@ -50,16 +50,18 @@ func (mp *MessagePool) pruneMessages(ctx context.Context, ts *types.TipSet) erro pending, _ := mp.getPendingMessages(ts, ts) - // priority actors -- not pruned - priority := make(map[address.Address]struct{}) + // protected actors -- not pruned + protected := make(map[address.Address]struct{}) + + // we never prune priority addresses for _, actor := range mp.cfg.PriorityAddrs { - priority[actor] = struct{}{} + protected[actor] = struct{}{} } // we also never prune locally published messages mp.lk.Lock() for actor := range mp.localAddrs { - priority[actor] = struct{}{} + protected[actor] = struct{}{} } mp.lk.Unlock() @@ -69,14 +71,14 @@ func (mp *MessagePool) pruneMessages(ctx context.Context, ts *types.TipSet) erro var chains []*msgChain for actor, mset := range pending { - // we never prune priority actors - _, keep := priority[actor] + // we never prune protected actors + _, keep := protected[actor] if keep { keepCount += len(mset) continue } - // not a priority actor, track the messages and create chains + // not a protected actor, track the messages and create chains for _, m := range mset { pruneMsgs[m.Message.Cid()] = m } From b78fe0b8980bbac20dd980d793cf006b2fd7f35c Mon Sep 17 00:00:00 2001 From: vyzo Date: Fri, 11 Sep 2020 21:40:25 +0300 Subject: [PATCH 073/885] fix deadlock --- chain/messagepool/pruning.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/chain/messagepool/pruning.go b/chain/messagepool/pruning.go index f40007f34..fd8199b89 100644 --- a/chain/messagepool/pruning.go +++ b/chain/messagepool/pruning.go @@ -59,11 +59,9 @@ func (mp *MessagePool) pruneMessages(ctx context.Context, ts *types.TipSet) erro } // we also never prune locally published messages - mp.lk.Lock() for actor := range mp.localAddrs { protected[actor] = struct{}{} } - mp.lk.Unlock() // Collect all messages to track which ones to remove and create chains for block inclusion pruneMsgs := make(map[cid.Cid]*types.SignedMessage, mp.currentSize) From d5aef296ea18cc39f99d9bbd0de4a71831e58faf Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 11 Sep 2020 21:59:51 +0200 Subject: [PATCH 074/885] Increase the FeeCap estimation to 20 blocks in a future FeeCap will be set to 10x the current running rate Signed-off-by: Jakub Sztandera --- node/impl/full/gas.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/impl/full/gas.go b/node/impl/full/gas.go index 40ab88b6b..c1f3bd16c 100644 --- a/node/impl/full/gas.go +++ b/node/impl/full/gas.go @@ -204,7 +204,7 @@ func (a *GasAPI) GasEstimateMessageGas(ctx context.Context, msg *types.Message, } if msg.GasFeeCap == types.EmptyInt || types.BigCmp(msg.GasFeeCap, types.NewInt(0)) == 0 { - feeCap, err := a.GasEstimateFeeCap(ctx, msg, 10, types.EmptyTSK) + feeCap, err := a.GasEstimateFeeCap(ctx, msg, 20, types.EmptyTSK) if err != nil { return nil, xerrors.Errorf("estimating fee cap: %w", err) } From 8b9a3ea6c396cdd20998333b6d14ee971c00c1d7 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Fri, 11 Sep 2020 20:01:37 -0700 Subject: [PATCH 075/885] implement initial lotus-gateway program --- cmd/lotus-gateway/api.go | 52 ++++++++++++++++++ cmd/lotus-gateway/main.go | 112 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 164 insertions(+) create mode 100644 cmd/lotus-gateway/api.go create mode 100644 cmd/lotus-gateway/main.go diff --git a/cmd/lotus-gateway/api.go b/cmd/lotus-gateway/api.go new file mode 100644 index 000000000..06c9a3ae5 --- /dev/null +++ b/cmd/lotus-gateway/api.go @@ -0,0 +1,52 @@ +package main + +import ( + "context" + "fmt" + "time" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/types" + + "go.opencensus.io/trace" +) + +const LookbackCap = time.Hour + +var ( + ErrLookbackTooLong = fmt.Errorf("lookbacks of more than %s are disallowed", LookbackCap) +) + +type GatewayAPI struct { + api api.FullNode +} + +func (a *GatewayAPI) getTipsetTimestamp(ctx context.Context, tsk types.TipSetKey) (time.Time, error) { + if tsk.IsEmpty() { + return time.Now(), nil + } + + ts, err := a.api.ChainGetTipSet(ctx, tsk) + if err != nil { + return time.Time{}, err + } + + return time.Unix(int64(ts.Blocks()[0].Timestamp), 0), nil +} + +func (a *GatewayAPI) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) { + ctx, span := trace.StartSpan(ctx, "StateGetActor") + defer span.End() + + when, err := a.getTipsetTimestamp(ctx, ts) + if err != nil { + return nil, err + } + + if time.Since(when) > time.Hour { + return nil, ErrLookbackTooLong + } + + return a.api.StateGetActor(ctx, actor, ts) +} diff --git a/cmd/lotus-gateway/main.go b/cmd/lotus-gateway/main.go new file mode 100644 index 000000000..7adf5dd8c --- /dev/null +++ b/cmd/lotus-gateway/main.go @@ -0,0 +1,112 @@ +package main + +import ( + "context" + "net" + "net/http" + "os" + + "github.com/filecoin-project/go-jsonrpc" + "github.com/filecoin-project/lotus/build" + lcli "github.com/filecoin-project/lotus/cli" + "github.com/filecoin-project/lotus/lib/lotuslog" + logging "github.com/ipfs/go-log" + + "github.com/gorilla/mux" + "github.com/urfave/cli/v2" +) + +var log = logging.Logger("gateway") + +func main() { + lotuslog.SetupLogLevels() + + local := []*cli.Command{ + runCmd, + } + + app := &cli.App{ + Name: "lotus-gateway", + Usage: "Public API server for lotus", + Version: build.UserVersion(), + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "repo", + EnvVars: []string{"LOTUS_PATH"}, + Value: "~/.lotus", // TODO: Consider XDG_DATA_HOME + }, + }, + + Commands: local, + } + app.Setup() + + if err := app.Run(os.Args); err != nil { + log.Warnf("%+v", err) + return + } +} + +var runCmd = &cli.Command{ + Name: "run", + Usage: "Start api server", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "listen", + Usage: "host address and port the api server will listen on", + Value: "0.0.0.0:1777", + }, + }, + Action: func(cctx *cli.Context) error { + log.Info("Starting lotus wallet") + + ctx := lcli.ReqContext(cctx) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + api, closer, err := lcli.GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + + address := cctx.String("listen") + mux := mux.NewRouter() + + log.Info("Setting up API endpoint at " + address) + + rpcServer := jsonrpc.NewServer() + rpcServer.Register("Filecoin", &GatewayAPI{api: api}) + + mux.Handle("/rpc/v0", rpcServer) + mux.PathPrefix("/").Handler(http.DefaultServeMux) + + /*ah := &auth.Handler{ + Verify: nodeApi.AuthVerify, + Next: mux.ServeHTTP, + }*/ + + srv := &http.Server{ + Handler: mux, + BaseContext: func(listener net.Listener) context.Context { + return ctx + }, + } + + go func() { + <-ctx.Done() + log.Warn("Shutting down...") + if err := srv.Shutdown(context.TODO()); err != nil { + log.Errorf("shutting down RPC server failed: %s", err) + } + log.Warn("Graceful shutdown successful") + }() + + nl, err := net.Listen("tcp", address) + if err != nil { + return err + } + + return srv.Serve(nl) + }, +} From d3594835c4a790026e4747badedf87af11d191f8 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 11 Sep 2020 20:07:52 -0700 Subject: [PATCH 076/885] [WIP] Network upgrade support This patch starts adding support for network upgrades. * It adds an actors abstraction layer for loading abstract (cross-version) actors. * It starts switching over to a shared deadline type. * It adds an abstraction for ADTs (hamt/amt). * It removes the callback-based API in the StateManager (difficult to abstract across actor versions). * It _does not_ actually add support for actors v2. We can do that in a followup patch but that should be relatively easy. This patch is heavily WIP and does not compile. Feel free to push changes directly to this branch. Notes: * State tree access now needs a network version, because the HAMT type will change. * I haven't figured out a nice way to abstract over changes to the _message_ types. However, many of them will be type aliased to actors v0 in actors v2 so we can likely continue using the v0 versions (or use the v2 versions everywhere). I've been renaming imports to `v0*` to make it clear that we're importing types from a _specific_ actors version. TODO: * Consider merging incremental improvements? We'd have to get this compiling again first but we could merge in the new abstractions, and slowly switch over. * Finish migrating to the new abstractions. * Remove all actor state types from the public API. See `miner.State.Info()` for the planned approach here. * Fix the tests. This is likely going to be a massive pain. --- api/api_full.go | 7 +- api/types.go | 47 +------- chain/actors/adt/adt.go | 57 +++++++++ chain/actors/adt/store.go | 17 +++ chain/actors/builtin/README.md | 29 +++++ chain/actors/builtin/builtin.go | 23 ++++ chain/actors/builtin/init/init.go | 32 +++++ chain/actors/builtin/init/v0.go | 22 ++++ chain/actors/builtin/market/market.go | 32 +++++ chain/actors/builtin/market/v0.go | 25 ++++ chain/actors/builtin/miner/miner.go | 59 ++++++++++ chain/actors/builtin/miner/v0.go | 112 ++++++++++++++++++ chain/actors/builtin/power/power.go | 28 +++++ chain/actors/builtin/power/v0.go | 16 +++ chain/actors/version.go | 24 ++++ chain/messagepool/provider.go | 4 +- chain/state/statetree.go | 31 ++--- chain/stmgr/read.go | 138 ++++------------------ chain/stmgr/stmgr.go | 30 ++--- chain/stmgr/utils.go | 24 ---- chain/sub/incoming.go | 16 +-- chain/sync.go | 3 +- chain/vm/invoker.go | 1 + cmd/lotus-shed/balances.go | 5 +- cmd/lotus-shed/genesis-verify.go | 2 +- cmd/lotus-storage-miner/proving.go | 161 +++++++++++--------------- extern/storage-sealing/checks.go | 4 +- go.mod | 4 +- go.sum | 6 + node/impl/full/state.go | 46 ++++---- storage/adapter_storage_miner.go | 13 +-- storage/wdpost_run.go | 103 +++++++++++----- storage/wdpost_sched.go | 6 +- 33 files changed, 735 insertions(+), 392 deletions(-) create mode 100644 chain/actors/adt/adt.go create mode 100644 chain/actors/adt/store.go create mode 100644 chain/actors/builtin/README.md create mode 100644 chain/actors/builtin/builtin.go create mode 100644 chain/actors/builtin/init/init.go create mode 100644 chain/actors/builtin/init/v0.go create mode 100644 chain/actors/builtin/market/market.go create mode 100644 chain/actors/builtin/market/v0.go create mode 100644 chain/actors/builtin/miner/miner.go create mode 100644 chain/actors/builtin/miner/v0.go create mode 100644 chain/actors/builtin/power/power.go create mode 100644 chain/actors/builtin/power/v0.go create mode 100644 chain/actors/version.go diff --git a/api/api_full.go b/api/api_full.go index 9d1d7ab63..e5147db47 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -18,6 +18,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" + "github.com/filecoin-project/go-state-types/dline" "github.com/filecoin-project/specs-actors/actors/builtin/market" "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/specs-actors/actors/builtin/paych" @@ -312,15 +313,11 @@ type FullNode interface { StateMinerActiveSectors(context.Context, address.Address, types.TipSetKey) ([]*ChainSectorInfo, error) // StateMinerProvingDeadline calculates the deadline at some epoch for a proving period // and returns the deadline-related calculations. - StateMinerProvingDeadline(context.Context, address.Address, types.TipSetKey) (*miner.DeadlineInfo, error) + StateMinerProvingDeadline(context.Context, address.Address, types.TipSetKey) (*dline.Info, error) // StateMinerPower returns the power of the indicated miner StateMinerPower(context.Context, address.Address, types.TipSetKey) (*MinerPower, error) // StateMinerInfo returns info about the indicated miner StateMinerInfo(context.Context, address.Address, types.TipSetKey) (MinerInfo, error) - // StateMinerDeadlines returns all the proving deadlines for the given miner - StateMinerDeadlines(context.Context, address.Address, types.TipSetKey) ([]*miner.Deadline, error) - // StateMinerPartitions loads miner partitions for the specified miner/deadline - StateMinerPartitions(context.Context, address.Address, uint64, types.TipSetKey) ([]*miner.Partition, error) // StateMinerFaults returns a bitfield indicating the faulty sectors of the given miner StateMinerFaults(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error) // StateAllMinerFaults returns all non-expired Faults that occur within lookback epochs of the given tipset diff --git a/api/types.go b/api/types.go index dc8432818..4133a7105 100644 --- a/api/types.go +++ b/api/types.go @@ -8,9 +8,10 @@ import ( datatransfer "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/ipfs/go-cid" + v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" + "github.com/libp2p/go-libp2p-core/peer" pubsub "github.com/libp2p/go-libp2p-pubsub" ma "github.com/multiformats/go-multiaddr" @@ -49,48 +50,6 @@ type PubsubScore struct { Score *pubsub.PeerScoreSnapshot } -type MinerInfo struct { - Owner address.Address // Must be an ID-address. - Worker address.Address // Must be an ID-address. - NewWorker address.Address // Must be an ID-address. - ControlAddresses []address.Address // Must be an ID-addresses. - WorkerChangeEpoch abi.ChainEpoch - PeerId *peer.ID - Multiaddrs []abi.Multiaddrs - SealProofType abi.RegisteredSealProof - SectorSize abi.SectorSize - WindowPoStPartitionSectors uint64 -} - -func NewApiMinerInfo(info *miner.MinerInfo) MinerInfo { - var pid *peer.ID - if peerID, err := peer.IDFromBytes(info.PeerId); err == nil { - pid = &peerID - } - - mi := MinerInfo{ - Owner: info.Owner, - Worker: info.Worker, - ControlAddresses: info.ControlAddresses, - - NewWorker: address.Undef, - WorkerChangeEpoch: -1, - - PeerId: pid, - Multiaddrs: info.Multiaddrs, - SealProofType: info.SealProofType, - SectorSize: info.SectorSize, - WindowPoStPartitionSectors: info.WindowPoStPartitionSectors, - } - - if info.PendingWorkerKey != nil { - mi.NewWorker = info.PendingWorkerKey.NewWorker - mi.WorkerChangeEpoch = info.PendingWorkerKey.EffectiveAt - } - - return mi -} - type MessageSendSpec struct { MaxFee abi.TokenAmount } @@ -151,3 +110,5 @@ func NewDataTransferChannel(hostID peer.ID, channelState datatransfer.ChannelSta } return channel } + +type diff --git a/chain/actors/adt/adt.go b/chain/actors/adt/adt.go new file mode 100644 index 000000000..144150659 --- /dev/null +++ b/chain/actors/adt/adt.go @@ -0,0 +1,57 @@ +package adt + +import ( + "github.com/ipfs/go-cid" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/cbor" + "github.com/filecoin-project/go-state-types/network" + "github.com/filecoin-project/lotus/chain/actors/builtin" + v0adt "github.com/filecoin-project/specs-actors/actors/util/adt" +) + +type Map interface { + Root() (cid.Cid, error) + + Put(k abi.Keyer, v cbor.Marshaler) error + Get(k abi.Keyer, v cbor.Unmarshaler) (bool, error) + Delete(k abi.Keyer) error + + ForEach(v cbor.Unmarshaler, fn func(key string) error) error +} + +func AsMap(store Store, root cid.Cid, version network.Version) (Map, error) { + switch builtin.VersionForNetwork(version) { + case builtin.Version0: + return v0adt.AsMap(store, root) + } + return nil, xerrors.Errorf("unknown network version: %d", version) +} + +func NewMap(store Store, version network.Version) (Map, error) { + switch builtin.VersionForNetwork(version) { + case builtin.Version0: + return v0adt.MakeEmptyMap(store) + } + return nil, xerrors.Errorf("unknown network version: %d", version) +} + +type Array interface { + Root() (cid.Cid, error) + + Set(idx uint64, v cbor.Marshaler) error + Get(idx uint64, v cbor.Unmarshaler) (bool, error) + Delete(idx uint64) error + Length() uint64 + + ForEach(v cbor.Unmarshaler, fn func(idx int) error) error +} + +func AsArray(store Store, root cid.Cid, version network.Version) (Array, error) { + switch builtin.VersionForNetwork(version) { + case builtin.Version0: + return v0adt.AsArray(store, root) + } + return nil, xerrors.Errorf("unknown network version: %d", version) +} diff --git a/chain/actors/adt/store.go b/chain/actors/adt/store.go new file mode 100644 index 000000000..8dd9841a1 --- /dev/null +++ b/chain/actors/adt/store.go @@ -0,0 +1,17 @@ +package adt + +import ( + "context" + + adt "github.com/filecoin-project/specs-actors/actors/util/adt" + cbor "github.com/ipfs/go-ipld-cbor" +) + +type Store interface { + Context() context.Context + cbor.IpldStore +} + +func WrapStore(ctx context.Context, store cbor.IpldStore) Store { + return adt.WrapStore(ctx, store) +} diff --git a/chain/actors/builtin/README.md b/chain/actors/builtin/README.md new file mode 100644 index 000000000..21b3fd38f --- /dev/null +++ b/chain/actors/builtin/README.md @@ -0,0 +1,29 @@ +# Actors + +This package contains shims for abstracting over different actor versions. + +## Design + +Shims in this package follow a few common design principles. + +### Structure Agnostic + +Shims interfaces defined in this package should (ideally) not change even if the +structure of the underlying data changes. For example: + +* All shims store an internal "store" object. That way, state can be moved into + a separate object without needing to add a store to the function signature. +* All functions must return an error, even if unused for now. + +### Minimal + +These interfaces should be expanded only as necessary to reduce maintenance burden. + +### Queries, not field assessors. + +When possible, functions should query the state instead of simply acting as +field assessors. These queries are more likely to remain stable across +specs-actor upgrades than specific state fields. + +Note: there is a trade-off here. Avoid implementing _complicated_ query logic +inside these shims, as it will need to be replicated in every shim. diff --git a/chain/actors/builtin/builtin.go b/chain/actors/builtin/builtin.go new file mode 100644 index 000000000..173f50ba6 --- /dev/null +++ b/chain/actors/builtin/builtin.go @@ -0,0 +1,23 @@ +package builtin + +import ( + "fmt" + + "github.com/filecoin-project/go-state-types/network" +) + +type Version int + +const ( + Version0 = iota +) + +// Converts a network version into a specs-actors version. +func VersionForNetwork(version network.Version) Version { + switch version { + case network.Version0, network.Version1: + return Version + default: + panic(fmt.Sprintf("unsupported network version %d", version)) + } +} diff --git a/chain/actors/builtin/init/init.go b/chain/actors/builtin/init/init.go new file mode 100644 index 000000000..de71a032c --- /dev/null +++ b/chain/actors/builtin/init/init.go @@ -0,0 +1,32 @@ +package init + +import ( + "github.com/filecoin-project/go-bitfield" + "github.com/filecoin-project/go-state-types/cbor" + "golang.org/x/xerrors" + + v0builtin "github.com/filecoin-project/specs-actors/actors/builtin" + + "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/chain/types" +) + +func Load(store adt.Store, act *types.Actor) (State, error) { + switch act.Code { + case v0builtin.InitActorCodeID: + out := v0State{store: store} + err := store.Get(store.Context(), act.Head, &out) + if err != nil { + return nil, err + } + return &out, nil + } + return nil, xerrors.Errorf("unknown actor code %s", act.Code) +} + +type State interface { + cbor.Marshaler + + ResolveAddress(address addr.Address) (address.Address, bool, error) + MapAddressToNewID(address addr.Address) (address.Address, error) +} diff --git a/chain/actors/builtin/init/v0.go b/chain/actors/builtin/init/v0.go new file mode 100644 index 000000000..3a94f59f0 --- /dev/null +++ b/chain/actors/builtin/init/v0.go @@ -0,0 +1,22 @@ +package init + +import ( + "github.com/filecoin-project/go-address" + + "github.com/filecoin-project/specs-actors/actors/builtin/init" + + "github.com/filecoin-project/lotus/chain/actors/adt" +) + +type v0State struct { + init.State + store adt.Store +} + +func (s *v0State) ResolveAddress(address address.Address) (address.Address, bool, error) { + return s.State.ResolveAddress(s.store, address) +} + +func (s *v0State) MapAddressToNewID(address address.Address) (address.Address, error) { + return s.State.MapAddressToNewID(s.store, address) +} diff --git a/chain/actors/builtin/market/market.go b/chain/actors/builtin/market/market.go new file mode 100644 index 000000000..663fa0e83 --- /dev/null +++ b/chain/actors/builtin/market/market.go @@ -0,0 +1,32 @@ +package market + +import ( + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/cbor" + "github.com/ipfs/go-cid" +) + +func Load(store adt.Store, act *types.Actor) (st State, err error) { + switch act.Code { + case v0builtin.MarketActorCodeID: + out := v0State{store: store} + err := store.Get(store.Context(), act.Head, &out) + if err != nil { + return nil, err + } + return &out, nil + } + return nil, xerrors.Errorf("unknown actor code %s", act.Code) +} + +type State interface { + cbor.Marshaler + EscrowTable() (BalanceTable, error) + LockedTable() (BalanceTable, error) + TotalLocked() (abi.TokenAmount, error) +} + +type BalanceTable interface { + Get(key address.Address) (abi.TokenAmount, error) +} diff --git a/chain/actors/builtin/market/v0.go b/chain/actors/builtin/market/v0.go new file mode 100644 index 000000000..23f33e3d3 --- /dev/null +++ b/chain/actors/builtin/market/v0.go @@ -0,0 +1,25 @@ +package market + +import ( + "github.com/filecoin-project/specs-actors/actors/builtin/market" + "github.com/filecoin-project/specs-actors/actors/util/adt" +) + +type v0State struct { + market.State + store adt.Store +} + +func (s *v0State) TotalLocked() (abi.TokenAmount, error) { + fml := types.BigAdd(s.TotalClientLockedCollateral, s.TotalProviderLockedCollateral) + fml = types.BigAdd(fml, s.TotalClientStorageFee) + return fml, nil +} + +func (s *v0State) EscrowTable() (BalanceTable, error) { + return adt.AsBalanceTable(s.store, s.State.EscrowTable) +} + +func (s *v0State) Lockedtable() (BalanceTable, error) { + return adt.AsBalanceTable(s.store, s.State.LockedTable) +} diff --git a/chain/actors/builtin/miner/miner.go b/chain/actors/builtin/miner/miner.go new file mode 100644 index 000000000..50453827d --- /dev/null +++ b/chain/actors/builtin/miner/miner.go @@ -0,0 +1,59 @@ +package miner + +import ( + "github.com/filecoin-project/go-bitfield" + "github.com/filecoin-project/go-state-types/abi" + "golang.org/x/xerrors" + + v0builtin "github.com/filecoin-project/specs-actors/actors/builtin" + + "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/chain/types" +) + +func Load(store adt.Store, act *types.Actor) (st State, err error) { + switch act.Code { + case v0builtin.StorageMinerActorCodeID: + out := v0State{store: store} + err := store.Get(store.Context(), act.Head, &out) + if err != nil { + return nil, err + } + return &out, nil + } + return nil, xerrors.Errorf("unknown actor code %s", act.Code) +} + +type State interface { + cbor.Marshaler + + LoadDeadline(idx uint64) (Deadline, error) + ForEachDeadline(cb func(idx uint64, dl Deadline) error) error + NumDeadlines() (uint64, error) +} + +type Deadline interface { + LoadPartition(idx uint64) (Partition, error) + ForEachPartition(cb func(idx uint64, part Partition) error) error +} + +type Partition interface { + AllSectors() (bitfield.BitField, error) + FaultySectors() (bitfield.BitField, error) + RecoveringSectors() (bitfield.BitField, error) + LiveSectors() (bitfield.BitField, error) + ActiveSectors() (bitfield.BitField, error) +} + +type MinerInfo struct { + Owner address.Address // Must be an ID-address. + Worker address.Address // Must be an ID-address. + NewWorker address.Address // Must be an ID-address. + ControlAddresses []address.Address // Must be an ID-addresses. + WorkerChangeEpoch abi.ChainEpoch + PeerId *peer.ID + Multiaddrs []abi.Multiaddrs + SealProofType abi.RegisteredSealProof + SectorSize abi.SectorSize + WindowPoStPartitionSectors uint64 +} diff --git a/chain/actors/builtin/miner/v0.go b/chain/actors/builtin/miner/v0.go new file mode 100644 index 000000000..8898a26a5 --- /dev/null +++ b/chain/actors/builtin/miner/v0.go @@ -0,0 +1,112 @@ +package miner + +import ( + "github.com/filecoin-project/go-bitfield" + "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/libp2p/go-libp2p-core/peer" + + "github.com/filecoin-project/specs-actors/actors/builtin/miner" +) + +type v0State struct { + miner.State + store adt.Store +} + +type v0Deadline struct { + miner.Deadline + store adt.Store +} + +type v0Partition struct { + miner.Partition + store adt.Store +} + +func (s *v0State) LoadDeadline(idx uint64) (Deadline, error) { + dls, err := s.State.LoadDeadlines(s.store) + if err != nil { + return nil, err + } + dl, err := dls.LoadDeadline(s.store, idx) + if err != nil { + return nil, err + } + return &v0Deadline{*dl, s.store}, nil +} + +func (s *v0State) ForEachDeadline(cb func(uint64, Deadline) error) error { + dls, err := s.State.LoadDeadlines(s.store) + if err != nil { + return err + } + return dls.ForEach(s.store, func(i uint64, dl *miner.Deadline) error { + return cb(i, &v0Deadline{*dl, s.store}) + }) +} + +func (s *v0State) NumDeadlines() (uint64, error) { + return miner.WPoStPeriodDeadlines, nil +} + +func (s *v0State) Info() (MinerInfo, error) { + info, err := s.State.GetInfo(s.store) + + var pid *peer.ID + if peerID, err := peer.IDFromBytes(info.PeerId); err == nil { + pid = &peerID + } + + mi := MinerInfo{ + Owner: info.Owner, + Worker: info.Worker, + ControlAddresses: info.ControlAddresses, + + NewWorker: address.Undef, + WorkerChangeEpoch: -1, + + PeerId: pid, + Multiaddrs: info.Multiaddrs, + SealProofType: info.SealProofType, + SectorSize: info.SectorSize, + WindowPoStPartitionSectors: info.WindowPoStPartitionSectors, + } + + if info.PendingWorkerKey != nil { + mi.NewWorker = info.PendingWorkerKey.NewWorker + mi.WorkerChangeEpoch = info.PendingWorkerKey.EffectiveAt + } + + return mi +} + +func (d *v0Deadline) LoadPartition(idx uint64) (Partition, error) { + p, err := d.Deadline.LoadPartition(d.store, idx) + if err != nil { + return nil, err + } + return &v0Partition{*p, d.store}, nil +} + +func (d *v0Deadline) ForEachPartition(cb func(uint64, Partition) error) error { + ps, err := d.Deadline.PartitionsArray(d.store) + if err != nil { + return err + } + var part miner.Partition + return ps.ForEach(&part, func(i int64) error { + return cb(uint64(i), &v0Partition{part, d.store}) + }) +} + +func (p *v0Partition) AllSectors() (bitfield.BitField, error) { + return p.Partition.Sectors, nil +} + +func (p *v0Partition) FaultySectors() (bitfield.BitField, error) { + return p.Partition.Faults, nil +} + +func (p *v0Partition) RecoveringSectors() (bitfield.BitField, error) { + return p.Partition.Recoveries, nil +} diff --git a/chain/actors/builtin/power/power.go b/chain/actors/builtin/power/power.go new file mode 100644 index 000000000..7671877e3 --- /dev/null +++ b/chain/actors/builtin/power/power.go @@ -0,0 +1,28 @@ +package power + +import ( + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/cbor" + "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/chain/types" + "golang.org/x/xerrors" +) + +func Load(store adt.Store, act *types.Actor) (st State, err error) { + switch act.Code { + case v0builtin.PowerActorCodeID: + out := v0State{store: store} + err := store.Get(store.Context(), act.Head, &out) + if err != nil { + return nil, err + } + return &out, nil + } + return nil, xerrors.Errorf("unknown actor code %s", act.Code) +} + +type State interface { + cbor.Marshaler + + TotalLocked() (abi.TokenAmount, error) +} diff --git a/chain/actors/builtin/power/v0.go b/chain/actors/builtin/power/v0.go new file mode 100644 index 000000000..8851080e6 --- /dev/null +++ b/chain/actors/builtin/power/v0.go @@ -0,0 +1,16 @@ +package power + +import ( + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/specs-actors/actors/builtin/power" +) + +type v0State struct { + power.State + store adt.Store +} + +func (s *v0State) TotalLocked() (abi.TokenAmount, error) { + return s.TotalPledgeCollateral, nil +} diff --git a/chain/actors/version.go b/chain/actors/version.go new file mode 100644 index 000000000..99cc59eaa --- /dev/null +++ b/chain/actors/version.go @@ -0,0 +1,24 @@ +package actors + +import ( + "fmt" + + "github.com/filecoin-project/go-state-types/network" +) + +type Version int + +const ( + Version0 = iota + Version1 +) + +// VersionForNetwork resolves the network version into an specs-actors version. +func VersionForNetwork(v network.Version) Version { + switch v { + case network.Version0, network.Version1: + return Version0 + default: + panic(fmt.Sprintf("unimplemented network version: %d", v)) + } +} diff --git a/chain/messagepool/provider.go b/chain/messagepool/provider.go index 80b9a4297..d67468d9a 100644 --- a/chain/messagepool/provider.go +++ b/chain/messagepool/provider.go @@ -52,8 +52,8 @@ func (mpp *mpoolProvider) GetActorAfter(addr address.Address, ts *types.TipSet) if err != nil { return nil, xerrors.Errorf("computing tipset state for GetActor: %w", err) } - - return &act, mpp.sm.WithStateTree(stcid, mpp.sm.WithActor(addr, stmgr.GetActor(&act))) + version := mpp.sm.GetNtwkVersion(context.TODO(), ts.Height()) + return &act, mpp.sm.WithStateTree(stcid, version, mpp.sm.WithActor(addr, stmgr.GetActor(&act))) } func (mpp *mpoolProvider) StateAccountKey(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) { diff --git a/chain/state/statetree.go b/chain/state/statetree.go index c083f1817..684e6ce5d 100644 --- a/chain/state/statetree.go +++ b/chain/state/statetree.go @@ -4,10 +4,12 @@ import ( "context" "fmt" + "github.com/filecoin-project/go-state-types/network" + "github.com/filecoin-project/lotus/chain/actors/builtin/init" + init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init" "github.com/filecoin-project/specs-actors/actors/builtin" - init_ "github.com/filecoin-project/specs-actors/actors/builtin/init" - "github.com/filecoin-project/specs-actors/actors/util/adt" + "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" logging "github.com/ipfs/go-log/v2" @@ -22,7 +24,7 @@ var log = logging.Logger("statetree") // StateTree stores actors state by their ID. type StateTree struct { - root *adt.Map + root adt.Map Store cbor.IpldStore snaps *stateSnaps @@ -115,17 +117,18 @@ func (ss *stateSnaps) deleteActor(addr address.Address) { ss.layers[len(ss.layers)-1].actors[addr] = streeOp{Delete: true} } -func NewStateTree(cst cbor.IpldStore) (*StateTree, error) { +func NewStateTree(cst cbor.IpldStore, version network.Version) (*StateTree, error) { return &StateTree{ - root: adt.MakeEmptyMap(adt.WrapStore(context.TODO(), cst)), + root: adt.NewMap(adt.WrapStore(context.TODO(), cst), version), Store: cst, snaps: newStateSnaps(), }, nil } -func LoadStateTree(cst cbor.IpldStore, c cid.Cid) (*StateTree, error) { - nd, err := adt.AsMap(adt.WrapStore(context.TODO(), cst), c) +func LoadStateTree(cst cbor.IpldStore, c cid.Cid, version network.Version) (*StateTree, error) { + // NETUPGRADE: switch map adt type on version upgrade. + nd, err := adt.AsMap(adt.WrapStore(context.TODO(), cst), c, version) if err != nil { log.Errorf("loading hamt node %s failed: %s", c, err) return nil, err @@ -165,12 +168,12 @@ func (st *StateTree) LookupID(addr address.Address) (address.Address, error) { return address.Undef, xerrors.Errorf("getting init actor: %w", err) } - var ias init_.State - if err := st.Store.Get(context.TODO(), act.Head, &ias); err != nil { + ias, err := init.Load(&AdtStore{st.Store}, &act) + if err != nil { return address.Undef, xerrors.Errorf("loading init actor state: %w", err) } - a, found, err := ias.ResolveAddress(&AdtStore{st.Store}, addr) + a, found, err := ias.ResolveAddress(addr) if err == nil && !found { err = types.ErrActorNotFound } @@ -283,18 +286,18 @@ func (st *StateTree) ClearSnapshot() { func (st *StateTree) RegisterNewAddress(addr address.Address) (address.Address, error) { var out address.Address err := st.MutateActor(builtin.InitActorAddr, func(initact *types.Actor) error { - var ias init_.State - if err := st.Store.Get(context.TODO(), initact.Head, &ias); err != nil { + ias, err := init.Load(&AdtStore{st.Store}, initact) + if err != nil { return err } - oaddr, err := ias.MapAddressToNewID(&AdtStore{st.Store}, addr) + oaddr, err := ias.MapAddressToNewID(addr) if err != nil { return err } out = oaddr - ncid, err := st.Store.Put(context.TODO(), &ias) + ncid, err := st.Store.Put(context.TODO(), ias) if err != nil { return err } diff --git a/chain/stmgr/read.go b/chain/stmgr/read.go index c707b5195..2daa9f79d 100644 --- a/chain/stmgr/read.go +++ b/chain/stmgr/read.go @@ -10,146 +10,54 @@ import ( cbor "github.com/ipfs/go-ipld-cbor" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/specs-actors/actors/util/adt" ) -type StateTreeCB func(state *state.StateTree) error - -func (sm *StateManager) WithParentStateTsk(tsk types.TipSetKey, cb StateTreeCB) error { +func (sm *StateManager) ParentStateTsk(tsk types.TipSetKey) (*state.StateTree, error) { ts, err := sm.cs.GetTipSetFromKey(tsk) if err != nil { - return xerrors.Errorf("loading tipset %s: %w", tsk, err) + return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) } + return sm.ParentState(ts, cb) +} +func (sm *StateManager) ParentState(ts *types.TipSet) (*state.StateTree, error) { cst := cbor.NewCborStore(sm.cs.Blockstore()) - state, err := state.LoadStateTree(cst, sm.parentState(ts)) + version := sm.GetNtwkVersion(context.TODO(), ts.Height()-1) + state, err := state.LoadStateTree(cst, sm.parentState(ts), version) if err != nil { - return xerrors.Errorf("load state tree: %w", err) + return nil, xerrors.Errorf("load state tree: %w", err) } - return cb(state) + return state, nil } -func (sm *StateManager) WithParentState(ts *types.TipSet, cb StateTreeCB) error { +func (sm *StateManager) StateTree(st cid.Cid, ntwkVersion network.Version) (*state.StateTree, error) { cst := cbor.NewCborStore(sm.cs.Blockstore()) - state, err := state.LoadStateTree(cst, sm.parentState(ts)) + state, err := state.LoadStateTree(cst, st, ntwkVersion) if err != nil { - return xerrors.Errorf("load state tree: %w", err) + return nil, xerrors.Errorf("load state tree: %w", err) } - return cb(state) + return state, nil } -func (sm *StateManager) WithStateTree(st cid.Cid, cb StateTreeCB) error { - cst := cbor.NewCborStore(sm.cs.Blockstore()) - state, err := state.LoadStateTree(cst, st) +func (sm *StateManager) LoadActor(_ context.Context, addr address.Address, ts *types.TipSet) (*types.Actor, error) { + state, err := sm.ParentState(ts) if err != nil { - return xerrors.Errorf("load state tree: %w", err) + return nil, err } - - return cb(state) + return state.GetActor(addr) } -type ActorCB func(act *types.Actor) error - -func GetActor(out *types.Actor) ActorCB { - return func(act *types.Actor) error { - *out = *act - return nil - } -} - -func (sm *StateManager) WithActor(addr address.Address, cb ActorCB) StateTreeCB { - return func(state *state.StateTree) error { - act, err := state.GetActor(addr) - if err != nil { - return xerrors.Errorf("get actor: %w", err) - } - - return cb(act) - } -} - -// WithActorState usage: -// Option 1: WithActorState(ctx, idAddr, func(store adt.Store, st *ActorStateType) error {...}) -// Option 2: WithActorState(ctx, idAddr, actorStatePtr) -func (sm *StateManager) WithActorState(ctx context.Context, out interface{}) ActorCB { - return func(act *types.Actor) error { - store := sm.cs.Store(ctx) - - outCallback := reflect.TypeOf(out).Kind() == reflect.Func - - var st reflect.Value - if outCallback { - st = reflect.New(reflect.TypeOf(out).In(1).Elem()) - } else { - st = reflect.ValueOf(out) - } - if err := store.Get(ctx, act.Head, st.Interface()); err != nil { - return xerrors.Errorf("read actor head: %w", err) - } - - if outCallback { - out := reflect.ValueOf(out).Call([]reflect.Value{reflect.ValueOf(store), st}) - if !out[0].IsNil() && out[0].Interface().(error) != nil { - return out[0].Interface().(error) - } - } - - return nil - } -} - -type DeadlinesCB func(store adt.Store, deadlines *miner.Deadlines) error - -func (sm *StateManager) WithDeadlines(cb DeadlinesCB) func(store adt.Store, mas *miner.State) error { - return func(store adt.Store, mas *miner.State) error { - deadlines, err := mas.LoadDeadlines(store) - if err != nil { - return err - } - - return cb(store, deadlines) - } -} - -type DeadlineCB func(store adt.Store, idx uint64, deadline *miner.Deadline) error - -func (sm *StateManager) WithDeadline(idx uint64, cb DeadlineCB) DeadlinesCB { - return func(store adt.Store, deadlines *miner.Deadlines) error { - d, err := deadlines.LoadDeadline(store, idx) - if err != nil { - return err - } - - return cb(store, idx, d) - } -} - -func (sm *StateManager) WithEachDeadline(cb DeadlineCB) DeadlinesCB { - return func(store adt.Store, deadlines *miner.Deadlines) error { - return deadlines.ForEach(store, func(dlIdx uint64, dl *miner.Deadline) error { - return cb(store, dlIdx, dl) - }) - } -} - -type PartitionCB func(store adt.Store, idx uint64, partition *miner.Partition) error - -func (sm *StateManager) WithEachPartition(cb PartitionCB) DeadlineCB { - return func(store adt.Store, idx uint64, deadline *miner.Deadline) error { - parts, err := deadline.PartitionsArray(store) - if err != nil { - return err - } - - var partition miner.Partition - return parts.ForEach(&partition, func(i int64) error { - p := partition - return cb(store, uint64(i), &p) - }) +func (sm *StateManager) LoadActorTsk(_ context.Context, addr address.Address, tsk types.TipSetKey) (*types.Actor, error) { + state, err := sm.ParentStateTsk(tsk) + if err != nil { + return nil, err } + return state.GetActor(addr) } diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index 929c9daf7..fa4b08147 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -7,14 +7,14 @@ import ( "github.com/filecoin-project/specs-actors/actors/runtime" - "github.com/filecoin-project/specs-actors/actors/builtin/power" - "github.com/filecoin-project/specs-actors/actors/builtin/multisig" "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/actors/builtin/market" + "github.com/filecoin-project/lotus/chain/actors/builtin/power" "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" @@ -23,7 +23,6 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/builtin/market" "github.com/filecoin-project/specs-actors/actors/builtin/reward" "github.com/filecoin-project/specs-actors/actors/util/adt" @@ -201,6 +200,7 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp for i := parentEpoch; i < epoch; i++ { // handle state forks + // XXX: The state tre err = sm.handleStateForks(ctx, vmi.StateTree(), i, ts) if err != nil { return cid.Undef, cid.Undef, xerrors.Errorf("error handling state forks: %w", err) @@ -711,11 +711,14 @@ func (sm *StateManager) ListAllActors(ctx context.Context, ts *types.TipSet) ([] } func (sm *StateManager) MarketBalance(ctx context.Context, addr address.Address, ts *types.TipSet) (api.MarketBalance, error) { - var state market.State - _, err := sm.LoadActorState(ctx, builtin.StorageMarketActorAddr, &state, ts) + st, err := sm.ParentState(ts) if err != nil { return api.MarketBalance{}, err } + act, err := st.GetActor(builtin.StorageMarketActorAddr) + if err != nil { + return nil, err + } addr, err = sm.LookupID(ctx, addr, ts) if err != nil { @@ -1016,19 +1019,17 @@ func GetFilMined(ctx context.Context, st *state.StateTree) (abi.TokenAmount, err } func getFilMarketLocked(ctx context.Context, st *state.StateTree) (abi.TokenAmount, error) { - mactor, err := st.GetActor(builtin.StorageMarketActorAddr) + act, err := st.GetActor(builtin.StorageMarketActorAddr) if err != nil { return big.Zero(), xerrors.Errorf("failed to load market actor: %w", err) } - var mst market.State - if err := st.Store.Get(ctx, mactor.Head, &mst); err != nil { + mst, err := market.Load(adt.WrapStore(ctx, st.Store), act) + if err != nil { return big.Zero(), xerrors.Errorf("failed to load market state: %w", err) } - fml := types.BigAdd(mst.TotalClientLockedCollateral, mst.TotalProviderLockedCollateral) - fml = types.BigAdd(fml, mst.TotalClientStorageFee) - return fml, nil + return mst.TotalLocked() } func getFilPowerLocked(ctx context.Context, st *state.StateTree) (abi.TokenAmount, error) { @@ -1037,11 +1038,12 @@ func getFilPowerLocked(ctx context.Context, st *state.StateTree) (abi.TokenAmoun return big.Zero(), xerrors.Errorf("failed to load power actor: %w", err) } - var pst power.State - if err := st.Store.Get(ctx, pactor.Head, &pst); err != nil { + pst, err := power.Load(adt.WrapStore(ctx, st.Store), act) + if err != nil { return big.Zero(), xerrors.Errorf("failed to load power state: %w", err) } - return pst.TotalPledgeCollateral, nil + + return pst.TotalLocked(), nil } func (sm *StateManager) GetFilLocked(ctx context.Context, st *state.StateTree) (abi.TokenAmount, error) { diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index b21400da6..a137afb51 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -56,30 +56,6 @@ func GetNetworkName(ctx context.Context, sm *StateManager, st cid.Cid) (dtypes.N return dtypes.NetworkName(state.NetworkName), nil } -func (sm *StateManager) LoadActorState(ctx context.Context, addr address.Address, out interface{}, ts *types.TipSet) (*types.Actor, error) { - var a *types.Actor - if err := sm.WithParentState(ts, sm.WithActor(addr, func(act *types.Actor) error { - a = act - return sm.WithActorState(ctx, out)(act) - })); err != nil { - return nil, err - } - - return a, nil -} - -func (sm *StateManager) LoadActorStateRaw(ctx context.Context, addr address.Address, out interface{}, st cid.Cid) (*types.Actor, error) { - var a *types.Actor - if err := sm.WithStateTree(st, sm.WithActor(addr, func(act *types.Actor) error { - a = act - return sm.WithActorState(ctx, out)(act) - })); err != nil { - return nil, err - } - - return a, nil -} - func GetMinerWorkerRaw(ctx context.Context, sm *StateManager, st cid.Cid, maddr address.Address) (address.Address, error) { var mas miner.State _, err := sm.LoadActorStateRaw(ctx, maddr, &mas, st) diff --git a/chain/sub/incoming.go b/chain/sub/incoming.go index 34dde227f..7c672bee2 100644 --- a/chain/sub/incoming.go +++ b/chain/sub/incoming.go @@ -11,7 +11,6 @@ import ( "golang.org/x/xerrors" address "github.com/filecoin-project/go-address" - miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/specs-actors/actors/util/adt" lru "github.com/hashicorp/golang-lru" blocks "github.com/ipfs/go-block-format" @@ -28,6 +27,7 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/messagepool" "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/stmgr" @@ -432,9 +432,10 @@ func (bv *BlockValidator) checkPowerAndGetWorkerKey(ctx context.Context, bh *typ if err != nil { return address.Undef, err } + buf := bufbstore.NewBufferedBstore(bv.chain.Blockstore()) cst := cbor.NewCborStore(buf) - state, err := state.LoadStateTree(cst, st) + state, err := state.LoadStateTree(cst, st, bv.stmgr.GetNtwkVersion(ctx, ts.Height())) if err != nil { return address.Undef, err } @@ -443,19 +444,12 @@ func (bv *BlockValidator) checkPowerAndGetWorkerKey(ctx context.Context, bh *typ return address.Undef, err } - blk, err := bv.chain.Blockstore().Get(act.Head) - if err != nil { - return address.Undef, err - } - aso := blk.RawData() - - var mst miner.State - err = mst.UnmarshalCBOR(bytes.NewReader(aso)) + mst, err := miner.Load(bv.chain.Store(ctx), act) if err != nil { return address.Undef, err } - info, err := mst.GetInfo(adt.WrapStore(ctx, cst)) + info, err := mst.Info() if err != nil { return address.Undef, err } diff --git a/chain/sync.go b/chain/sync.go index d2cf08b92..d5bd777b7 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -1027,9 +1027,10 @@ func (syncer *Syncer) checkBlockMessages(ctx context.Context, b *types.FullBlock if err != nil { return err } + nwVersion := syncer.sm.GetNtwkVersion(ctx, baseTs.Height()) cst := cbor.NewCborStore(syncer.store.Blockstore()) - st, err := state.LoadStateTree(cst, stateroot) + st, err := state.LoadStateTree(cst, stateroot, nwVersion) if err != nil { return xerrors.Errorf("failed to load base state tree: %w", err) } diff --git a/chain/vm/invoker.go b/chain/vm/invoker.go index 2ec56a9db..8583b0438 100644 --- a/chain/vm/invoker.go +++ b/chain/vm/invoker.go @@ -47,6 +47,7 @@ func NewInvoker() *Invoker { } // add builtInCode using: register(cid, singleton) + // NETUPGRADE: register code IDs for v2, etc. inv.Register(builtin.SystemActorCodeID, system.Actor{}, adt.EmptyValue{}) inv.Register(builtin.InitActorCodeID, init_.Actor{}, init_.State{}) inv.Register(builtin.RewardActorCodeID, reward.Actor{}, reward.State{}) diff --git a/cmd/lotus-shed/balances.go b/cmd/lotus-shed/balances.go index aad321783..f3ffd140f 100644 --- a/cmd/lotus-shed/balances.go +++ b/cmd/lotus-shed/balances.go @@ -12,6 +12,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/store" @@ -170,7 +171,9 @@ var chainBalanceStateCmd = &cli.Command{ sm := stmgr.NewStateManager(cs) - tree, err := state.LoadStateTree(cst, sroot) + // NETUPGRADE: FIXME. + // Options: (a) encode the version in the chain or (b) pass a flag. + tree, err := state.LoadStateTree(cst, sroot, network.Version0) if err != nil { return err } diff --git a/cmd/lotus-shed/genesis-verify.go b/cmd/lotus-shed/genesis-verify.go index 043cb72bb..62808db9b 100644 --- a/cmd/lotus-shed/genesis-verify.go +++ b/cmd/lotus-shed/genesis-verify.go @@ -79,7 +79,7 @@ var genesisVerifyCmd = &cli.Command{ cst := cbor.NewCborStore(bs) - stree, err := state.LoadStateTree(cst, ts.ParentState()) + stree, err := state.LoadStateTree(cst, ts.ParentState(), sm.GetNtwkVersion()) if err != nil { return err } diff --git a/cmd/lotus-storage-miner/proving.go b/cmd/lotus-storage-miner/proving.go index b5087556d..d9bf81376 100644 --- a/cmd/lotus-storage-miner/proving.go +++ b/cmd/lotus-storage-miner/proving.go @@ -12,9 +12,11 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" + "github.com/filecoin-project/lotus/api/apibstore" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" ) @@ -49,54 +51,41 @@ var provingFaultsCmd = &cli.Command{ ctx := lcli.ReqContext(cctx) + stor := store.ActorStore(ctx, apibstore.NewAPIBlockstore(api)) + maddr, err := getActorAddress(ctx, nodeApi, cctx.String("actor")) if err != nil { return err } - var mas miner.State - { - mact, err := api.StateGetActor(ctx, maddr, types.EmptyTSK) - if err != nil { - return err - } - rmas, err := api.ChainReadObj(ctx, mact.Head) - if err != nil { - return err - } - if err := mas.UnmarshalCBOR(bytes.NewReader(rmas)); err != nil { - return err - } + mact, err := api.StateGetActor(ctx, maddr, types.EmptyTSK) + if err != nil { + return err + } + + mas, err := miner.Load(stor, mact) + if err != nil { + return err } fmt.Printf("Miner: %s\n", color.BlueString("%s", maddr)) - head, err := api.ChainHead(ctx) - if err != nil { - return xerrors.Errorf("getting chain head: %w", err) - } - deadlines, err := api.StateMinerDeadlines(ctx, maddr, head.Key()) - if err != nil { - return xerrors.Errorf("getting miner deadlines: %w", err) - } tw := tabwriter.NewWriter(os.Stdout, 2, 4, 2, ' ', 0) _, _ = fmt.Fprintln(tw, "deadline\tpartition\tsectors") - for dlIdx := range deadlines { - partitions, err := api.StateMinerPartitions(ctx, maddr, uint64(dlIdx), types.EmptyTSK) - if err != nil { - return xerrors.Errorf("loading partitions for deadline %d: %w", dlIdx, err) - } - - for partIdx, partition := range partitions { - faulty, err := partition.Faults.All(10000000) + err = mas.ForEachDeadline(func(dlIdx uint64, dl miner.Deadline) error { + dl.ForEachPartition(func(partIdx uint64, part miner.Partition) error { + faults, err := part.Faults() if err != nil { return err } - - for _, num := range faulty { + faults.ForEach(func(num uint64) error { _, _ = fmt.Fprintf(tw, "%d\t%d\t%d\n", dlIdx, partIdx, num) - } - } + return nil + }) + }) + }) + if err != nil { + return err } return tw.Flush() }, @@ -132,67 +121,61 @@ var provingInfoCmd = &cli.Command{ return xerrors.Errorf("getting chain head: %w", err) } + mact, err := api.StateGetActor(ctx, maddr, head.Key()) + if err != nil { + return err + } + + stor := store.ActorStore(ctx, apibstore.NewAPIBlockstore(api)) + + mas, err := miner.Load(stor, mact) + if err != nil { + return err + } + cd, err := api.StateMinerProvingDeadline(ctx, maddr, head.Key()) if err != nil { return xerrors.Errorf("getting miner info: %w", err) } - deadlines, err := api.StateMinerDeadlines(ctx, maddr, head.Key()) - if err != nil { - return xerrors.Errorf("getting miner deadlines: %w", err) - } - fmt.Printf("Miner: %s\n", color.BlueString("%s", maddr)) - var mas miner.State - { - mact, err := api.StateGetActor(ctx, maddr, types.EmptyTSK) - if err != nil { - return err - } - rmas, err := api.ChainReadObj(ctx, mact.Head) - if err != nil { - return err - } - if err := mas.UnmarshalCBOR(bytes.NewReader(rmas)); err != nil { - return err - } - } - - parts := map[uint64][]*miner.Partition{} - for dlIdx := range deadlines { - part, err := api.StateMinerPartitions(ctx, maddr, uint64(dlIdx), types.EmptyTSK) - if err != nil { - return xerrors.Errorf("getting miner partition: %w", err) - } - - parts[uint64(dlIdx)] = part - } - proving := uint64(0) faults := uint64(0) recovering := uint64(0) + curDeadlineSectors := uint64(0) - for _, partitions := range parts { - for _, partition := range partitions { - sc, err := partition.Sectors.Count() - if err != nil { - return xerrors.Errorf("count partition sectors: %w", err) + if err := mas.ForEachDeadline(func(dlIdx uint64, dl miner.Deadline) error { + return dl.ForEachPartition(func(partIdx uint64, part miner.Partition) error { + if bf, err := part.LiveSectors(); err != nil { + return err + } else if count, err := bf.Count(); err != nil { + return err + } else { + proving += count + if dlIdx == cd.Index { + curDeadlineSectors += count + } } - proving += sc - fc, err := partition.Faults.Count() - if err != nil { - return xerrors.Errorf("count partition faults: %w", err) + if bf, err := part.Faults(); err != nil { + return err + } else if count, err := bf.Count(); err != nil { + return err + } else { + faults += count } - faults += fc - rc, err := partition.Recoveries.Count() - if err != nil { - return xerrors.Errorf("count partition recoveries: %w", err) + if bf, err := part.Recovering(); err != nil { + return err + } else if count, err := bf.Count(); err != nil { + return err + } else { + recovering += count } - recovering += rc - } + }) + }); err != nil { + return xerrors.Errorf("walking miner deadlines and partitions: %w", err) } var faultPerc float64 @@ -202,28 +185,15 @@ var provingInfoCmd = &cli.Command{ fmt.Printf("Current Epoch: %d\n", cd.CurrentEpoch) - fmt.Printf("Proving Period Boundary: %d\n", cd.PeriodStart%miner.WPoStProvingPeriod) + fmt.Printf("Proving Period Boundary: %d\n", cd.PeriodStart%cd.WPoStProvingPeriod) fmt.Printf("Proving Period Start: %s\n", epochTime(cd.CurrentEpoch, cd.PeriodStart)) - fmt.Printf("Next Period Start: %s\n\n", epochTime(cd.CurrentEpoch, cd.PeriodStart+miner.WPoStProvingPeriod)) + fmt.Printf("Next Period Start: %s\n\n", epochTime(cd.CurrentEpoch, cd.PeriodStart+cd.WPoStProvingPeriod)) fmt.Printf("Faults: %d (%.2f%%)\n", faults, faultPerc) fmt.Printf("Recovering: %d\n", recovering) fmt.Printf("Deadline Index: %d\n", cd.Index) - - if cd.Index < miner.WPoStPeriodDeadlines { - curDeadlineSectors := uint64(0) - for _, partition := range parts[cd.Index] { - sc, err := partition.Sectors.Count() - if err != nil { - return xerrors.Errorf("counting current deadline sectors: %w", err) - } - curDeadlineSectors += sc - } - - fmt.Printf("Deadline Sectors: %d\n", curDeadlineSectors) - } - + fmt.Printf("Deadline Sectors: %d\n", curDeadlineSectors) fmt.Printf("Deadline Open: %s\n", epochTime(cd.CurrentEpoch, cd.Open)) fmt.Printf("Deadline Close: %s\n", epochTime(cd.CurrentEpoch, cd.Close)) fmt.Printf("Deadline Challenge: %s\n", epochTime(cd.CurrentEpoch, cd.Challenge)) @@ -286,6 +256,7 @@ var provingDeadlinesCmd = &cli.Command{ if err != nil { return err } + miner.Load rmas, err := api.ChainReadObj(ctx, mact.Head) if err != nil { return err diff --git a/extern/storage-sealing/checks.go b/extern/storage-sealing/checks.go index 906c9c106..074c4cfcf 100644 --- a/extern/storage-sealing/checks.go +++ b/extern/storage-sealing/checks.go @@ -4,7 +4,7 @@ import ( "bytes" "context" - saproof "github.com/filecoin-project/specs-actors/actors/runtime/proof" + v0proof "github.com/filecoin-project/specs-actors/actors/runtime/proof" "golang.org/x/xerrors" @@ -170,7 +170,7 @@ func (m *Sealing) checkCommit(ctx context.Context, si SectorInfo, proof []byte, log.Warn("on-chain sealed CID doesn't match!") } - ok, err := m.verif.VerifySeal(saproof.SealVerifyInfo{ + ok, err := m.verif.VerifySeal(v0proof.SealVerifyInfo{ SectorID: m.minerSector(si.SectorNumber), SealedCID: pci.Info.SealedCID, SealProof: spt, diff --git a/go.mod b/go.mod index bc4067d67..9a7c5403b 100644 --- a/go.mod +++ b/go.mod @@ -34,11 +34,11 @@ require ( github.com/filecoin-project/go-multistore v0.0.3 github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 github.com/filecoin-project/go-paramfetch v0.0.2-0.20200701152213-3e0f0afdc261 - github.com/filecoin-project/go-state-types v0.0.0-20200905071437-95828685f9df + github.com/filecoin-project/go-state-types v0.0.0-20200911004822-964d6c679cfc github.com/filecoin-project/go-statemachine v0.0.0-20200813232949-df9b130df370 github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b - github.com/filecoin-project/specs-actors v0.9.7 + github.com/filecoin-project/specs-actors v0.9.9-0.20200911231631-727cd8845d30 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 github.com/filecoin-project/test-vectors/schema v0.0.1 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 diff --git a/go.sum b/go.sum index 0fa13fe70..106814b02 100644 --- a/go.sum +++ b/go.sum @@ -226,6 +226,8 @@ github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f h1 github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= github.com/filecoin-project/go-fil-markets v0.6.0 h1:gfxMweUHo4u+2BZh2Q7/7+cV0/ttikuJfhkkxLRsE2Q= github.com/filecoin-project/go-fil-markets v0.6.0/go.mod h1:LhSFYLkjaoe0vFRKABGYyw1Jz+9jCpF1sPA7yOftLTw= +github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= +github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24= github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200822201400-474f4fdccc52 h1:FXtCp0ybqdQL9knb3OGDpkNTaBbPxgkqPeWKotUwkH0= github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200822201400-474f4fdccc52/go.mod h1:XBBpuKIMaXIIzeqzO1iucq4GvbF8CxmXRFoezRh+Cx4= github.com/filecoin-project/go-multistore v0.0.3 h1:vaRBY4YiA2UZFPK57RNuewypB8u0DzzQwqsL0XarpnI= @@ -238,6 +240,8 @@ github.com/filecoin-project/go-state-types v0.0.0-20200903145444-247639ffa6ad/go github.com/filecoin-project/go-state-types v0.0.0-20200904021452-1883f36ca2f4/go.mod h1:IQ0MBPnonv35CJHtWSN3YY1Hz2gkPru1Q9qoaYLxx9I= github.com/filecoin-project/go-state-types v0.0.0-20200905071437-95828685f9df h1:m2esXSuGBkuXlRyCsl1a/7/FkFam63o1OzIgzaHtOfI= github.com/filecoin-project/go-state-types v0.0.0-20200905071437-95828685f9df/go.mod h1:IQ0MBPnonv35CJHtWSN3YY1Hz2gkPru1Q9qoaYLxx9I= +github.com/filecoin-project/go-state-types v0.0.0-20200911004822-964d6c679cfc h1:1vr/LoqGq5m5g37Q3sNSAjfwF1uJY0zmiHcvnxY6hik= +github.com/filecoin-project/go-state-types v0.0.0-20200911004822-964d6c679cfc/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-statemachine v0.0.0-20200714194326-a77c3ae20989/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= github.com/filecoin-project/go-statemachine v0.0.0-20200813232949-df9b130df370 h1:Jbburj7Ih2iaJ/o5Q9A+EAeTabME6YII7FLi9SKUf5c= github.com/filecoin-project/go-statemachine v0.0.0-20200813232949-df9b130df370/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= @@ -248,6 +252,8 @@ github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/ github.com/filecoin-project/specs-actors v0.9.4/go.mod h1:BStZQzx5x7TmCkLv0Bpa07U6cPKol6fd3w9KjMPZ6Z4= github.com/filecoin-project/specs-actors v0.9.7 h1:7PAZ8kdqwBdmgf/23FCkQZLCXcVu02XJrkpkhBikiA8= github.com/filecoin-project/specs-actors v0.9.7/go.mod h1:wM2z+kwqYgXn5Z7scV1YHLyd1Q1cy0R8HfTIWQ0BFGU= +github.com/filecoin-project/specs-actors v0.9.9-0.20200911231631-727cd8845d30 h1:6Kn6y3TpJbk5BsvhVha+3jr7C3gAAJq0rCnwUYOWRl0= +github.com/filecoin-project/specs-actors v0.9.9-0.20200911231631-727cd8845d30/go.mod h1:czlvLQGEX0fjLLfdNHD7xLymy6L3n7aQzRWzsYGf+ys= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= github.com/filecoin-project/test-vectors/schema v0.0.1 h1:5fNF76nl4qolEvcIsjc0kUADlTMVHO73tW4kXXPnsus= diff --git a/node/impl/full/state.go b/node/impl/full/state.go index c068bab93..e9e290fd2 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -20,7 +20,6 @@ import ( "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/builtin/market" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" samsig "github.com/filecoin-project/specs-actors/actors/builtin/multisig" "github.com/filecoin-project/specs-actors/actors/builtin/power" "github.com/filecoin-project/specs-actors/actors/builtin/reward" @@ -33,6 +32,7 @@ import ( "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/stmgr" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" @@ -112,6 +112,8 @@ func (a *StateAPI) StateMinerInfo(ctx context.Context, actor address.Address, ts return api.MinerInfo{}, xerrors.Errorf("loading tipset %s: %w", tsk, err) } + a.StateManager.LoadActorStateRaw(ctx context.Context, addr address.Address, out interface{}, st cid.Cid) + mi, err := stmgr.StateMinerInfo(ctx, a.StateManager, ts, actor) if err != nil { return api.MinerInfo{}, err @@ -119,17 +121,28 @@ func (a *StateAPI) StateMinerInfo(ctx context.Context, actor address.Address, ts return api.NewApiMinerInfo(mi), nil } -func (a *StateAPI) StateMinerDeadlines(ctx context.Context, m address.Address, tsk types.TipSetKey) ([]*miner.Deadline, error) { +func (a *StateAPI) StateMinerDeadlines(ctx context.Context, m address.Address, tsk types.TipSetKey) ([]miner.Deadline, error) { var out []*miner.Deadline - return out, a.StateManager.WithParentStateTsk(tsk, - a.StateManager.WithActor(m, - a.StateManager.WithActorState(ctx, - a.StateManager.WithDeadlines( - a.StateManager.WithEachDeadline( - func(store adt.Store, idx uint64, deadline *miner.Deadline) error { - out = append(out, deadline) - return nil - }))))) + state, err := a.StateManager.LoadParentStateTsk(tsk) + if err != nil { + return nil, err + } + act, err := state.GetActor(addr) + if err != nil { + return nil, err + } + mas, err := miner.Load(a.Chain.Store(ctx), act) + if err != nil { + return nil, err + } + var deadlines []miner.Deadline + if err := mas.ForEachDeadline(func(_ uint64, dl miner.Deadline) error { + deadlines = append(deadlines, dl) + return nil + }); err != nil { + return err + } + return deadlines, nil } func (a *StateAPI) StateMinerPartitions(ctx context.Context, m address.Address, dlIdx uint64, tsk types.TipSetKey) ([]*miner.Partition, error) { @@ -302,7 +315,7 @@ func (a *StateAPI) stateForTs(ctx context.Context, ts *types.TipSet) (*state.Sta buf := bufbstore.NewBufferedBstore(a.Chain.Blockstore()) cst := cbor.NewCborStore(buf) - return state.LoadStateTree(cst, st) + return state.LoadStateTree(cst, st, a.StateManager.GetNtwkVersion(ctx, ts.Height())) } func (a *StateAPI) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) { @@ -1169,16 +1182,9 @@ func (a *StateAPI) StateCirculatingSupply(ctx context.Context, tsk types.TipSetK return api.CirculatingSupply{}, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - st, _, err := a.StateManager.TipSetState(ctx, ts) + sTree, err := a.stateForTs(ctx, ts) if err != nil { return api.CirculatingSupply{}, err } - - cst := cbor.NewCborStore(a.Chain.Blockstore()) - sTree, err := state.LoadStateTree(cst, st) - if err != nil { - return api.CirculatingSupply{}, err - } - return a.StateManager.GetCirculatingSupplyDetailed(ctx, ts.Height(), sTree) } diff --git a/storage/adapter_storage_miner.go b/storage/adapter_storage_miner.go index 2869e48e5..706cd6e04 100644 --- a/storage/adapter_storage_miner.go +++ b/storage/adapter_storage_miner.go @@ -16,13 +16,13 @@ import ( "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/builtin/market" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/apibstore" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" sealing "github.com/filecoin-project/lotus/extern/storage-sealing" @@ -179,14 +179,11 @@ func (s SealingAPIAdapter) StateSectorPreCommitInfo(ctx context.Context, maddr a return nil, xerrors.Errorf("handleSealFailed(%d): temp error: %+v", sectorNumber, err) } - st, err := s.delegate.ChainReadObj(ctx, act.Head) - if err != nil { - return nil, xerrors.Errorf("handleSealFailed(%d): temp error: %+v", sectorNumber, err) - } + stor := store.ActorStore(ctx, apibstore.NewAPIBlockstore(s.delegate)) - var state miner.State - if err := state.UnmarshalCBOR(bytes.NewReader(st)); err != nil { - return nil, xerrors.Errorf("handleSealFailed(%d): temp error: unmarshaling miner state: %+v", sectorNumber, err) + state, err := miner.Load(stor, act) + if err != nil { + return nil, xerrors.Errorf("handleSealFailed(%d): temp error: loading miner state: %+v", sectorNumber, err) } stor := store.ActorStore(ctx, apibstore.NewAPIBlockstore(s.delegate)) precommits, err := adt.AsMap(stor, state.PreCommittedSectors) diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 51d71e331..0f215048a 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -6,28 +6,32 @@ import ( "errors" "time" - "github.com/filecoin-project/specs-actors/actors/runtime/proof" - "github.com/filecoin-project/go-bitfield" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" + "github.com/filecoin-project/go-state-types/dline" "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" "go.opencensus.io/trace" "golang.org/x/xerrors" + v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" + v0proof "github.com/filecoin-project/specs-actors/actors/runtime/proof" + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/api/apibstore" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" ) var errNoPartitions = errors.New("no partitions") -func (s *WindowPoStScheduler) failPost(deadline *miner.DeadlineInfo) { +func (s *WindowPoStScheduler) failPost(deadline *dline.Info) { log.Errorf("TODO") /*s.failLk.Lock() if eps > s.failed { @@ -36,7 +40,7 @@ func (s *WindowPoStScheduler) failPost(deadline *miner.DeadlineInfo) { s.failLk.Unlock()*/ } -func (s *WindowPoStScheduler) doPost(ctx context.Context, deadline *miner.DeadlineInfo, ts *types.TipSet) { +func (s *WindowPoStScheduler) doPost(ctx context.Context, deadline *dline.Info, ts *types.TipSet) { ctx, abort := context.WithCancel(ctx) s.abort = abort @@ -111,18 +115,26 @@ func (s *WindowPoStScheduler) checkSectors(ctx context.Context, check bitfield.B return sbf, nil } -func (s *WindowPoStScheduler) checkNextRecoveries(ctx context.Context, dlIdx uint64, partitions []*miner.Partition) error { +func (s *WindowPoStScheduler) checkNextRecoveries(ctx context.Context, dlIdx uint64, partitions []miner.Partition) error { ctx, span := trace.StartSpan(ctx, "storage.checkNextRecoveries") defer span.End() - params := &miner.DeclareFaultsRecoveredParams{ - Recoveries: []miner.RecoveryDeclaration{}, + params := &v0miner.DeclareFaultsRecoveredParams{ + Recoveries: []v0miner.RecoveryDeclaration{}, } faulty := uint64(0) for partIdx, partition := range partitions { - unrecovered, err := bitfield.SubtractBitField(partition.Faults, partition.Recoveries) + faults, err := partition.FaultySectors() + if err != nil { + return xerrors.Errorf("getting faults: %w", err) + } + recovering, err := partition.RecoveringSectors() + if err != nil { + return xerrors.Errorf("getting recovering: %w", err) + } + unrecovered, err := bitfield.SubtractBitField(faults, recovering) if err != nil { return xerrors.Errorf("subtracting recovered set from fault set: %w", err) } @@ -153,7 +165,7 @@ func (s *WindowPoStScheduler) checkNextRecoveries(ctx context.Context, dlIdx uin continue } - params.Recoveries = append(params.Recoveries, miner.RecoveryDeclaration{ + params.Recoveries = append(params.Recoveries, v0miner.RecoveryDeclaration{ Deadline: dlIdx, Partition: uint64(partIdx), Sectors: recovered, @@ -202,17 +214,17 @@ func (s *WindowPoStScheduler) checkNextRecoveries(ctx context.Context, dlIdx uin return nil } -func (s *WindowPoStScheduler) checkNextFaults(ctx context.Context, dlIdx uint64, partitions []*miner.Partition) error { +func (s *WindowPoStScheduler) checkNextFaults(ctx context.Context, dlIdx uint64, partitions []miner.Partition) error { ctx, span := trace.StartSpan(ctx, "storage.checkNextFaults") defer span.End() - params := &miner.DeclareFaultsParams{ - Faults: []miner.FaultDeclaration{}, + params := &v0miner.DeclareFaultsParams{ + Faults: []v0miner.FaultDeclaration{}, } bad := uint64(0) - for partIdx, partition := range partitions { + for _, partition := range partitions { toCheck, err := partition.ActiveSectors() if err != nil { return xerrors.Errorf("getting active sectors: %w", err) @@ -239,7 +251,7 @@ func (s *WindowPoStScheduler) checkNextFaults(ctx context.Context, dlIdx uint64, bad += c - params.Faults = append(params.Faults, miner.FaultDeclaration{ + params.Faults = append(params.Faults, v0miner.FaultDeclaration{ Deadline: dlIdx, Partition: uint64(partIdx), Sectors: faulty, @@ -286,20 +298,40 @@ func (s *WindowPoStScheduler) checkNextFaults(ctx context.Context, dlIdx uint64, return nil } -func (s *WindowPoStScheduler) runPost(ctx context.Context, di miner.DeadlineInfo, ts *types.TipSet) (*miner.SubmitWindowedPoStParams, error) { +func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *types.TipSet) (*v0miner.SubmitWindowedPoStParams, error) { ctx, span := trace.StartSpan(ctx, "storage.runPost") defer span.End() + stor := store.ActorStore(ctx, apibstore.NewAPIBlockstore(s.api)) + act, err := s.api.StateGetActor(context.TODO(), s.actor, ts.Key()) + if err != nil { + return nil, xerrors.Errorf("resolving actor: %w", err) + } + + mas, err := miner.Load(stor, act) + if err != nil { + return nil, xerrors.Errorf("getting miner state: %w", err) + } + go func() { // TODO: extract from runPost, run on fault cutoff boundaries // check faults / recoveries for the *next* deadline. It's already too // late to declare them for this deadline - declDeadline := (di.Index + 2) % miner.WPoStPeriodDeadlines + declDeadline := (di.Index + 2) % di.WPoStPeriodDeadlines - partitions, err := s.api.StateMinerPartitions(context.TODO(), s.actor, declDeadline, ts.Key()) + dl, err := mas.LoadDeadline(declDeadline) if err != nil { - log.Errorf("getting partitions: %v", err) + log.Errorf("loading deadline: %v", err) + return + } + var partitions []miner.Partition + err = dl.ForEachPartition(func(_ uint64, part miner.Partition) error { + partitions = append(partitions, part) + return nil + }) + if err != nil { + log.Errorf("loading partitions: %v", err) return } @@ -324,18 +356,27 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di miner.DeadlineInfo return nil, xerrors.Errorf("failed to get chain randomness for windowPost (ts=%d; deadline=%d): %w", ts.Height(), di, err) } - partitions, err := s.api.StateMinerPartitions(ctx, s.actor, di.Index, ts.Key()) + dl, err := mas.LoadDeadline(di.Index) if err != nil { - return nil, xerrors.Errorf("getting partitions: %w", err) + return nil, xerrors.Errorf("loading deadline: %w", err) } - params := &miner.SubmitWindowedPoStParams{ + var partitions []miner.Partitions + err = dl.ForEachPartition(func(_ uint64, part miner.Partition) error { + partitions = apppend(partitions, part) + return nil + }) + if err != nil { + return nil, xerrors.Errorf("loading partitions: %w", err) + } + + params := &v0miner.SubmitWindowedPoStParams{ Deadline: di.Index, - Partitions: make([]miner.PoStPartition, 0, len(partitions)), + Partitions: make([]v0miner.PoStPartition, 0, len(partitions)), Proofs: nil, } - var sinfos []proof.SectorInfo + var sinfos []v0proof.SectorInfo sidToPart := map[abi.SectorNumber]uint64{} skipCount := uint64(0) @@ -382,7 +423,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di miner.DeadlineInfo sidToPart[si.SectorNumber] = uint64(partIdx) } - params.Partitions = append(params.Partitions, miner.PoStPartition{ + params.Partitions = append(params.Partitions, v0miner.PoStPartition{ Index: uint64(partIdx), Skipped: skipped, }) @@ -436,7 +477,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di miner.DeadlineInfo return params, nil } -func (s *WindowPoStScheduler) sectorsForProof(ctx context.Context, goodSectors, allSectors bitfield.BitField, ts *types.TipSet) ([]proof.SectorInfo, error) { +func (s *WindowPoStScheduler) sectorsForProof(ctx context.Context, goodSectors, allSectors bitfield.BitField, ts *types.TipSet) ([]v0proof.SectorInfo, error) { sset, err := s.api.StateMinerSectors(ctx, s.actor, &goodSectors, false, ts.Key()) if err != nil { return nil, err @@ -446,22 +487,22 @@ func (s *WindowPoStScheduler) sectorsForProof(ctx context.Context, goodSectors, return nil, nil } - substitute := proof.SectorInfo{ + substitute := v0proof.SectorInfo{ SectorNumber: sset[0].ID, SealedCID: sset[0].Info.SealedCID, SealProof: sset[0].Info.SealProof, } - sectorByID := make(map[uint64]proof.SectorInfo, len(sset)) + sectorByID := make(map[uint64]v0proof.SectorInfo, len(sset)) for _, sector := range sset { - sectorByID[uint64(sector.ID)] = proof.SectorInfo{ + sectorByID[uint64(sector.ID)] = v0proof.SectorInfo{ SectorNumber: sector.ID, SealedCID: sector.Info.SealedCID, SealProof: sector.Info.SealProof, } } - proofSectors := make([]proof.SectorInfo, 0, len(sset)) + proofSectors := make([]v0proof.SectorInfo, 0, len(sset)) if err := allSectors.ForEach(func(sectorNo uint64) error { if info, found := sectorByID[sectorNo]; found { proofSectors = append(proofSectors, info) @@ -476,7 +517,7 @@ func (s *WindowPoStScheduler) sectorsForProof(ctx context.Context, goodSectors, return proofSectors, nil } -func (s *WindowPoStScheduler) submitPost(ctx context.Context, proof *miner.SubmitWindowedPoStParams) error { +func (s *WindowPoStScheduler) submitPost(ctx context.Context, proof *v0miner.SubmitWindowedPoStParams) error { ctx, span := trace.StartSpan(ctx, "storage.commitPost") defer span.End() diff --git a/storage/wdpost_sched.go b/storage/wdpost_sched.go index b238a490d..7f7c64926 100644 --- a/storage/wdpost_sched.go +++ b/storage/wdpost_sched.go @@ -8,7 +8,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" + "github.com/filecoin-project/go-state-types/dline" "github.com/filecoin-project/specs-storage/storage" "github.com/filecoin-project/lotus/api" @@ -37,7 +37,7 @@ type WindowPoStScheduler struct { cur *types.TipSet // if a post is in progress, this indicates for which ElectionPeriodStart - activeDeadline *miner.DeadlineInfo + activeDeadline *dline.Info abort context.CancelFunc //failed abi.ChainEpoch // eps @@ -68,7 +68,7 @@ func NewWindowedPoStScheduler(api storageMinerApi, fc config.MinerFeeConfig, sb }, nil } -func deadlineEquals(a, b *miner.DeadlineInfo) bool { +func deadlineEquals(a, b *dline.Info) bool { if a == nil || b == nil { return b == a } From 3401cb349b88c39ebf87a95151928c885b8cdf7e Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Fri, 11 Sep 2020 23:07:20 -0700 Subject: [PATCH 077/885] add mpool push and chainHead --- cmd/lotus-gateway/api.go | 52 ++++++++++++++++++++++++++++++++++------ 1 file changed, 45 insertions(+), 7 deletions(-) diff --git a/cmd/lotus-gateway/api.go b/cmd/lotus-gateway/api.go index 06c9a3ae5..42e9e4829 100644 --- a/cmd/lotus-gateway/api.go +++ b/cmd/lotus-gateway/api.go @@ -8,6 +8,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" "go.opencensus.io/trace" ) @@ -35,18 +36,55 @@ func (a *GatewayAPI) getTipsetTimestamp(ctx context.Context, tsk types.TipSetKey return time.Unix(int64(ts.Blocks()[0].Timestamp), 0), nil } +func (a *GatewayAPI) checkTipset(ctx context.Context, ts types.TipSetKey) error { + when, err := a.getTipsetTimestamp(ctx, ts) + if err != nil { + return err + } + + if time.Since(when) > time.Hour { + return ErrLookbackTooLong + } + + return nil +} + func (a *GatewayAPI) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) { ctx, span := trace.StartSpan(ctx, "StateGetActor") defer span.End() - when, err := a.getTipsetTimestamp(ctx, ts) - if err != nil { - return nil, err - } - - if time.Since(when) > time.Hour { - return nil, ErrLookbackTooLong + if err := a.checkTipset(ctx, ts); err != nil { + return nil, fmt.Errorf("bad tipset: %w", err) } return a.api.StateGetActor(ctx, actor, ts) } + +func (a *GatewayAPI) ChainHead(ctx context.Context) (*types.TipSet, error) { + ctx, span := trace.StartSpan(ctx, "ChainHead") + defer span.End() + // TODO: cache and invalidate cache when timestamp is up (or have internal ChainNotify) + + return a.api.ChainHead(ctx) +} + +func (a *GatewayAPI) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) { + ctx, span := trace.StartSpan(ctx, "ChainGetTipSet") + defer span.End() + + if err := a.checkTipset(ctx, tsk); err != nil { + return nil, fmt.Errorf("bad tipset: %w", err) + } + + // TODO: since we're limiting lookbacks, should just cache this (could really even cache the json response bytes) + return a.api.ChainGetTipSet(ctx, tsk) +} + +func (a *GatewayAPI) MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) { + ctx, span := trace.StartSpan(ctx, "MpoolPush") + defer span.End() + + // TODO: additional anti-spam checks + + return a.api.MpoolPush(ctx, sm) +} From 45a5eaf91306f57d00f59020049ce091fbaeb72a Mon Sep 17 00:00:00 2001 From: zgfzgf <1901989065@qq.com> Date: Mon, 14 Sep 2020 11:00:08 +0800 Subject: [PATCH 078/885] optimize Settings.nodeType --- node/builder.go | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/node/builder.go b/node/builder.go index 6c01258a6..d48cdef60 100644 --- a/node/builder.go +++ b/node/builder.go @@ -475,12 +475,18 @@ func Repo(r repo.Repo) Option { } func FullAPI(out *api.FullNode) Option { - return func(s *Settings) error { - resAPI := &impl.FullNodeAPI{} - s.invokes[ExtractApiKey] = fx.Extract(resAPI) - *out = resAPI - return nil - } + return Options( + func(s *Settings) error { + s.nodeType = repo.FullNode + return nil + }, + func(s *Settings) error { + resAPI := &impl.FullNodeAPI{} + s.invokes[ExtractApiKey] = fx.Extract(resAPI) + *out = resAPI + return nil + }, + ) } type StopFunc func(context.Context) error @@ -488,9 +494,8 @@ type StopFunc func(context.Context) error // New builds and starts new Filecoin node func New(ctx context.Context, opts ...Option) (StopFunc, error) { settings := Settings{ - modules: map[interface{}]fx.Option{}, - invokes: make([]fx.Option, _nInvokes), - nodeType: repo.FullNode, + modules: map[interface{}]fx.Option{}, + invokes: make([]fx.Option, _nInvokes), } // apply module options in the right order From 6c61ff0e9e0f7118cc0ccb3a42151dccbf8c1ce7 Mon Sep 17 00:00:00 2001 From: Aarsh Shah Date: Mon, 14 Sep 2020 12:04:50 +0530 Subject: [PATCH 079/885] log error --- markets/storageadapter/provider.go | 1 + 1 file changed, 1 insertion(+) diff --git a/markets/storageadapter/provider.go b/markets/storageadapter/provider.go index ed714ab61..50773af93 100644 --- a/markets/storageadapter/provider.go +++ b/markets/storageadapter/provider.go @@ -108,6 +108,7 @@ func (n *ProviderNodeAdapter) OnDealComplete(ctx context.Context, deal storagema curTime := time.Now() for time.Since(curTime) < addPieceRetryTimeout { if !xerrors.Is(err, sealing.ErrTooManySectorsSealing) { + log.Errorf("failed to addPiece for deal %d, err: %w", deal.DealID, err) break } select { From e45a7749b0d03c98fef4dd160edefff89306fe42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Albert=C2=B7Gou?= Date: Mon, 14 Sep 2020 16:20:01 +0800 Subject: [PATCH 080/885] Update sync.go --- cmd/lotus-chainwatch/syncer/sync.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/lotus-chainwatch/syncer/sync.go b/cmd/lotus-chainwatch/syncer/sync.go index 306cf8b3e..19ceddda9 100644 --- a/cmd/lotus-chainwatch/syncer/sync.go +++ b/cmd/lotus-chainwatch/syncer/sync.go @@ -5,6 +5,7 @@ import ( "context" "database/sql" "fmt" + cid "github.com/ipfs/go-cid/_rsrch/cidiface" "sync" "time" @@ -136,7 +137,7 @@ create unique index if not exists block_cid_uindex on blocks (cid,height); create materialized view if not exists state_heights - as select b.parentstateroot, min(b.height) height + as select min(b.height) height, b.parentstateroot from blocks b group by b.parentstateroot; create index if not exists state_heights_height_index From dcae513487524eb4dd3feb6cacaf205fe7b9a19d Mon Sep 17 00:00:00 2001 From: Frank Date: Mon, 14 Sep 2020 16:49:35 +0800 Subject: [PATCH 081/885] make nested dir --- node/repo/fsrepo.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index 14085d4ac..709d78d3a 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -101,7 +101,7 @@ func (fsr *FsRepo) Init(t RepoType) error { } log.Infof("Initializing repo at '%s'", fsr.path) - err = os.Mkdir(fsr.path, 0755) //nolint: gosec + err = os.MkdirAll(fsr.path, 0755) //nolint: gosec if err != nil && !os.IsExist(err) { return err } From 747b34c2c89d83b91bdb45d279e58c4329aa181a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Albert=C2=B7Gou?= Date: Mon, 14 Sep 2020 18:00:17 +0800 Subject: [PATCH 082/885] Update sync.go --- cmd/lotus-chainwatch/syncer/sync.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/lotus-chainwatch/syncer/sync.go b/cmd/lotus-chainwatch/syncer/sync.go index 19ceddda9..609b71088 100644 --- a/cmd/lotus-chainwatch/syncer/sync.go +++ b/cmd/lotus-chainwatch/syncer/sync.go @@ -5,7 +5,6 @@ import ( "context" "database/sql" "fmt" - cid "github.com/ipfs/go-cid/_rsrch/cidiface" "sync" "time" From 38f87981c1c5c2a60d65acb22df75a4058b41334 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 14 Sep 2020 13:14:06 +0200 Subject: [PATCH 083/885] Fix some build errors --- api/api_full.go | 6 +++--- api/apistruct/struct.go | 20 ++++++------------- api/types.go | 5 +---- build/version.go | 2 +- chain/actors/adt/adt.go | 4 ++-- chain/actors/builtin/builtin.go | 2 +- chain/actors/builtin/init/init.go | 8 ++++---- chain/actors/builtin/init/v0.go | 4 ++-- chain/actors/builtin/market/market.go | 9 +++++++-- chain/actors/builtin/market/v0.go | 4 +++- chain/actors/builtin/miner/miner.go | 7 +++++-- chain/actors/builtin/miner/v0.go | 6 +++++- chain/actors/builtin/power/power.go | 7 +++++-- chain/state/statetree.go | 28 +++++++++++++++------------ go.sum | 7 ------- 15 files changed, 61 insertions(+), 58 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index e5147db47..8c7340971 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -5,8 +5,6 @@ import ( "fmt" "time" - "github.com/filecoin-project/specs-actors/actors/runtime/proof" - "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p-core/peer" @@ -24,7 +22,9 @@ import ( "github.com/filecoin-project/specs-actors/actors/builtin/paych" "github.com/filecoin-project/specs-actors/actors/builtin/power" "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" + "github.com/filecoin-project/specs-actors/actors/runtime/proof" + miner2 "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" marketevents "github.com/filecoin-project/lotus/markets/loggers" "github.com/filecoin-project/lotus/node/modules/dtypes" @@ -317,7 +317,7 @@ type FullNode interface { // StateMinerPower returns the power of the indicated miner StateMinerPower(context.Context, address.Address, types.TipSetKey) (*MinerPower, error) // StateMinerInfo returns info about the indicated miner - StateMinerInfo(context.Context, address.Address, types.TipSetKey) (MinerInfo, error) + StateMinerInfo(context.Context, address.Address, types.TipSetKey) (miner2.MinerInfo, error) // StateMinerFaults returns a bitfield indicating the faulty sectors of the given miner StateMinerFaults(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error) // StateAllMinerFaults returns all non-expired Faults that occur within lookback epochs of the given tipset diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 3cf9a0add..45c3f7d63 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -2,6 +2,8 @@ package apistruct import ( "context" + "github.com/filecoin-project/go-state-types/dline" + miner2 "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "io" "time" @@ -160,11 +162,9 @@ type FullNodeStruct struct { StateNetworkName func(context.Context) (dtypes.NetworkName, error) `perm:"read"` StateMinerSectors func(context.Context, address.Address, *bitfield.BitField, bool, types.TipSetKey) ([]*api.ChainSectorInfo, error) `perm:"read"` StateMinerActiveSectors func(context.Context, address.Address, types.TipSetKey) ([]*api.ChainSectorInfo, error) `perm:"read"` - StateMinerProvingDeadline func(context.Context, address.Address, types.TipSetKey) (*miner.DeadlineInfo, error) `perm:"read"` + StateMinerProvingDeadline func(context.Context, address.Address, types.TipSetKey) (*dline.Info, error) `perm:"read"` StateMinerPower func(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error) `perm:"read"` - StateMinerInfo func(context.Context, address.Address, types.TipSetKey) (api.MinerInfo, error) `perm:"read"` - StateMinerDeadlines func(context.Context, address.Address, types.TipSetKey) ([]*miner.Deadline, error) `perm:"read"` - StateMinerPartitions func(context.Context, address.Address, uint64, types.TipSetKey) ([]*miner.Partition, error) `perm:"read"` + StateMinerInfo func(context.Context, address.Address, types.TipSetKey) (miner2.MinerInfo, error) `perm:"read"` StateMinerFaults func(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error) `perm:"read"` StateAllMinerFaults func(context.Context, abi.ChainEpoch, types.TipSetKey) ([]*api.Fault, error) `perm:"read"` StateMinerRecoveries func(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error) `perm:"read"` @@ -724,7 +724,7 @@ func (c *FullNodeStruct) StateMinerActiveSectors(ctx context.Context, addr addre return c.Internal.StateMinerActiveSectors(ctx, addr, tsk) } -func (c *FullNodeStruct) StateMinerProvingDeadline(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*miner.DeadlineInfo, error) { +func (c *FullNodeStruct) StateMinerProvingDeadline(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*dline.Info, error) { return c.Internal.StateMinerProvingDeadline(ctx, addr, tsk) } @@ -732,18 +732,10 @@ func (c *FullNodeStruct) StateMinerPower(ctx context.Context, a address.Address, return c.Internal.StateMinerPower(ctx, a, tsk) } -func (c *FullNodeStruct) StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (api.MinerInfo, error) { +func (c *FullNodeStruct) StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (miner2.MinerInfo, error) { return c.Internal.StateMinerInfo(ctx, actor, tsk) } -func (c *FullNodeStruct) StateMinerDeadlines(ctx context.Context, m address.Address, tsk types.TipSetKey) ([]*miner.Deadline, error) { - return c.Internal.StateMinerDeadlines(ctx, m, tsk) -} - -func (c *FullNodeStruct) StateMinerPartitions(ctx context.Context, m address.Address, dlIdx uint64, tsk types.TipSetKey) ([]*miner.Partition, error) { - return c.Internal.StateMinerPartitions(ctx, m, dlIdx, tsk) -} - func (c *FullNodeStruct) StateMinerFaults(ctx context.Context, actor address.Address, tsk types.TipSetKey) (bitfield.BitField, error) { return c.Internal.StateMinerFaults(ctx, actor, tsk) } diff --git a/api/types.go b/api/types.go index 4133a7105..53c0fe203 100644 --- a/api/types.go +++ b/api/types.go @@ -4,14 +4,11 @@ import ( "encoding/json" "fmt" - "github.com/filecoin-project/go-address" datatransfer "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/build" "github.com/ipfs/go-cid" - v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" - "github.com/libp2p/go-libp2p-core/peer" pubsub "github.com/libp2p/go-libp2p-pubsub" ma "github.com/multiformats/go-multiaddr" @@ -111,4 +108,4 @@ func NewDataTransferChannel(hostID peer.ID, channelState datatransfer.ChannelSta return channel } -type +// type TODO (stebalien): this was here, why was this here? diff --git a/build/version.go b/build/version.go index 338be1263..1e55b9889 100644 --- a/build/version.go +++ b/build/version.go @@ -53,7 +53,7 @@ func (ve Version) EqMajorMinor(v2 Version) bool { } // APIVersion is a semver version of the rpc api exposed -var APIVersion Version = newVer(0, 14, 0) +var APIVersion Version = newVer(0, 15, 0) //nolint:varcheck,deadcode const ( diff --git a/chain/actors/adt/adt.go b/chain/actors/adt/adt.go index 144150659..179c73b33 100644 --- a/chain/actors/adt/adt.go +++ b/chain/actors/adt/adt.go @@ -32,7 +32,7 @@ func AsMap(store Store, root cid.Cid, version network.Version) (Map, error) { func NewMap(store Store, version network.Version) (Map, error) { switch builtin.VersionForNetwork(version) { case builtin.Version0: - return v0adt.MakeEmptyMap(store) + return v0adt.MakeEmptyMap(store), nil } return nil, xerrors.Errorf("unknown network version: %d", version) } @@ -45,7 +45,7 @@ type Array interface { Delete(idx uint64) error Length() uint64 - ForEach(v cbor.Unmarshaler, fn func(idx int) error) error + ForEach(v cbor.Unmarshaler, fn func(idx int64) error) error } func AsArray(store Store, root cid.Cid, version network.Version) (Array, error) { diff --git a/chain/actors/builtin/builtin.go b/chain/actors/builtin/builtin.go index 173f50ba6..accc4e7e6 100644 --- a/chain/actors/builtin/builtin.go +++ b/chain/actors/builtin/builtin.go @@ -16,7 +16,7 @@ const ( func VersionForNetwork(version network.Version) Version { switch version { case network.Version0, network.Version1: - return Version + return Version0 default: panic(fmt.Sprintf("unsupported network version %d", version)) } diff --git a/chain/actors/builtin/init/init.go b/chain/actors/builtin/init/init.go index de71a032c..a96c0dfd6 100644 --- a/chain/actors/builtin/init/init.go +++ b/chain/actors/builtin/init/init.go @@ -1,10 +1,10 @@ package init import ( - "github.com/filecoin-project/go-bitfield" - "github.com/filecoin-project/go-state-types/cbor" "golang.org/x/xerrors" + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/cbor" v0builtin "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/adt" @@ -27,6 +27,6 @@ func Load(store adt.Store, act *types.Actor) (State, error) { type State interface { cbor.Marshaler - ResolveAddress(address addr.Address) (address.Address, bool, error) - MapAddressToNewID(address addr.Address) (address.Address, error) + ResolveAddress(address address.Address) (address.Address, bool, error) + MapAddressToNewID(address address.Address) (address.Address, error) } diff --git a/chain/actors/builtin/init/v0.go b/chain/actors/builtin/init/v0.go index 3a94f59f0..5001e9806 100644 --- a/chain/actors/builtin/init/v0.go +++ b/chain/actors/builtin/init/v0.go @@ -3,13 +3,13 @@ package init import ( "github.com/filecoin-project/go-address" - "github.com/filecoin-project/specs-actors/actors/builtin/init" + init_ "github.com/filecoin-project/specs-actors/actors/builtin/init" "github.com/filecoin-project/lotus/chain/actors/adt" ) type v0State struct { - init.State + init_.State store adt.Store } diff --git a/chain/actors/builtin/market/market.go b/chain/actors/builtin/market/market.go index 663fa0e83..3d12ac9a8 100644 --- a/chain/actors/builtin/market/market.go +++ b/chain/actors/builtin/market/market.go @@ -1,15 +1,20 @@ package market import ( + "golang.org/x/xerrors" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/cbor" - "github.com/ipfs/go-cid" + v0builtin "github.com/filecoin-project/specs-actors/actors/builtin" + + "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/chain/types" ) func Load(store adt.Store, act *types.Actor) (st State, err error) { switch act.Code { - case v0builtin.MarketActorCodeID: + case v0builtin.StorageMarketActorCodeID: out := v0State{store: store} err := store.Get(store.Context(), act.Head, &out) if err != nil { diff --git a/chain/actors/builtin/market/v0.go b/chain/actors/builtin/market/v0.go index 23f33e3d3..da86cda0f 100644 --- a/chain/actors/builtin/market/v0.go +++ b/chain/actors/builtin/market/v0.go @@ -1,6 +1,8 @@ package market import ( + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/specs-actors/actors/builtin/market" "github.com/filecoin-project/specs-actors/actors/util/adt" ) @@ -20,6 +22,6 @@ func (s *v0State) EscrowTable() (BalanceTable, error) { return adt.AsBalanceTable(s.store, s.State.EscrowTable) } -func (s *v0State) Lockedtable() (BalanceTable, error) { +func (s *v0State) LockedTable() (BalanceTable, error) { return adt.AsBalanceTable(s.store, s.State.LockedTable) } diff --git a/chain/actors/builtin/miner/miner.go b/chain/actors/builtin/miner/miner.go index 50453827d..11dc3158a 100644 --- a/chain/actors/builtin/miner/miner.go +++ b/chain/actors/builtin/miner/miner.go @@ -1,10 +1,13 @@ package miner import ( - "github.com/filecoin-project/go-bitfield" - "github.com/filecoin-project/go-state-types/abi" + "github.com/libp2p/go-libp2p-core/peer" "golang.org/x/xerrors" + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-bitfield" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/cbor" v0builtin "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/adt" diff --git a/chain/actors/builtin/miner/v0.go b/chain/actors/builtin/miner/v0.go index 8898a26a5..1d8a68183 100644 --- a/chain/actors/builtin/miner/v0.go +++ b/chain/actors/builtin/miner/v0.go @@ -1,6 +1,7 @@ package miner import ( + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-bitfield" "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/libp2p/go-libp2p-core/peer" @@ -51,6 +52,9 @@ func (s *v0State) NumDeadlines() (uint64, error) { func (s *v0State) Info() (MinerInfo, error) { info, err := s.State.GetInfo(s.store) + if err != nil { + return MinerInfo{}, err + } var pid *peer.ID if peerID, err := peer.IDFromBytes(info.PeerId); err == nil { @@ -77,7 +81,7 @@ func (s *v0State) Info() (MinerInfo, error) { mi.WorkerChangeEpoch = info.PendingWorkerKey.EffectiveAt } - return mi + return mi, nil } func (d *v0Deadline) LoadPartition(idx uint64) (Partition, error) { diff --git a/chain/actors/builtin/power/power.go b/chain/actors/builtin/power/power.go index 7671877e3..b7bb9329e 100644 --- a/chain/actors/builtin/power/power.go +++ b/chain/actors/builtin/power/power.go @@ -1,16 +1,19 @@ package power import ( + "golang.org/x/xerrors" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/cbor" + v0builtin "github.com/filecoin-project/specs-actors/actors/builtin" + "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/types" - "golang.org/x/xerrors" ) func Load(store adt.Store, act *types.Actor) (st State, err error) { switch act.Code { - case v0builtin.PowerActorCodeID: + case v0builtin.StoragePowerActorCodeID: out := v0State{store: store} err := store.Get(store.Context(), act.Head, &out) if err != nil { diff --git a/chain/state/statetree.go b/chain/state/statetree.go index 684e6ce5d..7aa7701fd 100644 --- a/chain/state/statetree.go +++ b/chain/state/statetree.go @@ -4,12 +4,6 @@ import ( "context" "fmt" - "github.com/filecoin-project/go-state-types/network" - "github.com/filecoin-project/lotus/chain/actors/builtin/init" - init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init" - "github.com/filecoin-project/specs-actors/actors/builtin" - - "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" logging "github.com/ipfs/go-log/v2" @@ -17,6 +11,12 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/network" + init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init" + "github.com/filecoin-project/specs-actors/actors/builtin" + + "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/types" ) @@ -118,9 +118,13 @@ func (ss *stateSnaps) deleteActor(addr address.Address) { } func NewStateTree(cst cbor.IpldStore, version network.Version) (*StateTree, error) { + root, err := adt.NewMap(adt.WrapStore(context.TODO(), cst), version) + if err != nil { + return nil, err + } return &StateTree{ - root: adt.NewMap(adt.WrapStore(context.TODO(), cst), version), + root: root, Store: cst, snaps: newStateSnaps(), }, nil @@ -168,7 +172,7 @@ func (st *StateTree) LookupID(addr address.Address) (address.Address, error) { return address.Undef, xerrors.Errorf("getting init actor: %w", err) } - ias, err := init.Load(&AdtStore{st.Store}, &act) + ias, err := init_.Load(&AdtStore{st.Store}, act) if err != nil { return address.Undef, xerrors.Errorf("loading init actor state: %w", err) } @@ -212,7 +216,7 @@ func (st *StateTree) GetActor(addr address.Address) (*types.Actor, error) { } var act types.Actor - if found, err := st.root.Get(adt.AddrKey(addr), &act); err != nil { + if found, err := st.root.Get(abi.AddrKey(addr), &act); err != nil { return nil, xerrors.Errorf("hamt find failed: %w", err) } else if !found { return nil, types.ErrActorNotFound @@ -257,11 +261,11 @@ func (st *StateTree) Flush(ctx context.Context) (cid.Cid, error) { for addr, sto := range st.snaps.layers[0].actors { if sto.Delete { - if err := st.root.Delete(adt.AddrKey(addr)); err != nil { + if err := st.root.Delete(abi.AddrKey(addr)); err != nil { return cid.Undef, err } } else { - if err := st.root.Put(adt.AddrKey(addr), &sto.Act); err != nil { + if err := st.root.Put(abi.AddrKey(addr), &sto.Act); err != nil { return cid.Undef, err } } @@ -286,7 +290,7 @@ func (st *StateTree) ClearSnapshot() { func (st *StateTree) RegisterNewAddress(addr address.Address) (address.Address, error) { var out address.Address err := st.MutateActor(builtin.InitActorAddr, func(initact *types.Actor) error { - ias, err := init.Load(&AdtStore{st.Store}, initact) + ias, err := init_.Load(&AdtStore{st.Store}, initact) if err != nil { return err } diff --git a/go.sum b/go.sum index 106814b02..0ae61535a 100644 --- a/go.sum +++ b/go.sum @@ -249,11 +249,6 @@ github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIi github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b h1:fkRZSPrYpk42PV3/lIXiL0LHetxde7vyYYvSsttQtfg= github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/go.mod h1:Q0GQOBtKf1oE10eSXSlhN45kDBdGvEcVOqMiffqX+N8= -github.com/filecoin-project/specs-actors v0.9.4/go.mod h1:BStZQzx5x7TmCkLv0Bpa07U6cPKol6fd3w9KjMPZ6Z4= -github.com/filecoin-project/specs-actors v0.9.7 h1:7PAZ8kdqwBdmgf/23FCkQZLCXcVu02XJrkpkhBikiA8= -github.com/filecoin-project/specs-actors v0.9.7/go.mod h1:wM2z+kwqYgXn5Z7scV1YHLyd1Q1cy0R8HfTIWQ0BFGU= -github.com/filecoin-project/specs-actors v0.9.9-0.20200911231631-727cd8845d30 h1:6Kn6y3TpJbk5BsvhVha+3jr7C3gAAJq0rCnwUYOWRl0= -github.com/filecoin-project/specs-actors v0.9.9-0.20200911231631-727cd8845d30/go.mod h1:czlvLQGEX0fjLLfdNHD7xLymy6L3n7aQzRWzsYGf+ys= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= github.com/filecoin-project/test-vectors/schema v0.0.1 h1:5fNF76nl4qolEvcIsjc0kUADlTMVHO73tW4kXXPnsus= @@ -503,8 +498,6 @@ github.com/ipfs/go-fs-lock v0.0.6/go.mod h1:OTR+Rj9sHiRubJh3dRhD15Juhd/+w6VPOY28 github.com/ipfs/go-graphsync v0.1.0/go.mod h1:jMXfqIEDFukLPZHqDPp8tJMbHO9Rmeb9CEGevngQbmE= github.com/ipfs/go-graphsync v0.1.2 h1:25Ll9kIXCE+DY0dicvfS3KMw+U5sd01b/FJbA7KAbhg= github.com/ipfs/go-graphsync v0.1.2/go.mod h1:sLXVXm1OxtE2XYPw62MuXCdAuNwkAdsbnfrmos5odbA= -github.com/ipfs/go-hamt-ipld v0.1.1 h1:0IQdvwnAAUKmDE+PMJa5y1QiwOPHpI9+eAbQEEEYthk= -github.com/ipfs/go-hamt-ipld v0.1.1/go.mod h1:1EZCr2v0jlCnhpa+aZ0JZYp8Tt2w16+JJOAVz17YcDk= github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw= github.com/ipfs/go-ipfs-blockstore v0.1.4/go.mod h1:Jxm3XMVjh6R17WvxFEiyKBLUGr86HgIYJW/D/MwqeYQ= From 68097132fe64dd9454aae00ae86bf6f4b2b36f96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 14 Sep 2020 13:45:20 +0200 Subject: [PATCH 084/885] Fix vm build --- chain/gen/genesis/genesis.go | 7 +- chain/gen/genesis/miners.go | 4 +- chain/stmgr/stmgr.go | 39 +++++----- chain/vm/invoker.go | 8 +-- chain/vm/runtime.go | 135 ++++++++++++++++------------------- chain/vm/vm.go | 28 ++++---- 6 files changed, 99 insertions(+), 122 deletions(-) diff --git a/chain/gen/genesis/genesis.go b/chain/gen/genesis/genesis.go index ac22b5b19..3c4632437 100644 --- a/chain/gen/genesis/genesis.go +++ b/chain/gen/genesis/genesis.go @@ -6,8 +6,6 @@ import ( "encoding/json" "fmt" - "github.com/filecoin-project/specs-actors/actors/runtime" - "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" cbor "github.com/ipfs/go-ipld-cbor" @@ -19,6 +17,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" + "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/builtin/account" "github.com/filecoin-project/specs-actors/actors/builtin/multisig" @@ -406,8 +405,8 @@ func VerifyPreSealedData(ctx context.Context, cs *store.ChainStore, stateroot ci verifNeeds := make(map[address.Address]abi.PaddedPieceSize) var sum abi.PaddedPieceSize - nwv := func(context.Context, abi.ChainEpoch) runtime.NetworkVersion { - return runtime.NetworkVersion1 + nwv := func(context.Context, abi.ChainEpoch) network.Version { + return network.Version1 } vmopt := vm.VMOpts{ diff --git a/chain/gen/genesis/miners.go b/chain/gen/genesis/miners.go index d8441c66c..6d041bb4d 100644 --- a/chain/gen/genesis/miners.go +++ b/chain/gen/genesis/miners.go @@ -61,8 +61,8 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid return big.Zero(), nil } - nwv := func(context.Context, abi.ChainEpoch) runtime.NetworkVersion { - return runtime.NetworkVersion1 + nwv := func(context.Context, abi.ChainEpoch) network.Version { + return network.Version1 } vmopt := &vm.VMOpts{ diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index fa4b08147..f5b043dc5 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -5,11 +5,22 @@ import ( "fmt" "sync" - "github.com/filecoin-project/specs-actors/actors/runtime" - - "github.com/filecoin-project/specs-actors/actors/builtin/multisig" + "github.com/ipfs/go-cid" + cbor "github.com/ipfs/go-ipld-cbor" + logging "github.com/ipfs/go-log/v2" + cbg "github.com/whyrusleeping/cbor-gen" + "go.opencensus.io/trace" + "golang.org/x/xerrors" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/go-state-types/network" + "github.com/filecoin-project/specs-actors/actors/builtin" + "github.com/filecoin-project/specs-actors/actors/builtin/multisig" + "github.com/filecoin-project/specs-actors/actors/builtin/reward" + "github.com/filecoin-project/specs-actors/actors/util/adt" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" @@ -19,20 +30,6 @@ import ( "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" - - "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/big" - "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/builtin/reward" - "github.com/filecoin-project/specs-actors/actors/util/adt" - - "golang.org/x/xerrors" - - "github.com/ipfs/go-cid" - cbor "github.com/ipfs/go-ipld-cbor" - logging "github.com/ipfs/go-log/v2" - cbg "github.com/whyrusleeping/cbor-gen" - "go.opencensus.io/trace" ) var log = logging.Logger("statemgr") @@ -1126,14 +1123,14 @@ func (sm *StateManager) GetCirculatingSupply(ctx context.Context, height abi.Cha return csi.FilCirculating, nil } -func (sm *StateManager) GetNtwkVersion(ctx context.Context, height abi.ChainEpoch) runtime.NetworkVersion { +func (sm *StateManager) GetNtwkVersion(ctx context.Context, height abi.ChainEpoch) network.Version { if build.UpgradeBreezeHeight == 0 { - return runtime.NetworkVersion1 + return network.Version1 } if height <= build.UpgradeBreezeHeight { - return runtime.NetworkVersion0 + return network.Version0 } - return runtime.NetworkVersion1 + return network.Version1 } diff --git a/chain/vm/invoker.go b/chain/vm/invoker.go index 8583b0438..9f5f8e2d9 100644 --- a/chain/vm/invoker.go +++ b/chain/vm/invoker.go @@ -15,6 +15,7 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/actors/aerrors" "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/builtin/cron" init_ "github.com/filecoin-project/specs-actors/actors/builtin/init" @@ -27,9 +28,6 @@ import ( "github.com/filecoin-project/specs-actors/actors/builtin/system" "github.com/filecoin-project/specs-actors/actors/runtime" vmr "github.com/filecoin-project/specs-actors/actors/runtime" - "github.com/filecoin-project/specs-actors/actors/util/adt" - - "github.com/filecoin-project/lotus/chain/actors/aerrors" ) type Invoker struct { @@ -48,7 +46,7 @@ func NewInvoker() *Invoker { // add builtInCode using: register(cid, singleton) // NETUPGRADE: register code IDs for v2, etc. - inv.Register(builtin.SystemActorCodeID, system.Actor{}, adt.EmptyValue{}) + inv.Register(builtin.SystemActorCodeID, system.Actor{}, abi.EmptyValue{}) inv.Register(builtin.InitActorCodeID, init_.Actor{}, init_.State{}) inv.Register(builtin.RewardActorCodeID, reward.Actor{}, reward.State{}) inv.Register(builtin.CronActorCodeID, cron.Actor{}, cron.State{}) @@ -67,7 +65,7 @@ func (inv *Invoker) Invoke(codeCid cid.Cid, rt runtime.Runtime, method abi.Metho code, ok := inv.builtInCode[codeCid] if !ok { - log.Errorf("no code for actor %s (Addr: %s)", codeCid, rt.Message().Receiver()) + log.Errorf("no code for actor %s (Addr: %s)", codeCid, rt.Receiver()) return nil, aerrors.Newf(exitcode.SysErrorIllegalActor, "no code for actor %s(%d)(%s)", codeCid, method, hex.EncodeToString(params)) } if method >= abi.MethodNum(len(code)) || code[method] == nil { diff --git a/chain/vm/runtime.go b/chain/vm/runtime.go index 043ea3a45..87d91e230 100644 --- a/chain/vm/runtime.go +++ b/chain/vm/runtime.go @@ -8,21 +8,24 @@ import ( gruntime "runtime" "time" - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/big" - "github.com/filecoin-project/go-state-types/crypto" - "github.com/filecoin-project/go-state-types/exitcode" - "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/runtime" - vmr "github.com/filecoin-project/specs-actors/actors/runtime" - "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" cbg "github.com/whyrusleeping/cbor-gen" "go.opencensus.io/trace" "golang.org/x/xerrors" + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" + statecbor "github.com/filecoin-project/go-state-types/cbor" + "github.com/filecoin-project/go-state-types/crypto" + "github.com/filecoin-project/go-state-types/exitcode" + "github.com/filecoin-project/go-state-types/network" + rtt "github.com/filecoin-project/go-state-types/rt" + "github.com/filecoin-project/specs-actors/actors/builtin" + "github.com/filecoin-project/specs-actors/actors/runtime" + vmr "github.com/filecoin-project/specs-actors/actors/runtime" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors/aerrors" "github.com/filecoin-project/lotus/chain/state" @@ -34,15 +37,15 @@ type Runtime struct { vm *VM state *state.StateTree - vmsg vmr.Message height abi.ChainEpoch cst cbor.IpldStore pricelist Pricelist + vmr.Message gasAvailable int64 gasUsed int64 - sys runtime.Syscalls + runtime.Syscalls // address that started invoke chain origin address.Address @@ -56,7 +59,7 @@ type Runtime struct { lastGasCharge *types.GasTrace } -func (rt *Runtime) NetworkVersion() vmr.NetworkVersion { +func (rt *Runtime) NetworkVersion() network.Version { return rt.vm.GetNtwkVersion(rt.ctx, rt.CurrEpoch()) } @@ -84,7 +87,7 @@ type notFoundErr interface { IsNotFound() bool } -func (rt *Runtime) Get(c cid.Cid, o vmr.CBORUnmarshaler) bool { +func (rt *Runtime) StoreGet(c cid.Cid, o statecbor.Unmarshaler) bool { if err := rt.cst.Get(context.TODO(), c, o); err != nil { var nfe notFoundErr if xerrors.As(err, &nfe) && nfe.IsNotFound() { @@ -99,7 +102,7 @@ func (rt *Runtime) Get(c cid.Cid, o vmr.CBORUnmarshaler) bool { return true } -func (rt *Runtime) Put(x vmr.CBORMarshaler) cid.Cid { +func (rt *Runtime) StorePut(x statecbor.Marshaler) cid.Cid { c, err := rt.cst.Put(context.TODO(), x) if err != nil { if xerrors.As(err, new(cbor.SerializationError)) { @@ -136,7 +139,7 @@ func (rt *Runtime) shimCall(f func() interface{}) (rval []byte, aerr aerrors.Act switch ret := ret.(type) { case []byte: return ret, nil - case *adt.EmptyValue: + case *abi.EmptyValue: return nil, nil case cbg.CBORMarshaler: buf := new(bytes.Buffer) @@ -151,17 +154,13 @@ func (rt *Runtime) shimCall(f func() interface{}) (rval []byte, aerr aerrors.Act } } -func (rt *Runtime) Message() vmr.Message { - return rt.vmsg -} - func (rt *Runtime) ValidateImmediateCallerAcceptAny() { rt.abortIfAlreadyValidated() return } func (rt *Runtime) CurrentBalance() abi.TokenAmount { - b, err := rt.GetBalance(rt.Message().Receiver()) + b, err := rt.GetBalance(rt.Receiver()) if err != nil { rt.Abortf(err.RetCode(), "get current balance: %v", err) } @@ -257,7 +256,7 @@ func (rt *Runtime) CreateActor(codeID cid.Cid, address address.Address) { // May only be called by the actor itself. func (rt *Runtime) DeleteActor(beneficiary address.Address) { rt.chargeGas(rt.Pricelist().OnDeleteActor()) - act, err := rt.state.GetActor(rt.Message().Receiver()) + act, err := rt.state.GetActor(rt.Receiver()) if err != nil { if xerrors.Is(err, types.ErrActorNotFound) { rt.Abortf(exitcode.SysErrorIllegalActor, "failed to load actor in delete actor: %s", err) @@ -266,36 +265,32 @@ func (rt *Runtime) DeleteActor(beneficiary address.Address) { } if !act.Balance.IsZero() { // Transfer the executing actor's balance to the beneficiary - if err := rt.vm.transfer(rt.Message().Receiver(), beneficiary, act.Balance); err != nil { + if err := rt.vm.transfer(rt.Receiver(), beneficiary, act.Balance); err != nil { panic(aerrors.Fatalf("failed to transfer balance to beneficiary actor: %s", err)) } } // Delete the executing actor - if err := rt.state.DeleteActor(rt.Message().Receiver()); err != nil { + if err := rt.state.DeleteActor(rt.Receiver()); err != nil { panic(aerrors.Fatalf("failed to delete actor: %s", err)) } _ = rt.chargeGasSafe(gasOnActorExec) } -func (rt *Runtime) Syscalls() vmr.Syscalls { - return rt.sys -} - -func (rt *Runtime) StartSpan(name string) vmr.TraceSpan { +func (rt *Runtime) StartSpan(name string) func() { panic("implement me") } func (rt *Runtime) ValidateImmediateCallerIs(as ...address.Address) { rt.abortIfAlreadyValidated() - imm := rt.Message().Caller() + imm := rt.Caller() for _, a := range as { if imm == a { return } } - rt.Abortf(exitcode.SysErrForbidden, "caller %s is not one of %s", rt.Message().Caller(), as) + rt.Abortf(exitcode.SysErrForbidden, "caller %s is not one of %s", rt.Caller(), as) } func (rt *Runtime) Context() context.Context { @@ -313,7 +308,7 @@ func (rt *Runtime) AbortStateMsg(msg string) { func (rt *Runtime) ValidateImmediateCallerType(ts ...cid.Cid) { rt.abortIfAlreadyValidated() - callerCid, ok := rt.GetActorCodeCID(rt.Message().Caller()) + callerCid, ok := rt.GetActorCodeCID(rt.Caller()) if !ok { panic(aerrors.Fatalf("failed to lookup code cid for caller")) } @@ -329,15 +324,7 @@ func (rt *Runtime) CurrEpoch() abi.ChainEpoch { return rt.height } -type dumbWrapperType struct { - val []byte -} - -func (dwt *dumbWrapperType) Into(um vmr.CBORUnmarshaler) error { - return um.UnmarshalCBOR(bytes.NewReader(dwt.val)) -} - -func (rt *Runtime) Send(to address.Address, method abi.MethodNum, m vmr.CBORMarshaler, value abi.TokenAmount) (vmr.SendReturn, exitcode.ExitCode) { +func (rt *Runtime) Send(to address.Address, method abi.MethodNum, m statecbor.Marshaler, value abi.TokenAmount, out statecbor.Er) exitcode.ExitCode { if !rt.allowInternal { rt.Abortf(exitcode.SysErrorIllegalActor, "runtime.Send() is currently disallowed") } @@ -350,16 +337,22 @@ func (rt *Runtime) Send(to address.Address, method abi.MethodNum, m vmr.CBORMars params = buf.Bytes() } - ret, err := rt.internalSend(rt.Message().Receiver(), to, method, value, params) + ret, err := rt.internalSend(rt.Receiver(), to, method, value, params) if err != nil { if err.IsFatal() { panic(err) } log.Warnf("vmctx send failed: to: %s, method: %d: ret: %d, err: %s", to, method, ret, err) - return &dumbWrapperType{nil}, err.RetCode() + return err.RetCode() } _ = rt.chargeGasSafe(gasOnActorExec) - return &dumbWrapperType{ret}, 0 + + if err := out.UnmarshalCBOR(bytes.NewReader(ret)); err != nil { + // REVIEW: always fatal? + panic(err) + } + + return 0 } func (rt *Runtime) internalSend(from, to address.Address, method abi.MethodNum, value types.BigInt, params []byte) ([]byte, aerrors.ActorError) { @@ -398,54 +391,46 @@ func (rt *Runtime) internalSend(from, to address.Address, method abi.MethodNum, if subrt != nil { rt.numActorsCreated = subrt.numActorsCreated + rt.executionTrace.Subcalls = append(rt.executionTrace.Subcalls, subrt.executionTrace) } - rt.executionTrace.Subcalls = append(rt.executionTrace.Subcalls, subrt.executionTrace) return ret, errSend } -func (rt *Runtime) State() vmr.StateHandle { - return &shimStateHandle{rt: rt} -} - -type shimStateHandle struct { - rt *Runtime -} - -func (ssh *shimStateHandle) Create(obj vmr.CBORMarshaler) { - c := ssh.rt.Put(obj) - err := ssh.rt.stateCommit(EmptyObjectCid, c) +func (rt *Runtime) StateCreate(obj statecbor.Marshaler) { + c := rt.StorePut(obj) + err := rt.stateCommit(EmptyObjectCid, c) if err != nil { panic(fmt.Errorf("failed to commit state after creating object: %w", err)) } } -func (ssh *shimStateHandle) Readonly(obj vmr.CBORUnmarshaler) { - act, err := ssh.rt.state.GetActor(ssh.rt.Message().Receiver()) +func (rt *Runtime) StateReadonly(obj statecbor.Unmarshaler) { + act, err := rt.state.GetActor(rt.Receiver()) if err != nil { - ssh.rt.Abortf(exitcode.SysErrorIllegalArgument, "failed to get actor for Readonly state: %s", err) + rt.Abortf(exitcode.SysErrorIllegalArgument, "failed to get actor for Readonly state: %s", err) } - ssh.rt.Get(act.Head, obj) + rt.StoreGet(act.Head, obj) } -func (ssh *shimStateHandle) Transaction(obj vmr.CBORer, f func()) { +func (rt *Runtime) StateTransaction(obj statecbor.Er, f func()) { if obj == nil { - ssh.rt.Abortf(exitcode.SysErrorIllegalActor, "Must not pass nil to Transaction()") + rt.Abortf(exitcode.SysErrorIllegalActor, "Must not pass nil to Transaction()") } - act, err := ssh.rt.state.GetActor(ssh.rt.Message().Receiver()) + act, err := rt.state.GetActor(rt.Receiver()) if err != nil { - ssh.rt.Abortf(exitcode.SysErrorIllegalActor, "failed to get actor for Transaction: %s", err) + rt.Abortf(exitcode.SysErrorIllegalActor, "failed to get actor for Transaction: %s", err) } baseState := act.Head - ssh.rt.Get(baseState, obj) + rt.StoreGet(baseState, obj) - ssh.rt.allowInternal = false + rt.allowInternal = false f() - ssh.rt.allowInternal = true + rt.allowInternal = true - c := ssh.rt.Put(obj) + c := rt.StorePut(obj) - err = ssh.rt.stateCommit(baseState, c) + err = rt.stateCommit(baseState, c) if err != nil { panic(fmt.Errorf("failed to commit state after transaction: %w", err)) } @@ -465,7 +450,7 @@ func (rt *Runtime) GetBalance(a address.Address) (types.BigInt, aerrors.ActorErr func (rt *Runtime) stateCommit(oldh, newh cid.Cid) aerrors.ActorError { // TODO: we can make this more efficient in the future... - act, err := rt.state.GetActor(rt.Message().Receiver()) + act, err := rt.state.GetActor(rt.Receiver()) if err != nil { return aerrors.Escalate(err, "failed to get actor to commit state") } @@ -476,7 +461,7 @@ func (rt *Runtime) stateCommit(oldh, newh cid.Cid) aerrors.ActorError { act.Head = newh - if err := rt.state.SetActor(rt.Message().Receiver(), act); err != nil { + if err := rt.state.SetActor(rt.Receiver(), act); err != nil { return aerrors.Fatalf("failed to set actor in commit state: %s", err) } @@ -571,15 +556,15 @@ func (rt *Runtime) abortIfAlreadyValidated() { rt.callerValidated = true } -func (rt *Runtime) Log(level vmr.LogLevel, msg string, args ...interface{}) { +func (rt *Runtime) Log(level rtt.LogLevel, msg string, args ...interface{}) { switch level { - case vmr.DEBUG: + case rtt.DEBUG: actorLog.Debugf(msg, args...) - case vmr.INFO: + case rtt.INFO: actorLog.Infof(msg, args...) - case vmr.WARN: + case rtt.WARN: actorLog.Warnf(msg, args...) - case vmr.ERROR: + case rtt.ERROR: actorLog.Errorf(msg, args...) } } diff --git a/chain/vm/vm.go b/chain/vm/vm.go index eb6c2f354..a125df053 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -7,13 +7,6 @@ import ( "reflect" "time" - "github.com/filecoin-project/specs-actors/actors/runtime" - - bstore "github.com/filecoin-project/lotus/lib/blockstore" - - "github.com/filecoin-project/go-state-types/big" - "github.com/filecoin-project/specs-actors/actors/builtin" - block "github.com/ipfs/go-block-format" cid "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" @@ -25,8 +18,11 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/go-state-types/exitcode" + "github.com/filecoin-project/go-state-types/network" + "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/builtin/account" "github.com/filecoin-project/lotus/build" @@ -34,6 +30,7 @@ import ( "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/blockstore" + bstore "github.com/filecoin-project/lotus/lib/blockstore" "github.com/filecoin-project/lotus/lib/bufbstore" ) @@ -116,7 +113,7 @@ func (vm *VM) makeRuntime(ctx context.Context, msg *types.Message, origin addres Blocks: &gasChargingBlocks{rt.chargeGasFunc(2), rt.pricelist, vm.cst.Blocks}, Atlas: vm.cst.Atlas, } - rt.sys = pricedSyscalls{ + rt.Syscalls = pricedSyscalls{ under: vm.Syscalls(ctx, vm.cstate, rt.cst), chargeGas: rt.chargeGasFunc(1), pl: rt.pricelist, @@ -128,7 +125,7 @@ func (vm *VM) makeRuntime(ctx context.Context, msg *types.Message, origin addres rt.Abortf(exitcode.SysErrInvalidReceiver, "resolve msg.From address failed") } vmm.From = resF - rt.vmsg = &vmm + rt.Message = &vmm return rt } @@ -142,7 +139,7 @@ func (vm *UnsafeVM) MakeRuntime(ctx context.Context, msg *types.Message, origin } type CircSupplyCalculator func(context.Context, abi.ChainEpoch, *state.StateTree) (abi.TokenAmount, error) -type NtwkVersionGetter func(context.Context, abi.ChainEpoch) runtime.NetworkVersion +type NtwkVersionGetter func(context.Context, abi.ChainEpoch) network.Version type VM struct { cstate *state.StateTree @@ -170,10 +167,11 @@ type VMOpts struct { BaseFee abi.TokenAmount } -func NewVM(opts *VMOpts) (*VM, error) { +func NewVM(ctx context.Context, opts *VMOpts) (*VM, error) { buf := bufbstore.NewBufferedBstore(opts.Bstore) cst := cbor.NewCborStore(buf) - state, err := state.LoadStateTree(cst, opts.StateBase) + nwv := opts.NtwkVersion(ctx, opts.Epoch) // TODO: why do we need a context for this? + state, err := state.LoadStateTree(cst, opts.StateBase, nwv) if err != nil { return nil, err } @@ -700,9 +698,9 @@ func (vm *VM) Invoke(act *types.Actor, rt *Runtime, method abi.MethodNum, params defer span.End() if span.IsRecordingEvents() { span.AddAttributes( - trace.StringAttribute("to", rt.Message().Receiver().String()), + trace.StringAttribute("to", rt.Receiver().String()), trace.Int64Attribute("method", int64(method)), - trace.StringAttribute("value", rt.Message().ValueReceived().String()), + trace.StringAttribute("value", rt.ValueReceived().String()), ) } @@ -722,7 +720,7 @@ func (vm *VM) SetInvoker(i *Invoker) { vm.inv = i } -func (vm *VM) GetNtwkVersion(ctx context.Context, ce abi.ChainEpoch) runtime.NetworkVersion { +func (vm *VM) GetNtwkVersion(ctx context.Context, ce abi.ChainEpoch) network.Version { return vm.ntwkVersion(ctx, ce) } From 05aa5f2d385c85ff8cdf9ffd8ce20149d446ea85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Mon, 14 Sep 2020 13:01:03 +0100 Subject: [PATCH 085/885] allow customizing disabled journal events + tests. --- journal/registry_test.go | 49 ++++++++++++++++++++++++++++++++++++++++ journal/types.go | 21 +++++++++++++++++ node/builder.go | 15 +++++++++++- 3 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 journal/registry_test.go diff --git a/journal/registry_test.go b/journal/registry_test.go new file mode 100644 index 000000000..9d2085c12 --- /dev/null +++ b/journal/registry_test.go @@ -0,0 +1,49 @@ +package journal + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestDisabledEvents(t *testing.T) { + req := require.New(t) + + test := func(dis DisabledEvents) func(*testing.T) { + return func(t *testing.T) { + registry := NewEventTypeRegistry(dis) + + reg1 := registry.RegisterEventType("system1", "disabled1") + reg2 := registry.RegisterEventType("system2", "disabled2") + + req.False(reg1.Enabled()) + req.False(reg2.Enabled()) + req.True(reg1.safe) + req.True(reg2.safe) + + reg3 := registry.RegisterEventType("system3", "enabled3") + req.True(reg3.Enabled()) + req.True(reg3.safe) + } + } + + t.Run("direct", test(DisabledEvents{ + EventType{System: "system1", Event: "disabled1"}, + EventType{System: "system2", Event: "disabled2"}, + })) + + dis, err := ParseDisabledEvents("system1:disabled1,system2:disabled2") + req.NoError(err) + + t.Run("parsed", test(dis)) + + dis, err = ParseDisabledEvents(" system1:disabled1 , system2:disabled2 ") + req.NoError(err) + + t.Run("parsed_spaces", test(dis)) +} + +func TestParseDisableEvents(t *testing.T) { + _, err := ParseDisabledEvents("system1:disabled1:failed,system2:disabled2") + require.Error(t, err) +} diff --git a/journal/types.go b/journal/types.go index c81fb8c89..56c7d4bc0 100644 --- a/journal/types.go +++ b/journal/types.go @@ -1,6 +1,8 @@ package journal import ( + "fmt" + "strings" "time" logging "github.com/ipfs/go-log" @@ -20,6 +22,25 @@ var ( // DisabledEvents is the set of event types whose journaling is suppressed. type DisabledEvents []EventType +// ParseDisabledEvents parses a string of the form: "system1:event1,system2:event2[,...]" +// into a DisabledEvents object, returning an error if the string failed to parse. +// +// It sanitizes strings via strings.TrimSpace. +func ParseDisabledEvents(s string) (DisabledEvents, error) { + s = strings.TrimSpace(s) // sanitize + evts := strings.Split(s, ",") + ret := make(DisabledEvents, 0, len(evts)) + for _, evt := range evts { + evt = strings.TrimSpace(evt) // sanitize + s := strings.Split(evt, ":") + if len(s) != 2 { + return nil, fmt.Errorf("invalid event type: %s", s) + } + ret = append(ret, EventType{System: s[0], Event: s[1]}) + } + return ret, nil +} + // EventType represents the signature of an event. type EventType struct { System string diff --git a/node/builder.go b/node/builder.go index 08c48d408..028c9212e 100644 --- a/node/builder.go +++ b/node/builder.go @@ -3,6 +3,7 @@ package node import ( "context" "errors" + "os" "time" logging "github.com/ipfs/go-log" @@ -68,6 +69,10 @@ import ( "github.com/filecoin-project/lotus/storage/sectorblocks" ) +// EnvJournalDisabledEvents is the environment variable through which disabled +// journal events can be customized. +const EnvJournalDisabledEvents = "LOTUS_JOURNAL_DISABLED_EVENTS" + //nolint:deadcode,varcheck var log = logging.Logger("builder") @@ -157,7 +162,15 @@ type Settings struct { func defaults() []Option { return []Option{ // global system journal. - Override(new(journal.DisabledEvents), journal.DefaultDisabledEvents), + Override(new(journal.DisabledEvents), func() journal.DisabledEvents { + if env, ok := os.LookupEnv(EnvJournalDisabledEvents); ok { + if ret, err := journal.ParseDisabledEvents(env); err == nil { + return ret + } + } + // fallback if env variable is not set, or if it failed to parse. + return journal.DefaultDisabledEvents + }), Override(new(journal.Journal), modules.OpenFilesystemJournal), Override(InitJournalKey, func(j journal.Journal) { journal.J = j // eagerly sets the global journal through fx.Invoke. From 86607452d7bbe139520aebcb1b9abb81f1a8aac6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Mon, 14 Sep 2020 13:18:53 +0100 Subject: [PATCH 086/885] circleci: make lint expend all cores. (#3831) * circleci: make lint expend all cores. * i hate yaml. * now i hate it even more. --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d8c7abf16..4fdb3e38f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -400,7 +400,8 @@ workflows: version: 2.1 ci: jobs: - - lint-all + - lint-all: + concurrency: "16" # expend all docker 2xlarge CPUs. - mod-tidy-check - gofmt - cbor-gen-check From 683a58195ec966757e1360a5fe0a2d973791d053 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 14 Sep 2020 14:17:45 +0200 Subject: [PATCH 087/885] More terraforming in chain/ --- chain/actors/builtin/power/power.go | 2 ++ chain/actors/builtin/power/v0.go | 5 ++++ chain/gen/genesis/genesis.go | 11 +++------ chain/gen/genesis/miners.go | 8 ++----- chain/gen/genesis/util.go | 6 +++++ chain/stmgr/call.go | 4 ++-- chain/stmgr/forks.go | 28 ++++++++++------------ chain/stmgr/read.go | 6 +---- chain/stmgr/stmgr.go | 37 +++++++++++++++++------------ chain/stmgr/utils.go | 11 ++++++--- chain/store/store.go | 22 ++++++++++++++++- chain/store/weight.go | 2 +- chain/vm/vm.go | 2 +- 13 files changed, 86 insertions(+), 58 deletions(-) diff --git a/chain/actors/builtin/power/power.go b/chain/actors/builtin/power/power.go index b7bb9329e..0fcbaf971 100644 --- a/chain/actors/builtin/power/power.go +++ b/chain/actors/builtin/power/power.go @@ -1,6 +1,7 @@ package power import ( + "github.com/filecoin-project/go-address" "golang.org/x/xerrors" "github.com/filecoin-project/go-state-types/abi" @@ -28,4 +29,5 @@ type State interface { cbor.Marshaler TotalLocked() (abi.TokenAmount, error) + MinerNominalPowerMeetsConsensusMinimum(adt.Store, address.Address) (bool, error) } diff --git a/chain/actors/builtin/power/v0.go b/chain/actors/builtin/power/v0.go index 8851080e6..879e3e446 100644 --- a/chain/actors/builtin/power/v0.go +++ b/chain/actors/builtin/power/v0.go @@ -1,6 +1,7 @@ package power import ( + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/specs-actors/actors/builtin/power" @@ -14,3 +15,7 @@ type v0State struct { func (s *v0State) TotalLocked() (abi.TokenAmount, error) { return s.TotalPledgeCollateral, nil } + +func (s *v0State) MinerNominalPowerMeetsConsensusMinimum(st adt.Store, a address.Address) (bool, error) { + return s.State.MinerNominalPowerMeetsConsensusMinimum(st, a) +} diff --git a/chain/gen/genesis/genesis.go b/chain/gen/genesis/genesis.go index 3c4632437..d726dfa50 100644 --- a/chain/gen/genesis/genesis.go +++ b/chain/gen/genesis/genesis.go @@ -17,7 +17,6 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" - "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/builtin/account" "github.com/filecoin-project/specs-actors/actors/builtin/multisig" @@ -115,7 +114,7 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge return nil, nil, xerrors.Errorf("putting empty object: %w", err) } - state, err := state.NewStateTree(cst) + state, err := state.NewStateTree(cst, GenesisNetworkVersion) if err != nil { return nil, nil, xerrors.Errorf("making new state tree: %w", err) } @@ -405,10 +404,6 @@ func VerifyPreSealedData(ctx context.Context, cs *store.ChainStore, stateroot ci verifNeeds := make(map[address.Address]abi.PaddedPieceSize) var sum abi.PaddedPieceSize - nwv := func(context.Context, abi.ChainEpoch) network.Version { - return network.Version1 - } - vmopt := vm.VMOpts{ StateBase: stateroot, Epoch: 0, @@ -416,10 +411,10 @@ func VerifyPreSealedData(ctx context.Context, cs *store.ChainStore, stateroot ci Bstore: cs.Blockstore(), Syscalls: mkFakedSigSyscalls(cs.VMSys()), CircSupplyCalc: nil, - NtwkVersion: nwv, + NtwkVersion: genesisNetworkVersion, BaseFee: types.NewInt(0), } - vm, err := vm.NewVM(&vmopt) + vm, err := vm.NewVM(ctx, &vmopt) if err != nil { return cid.Undef, xerrors.Errorf("failed to create NewVM: %w", err) } diff --git a/chain/gen/genesis/miners.go b/chain/gen/genesis/miners.go index 6d041bb4d..ff4668d7f 100644 --- a/chain/gen/genesis/miners.go +++ b/chain/gen/genesis/miners.go @@ -61,10 +61,6 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid return big.Zero(), nil } - nwv := func(context.Context, abi.ChainEpoch) network.Version { - return network.Version1 - } - vmopt := &vm.VMOpts{ StateBase: sroot, Epoch: 0, @@ -72,11 +68,11 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid Bstore: cs.Blockstore(), Syscalls: mkFakedSigSyscalls(cs.VMSys()), CircSupplyCalc: csc, - NtwkVersion: nwv, + NtwkVersion: genesisNetworkVersion, BaseFee: types.NewInt(0), } - vm, err := vm.NewVM(vmopt) + vm, err := vm.NewVM(ctx, vmopt) if err != nil { return cid.Undef, xerrors.Errorf("failed to create NewVM: %w", err) } diff --git a/chain/gen/genesis/util.go b/chain/gen/genesis/util.go index 67a4e9579..54fc4abfe 100644 --- a/chain/gen/genesis/util.go +++ b/chain/gen/genesis/util.go @@ -2,6 +2,7 @@ package genesis import ( "context" + "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" @@ -46,3 +47,8 @@ func doExecValue(ctx context.Context, vm *vm.VM, to, from address.Address, value return ret.Return, nil } + +var GenesisNetworkVersion = network.Version1 +func genesisNetworkVersion(context.Context, abi.ChainEpoch) network.Version { + return GenesisNetworkVersion // TODO: Get from build/ +} diff --git a/chain/stmgr/call.go b/chain/stmgr/call.go index 4b83842b4..f4dfc7115 100644 --- a/chain/stmgr/call.go +++ b/chain/stmgr/call.go @@ -33,7 +33,7 @@ func (sm *StateManager) CallRaw(ctx context.Context, msg *types.Message, bstate BaseFee: types.NewInt(0), } - vmi, err := vm.NewVM(vmopt) + vmi, err := vm.NewVM(ctx, vmopt) if err != nil { return nil, xerrors.Errorf("failed to set up vm: %w", err) } @@ -134,7 +134,7 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri NtwkVersion: sm.GetNtwkVersion, BaseFee: ts.Blocks()[0].ParentBaseFee, } - vmi, err := vm.NewVM(vmopt) + vmi, err := vm.NewVM(ctx, vmopt) if err != nil { return nil, xerrors.Errorf("failed to set up vm: %w", err) } diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index addaba90f..650536718 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -7,11 +7,10 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" - "github.com/filecoin-project/specs-actors/actors/builtin/power" + v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" + v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" "github.com/filecoin-project/specs-actors/actors/util/adt" cbor "github.com/ipfs/go-ipld-cbor" "golang.org/x/xerrors" @@ -96,11 +95,8 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, tree types return xerrors.Errorf("failed to get tipset at lookback height: %w", err) } - var lbtree *state.StateTree - if err = sm.WithStateTree(lbts.ParentState(), func(state *state.StateTree) error { - lbtree = state - return nil - }); err != nil { + lbtree, err := sm.ParentState(lbts) + if err != nil { return xerrors.Errorf("loading state tree failed: %w", err) } @@ -139,8 +135,8 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, tree types }) } case builtin.StorageMinerActorCodeID: - var st miner.State - if err := sm.WithActorState(ctx, &st)(act); err != nil { + var st v0miner.State + if err := sm.ChainStore().Store(ctx).Get(ctx, act.Head, &st); err != nil { return xerrors.Errorf("failed to load miner state: %w", err) } @@ -176,7 +172,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, tree types } // pull up power table to give miners back some funds proportional to their power - var ps power.State + var ps v0power.State powAct, err := tree.GetActor(builtin.StoragePowerActorAddr) if err != nil { return xerrors.Errorf("failed to load power actor: %w", err) @@ -215,12 +211,12 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, tree types }) } case builtin.StorageMinerActorCodeID: - var st miner.State - if err := sm.WithActorState(ctx, &st)(act); err != nil { + var st v0miner.State + if err := sm.ChainStore().Store(ctx).Get(ctx, act.Head, &st); err != nil { return xerrors.Errorf("failed to load miner state: %w", err) } - var minfo miner.MinerInfo + var minfo v0miner.MinerInfo if err := cst.Get(ctx, st.Info, &minfo); err != nil { return xerrors.Errorf("failed to get miner info: %w", err) } @@ -244,8 +240,8 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, tree types // Now make sure to give each miner who had power at the lookback some FIL lbact, err := lbtree.GetActor(addr) if err == nil { - var lbst miner.State - if err := sm.WithActorState(ctx, &lbst)(lbact); err != nil { + var lbst v0miner.State + if err := sm.ChainStore().Store(ctx).Get(ctx, lbact.Head, &lbst); err != nil { return xerrors.Errorf("failed to load miner state: %w", err) } diff --git a/chain/stmgr/read.go b/chain/stmgr/read.go index 2daa9f79d..7bf757fb8 100644 --- a/chain/stmgr/read.go +++ b/chain/stmgr/read.go @@ -2,8 +2,6 @@ package stmgr import ( "context" - "reflect" - "golang.org/x/xerrors" "github.com/ipfs/go-cid" @@ -13,8 +11,6 @@ import ( "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" - "github.com/filecoin-project/specs-actors/actors/util/adt" ) func (sm *StateManager) ParentStateTsk(tsk types.TipSetKey) (*state.StateTree, error) { @@ -22,7 +18,7 @@ func (sm *StateManager) ParentStateTsk(tsk types.TipSetKey) (*state.StateTree, e if err != nil { return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - return sm.ParentState(ts, cb) + return sm.ParentState(ts) } func (sm *StateManager) ParentState(ts *types.TipSet) (*state.StateTree, error) { diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index f5b043dc5..489eaab3d 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -41,7 +41,7 @@ type StateManager struct { compWait map[string]chan struct{} stlk sync.Mutex genesisMsigLk sync.Mutex - newVM func(*vm.VMOpts) (*vm.VM, error) + newVM func(context.Context, *vm.VMOpts) (*vm.VM, error) genInfo *genesisInfo } @@ -156,7 +156,7 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp BaseFee: baseFee, } - vmi, err := sm.newVM(vmopt) + vmi, err := sm.newVM(ctx, vmopt) if err != nil { return cid.Undef, cid.Undef, xerrors.Errorf("instantiating VM failed: %w", err) } @@ -383,7 +383,7 @@ func (sm *StateManager) ResolveToKeyAddress(ctx context.Context, addr address.Ad } cst := cbor.NewCborStore(sm.cs.Blockstore()) - tree, err := state.LoadStateTree(cst, st) + tree, err := state.LoadStateTree(cst, st, sm.GetNtwkVersion(ctx, ts.Height())) if err != nil { return address.Undef, xerrors.Errorf("failed to load state tree") } @@ -406,7 +406,7 @@ func (sm *StateManager) GetBlsPublicKey(ctx context.Context, addr address.Addres func (sm *StateManager) LookupID(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) { cst := cbor.NewCborStore(sm.cs.Blockstore()) - state, err := state.LoadStateTree(cst, sm.parentState(ts)) + state, err := state.LoadStateTree(cst, sm.parentState(ts), sm.GetNtwkVersion(ctx, ts.Height())) if err != nil { return address.Undef, xerrors.Errorf("load state tree: %w", err) } @@ -598,10 +598,9 @@ func (sm *StateManager) searchBackForMsg(ctx context.Context, from *types.TipSet default: } - var act types.Actor - err := sm.WithParentState(cur, sm.WithActor(m.VMMessage().From, GetActor(&act))) + act, err := sm.LoadActor(ctx, m.VMMessage().From, cur) if err != nil { - return nil, nil, cid.Undef, err + return nil, nil, cid.Cid{}, err } // we either have no messages from the sender, or the latest message we found has a lower nonce than the one being searched for, @@ -712,9 +711,15 @@ func (sm *StateManager) MarketBalance(ctx context.Context, addr address.Address, if err != nil { return api.MarketBalance{}, err } + act, err := st.GetActor(builtin.StorageMarketActorAddr) if err != nil { - return nil, err + return api.MarketBalance{}, err + } + + mstate, err := market.Load(sm.cs.Store(ctx), act) + if err != nil { + return api.MarketBalance{}, err } addr, err = sm.LookupID(ctx, addr, ts) @@ -724,7 +729,7 @@ func (sm *StateManager) MarketBalance(ctx context.Context, addr address.Address, var out api.MarketBalance - et, err := adt.AsBalanceTable(sm.cs.Store(ctx), state.EscrowTable) + et, err := mstate.EscrowTable() if err != nil { return api.MarketBalance{}, err } @@ -733,7 +738,7 @@ func (sm *StateManager) MarketBalance(ctx context.Context, addr address.Address, return api.MarketBalance{}, xerrors.Errorf("getting escrow balance: %w", err) } - lt, err := adt.AsBalanceTable(sm.cs.Store(ctx), state.LockedTable) + lt, err := mstate.LockedTable() if err != nil { return api.MarketBalance{}, err } @@ -774,7 +779,7 @@ func (sm *StateManager) ValidateChain(ctx context.Context, ts *types.TipSet) err return nil } -func (sm *StateManager) SetVMConstructor(nvm func(*vm.VMOpts) (*vm.VM, error)) { +func (sm *StateManager) SetVMConstructor(nvm func(context.Context, *vm.VMOpts) (*vm.VM, error)) { sm.newVM = nvm } @@ -812,7 +817,7 @@ func (sm *StateManager) setupGenesisActors(ctx context.Context) error { } cst := cbor.NewCborStore(sm.cs.Blockstore()) - sTree, err := state.LoadStateTree(cst, st) + sTree, err := state.LoadStateTree(cst, st, sm.GetNtwkVersion(ctx, gts.Height())) if err != nil { return xerrors.Errorf("loading state tree: %w", err) } @@ -918,7 +923,7 @@ func (sm *StateManager) setupGenesisActorsTestnet(ctx context.Context) error { } cst := cbor.NewCborStore(sm.cs.Blockstore()) - sTree, err := state.LoadStateTree(cst, st) + sTree, err := state.LoadStateTree(cst, st, sm.GetNtwkVersion(ctx, gts.Height())) if err != nil { return xerrors.Errorf("loading state tree: %w", err) } @@ -1035,12 +1040,12 @@ func getFilPowerLocked(ctx context.Context, st *state.StateTree) (abi.TokenAmoun return big.Zero(), xerrors.Errorf("failed to load power actor: %w", err) } - pst, err := power.Load(adt.WrapStore(ctx, st.Store), act) + pst, err := power.Load(adt.WrapStore(ctx, st.Store), pactor) if err != nil { return big.Zero(), xerrors.Errorf("failed to load power state: %w", err) } - return pst.TotalLocked(), nil + return pst.TotalLocked() } func (sm *StateManager) GetFilLocked(ctx context.Context, st *state.StateTree) (abi.TokenAmount, error) { @@ -1124,6 +1129,8 @@ func (sm *StateManager) GetCirculatingSupply(ctx context.Context, height abi.Cha } func (sm *StateManager) GetNtwkVersion(ctx context.Context, height abi.ChainEpoch) network.Version { + // TODO: move hard fork epoch checks to a schedule defined in build/ + if build.UpgradeBreezeHeight == 0 { return network.Version1 } diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index a137afb51..a6fdce31e 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -21,6 +21,7 @@ import ( "github.com/filecoin-project/go-bitfield" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/crypto" + power2 "github.com/filecoin-project/lotus/chain/actors/builtin/power" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/builtin/account" @@ -670,17 +671,21 @@ func GetReturnType(ctx context.Context, sm *StateManager, to address.Address, me } func MinerHasMinPower(ctx context.Context, sm *StateManager, addr address.Address, ts *types.TipSet) (bool, error) { - var ps power.State - _, err := sm.LoadActorState(ctx, builtin.StoragePowerActorAddr, &ps, ts) + pact, err := sm.LoadActor(ctx, builtin.StoragePowerActorAddr, ts) if err != nil { return false, xerrors.Errorf("loading power actor state: %w", err) } + ps, err := power2.Load(sm.cs.Store(ctx), pact) + if err != nil { + return false, err + } + return ps.MinerNominalPowerMeetsConsensusMinimum(sm.ChainStore().Store(ctx), addr) } func CheckTotalFIL(ctx context.Context, sm *StateManager, ts *types.TipSet) (abi.TokenAmount, error) { - str, err := state.LoadStateTree(sm.ChainStore().Store(ctx), ts.ParentState()) + str, err := state.LoadStateTree(sm.ChainStore().Store(ctx), ts.ParentState(), sm.GetNtwkVersion(ctx, ts.Height())) if err != nil { return abi.TokenAmount{}, err } diff --git a/chain/store/store.go b/chain/store/store.go index e0a997686..390242294 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -5,6 +5,8 @@ import ( "context" "encoding/binary" "encoding/json" + "github.com/filecoin-project/go-state-types/network" + "github.com/filecoin-project/lotus/build" "io" "os" "strconv" @@ -741,11 +743,29 @@ type BlockMessages struct { WinCount int64 } +// TODO: temp hack until #3682 is merged +func hackgetNtwkVersionhack(ctx context.Context, height abi.ChainEpoch) network.Version { + // TODO: move hard fork epoch checks to a schedule defined in build/ + + if build.UpgradeBreezeHeight == 0 { + return network.Version1 + } + + if height <= build.UpgradeBreezeHeight { + return network.Version0 + } + + return network.Version1 +} + func (cs *ChainStore) BlockMsgsForTipset(ts *types.TipSet) ([]BlockMessages, error) { applied := make(map[address.Address]uint64) cst := cbor.NewCborStore(cs.bs) - st, err := state.LoadStateTree(cst, ts.Blocks()[0].ParentStateRoot) + + nv := hackgetNtwkVersionhack(context.TODO(), ts.Height()) // TODO: part of the temp hack from above + + st, err := state.LoadStateTree(cst, ts.Blocks()[0].ParentStateRoot, nv) if err != nil { return nil, xerrors.Errorf("failed to load state tree") } diff --git a/chain/store/weight.go b/chain/store/weight.go index 5249df011..e27d3fd37 100644 --- a/chain/store/weight.go +++ b/chain/store/weight.go @@ -29,7 +29,7 @@ func (cs *ChainStore) Weight(ctx context.Context, ts *types.TipSet) (types.BigIn tpow := big2.Zero() { cst := cbor.NewCborStore(cs.Blockstore()) - state, err := state.LoadStateTree(cst, ts.ParentState()) + state, err := state.LoadStateTree(cst, ts.ParentState(), hackgetNtwkVersionhack(nil, ts.Height())) // TODO: hackgetNtwkVersionhack: HELP if err != nil { return types.NewInt(0), xerrors.Errorf("load state tree: %w", err) } diff --git a/chain/vm/vm.go b/chain/vm/vm.go index a125df053..ced11e04d 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -163,7 +163,7 @@ type VMOpts struct { Bstore bstore.Blockstore Syscalls SyscallBuilder CircSupplyCalc CircSupplyCalculator - NtwkVersion NtwkVersionGetter + NtwkVersion NtwkVersionGetter // TODO: stebalien: In what cases do we actually need this? It seems like even when creating new networks we want to use the 'global'/build-default version getter BaseFee abi.TokenAmount } From fb3f7105238ff59ff91fed05aec4eee5ae391dec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 14 Sep 2020 15:17:00 +0200 Subject: [PATCH 088/885] Some post-master merge fixes --- api/apistruct/struct.go | 8 +++----- chain/gen/genesis/miners.go | 7 +------ chain/gen/genesis/util.go | 13 ++++++++++--- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 151e7d99a..1eac858a0 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -2,13 +2,9 @@ package apistruct import ( "context" - "github.com/filecoin-project/go-state-types/dline" - miner2 "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "io" "time" - "github.com/filecoin-project/go-state-types/dline" - "github.com/ipfs/go-cid" metrics "github.com/libp2p/go-libp2p-core/metrics" "github.com/libp2p/go-libp2p-core/network" @@ -25,6 +21,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" + "github.com/filecoin-project/go-state-types/dline" "github.com/filecoin-project/lotus/extern/sector-storage/fsutil" "github.com/filecoin-project/lotus/extern/sector-storage/sealtasks" "github.com/filecoin-project/lotus/extern/sector-storage/stores" @@ -37,6 +34,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" + miner2 "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/modules/dtypes" ) @@ -168,7 +166,7 @@ type FullNodeStruct struct { StateMinerActiveSectors func(context.Context, address.Address, types.TipSetKey) ([]*api.ChainSectorInfo, error) `perm:"read"` StateMinerProvingDeadline func(context.Context, address.Address, types.TipSetKey) (*dline.Info, error) `perm:"read"` StateMinerPower func(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error) `perm:"read"` - StateMinerInfo func(context.Context, address.Address, types.TipSetKey) (miner2.MinerInfo, error) `perm:"read"` + StateMinerInfo func(context.Context, address.Address, types.TipSetKey) (miner2.MinerInfo, error) `perm:"read"` StateMinerFaults func(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error) `perm:"read"` StateAllMinerFaults func(context.Context, abi.ChainEpoch, types.TipSetKey) ([]*api.Fault, error) `perm:"read"` StateMinerRecoveries func(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error) `perm:"read"` diff --git a/chain/gen/genesis/miners.go b/chain/gen/genesis/miners.go index 6c1fec23c..3bed1c1a9 100644 --- a/chain/gen/genesis/miners.go +++ b/chain/gen/genesis/miners.go @@ -6,12 +6,6 @@ import ( "fmt" "math/rand" - "github.com/filecoin-project/lotus/build" - - "github.com/filecoin-project/go-state-types/network" - - "github.com/filecoin-project/lotus/chain/state" - "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" cbg "github.com/whyrusleeping/cbor-gen" @@ -29,6 +23,7 @@ import ( "github.com/filecoin-project/specs-actors/actors/builtin/reward" "github.com/filecoin-project/specs-actors/actors/runtime" + "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" diff --git a/chain/gen/genesis/util.go b/chain/gen/genesis/util.go index 54fc4abfe..6a27f2f29 100644 --- a/chain/gen/genesis/util.go +++ b/chain/gen/genesis/util.go @@ -3,6 +3,7 @@ package genesis import ( "context" "github.com/filecoin-project/go-state-types/network" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" @@ -48,7 +49,13 @@ func doExecValue(ctx context.Context, vm *vm.VM, to, from address.Address, value return ret.Return, nil } -var GenesisNetworkVersion = network.Version1 -func genesisNetworkVersion(context.Context, abi.ChainEpoch) network.Version { +var GenesisNetworkVersion = func() network.Version {// TODO: Get from build/ + if build.UseNewestNetwork() {// TODO: Get from build/ + return build.NewestNetworkVersion// TODO: Get from build/ + }// TODO: Get from build/ + return network.Version1// TODO: Get from build/ +}()// TODO: Get from build/ + +func genesisNetworkVersion(context.Context, abi.ChainEpoch) network.Version {// TODO: Get from build/ return GenesisNetworkVersion // TODO: Get from build/ -} +}// TODO: Get from build/ From 09e9d6d7781a221b842d34fd43f49e172f387419 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Mon, 14 Sep 2020 16:20:01 +0100 Subject: [PATCH 089/885] deal journal events: wire into markets subscriptions. --- markets/loggers/journal.go | 76 ++++++++++++++++++++++++ markets/storageadapter/client.go | 23 +------ markets/storageadapter/journal_events.go | 64 -------------------- markets/storageadapter/provider.go | 45 ++------------ node/modules/client.go | 9 +++ node/modules/storageminer.go | 9 +++ 6 files changed, 99 insertions(+), 127 deletions(-) create mode 100644 markets/loggers/journal.go delete mode 100644 markets/storageadapter/journal_events.go diff --git a/markets/loggers/journal.go b/markets/loggers/journal.go new file mode 100644 index 000000000..12948fe7b --- /dev/null +++ b/markets/loggers/journal.go @@ -0,0 +1,76 @@ +package marketevents + +import ( + "github.com/filecoin-project/go-fil-markets/retrievalmarket" + "github.com/filecoin-project/go-fil-markets/storagemarket" + + "github.com/filecoin-project/lotus/journal" +) + +type StorageClientEvt struct { + Event string + Deal storagemarket.ClientDeal +} + +type StorageProviderEvt struct { + Event string + Deal storagemarket.MinerDeal +} + +type RetrievalClientEvt struct { + Event string + Deal retrievalmarket.ClientDealState +} + +type RetrievalProviderEvt struct { + Event string + Deal retrievalmarket.ProviderDealState +} + +// StorageClientJournaler records journal events from the storage client. +func StorageClientJournaler(evtType journal.EventType) func(event storagemarket.ClientEvent, deal storagemarket.ClientDeal) { + return func(event storagemarket.ClientEvent, deal storagemarket.ClientDeal) { + journal.J.RecordEvent(evtType, func() interface{} { + return StorageClientEvt{ + Event: storagemarket.ClientEvents[event], + Deal: deal, + } + }) + } +} + +// StorageProviderJournaler records journal events from the storage provider. +func StorageProviderJournaler(evtType journal.EventType) func(event storagemarket.ProviderEvent, deal storagemarket.MinerDeal) { + return func(event storagemarket.ProviderEvent, deal storagemarket.MinerDeal) { + journal.J.RecordEvent(evtType, func() interface{} { + return StorageProviderEvt{ + Event: storagemarket.ProviderEvents[event], + Deal: deal, + } + }) + } +} + +// RetrievalClientJournaler records journal events from the retrieval client. +func RetrievalClientJournaler(evtType journal.EventType) func(event retrievalmarket.ClientEvent, deal retrievalmarket.ClientDealState) { + return func(event retrievalmarket.ClientEvent, deal retrievalmarket.ClientDealState) { + journal.J.RecordEvent(evtType, func() interface{} { + return RetrievalClientEvt{ + Event: retrievalmarket.ClientEvents[event], + Deal: deal, + } + }) + } +} + +// RetrievalProviderJournaler records journal events from the retrieval provider. +func RetrievalProviderJournaler(evtType journal.EventType) func(event retrievalmarket.ProviderEvent, deal retrievalmarket.ProviderDealState) { + return func(event retrievalmarket.ProviderEvent, deal retrievalmarket.ProviderDealState) { + journal.J.RecordEvent(evtType, func() interface{} { + return RetrievalProviderEvt{ + Event: retrievalmarket.ProviderEvents[event], + Deal: deal, + } + }) + } +} diff --git a/markets/storageadapter/client.go b/markets/storageadapter/client.go index f66fd3ef9..4168792da 100644 --- a/markets/storageadapter/client.go +++ b/markets/storageadapter/client.go @@ -24,7 +24,6 @@ import ( "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/lib/sigs" "github.com/filecoin-project/lotus/markets/utils" "github.com/filecoin-project/lotus/node/impl/full" @@ -43,9 +42,6 @@ type ClientNodeAdapter struct { cs *store.ChainStore fm *market.FundMgr ev *events.Events - - // index 0 is unused, as it corresponds to evtTypeDealAccepted, a provider-only event. - evtTypes [4]journal.EventType } type clientApi struct { @@ -63,12 +59,6 @@ func NewClientNodeAdapter(state full.StateAPI, chain full.ChainAPI, mpool full.M cs: cs, fm: fm, ev: events.NewEvents(context.TODO(), &clientApi{chain, state}), - - evtTypes: [...]journal.EventType{ - evtTypeDealSectorCommitted: journal.J.RegisterEventType("markets:storage:client", "deal_sector_committed"), - evtTypeDealExpired: journal.J.RegisterEventType("markets:storage:client", "deal_expired"), - evtTypeDealSlashed: journal.J.RegisterEventType("markets:storage:client", "deal_slashed"), - }, } } @@ -233,14 +223,7 @@ func (c *ClientNodeAdapter) ValidatePublishedDeal(ctx context.Context, deal stor return 0, err } - dealID := res.IDs[dealIdx] - journal.J.RecordEvent(c.evtTypes[evtTypeDealAccepted], func() interface{} { - deal := deal // copy and strip fields we don't want to log to the journal - deal.ClientSignature = crypto.Signature{} - return ClientDealAcceptedEvt{ID: dealID, Deal: deal, Height: c.cs.GetHeaviestTipSet().Height()} - }) - - return dealID, nil + return res.IDs[dealIdx], nil } const clientOverestimation = 2 @@ -294,10 +277,6 @@ func (c *ClientNodeAdapter) OnDealSectorCommitted(ctx context.Context, provider log.Infof("Storage deal %d activated at epoch %d", dealId, sd.State.SectorStartEpoch) - journal.J.RecordEvent(c.evtTypes[evtTypeDealSectorCommitted], func() interface{} { - return ClientDealSectorCommittedEvt{ID: dealId, State: sd.State, Height: curH} - }) - cb(nil) return false, nil diff --git a/markets/storageadapter/journal_events.go b/markets/storageadapter/journal_events.go deleted file mode 100644 index b74c50301..000000000 --- a/markets/storageadapter/journal_events.go +++ /dev/null @@ -1,64 +0,0 @@ -package storageadapter - -import ( - "github.com/filecoin-project/go-fil-markets/storagemarket" - "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/specs-actors/actors/builtin/market" -) - -// Journal entry types emitted from this module. -const ( - evtTypeDealAccepted = iota - evtTypeDealSectorCommitted - evtTypeDealExpired - evtTypeDealSlashed -) - -type ClientDealAcceptedEvt struct { - ID abi.DealID - Deal storagemarket.ClientDeal - Height abi.ChainEpoch -} - -type ClientDealSectorCommittedEvt struct { - ID abi.DealID - State market.DealState - Height abi.ChainEpoch -} - -type ClientDealExpiredEvt struct { - ID abi.DealID - State market.DealState - Height abi.ChainEpoch -} - -type ClientDealSlashedEvt struct { - ID abi.DealID - State market.DealState - Height abi.ChainEpoch -} - -type MinerDealAcceptedEvt struct { - ID abi.DealID - Deal storagemarket.MinerDeal - State market.DealState - Height abi.ChainEpoch -} - -type MinerDealSectorCommittedEvt struct { - ID abi.DealID - State market.DealState - Height abi.ChainEpoch -} - -type MinerDealExpiredEvt struct { - ID abi.DealID - State market.DealState - Height abi.ChainEpoch -} - -type MinerDealSlashedEvt struct { - ID abi.DealID - State market.DealState - Height abi.ChainEpoch -} diff --git a/markets/storageadapter/provider.go b/markets/storageadapter/provider.go index 5b8ead5be..7af1808c1 100644 --- a/markets/storageadapter/provider.go +++ b/markets/storageadapter/provider.go @@ -30,7 +30,6 @@ import ( "github.com/filecoin-project/lotus/chain/events/state" "github.com/filecoin-project/lotus/chain/types" sealing "github.com/filecoin-project/lotus/extern/storage-sealing" - "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/lib/sigs" "github.com/filecoin-project/lotus/markets/utils" "github.com/filecoin-project/lotus/node/modules/dtypes" @@ -49,8 +48,6 @@ type ProviderNodeAdapter struct { secb *sectorblocks.SectorBlocks ev *events.Events - - evtTypes [4]journal.EventType } func NewProviderNodeAdapter(dag dtypes.StagingDAG, secb *sectorblocks.SectorBlocks, full api.FullNode) storagemarket.StorageProviderNode { @@ -59,13 +56,6 @@ func NewProviderNodeAdapter(dag dtypes.StagingDAG, secb *sectorblocks.SectorBloc dag: dag, secb: secb, ev: events.NewEvents(context.TODO(), full), - - evtTypes: [...]journal.EventType{ - evtTypeDealAccepted: journal.J.RegisterEventType("markets:storage:provider", "deal_complete"), - evtTypeDealSectorCommitted: journal.J.RegisterEventType("markets:storage:provider", "deal_sector_committed"), - evtTypeDealExpired: journal.J.RegisterEventType("markets:storage:provider", "deal_expired"), - evtTypeDealSlashed: journal.J.RegisterEventType("markets:storage:provider", "deal_slashed"), - }, } } @@ -132,15 +122,8 @@ func (n *ProviderNodeAdapter) OnDealComplete(ctx context.Context, deal storagema if err != nil { return nil, xerrors.Errorf("AddPiece failed: %s", err) } - log.Warnf("New Deal: deal %d", deal.DealID) - journal.J.RecordEvent(n.evtTypes[evtTypeDealAccepted], func() interface{} { - deal := deal // copy and strip fields we don't want to log to the journal - deal.ClientSignature = crypto.Signature{} - return MinerDealAcceptedEvt{ID: deal.DealID, Deal: deal} - }) - return &storagemarket.PackingResult{ SectorNumber: p, Offset: offset, @@ -319,10 +302,6 @@ func (n *ProviderNodeAdapter) OnDealSectorCommitted(ctx context.Context, provide log.Infof("Storage deal %d activated at epoch %d", dealID, sd.State.SectorStartEpoch) - journal.J.RecordEvent(n.evtTypes[evtTypeDealSectorCommitted], func() interface{} { - return MinerDealSectorCommittedEvt{ID: dealID, State: sd.State, Height: curH} - }) - cb(nil) return false, nil @@ -429,23 +408,15 @@ func (n *ProviderNodeAdapter) OnDealExpiredOrSlashed(ctx context.Context, dealID return false, true, nil } - height := ts.Height() - // Check if the deal has already expired - if sd.Proposal.EndEpoch <= height { + if sd.Proposal.EndEpoch <= ts.Height() { onDealExpired(nil) - journal.J.RecordEvent(n.evtTypes[evtTypeDealExpired], func() interface{} { - return MinerDealExpiredEvt{ID: dealID, State: sd.State, Height: height} - }) return true, false, nil } // If there is no deal assume it's already been slashed if sd.State.SectorStartEpoch < 0 { - onDealSlashed(height, nil) - journal.J.RecordEvent(n.evtTypes[evtTypeDealSlashed], func() interface{} { - return MinerDealSlashedEvt{ID: dealID, State: sd.State, Height: height} - }) + onDealSlashed(ts.Height(), nil) return true, false, nil } @@ -457,14 +428,9 @@ func (n *ProviderNodeAdapter) OnDealExpiredOrSlashed(ctx context.Context, dealID // Called when there was a match against the state change we're looking for // and the chain has advanced to the confidence height stateChanged := func(ts *types.TipSet, ts2 *types.TipSet, states events.StateChange, h abi.ChainEpoch) (more bool, err error) { - height := ts2.Height() - // Check if the deal has already expired - if sd.Proposal.EndEpoch <= height { + if sd.Proposal.EndEpoch <= ts2.Height() { onDealExpired(nil) - journal.J.RecordEvent(n.evtTypes[evtTypeDealExpired], func() interface{} { - return MinerDealExpiredEvt{ID: dealID, State: sd.State, Height: height} - }) return false, nil } @@ -487,10 +453,7 @@ func (n *ProviderNodeAdapter) OnDealExpiredOrSlashed(ctx context.Context, dealID // Deal was slashed if deal.To == nil { - onDealSlashed(height, nil) - journal.J.RecordEvent(n.evtTypes[evtTypeDealSlashed], func() interface{} { - return MinerDealSlashedEvt{ID: dealID, State: sd.State, Height: height} - }) + onDealSlashed(ts2.Height(), nil) return false, nil } diff --git a/node/modules/client.go b/node/modules/client.go index 63633c0e3..7d1ac942e 100644 --- a/node/modules/client.go +++ b/node/modules/client.go @@ -26,6 +26,7 @@ import ( "github.com/ipfs/go-datastore/namespace" "github.com/libp2p/go-libp2p-core/host" + "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/lib/blockstore" marketevents "github.com/filecoin-project/lotus/markets/loggers" "github.com/filecoin-project/lotus/markets/retrievaladapter" @@ -119,6 +120,10 @@ func StorageClient(lc fx.Lifecycle, h host.Host, ibs dtypes.ClientBlockstore, md lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { c.SubscribeToEvents(marketevents.StorageClientLogger) + + evtType := journal.J.RegisterEventType("markets/storage/client", "state_change") + c.SubscribeToEvents(marketevents.StorageClientJournaler(evtType)) + return c.Start(ctx) }, OnStop: func(context.Context) error { @@ -140,6 +145,10 @@ func RetrievalClient(lc fx.Lifecycle, h host.Host, mds dtypes.ClientMultiDstore, lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { client.SubscribeToEvents(marketevents.RetrievalClientLogger) + + evtType := journal.J.RegisterEventType("markets/retrieval/client", "state_change") + client.SubscribeToEvents(marketevents.RetrievalClientJournaler(evtType)) + return nil }, }) diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 5f00fffc0..eb512da07 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -49,6 +49,7 @@ import ( "github.com/filecoin-project/lotus/extern/sector-storage/stores" sealing "github.com/filecoin-project/lotus/extern/storage-sealing" "github.com/filecoin-project/lotus/extern/storage-sealing/sealiface" + "github.com/filecoin-project/lotus/journal" lapi "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" @@ -213,6 +214,10 @@ func HandleRetrieval(host host.Host, lc fx.Lifecycle, m retrievalmarket.Retrieva lc.Append(fx.Hook{ OnStart: func(context.Context) error { m.SubscribeToEvents(marketevents.RetrievalProviderLogger) + + evtType := journal.J.RegisterEventType("markets/retrieval/provider", "state_change") + m.SubscribeToEvents(marketevents.RetrievalProviderJournaler(evtType)) + return m.Start() }, OnStop: func(context.Context) error { @@ -227,6 +232,10 @@ func HandleDeals(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, h sto lc.Append(fx.Hook{ OnStart: func(context.Context) error { h.SubscribeToEvents(marketevents.StorageProviderLogger) + + evtType := journal.J.RegisterEventType("markets/storage/provider", "state_change") + h.SubscribeToEvents(marketevents.StorageProviderJournaler(evtType)) + return h.Start(ctx) }, OnStop: func(context.Context) error { From 954ff325034abd98c73a87b67330c9ea9b9af4fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Mon, 14 Sep 2020 19:01:35 +0100 Subject: [PATCH 090/885] fix transitive api dependency on ffi. --- markets/{loggers => }/journal.go | 2 +- node/modules/client.go | 5 +++-- node/modules/storageminer.go | 5 +++-- 3 files changed, 7 insertions(+), 5 deletions(-) rename markets/{loggers => }/journal.go (99%) diff --git a/markets/loggers/journal.go b/markets/journal.go similarity index 99% rename from markets/loggers/journal.go rename to markets/journal.go index 12948fe7b..a73d28011 100644 --- a/markets/loggers/journal.go +++ b/markets/journal.go @@ -1,4 +1,4 @@ -package marketevents +package markets import ( "github.com/filecoin-project/go-fil-markets/retrievalmarket" diff --git a/node/modules/client.go b/node/modules/client.go index 7d1ac942e..6aa3cbb0e 100644 --- a/node/modules/client.go +++ b/node/modules/client.go @@ -28,6 +28,7 @@ import ( "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/lib/blockstore" + "github.com/filecoin-project/lotus/markets" marketevents "github.com/filecoin-project/lotus/markets/loggers" "github.com/filecoin-project/lotus/markets/retrievaladapter" "github.com/filecoin-project/lotus/node/impl/full" @@ -122,7 +123,7 @@ func StorageClient(lc fx.Lifecycle, h host.Host, ibs dtypes.ClientBlockstore, md c.SubscribeToEvents(marketevents.StorageClientLogger) evtType := journal.J.RegisterEventType("markets/storage/client", "state_change") - c.SubscribeToEvents(marketevents.StorageClientJournaler(evtType)) + c.SubscribeToEvents(markets.StorageClientJournaler(evtType)) return c.Start(ctx) }, @@ -147,7 +148,7 @@ func RetrievalClient(lc fx.Lifecycle, h host.Host, mds dtypes.ClientMultiDstore, client.SubscribeToEvents(marketevents.RetrievalClientLogger) evtType := journal.J.RegisterEventType("markets/retrieval/client", "state_change") - client.SubscribeToEvents(marketevents.RetrievalClientJournaler(evtType)) + client.SubscribeToEvents(markets.RetrievalClientJournaler(evtType)) return nil }, diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index eb512da07..9a94a56a5 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -50,6 +50,7 @@ import ( sealing "github.com/filecoin-project/lotus/extern/storage-sealing" "github.com/filecoin-project/lotus/extern/storage-sealing/sealiface" "github.com/filecoin-project/lotus/journal" + "github.com/filecoin-project/lotus/markets" lapi "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" @@ -216,7 +217,7 @@ func HandleRetrieval(host host.Host, lc fx.Lifecycle, m retrievalmarket.Retrieva m.SubscribeToEvents(marketevents.RetrievalProviderLogger) evtType := journal.J.RegisterEventType("markets/retrieval/provider", "state_change") - m.SubscribeToEvents(marketevents.RetrievalProviderJournaler(evtType)) + m.SubscribeToEvents(markets.RetrievalProviderJournaler(evtType)) return m.Start() }, @@ -234,7 +235,7 @@ func HandleDeals(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, h sto h.SubscribeToEvents(marketevents.StorageProviderLogger) evtType := journal.J.RegisterEventType("markets/storage/provider", "state_change") - h.SubscribeToEvents(marketevents.StorageProviderJournaler(evtType)) + h.SubscribeToEvents(markets.StorageProviderJournaler(evtType)) return h.Start(ctx) }, From 8a05fe86d6e6504004aae927f2e27dd02a4c6159 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 14 Sep 2020 20:33:54 +0200 Subject: [PATCH 091/885] wdpost: Skip sectors correctly --- storage/wdpost_run.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 8f15c4796..08a7437a9 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -430,16 +430,16 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty return nil, xerrors.Errorf("adding recoveries to set of sectors to prove: %w", err) } - toProve, err = bitfield.SubtractBitField(toProve, postSkipped) - if err != nil { - return nil, xerrors.Errorf("toProve - postSkipped: %w", err) - } - good, err := s.checkSectors(ctx, toProve) if err != nil { return nil, xerrors.Errorf("checking sectors to skip: %w", err) } + good, err = bitfield.SubtractBitField(good, postSkipped) + if err != nil { + return nil, xerrors.Errorf("toProve - postSkipped: %w", err) + } + skipped, err := bitfield.SubtractBitField(toProve, good) if err != nil { return nil, xerrors.Errorf("toProve - good: %w", err) From 60a031a7134b673f4ebb31f61e3536969bc54634 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Mon, 14 Sep 2020 00:25:54 -0400 Subject: [PATCH 092/885] Lower devnet block time --- build/params_2k.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/params_2k.go b/build/params_2k.go index 0ef1d9b34..9913374e1 100644 --- a/build/params_2k.go +++ b/build/params_2k.go @@ -29,7 +29,7 @@ func init() { BuildType |= Build2k } -const BlockDelaySecs = uint64(30) +const BlockDelaySecs = uint64(4) const PropagationDelaySecs = uint64(1) From d1850ea27d1488990f2395656a4154eabe0c9608 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Sun, 13 Sep 2020 01:26:44 -0400 Subject: [PATCH 093/885] Add an API to get detailed gas costs for a message --- api/api_full.go | 13 ++++++++ api/apistruct/struct.go | 5 +++ chain/stmgr/stmgr.go | 4 +-- chain/vm/vm.go | 35 +++++++++++---------- cli/state.go | 55 +++++++++++++++++++++++++++++++++ documentation/en/api-methods.md | 35 +++++++++++++++++++++ node/impl/full/state.go | 40 ++++++++++++++++++++++++ 7 files changed, 169 insertions(+), 18 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 23226443a..588e567a8 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -354,6 +354,8 @@ type FullNode interface { StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*SectorLocation, error) // StateSearchMsg searches for a message in the chain, and returns its receipt and the tipset where it was executed StateSearchMsg(context.Context, cid.Cid) (*MsgLookup, error) + // StateMsgGasCost searches for a message in the chain, and returns details of the messages gas costs, including the penalty and miner tip + StateMsgGasCost(context.Context, cid.Cid, types.TipSetKey) (*MsgGasCost, error) // StateWaitMsg looks back in the chain for a message. If not found, it blocks until the // message arrives on chain, and gets to the indicated confidence depth. StateWaitMsg(ctx context.Context, cid cid.Cid, confidence uint64) (*MsgLookup, error) @@ -531,6 +533,17 @@ type MsgLookup struct { Height abi.ChainEpoch } +type MsgGasCost struct { + Message cid.Cid // Can be different than requested, in case it was replaced, but only gas values changed + GasUsed abi.TokenAmount + BaseFeeBurn abi.TokenAmount + OverEstimationBurn abi.TokenAmount + MinerPenalty abi.TokenAmount + MinerTip abi.TokenAmount + Refund abi.TokenAmount + TotalCost abi.TokenAmount +} + type BlockMessages struct { BlsMessages []*types.Message SecpkMessages []*types.SignedMessage diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index fdb9843ff..9f2b7bc4b 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -183,6 +183,7 @@ type FullNodeStruct struct { StateReplay func(context.Context, types.TipSetKey, cid.Cid) (*api.InvocResult, error) `perm:"read"` StateGetActor func(context.Context, address.Address, types.TipSetKey) (*types.Actor, error) `perm:"read"` StateReadState func(context.Context, address.Address, types.TipSetKey) (*api.ActorState, error) `perm:"read"` + StateMsgGasCost func(context.Context, cid.Cid, types.TipSetKey) (*api.MsgGasCost, error) `perm:"read"` StateWaitMsg func(ctx context.Context, cid cid.Cid, confidence uint64) (*api.MsgLookup, error) `perm:"read"` StateSearchMsg func(context.Context, cid.Cid) (*api.MsgLookup, error) `perm:"read"` StateListMiners func(context.Context, types.TipSetKey) ([]address.Address, error) `perm:"read"` @@ -816,6 +817,10 @@ func (c *FullNodeStruct) StateReadState(ctx context.Context, addr address.Addres return c.Internal.StateReadState(ctx, addr, tsk) } +func (c *FullNodeStruct) StateMsgGasCost(ctx context.Context, msgc cid.Cid, tsk types.TipSetKey) (*api.MsgGasCost, error) { + return c.Internal.StateMsgGasCost(ctx, msgc, tsk) +} + func (c *FullNodeStruct) StateWaitMsg(ctx context.Context, msgc cid.Cid, confidence uint64) (*api.MsgLookup, error) { return c.Internal.StateWaitMsg(ctx, msgc, confidence) } diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index e6103e2b3..a7f94e6ee 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -233,8 +233,8 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp } receipts = append(receipts, &r.MessageReceipt) - gasReward = big.Add(gasReward, r.MinerTip) - penalty = big.Add(penalty, r.Penalty) + gasReward = big.Add(gasReward, r.GasCosts.MinerTip) + penalty = big.Add(penalty, r.GasCosts.MinerPenalty) if cb != nil { if err := cb(cm.Cid(), m, r); err != nil { diff --git a/chain/vm/vm.go b/chain/vm/vm.go index e389a3531..eb1ef70ce 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -201,10 +201,9 @@ type Rand interface { type ApplyRet struct { types.MessageReceipt ActorErr aerrors.ActorError - Penalty types.BigInt - MinerTip types.BigInt ExecutionTrace types.ExecutionTrace Duration time.Duration + GasCosts GasOutputs } func (vm *VM) send(ctx context.Context, msg *types.Message, parent *Runtime, @@ -328,8 +327,7 @@ func (vm *VM) ApplyImplicitMessage(ctx context.Context, msg *types.Message) (*Ap }, ActorErr: actorErr, ExecutionTrace: rt.executionTrace, - Penalty: types.NewInt(0), - MinerTip: types.NewInt(0), + GasCosts: GasOutputs{}, Duration: time.Since(start), }, actorErr } @@ -362,9 +360,10 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, ExitCode: exitcode.SysErrOutOfGas, GasUsed: 0, }, - Penalty: types.BigMul(vm.baseFee, abi.NewTokenAmount(msgGasCost)), + GasCosts: GasOutputs{ + MinerPenalty: types.BigMul(vm.baseFee, abi.NewTokenAmount(msgGasCost)), + }, Duration: time.Since(start), - MinerTip: big.Zero(), }, nil } @@ -381,9 +380,10 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, GasUsed: 0, }, ActorErr: aerrors.Newf(exitcode.SysErrSenderInvalid, "actor not found: %s", msg.From), - Penalty: minerPenaltyAmount, + GasCosts: GasOutputs{ + MinerPenalty: minerPenaltyAmount, + }, Duration: time.Since(start), - MinerTip: big.Zero(), }, nil } return nil, xerrors.Errorf("failed to look up from actor: %w", err) @@ -397,9 +397,10 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, GasUsed: 0, }, ActorErr: aerrors.Newf(exitcode.SysErrSenderInvalid, "send from not account actor: %s", fromActor.Code), - Penalty: minerPenaltyAmount, + GasCosts: GasOutputs{ + MinerPenalty: minerPenaltyAmount, + }, Duration: time.Since(start), - MinerTip: big.Zero(), }, nil } @@ -411,9 +412,11 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, }, ActorErr: aerrors.Newf(exitcode.SysErrSenderStateInvalid, "actor nonce invalid: msg:%d != state:%d", msg.Nonce, fromActor.Nonce), - Penalty: minerPenaltyAmount, + + GasCosts: GasOutputs{ + MinerPenalty: minerPenaltyAmount, + }, Duration: time.Since(start), - MinerTip: big.Zero(), }, nil } @@ -426,9 +429,10 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, }, ActorErr: aerrors.Newf(exitcode.SysErrSenderStateInvalid, "actor balance less than needed: %s < %s", types.FIL(fromActor.Balance), types.FIL(gascost)), - Penalty: minerPenaltyAmount, + GasCosts: GasOutputs{ + MinerPenalty: minerPenaltyAmount, + }, Duration: time.Since(start), - MinerTip: big.Zero(), }, nil } @@ -521,8 +525,7 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, }, ActorErr: actorErr, ExecutionTrace: rt.executionTrace, - Penalty: gasOutputs.MinerPenalty, - MinerTip: gasOutputs.MinerTip, + GasCosts: gasOutputs, Duration: time.Since(start), }, nil } diff --git a/cli/state.go b/cli/state.go index 1036e8fe5..7729e729a 100644 --- a/cli/state.go +++ b/cli/state.go @@ -65,6 +65,7 @@ var stateCmd = &cli.Command{ stateGetDealSetCmd, stateWaitMsgCmd, stateSearchMsgCmd, + stateMsgCostCmd, stateMinerInfo, stateMarketCmd, }, @@ -1304,6 +1305,60 @@ var stateSearchMsgCmd = &cli.Command{ }, } +var stateMsgCostCmd = &cli.Command{ + Name: "msg-cost", + Usage: "Get the detailed gas costs of a message", + ArgsUsage: "[messageCid]", + Action: func(cctx *cli.Context) error { + if !cctx.Args().Present() { + return fmt.Errorf("must specify message cid to get gas costs for") + } + + api, closer, err := GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + + ctx := ReqContext(cctx) + + msg, err := cid.Decode(cctx.Args().First()) + if err != nil { + return err + } + + tsk := types.EmptyTSK + + ts, err := LoadTipSet(ctx, cctx, api) + if err != nil { + return err + } + + if ts != nil { + tsk = ts.Key() + } + + mgc, err := api.StateMsgGasCost(ctx, msg, tsk) + if err != nil { + return err + } + + if mgc != nil { + fmt.Printf("\nMessage CID: %d", mgc.Message) + fmt.Printf("\nGas Used: %d", mgc.GasUsed) + fmt.Printf("\nBase Fee Burn: %d", mgc.BaseFeeBurn) + fmt.Printf("\nOverestimation Burn: %d", mgc.OverEstimationBurn) + fmt.Printf("\nMiner Tip: %d", mgc.MinerTip) + fmt.Printf("\nRefundd: %d", mgc.Refund) + fmt.Printf("\nTotal Cost: %d", mgc.TotalCost) + fmt.Printf("\nMiner Penalty: %d", mgc.MinerPenalty) + } else { + fmt.Print("message was not found on chain") + } + return nil + }, +} + var stateCallCmd = &cli.Command{ Name: "call", Usage: "Invoke a method on an actor locally", diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index 27875eca1..ec298d635 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -148,6 +148,7 @@ * [StateMinerRecoveries](#StateMinerRecoveries) * [StateMinerSectorCount](#StateMinerSectorCount) * [StateMinerSectors](#StateMinerSectors) + * [StateMsgGasCost](#StateMsgGasCost) * [StateNetworkName](#StateNetworkName) * [StateReadState](#StateReadState) * [StateReplay](#StateReplay) @@ -3731,6 +3732,40 @@ Inputs: Response: `null` +### StateMsgGasCost +StateMsgGasCost searches for a message in the chain, and returns details of the messages gas costs, including the penalty and miner tip + + +Perms: read + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: +```json +{ + "Message": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Receipt": { + "ExitCode": 0, + "Return": "Ynl0ZSBhcnJheQ==", + "GasUsed": 9 + }, + "BaseFeeBurn": "0", + "OverEstimationBurn": "0", + "MinerPenalty": "0", + "MinerTip": "0", + "Refund": "0" +} +``` + ### StateNetworkName StateNetworkName returns the name of the network the node is synced to diff --git a/node/impl/full/state.go b/node/impl/full/state.go index e183fafa4..1fb078001 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -1226,3 +1226,43 @@ func (a *StateAPI) StateCirculatingSupply(ctx context.Context, tsk types.TipSetK return a.StateManager.GetCirculatingSupplyDetailed(ctx, ts.Height(), sTree) } + +func (a *StateAPI) StateMsgGasCost(ctx context.Context, inputMsg cid.Cid, tsk types.TipSetKey) (*api.MsgGasCost, error) { + var msg cid.Cid + var ts *types.TipSet + var err error + if tsk != types.EmptyTSK { + msg = inputMsg + ts, err = a.Chain.LoadTipSet(tsk) + if err != nil { + return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) + } + } else { + msg, err := a.StateSearchMsg(ctx, inputMsg) + if err != nil { + return nil, xerrors.Errorf("searching for msg %s: %w", inputMsg, err) + } + + ts, err = a.Chain.GetTipSetFromKey(msg.TipSet) + if err != nil { + return nil, xerrors.Errorf("loading tipset %s: %w", msg.TipSet, err) + } + } + + m, r, err := a.StateManager.Replay(ctx, ts, msg) + if err != nil { + return nil, err + } + + gasSpent := big.Sub(big.NewInt(m.GasLimit), r.GasCosts.Refund) + return &api.MsgGasCost{ + Message: msg, + GasUsed: big.NewInt(r.GasUsed), + BaseFeeBurn: r.GasCosts.BaseFeeBurn, + OverEstimationBurn: r.GasCosts.OverEstimationBurn, + MinerPenalty: r.GasCosts.MinerPenalty, + MinerTip: r.GasCosts.MinerTip, + Refund: r.GasCosts.Refund, + TotalCost: big.Mul(gasSpent, m.GasFeeCap), + }, nil +} From 0edcae9e046346217c482b7d1c093d5a6dc77ac8 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Mon, 14 Sep 2020 01:23:29 -0400 Subject: [PATCH 094/885] Fixup new command --- cli/state.go | 4 ++-- documentation/en/api-methods.md | 19 ++++++++++++------- extern/storage-sealing/upgrade_queue.go | 2 ++ node/impl/full/state.go | 23 ++++++++++++++++------- 4 files changed, 32 insertions(+), 16 deletions(-) diff --git a/cli/state.go b/cli/state.go index 7729e729a..62d751204 100644 --- a/cli/state.go +++ b/cli/state.go @@ -1344,12 +1344,12 @@ var stateMsgCostCmd = &cli.Command{ } if mgc != nil { - fmt.Printf("\nMessage CID: %d", mgc.Message) + fmt.Printf("Message CID: %s", mgc.Message) fmt.Printf("\nGas Used: %d", mgc.GasUsed) fmt.Printf("\nBase Fee Burn: %d", mgc.BaseFeeBurn) fmt.Printf("\nOverestimation Burn: %d", mgc.OverEstimationBurn) fmt.Printf("\nMiner Tip: %d", mgc.MinerTip) - fmt.Printf("\nRefundd: %d", mgc.Refund) + fmt.Printf("\nRefund: %d", mgc.Refund) fmt.Printf("\nTotal Cost: %d", mgc.TotalCost) fmt.Printf("\nMiner Penalty: %d", mgc.MinerPenalty) } else { diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index ec298d635..5bba832df 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -3743,7 +3743,15 @@ Inputs: [ { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } + }, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] ] ``` @@ -3753,16 +3761,13 @@ Response: "Message": { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" }, - "Receipt": { - "ExitCode": 0, - "Return": "Ynl0ZSBhcnJheQ==", - "GasUsed": 9 - }, + "GasUsed": "0", "BaseFeeBurn": "0", "OverEstimationBurn": "0", "MinerPenalty": "0", "MinerTip": "0", - "Refund": "0" + "Refund": "0", + "TotalCost": "0" } ``` diff --git a/extern/storage-sealing/upgrade_queue.go b/extern/storage-sealing/upgrade_queue.go index 650fdc83d..af2d1827d 100644 --- a/extern/storage-sealing/upgrade_queue.go +++ b/extern/storage-sealing/upgrade_queue.go @@ -67,6 +67,8 @@ func (m *Sealing) tryUpgradeSector(ctx context.Context, params *miner.SectorPreC params.ReplaceSectorDeadline = loc.Deadline params.ReplaceSectorPartition = loc.Partition + log.Infof("replacing sector %d with %d", *replace, params.SectorNumber) + ri, err := m.api.StateSectorGetInfo(ctx, m.maddr, *replace, nil) if err != nil { log.Errorf("error calling StateSectorGetInfo for replaced sector: %+v", err) diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 1fb078001..c37ab9c3d 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -1238,15 +1238,25 @@ func (a *StateAPI) StateMsgGasCost(ctx context.Context, inputMsg cid.Cid, tsk ty return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) } } else { - msg, err := a.StateSearchMsg(ctx, inputMsg) + mlkp, err := a.StateSearchMsg(ctx, inputMsg) if err != nil { return nil, xerrors.Errorf("searching for msg %s: %w", inputMsg, err) } - - ts, err = a.Chain.GetTipSetFromKey(msg.TipSet) - if err != nil { - return nil, xerrors.Errorf("loading tipset %s: %w", msg.TipSet, err) + if mlkp == nil { + return nil, xerrors.Errorf("didn't find msg %s", inputMsg) } + + executionTs, err := a.Chain.GetTipSetFromKey(mlkp.TipSet) + if err != nil { + return nil, xerrors.Errorf("loading tipset %s: %w", mlkp.TipSet, err) + } + + ts, err = a.Chain.LoadTipSet(executionTs.Parents()) + if err != nil { + return nil, xerrors.Errorf("loading parent tipset %s: %w", mlkp.TipSet, err) + } + + msg = mlkp.Message } m, r, err := a.StateManager.Replay(ctx, ts, msg) @@ -1254,7 +1264,6 @@ func (a *StateAPI) StateMsgGasCost(ctx context.Context, inputMsg cid.Cid, tsk ty return nil, err } - gasSpent := big.Sub(big.NewInt(m.GasLimit), r.GasCosts.Refund) return &api.MsgGasCost{ Message: msg, GasUsed: big.NewInt(r.GasUsed), @@ -1263,6 +1272,6 @@ func (a *StateAPI) StateMsgGasCost(ctx context.Context, inputMsg cid.Cid, tsk ty MinerPenalty: r.GasCosts.MinerPenalty, MinerTip: r.GasCosts.MinerTip, Refund: r.GasCosts.Refund, - TotalCost: big.Mul(gasSpent, m.GasFeeCap), + TotalCost: big.Sub(m.RequiredFunds(), r.GasCosts.Refund), }, nil } From d68ec379429b1081107b78b7f0237b51d7485545 Mon Sep 17 00:00:00 2001 From: vyzo Date: Tue, 8 Sep 2020 09:33:21 +0300 Subject: [PATCH 095/885] reenable baseFee lower bound check --- chain/messagepool/messagepool.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/chain/messagepool/messagepool.go b/chain/messagepool/messagepool.go index f6799be73..033d30059 100644 --- a/chain/messagepool/messagepool.go +++ b/chain/messagepool/messagepool.go @@ -104,10 +104,6 @@ type MessagePoolEvtMessage struct { CID cid.Cid } -// this is *temporary* mutilation until we have implemented uncapped miner penalties -- it will go -// away in the next fork. -var strictBaseFeeValidation = false - func init() { // if the republish interval is too short compared to the pubsub timecache, adjust it minInterval := pubsub.TimeCacheDuration + time.Duration(build.PropagationDelaySecs) @@ -444,7 +440,7 @@ func (mp *MessagePool) verifyMsgBeforeAdd(m *types.SignedMessage, curTs *types.T // Note that for local messages, we always add them so that they can be accepted and republished // automatically. publish := local - if strictBaseFeeValidation && len(curTs.Blocks()) > 0 { + if len(curTs.Blocks()) > 0 { baseFee := curTs.Blocks()[0].ParentBaseFee baseFeeLowerBound := getBaseFeeLowerBound(baseFee) if m.Message.GasFeeCap.LessThan(baseFeeLowerBound) { From 044202b37f77677b5b725ab2b721c70593a6e3a6 Mon Sep 17 00:00:00 2001 From: vyzo Date: Fri, 11 Sep 2020 17:11:57 +0300 Subject: [PATCH 096/885] use conservative base fee lower bound factor for strict checks --- chain/messagepool/messagepool.go | 5 +++-- chain/messagepool/pruning.go | 2 +- chain/messagepool/repub.go | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/chain/messagepool/messagepool.go b/chain/messagepool/messagepool.go index 033d30059..03584d275 100644 --- a/chain/messagepool/messagepool.go +++ b/chain/messagepool/messagepool.go @@ -52,6 +52,7 @@ var RepublishInterval = time.Duration(10*build.BlockDelaySecs+build.PropagationD var minimumBaseFee = types.NewInt(uint64(build.MinimumBaseFee)) var baseFeeLowerBoundFactor = types.NewInt(10) +var baseFeeLowerBoundFactorConservative = types.NewInt(100) var MaxActorPendingMessages = 1000 @@ -442,7 +443,7 @@ func (mp *MessagePool) verifyMsgBeforeAdd(m *types.SignedMessage, curTs *types.T publish := local if len(curTs.Blocks()) > 0 { baseFee := curTs.Blocks()[0].ParentBaseFee - baseFeeLowerBound := getBaseFeeLowerBound(baseFee) + baseFeeLowerBound := getBaseFeeLowerBound(baseFee, baseFeeLowerBoundFactorConservative) if m.Message.GasFeeCap.LessThan(baseFeeLowerBound) { if local { log.Warnf("local message will not be immediately published because GasFeeCap doesn't meet the lower bound for inclusion in the next 20 blocks (GasFeeCap: %s, baseFeeLowerBound: %s)", @@ -1340,7 +1341,7 @@ func (mp *MessagePool) Clear(local bool) { } } -func getBaseFeeLowerBound(baseFee types.BigInt) types.BigInt { +func getBaseFeeLowerBound(baseFee, factor types.BigInt) types.BigInt { baseFeeLowerBound := types.BigDiv(baseFee, baseFeeLowerBoundFactor) if baseFeeLowerBound.LessThan(minimumBaseFee) { baseFeeLowerBound = minimumBaseFee diff --git a/chain/messagepool/pruning.go b/chain/messagepool/pruning.go index fd8199b89..d0e53795a 100644 --- a/chain/messagepool/pruning.go +++ b/chain/messagepool/pruning.go @@ -46,7 +46,7 @@ func (mp *MessagePool) pruneMessages(ctx context.Context, ts *types.TipSet) erro if err != nil { return xerrors.Errorf("computing basefee: %w", err) } - baseFeeLowerBound := getBaseFeeLowerBound(baseFee) + baseFeeLowerBound := getBaseFeeLowerBound(baseFee, baseFeeLowerBoundFactor) pending, _ := mp.getPendingMessages(ts, ts) diff --git a/chain/messagepool/repub.go b/chain/messagepool/repub.go index 375360bce..db31e18c2 100644 --- a/chain/messagepool/repub.go +++ b/chain/messagepool/repub.go @@ -28,7 +28,7 @@ func (mp *MessagePool) republishPendingMessages() error { mp.curTsLk.Unlock() return xerrors.Errorf("computing basefee: %w", err) } - baseFeeLowerBound := getBaseFeeLowerBound(baseFee) + baseFeeLowerBound := getBaseFeeLowerBound(baseFee, baseFeeLowerBoundFactor) pending := make(map[address.Address]map[uint64]*types.SignedMessage) mp.lk.Lock() From 0a5494dd7962a517d3531ef28366d3bf359e8720 Mon Sep 17 00:00:00 2001 From: vyzo Date: Mon, 14 Sep 2020 22:05:24 +0300 Subject: [PATCH 097/885] use the factor in getBaseFeeLowerBound --- chain/messagepool/messagepool.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chain/messagepool/messagepool.go b/chain/messagepool/messagepool.go index 03584d275..984f696da 100644 --- a/chain/messagepool/messagepool.go +++ b/chain/messagepool/messagepool.go @@ -1342,7 +1342,7 @@ func (mp *MessagePool) Clear(local bool) { } func getBaseFeeLowerBound(baseFee, factor types.BigInt) types.BigInt { - baseFeeLowerBound := types.BigDiv(baseFee, baseFeeLowerBoundFactor) + baseFeeLowerBound := types.BigDiv(baseFee, factor) if baseFeeLowerBound.LessThan(minimumBaseFee) { baseFeeLowerBound = minimumBaseFee } From 172bfacae0cdc7ee51031c71bfdbacac77ddfc7c Mon Sep 17 00:00:00 2001 From: vyzo Date: Mon, 14 Sep 2020 22:20:26 +0300 Subject: [PATCH 098/885] compute baseFee for check even in null rounds --- chain/messagepool/messagepool.go | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/chain/messagepool/messagepool.go b/chain/messagepool/messagepool.go index 984f696da..96900925f 100644 --- a/chain/messagepool/messagepool.go +++ b/chain/messagepool/messagepool.go @@ -441,18 +441,27 @@ func (mp *MessagePool) verifyMsgBeforeAdd(m *types.SignedMessage, curTs *types.T // Note that for local messages, we always add them so that they can be accepted and republished // automatically. publish := local + + var baseFee big.Int if len(curTs.Blocks()) > 0 { - baseFee := curTs.Blocks()[0].ParentBaseFee - baseFeeLowerBound := getBaseFeeLowerBound(baseFee, baseFeeLowerBoundFactorConservative) - if m.Message.GasFeeCap.LessThan(baseFeeLowerBound) { - if local { - log.Warnf("local message will not be immediately published because GasFeeCap doesn't meet the lower bound for inclusion in the next 20 blocks (GasFeeCap: %s, baseFeeLowerBound: %s)", - m.Message.GasFeeCap, baseFeeLowerBound) - publish = false - } else { - return false, xerrors.Errorf("GasFeeCap doesn't meet base fee lower bound for inclusion in the next 20 blocks (GasFeeCap: %s, baseFeeLowerBound: %s): %w", - m.Message.GasFeeCap, baseFeeLowerBound, ErrSoftValidationFailure) - } + baseFee = curTs.Blocks()[0].ParentBaseFee + } else { + var err error + baseFee, err = mp.api.ChainComputeBaseFee(context.TODO(), curTs) + if err != nil { + return false, xerrors.Errorf("computing basefee: %w", err) + } + } + + baseFeeLowerBound := getBaseFeeLowerBound(baseFee, baseFeeLowerBoundFactorConservative) + if m.Message.GasFeeCap.LessThan(baseFeeLowerBound) { + if local { + log.Warnf("local message will not be immediately published because GasFeeCap doesn't meet the lower bound for inclusion in the next 20 blocks (GasFeeCap: %s, baseFeeLowerBound: %s)", + m.Message.GasFeeCap, baseFeeLowerBound) + publish = false + } else { + return false, xerrors.Errorf("GasFeeCap doesn't meet base fee lower bound for inclusion in the next 20 blocks (GasFeeCap: %s, baseFeeLowerBound: %s): %w", + m.Message.GasFeeCap, baseFeeLowerBound, ErrSoftValidationFailure) } } From 32eeb96ce72aff598c15f16699d4916936069c32 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 14 Sep 2020 12:41:55 -0700 Subject: [PATCH 099/885] Update to specs-actors 0.9.9 This patch changes the runtime interfaces, to make it possible to abstract over them. --- chain/stmgr/forks_test.go | 6 +- chain/vm/invoker.go | 2 +- chain/vm/runtime.go | 124 ++++++++++++++------------------ chain/vm/runtime_test.go | 2 +- chain/vm/vm.go | 8 +-- conformance/chaos/actor.go | 16 ++--- conformance/chaos/actor_test.go | 6 +- go.mod | 4 +- go.sum | 8 +-- 9 files changed, 78 insertions(+), 98 deletions(-) diff --git a/chain/stmgr/forks_test.go b/chain/stmgr/forks_test.go index e96d3f316..2fc11b3d7 100644 --- a/chain/stmgr/forks_test.go +++ b/chain/stmgr/forks_test.go @@ -74,8 +74,8 @@ func (ta *testActor) Exports() []interface{} { func (ta *testActor) Constructor(rt runtime.Runtime, params *abi.EmptyValue) *abi.EmptyValue { rt.ValidateImmediateCallerAcceptAny() - rt.State().Create(&testActorState{11}) - fmt.Println("NEW ACTOR ADDRESS IS: ", rt.Message().Receiver()) + rt.StateCreate(&testActorState{11}) + fmt.Println("NEW ACTOR ADDRESS IS: ", rt.Receiver()) return abi.Empty } @@ -83,7 +83,7 @@ func (ta *testActor) Constructor(rt runtime.Runtime, params *abi.EmptyValue) *ab func (ta *testActor) TestMethod(rt runtime.Runtime, params *abi.EmptyValue) *abi.EmptyValue { rt.ValidateImmediateCallerAcceptAny() var st testActorState - rt.State().Readonly(&st) + rt.StateReadonly(&st) if rt.CurrEpoch() > testForkHeight { if st.HasUpgraded != 55 { diff --git a/chain/vm/invoker.go b/chain/vm/invoker.go index c9d22cd4c..1a9a2090a 100644 --- a/chain/vm/invoker.go +++ b/chain/vm/invoker.go @@ -64,7 +64,7 @@ func (inv *Invoker) Invoke(codeCid cid.Cid, rt runtime.Runtime, method abi.Metho code, ok := inv.builtInCode[codeCid] if !ok { - log.Errorf("no code for actor %s (Addr: %s)", codeCid, rt.Message().Receiver()) + log.Errorf("no code for actor %s (Addr: %s)", codeCid, rt.Receiver()) return nil, aerrors.Newf(exitcode.SysErrorIllegalActor, "no code for actor %s(%d)(%s)", codeCid, method, hex.EncodeToString(params)) } if method >= abi.MethodNum(len(code)) || code[method] == nil { diff --git a/chain/vm/runtime.go b/chain/vm/runtime.go index 7e9dd894b..d3757f458 100644 --- a/chain/vm/runtime.go +++ b/chain/vm/runtime.go @@ -8,7 +8,9 @@ import ( gruntime "runtime" "time" + "github.com/filecoin-project/go-state-types/cbor" "github.com/filecoin-project/go-state-types/network" + rtt "github.com/filecoin-project/go-state-types/rt" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" @@ -16,11 +18,9 @@ import ( "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/go-state-types/exitcode" "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/runtime" - vmr "github.com/filecoin-project/specs-actors/actors/runtime" + rt0 "github.com/filecoin-project/specs-actors/actors/runtime" "github.com/ipfs/go-cid" - cbor "github.com/ipfs/go-ipld-cbor" - cbg "github.com/whyrusleeping/cbor-gen" + ipldcbor "github.com/ipfs/go-ipld-cbor" "go.opencensus.io/trace" "golang.org/x/xerrors" @@ -31,20 +31,20 @@ import ( ) type Runtime struct { + types.Message + rt0.Syscalls + ctx context.Context vm *VM state *state.StateTree - vmsg vmr.Message height abi.ChainEpoch - cst cbor.IpldStore + cst ipldcbor.IpldStore pricelist Pricelist gasAvailable int64 gasUsed int64 - sys runtime.Syscalls - // address that started invoke chain origin address.Address originNonce uint64 @@ -85,11 +85,11 @@ type notFoundErr interface { IsNotFound() bool } -func (rt *Runtime) Get(c cid.Cid, o vmr.CBORUnmarshaler) bool { +func (rt *Runtime) StoreGet(c cid.Cid, o cbor.Unmarshaler) bool { if err := rt.cst.Get(context.TODO(), c, o); err != nil { var nfe notFoundErr if xerrors.As(err, &nfe) && nfe.IsNotFound() { - if xerrors.As(err, new(cbor.SerializationError)) { + if xerrors.As(err, new(ipldcbor.SerializationError)) { panic(aerrors.Newf(exitcode.ErrSerialization, "failed to unmarshal cbor object %s", err)) } return false @@ -100,10 +100,10 @@ func (rt *Runtime) Get(c cid.Cid, o vmr.CBORUnmarshaler) bool { return true } -func (rt *Runtime) Put(x vmr.CBORMarshaler) cid.Cid { +func (rt *Runtime) StorePut(x cbor.Marshaler) cid.Cid { c, err := rt.cst.Put(context.TODO(), x) if err != nil { - if xerrors.As(err, new(cbor.SerializationError)) { + if xerrors.As(err, new(ipldcbor.SerializationError)) { panic(aerrors.Newf(exitcode.ErrSerialization, "failed to marshal cbor object %s", err)) } panic(aerrors.Fatalf("failed to put cbor object: %s", err)) @@ -111,7 +111,7 @@ func (rt *Runtime) Put(x vmr.CBORMarshaler) cid.Cid { return c } -var _ vmr.Runtime = (*Runtime)(nil) +var _ rt0.Runtime = (*Runtime)(nil) func (rt *Runtime) shimCall(f func() interface{}) (rval []byte, aerr aerrors.ActorError) { defer func() { @@ -139,7 +139,7 @@ func (rt *Runtime) shimCall(f func() interface{}) (rval []byte, aerr aerrors.Act return ret, nil case *abi.EmptyValue: return nil, nil - case cbg.CBORMarshaler: + case cbor.Marshaler: buf := new(bytes.Buffer) if err := ret.MarshalCBOR(buf); err != nil { return nil, aerrors.Absorb(err, 2, "failed to marshal response to cbor") @@ -152,17 +152,13 @@ func (rt *Runtime) shimCall(f func() interface{}) (rval []byte, aerr aerrors.Act } } -func (rt *Runtime) Message() vmr.Message { - return rt.vmsg -} - func (rt *Runtime) ValidateImmediateCallerAcceptAny() { rt.abortIfAlreadyValidated() return } func (rt *Runtime) CurrentBalance() abi.TokenAmount { - b, err := rt.GetBalance(rt.Message().Receiver()) + b, err := rt.GetBalance(rt.Receiver()) if err != nil { rt.Abortf(err.RetCode(), "get current balance: %v", err) } @@ -198,10 +194,6 @@ func (rt *Runtime) GetRandomnessFromBeacon(personalization crypto.DomainSeparati return res } -func (rt *Runtime) Store() vmr.Store { - return rt -} - func (rt *Runtime) NewActorAddress() address.Address { var b bytes.Buffer oa, _ := ResolveToKeyAddr(rt.vm.cstate, rt.vm.cst, rt.origin) @@ -258,7 +250,7 @@ func (rt *Runtime) CreateActor(codeID cid.Cid, address address.Address) { // May only be called by the actor itself. func (rt *Runtime) DeleteActor(beneficiary address.Address) { rt.chargeGas(rt.Pricelist().OnDeleteActor()) - act, err := rt.state.GetActor(rt.Message().Receiver()) + act, err := rt.state.GetActor(rt.Receiver()) if err != nil { if xerrors.Is(err, types.ErrActorNotFound) { rt.Abortf(exitcode.SysErrorIllegalActor, "failed to load actor in delete actor: %s", err) @@ -267,36 +259,32 @@ func (rt *Runtime) DeleteActor(beneficiary address.Address) { } if !act.Balance.IsZero() { // Transfer the executing actor's balance to the beneficiary - if err := rt.vm.transfer(rt.Message().Receiver(), beneficiary, act.Balance); err != nil { + if err := rt.vm.transfer(rt.Receiver(), beneficiary, act.Balance); err != nil { panic(aerrors.Fatalf("failed to transfer balance to beneficiary actor: %s", err)) } } // Delete the executing actor - if err := rt.state.DeleteActor(rt.Message().Receiver()); err != nil { + if err := rt.state.DeleteActor(rt.Receiver()); err != nil { panic(aerrors.Fatalf("failed to delete actor: %s", err)) } _ = rt.chargeGasSafe(gasOnActorExec) } -func (rt *Runtime) Syscalls() vmr.Syscalls { - return rt.sys -} - -func (rt *Runtime) StartSpan(name string) vmr.TraceSpan { +func (rt *Runtime) StartSpan(name string) func() { panic("implement me") } func (rt *Runtime) ValidateImmediateCallerIs(as ...address.Address) { rt.abortIfAlreadyValidated() - imm := rt.Message().Caller() + imm := rt.Caller() for _, a := range as { if imm == a { return } } - rt.Abortf(exitcode.SysErrForbidden, "caller %s is not one of %s", rt.Message().Caller(), as) + rt.Abortf(exitcode.SysErrForbidden, "caller %s is not one of %s", rt.Caller(), as) } func (rt *Runtime) Context() context.Context { @@ -314,7 +302,7 @@ func (rt *Runtime) AbortStateMsg(msg string) { func (rt *Runtime) ValidateImmediateCallerType(ts ...cid.Cid) { rt.abortIfAlreadyValidated() - callerCid, ok := rt.GetActorCodeCID(rt.Message().Caller()) + callerCid, ok := rt.GetActorCodeCID(rt.Caller()) if !ok { panic(aerrors.Fatalf("failed to lookup code cid for caller")) } @@ -334,11 +322,11 @@ type dumbWrapperType struct { val []byte } -func (dwt *dumbWrapperType) Into(um vmr.CBORUnmarshaler) error { +func (dwt *dumbWrapperType) Into(um cbor.Unmarshaler) error { return um.UnmarshalCBOR(bytes.NewReader(dwt.val)) } -func (rt *Runtime) Send(to address.Address, method abi.MethodNum, m vmr.CBORMarshaler, value abi.TokenAmount) (vmr.SendReturn, exitcode.ExitCode) { +func (rt *Runtime) Send(to address.Address, method abi.MethodNum, m cbor.Marshaler, value abi.TokenAmount, out cbor.Er) exitcode.ExitCode { if !rt.allowInternal { rt.Abortf(exitcode.SysErrorIllegalActor, "runtime.Send() is currently disallowed") } @@ -351,16 +339,20 @@ func (rt *Runtime) Send(to address.Address, method abi.MethodNum, m vmr.CBORMars params = buf.Bytes() } - ret, err := rt.internalSend(rt.Message().Receiver(), to, method, value, params) + ret, err := rt.internalSend(rt.Receiver(), to, method, value, params) if err != nil { if err.IsFatal() { panic(err) } log.Warnf("vmctx send failed: to: %s, method: %d: ret: %d, err: %s", to, method, ret, err) - return &dumbWrapperType{nil}, err.RetCode() + return err.RetCode() } _ = rt.chargeGasSafe(gasOnActorExec) - return &dumbWrapperType{ret}, 0 + + if err := out.UnmarshalCBOR(bytes.NewReader(ret)); err != nil { + rt.Abortf(exitcode.ErrSerialization, "failed to unmarshal return value: %s", err) + } + return 0 } func (rt *Runtime) internalSend(from, to address.Address, method abi.MethodNum, value types.BigInt, params []byte) ([]byte, aerrors.ActorError) { @@ -404,49 +396,41 @@ func (rt *Runtime) internalSend(from, to address.Address, method abi.MethodNum, return ret, errSend } -func (rt *Runtime) State() vmr.StateHandle { - return &shimStateHandle{rt: rt} -} - -type shimStateHandle struct { - rt *Runtime -} - -func (ssh *shimStateHandle) Create(obj vmr.CBORMarshaler) { - c := ssh.rt.Put(obj) - err := ssh.rt.stateCommit(EmptyObjectCid, c) +func (rt *Runtime) StateCreate(obj cbor.Marshaler) { + c := rt.StorePut(obj) + err := rt.stateCommit(EmptyObjectCid, c) if err != nil { panic(fmt.Errorf("failed to commit state after creating object: %w", err)) } } -func (ssh *shimStateHandle) Readonly(obj vmr.CBORUnmarshaler) { - act, err := ssh.rt.state.GetActor(ssh.rt.Message().Receiver()) +func (rt *Runtime) StateReadonly(obj cbor.Unmarshaler) { + act, err := rt.state.GetActor(rt.Receiver()) if err != nil { - ssh.rt.Abortf(exitcode.SysErrorIllegalArgument, "failed to get actor for Readonly state: %s", err) + rt.Abortf(exitcode.SysErrorIllegalArgument, "failed to get actor for Readonly state: %s", err) } - ssh.rt.Get(act.Head, obj) + rt.StoreGet(act.Head, obj) } -func (ssh *shimStateHandle) Transaction(obj vmr.CBORer, f func()) { +func (rt *Runtime) StateTransaction(obj cbor.Er, f func()) { if obj == nil { - ssh.rt.Abortf(exitcode.SysErrorIllegalActor, "Must not pass nil to Transaction()") + rt.Abortf(exitcode.SysErrorIllegalActor, "Must not pass nil to Transaction()") } - act, err := ssh.rt.state.GetActor(ssh.rt.Message().Receiver()) + act, err := rt.state.GetActor(rt.Receiver()) if err != nil { - ssh.rt.Abortf(exitcode.SysErrorIllegalActor, "failed to get actor for Transaction: %s", err) + rt.Abortf(exitcode.SysErrorIllegalActor, "failed to get actor for Transaction: %s", err) } baseState := act.Head - ssh.rt.Get(baseState, obj) + rt.StoreGet(baseState, obj) - ssh.rt.allowInternal = false + rt.allowInternal = false f() - ssh.rt.allowInternal = true + rt.allowInternal = true - c := ssh.rt.Put(obj) + c := rt.StorePut(obj) - err = ssh.rt.stateCommit(baseState, c) + err = rt.stateCommit(baseState, c) if err != nil { panic(fmt.Errorf("failed to commit state after transaction: %w", err)) } @@ -466,7 +450,7 @@ func (rt *Runtime) GetBalance(a address.Address) (types.BigInt, aerrors.ActorErr func (rt *Runtime) stateCommit(oldh, newh cid.Cid) aerrors.ActorError { // TODO: we can make this more efficient in the future... - act, err := rt.state.GetActor(rt.Message().Receiver()) + act, err := rt.state.GetActor(rt.Receiver()) if err != nil { return aerrors.Escalate(err, "failed to get actor to commit state") } @@ -477,7 +461,7 @@ func (rt *Runtime) stateCommit(oldh, newh cid.Cid) aerrors.ActorError { act.Head = newh - if err := rt.state.SetActor(rt.Message().Receiver(), act); err != nil { + if err := rt.state.SetActor(rt.Receiver(), act); err != nil { return aerrors.Fatalf("failed to set actor in commit state: %s", err) } @@ -572,15 +556,15 @@ func (rt *Runtime) abortIfAlreadyValidated() { rt.callerValidated = true } -func (rt *Runtime) Log(level vmr.LogLevel, msg string, args ...interface{}) { +func (rt *Runtime) Log(level rtt.LogLevel, msg string, args ...interface{}) { switch level { - case vmr.DEBUG: + case rtt.DEBUG: actorLog.Debugf(msg, args...) - case vmr.INFO: + case rtt.INFO: actorLog.Infof(msg, args...) - case vmr.WARN: + case rtt.WARN: actorLog.Warnf(msg, args...) - case vmr.ERROR: + case rtt.ERROR: actorLog.Errorf(msg, args...) } } diff --git a/chain/vm/runtime_test.go b/chain/vm/runtime_test.go index 2f9704394..c22a8b615 100644 --- a/chain/vm/runtime_test.go +++ b/chain/vm/runtime_test.go @@ -42,6 +42,6 @@ func TestRuntimePutErrors(t *testing.T) { cst: cbor.NewCborStore(nil), } - rt.Put(&NotAVeryGoodMarshaler{}) + rt.StorePut(&NotAVeryGoodMarshaler{}) t.Error("expected panic") } diff --git a/chain/vm/vm.go b/chain/vm/vm.go index e389a3531..c9acc8658 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -116,7 +116,7 @@ func (vm *VM) makeRuntime(ctx context.Context, msg *types.Message, origin addres Blocks: &gasChargingBlocks{rt.chargeGasFunc(2), rt.pricelist, vm.cst.Blocks}, Atlas: vm.cst.Atlas, } - rt.sys = pricedSyscalls{ + rt.Syscalls = pricedSyscalls{ under: vm.Syscalls(ctx, vm.cstate, rt.cst), chargeGas: rt.chargeGasFunc(1), pl: rt.pricelist, @@ -128,7 +128,7 @@ func (vm *VM) makeRuntime(ctx context.Context, msg *types.Message, origin addres rt.Abortf(exitcode.SysErrInvalidReceiver, "resolve msg.From address failed") } vmm.From = resF - rt.vmsg = &vmm + rt.Message = vmm return rt } @@ -700,9 +700,9 @@ func (vm *VM) Invoke(act *types.Actor, rt *Runtime, method abi.MethodNum, params defer span.End() if span.IsRecordingEvents() { span.AddAttributes( - trace.StringAttribute("to", rt.Message().Receiver().String()), + trace.StringAttribute("to", rt.Receiver().String()), trace.Int64Attribute("method", int64(method)), - trace.StringAttribute("value", rt.Message().ValueReceived().String()), + trace.StringAttribute("value", rt.ValueReceived().String()), ) } diff --git a/conformance/chaos/actor.go b/conformance/chaos/actor.go index b9a181ba7..daff26694 100644 --- a/conformance/chaos/actor.go +++ b/conformance/chaos/actor.go @@ -100,18 +100,14 @@ type SendReturn struct { // passed parameters. func (a Actor) Send(rt runtime.Runtime, args *SendArgs) *SendReturn { rt.ValidateImmediateCallerAcceptAny() - ret, code := rt.Send( + var out runtime.CBORBytes + code := rt.Send( args.To, args.Method, runtime.CBORBytes(args.Params), args.Value, + &out, ) - var out runtime.CBORBytes - if ret != nil { - if err := ret.Into(&out); err != nil { - rt.Abortf(exitcode.ErrIllegalState, "failed to unmarshal send return: %v", err) - } - } return &SendReturn{ Return: out, Code: code, @@ -217,14 +213,14 @@ func (a Actor) MutateState(rt runtime.Runtime, args *MutateStateArgs) *abi.Empty var st State switch args.Branch { case MutateInTransaction: - rt.State().Transaction(&st, func() { + rt.StateTransaction(&st, func() { st.Value = args.Value }) case MutateReadonly: - rt.State().Readonly(&st) + rt.StateReadonly(&st) st.Value = args.Value case MutateAfterTransaction: - rt.State().Transaction(&st, func() { + rt.StateTransaction(&st, func() { st.Value = args.Value + "-in" }) st.Value = args.Value diff --git a/conformance/chaos/actor_test.go b/conformance/chaos/actor_test.go index a08267022..67ced2899 100644 --- a/conformance/chaos/actor_test.go +++ b/conformance/chaos/actor_test.go @@ -46,7 +46,7 @@ func TestMutateStateInTransaction(t *testing.T) { var a Actor rt.ExpectValidateCallerAny() - rt.Create(&State{}) + rt.StateCreate(&State{}) val := "__mutstat test" rt.Call(a.MutateState, &MutateStateArgs{ @@ -72,7 +72,7 @@ func TestMutateStateAfterTransaction(t *testing.T) { var a Actor rt.ExpectValidateCallerAny() - rt.Create(&State{}) + rt.StateCreate(&State{}) val := "__mutstat test" rt.Call(a.MutateState, &MutateStateArgs{ @@ -99,7 +99,7 @@ func TestMutateStateReadonly(t *testing.T) { var a Actor rt.ExpectValidateCallerAny() - rt.Create(&State{}) + rt.StateCreate(&State{}) val := "__mutstat test" rt.Call(a.MutateState, &MutateStateArgs{ diff --git a/go.mod b/go.mod index 4cedea65f..5470e226a 100644 --- a/go.mod +++ b/go.mod @@ -32,11 +32,11 @@ require ( github.com/filecoin-project/go-multistore v0.0.3 github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 github.com/filecoin-project/go-paramfetch v0.0.2-0.20200701152213-3e0f0afdc261 - github.com/filecoin-project/go-state-types v0.0.0-20200909080127-001afaca718c + github.com/filecoin-project/go-state-types v0.0.0-20200911004822-964d6c679cfc github.com/filecoin-project/go-statemachine v0.0.0-20200813232949-df9b130df370 github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b - github.com/filecoin-project/specs-actors v0.9.8 + github.com/filecoin-project/specs-actors v0.9.9 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 github.com/filecoin-project/test-vectors/schema v0.0.1 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 diff --git a/go.sum b/go.sum index 215eec431..5e479db82 100644 --- a/go.sum +++ b/go.sum @@ -241,8 +241,8 @@ github.com/filecoin-project/go-paramfetch v0.0.2-0.20200701152213-3e0f0afdc261/g github.com/filecoin-project/go-state-types v0.0.0-20200903145444-247639ffa6ad/go.mod h1:IQ0MBPnonv35CJHtWSN3YY1Hz2gkPru1Q9qoaYLxx9I= github.com/filecoin-project/go-state-types v0.0.0-20200904021452-1883f36ca2f4/go.mod h1:IQ0MBPnonv35CJHtWSN3YY1Hz2gkPru1Q9qoaYLxx9I= github.com/filecoin-project/go-state-types v0.0.0-20200905071437-95828685f9df/go.mod h1:IQ0MBPnonv35CJHtWSN3YY1Hz2gkPru1Q9qoaYLxx9I= -github.com/filecoin-project/go-state-types v0.0.0-20200909080127-001afaca718c h1:HHRMFpU8OrODDUja5NmGWNBAVGoSy4MRjxgZa+a0qIw= -github.com/filecoin-project/go-state-types v0.0.0-20200909080127-001afaca718c/go.mod h1:IQ0MBPnonv35CJHtWSN3YY1Hz2gkPru1Q9qoaYLxx9I= +github.com/filecoin-project/go-state-types v0.0.0-20200911004822-964d6c679cfc h1:1vr/LoqGq5m5g37Q3sNSAjfwF1uJY0zmiHcvnxY6hik= +github.com/filecoin-project/go-state-types v0.0.0-20200911004822-964d6c679cfc/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-statemachine v0.0.0-20200714194326-a77c3ae20989/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= github.com/filecoin-project/go-statemachine v0.0.0-20200813232949-df9b130df370 h1:Jbburj7Ih2iaJ/o5Q9A+EAeTabME6YII7FLi9SKUf5c= github.com/filecoin-project/go-statemachine v0.0.0-20200813232949-df9b130df370/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= @@ -252,8 +252,8 @@ github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/go.mod h1:Q0GQOBtKf1oE10eSXSlhN45kDBdGvEcVOqMiffqX+N8= github.com/filecoin-project/specs-actors v0.9.4/go.mod h1:BStZQzx5x7TmCkLv0Bpa07U6cPKol6fd3w9KjMPZ6Z4= github.com/filecoin-project/specs-actors v0.9.7/go.mod h1:wM2z+kwqYgXn5Z7scV1YHLyd1Q1cy0R8HfTIWQ0BFGU= -github.com/filecoin-project/specs-actors v0.9.8 h1:45fnx/BsseFL3CtvSoR6CszFY26TFtsh9AHwCW2vkg8= -github.com/filecoin-project/specs-actors v0.9.8/go.mod h1:xFObDoWPySBNTNBrGXVVrutmgSZH/mMo46Q1bec/0hw= +github.com/filecoin-project/specs-actors v0.9.9 h1:hc1iCks6dv7mpGQXcbYU3hL1WfBRZaVTYE4x2d1l2yw= +github.com/filecoin-project/specs-actors v0.9.9/go.mod h1:czlvLQGEX0fjLLfdNHD7xLymy6L3n7aQzRWzsYGf+ys= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= github.com/filecoin-project/test-vectors/schema v0.0.1 h1:5fNF76nl4qolEvcIsjc0kUADlTMVHO73tW4kXXPnsus= From dd841f32dbccc7dbf7360ba17dcd94907be888a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Mon, 14 Sep 2020 21:58:59 +0100 Subject: [PATCH 100/885] syncer: make SyncManager an interface. --- chain/sync.go | 27 ++++++++-------- chain/sync_manager.go | 64 ++++++++++++++++++++++++++++---------- chain/sync_manager_test.go | 14 ++++----- node/builder.go | 3 ++ node/modules/chain.go | 27 ++++++++++++++-- 5 files changed, 97 insertions(+), 38 deletions(-) diff --git a/chain/sync.go b/chain/sync.go index 9864600dd..ae3f8ee80 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -58,7 +58,14 @@ import ( // the theoretical max height based on systime are quickly rejected const MaxHeightDrift = 5 -var defaultMessageFetchWindowSize = 200 +var ( + // LocalIncoming is the _local_ pubsub (unrelated to libp2p pubsub) topic + // where the Syncer publishes candidate chain heads to be synced. + LocalIncoming = "incoming" + + log = logging.Logger("chain") + defaultMessageFetchWindowSize = 200 +) func init() { if s := os.Getenv("LOTUS_BSYNC_MSG_WINDOW"); s != "" { @@ -71,10 +78,6 @@ func init() { } } -var log = logging.Logger("chain") - -var LocalIncoming = "incoming" - // Syncer is in charge of running the chain synchronization logic. As such, it // is tasked with these functions, amongst others: // @@ -119,7 +122,7 @@ type Syncer struct { self peer.ID - syncmgr *SyncManager + syncmgr SyncManager connmgr connmgr.ConnManager @@ -140,8 +143,10 @@ type Syncer struct { ds dtypes.MetadataDS } +type SyncManagerCtor func(syncFn SyncFunc) SyncManager + // NewSyncer creates a new Syncer object. -func NewSyncer(ds dtypes.MetadataDS, sm *stmgr.StateManager, exchange exchange.Client, connmgr connmgr.ConnManager, self peer.ID, beacon beacon.Schedule, verifier ffiwrapper.Verifier) (*Syncer, error) { +func NewSyncer(ds dtypes.MetadataDS, sm *stmgr.StateManager, exchange exchange.Client, syncMgrCtor SyncManagerCtor, connmgr connmgr.ConnManager, self peer.ID, beacon beacon.Schedule, verifier ffiwrapper.Verifier) (*Syncer, error) { gen, err := sm.ChainStore().GetGenesis() if err != nil { return nil, xerrors.Errorf("getting genesis block: %w", err) @@ -181,7 +186,7 @@ func NewSyncer(ds dtypes.MetadataDS, sm *stmgr.StateManager, exchange exchange.C log.Warn("*********************************************************************************************") } - s.syncmgr = NewSyncManager(s.Sync) + s.syncmgr = syncMgrCtor(s.Sync) return s, nil } @@ -1665,11 +1670,7 @@ func VerifyElectionPoStVRF(ctx context.Context, worker address.Address, rand []b } func (syncer *Syncer) State() []SyncerState { - var out []SyncerState - for _, ss := range syncer.syncmgr.syncStates { - out = append(out, ss.Snapshot()) - } - return out + return syncer.syncmgr.State() } // MarkBad manually adds a block to the "bad blocks" cache. diff --git a/chain/sync_manager.go b/chain/sync_manager.go index 8c77b47c5..811092bc7 100644 --- a/chain/sync_manager.go +++ b/chain/sync_manager.go @@ -20,7 +20,28 @@ const ( type SyncFunc func(context.Context, *types.TipSet) error -type SyncManager struct { +// SyncManager manages the chain synchronization process, both at bootstrap time +// and during ongoing operation. +// +// It receives candidate chain heads in the form of tipsets from peers, +// and schedules them onto sync workers, deduplicating processing for +// already-active syncs. +type SyncManager interface { + // Start starts the SyncManager. + Start() + + // Stop stops the SyncManager. + Stop() + + // SetPeerHead informs the SyncManager that the supplied peer reported the + // supplied tipset. + SetPeerHead(ctx context.Context, p peer.ID, ts *types.TipSet) + + // State retrieves the state of the sync workers. + State() []SyncerState +} + +type syncManager struct { lk sync.Mutex peerHeads map[peer.ID]*types.TipSet @@ -48,6 +69,8 @@ type SyncManager struct { workerChan chan *types.TipSet } +var _ SyncManager = (*syncManager)(nil) + type syncResult struct { ts *types.TipSet success bool @@ -55,8 +78,8 @@ type syncResult struct { const syncWorkerCount = 3 -func NewSyncManager(sync SyncFunc) *SyncManager { - return &SyncManager{ +func NewSyncManager(sync SyncFunc) SyncManager { + return &syncManager{ bspThresh: 1, peerHeads: make(map[peer.ID]*types.TipSet), syncTargets: make(chan *types.TipSet), @@ -69,18 +92,18 @@ func NewSyncManager(sync SyncFunc) *SyncManager { } } -func (sm *SyncManager) Start() { +func (sm *syncManager) Start() { go sm.syncScheduler() for i := 0; i < syncWorkerCount; i++ { go sm.syncWorker(i) } } -func (sm *SyncManager) Stop() { +func (sm *syncManager) Stop() { close(sm.stop) } -func (sm *SyncManager) SetPeerHead(ctx context.Context, p peer.ID, ts *types.TipSet) { +func (sm *syncManager) SetPeerHead(ctx context.Context, p peer.ID, ts *types.TipSet) { sm.lk.Lock() defer sm.lk.Unlock() sm.peerHeads[p] = ts @@ -105,6 +128,14 @@ func (sm *SyncManager) SetPeerHead(ctx context.Context, p peer.ID, ts *types.Tip sm.incomingTipSets <- ts } +func (sm *syncManager) State() []SyncerState { + ret := make([]SyncerState, 0, len(sm.syncStates)) + for _, s := range sm.syncStates { + ret = append(ret, s.Snapshot()) + } + return ret +} + type syncBucketSet struct { buckets []*syncTargetBucket } @@ -234,7 +265,7 @@ func (stb *syncTargetBucket) heaviestTipSet() *types.TipSet { return best } -func (sm *SyncManager) selectSyncTarget() (*types.TipSet, error) { +func (sm *syncManager) selectSyncTarget() (*types.TipSet, error) { var buckets syncBucketSet var peerHeads []*types.TipSet @@ -258,7 +289,7 @@ func (sm *SyncManager) selectSyncTarget() (*types.TipSet, error) { return buckets.Heaviest(), nil } -func (sm *SyncManager) syncScheduler() { +func (sm *syncManager) syncScheduler() { for { select { @@ -280,7 +311,7 @@ func (sm *SyncManager) syncScheduler() { } } -func (sm *SyncManager) scheduleIncoming(ts *types.TipSet) { +func (sm *syncManager) scheduleIncoming(ts *types.TipSet) { log.Debug("scheduling incoming tipset sync: ", ts.Cids()) if sm.getBootstrapState() == BSStateSelected { sm.setBootstrapState(BSStateScheduled) @@ -328,10 +359,11 @@ func (sm *SyncManager) scheduleIncoming(ts *types.TipSet) { } } -func (sm *SyncManager) scheduleProcessResult(res *syncResult) { +func (sm *syncManager) scheduleProcessResult(res *syncResult) { if res.success && sm.getBootstrapState() != BSStateComplete { sm.setBootstrapState(BSStateComplete) } + delete(sm.activeSyncs, res.ts.Key()) relbucket := sm.activeSyncTips.PopRelated(res.ts) if relbucket != nil { @@ -360,7 +392,7 @@ func (sm *SyncManager) scheduleProcessResult(res *syncResult) { } } -func (sm *SyncManager) scheduleWorkSent() { +func (sm *syncManager) scheduleWorkSent() { hts := sm.nextSyncTarget.heaviestTipSet() sm.activeSyncs[hts.Key()] = hts @@ -372,7 +404,7 @@ func (sm *SyncManager) scheduleWorkSent() { } } -func (sm *SyncManager) syncWorker(id int) { +func (sm *syncManager) syncWorker(id int) { ss := &SyncerState{} sm.syncStates[id] = ss for { @@ -397,7 +429,7 @@ func (sm *SyncManager) syncWorker(id int) { } } -func (sm *SyncManager) syncedPeerCount() int { +func (sm *syncManager) syncedPeerCount() int { var count int for _, ts := range sm.peerHeads { if ts.Height() > 0 { @@ -407,19 +439,19 @@ func (sm *SyncManager) syncedPeerCount() int { return count } -func (sm *SyncManager) getBootstrapState() int { +func (sm *syncManager) getBootstrapState() int { sm.bssLk.Lock() defer sm.bssLk.Unlock() return sm.bootstrapState } -func (sm *SyncManager) setBootstrapState(v int) { +func (sm *syncManager) setBootstrapState(v int) { sm.bssLk.Lock() defer sm.bssLk.Unlock() sm.bootstrapState = v } -func (sm *SyncManager) IsBootstrapped() bool { +func (sm *syncManager) IsBootstrapped() bool { sm.bssLk.Lock() defer sm.bssLk.Unlock() return sm.bootstrapState == BSStateComplete diff --git a/chain/sync_manager_test.go b/chain/sync_manager_test.go index ca2ced856..269b3a62e 100644 --- a/chain/sync_manager_test.go +++ b/chain/sync_manager_test.go @@ -17,7 +17,7 @@ type syncOp struct { done func() } -func runSyncMgrTest(t *testing.T, tname string, thresh int, tf func(*testing.T, *SyncManager, chan *syncOp)) { +func runSyncMgrTest(t *testing.T, tname string, thresh int, tf func(*testing.T, *syncManager, chan *syncOp)) { syncTargets := make(chan *syncOp) sm := NewSyncManager(func(ctx context.Context, ts *types.TipSet) error { ch := make(chan struct{}) @@ -27,7 +27,7 @@ func runSyncMgrTest(t *testing.T, tname string, thresh int, tf func(*testing.T, } <-ch return nil - }) + }).(*syncManager) sm.bspThresh = thresh sm.Start() @@ -77,12 +77,12 @@ func TestSyncManager(t *testing.T) { c3 := mock.TipSet(mock.MkBlock(b, 3, 5)) d := mock.TipSet(mock.MkBlock(c1, 4, 5)) - runSyncMgrTest(t, "testBootstrap", 1, func(t *testing.T, sm *SyncManager, stc chan *syncOp) { + runSyncMgrTest(t, "testBootstrap", 1, func(t *testing.T, sm *syncManager, stc chan *syncOp) { sm.SetPeerHead(ctx, "peer1", c1) assertGetSyncOp(t, stc, c1) }) - runSyncMgrTest(t, "testBootstrap", 2, func(t *testing.T, sm *SyncManager, stc chan *syncOp) { + runSyncMgrTest(t, "testBootstrap", 2, func(t *testing.T, sm *syncManager, stc chan *syncOp) { sm.SetPeerHead(ctx, "peer1", c1) assertNoOp(t, stc) @@ -90,7 +90,7 @@ func TestSyncManager(t *testing.T) { assertGetSyncOp(t, stc, c1) }) - runSyncMgrTest(t, "testSyncAfterBootstrap", 1, func(t *testing.T, sm *SyncManager, stc chan *syncOp) { + runSyncMgrTest(t, "testSyncAfterBootstrap", 1, func(t *testing.T, sm *syncManager, stc chan *syncOp) { sm.SetPeerHead(ctx, "peer1", b) assertGetSyncOp(t, stc, b) @@ -101,7 +101,7 @@ func TestSyncManager(t *testing.T) { assertGetSyncOp(t, stc, c2) }) - runSyncMgrTest(t, "testCoalescing", 1, func(t *testing.T, sm *SyncManager, stc chan *syncOp) { + runSyncMgrTest(t, "testCoalescing", 1, func(t *testing.T, sm *syncManager, stc chan *syncOp) { sm.SetPeerHead(ctx, "peer1", a) assertGetSyncOp(t, stc, a) @@ -122,7 +122,7 @@ func TestSyncManager(t *testing.T) { assertGetSyncOp(t, stc, d) }) - runSyncMgrTest(t, "testSyncIncomingTipset", 1, func(t *testing.T, sm *SyncManager, stc chan *syncOp) { + runSyncMgrTest(t, "testSyncIncomingTipset", 1, func(t *testing.T, sm *syncManager, stc chan *syncOp) { sm.SetPeerHead(ctx, "peer1", a) assertGetSyncOp(t, stc, a) diff --git a/node/builder.go b/node/builder.go index 6c01258a6..bf11145df 100644 --- a/node/builder.go +++ b/node/builder.go @@ -242,6 +242,9 @@ func Online() Option { Override(new(dtypes.ChainBlockService), modules.ChainBlockService), // Filecoin services + // We don't want the SyncManagerCtor to be used as an fx constructor, but rather as a value. + // It will be called implicitly by the Syncer constructor. + Override(new(chain.SyncManagerCtor), func() chain.SyncManagerCtor { return chain.NewSyncManager }), Override(new(*chain.Syncer), modules.NewSyncer), Override(new(exchange.Client), exchange.NewClient), Override(new(*messagepool.MessagePool), modules.MessagePool), diff --git a/node/modules/chain.go b/node/modules/chain.go index 7b7e03e44..5eda51078 100644 --- a/node/modules/chain.go +++ b/node/modules/chain.go @@ -163,8 +163,31 @@ func NetworkName(mctx helpers.MetricsCtx, lc fx.Lifecycle, cs *store.ChainStore, return netName, err } -func NewSyncer(lc fx.Lifecycle, ds dtypes.MetadataDS, sm *stmgr.StateManager, exchange exchange.Client, h host.Host, beacon beacon.Schedule, verifier ffiwrapper.Verifier) (*chain.Syncer, error) { - syncer, err := chain.NewSyncer(ds, sm, exchange, h.ConnManager(), h.ID(), beacon, verifier) +type SyncerParams struct { + fx.In + + Lifecycle fx.Lifecycle + MetadataDS dtypes.MetadataDS + StateManager *stmgr.StateManager + ChainXchg exchange.Client + SyncMgrCtor chain.SyncManagerCtor + Host host.Host + Beacon beacon.Schedule + Verifier ffiwrapper.Verifier +} + +func NewSyncer(params SyncerParams) (*chain.Syncer, error) { + var ( + lc = params.Lifecycle + ds = params.MetadataDS + sm = params.StateManager + ex = params.ChainXchg + smCtor = params.SyncMgrCtor + h = params.Host + b = params.Beacon + v = params.Verifier + ) + syncer, err := chain.NewSyncer(ds, sm, ex, smCtor, h.ConnManager(), h.ID(), b, v) if err != nil { return nil, err } From 9804310cc8243dc34e2944fc4ee839fbf3251c2c Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 14 Sep 2020 13:31:03 -0700 Subject: [PATCH 101/885] Update specs-actors to 0.9.10 Moves the Keyer type (and helpers) into go-state-types. --- chain/events/state/diff_adt.go | 3 ++- chain/events/state/diff_adt_test.go | 33 +++++++++++++------------ chain/events/state/predicates.go | 8 +++--- chain/gen/genesis/t01_init.go | 9 ++++--- chain/state/statetree.go | 7 +++--- chain/stmgr/utils.go | 4 +-- chain/sync.go | 2 +- cli/chain.go | 2 +- cmd/lotus-chainwatch/processor/miner.go | 2 +- cmd/lotus-shed/verifreg.go | 3 ++- go.mod | 2 +- go.sum | 4 +-- node/impl/full/chain.go | 4 +-- node/impl/full/state.go | 6 ++--- storage/adapter_storage_miner.go | 2 +- 15 files changed, 48 insertions(+), 43 deletions(-) diff --git a/chain/events/state/diff_adt.go b/chain/events/state/diff_adt.go index 39d7e8556..519c61c2d 100644 --- a/chain/events/state/diff_adt.go +++ b/chain/events/state/diff_adt.go @@ -3,6 +3,7 @@ package state import ( "bytes" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/specs-actors/actors/util/adt" typegen "github.com/whyrusleeping/cbor-gen" ) @@ -69,7 +70,7 @@ func DiffAdtArray(preArr, curArr *adt.Array, out AdtArrayDiff) error { // Modify should be called when a value is modified in the map // Remove should be called when a value is removed from the map type AdtMapDiff interface { - AsKey(key string) (adt.Keyer, error) + AsKey(key string) (abi.Keyer, error) Add(key string, val *typegen.Deferred) error Modify(key string, from, to *typegen.Deferred) error Remove(key string, val *typegen.Deferred) error diff --git a/chain/events/state/diff_adt_test.go b/chain/events/state/diff_adt_test.go index 56a03bf33..55926afb9 100644 --- a/chain/events/state/diff_adt_test.go +++ b/chain/events/state/diff_adt_test.go @@ -11,6 +11,7 @@ import ( cbornode "github.com/ipfs/go-ipld-cbor" typegen "github.com/whyrusleeping/cbor-gen" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/specs-actors/actors/runtime" "github.com/filecoin-project/specs-actors/actors/util/adt" @@ -78,21 +79,21 @@ func TestDiffAdtMap(t *testing.T) { mapA := adt.MakeEmptyMap(ctxstoreA) mapB := adt.MakeEmptyMap(ctxstoreB) - require.NoError(t, mapA.Put(adt.UIntKey(0), runtime.CBORBytes([]byte{0}))) // delete + require.NoError(t, mapA.Put(abi.UIntKey(0), runtime.CBORBytes([]byte{0}))) // delete - require.NoError(t, mapA.Put(adt.UIntKey(1), runtime.CBORBytes([]byte{0}))) // modify - require.NoError(t, mapB.Put(adt.UIntKey(1), runtime.CBORBytes([]byte{1}))) + require.NoError(t, mapA.Put(abi.UIntKey(1), runtime.CBORBytes([]byte{0}))) // modify + require.NoError(t, mapB.Put(abi.UIntKey(1), runtime.CBORBytes([]byte{1}))) - require.NoError(t, mapA.Put(adt.UIntKey(2), runtime.CBORBytes([]byte{1}))) // delete + require.NoError(t, mapA.Put(abi.UIntKey(2), runtime.CBORBytes([]byte{1}))) // delete - require.NoError(t, mapA.Put(adt.UIntKey(3), runtime.CBORBytes([]byte{0}))) // noop - require.NoError(t, mapB.Put(adt.UIntKey(3), runtime.CBORBytes([]byte{0}))) + require.NoError(t, mapA.Put(abi.UIntKey(3), runtime.CBORBytes([]byte{0}))) // noop + require.NoError(t, mapB.Put(abi.UIntKey(3), runtime.CBORBytes([]byte{0}))) - require.NoError(t, mapA.Put(adt.UIntKey(4), runtime.CBORBytes([]byte{0}))) // modify - require.NoError(t, mapB.Put(adt.UIntKey(4), runtime.CBORBytes([]byte{6}))) + require.NoError(t, mapA.Put(abi.UIntKey(4), runtime.CBORBytes([]byte{0}))) // modify + require.NoError(t, mapB.Put(abi.UIntKey(4), runtime.CBORBytes([]byte{6}))) - require.NoError(t, mapB.Put(adt.UIntKey(5), runtime.CBORBytes{8})) // add - require.NoError(t, mapB.Put(adt.UIntKey(6), runtime.CBORBytes{9})) // add + require.NoError(t, mapB.Put(abi.UIntKey(5), runtime.CBORBytes{8})) // add + require.NoError(t, mapB.Put(abi.UIntKey(6), runtime.CBORBytes{9})) // add changes := new(TestDiffMap) @@ -134,12 +135,12 @@ type TestDiffMap struct { var _ AdtMapDiff = &TestDiffMap{} -func (t *TestDiffMap) AsKey(key string) (adt.Keyer, error) { - k, err := adt.ParseUIntKey(key) +func (t *TestDiffMap) AsKey(key string) (abi.Keyer, error) { + k, err := abi.ParseUIntKey(key) if err != nil { return nil, err } - return adt.UIntKey(k), nil + return abi.UIntKey(k), nil } func (t *TestDiffMap) Add(key string, val *typegen.Deferred) error { @@ -148,7 +149,7 @@ func (t *TestDiffMap) Add(key string, val *typegen.Deferred) error { if err != nil { return err } - k, err := adt.ParseUIntKey(key) + k, err := abi.ParseUIntKey(key) if err != nil { return err } @@ -172,7 +173,7 @@ func (t *TestDiffMap) Modify(key string, from, to *typegen.Deferred) error { return err } - k, err := adt.ParseUIntKey(key) + k, err := abi.ParseUIntKey(key) if err != nil { return err } @@ -198,7 +199,7 @@ func (t *TestDiffMap) Remove(key string, val *typegen.Deferred) error { if err != nil { return err } - k, err := adt.ParseUIntKey(key) + k, err := abi.ParseUIntKey(key) if err != nil { return err } diff --git a/chain/events/state/predicates.go b/chain/events/state/predicates.go index b30e69b48..0858793d8 100644 --- a/chain/events/state/predicates.go +++ b/chain/events/state/predicates.go @@ -537,8 +537,8 @@ type MinerPreCommitChanges struct { Removed []miner.SectorPreCommitOnChainInfo } -func (m *MinerPreCommitChanges) AsKey(key string) (adt.Keyer, error) { - sector, err := adt.ParseUIntKey(key) +func (m *MinerPreCommitChanges) AsKey(key string) (abi.Keyer, error) { + sector, err := abi.ParseUIntKey(key) if err != nil { return nil, err } @@ -662,12 +662,12 @@ type AddressChange struct { type DiffInitActorStateFunc func(ctx context.Context, oldState *init_.State, newState *init_.State) (changed bool, user UserData, err error) -func (i *InitActorAddressChanges) AsKey(key string) (adt.Keyer, error) { +func (i *InitActorAddressChanges) AsKey(key string) (abi.Keyer, error) { addr, err := address.NewFromBytes([]byte(key)) if err != nil { return nil, err } - return adt.AddrKey(addr), nil + return abi.AddrKey(addr), nil } func (i *InitActorAddressChanges) Add(key string, val *typegen.Deferred) error { diff --git a/chain/gen/genesis/t01_init.go b/chain/gen/genesis/t01_init.go index 1686102fe..667079a6d 100644 --- a/chain/gen/genesis/t01_init.go +++ b/chain/gen/genesis/t01_init.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/util/adt" @@ -50,7 +51,7 @@ func SetupInitActor(bs bstore.Blockstore, netname string, initialActors []genesi fmt.Printf("init set %s t0%d\n", e, counter) value := cbg.CborInt(counter) - if err := amap.Put(adt.AddrKey(e), &value); err != nil { + if err := amap.Put(abi.AddrKey(e), &value); err != nil { return 0, nil, nil, err } counter = counter + 1 @@ -77,7 +78,7 @@ func SetupInitActor(bs bstore.Blockstore, netname string, initialActors []genesi fmt.Printf("init set %s t0%d\n", ainfo.Owner, counter) value := cbg.CborInt(counter) - if err := amap.Put(adt.AddrKey(ainfo.Owner), &value); err != nil { + if err := amap.Put(abi.AddrKey(ainfo.Owner), &value); err != nil { return 0, nil, nil, err } counter = counter + 1 @@ -95,7 +96,7 @@ func SetupInitActor(bs bstore.Blockstore, netname string, initialActors []genesi return 0, nil, nil, xerrors.Errorf("unmarshaling account meta: %w", err) } value := cbg.CborInt(80) - if err := amap.Put(adt.AddrKey(ainfo.Owner), &value); err != nil { + if err := amap.Put(abi.AddrKey(ainfo.Owner), &value); err != nil { return 0, nil, nil, err } } else if rootVerifier.Type == genesis.TMultisig { @@ -110,7 +111,7 @@ func SetupInitActor(bs bstore.Blockstore, netname string, initialActors []genesi fmt.Printf("init set %s t0%d\n", e, counter) value := cbg.CborInt(counter) - if err := amap.Put(adt.AddrKey(e), &value); err != nil { + if err := amap.Put(abi.AddrKey(e), &value); err != nil { return 0, nil, nil, err } counter = counter + 1 diff --git a/chain/state/statetree.go b/chain/state/statetree.go index c083f1817..445f3765a 100644 --- a/chain/state/statetree.go +++ b/chain/state/statetree.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/specs-actors/actors/builtin" init_ "github.com/filecoin-project/specs-actors/actors/builtin/init" @@ -209,7 +210,7 @@ func (st *StateTree) GetActor(addr address.Address) (*types.Actor, error) { } var act types.Actor - if found, err := st.root.Get(adt.AddrKey(addr), &act); err != nil { + if found, err := st.root.Get(abi.AddrKey(addr), &act); err != nil { return nil, xerrors.Errorf("hamt find failed: %w", err) } else if !found { return nil, types.ErrActorNotFound @@ -254,11 +255,11 @@ func (st *StateTree) Flush(ctx context.Context) (cid.Cid, error) { for addr, sto := range st.snaps.layers[0].actors { if sto.Delete { - if err := st.root.Delete(adt.AddrKey(addr)); err != nil { + if err := st.root.Delete(abi.AddrKey(addr)); err != nil { return cid.Undef, err } } else { - if err := st.root.Put(adt.AddrKey(addr), &sto.Act); err != nil { + if err := st.root.Put(abi.AddrKey(addr), &sto.Act); err != nil { return cid.Undef, err } } diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index f77ec20ff..27b71f358 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -120,7 +120,7 @@ func GetPowerRaw(ctx context.Context, sm *StateManager, st cid.Cid, maddr addres } var claim power.Claim - if _, err := cm.Get(adt.AddrKey(maddr), &claim); err != nil { + if _, err := cm.Get(abi.AddrKey(maddr), &claim); err != nil { return power.Claim{}, power.Claim{}, err } @@ -312,7 +312,7 @@ func GetMinerSlashed(ctx context.Context, sm *StateManager, ts *types.TipSet, ma return false, err } - ok, err := claims.Get(power.AddrKey(maddr), nil) + ok, err := claims.Get(abi.AddrKey(maddr), nil) if err != nil { return false, err } diff --git a/chain/sync.go b/chain/sync.go index 9864600dd..bb3e50bdb 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -648,7 +648,7 @@ func (syncer *Syncer) minerIsValid(ctx context.Context, maddr address.Address, b } var claim power.Claim - exist, err := cm.Get(adt.AddrKey(maddr), &claim) + exist, err := cm.Get(abi.AddrKey(maddr), &claim) if err != nil { return err } diff --git a/cli/chain.go b/cli/chain.go index 36288c7d7..278a8e29b 100644 --- a/cli/chain.go +++ b/cli/chain.go @@ -659,7 +659,7 @@ func handleHamtEpoch(ctx context.Context, api api.FullNode, r cid.Cid) error { } return mp.ForEach(nil, func(key string) error { - ik, err := adt.ParseIntKey(key) + ik, err := abi.ParseIntKey(key) if err != nil { return err } diff --git a/cmd/lotus-chainwatch/processor/miner.go b/cmd/lotus-chainwatch/processor/miner.go index e063db19f..71d881927 100644 --- a/cmd/lotus-chainwatch/processor/miner.go +++ b/cmd/lotus-chainwatch/processor/miner.go @@ -221,7 +221,7 @@ func (p *Processor) processMiners(ctx context.Context, minerTips map[types.TipSe var claim power.Claim // get miner claim from power actors claim map and store if found, else the miner had no claim at // this tipset - found, err := minersClaims.Get(adt.AddrKey(act.addr), &claim) + found, err := minersClaims.Get(abi.AddrKey(act.addr), &claim) if err != nil { return nil, err } diff --git a/cmd/lotus-shed/verifreg.go b/cmd/lotus-shed/verifreg.go index 6e24dc0b6..9f475261d 100644 --- a/cmd/lotus-shed/verifreg.go +++ b/cmd/lotus-shed/verifreg.go @@ -7,6 +7,7 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" "github.com/filecoin-project/specs-actors/actors/util/adt" @@ -336,7 +337,7 @@ var verifRegCheckVerifierCmd = &cli.Command{ } var dcap verifreg.DataCap - if found, err := vh.Get(adt.AddrKey(vaddr), &dcap); err != nil { + if found, err := vh.Get(abi.AddrKey(vaddr), &dcap); err != nil { return err } else if !found { return fmt.Errorf("not found") diff --git a/go.mod b/go.mod index 5470e226a..65c2addb8 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/filecoin-project/go-statemachine v0.0.0-20200813232949-df9b130df370 github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b - github.com/filecoin-project/specs-actors v0.9.9 + github.com/filecoin-project/specs-actors v0.9.10 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 github.com/filecoin-project/test-vectors/schema v0.0.1 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 diff --git a/go.sum b/go.sum index 5e479db82..77b556fb2 100644 --- a/go.sum +++ b/go.sum @@ -252,8 +252,8 @@ github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/go.mod h1:Q0GQOBtKf1oE10eSXSlhN45kDBdGvEcVOqMiffqX+N8= github.com/filecoin-project/specs-actors v0.9.4/go.mod h1:BStZQzx5x7TmCkLv0Bpa07U6cPKol6fd3w9KjMPZ6Z4= github.com/filecoin-project/specs-actors v0.9.7/go.mod h1:wM2z+kwqYgXn5Z7scV1YHLyd1Q1cy0R8HfTIWQ0BFGU= -github.com/filecoin-project/specs-actors v0.9.9 h1:hc1iCks6dv7mpGQXcbYU3hL1WfBRZaVTYE4x2d1l2yw= -github.com/filecoin-project/specs-actors v0.9.9/go.mod h1:czlvLQGEX0fjLLfdNHD7xLymy6L3n7aQzRWzsYGf+ys= +github.com/filecoin-project/specs-actors v0.9.10 h1:gU0TrRhgkCsBEOP42sGDE7RQuR0Cov9hJhBqq+RJmjU= +github.com/filecoin-project/specs-actors v0.9.10/go.mod h1:czlvLQGEX0fjLLfdNHD7xLymy6L3n7aQzRWzsYGf+ys= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= github.com/filecoin-project/test-vectors/schema v0.0.1 h1:5fNF76nl4qolEvcIsjc0kUADlTMVHO73tW4kXXPnsus= diff --git a/node/impl/full/chain.go b/node/impl/full/chain.go index ad2c2944c..0d6f923d8 100644 --- a/node/impl/full/chain.go +++ b/node/impl/full/chain.go @@ -300,7 +300,7 @@ func resolveOnce(bs blockstore.Blockstore) func(ctx context.Context, ds ipld.Nod return nil, nil, xerrors.Errorf("parsing int64: %w", err) } - ik := adt.IntKey(i) + ik := abi.IntKey(i) names[0] = "@H:" + ik.Key() } @@ -311,7 +311,7 @@ func resolveOnce(bs blockstore.Blockstore) func(ctx context.Context, ds ipld.Nod return nil, nil, xerrors.Errorf("parsing uint64: %w", err) } - ik := adt.UIntKey(i) + ik := abi.UIntKey(i) names[0] = "@H:" + ik.Key() } diff --git a/node/impl/full/state.go b/node/impl/full/state.go index e183fafa4..e82086053 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -509,7 +509,7 @@ func (a *StateAPI) StateMarketParticipants(ctx context.Context, tsk types.TipSet return err } - if found, err := locked.Get(adt.AddrKey(a), &lk); err != nil { + if found, err := locked.Get(abi.AddrKey(a), &lk); err != nil { return err } else if !found { return fmt.Errorf("locked funds not found") @@ -604,7 +604,7 @@ func (a *StateAPI) StateChangedActors(ctx context.Context, old cid.Cid, new cid. return xerrors.Errorf("address in state tree was not valid: %w", err) } - found, err := oh.Get(adt.AddrKey(addr), &ocval) + found, err := oh.Get(abi.AddrKey(addr), &ocval) if err != nil { return err } @@ -1150,7 +1150,7 @@ func (a *StateAPI) StateVerifiedClientStatus(ctx context.Context, addr address.A } var dcap verifreg.DataCap - if found, err := vh.Get(adt.AddrKey(aid), &dcap); err != nil { + if found, err := vh.Get(abi.AddrKey(aid), &dcap); err != nil { return nil, err } else if !found { return nil, nil diff --git a/storage/adapter_storage_miner.go b/storage/adapter_storage_miner.go index 2869e48e5..f7431299d 100644 --- a/storage/adapter_storage_miner.go +++ b/storage/adapter_storage_miner.go @@ -195,7 +195,7 @@ func (s SealingAPIAdapter) StateSectorPreCommitInfo(ctx context.Context, maddr a } var pci miner.SectorPreCommitOnChainInfo - ok, err := precommits.Get(adt.UIntKey(uint64(sectorNumber)), &pci) + ok, err := precommits.Get(abi.UIntKey(uint64(sectorNumber)), &pci) if err != nil { return nil, err } From 2088335b9df1362f4a4cfba16c1a9d9dcec406dd Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 14 Sep 2020 14:40:52 -0700 Subject: [PATCH 102/885] rename to simplify merge --- storage/wdpost_run.go | 52 +++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index f314dce9a..083c793e8 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -17,14 +17,14 @@ import ( "go.opencensus.io/trace" "golang.org/x/xerrors" - v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" - v0proof "github.com/filecoin-project/specs-actors/actors/runtime/proof" + "github.com/filecoin-project/specs-actors/actors/builtin/miner" + "github.com/filecoin-project/specs-actors/actors/runtime/proof" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/apibstore" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + iminer "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" ) @@ -115,12 +115,12 @@ func (s *WindowPoStScheduler) checkSectors(ctx context.Context, check bitfield.B return sbf, nil } -func (s *WindowPoStScheduler) checkNextRecoveries(ctx context.Context, dlIdx uint64, partitions []miner.Partition) error { +func (s *WindowPoStScheduler) checkNextRecoveries(ctx context.Context, dlIdx uint64, partitions []iminer.Partition) error { ctx, span := trace.StartSpan(ctx, "storage.checkNextRecoveries") defer span.End() - params := &v0miner.DeclareFaultsRecoveredParams{ - Recoveries: []v0miner.RecoveryDeclaration{}, + params := &miner.DeclareFaultsRecoveredParams{ + Recoveries: []miner.RecoveryDeclaration{}, } faulty := uint64(0) @@ -165,7 +165,7 @@ func (s *WindowPoStScheduler) checkNextRecoveries(ctx context.Context, dlIdx uin continue } - params.Recoveries = append(params.Recoveries, v0miner.RecoveryDeclaration{ + params.Recoveries = append(params.Recoveries, miner.RecoveryDeclaration{ Deadline: dlIdx, Partition: uint64(partIdx), Sectors: recovered, @@ -214,12 +214,12 @@ func (s *WindowPoStScheduler) checkNextRecoveries(ctx context.Context, dlIdx uin return nil } -func (s *WindowPoStScheduler) checkNextFaults(ctx context.Context, dlIdx uint64, partitions []miner.Partition) error { +func (s *WindowPoStScheduler) checkNextFaults(ctx context.Context, dlIdx uint64, partitions []iminer.Partition) error { ctx, span := trace.StartSpan(ctx, "storage.checkNextFaults") defer span.End() - params := &v0miner.DeclareFaultsParams{ - Faults: []v0miner.FaultDeclaration{}, + params := &miner.DeclareFaultsParams{ + Faults: []miner.FaultDeclaration{}, } bad := uint64(0) @@ -251,7 +251,7 @@ func (s *WindowPoStScheduler) checkNextFaults(ctx context.Context, dlIdx uint64, bad += c - params.Faults = append(params.Faults, v0miner.FaultDeclaration{ + params.Faults = append(params.Faults, miner.FaultDeclaration{ Deadline: dlIdx, Partition: uint64(partIdx), Sectors: faulty, @@ -298,7 +298,7 @@ func (s *WindowPoStScheduler) checkNextFaults(ctx context.Context, dlIdx uint64, return nil } -func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *types.TipSet) (*v0miner.SubmitWindowedPoStParams, error) { +func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *types.TipSet) (*miner.SubmitWindowedPoStParams, error) { ctx, span := trace.StartSpan(ctx, "storage.runPost") defer span.End() @@ -308,7 +308,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty return nil, xerrors.Errorf("resolving actor: %w", err) } - mas, err := miner.Load(stor, act) + mas, err := iminer.Load(stor, act) if err != nil { return nil, xerrors.Errorf("getting miner state: %w", err) } @@ -325,8 +325,8 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty log.Errorf("loading deadline: %v", err) return } - var partitions []miner.Partition - err = dl.ForEachPartition(func(_ uint64, part miner.Partition) error { + var partitions []iminer.Partition + err = dl.ForEachPartition(func(_ uint64, part iminer.Partition) error { partitions = append(partitions, part) return nil }) @@ -361,8 +361,8 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty return nil, xerrors.Errorf("loading deadline: %w", err) } - var partitions []miner.Partitions - err = dl.ForEachPartition(func(_ uint64, part miner.Partition) error { + var partitions []iminer.Partitions + err = dl.ForEachPartition(func(_ uint64, part iminer.Partition) error { partitions = apppend(partitions, part) return nil }) @@ -370,9 +370,9 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty return nil, xerrors.Errorf("loading partitions: %w", err) } - params := &v0miner.SubmitWindowedPoStParams{ + params := &miner.SubmitWindowedPoStParams{ Deadline: di.Index, - Partitions: make([]v0miner.PoStPartition, 0, len(partitions)), + Partitions: make([]miner.PoStPartition, 0, len(partitions)), Proofs: nil, } @@ -432,7 +432,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty sidToPart[si.SectorNumber] = partIdx } - params.Partitions = append(params.Partitions, v0miner.PoStPartition{ + params.Partitions = append(params.Partitions, miner.PoStPartition{ Index: uint64(partIdx), Skipped: skipped, }) @@ -497,7 +497,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty return params, nil } -func (s *WindowPoStScheduler) sectorsForProof(ctx context.Context, goodSectors, allSectors bitfield.BitField, ts *types.TipSet) ([]v0proof.SectorInfo, error) { +func (s *WindowPoStScheduler) sectorsForProof(ctx context.Context, goodSectors, allSectors bitfield.BitField, ts *types.TipSet) ([]proof.SectorInfo, error) { sset, err := s.api.StateMinerSectors(ctx, s.actor, &goodSectors, false, ts.Key()) if err != nil { return nil, err @@ -507,22 +507,22 @@ func (s *WindowPoStScheduler) sectorsForProof(ctx context.Context, goodSectors, return nil, nil } - substitute := v0proof.SectorInfo{ + substitute := proof.SectorInfo{ SectorNumber: sset[0].ID, SealedCID: sset[0].Info.SealedCID, SealProof: sset[0].Info.SealProof, } - sectorByID := make(map[uint64]v0proof.SectorInfo, len(sset)) + sectorByID := make(map[uint64]proof.SectorInfo, len(sset)) for _, sector := range sset { - sectorByID[uint64(sector.ID)] = v0proof.SectorInfo{ + sectorByID[uint64(sector.ID)] = proof.SectorInfo{ SectorNumber: sector.ID, SealedCID: sector.Info.SealedCID, SealProof: sector.Info.SealProof, } } - proofSectors := make([]v0proof.SectorInfo, 0, len(sset)) + proofSectors := make([]proof.SectorInfo, 0, len(sset)) if err := allSectors.ForEach(func(sectorNo uint64) error { if info, found := sectorByID[sectorNo]; found { proofSectors = append(proofSectors, info) @@ -537,7 +537,7 @@ func (s *WindowPoStScheduler) sectorsForProof(ctx context.Context, goodSectors, return proofSectors, nil } -func (s *WindowPoStScheduler) submitPost(ctx context.Context, proof *v0miner.SubmitWindowedPoStParams) error { +func (s *WindowPoStScheduler) submitPost(ctx context.Context, proof *miner.SubmitWindowedPoStParams) error { ctx, span := trace.StartSpan(ctx, "storage.commitPost") defer span.End() From cc4d5306ebc363fa912310a2b2f96c4db2486c53 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 14 Sep 2020 15:43:12 -0700 Subject: [PATCH 103/885] Progress --- chain/actors/adt/adt.go | 8 ++-- chain/actors/builtin/init/init.go | 4 ++ chain/actors/builtin/init/v0.go | 5 +++ chain/actors/builtin/miner/miner.go | 2 + chain/actors/builtin/power/power.go | 12 ++++- chain/actors/builtin/power/v0.go | 28 ++++++++++-- chain/actors/version.go | 24 ---------- chain/gen/genesis/genesis.go | 63 +++++++++++++------------- chain/state/statetree.go | 70 ++++++++++++++++++++++------- chain/stmgr/read.go | 16 ++++--- chain/stmgr/stmgr.go | 8 ++-- chain/stmgr/utils.go | 60 ++++++++++++------------- chain/store/store.go | 21 +-------- chain/store/weight.go | 2 +- chain/sub/incoming.go | 2 +- chain/sync.go | 3 +- chain/types/state.go | 15 +++++++ chain/vm/vm.go | 3 +- cmd/lotus-shed/balances.go | 3 +- cmd/lotus-shed/genesis-verify.go | 2 +- gen/main.go | 2 + node/impl/full/state.go | 2 +- 22 files changed, 205 insertions(+), 150 deletions(-) delete mode 100644 chain/actors/version.go create mode 100644 chain/types/state.go diff --git a/chain/actors/adt/adt.go b/chain/actors/adt/adt.go index 179c73b33..ebf32c3c4 100644 --- a/chain/actors/adt/adt.go +++ b/chain/actors/adt/adt.go @@ -21,16 +21,16 @@ type Map interface { ForEach(v cbor.Unmarshaler, fn func(key string) error) error } -func AsMap(store Store, root cid.Cid, version network.Version) (Map, error) { - switch builtin.VersionForNetwork(version) { +func AsMap(store Store, root cid.Cid, version builtin.Version) (Map, error) { + switch version { case builtin.Version0: return v0adt.AsMap(store, root) } return nil, xerrors.Errorf("unknown network version: %d", version) } -func NewMap(store Store, version network.Version) (Map, error) { - switch builtin.VersionForNetwork(version) { +func NewMap(store Store, version builtin.Version) (Map, error) { + switch version { case builtin.Version0: return v0adt.MakeEmptyMap(store), nil } diff --git a/chain/actors/builtin/init/init.go b/chain/actors/builtin/init/init.go index a96c0dfd6..3b1a49564 100644 --- a/chain/actors/builtin/init/init.go +++ b/chain/actors/builtin/init/init.go @@ -9,8 +9,11 @@ import ( "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/node/modules/dtypes" ) +var Address = v0builtin.InitActorAddr + func Load(store adt.Store, act *types.Actor) (State, error) { switch act.Code { case v0builtin.InitActorCodeID: @@ -29,4 +32,5 @@ type State interface { ResolveAddress(address address.Address) (address.Address, bool, error) MapAddressToNewID(address address.Address) (address.Address, error) + NetworkName() (dtypes.NetworkName, error) } diff --git a/chain/actors/builtin/init/v0.go b/chain/actors/builtin/init/v0.go index 5001e9806..711b702e2 100644 --- a/chain/actors/builtin/init/v0.go +++ b/chain/actors/builtin/init/v0.go @@ -6,6 +6,7 @@ import ( init_ "github.com/filecoin-project/specs-actors/actors/builtin/init" "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/node/modules/dtypes" ) type v0State struct { @@ -20,3 +21,7 @@ func (s *v0State) ResolveAddress(address address.Address) (address.Address, bool func (s *v0State) MapAddressToNewID(address address.Address) (address.Address, error) { return s.State.MapAddressToNewID(s.store, address) } + +func (s *v0State) NetworkName() (dtypes.NetworkName, error) { + return dtypes.NetworkName(s.State.NetworkName), nil +} diff --git a/chain/actors/builtin/miner/miner.go b/chain/actors/builtin/miner/miner.go index 11dc3158a..76503d80e 100644 --- a/chain/actors/builtin/miner/miner.go +++ b/chain/actors/builtin/miner/miner.go @@ -14,6 +14,8 @@ import ( "github.com/filecoin-project/lotus/chain/types" ) +var Address = v0builtin.InitActorAddr + func Load(store adt.Store, act *types.Actor) (st State, err error) { switch act.Code { case v0builtin.StorageMinerActorCodeID: diff --git a/chain/actors/builtin/power/power.go b/chain/actors/builtin/power/power.go index 0fcbaf971..7588615a6 100644 --- a/chain/actors/builtin/power/power.go +++ b/chain/actors/builtin/power/power.go @@ -29,5 +29,15 @@ type State interface { cbor.Marshaler TotalLocked() (abi.TokenAmount, error) - MinerNominalPowerMeetsConsensusMinimum(adt.Store, address.Address) (bool, error) + TotalPower() (Claim, error) + MinerPower(address.Address) (Claim, error) + MinerNominalPowerMeetsConsensusMinimum(address.Address) (bool, error) +} + +type Claim struct { + // Sum of raw byte power for a miner's sectors. + RawBytePower abi.StoragePower + + // Sum of quality adjusted power for a miner's sectors. + QualityAdjPower abi.StoragePower } diff --git a/chain/actors/builtin/power/v0.go b/chain/actors/builtin/power/v0.go index 879e3e446..f3152eb6a 100644 --- a/chain/actors/builtin/power/v0.go +++ b/chain/actors/builtin/power/v0.go @@ -3,8 +3,8 @@ package power import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/specs-actors/actors/builtin/power" + "github.com/filecoin-project/specs-actors/actors/util/adt" ) type v0State struct { @@ -16,6 +16,28 @@ func (s *v0State) TotalLocked() (abi.TokenAmount, error) { return s.TotalPledgeCollateral, nil } -func (s *v0State) MinerNominalPowerMeetsConsensusMinimum(st adt.Store, a address.Address) (bool, error) { - return s.State.MinerNominalPowerMeetsConsensusMinimum(st, a) +func (s *v0State) TotalPower() (Claim, error) { + return Claim{ + RawBytePower: s.TotalRawBytePower, + QualityAdjPower: s.TotalQualityAdjPower, + }, nil +} + +func (s *v0State) MinerPower(addr address.Address) (Claim, error) { + claims, err := adt.AsMap(s.store, s.Claims) + if err != nil { + return Claim{}, err + } + var claim power.Claim + if _, err := claims.Get(abi.AddrKey(addr), &claim); err != nil { + return Claim{}, err + } + return Claim{ + RawBytePower: claim.RawBytePower, + QualityAdjPower: claim.QualityAdjPower, + }, nil +} + +func (s *v0State) MinerNominalPowerMeetsConsensusMinimum(a address.Address) (bool, error) { + return s.State.MinerNominalPowerMeetsConsensusMinimum(s.store, a) } diff --git a/chain/actors/version.go b/chain/actors/version.go deleted file mode 100644 index 99cc59eaa..000000000 --- a/chain/actors/version.go +++ /dev/null @@ -1,24 +0,0 @@ -package actors - -import ( - "fmt" - - "github.com/filecoin-project/go-state-types/network" -) - -type Version int - -const ( - Version0 = iota - Version1 -) - -// VersionForNetwork resolves the network version into an specs-actors version. -func VersionForNetwork(v network.Version) Version { - switch v { - case network.Version0, network.Version1: - return Version0 - default: - panic(fmt.Sprintf("unimplemented network version: %d", v)) - } -} diff --git a/chain/gen/genesis/genesis.go b/chain/gen/genesis/genesis.go index d726dfa50..be2ed70aa 100644 --- a/chain/gen/genesis/genesis.go +++ b/chain/gen/genesis/genesis.go @@ -17,13 +17,14 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" - "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/builtin/account" - "github.com/filecoin-project/specs-actors/actors/builtin/multisig" - "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" - "github.com/filecoin-project/specs-actors/actors/util/adt" + v0builtin "github.com/filecoin-project/specs-actors/actors/builtin" + v0account "github.com/filecoin-project/specs-actors/actors/builtin/account" + v0multisig "github.com/filecoin-project/specs-actors/actors/builtin/multisig" + v0verifreg "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" + v0adt "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" @@ -114,7 +115,7 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge return nil, nil, xerrors.Errorf("putting empty object: %w", err) } - state, err := state.NewStateTree(cst, GenesisNetworkVersion) + state, err := state.NewStateTree(cst, builtin.Version0) if err != nil { return nil, nil, xerrors.Errorf("making new state tree: %w", err) } @@ -125,7 +126,7 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge if err != nil { return nil, nil, xerrors.Errorf("setup init actor: %w", err) } - if err := state.SetActor(builtin.SystemActorAddr, sysact); err != nil { + if err := state.SetActor(v0builtin.SystemActorAddr, sysact); err != nil { return nil, nil, xerrors.Errorf("set init actor: %w", err) } @@ -135,7 +136,7 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge if err != nil { return nil, nil, xerrors.Errorf("setup init actor: %w", err) } - if err := state.SetActor(builtin.InitActorAddr, initact); err != nil { + if err := state.SetActor(v0builtin.InitActorAddr, initact); err != nil { return nil, nil, xerrors.Errorf("set init actor: %w", err) } @@ -146,7 +147,7 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge return nil, nil, xerrors.Errorf("setup init actor: %w", err) } - err = state.SetActor(builtin.RewardActorAddr, rewact) + err = state.SetActor(v0builtin.RewardActorAddr, rewact) if err != nil { return nil, nil, xerrors.Errorf("set network account actor: %w", err) } @@ -156,7 +157,7 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge if err != nil { return nil, nil, xerrors.Errorf("setup cron actor: %w", err) } - if err := state.SetActor(builtin.CronActorAddr, cronact); err != nil { + if err := state.SetActor(v0builtin.CronActorAddr, cronact); err != nil { return nil, nil, xerrors.Errorf("set cron actor: %w", err) } @@ -165,7 +166,7 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge if err != nil { return nil, nil, xerrors.Errorf("setup storage market actor: %w", err) } - if err := state.SetActor(builtin.StoragePowerActorAddr, spact); err != nil { + if err := state.SetActor(v0builtin.StoragePowerActorAddr, spact); err != nil { return nil, nil, xerrors.Errorf("set storage market actor: %w", err) } @@ -174,7 +175,7 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge if err != nil { return nil, nil, xerrors.Errorf("setup storage market actor: %w", err) } - if err := state.SetActor(builtin.StorageMarketActorAddr, marketact); err != nil { + if err := state.SetActor(v0builtin.StorageMarketActorAddr, marketact); err != nil { return nil, nil, xerrors.Errorf("set market actor: %w", err) } @@ -183,20 +184,20 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge if err != nil { return nil, nil, xerrors.Errorf("setup storage market actor: %w", err) } - if err := state.SetActor(builtin.VerifiedRegistryActorAddr, verifact); err != nil { + if err := state.SetActor(v0builtin.VerifiedRegistryActorAddr, verifact); err != nil { return nil, nil, xerrors.Errorf("set market actor: %w", err) } - burntRoot, err := cst.Put(ctx, &account.State{ - Address: builtin.BurntFundsActorAddr, + burntRoot, err := cst.Put(ctx, &v0account.State{ + Address: v0builtin.BurntFundsActorAddr, }) if err != nil { return nil, nil, xerrors.Errorf("failed to setup burnt funds actor state: %w", err) } // Setup burnt-funds - err = state.SetActor(builtin.BurntFundsActorAddr, &types.Actor{ - Code: builtin.AccountActorCodeID, + err = state.SetActor(v0builtin.BurntFundsActorAddr, &types.Actor{ + Code: v0builtin.AccountActorCodeID, Balance: types.NewInt(0), Head: burntRoot, }) @@ -261,13 +262,13 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge return nil, nil, err } - verifierState, err := cst.Put(ctx, &account.State{Address: verifierAd}) + verifierState, err := cst.Put(ctx, &v0account.State{Address: verifierAd}) if err != nil { return nil, nil, err } err = state.SetActor(verifierId, &types.Actor{ - Code: builtin.AccountActorCodeID, + Code: v0builtin.AccountActorCodeID, Balance: types.NewInt(0), Head: verifierState, }) @@ -314,7 +315,7 @@ func createAccountActor(ctx context.Context, cst cbor.IpldStore, state *state.St if err := json.Unmarshal(info.Meta, &ainfo); err != nil { return xerrors.Errorf("unmarshaling account meta: %w", err) } - st, err := cst.Put(ctx, &account.State{Address: ainfo.Owner}) + st, err := cst.Put(ctx, &v0account.State{Address: ainfo.Owner}) if err != nil { return err } @@ -325,7 +326,7 @@ func createAccountActor(ctx context.Context, cst cbor.IpldStore, state *state.St } err = state.SetActor(ida, &types.Actor{ - Code: builtin.AccountActorCodeID, + Code: v0builtin.AccountActorCodeID, Balance: info.Balance, Head: st, }) @@ -343,7 +344,7 @@ func createMultisigAccount(ctx context.Context, bs bstore.Blockstore, cst cbor.I if err := json.Unmarshal(info.Meta, &ainfo); err != nil { return xerrors.Errorf("unmarshaling account meta: %w", err) } - pending, err := adt.MakeEmptyMap(adt.WrapStore(ctx, cst)).Root() + pending, err := v0adt.MakeEmptyMap(v0adt.WrapStore(ctx, cst)).Root() if err != nil { return xerrors.Errorf("failed to create empty map: %v", err) } @@ -363,12 +364,12 @@ func createMultisigAccount(ctx context.Context, bs bstore.Blockstore, cst cbor.I continue } - st, err := cst.Put(ctx, &account.State{Address: e}) + st, err := cst.Put(ctx, &v0account.State{Address: e}) if err != nil { return err } err = state.SetActor(idAddress, &types.Actor{ - Code: builtin.AccountActorCodeID, + Code: v0builtin.AccountActorCodeID, Balance: types.NewInt(0), Head: st, }) @@ -378,7 +379,7 @@ func createMultisigAccount(ctx context.Context, bs bstore.Blockstore, cst cbor.I signers = append(signers, idAddress) } - st, err := cst.Put(ctx, &multisig.State{ + st, err := cst.Put(ctx, &v0multisig.State{ Signers: signers, NumApprovalsThreshold: uint64(ainfo.Threshold), StartEpoch: abi.ChainEpoch(ainfo.VestingStart), @@ -390,7 +391,7 @@ func createMultisigAccount(ctx context.Context, bs bstore.Blockstore, cst cbor.I return err } err = state.SetActor(ida, &types.Actor{ - Code: builtin.MultisigActorCodeID, + Code: v0builtin.MultisigActorCodeID, Balance: info.Balance, Head: st, }) @@ -441,7 +442,7 @@ func VerifyPreSealedData(ctx context.Context, cs *store.ChainStore, stateroot ci return cid.Undef, err } - _, err = doExecValue(ctx, vm, builtin.VerifiedRegistryActorAddr, verifregRoot, types.NewInt(0), builtin.MethodsVerifiedRegistry.AddVerifier, mustEnc(&verifreg.AddVerifierParams{ + _, err = doExecValue(ctx, vm, v0builtin.VerifiedRegistryActorAddr, verifregRoot, types.NewInt(0), v0builtin.MethodsVerifiedRegistry.AddVerifier, mustEnc(&v0verifreg.AddVerifierParams{ Address: verifier, Allowance: abi.NewStoragePower(int64(sum)), // eh, close enough @@ -452,7 +453,7 @@ func VerifyPreSealedData(ctx context.Context, cs *store.ChainStore, stateroot ci } for c, amt := range verifNeeds { - _, err := doExecValue(ctx, vm, builtin.VerifiedRegistryActorAddr, verifier, types.NewInt(0), builtin.MethodsVerifiedRegistry.AddVerifiedClient, mustEnc(&verifreg.AddVerifiedClientParams{ + _, err := doExecValue(ctx, vm, v0builtin.VerifiedRegistryActorAddr, verifier, types.NewInt(0), v0builtin.MethodsVerifiedRegistry.AddVerifiedClient, mustEnc(&v0verifreg.AddVerifiedClientParams{ Address: c, Allowance: abi.NewStoragePower(int64(amt)), })) @@ -494,8 +495,8 @@ func MakeGenesisBlock(ctx context.Context, bs bstore.Blockstore, sys vm.SyscallB return nil, xerrors.Errorf("setup miners failed: %w", err) } - store := adt.WrapStore(ctx, cbor.NewCborStore(bs)) - emptyroot, err := adt.MakeEmptyArray(store).Root() + store := v0adt.WrapStore(ctx, cbor.NewCborStore(bs)) + emptyroot, err := v0adt.MakeEmptyArray(store).Root() if err != nil { return nil, xerrors.Errorf("amt build failed: %w", err) } @@ -543,7 +544,7 @@ func MakeGenesisBlock(ctx context.Context, bs bstore.Blockstore, sys vm.SyscallB } b := &types.BlockHeader{ - Miner: builtin.SystemActorAddr, + Miner: v0builtin.SystemActorAddr, Ticket: genesisticket, Parents: []cid.Cid{filecoinGenesisCid}, Height: 0, diff --git a/chain/state/statetree.go b/chain/state/statetree.go index 7aa7701fd..b3a3b7cee 100644 --- a/chain/state/statetree.go +++ b/chain/state/statetree.go @@ -12,9 +12,8 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/network" + "github.com/filecoin-project/lotus/chain/actors/builtin" init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init" - "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/types" @@ -24,8 +23,10 @@ var log = logging.Logger("statetree") // StateTree stores actors state by their ID. type StateTree struct { - root adt.Map - Store cbor.IpldStore + root adt.Map + version builtin.Version // TODO + info cid.Cid + Store cbor.IpldStore snaps *stateSnaps } @@ -117,31 +118,57 @@ func (ss *stateSnaps) deleteActor(addr address.Address) { ss.layers[len(ss.layers)-1].actors[addr] = streeOp{Delete: true} } -func NewStateTree(cst cbor.IpldStore, version network.Version) (*StateTree, error) { +func NewStateTree(cst cbor.IpldStore, version builtin.Version) (*StateTree, error) { + var info cid.Cid + switch version { + case builtin.Version0: + // info is undefined + default: + return nil, xerrors.Errorf("unsupported state tree version: %d", version) + } root, err := adt.NewMap(adt.WrapStore(context.TODO(), cst), version) if err != nil { return nil, err } return &StateTree{ - root: root, - Store: cst, - snaps: newStateSnaps(), + root: root, + info: info, + version: version, + Store: cst, + snaps: newStateSnaps(), }, nil } -func LoadStateTree(cst cbor.IpldStore, c cid.Cid, version network.Version) (*StateTree, error) { - // NETUPGRADE: switch map adt type on version upgrade. - nd, err := adt.AsMap(adt.WrapStore(context.TODO(), cst), c, version) +func LoadStateTree(cst cbor.IpldStore, c cid.Cid) (*StateTree, error) { + var root types.StateRoot + // Try loading as a new-style state-tree (version/actors tuple). + if err := cst.Get(context.TODO(), c, &root); err != nil { + // We failed to decode as the new version, must be an old version. + root.Actors = c + root.Version = builtin.Version0 + } + + // If that fails, load as an old-style state-tree (direct hampt, version 0. + nd, err := adt.AsMap(adt.WrapStore(context.TODO(), cst), root.Actors, builtin.Version(root.Version)) if err != nil { log.Errorf("loading hamt node %s failed: %s", c, err) return nil, err } + switch root.Version { + case builtin.Version0: + // supported + default: + return nil, xerrors.Errorf("unsupported state tree version: %d", root.Version) + } + return &StateTree{ - root: nd, - Store: cst, - snaps: newStateSnaps(), + root: nd, + info: root.Info, + version: builtin.Version(root.Version), + Store: cst, + snaps: newStateSnaps(), }, nil } @@ -167,7 +194,7 @@ func (st *StateTree) LookupID(addr address.Address) (address.Address, error) { return resa, nil } - act, err := st.GetActor(builtin.InitActorAddr) + act, err := st.GetActor(init_.Address) if err != nil { return address.Undef, xerrors.Errorf("getting init actor: %w", err) } @@ -271,7 +298,16 @@ func (st *StateTree) Flush(ctx context.Context) (cid.Cid, error) { } } - return st.root.Root() + root, err := st.root.Root() + if err != nil { + return cid.Undef, xerrors.Errorf("failed to flush state-tree hamt: %w", err) + } + // If we're version 0, return a raw tree. + if st.version == builtin.Version0 { + return root, nil + } + // Otherwise, return a versioned tree. + return st.Store.Put(ctx, &types.StateRoot{Version: uint64(st.version), Actors: root, Info: st.info}) } func (st *StateTree) Snapshot(ctx context.Context) error { @@ -289,7 +325,7 @@ func (st *StateTree) ClearSnapshot() { func (st *StateTree) RegisterNewAddress(addr address.Address) (address.Address, error) { var out address.Address - err := st.MutateActor(builtin.InitActorAddr, func(initact *types.Actor) error { + err := st.MutateActor(init_.Address, func(initact *types.Actor) error { ias, err := init_.Load(&AdtStore{st.Store}, initact) if err != nil { return err diff --git a/chain/stmgr/read.go b/chain/stmgr/read.go index 7bf757fb8..53eb8d384 100644 --- a/chain/stmgr/read.go +++ b/chain/stmgr/read.go @@ -8,7 +8,6 @@ import ( cbor "github.com/ipfs/go-ipld-cbor" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/types" ) @@ -23,8 +22,7 @@ func (sm *StateManager) ParentStateTsk(tsk types.TipSetKey) (*state.StateTree, e func (sm *StateManager) ParentState(ts *types.TipSet) (*state.StateTree, error) { cst := cbor.NewCborStore(sm.cs.Blockstore()) - version := sm.GetNtwkVersion(context.TODO(), ts.Height()-1) - state, err := state.LoadStateTree(cst, sm.parentState(ts), version) + state, err := state.LoadStateTree(cst, sm.parentState(ts)) if err != nil { return nil, xerrors.Errorf("load state tree: %w", err) } @@ -32,9 +30,9 @@ func (sm *StateManager) ParentState(ts *types.TipSet) (*state.StateTree, error) return state, nil } -func (sm *StateManager) StateTree(st cid.Cid, ntwkVersion network.Version) (*state.StateTree, error) { +func (sm *StateManager) StateTree(st cid.Cid) (*state.StateTree, error) { cst := cbor.NewCborStore(sm.cs.Blockstore()) - state, err := state.LoadStateTree(cst, st, ntwkVersion) + state, err := state.LoadStateTree(cst, st) if err != nil { return nil, xerrors.Errorf("load state tree: %w", err) } @@ -57,3 +55,11 @@ func (sm *StateManager) LoadActorTsk(_ context.Context, addr address.Address, ts } return state.GetActor(addr) } + +func (sm *StateManager) LoadActorRaw(_ context.Context, addr address.Address, st cid.Cid) (*types.Actor, error) { + state, err := sm.StateTree(st) + if err != nil { + return nil, err + } + return state.GetActor(addr) +} diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index 23e8e92a0..f71b788c8 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -383,7 +383,7 @@ func (sm *StateManager) ResolveToKeyAddress(ctx context.Context, addr address.Ad } cst := cbor.NewCborStore(sm.cs.Blockstore()) - tree, err := state.LoadStateTree(cst, st, sm.GetNtwkVersion(ctx, ts.Height())) + tree, err := state.LoadStateTree(cst, st) if err != nil { return address.Undef, xerrors.Errorf("failed to load state tree") } @@ -406,7 +406,7 @@ func (sm *StateManager) GetBlsPublicKey(ctx context.Context, addr address.Addres func (sm *StateManager) LookupID(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) { cst := cbor.NewCborStore(sm.cs.Blockstore()) - state, err := state.LoadStateTree(cst, sm.parentState(ts), sm.GetNtwkVersion(ctx, ts.Height())) + state, err := state.LoadStateTree(cst, sm.parentState(ts)) if err != nil { return address.Undef, xerrors.Errorf("load state tree: %w", err) } @@ -817,7 +817,7 @@ func (sm *StateManager) setupGenesisActors(ctx context.Context) error { } cst := cbor.NewCborStore(sm.cs.Blockstore()) - sTree, err := state.LoadStateTree(cst, st, sm.GetNtwkVersion(ctx, gts.Height())) + sTree, err := state.LoadStateTree(cst, st) if err != nil { return xerrors.Errorf("loading state tree: %w", err) } @@ -923,7 +923,7 @@ func (sm *StateManager) setupGenesisActorsTestnet(ctx context.Context) error { } cst := cbor.NewCborStore(sm.cs.Blockstore()) - sTree, err := state.LoadStateTree(cst, st, sm.GetNtwkVersion(ctx, gts.Height())) + sTree, err := state.LoadStateTree(cst, st) if err != nil { return xerrors.Errorf("loading state tree: %w", err) } diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index 81bcb0900..63e3e0f74 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -21,23 +21,22 @@ import ( "github.com/filecoin-project/go-bitfield" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/crypto" - power2 "github.com/filecoin-project/lotus/chain/actors/builtin/power" + "github.com/filecoin-project/lotus/chain/actors/builtin/power" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/builtin/account" "github.com/filecoin-project/specs-actors/actors/builtin/cron" - init_ "github.com/filecoin-project/specs-actors/actors/builtin/init" "github.com/filecoin-project/specs-actors/actors/builtin/market" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/specs-actors/actors/builtin/multisig" "github.com/filecoin-project/specs-actors/actors/builtin/paych" - "github.com/filecoin-project/specs-actors/actors/builtin/power" "github.com/filecoin-project/specs-actors/actors/builtin/reward" "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" + init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/beacon" "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/store" @@ -48,8 +47,11 @@ import ( ) func GetNetworkName(ctx context.Context, sm *StateManager, st cid.Cid) (dtypes.NetworkName, error) { - var state init_.State - err := sm.WithStateTree(st, sm.WithActor(builtin.InitActorAddr, sm.WithActorState(ctx, &state))) + act, err := sm.LoadActorRaw(ctx, init_.Address, st) + if err != nil { + return "", err + } + ias, err := init_.Load(sm.cs.Store(ctx), act) if err != nil { return "", err } @@ -58,21 +60,18 @@ func GetNetworkName(ctx context.Context, sm *StateManager, st cid.Cid) (dtypes.N } func GetMinerWorkerRaw(ctx context.Context, sm *StateManager, st cid.Cid, maddr address.Address) (address.Address, error) { - var mas miner.State - _, err := sm.LoadActorStateRaw(ctx, maddr, &mas, st) + act, err := sm.LoadActorRaw(ctx, sm, maddr, st) if err != nil { return address.Undef, xerrors.Errorf("(get sset) failed to load miner actor state: %w", err) } - - cst := cbor.NewCborStore(sm.cs.Blockstore()) - state, err := state.LoadStateTree(cst, st) + mas, err := miner.Load(sm.cs.Store(ctx), act) if err != nil { return address.Undef, xerrors.Errorf("load state tree: %w", err) } - info, err := mas.GetInfo(sm.cs.Store(ctx)) + info, err := mas.Info() if err != nil { - return address.Address{}, err + return address.Undef, xerrors.Errorf("failed to load actor info: %w", err) } return vm.ResolveToKeyAddr(state, cst, info.Worker) @@ -83,36 +82,35 @@ func GetPower(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr add } func GetPowerRaw(ctx context.Context, sm *StateManager, st cid.Cid, maddr address.Address) (power.Claim, power.Claim, error) { - var ps power.State - _, err := sm.LoadActorStateRaw(ctx, builtin.StoragePowerActorAddr, &ps, st) + act, err := sm.LoadActorRaw(ctx, builtin.StoragePowerActorAddr, st) if err != nil { return power.Claim{}, power.Claim{}, xerrors.Errorf("(get sset) failed to load power actor state: %w", err) } + mas, err := power.Load(sm.cs.Store(ctx), act) + if err != nil { + return power.Claim{}, power.Claim{}, err + } + + tpow, err := mas.TotalPower() + if err != nil { + return power.Claim{}, power.Claim{}, err + } + var mpow power.Claim if maddr != address.Undef { - cm, err := adt.AsMap(sm.cs.Store(ctx), ps.Claims) + mpow, err = mas.MinerPower(maddr) if err != nil { return power.Claim{}, power.Claim{}, err } - - var claim power.Claim - if _, err := cm.Get(abi.AddrKey(maddr), &claim); err != nil { - return power.Claim{}, power.Claim{}, err - } - - mpow = claim } - return mpow, power.Claim{ - RawBytePower: ps.TotalRawBytePower, - QualityAdjPower: ps.TotalQualityAdjPower, - }, nil + return mpow, tpow, nil } func PreCommitInfo(ctx context.Context, sm *StateManager, maddr address.Address, sid abi.SectorNumber, ts *types.TipSet) (miner.SectorPreCommitOnChainInfo, error) { var mas miner.State - _, err := sm.LoadActorState(ctx, maddr, &mas, ts) + act, err := sm.LoadActor(ctx, maddr, ts) if err != nil { return miner.SectorPreCommitOnChainInfo{}, xerrors.Errorf("(get sset) failed to load miner actor state: %w", err) } @@ -676,16 +674,16 @@ func MinerHasMinPower(ctx context.Context, sm *StateManager, addr address.Addres return false, xerrors.Errorf("loading power actor state: %w", err) } - ps, err := power2.Load(sm.cs.Store(ctx), pact) + ps, err := power.Load(sm.cs.Store(ctx), pact) if err != nil { return false, err } - return ps.MinerNominalPowerMeetsConsensusMinimum(sm.ChainStore().Store(ctx), addr) + return ps.MinerNominalPowerMeetsConsensusMinimum(addr) } func CheckTotalFIL(ctx context.Context, sm *StateManager, ts *types.TipSet) (abi.TokenAmount, error) { - str, err := state.LoadStateTree(sm.ChainStore().Store(ctx), ts.ParentState(), sm.GetNtwkVersion(ctx, ts.Height())) + str, err := state.LoadStateTree(sm.ChainStore().Store(ctx), ts.ParentState()) if err != nil { return abi.TokenAmount{}, err } diff --git a/chain/store/store.go b/chain/store/store.go index 2c97e2de5..66e31569b 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -5,8 +5,6 @@ import ( "context" "encoding/binary" "encoding/json" - "github.com/filecoin-project/go-state-types/network" - "github.com/filecoin-project/lotus/build" "io" "os" "strconv" @@ -766,29 +764,12 @@ type BlockMessages struct { WinCount int64 } -// TODO: temp hack until #3682 is merged -func hackgetNtwkVersionhack(ctx context.Context, height abi.ChainEpoch) network.Version { - // TODO: move hard fork epoch checks to a schedule defined in build/ - - if build.UpgradeBreezeHeight == 0 { - return network.Version1 - } - - if height <= build.UpgradeBreezeHeight { - return network.Version0 - } - - return network.Version1 -} - func (cs *ChainStore) BlockMsgsForTipset(ts *types.TipSet) ([]BlockMessages, error) { applied := make(map[address.Address]uint64) cst := cbor.NewCborStore(cs.bs) - nv := hackgetNtwkVersionhack(context.TODO(), ts.Height()) // TODO: part of the temp hack from above - - st, err := state.LoadStateTree(cst, ts.Blocks()[0].ParentStateRoot, nv) + st, err := state.LoadStateTree(cst, ts.Blocks()[0].ParentStateRoot) if err != nil { return nil, xerrors.Errorf("failed to load state tree") } diff --git a/chain/store/weight.go b/chain/store/weight.go index e27d3fd37..5249df011 100644 --- a/chain/store/weight.go +++ b/chain/store/weight.go @@ -29,7 +29,7 @@ func (cs *ChainStore) Weight(ctx context.Context, ts *types.TipSet) (types.BigIn tpow := big2.Zero() { cst := cbor.NewCborStore(cs.Blockstore()) - state, err := state.LoadStateTree(cst, ts.ParentState(), hackgetNtwkVersionhack(nil, ts.Height())) // TODO: hackgetNtwkVersionhack: HELP + state, err := state.LoadStateTree(cst, ts.ParentState()) if err != nil { return types.NewInt(0), xerrors.Errorf("load state tree: %w", err) } diff --git a/chain/sub/incoming.go b/chain/sub/incoming.go index 7c672bee2..1af5d8188 100644 --- a/chain/sub/incoming.go +++ b/chain/sub/incoming.go @@ -435,7 +435,7 @@ func (bv *BlockValidator) checkPowerAndGetWorkerKey(ctx context.Context, bh *typ buf := bufbstore.NewBufferedBstore(bv.chain.Blockstore()) cst := cbor.NewCborStore(buf) - state, err := state.LoadStateTree(cst, st, bv.stmgr.GetNtwkVersion(ctx, ts.Height())) + state, err := state.LoadStateTree(cst, st) if err != nil { return address.Undef, err } diff --git a/chain/sync.go b/chain/sync.go index 1d8e456d8..bb3e50bdb 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -1045,10 +1045,9 @@ func (syncer *Syncer) checkBlockMessages(ctx context.Context, b *types.FullBlock if err != nil { return err } - nwVersion := syncer.sm.GetNtwkVersion(ctx, baseTs.Height()) cst := cbor.NewCborStore(syncer.store.Blockstore()) - st, err := state.LoadStateTree(cst, stateroot, nwVersion) + st, err := state.LoadStateTree(cst, stateroot) if err != nil { return xerrors.Errorf("failed to load base state tree: %w", err) } diff --git a/chain/types/state.go b/chain/types/state.go new file mode 100644 index 000000000..b99eb19c2 --- /dev/null +++ b/chain/types/state.go @@ -0,0 +1,15 @@ +package types + +import "github.com/ipfs/go-cid" + +type StateRoot struct { + // State root version. Versioned along with actors (for now). + Version uint64 + // Actors tree. The structure depends on the state root version. + Actors cid.Cid + // Info. The structure depends on the state root version. + Info cid.Cid +} + +// TODO: version this. +type StateInfo struct{} diff --git a/chain/vm/vm.go b/chain/vm/vm.go index aa334aad5..5afde5f49 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -170,8 +170,7 @@ type VMOpts struct { func NewVM(ctx context.Context, opts *VMOpts) (*VM, error) { buf := bufbstore.NewBufferedBstore(opts.Bstore) cst := cbor.NewCborStore(buf) - nwv := opts.NtwkVersion(ctx, opts.Epoch) // TODO: why do we need a context for this? - state, err := state.LoadStateTree(cst, opts.StateBase, nwv) + state, err := state.LoadStateTree(cst, opts.StateBase) if err != nil { return nil, err } diff --git a/cmd/lotus-shed/balances.go b/cmd/lotus-shed/balances.go index f3ffd140f..7dbfe2eb7 100644 --- a/cmd/lotus-shed/balances.go +++ b/cmd/lotus-shed/balances.go @@ -171,9 +171,8 @@ var chainBalanceStateCmd = &cli.Command{ sm := stmgr.NewStateManager(cs) - // NETUPGRADE: FIXME. // Options: (a) encode the version in the chain or (b) pass a flag. - tree, err := state.LoadStateTree(cst, sroot, network.Version0) + tree, err := state.LoadStateTree(cst, sroot) if err != nil { return err } diff --git a/cmd/lotus-shed/genesis-verify.go b/cmd/lotus-shed/genesis-verify.go index 62808db9b..043cb72bb 100644 --- a/cmd/lotus-shed/genesis-verify.go +++ b/cmd/lotus-shed/genesis-verify.go @@ -79,7 +79,7 @@ var genesisVerifyCmd = &cli.Command{ cst := cbor.NewCborStore(bs) - stree, err := state.LoadStateTree(cst, ts.ParentState(), sm.GetNtwkVersion()) + stree, err := state.LoadStateTree(cst, ts.ParentState()) if err != nil { return err } diff --git a/gen/main.go b/gen/main.go index e7586a92a..be227663c 100644 --- a/gen/main.go +++ b/gen/main.go @@ -26,6 +26,8 @@ func main() { types.BlockMsg{}, types.ExpTipSet{}, types.BeaconEntry{}, + types.StateRoot{}, + types.StateInfo{}, ) if err != nil { fmt.Println(err) diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 38e62f905..75f06e636 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -317,7 +317,7 @@ func (a *StateAPI) stateForTs(ctx context.Context, ts *types.TipSet) (*state.Sta buf := bufbstore.NewBufferedBstore(a.Chain.Blockstore()) cst := cbor.NewCborStore(buf) - return state.LoadStateTree(cst, st, a.StateManager.GetNtwkVersion(ctx, ts.Height())) + return state.LoadStateTree(cst, st) } func (a *StateAPI) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) { From 7fa6c1e8d04c3a68c1a37928af3c10c44b5fcff6 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 14 Sep 2020 15:45:00 -0700 Subject: [PATCH 104/885] add info method to miner state API --- chain/actors/builtin/miner/miner.go | 1 + 1 file changed, 1 insertion(+) diff --git a/chain/actors/builtin/miner/miner.go b/chain/actors/builtin/miner/miner.go index 76503d80e..154648986 100644 --- a/chain/actors/builtin/miner/miner.go +++ b/chain/actors/builtin/miner/miner.go @@ -35,6 +35,7 @@ type State interface { LoadDeadline(idx uint64) (Deadline, error) ForEachDeadline(cb func(idx uint64, dl Deadline) error) error NumDeadlines() (uint64, error) + Info() (MinerInfo, error) } type Deadline interface { From 7fd5c8167477911ddbaee5119755158167481350 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 15 Sep 2020 01:36:36 +0200 Subject: [PATCH 105/885] cli: state sector command --- cli/state.go | 79 ++++++++++++++++++++++++++++++ cli/util.go | 21 +++++++- cmd/lotus-storage-miner/proving.go | 33 +++---------- 3 files changed, 107 insertions(+), 26 deletions(-) diff --git a/cli/state.go b/cli/state.go index 1036e8fe5..bfcbdf94a 100644 --- a/cli/state.go +++ b/cli/state.go @@ -54,6 +54,7 @@ var stateCmd = &cli.Command{ stateListActorsCmd, stateListMinersCmd, stateCircSupplyCmd, + stateSectorCmd, stateGetActorCmd, stateLookupIDCmd, stateReplaySetCmd, @@ -119,6 +120,13 @@ var stateMinerInfo = &cli.Command{ } fmt.Println() + cd, err := api.StateMinerProvingDeadline(ctx, addr, ts.Key()) + if err != nil { + return xerrors.Errorf("getting miner info: %w", err) + } + + fmt.Printf("Proving Period Start:\t%s\n", EpochTime(cd.CurrentEpoch, cd.PeriodStart)) + return nil }, } @@ -1554,6 +1562,77 @@ var stateCircSupplyCmd = &cli.Command{ }, } +var stateSectorCmd = &cli.Command{ + Name: "sector", + Usage: "Get miner sector info", + ArgsUsage: "[miner address] [sector number]", + Action: func(cctx *cli.Context) error { + api, closer, err := GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + + ctx := ReqContext(cctx) + + if cctx.Args().Len() != 2 { + return xerrors.Errorf("expected 2 params") + } + + ts, err := LoadTipSet(ctx, cctx, api) + if err != nil { + return err + } + + if ts == nil { + ts, err = api.ChainHead(ctx) + if err != nil { + return err + } + } + + maddr, err := address.NewFromString(cctx.Args().Get(0)) + if err != nil { + return err + } + + sid, err := strconv.ParseInt(cctx.Args().Get(1), 10, 64) + if err != nil { + return err + } + + si, err := api.StateSectorGetInfo(ctx, maddr, abi.SectorNumber(sid), ts.Key()) + if err != nil { + return err + } + + fmt.Println("SectorNumber: ", si.SectorNumber) + fmt.Println("SealProof: ", si.SealProof) + fmt.Println("SealedCID: ", si.SealedCID) + fmt.Println("DealIDs: ", si.DealIDs) + fmt.Println() + fmt.Println("Activation: ", EpochTime(ts.Height(), si.Activation)) + fmt.Println("Expiration: ", EpochTime(ts.Height(), si.Expiration)) + fmt.Println() + fmt.Println("DealWeight: ", si.DealWeight) + fmt.Println("VerifiedDealWeight: ", si.VerifiedDealWeight) + fmt.Println("InitialPledge: ", types.FIL(si.InitialPledge)) + fmt.Println("ExpectedDayReward: ", types.FIL(si.ExpectedDayReward)) + fmt.Println("ExpectedStoragePledge: ", types.FIL(si.ExpectedStoragePledge)) + fmt.Println() + + sp, err := api.StateSectorPartition(ctx, maddr, abi.SectorNumber(sid), ts.Key()) + if err != nil { + return err + } + + fmt.Println("Deadline: ", sp.Deadline) + fmt.Println("Partition: ", sp.Partition) + + return nil + }, +} + var stateMarketCmd = &cli.Command{ Name: "market", Usage: "Inspect the storage market actor", diff --git a/cli/util.go b/cli/util.go index 4371f8bbc..762cdb565 100644 --- a/cli/util.go +++ b/cli/util.go @@ -2,10 +2,16 @@ package cli import ( "context" + "fmt" + "time" + + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" - "github.com/ipfs/go-cid" ) func parseTipSet(ctx context.Context, api api.FullNode, vals []string) (*types.TipSet, error) { @@ -26,3 +32,16 @@ func parseTipSet(ctx context.Context, api api.FullNode, vals []string) (*types.T return types.NewTipSet(headers) } + +func EpochTime(curr, e abi.ChainEpoch) string { + switch { + case curr > e: + return fmt.Sprintf("%d (%s ago)", e, time.Second*time.Duration(int64(build.BlockDelaySecs)*int64(curr-e))) + case curr == e: + return fmt.Sprintf("%d (now)", e) + case curr < e: + return fmt.Sprintf("%d (in %s)", e, time.Second*time.Duration(int64(build.BlockDelaySecs)*int64(e-curr))) + } + + panic("math broke") +} diff --git a/cmd/lotus-storage-miner/proving.go b/cmd/lotus-storage-miner/proving.go index b5087556d..bf5c709e4 100644 --- a/cmd/lotus-storage-miner/proving.go +++ b/cmd/lotus-storage-miner/proving.go @@ -3,18 +3,14 @@ package main import ( "bytes" "fmt" - "os" - "text/tabwriter" - "time" - "github.com/fatih/color" "github.com/urfave/cli/v2" "golang.org/x/xerrors" + "os" + "text/tabwriter" - "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/specs-actors/actors/builtin/miner" - "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" ) @@ -203,8 +199,8 @@ var provingInfoCmd = &cli.Command{ fmt.Printf("Current Epoch: %d\n", cd.CurrentEpoch) fmt.Printf("Proving Period Boundary: %d\n", cd.PeriodStart%miner.WPoStProvingPeriod) - fmt.Printf("Proving Period Start: %s\n", epochTime(cd.CurrentEpoch, cd.PeriodStart)) - fmt.Printf("Next Period Start: %s\n\n", epochTime(cd.CurrentEpoch, cd.PeriodStart+miner.WPoStProvingPeriod)) + fmt.Printf("Proving Period Start: %s\n", lcli.EpochTime(cd.CurrentEpoch, cd.PeriodStart)) + fmt.Printf("Next Period Start: %s\n\n", lcli.EpochTime(cd.CurrentEpoch, cd.PeriodStart+miner.WPoStProvingPeriod)) fmt.Printf("Faults: %d (%.2f%%)\n", faults, faultPerc) fmt.Printf("Recovering: %d\n", recovering) @@ -224,27 +220,14 @@ var provingInfoCmd = &cli.Command{ fmt.Printf("Deadline Sectors: %d\n", curDeadlineSectors) } - fmt.Printf("Deadline Open: %s\n", epochTime(cd.CurrentEpoch, cd.Open)) - fmt.Printf("Deadline Close: %s\n", epochTime(cd.CurrentEpoch, cd.Close)) - fmt.Printf("Deadline Challenge: %s\n", epochTime(cd.CurrentEpoch, cd.Challenge)) - fmt.Printf("Deadline FaultCutoff: %s\n", epochTime(cd.CurrentEpoch, cd.FaultCutoff)) + fmt.Printf("Deadline Open: %s\n", lcli.EpochTime(cd.CurrentEpoch, cd.Open)) + fmt.Printf("Deadline Close: %s\n", lcli.EpochTime(cd.CurrentEpoch, cd.Close)) + fmt.Printf("Deadline Challenge: %s\n", lcli.EpochTime(cd.CurrentEpoch, cd.Challenge)) + fmt.Printf("Deadline FaultCutoff: %s\n", lcli.EpochTime(cd.CurrentEpoch, cd.FaultCutoff)) return nil }, } -func epochTime(curr, e abi.ChainEpoch) string { - switch { - case curr > e: - return fmt.Sprintf("%d (%s ago)", e, time.Second*time.Duration(int64(build.BlockDelaySecs)*int64(curr-e))) - case curr == e: - return fmt.Sprintf("%d (now)", e) - case curr < e: - return fmt.Sprintf("%d (in %s)", e, time.Second*time.Duration(int64(build.BlockDelaySecs)*int64(e-curr))) - } - - panic("math broke") -} - var provingDeadlinesCmd = &cli.Command{ Name: "deadlines", Usage: "View the current proving period deadlines information", From 6acd3d602883d4c86182d3a3c93b8caa69a83733 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Mon, 14 Sep 2020 21:12:32 -0400 Subject: [PATCH 106/885] Fix MsigGetAvailableBalance --- node/impl/full/state.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/node/impl/full/state.go b/node/impl/full/state.go index e82086053..f9fcdab8a 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -883,9 +883,8 @@ func (a *StateAPI) MsigGetAvailableBalance(ctx context.Context, addr address.Add return act.Balance, nil } - minBalance := types.BigDiv(st.InitialBalance, types.NewInt(uint64(st.UnlockDuration))) - minBalance = types.BigMul(minBalance, types.NewInt(uint64(offset))) - return types.BigSub(act.Balance, minBalance), nil + al := st.AmountLocked(offset) + return types.BigSub(act.Balance, al), nil } func (a *StateAPI) MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error) { From 48e47ddc23f7330c92304b3c69140338859ecaf6 Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Fri, 11 Sep 2020 15:52:09 -0700 Subject: [PATCH 107/885] fix(paych): fix paych status command line --- api/api_full.go | 4 ++-- api/apistruct/struct.go | 12 ++++++------ cli/paych.go | 6 ++++-- markets/retrievaladapter/client.go | 2 +- node/impl/paych/paych.go | 4 ++-- 5 files changed, 15 insertions(+), 13 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 23226443a..cbf05b363 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -452,8 +452,8 @@ type FullNode interface { PaychGet(ctx context.Context, from, to address.Address, amt types.BigInt) (*ChannelInfo, error) PaychGetWaitReady(context.Context, cid.Cid) (address.Address, error) - PaychAvailableFunds(ch address.Address) (*ChannelAvailableFunds, error) - PaychAvailableFundsByFromTo(from, to address.Address) (*ChannelAvailableFunds, error) + PaychAvailableFunds(ctx context.Context, ch address.Address) (*ChannelAvailableFunds, error) + PaychAvailableFundsByFromTo(ctx context.Context, from, to address.Address) (*ChannelAvailableFunds, error) PaychList(context.Context) ([]address.Address, error) PaychStatus(context.Context, address.Address) (*PaychStatus, error) PaychSettle(context.Context, address.Address) (cid.Cid, error) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index fdb9843ff..53c8414ba 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -219,8 +219,8 @@ type FullNodeStruct struct { PaychGet func(ctx context.Context, from, to address.Address, amt types.BigInt) (*api.ChannelInfo, error) `perm:"sign"` PaychGetWaitReady func(context.Context, cid.Cid) (address.Address, error) `perm:"sign"` - PaychAvailableFunds func(address.Address) (*api.ChannelAvailableFunds, error) `perm:"sign"` - PaychAvailableFundsByFromTo func(address.Address, address.Address) (*api.ChannelAvailableFunds, error) `perm:"sign"` + PaychAvailableFunds func(context.Context, address.Address) (*api.ChannelAvailableFunds, error) `perm:"sign"` + PaychAvailableFundsByFromTo func(context.Context, address.Address, address.Address) (*api.ChannelAvailableFunds, error) `perm:"sign"` PaychList func(context.Context) ([]address.Address, error) `perm:"read"` PaychStatus func(context.Context, address.Address) (*api.PaychStatus, error) `perm:"read"` PaychSettle func(context.Context, address.Address) (cid.Cid, error) `perm:"sign"` @@ -944,12 +944,12 @@ func (c *FullNodeStruct) PaychGetWaitReady(ctx context.Context, sentinel cid.Cid return c.Internal.PaychGetWaitReady(ctx, sentinel) } -func (c *FullNodeStruct) PaychAvailableFunds(ch address.Address) (*api.ChannelAvailableFunds, error) { - return c.Internal.PaychAvailableFunds(ch) +func (c *FullNodeStruct) PaychAvailableFunds(ctx context.Context, ch address.Address) (*api.ChannelAvailableFunds, error) { + return c.Internal.PaychAvailableFunds(ctx, ch) } -func (c *FullNodeStruct) PaychAvailableFundsByFromTo(from, to address.Address) (*api.ChannelAvailableFunds, error) { - return c.Internal.PaychAvailableFundsByFromTo(from, to) +func (c *FullNodeStruct) PaychAvailableFundsByFromTo(ctx context.Context, from, to address.Address) (*api.ChannelAvailableFunds, error) { + return c.Internal.PaychAvailableFundsByFromTo(ctx, from, to) } func (c *FullNodeStruct) PaychList(ctx context.Context) ([]address.Address, error) { diff --git a/cli/paych.go b/cli/paych.go index 57fd1c142..12bcb20d7 100644 --- a/cli/paych.go +++ b/cli/paych.go @@ -103,6 +103,7 @@ var paychStatusByFromToCmd = &cli.Command{ if cctx.Args().Len() != 2 { return ShowHelp(cctx, fmt.Errorf("must pass two arguments: ")) } + ctx := ReqContext(cctx) from, err := address.NewFromString(cctx.Args().Get(0)) if err != nil { @@ -120,7 +121,7 @@ var paychStatusByFromToCmd = &cli.Command{ } defer closer() - avail, err := api.PaychAvailableFundsByFromTo(from, to) + avail, err := api.PaychAvailableFundsByFromTo(ctx, from, to) if err != nil { return err } @@ -138,6 +139,7 @@ var paychStatusCmd = &cli.Command{ if cctx.Args().Len() != 1 { return ShowHelp(cctx, fmt.Errorf("must pass an argument: ")) } + ctx := ReqContext(cctx) ch, err := address.NewFromString(cctx.Args().Get(0)) if err != nil { @@ -150,7 +152,7 @@ var paychStatusCmd = &cli.Command{ } defer closer() - avail, err := api.PaychAvailableFunds(ch) + avail, err := api.PaychAvailableFunds(ctx, ch) if err != nil { return err } diff --git a/markets/retrievaladapter/client.go b/markets/retrievaladapter/client.go index 17c56c167..ab81dd8c9 100644 --- a/markets/retrievaladapter/client.go +++ b/markets/retrievaladapter/client.go @@ -80,7 +80,7 @@ func (rcn *retrievalClientNode) WaitForPaymentChannelReady(ctx context.Context, func (rcn *retrievalClientNode) CheckAvailableFunds(ctx context.Context, paymentChannel address.Address) (retrievalmarket.ChannelAvailableFunds, error) { - channelAvailableFunds, err := rcn.payAPI.PaychAvailableFunds(paymentChannel) + channelAvailableFunds, err := rcn.payAPI.PaychAvailableFunds(ctx, paymentChannel) if err != nil { return retrievalmarket.ChannelAvailableFunds{}, err } diff --git a/node/impl/paych/paych.go b/node/impl/paych/paych.go index 84c2cc030..94fcc320d 100644 --- a/node/impl/paych/paych.go +++ b/node/impl/paych/paych.go @@ -39,11 +39,11 @@ func (a *PaychAPI) PaychGet(ctx context.Context, from, to address.Address, amt t }, nil } -func (a *PaychAPI) PaychAvailableFunds(ch address.Address) (*api.ChannelAvailableFunds, error) { +func (a *PaychAPI) PaychAvailableFunds(ctx context.Context, ch address.Address) (*api.ChannelAvailableFunds, error) { return a.PaychMgr.AvailableFunds(ch) } -func (a *PaychAPI) PaychAvailableFundsByFromTo(from, to address.Address) (*api.ChannelAvailableFunds, error) { +func (a *PaychAPI) PaychAvailableFundsByFromTo(ctx context.Context, from, to address.Address) (*api.ChannelAvailableFunds, error) { return a.PaychMgr.AvailableFundsByFromTo(from, to) } From ac8ef29d351b61d5d6cedcb8156aabb8d14c44f6 Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Fri, 11 Sep 2020 15:53:11 -0700 Subject: [PATCH 108/885] fix(paych): add paych status from to cmd add missing from to status to list --- cli/paych.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cli/paych.go b/cli/paych.go index 12bcb20d7..dc5b54c4a 100644 --- a/cli/paych.go +++ b/cli/paych.go @@ -29,6 +29,7 @@ var paychCmd = &cli.Command{ paychVoucherCmd, paychSettleCmd, paychStatusCmd, + paychStatusByFromToCmd, paychCloseCmd, }, } From 17e44af542404b9ffa6a7ca3a346beb1c2c98f6b Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Mon, 14 Sep 2020 18:44:45 -0700 Subject: [PATCH 109/885] docs(api): fix api docs --- documentation/en/api-methods.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index 27875eca1..d72beca98 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -2375,7 +2375,12 @@ There are not yet any comments for this method. Perms: sign -Inputs: `null` +Inputs: +```json +[ + "t01234" +] +``` Response: ```json @@ -2399,6 +2404,7 @@ Perms: sign Inputs: ```json [ + "t01234", "t01234" ] ``` From 7dc091052ad86d0f9f14f7b586340904d531656e Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Fri, 11 Sep 2020 12:04:54 -0700 Subject: [PATCH 110/885] feat(manager): less restrictive storage lock Use initial less restrictive storage lock when trying to read unsealed data before acquiring more restrictive lock needed for unsealing --- extern/sector-storage/manager.go | 44 ++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/extern/sector-storage/manager.go b/extern/sector-storage/manager.go index fe9bf5d45..73a5eb51e 100644 --- a/extern/sector-storage/manager.go +++ b/extern/sector-storage/manager.go @@ -203,25 +203,26 @@ func schedFetch(sector abi.SectorID, ft stores.SectorFileType, ptype stores.Path } } -func (m *Manager) ReadPiece(ctx context.Context, sink io.Writer, sector abi.SectorID, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, ticket abi.SealRandomness, unsealed cid.Cid) error { +func (m *Manager) tryReadUnsealedPiece(ctx context.Context, sink io.Writer, sector abi.SectorID, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (foundUnsealed bool, readOk bool, selector WorkerSelector, returnErr error) { + + // acquire a lock purely for reading unsealed sectors ctx, cancel := context.WithCancel(ctx) defer cancel() - if err := m.index.StorageLock(ctx, sector, stores.FTSealed|stores.FTCache, stores.FTUnsealed); err != nil { - return xerrors.Errorf("acquiring sector lock: %w", err) + if err := m.index.StorageLock(ctx, sector, stores.FTUnsealed, stores.FTNone); err != nil { + returnErr = xerrors.Errorf("acquiring read sector lock: %w", err) + return } // passing 0 spt because we only need it when allowFetch is true best, err := m.index.StorageFindSector(ctx, sector, stores.FTUnsealed, 0, false) if err != nil { - return xerrors.Errorf("read piece: checking for already existing unsealed sector: %w", err) + returnErr = xerrors.Errorf("read piece: checking for already existing unsealed sector: %w", err) + return } - var readOk bool - var selector WorkerSelector - if len(best) == 0 { // new - selector = newAllocSelector(m.index, stores.FTUnsealed, stores.PathSealing) - } else { // append to existing + foundUnsealed = len(best) > 0 + if foundUnsealed { // append to existing // There is unsealed sector, see if we can read from it selector = newExistingSelector(m.index, sector, stores.FTUnsealed, false) @@ -231,12 +232,27 @@ func (m *Manager) ReadPiece(ctx context.Context, sink io.Writer, sector abi.Sect return err }) if err != nil { - return xerrors.Errorf("reading piece from sealed sector: %w", err) + returnErr = xerrors.Errorf("reading piece from sealed sector: %w", err) } + } else { + selector = newAllocSelector(m.index, stores.FTUnsealed, stores.PathSealing) + } + return +} - if readOk { - return nil - } +func (m *Manager) ReadPiece(ctx context.Context, sink io.Writer, sector abi.SectorID, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, ticket abi.SealRandomness, unsealed cid.Cid) error { + foundUnsealed, readOk, selector, err := m.tryReadUnsealedPiece(ctx, sink, sector, offset, size) + if err != nil { + return err + } + if readOk { + return nil + } + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + if err := m.index.StorageLock(ctx, sector, stores.FTSealed|stores.FTCache, stores.FTUnsealed); err != nil { + return xerrors.Errorf("acquiring unseal sector lock: %w", err) } unsealFetch := func(ctx context.Context, worker Worker) error { @@ -244,7 +260,7 @@ func (m *Manager) ReadPiece(ctx context.Context, sink io.Writer, sector abi.Sect return xerrors.Errorf("copy sealed/cache sector data: %w", err) } - if len(best) > 0 { + if foundUnsealed { if err := worker.Fetch(ctx, sector, stores.FTUnsealed, stores.PathSealing, stores.AcquireMove); err != nil { return xerrors.Errorf("copy unsealed sector data: %w", err) } From 36f920bcd7487c0bb0fb81326109882fda9b98b9 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 14 Sep 2020 21:55:49 -0700 Subject: [PATCH 111/885] progress --- api/api_full.go | 2 +- chain/actors/builtin/miner/miner.go | 8 +++++ chain/actors/builtin/miner/v0.go | 19 +++++++++++ chain/stmgr/utils.go | 52 +++++++++++++---------------- extern/storage-sealing/sealing.go | 1 - storage/miner.go | 2 -- 6 files changed, 52 insertions(+), 32 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index a5cf2423d..035c7de43 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -17,8 +17,8 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/go-state-types/dline" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/specs-actors/actors/builtin/market" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/specs-actors/actors/builtin/paych" "github.com/filecoin-project/specs-actors/actors/builtin/power" "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" diff --git a/chain/actors/builtin/miner/miner.go b/chain/actors/builtin/miner/miner.go index 154648986..eb70d1457 100644 --- a/chain/actors/builtin/miner/miner.go +++ b/chain/actors/builtin/miner/miner.go @@ -9,6 +9,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/cbor" v0builtin "github.com/filecoin-project/specs-actors/actors/builtin" + v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/types" @@ -32,6 +33,9 @@ func Load(store adt.Store, act *types.Actor) (st State, err error) { type State interface { cbor.Marshaler + GetSector(abi.SectorNumber) (*SectorOnChainInfo, error) + GetPrecommittedSector(abi.SectorNumber) (*SectorPreCommitOnChainInfo, error) + LoadDeadline(idx uint64) (Deadline, error) ForEachDeadline(cb func(idx uint64, dl Deadline) error) error NumDeadlines() (uint64, error) @@ -51,6 +55,10 @@ type Partition interface { ActiveSectors() (bitfield.BitField, error) } +type SectorOnChainInfo = v0miner.SectorOnChainInfo +type SectorPreCommitInfo = v0miner.SectorPreCommitInfo +type SectorPreCommitOnChainInfo = v0miner.SectorPreCommitOnChainInfo + type MinerInfo struct { Owner address.Address // Must be an ID-address. Worker address.Address // Must be an ID-address. diff --git a/chain/actors/builtin/miner/v0.go b/chain/actors/builtin/miner/v0.go index 1d8a68183..40b9e4cc6 100644 --- a/chain/actors/builtin/miner/v0.go +++ b/chain/actors/builtin/miner/v0.go @@ -3,6 +3,7 @@ package miner import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-bitfield" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/libp2p/go-libp2p-core/peer" @@ -24,6 +25,24 @@ type v0Partition struct { store adt.Store } +func (s *v0State) GetSector(num abi.SectorNumber) (*SectorOnChainInfo, error) { + info, ok, err := s.State.GetSector(s.store, num) + if !ok || err != nil { + return nil, err + } + + return info, nil +} + +func (s *v0State) GetPrecommittedSector(num abi.SectorNumber) (*SectorPreCommitOnChainInfo, error) { + info, ok, err := s.State.GetPrecommittedSector(s.store, num) + if !ok || err != nil { + return nil, err + } + + return info, nil +} + func (s *v0State) LoadDeadline(idx uint64) (Deadline, error) { dls, err := s.State.LoadDeadlines(s.store) if err != nil { diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index 63e3e0f74..11eedb512 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -56,17 +56,21 @@ func GetNetworkName(ctx context.Context, sm *StateManager, st cid.Cid) (dtypes.N return "", err } - return dtypes.NetworkName(state.NetworkName), nil + return ias.NetworkName() } func GetMinerWorkerRaw(ctx context.Context, sm *StateManager, st cid.Cid, maddr address.Address) (address.Address, error) { - act, err := sm.LoadActorRaw(ctx, sm, maddr, st) + state, err := sm.StateTree(st) if err != nil { - return address.Undef, xerrors.Errorf("(get sset) failed to load miner actor state: %w", err) + return address.Undef, xerrors.Errorf("(get sset) failed to load state tree: %w", err) + } + act, err := state.GetActor(maddr) + if err != nil { + return address.Undef, xerrors.Errorf("(get sset) failed to load miner actor: %w", err) } mas, err := miner.Load(sm.cs.Store(ctx), act) if err != nil { - return address.Undef, xerrors.Errorf("load state tree: %w", err) + return address.Undef, xerrors.Errorf("(get sset) failed to load miner actor state: %w", err) } info, err := mas.Info() @@ -74,7 +78,7 @@ func GetMinerWorkerRaw(ctx context.Context, sm *StateManager, st cid.Cid, maddr return address.Undef, xerrors.Errorf("failed to load actor info: %w", err) } - return vm.ResolveToKeyAddr(state, cst, info.Worker) + return vm.ResolveToKeyAddr(state, sm.cs.Store(ctx), info.Worker) } func GetPower(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) (power.Claim, power.Claim, error) { @@ -108,40 +112,32 @@ func GetPowerRaw(ctx context.Context, sm *StateManager, st cid.Cid, maddr addres return mpow, tpow, nil } -func PreCommitInfo(ctx context.Context, sm *StateManager, maddr address.Address, sid abi.SectorNumber, ts *types.TipSet) (miner.SectorPreCommitOnChainInfo, error) { - var mas miner.State +func PreCommitInfo(ctx context.Context, sm *StateManager, maddr address.Address, sid abi.SectorNumber, ts *types.TipSet) (*miner.SectorPreCommitOnChainInfo, error) { act, err := sm.LoadActor(ctx, maddr, ts) if err != nil { - return miner.SectorPreCommitOnChainInfo{}, xerrors.Errorf("(get sset) failed to load miner actor state: %w", err) + return nil, xerrors.Errorf("(get sset) failed to load miner actor: %w", err) } - i, ok, err := mas.GetPrecommittedSector(sm.cs.Store(ctx), sid) - if err != nil { - return miner.SectorPreCommitOnChainInfo{}, err - } - if !ok { - return miner.SectorPreCommitOnChainInfo{}, xerrors.New("precommit not found") - } - - return *i, nil -} - -func MinerSectorInfo(ctx context.Context, sm *StateManager, maddr address.Address, sid abi.SectorNumber, ts *types.TipSet) (*miner.SectorOnChainInfo, error) { - var mas miner.State - _, err := sm.LoadActorState(ctx, maddr, &mas, ts) + mas, err := miner.Load(sm.cs.Store(ctx), act) if err != nil { return nil, xerrors.Errorf("(get sset) failed to load miner actor state: %w", err) } - sectorInfo, ok, err := mas.GetSector(sm.cs.Store(ctx), sid) + return mas.GetPrecommittedSector(sid) +} + +func MinerSectorInfo(ctx context.Context, sm *StateManager, maddr address.Address, sid abi.SectorNumber, ts *types.TipSet) (*miner.SectorOnChainInfo, error) { + act, err := sm.LoadActor(ctx, maddr, ts) if err != nil { - return nil, err - } - if !ok { - return nil, nil + return nil, xerrors.Errorf("(get sset) failed to load miner actor: %w", err) } - return sectorInfo, nil + mas, err := miner.Load(sm.cs.Store(ctx), act) + if err != nil { + return nil, xerrors.Errorf("(get sset) failed to load miner actor state: %w", err) + } + + return mas.GetSector(sid) } func GetMinerSectorSet(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address, filter *bitfield.BitField, filterOut bool) ([]*api.ChainSectorInfo, error) { diff --git a/extern/storage-sealing/sealing.go b/extern/storage-sealing/sealing.go index cb73182d3..533333860 100644 --- a/extern/storage-sealing/sealing.go +++ b/extern/storage-sealing/sealing.go @@ -50,7 +50,6 @@ type SealingAPI interface { StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok TipSetToken) (*SectorLocation, error) StateMinerSectorSize(context.Context, address.Address, TipSetToken) (abi.SectorSize, error) StateMinerWorkerAddress(ctx context.Context, maddr address.Address, tok TipSetToken) (address.Address, error) - StateMinerDeadlines(ctx context.Context, maddr address.Address, tok TipSetToken) ([]*miner.Deadline, error) StateMinerPreCommitDepositForPower(context.Context, address.Address, miner.SectorPreCommitInfo, TipSetToken) (big.Int, error) StateMinerInitialPledgeCollateral(context.Context, address.Address, miner.SectorPreCommitInfo, TipSetToken) (big.Int, error) StateMarketStorageDeal(context.Context, abi.DealID, TipSetToken) (market.DealProposal, error) diff --git a/storage/miner.go b/storage/miner.go index bf026fc30..d2af4be2c 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -67,8 +67,6 @@ type SealingStateEvt struct { type storageMinerApi interface { // Call a read only method on actors (no interaction with the chain required) StateCall(context.Context, *types.Message, types.TipSetKey) (*api.InvocResult, error) - StateMinerDeadlines(ctx context.Context, maddr address.Address, tok types.TipSetKey) ([]*miner.Deadline, error) - StateMinerPartitions(context.Context, address.Address, uint64, types.TipSetKey) ([]*miner.Partition, error) StateMinerSectors(context.Context, address.Address, *bitfield.BitField, bool, types.TipSetKey) ([]*api.ChainSectorInfo, error) StateSectorPreCommitInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) StateSectorGetInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorOnChainInfo, error) From f0f15f899c2c450ee923e33f9d9b269384dc5d8c Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Wed, 9 Sep 2020 15:01:37 +0200 Subject: [PATCH 112/885] feat: split window PoST messages into batches --- storage/wdpost_run.go | 288 ++++++++++++++++++++-------------- storage/wdpost_run_test.go | 305 +++++++++++++++++++++++++++++++++++++ 2 files changed, 480 insertions(+), 113 deletions(-) create mode 100644 storage/wdpost_run_test.go diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 08a7437a9..1c5d5650c 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -3,7 +3,6 @@ package storage import ( "bytes" "context" - "errors" "time" "github.com/filecoin-project/go-state-types/dline" @@ -30,8 +29,6 @@ import ( "github.com/filecoin-project/lotus/journal" ) -var errNoPartitions = errors.New("no partitions") - func (s *WindowPoStScheduler) failPost(err error, deadline *dline.Info) { journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} { return WdPoStSchedulerEvt{ @@ -79,25 +76,28 @@ func (s *WindowPoStScheduler) doPost(ctx context.Context, deadline *dline.Info, }) } - proof, err := s.runPost(ctx, *deadline, ts) - switch err { - case errNoPartitions: - recordProofsEvent(nil, cid.Undef) - return - case nil: - sm, err := s.submitPost(ctx, proof) - if err != nil { - log.Errorf("submitPost failed: %+v", err) - s.failPost(err, deadline) - return - } - recordProofsEvent(proof.Partitions, sm.Cid()) - default: + posts, err := s.runPost(ctx, *deadline, ts) + if err != nil { log.Errorf("runPost failed: %+v", err) s.failPost(err, deadline) return } + if len(posts) == 0 { + recordProofsEvent(nil, cid.Undef) + return + } + + for i := range posts { + post := &posts[i] + sm, err := s.submitPost(ctx, &posts[i]) + if err != nil { + log.Errorf("submitPost failed: %+v", err) + s.failPost(err, deadline) + } + recordProofsEvent(post.Partitions, sm.Cid()) + } + journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} { return WdPoStSchedulerEvt{ evtCommon: s.getEvtCommon(nil), @@ -327,7 +327,7 @@ func (s *WindowPoStScheduler) checkNextFaults(ctx context.Context, dlIdx uint64, return faults, sm, nil } -func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *types.TipSet) (*miner.SubmitWindowedPoStParams, error) { +func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *types.TipSet) ([]miner.SubmitWindowedPoStParams, error) { ctx, span := trace.StartSpan(ctx, "storage.runPost") defer span.End() @@ -399,136 +399,198 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty return nil, xerrors.Errorf("failed to get chain randomness for windowPost (ts=%d; deadline=%d): %w", ts.Height(), di, err) } + // Get the partitions for the given deadline partitions, err := s.api.StateMinerPartitions(ctx, s.actor, di.Index, ts.Key()) if err != nil { return nil, xerrors.Errorf("getting partitions: %w", err) } - params := &miner.SubmitWindowedPoStParams{ - Deadline: di.Index, - Partitions: make([]miner.PoStPartition, 0, len(partitions)), - Proofs: nil, + // Split partitions into batches, so as not to exceed the number of sectors + // allowed in a single message + partitionBatches, err := s.batchPartitions(partitions) + if err != nil { + return nil, err } - skipCount := uint64(0) - postSkipped := bitfield.New() - var postOut []proof.PoStProof + posts := make([]miner.SubmitWindowedPoStParams, 0, len(partitionBatches)) + for batchIdx, batch := range partitionBatches { + batchPartitionStartIdx := 0 + for _, batch := range partitionBatches[:batchIdx] { + batchPartitionStartIdx += len(batch) + } - for retries := 0; retries < 5; retries++ { + params := miner.SubmitWindowedPoStParams{ + Deadline: di.Index, + Partitions: make([]miner.PoStPartition, 0, len(batch)), + Proofs: nil, + } + + //var sinfos []proof.SectorInfo + //sidToPart := map[abi.SectorNumber]uint64{} + //skipCount := uint64(0) + + skipCount := uint64(0) + postSkipped := bitfield.New() + var postOut []proof.PoStProof var sinfos []proof.SectorInfo - sidToPart := map[abi.SectorNumber]int{} - for partIdx, partition := range partitions { - // TODO: Can do this in parallel - toProve, err := partition.ActiveSectors() + for retries := 0; retries < 5; retries++ { + sinfos = make([]proof.SectorInfo, 0) + + for partIdx, partition := range batch { + // TODO: Can do this in parallel + toProve, err := partition.ActiveSectors() + if err != nil { + return nil, xerrors.Errorf("getting active sectors: %w", err) + } + + toProve, err = bitfield.MergeBitFields(toProve, partition.Recoveries) + if err != nil { + return nil, xerrors.Errorf("adding recoveries to set of sectors to prove: %w", err) + } + + good, err := s.checkSectors(ctx, toProve) + if err != nil { + return nil, xerrors.Errorf("checking sectors to skip: %w", err) + } + + good, err = bitfield.SubtractBitField(good, postSkipped) + if err != nil { + return nil, xerrors.Errorf("toProve - postSkipped: %w", err) + } + + skipped, err := bitfield.SubtractBitField(toProve, good) + if err != nil { + return nil, xerrors.Errorf("toProve - good: %w", err) + } + + sc, err := skipped.Count() + if err != nil { + return nil, xerrors.Errorf("getting skipped sector count: %w", err) + } + + skipCount += sc + + ssi, err := s.sectorsForProof(ctx, good, partition.Sectors, ts) + if err != nil { + return nil, xerrors.Errorf("getting sorted sector info: %w", err) + } + + if len(ssi) == 0 { + continue + } + + sinfos = append(sinfos, ssi...) + params.Partitions = append(params.Partitions, miner.PoStPartition{ + Index: uint64(batchPartitionStartIdx + partIdx), + Skipped: skipped, + }) + } + + if len(sinfos) == 0 { + // nothing to prove.. + //return nil, errNoPartitions + break + } + + log.Infow("running windowPost", + "chain-random", rand, + "deadline", di, + "height", ts.Height(), + "skipped", skipCount) + + tsStart := build.Clock.Now() + + mid, err := address.IDFromAddress(s.actor) if err != nil { - return nil, xerrors.Errorf("getting active sectors: %w", err) + return nil, err } - toProve, err = bitfield.MergeBitFields(toProve, partition.Recoveries) - if err != nil { - return nil, xerrors.Errorf("adding recoveries to set of sectors to prove: %w", err) + var ps []abi.SectorID + postOut, ps, err = s.prover.GenerateWindowPoSt(ctx, abi.ActorID(mid), sinfos, abi.PoStRandomness(rand)) + elapsed := time.Since(tsStart) + + log.Infow("computing window PoSt", "batch", batchIdx, "elapsed", elapsed) + + if err == nil { + break } - good, err := s.checkSectors(ctx, toProve) - if err != nil { - return nil, xerrors.Errorf("checking sectors to skip: %w", err) + if len(ps) == 0 { + return nil, xerrors.Errorf("running post failed: %w", err) } - good, err = bitfield.SubtractBitField(good, postSkipped) - if err != nil { - return nil, xerrors.Errorf("toProve - postSkipped: %w", err) + log.Warnw("generate window PoSt skipped sectors", "sectors", ps, "error", err, "try", retries) + + skipCount += uint64(len(ps)) + for _, sector := range ps { + postSkipped.Set(uint64(sector.Number)) } - - skipped, err := bitfield.SubtractBitField(toProve, good) - if err != nil { - return nil, xerrors.Errorf("toProve - good: %w", err) - } - - sc, err := skipped.Count() - if err != nil { - return nil, xerrors.Errorf("getting skipped sector count: %w", err) - } - - skipCount += sc - - ssi, err := s.sectorsForProof(ctx, good, partition.Sectors, ts) - if err != nil { - return nil, xerrors.Errorf("getting sorted sector info: %w", err) - } - - if len(ssi) == 0 { - continue - } - - sinfos = append(sinfos, ssi...) - for _, si := range ssi { - sidToPart[si.SectorNumber] = partIdx - } - - params.Partitions = append(params.Partitions, miner.PoStPartition{ - Index: uint64(partIdx), - Skipped: skipped, - }) } if len(sinfos) == 0 { - // nothing to prove.. - return nil, errNoPartitions + continue } - log.Infow("running windowPost", - "chain-random", rand, - "deadline", di, - "height", ts.Height(), - "skipped", skipCount) - - tsStart := build.Clock.Now() - - mid, err := address.IDFromAddress(s.actor) - if err != nil { - return nil, err + if len(postOut) == 0 { + return nil, xerrors.Errorf("received no proofs back from generate window post") } - var ps []abi.SectorID - postOut, ps, err = s.prover.GenerateWindowPoSt(ctx, abi.ActorID(mid), sinfos, abi.PoStRandomness(rand)) - elapsed := time.Since(tsStart) + params.Proofs = postOut - log.Infow("computing window PoSt", "elapsed", elapsed) - - if err == nil { - break - } - - if len(ps) == 0 { - return nil, xerrors.Errorf("running post failed: %w", err) - } - - log.Warnw("generate window PoSt skipped sectors", "sectors", ps, "error", err, "try", retries) - - skipCount += uint64(len(ps)) - for _, sector := range ps { - postSkipped.Set(uint64(sector.Number)) - } + posts = append(posts, params) } - if len(postOut) == 0 { - return nil, xerrors.Errorf("received no proofs back from generate window post") - } - - params.Proofs = postOut - commEpoch := di.Open commRand, err := s.api.ChainGetRandomnessFromTickets(ctx, ts.Key(), crypto.DomainSeparationTag_PoStChainCommit, commEpoch, nil) if err != nil { return nil, xerrors.Errorf("failed to get chain randomness for windowPost (ts=%d; deadline=%d): %w", ts.Height(), di, err) } - params.ChainCommitEpoch = commEpoch - params.ChainCommitRand = commRand - log.Infow("submitting window PoSt") + for i := range posts { + posts[i].ChainCommitEpoch = commEpoch + posts[i].ChainCommitRand = commRand + } - return params, nil + return posts, nil +} + +func (s *WindowPoStScheduler) batchPartitions(partitions []*miner.Partition) ([][]*miner.Partition, error) { + // Get the number of sectors allowed in a partition, for this proof size + sectorsPerPartition, err := builtin.PoStProofWindowPoStPartitionSectors(s.proofType) + if err != nil { + return nil, xerrors.Errorf("getting sectors per partition: %w", err) + } + + // We don't want to exceed the number of sectors allowed in a message. + // So given the number of sectors in a partition, work out the number of + // partitions that can be in a message without exceeding sectors per + // message: + // floor(number of sectors allowed in a message / sectors per partition) + // eg: + // max sectors per message 7: ooooooo + // sectors per partition 3: ooo + // partitions per message 2: oooOOO + // <1><2> (3rd doesn't fit) + partitionsPerMsg := int(miner.AddressedSectorsMax / sectorsPerPartition) + + // The number of messages will be: + // ceiling(number of partitions / partitions per message) + batchCount := len(partitions) / partitionsPerMsg + if len(partitions)%partitionsPerMsg != 0 { + batchCount++ + } + + // Split the partitions into batches + batches := make([][]*miner.Partition, 0, batchCount) + for i := 0; i < len(partitions); i += partitionsPerMsg { + end := i + partitionsPerMsg + if end > len(partitions) { + end = len(partitions) + } + batches = append(batches, partitions[i:end]) + } + return batches, nil } func (s *WindowPoStScheduler) sectorsForProof(ctx context.Context, goodSectors, allSectors bitfield.BitField, ts *types.TipSet) ([]proof.SectorInfo, error) { diff --git a/storage/wdpost_run_test.go b/storage/wdpost_run_test.go new file mode 100644 index 000000000..af529a75e --- /dev/null +++ b/storage/wdpost_run_test.go @@ -0,0 +1,305 @@ +package storage + +import ( + "bytes" + "context" + "testing" + + "github.com/filecoin-project/go-state-types/dline" + + "golang.org/x/xerrors" + + "github.com/stretchr/testify/require" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-bitfield" + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/specs-actors/actors/builtin" + "github.com/filecoin-project/specs-actors/actors/builtin/miner" + "github.com/filecoin-project/specs-actors/actors/runtime/proof" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/crypto" + tutils "github.com/filecoin-project/specs-actors/support/testing" +) + +type mockStorageMinerAPI struct { + partitions []*miner.Partition + pushedMessages chan *types.Message +} + +func newMockStorageMinerAPI() *mockStorageMinerAPI { + return &mockStorageMinerAPI{ + pushedMessages: make(chan *types.Message), + } +} + +func (m *mockStorageMinerAPI) ChainGetRandomnessFromTickets(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error) { + return abi.Randomness("ticket rand"), nil +} + +func (m *mockStorageMinerAPI) ChainGetRandomnessFromBeacon(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error) { + return abi.Randomness("beacon rand"), nil +} + +func (m *mockStorageMinerAPI) setPartitions(ps []*miner.Partition) { + m.partitions = append(m.partitions, ps...) +} + +func (m *mockStorageMinerAPI) StateMinerPartitions(ctx context.Context, address address.Address, u uint64, key types.TipSetKey) ([]*miner.Partition, error) { + return m.partitions, nil +} + +func (m *mockStorageMinerAPI) StateMinerSectors(ctx context.Context, address address.Address, field *bitfield.BitField, b bool, key types.TipSetKey) ([]*api.ChainSectorInfo, error) { + var sis []*api.ChainSectorInfo + _ = field.ForEach(func(i uint64) error { + sis = append(sis, &api.ChainSectorInfo{ + Info: miner.SectorOnChainInfo{ + SectorNumber: abi.SectorNumber(i), + }, + ID: abi.SectorNumber(i), + }) + return nil + }) + return sis, nil +} + +func (m *mockStorageMinerAPI) StateMinerInfo(ctx context.Context, address address.Address, key types.TipSetKey) (api.MinerInfo, error) { + return api.MinerInfo{}, xerrors.Errorf("err") +} + +func (m *mockStorageMinerAPI) MpoolPushMessage(ctx context.Context, message *types.Message, spec *api.MessageSendSpec) (*types.SignedMessage, error) { + m.pushedMessages <- message + return &types.SignedMessage{ + Message: *message, + }, nil +} + +func (m *mockStorageMinerAPI) StateWaitMsg(ctx context.Context, cid cid.Cid, confidence uint64) (*api.MsgLookup, error) { + return &api.MsgLookup{ + Receipt: types.MessageReceipt{ + ExitCode: 0, + }, + }, nil +} + +type mockProver struct { +} + +func (m *mockProver) GenerateWinningPoSt(context.Context, abi.ActorID, []proof.SectorInfo, abi.PoStRandomness) ([]proof.PoStProof, error) { + panic("implement me") +} + +func (m *mockProver) GenerateWindowPoSt(ctx context.Context, aid abi.ActorID, sis []proof.SectorInfo, pr abi.PoStRandomness) ([]proof.PoStProof, []abi.SectorID, error) { + return []proof.PoStProof{ + { + PoStProof: abi.RegisteredPoStProof_StackedDrgWindow2KiBV1, + ProofBytes: []byte("post-proof"), + }, + }, nil, nil +} + +type mockFaultTracker struct { +} + +func (m mockFaultTracker) CheckProvable(ctx context.Context, spt abi.RegisteredSealProof, sectors []abi.SectorID) ([]abi.SectorID, error) { + // Returns "bad" sectors so just return nil meaning all sectors are good + return nil, nil +} + +// TestWDPostDoPost verifies that doPost will send the correct number of window +// PoST messages for a given number of partitions +func TestWDPostDoPost(t *testing.T) { + ctx := context.Background() + expectedMsgCount := 5 + + proofType := abi.RegisteredPoStProof_StackedDrgWindow2KiBV1 + postAct := tutils.NewIDAddr(t, 100) + workerAct := tutils.NewIDAddr(t, 101) + + mockStgMinerAPI := newMockStorageMinerAPI() + + // Get the number of sectors allowed in a partition for this proof type + sectorsPerPartition, err := builtin.PoStProofWindowPoStPartitionSectors(proofType) + require.NoError(t, err) + // Work out the number of partitions that can be included in a message + // without exceeding the message sector limit + partitionsPerMsg := int(miner.AddressedSectorsMax / sectorsPerPartition) + + // Enough partitions to fill expectedMsgCount-1 messages + partitionCount := (expectedMsgCount - 1) * partitionsPerMsg + // Add an extra partition that should be included in the last message + partitionCount++ + + var partitions []*miner.Partition + for p := 0; p < partitionCount; p++ { + sectors := bitfield.New() + for s := uint64(0); s < sectorsPerPartition; s++ { + sectors.Set(s) + } + partitions = append(partitions, &miner.Partition{ + Sectors: sectors, + }) + } + mockStgMinerAPI.setPartitions(partitions) + + // Run window PoST + scheduler := &WindowPoStScheduler{ + api: mockStgMinerAPI, + prover: &mockProver{}, + faultTracker: &mockFaultTracker{}, + proofType: proofType, + actor: postAct, + worker: workerAct, + } + + di := &dline.Info{} + ts := mockTipSet(t) + scheduler.doPost(ctx, di, ts) + + // Read the window PoST messages + for i := 0; i < expectedMsgCount; i++ { + msg := <-mockStgMinerAPI.pushedMessages + require.Equal(t, builtin.MethodsMiner.SubmitWindowedPoSt, msg.Method) + var params miner.SubmitWindowedPoStParams + err := params.UnmarshalCBOR(bytes.NewReader(msg.Params)) + require.NoError(t, err) + + if i == expectedMsgCount-1 { + // In the last message we only included a single partition (see above) + require.Len(t, params.Partitions, 1) + } else { + // All previous messages should include the full number of partitions + require.Len(t, params.Partitions, partitionsPerMsg) + } + } +} + +func mockTipSet(t *testing.T) *types.TipSet { + minerAct := tutils.NewActorAddr(t, "miner") + c, err := cid.Decode("QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH") + require.NoError(t, err) + blks := []*types.BlockHeader{ + { + Miner: minerAct, + Height: abi.ChainEpoch(1), + ParentStateRoot: c, + ParentMessageReceipts: c, + Messages: c, + }, + } + ts, err := types.NewTipSet(blks) + require.NoError(t, err) + return ts +} + +// +// All the mock methods below here are unused +// + +func (m *mockStorageMinerAPI) StateCall(ctx context.Context, message *types.Message, key types.TipSetKey) (*api.InvocResult, error) { + panic("implement me") +} + +func (m *mockStorageMinerAPI) StateMinerDeadlines(ctx context.Context, maddr address.Address, tok types.TipSetKey) ([]*miner.Deadline, error) { + panic("implement me") +} + +func (m *mockStorageMinerAPI) StateSectorPreCommitInfo(ctx context.Context, address address.Address, number abi.SectorNumber, key types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) { + panic("implement me") +} + +func (m *mockStorageMinerAPI) StateSectorGetInfo(ctx context.Context, address address.Address, number abi.SectorNumber, key types.TipSetKey) (*miner.SectorOnChainInfo, error) { + panic("implement me") +} + +func (m *mockStorageMinerAPI) StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*api.SectorLocation, error) { + panic("implement me") +} + +func (m *mockStorageMinerAPI) StateMinerProvingDeadline(ctx context.Context, address address.Address, key types.TipSetKey) (*dline.Info, error) { + panic("implement me") +} + +func (m *mockStorageMinerAPI) StateMinerPreCommitDepositForPower(ctx context.Context, address address.Address, info miner.SectorPreCommitInfo, key types.TipSetKey) (types.BigInt, error) { + panic("implement me") +} + +func (m *mockStorageMinerAPI) StateMinerInitialPledgeCollateral(ctx context.Context, address address.Address, info miner.SectorPreCommitInfo, key types.TipSetKey) (types.BigInt, error) { + panic("implement me") +} + +func (m *mockStorageMinerAPI) StateSearchMsg(ctx context.Context, cid cid.Cid) (*api.MsgLookup, error) { + panic("implement me") +} + +func (m *mockStorageMinerAPI) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) { + panic("implement me") +} + +func (m *mockStorageMinerAPI) StateGetReceipt(ctx context.Context, cid cid.Cid, key types.TipSetKey) (*types.MessageReceipt, error) { + panic("implement me") +} + +func (m *mockStorageMinerAPI) StateMarketStorageDeal(ctx context.Context, id abi.DealID, key types.TipSetKey) (*api.MarketDeal, error) { + panic("implement me") +} + +func (m *mockStorageMinerAPI) StateMinerFaults(ctx context.Context, address address.Address, key types.TipSetKey) (bitfield.BitField, error) { + panic("implement me") +} + +func (m *mockStorageMinerAPI) StateMinerRecoveries(ctx context.Context, address address.Address, key types.TipSetKey) (bitfield.BitField, error) { + panic("implement me") +} + +func (m *mockStorageMinerAPI) StateAccountKey(ctx context.Context, address address.Address, key types.TipSetKey) (address.Address, error) { + panic("implement me") +} + +func (m *mockStorageMinerAPI) GasEstimateMessageGas(ctx context.Context, message *types.Message, spec *api.MessageSendSpec, key types.TipSetKey) (*types.Message, error) { + panic("implement me") +} + +func (m *mockStorageMinerAPI) ChainHead(ctx context.Context) (*types.TipSet, error) { + panic("implement me") +} + +func (m *mockStorageMinerAPI) ChainNotify(ctx context.Context) (<-chan []*api.HeadChange, error) { + panic("implement me") +} + +func (m *mockStorageMinerAPI) ChainGetTipSetByHeight(ctx context.Context, epoch abi.ChainEpoch, key types.TipSetKey) (*types.TipSet, error) { + panic("implement me") +} + +func (m *mockStorageMinerAPI) ChainGetBlockMessages(ctx context.Context, cid cid.Cid) (*api.BlockMessages, error) { + panic("implement me") +} + +func (m *mockStorageMinerAPI) ChainReadObj(ctx context.Context, cid cid.Cid) ([]byte, error) { + panic("implement me") +} + +func (m *mockStorageMinerAPI) ChainHasObj(ctx context.Context, cid cid.Cid) (bool, error) { + panic("implement me") +} + +func (m *mockStorageMinerAPI) ChainGetTipSet(ctx context.Context, key types.TipSetKey) (*types.TipSet, error) { + panic("implement me") +} + +func (m *mockStorageMinerAPI) WalletSign(ctx context.Context, address address.Address, bytes []byte) (*crypto.Signature, error) { + panic("implement me") +} + +func (m *mockStorageMinerAPI) WalletBalance(ctx context.Context, address address.Address) (types.BigInt, error) { + panic("implement me") +} + +func (m *mockStorageMinerAPI) WalletHas(ctx context.Context, address address.Address) (bool, error) { + panic("implement me") +} From db998e9c0f06d9282ef7751cf384253f3b7c2d2c Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Thu, 10 Sep 2020 10:21:07 +0200 Subject: [PATCH 113/885] fix: storage - move rand generation after proofs --- storage/wdpost_run.go | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 1c5d5650c..7aa90bbba 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -90,12 +90,13 @@ func (s *WindowPoStScheduler) doPost(ctx context.Context, deadline *dline.Info, for i := range posts { post := &posts[i] - sm, err := s.submitPost(ctx, &posts[i]) + sm, err := s.submitPost(ctx, post) if err != nil { log.Errorf("submitPost failed: %+v", err) s.failPost(err, deadline) + } else { + recordProofsEvent(post.Partitions, sm.Cid()) } - recordProofsEvent(post.Partitions, sm.Cid()) } journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} { @@ -412,6 +413,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty return nil, err } + // Generate proofs in batches posts := make([]miner.SubmitWindowedPoStParams, 0, len(partitionBatches)) for batchIdx, batch := range partitionBatches { batchPartitionStartIdx := 0 @@ -425,18 +427,12 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty Proofs: nil, } - //var sinfos []proof.SectorInfo - //sidToPart := map[abi.SectorNumber]uint64{} - //skipCount := uint64(0) - skipCount := uint64(0) postSkipped := bitfield.New() var postOut []proof.PoStProof - var sinfos []proof.SectorInfo - + somethingToProve := true for retries := 0; retries < 5; retries++ { - sinfos = make([]proof.SectorInfo, 0) - + var sinfos []proof.SectorInfo for partIdx, partition := range batch { // TODO: Can do this in parallel toProve, err := partition.ActiveSectors() @@ -488,11 +484,12 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty } if len(sinfos) == 0 { - // nothing to prove.. - //return nil, errNoPartitions + // nothing to prove for this batch + somethingToProve = false break } + // Generate proof log.Infow("running windowPost", "chain-random", rand, "deadline", di, @@ -513,9 +510,12 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty log.Infow("computing window PoSt", "batch", batchIdx, "elapsed", elapsed) if err == nil { + // Proof generation successful, stop retrying break } + // Proof generation failed, so retry + if len(ps) == 0 { return nil, xerrors.Errorf("running post failed: %w", err) } @@ -528,7 +528,8 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty } } - if len(sinfos) == 0 { + // Nothing to prove for this batch, try the next batch + if !somethingToProve { continue } @@ -541,10 +542,12 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty posts = append(posts, params) } + // Compute randomness after generating proofs so as to reduce the impact + // of chain reorgs (which change randomness) commEpoch := di.Open commRand, err := s.api.ChainGetRandomnessFromTickets(ctx, ts.Key(), crypto.DomainSeparationTag_PoStChainCommit, commEpoch, nil) if err != nil { - return nil, xerrors.Errorf("failed to get chain randomness for windowPost (ts=%d; deadline=%d): %w", ts.Height(), di, err) + return nil, xerrors.Errorf("failed to get chain randomness for windowPost (ts=%d; deadline=%d): %w", ts.Height(), commEpoch, err) } for i := range posts { From 285fc69f6a3f00979cb3041bce63ac4af0041a06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 15 Sep 2020 11:45:03 +0200 Subject: [PATCH 114/885] gateway: address review --- .gitignore | 1 + Makefile | 6 ++++++ cmd/lotus-gateway/main.go | 4 ++-- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 0424c1f24..940f37b3f 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ /lotus-fountain /lotus-stats /lotus-bench +/lotus-gateway /bench.json /lotuspond/front/node_modules /lotuspond/front/build diff --git a/Makefile b/Makefile index 4f6ece417..56ab361ec 100644 --- a/Makefile +++ b/Makefile @@ -92,6 +92,12 @@ lotus-shed: $(BUILD_DEPS) .PHONY: lotus-shed BINS+=lotus-shed +lotus-gateway: $(BUILD_DEPS) + rm -f lotus-gateway + go build $(GOFLAGS) -o lotus-gateway ./cmd/lotus-gateway +.PHONY: lotus-gateway +BINS+=lotus-gateway + build: lotus lotus-miner lotus-worker @[[ $$(type -P "lotus") ]] && echo "Caution: you have \ an existing lotus binary in your PATH. This may cause problems if you don't run 'sudo make install'" || true diff --git a/cmd/lotus-gateway/main.go b/cmd/lotus-gateway/main.go index 7adf5dd8c..c19599084 100644 --- a/cmd/lotus-gateway/main.go +++ b/cmd/lotus-gateway/main.go @@ -54,11 +54,11 @@ var runCmd = &cli.Command{ &cli.StringFlag{ Name: "listen", Usage: "host address and port the api server will listen on", - Value: "0.0.0.0:1777", + Value: "0.0.0.0:2346", }, }, Action: func(cctx *cli.Context) error { - log.Info("Starting lotus wallet") + log.Info("Starting lotus gateway") ctx := lcli.ReqContext(cctx) ctx, cancel := context.WithCancel(ctx) From 84fa22110fc50d71e41c83f453a22c235d058e95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 15 Sep 2020 12:09:43 +0200 Subject: [PATCH 115/885] fix lint --- cmd/lotus-storage-miner/proving.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/lotus-storage-miner/proving.go b/cmd/lotus-storage-miner/proving.go index bf5c709e4..c828f7ef8 100644 --- a/cmd/lotus-storage-miner/proving.go +++ b/cmd/lotus-storage-miner/proving.go @@ -3,11 +3,12 @@ package main import ( "bytes" "fmt" + "os" + "text/tabwriter" + "github.com/fatih/color" "github.com/urfave/cli/v2" "golang.org/x/xerrors" - "os" - "text/tabwriter" "github.com/filecoin-project/specs-actors/actors/builtin/miner" From 4dabab5ce649137ee0f240122e1dbbbea1e84182 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 15 Sep 2020 13:04:45 +0200 Subject: [PATCH 116/885] state manager progress --- api/api_full.go | 9 +- api/apistruct/struct.go | 30 ++--- chain/actors/builtin/miner/miner.go | 6 + chain/actors/builtin/miner/utils.go | 28 +++++ chain/actors/builtin/miner/v0.go | 39 +++++++ chain/stmgr/utils.go | 167 ++++++++++++++-------------- node/impl/full/state.go | 8 +- storage/miner.go | 3 +- 8 files changed, 178 insertions(+), 112 deletions(-) create mode 100644 chain/actors/builtin/miner/utils.go diff --git a/api/api_full.go b/api/api_full.go index 035c7de43..c76b9038c 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -315,9 +315,9 @@ type FullNode interface { // StateMinerSectors returns info about the given miner's sectors. If the filter bitfield is nil, all sectors are included. // If the filterOut boolean is set to true, any sectors in the filter are excluded. // If false, only those sectors in the filter are included. - StateMinerSectors(context.Context, address.Address, *bitfield.BitField, bool, types.TipSetKey) ([]*ChainSectorInfo, error) + StateMinerSectors(context.Context, address.Address, *bitfield.BitField, bool, types.TipSetKey) ([]*miner2.ChainSectorInfo, error) // StateMinerActiveSectors returns info about sectors that a given miner is actively proving. - StateMinerActiveSectors(context.Context, address.Address, types.TipSetKey) ([]*ChainSectorInfo, error) + StateMinerActiveSectors(context.Context, address.Address, types.TipSetKey) ([]*miner2.ChainSectorInfo, error) // StateMinerProvingDeadline calculates the deadline at some epoch for a proving period // and returns the deadline-related calculations. StateMinerProvingDeadline(context.Context, address.Address, types.TipSetKey) (*dline.Info, error) @@ -538,11 +538,6 @@ type Message struct { Message *types.Message } -type ChainSectorInfo struct { - Info miner.SectorOnChainInfo - ID abi.SectorNumber -} - type ActorState struct { Balance types.BigInt State interface{} diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 1eac858a0..ea8c311a0 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -161,19 +161,19 @@ type FullNodeStruct struct { ClientDataTransferUpdates func(ctx context.Context) (<-chan api.DataTransferChannel, error) `perm:"write"` ClientRetrieveTryRestartInsufficientFunds func(ctx context.Context, paymentChannel address.Address) error `perm:"write"` - StateNetworkName func(context.Context) (dtypes.NetworkName, error) `perm:"read"` - StateMinerSectors func(context.Context, address.Address, *bitfield.BitField, bool, types.TipSetKey) ([]*api.ChainSectorInfo, error) `perm:"read"` - StateMinerActiveSectors func(context.Context, address.Address, types.TipSetKey) ([]*api.ChainSectorInfo, error) `perm:"read"` - StateMinerProvingDeadline func(context.Context, address.Address, types.TipSetKey) (*dline.Info, error) `perm:"read"` - StateMinerPower func(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error) `perm:"read"` - StateMinerInfo func(context.Context, address.Address, types.TipSetKey) (miner2.MinerInfo, error) `perm:"read"` - StateMinerFaults func(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error) `perm:"read"` - StateAllMinerFaults func(context.Context, abi.ChainEpoch, types.TipSetKey) ([]*api.Fault, error) `perm:"read"` - StateMinerRecoveries func(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error) `perm:"read"` - StateMinerPreCommitDepositForPower func(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error) `perm:"read"` - StateMinerInitialPledgeCollateral func(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error) `perm:"read"` - StateMinerAvailableBalance func(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) `perm:"read"` - StateSectorPreCommitInfo func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) `perm:"read"` + StateNetworkName func(context.Context) (dtypes.NetworkName, error) `perm:"read"` + StateMinerSectors func(context.Context, address.Address, *bitfield.BitField, bool, types.TipSetKey) ([]*miner2.ChainSectorInfo, error) `perm:"read"` + StateMinerActiveSectors func(context.Context, address.Address, types.TipSetKey) ([]*miner2.ChainSectorInfo, error) `perm:"read"` + StateMinerProvingDeadline func(context.Context, address.Address, types.TipSetKey) (*dline.Info, error) `perm:"read"` + StateMinerPower func(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error) `perm:"read"` + StateMinerInfo func(context.Context, address.Address, types.TipSetKey) (miner2.MinerInfo, error) `perm:"read"` + StateMinerFaults func(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error) `perm:"read"` + StateAllMinerFaults func(context.Context, abi.ChainEpoch, types.TipSetKey) ([]*api.Fault, error) `perm:"read"` + StateMinerRecoveries func(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error) `perm:"read"` + StateMinerPreCommitDepositForPower func(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error) `perm:"read"` + StateMinerInitialPledgeCollateral func(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error) `perm:"read"` + StateMinerAvailableBalance func(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) `perm:"read"` + StateSectorPreCommitInfo func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) `perm:"read"` StateSectorGetInfo func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorOnChainInfo, error) `perm:"read"` StateSectorExpiration func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*api.SectorExpiration, error) `perm:"read"` StateSectorPartition func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*api.SectorLocation, error) `perm:"read"` @@ -730,11 +730,11 @@ func (c *FullNodeStruct) StateNetworkName(ctx context.Context) (dtypes.NetworkNa return c.Internal.StateNetworkName(ctx) } -func (c *FullNodeStruct) StateMinerSectors(ctx context.Context, addr address.Address, filter *bitfield.BitField, filterOut bool, tsk types.TipSetKey) ([]*api.ChainSectorInfo, error) { +func (c *FullNodeStruct) StateMinerSectors(ctx context.Context, addr address.Address, filter *bitfield.BitField, filterOut bool, tsk types.TipSetKey) ([]*miner2.ChainSectorInfo, error) { return c.Internal.StateMinerSectors(ctx, addr, filter, filterOut, tsk) } -func (c *FullNodeStruct) StateMinerActiveSectors(ctx context.Context, addr address.Address, tsk types.TipSetKey) ([]*api.ChainSectorInfo, error) { +func (c *FullNodeStruct) StateMinerActiveSectors(ctx context.Context, addr address.Address, tsk types.TipSetKey) ([]*miner2.ChainSectorInfo, error) { return c.Internal.StateMinerActiveSectors(ctx, addr, tsk) } diff --git a/chain/actors/builtin/miner/miner.go b/chain/actors/builtin/miner/miner.go index eb70d1457..6f3bbe4bb 100644 --- a/chain/actors/builtin/miner/miner.go +++ b/chain/actors/builtin/miner/miner.go @@ -35,6 +35,7 @@ type State interface { GetSector(abi.SectorNumber) (*SectorOnChainInfo, error) GetPrecommittedSector(abi.SectorNumber) (*SectorPreCommitOnChainInfo, error) + LoadSectorsFromSet(filter *bitfield.BitField, filterOut bool) ([]*ChainSectorInfo, error) LoadDeadline(idx uint64) (Deadline, error) ForEachDeadline(cb func(idx uint64, dl Deadline) error) error @@ -71,3 +72,8 @@ type MinerInfo struct { SectorSize abi.SectorSize WindowPoStPartitionSectors uint64 } + +type ChainSectorInfo struct { + Info SectorOnChainInfo + ID abi.SectorNumber +} diff --git a/chain/actors/builtin/miner/utils.go b/chain/actors/builtin/miner/utils.go new file mode 100644 index 000000000..052d2da60 --- /dev/null +++ b/chain/actors/builtin/miner/utils.go @@ -0,0 +1,28 @@ +package miner + +import ( + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-bitfield" +) + +func AllPartSectors(mas State, sget func(Partition) (bitfield.BitField, error)) (bitfield.BitField, error) { + var parts []bitfield.BitField + + err := mas.ForEachDeadline(func(dlidx uint64, dl Deadline) error { + return dl.ForEachPartition(func(partidx uint64, part Partition) error { + s, err := sget(part) + if err != nil { + return xerrors.Errorf("getting sector list (dl: %d, part %d): %w", dlidx, partidx, err) + } + + parts = append(parts, s) + return nil + }) + }) + if err != nil { + return bitfield.BitField{}, err + } + + return bitfield.MultiMerge(parts...) +} \ No newline at end of file diff --git a/chain/actors/builtin/miner/v0.go b/chain/actors/builtin/miner/v0.go index 40b9e4cc6..0fb75795c 100644 --- a/chain/actors/builtin/miner/v0.go +++ b/chain/actors/builtin/miner/v0.go @@ -5,7 +5,11 @@ import ( "github.com/filecoin-project/go-bitfield" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/actors/adt" + v0adt "github.com/filecoin-project/specs-actors/actors/util/adt" + cbor "github.com/ipfs/go-ipld-cbor" "github.com/libp2p/go-libp2p-core/peer" + cbg "github.com/whyrusleeping/cbor-gen" + "golang.org/x/xerrors" "github.com/filecoin-project/specs-actors/actors/builtin/miner" ) @@ -43,6 +47,41 @@ func (s *v0State) GetPrecommittedSector(num abi.SectorNumber) (*SectorPreCommitO return info, nil } +func (s *v0State) LoadSectorsFromSet(filter *bitfield.BitField, filterOut bool) ([]*ChainSectorInfo, error) { + a, err := v0adt.AsArray(s.store, s.State.Sectors) + if err != nil { + return nil, err + } + + var sset []*ChainSectorInfo + var v cbg.Deferred + if err := a.ForEach(&v, func(i int64) error { + if filter != nil { + set, err := filter.IsSet(uint64(i)) + if err != nil { + return xerrors.Errorf("filter check error: %w", err) + } + if set == filterOut { + return nil + } + } + + var oci miner.SectorOnChainInfo + if err := cbor.DecodeInto(v.Raw, &oci); err != nil { + return err + } + sset = append(sset, &ChainSectorInfo{ + Info: oci, + ID: abi.SectorNumber(i), + }) + return nil + }); err != nil { + return nil, err + } + + return sset, nil +} + func (s *v0State) LoadDeadline(idx uint64) (Deadline, error) { dls, err := s.State.LoadDeadlines(s.store) if err != nil { diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index 11eedb512..0217bd91e 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -140,60 +140,35 @@ func MinerSectorInfo(ctx context.Context, sm *StateManager, maddr address.Addres return mas.GetSector(sid) } -func GetMinerSectorSet(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address, filter *bitfield.BitField, filterOut bool) ([]*api.ChainSectorInfo, error) { - var mas miner.State - _, err := sm.LoadActorState(ctx, maddr, &mas, ts) +func GetMinerSectorSet(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address, filter *bitfield.BitField, filterOut bool) ([]*miner.ChainSectorInfo, error) { + act, err := sm.LoadActor(ctx, maddr, ts) + if err != nil { + return nil, xerrors.Errorf("(get sset) failed to load miner actor: %w", err) + } + + mas, err := miner.Load(sm.cs.Store(ctx), act) if err != nil { return nil, xerrors.Errorf("(get sset) failed to load miner actor state: %w", err) } - return LoadSectorsFromSet(ctx, sm.ChainStore().Blockstore(), mas.Sectors, filter, filterOut) + return mas.LoadSectorsFromSet(filter, filterOut) } func GetSectorsForWinningPoSt(ctx context.Context, pv ffiwrapper.Verifier, sm *StateManager, st cid.Cid, maddr address.Address, rand abi.PoStRandomness) ([]proof.SectorInfo, error) { - var partsProving []bitfield.BitField - var mas *miner.State - var info *miner.MinerInfo - - err := sm.WithStateTree(st, sm.WithActor(maddr, sm.WithActorState(ctx, func(store adt.Store, mst *miner.State) error { - var err error - - mas = mst - - info, err = mas.GetInfo(store) - if err != nil { - return xerrors.Errorf("getting miner info: %w", err) - } - - deadlines, err := mas.LoadDeadlines(store) - if err != nil { - return xerrors.Errorf("loading deadlines: %w", err) - } - - return deadlines.ForEach(store, func(dlIdx uint64, deadline *miner.Deadline) error { - partitions, err := deadline.PartitionsArray(store) - if err != nil { - return xerrors.Errorf("getting partition array: %w", err) - } - - var partition miner.Partition - return partitions.ForEach(&partition, func(partIdx int64) error { - p, err := bitfield.SubtractBitField(partition.Sectors, partition.Faults) - if err != nil { - return xerrors.Errorf("subtract faults from partition sectors: %w", err) - } - - partsProving = append(partsProving, p) - - return nil - }) - }) - }))) + act, err := sm.LoadActorRaw(ctx, maddr, st) if err != nil { - return nil, err + return nil, xerrors.Errorf("failed to load miner actor: %w", err) } - provingSectors, err := bitfield.MultiMerge(partsProving...) + mas, err := miner.Load(sm.cs.Store(ctx), act) + if err != nil { + return nil, xerrors.Errorf("failed to load miner actor state: %w", err) + } + + // TODO (!!): This was partition.Sectors-partition.Faults originally, which was likely very wrong, and will need to be an upgrade + // ^ THE CURRENT THING HERE WON'T SYNC v + + provingSectors, err := miner.AllPartSectors(mas, miner.Partition.ActiveSectors) if err != nil { return nil, xerrors.Errorf("merge partition proving sets: %w", err) } @@ -208,6 +183,11 @@ func GetSectorsForWinningPoSt(ctx context.Context, pv ffiwrapper.Verifier, sm *S return nil, nil } + info, err := mas.Info() + if err != nil { + return nil, xerrors.Errorf("getting miner info: %w", err) + } + spt, err := ffiwrapper.SealProofTypeFromSectorSize(info.SectorSize) if err != nil { return nil, xerrors.Errorf("getting seal proof type: %w", err) @@ -228,31 +208,19 @@ func GetSectorsForWinningPoSt(ctx context.Context, pv ffiwrapper.Verifier, sm *S return nil, xerrors.Errorf("generating winning post challenges: %w", err) } - sectors, err := provingSectors.All(miner.SectorsMax) + sectors, err := mas.LoadSectorsFromSet(&provingSectors, false) if err != nil { - return nil, xerrors.Errorf("failed to enumerate all sector IDs: %w", err) - } - - sectorAmt, err := adt.AsArray(sm.cs.Store(ctx), mas.Sectors) - if err != nil { - return nil, xerrors.Errorf("failed to load sectors amt: %w", err) + return nil, xerrors.Errorf("loading proving sectors: %w", err) } out := make([]proof.SectorInfo, len(ids)) for i, n := range ids { - sid := sectors[n] - - var sinfo miner.SectorOnChainInfo - if found, err := sectorAmt.Get(sid, &sinfo); err != nil { - return nil, xerrors.Errorf("failed to get sector %d: %w", sid, err) - } else if !found { - return nil, xerrors.Errorf("failed to find sector %d", sid) - } + s := sectors[n] out[i] = proof.SectorInfo{ SealProof: spt, - SectorNumber: sinfo.SectorNumber, - SealedCID: sinfo.SealedCID, + SectorNumber: s.ID, + SealedCID: s.Info.SealedCID, } } @@ -260,20 +228,33 @@ func GetSectorsForWinningPoSt(ctx context.Context, pv ffiwrapper.Verifier, sm *S } func StateMinerInfo(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) (*miner.MinerInfo, error) { - var mas miner.State - _, err := sm.LoadActorStateRaw(ctx, maddr, &mas, ts.ParentState()) + act, err := sm.LoadActor(ctx, maddr, ts) if err != nil { - return nil, xerrors.Errorf("(get ssize) failed to load miner actor state: %w", err) + return nil, xerrors.Errorf("failed to load miner actor: %w", err) } - return mas.GetInfo(sm.cs.Store(ctx)) + mas, err := miner.Load(sm.cs.Store(ctx), act) + if err != nil { + return nil, xerrors.Errorf("failed to load miner actor state: %w", err) + } + + mi, err := mas.Info() + if err != nil { + return nil, err + } + + return &mi, err } func GetMinerSlashed(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) (bool, error) { - var spas power.State - _, err := sm.LoadActorState(ctx, builtin.StoragePowerActorAddr, &spas, ts) + act, err := sm.LoadActor(ctx, builtin.StoragePowerActorAddr, ts) if err != nil { - return false, xerrors.Errorf("(get miner slashed) failed to load power actor state") + return false, xerrors.Errorf("failed to load power actor: %w", err) + } + + spas, err := power.Load(sm.cs.Store(ctx), act) + if err != nil { + return false, xerrors.Errorf("failed to load power actor state: %w", err) } store := sm.cs.Store(ctx) @@ -295,10 +276,16 @@ func GetMinerSlashed(ctx context.Context, sm *StateManager, ts *types.TipSet, ma } func GetStorageDeal(ctx context.Context, sm *StateManager, dealID abi.DealID, ts *types.TipSet) (*api.MarketDeal, error) { - var state market.State - if _, err := sm.LoadActorState(ctx, builtin.StorageMarketActorAddr, &state, ts); err != nil { - return nil, err + act, err := sm.LoadActor(ctx, builtin.StorageMarketActorAddr, ts) + if err != nil { + return nil, xerrors.Errorf("failed to load market actor: %w", err) } + + state, err := market.Load(sm.cs.Store(ctx), act) + if err != nil { + return nil, xerrors.Errorf("failed to load market actor state: %w", err) + } + store := sm.ChainStore().Store(ctx) da, err := adt.AsArray(store, state.Proposals) @@ -338,9 +325,14 @@ func GetStorageDeal(ctx context.Context, sm *StateManager, dealID abi.DealID, ts } func ListMinerActors(ctx context.Context, sm *StateManager, ts *types.TipSet) ([]address.Address, error) { - var state power.State - if _, err := sm.LoadActorState(ctx, builtin.StoragePowerActorAddr, &state, ts); err != nil { - return nil, err + act, err := sm.LoadActor(ctx, builtin.StoragePowerActorAddr, ts) + if err != nil { + return nil, xerrors.Errorf("failed to load power actor: %w", err) + } + + state, err := power.Load(sm.cs.Store(ctx), act) + if err != nil { + return nil, xerrors.Errorf("failed to load power actor state: %w", err) } m, err := adt.AsMap(sm.cs.Store(ctx), state.Claims) @@ -364,13 +356,13 @@ func ListMinerActors(ctx context.Context, sm *StateManager, ts *types.TipSet) ([ return miners, nil } -func LoadSectorsFromSet(ctx context.Context, bs blockstore.Blockstore, ssc cid.Cid, filter *bitfield.BitField, filterOut bool) ([]*api.ChainSectorInfo, error) { +func LoadSectorsFromSet(ctx context.Context, bs blockstore.Blockstore, ssc cid.Cid, filter *bitfield.BitField, filterOut bool) ([]*miner.ChainSectorInfo, error) { a, err := adt.AsArray(store.ActorStore(ctx, bs), ssc) if err != nil { return nil, err } - var sset []*api.ChainSectorInfo + var sset []*miner.ChainSectorInfo var v cbg.Deferred if err := a.ForEach(&v, func(i int64) error { if filter != nil { @@ -387,7 +379,7 @@ func LoadSectorsFromSet(ctx context.Context, bs blockstore.Blockstore, ssc cid.C if err := cbor.DecodeInto(v.Raw, &oci); err != nil { return err } - sset = append(sset, &api.ChainSectorInfo{ + sset = append(sset, &miner.ChainSectorInfo{ Info: oci, ID: abi.SectorNumber(i), }) @@ -420,7 +412,7 @@ func ComputeState(ctx context.Context, sm *StateManager, height abi.ChainEpoch, NtwkVersion: sm.GetNtwkVersion, BaseFee: ts.Blocks()[0].ParentBaseFee, } - vmi, err := vm.NewVM(vmopt) + vmi, err := vm.NewVM(ctx, vmopt) if err != nil { return cid.Undef, nil, err } @@ -508,9 +500,14 @@ func MinerGetBaseInfo(ctx context.Context, sm *StateManager, bcs beacon.Schedule return nil, err } - var mas miner.State - if _, err := sm.LoadActorStateRaw(ctx, maddr, &mas, lbst); err != nil { - return nil, err + act, err := sm.LoadActorRaw(ctx, maddr, lbst) + if err != nil { + return nil, xerrors.Errorf("failed to load miner actor: %w", err) + } + + mas, err := miner.Load(sm.cs.Store(ctx), act) + if err != nil { + return nil, xerrors.Errorf("failed to load miner actor state: %w", err) } buf := new(bytes.Buffer) @@ -537,7 +534,7 @@ func MinerGetBaseInfo(ctx context.Context, sm *StateManager, bcs beacon.Schedule return nil, xerrors.Errorf("failed to get power: %w", err) } - info, err := mas.GetInfo(sm.cs.Store(ctx)) + info, err := mas.Info() if err != nil { return nil, err } @@ -652,9 +649,9 @@ func init() { } func GetReturnType(ctx context.Context, sm *StateManager, to address.Address, method abi.MethodNum, ts *types.TipSet) (cbg.CBORUnmarshaler, error) { - var act types.Actor - if err := sm.WithParentState(ts, sm.WithActor(to, GetActor(&act))); err != nil { - return nil, xerrors.Errorf("getting actor: %w", err) + act, err := sm.LoadActor(ctx, to, ts) + if err != nil { + return nil, xerrors.Errorf("(get sset) failed to load miner actor: %w", err) } m, found := MethodsMap[act.Code][method] diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 75f06e636..f61ebea49 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -30,11 +30,11 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/beacon" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/stmgr" - "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" @@ -62,7 +62,7 @@ func (a *StateAPI) StateNetworkName(ctx context.Context) (dtypes.NetworkName, er return stmgr.GetNetworkName(ctx, a.StateManager, a.Chain.GetHeaviestTipSet().ParentState()) } -func (a *StateAPI) StateMinerSectors(ctx context.Context, addr address.Address, filter *bitfield.BitField, filterOut bool, tsk types.TipSetKey) ([]*api.ChainSectorInfo, error) { +func (a *StateAPI) StateMinerSectors(ctx context.Context, addr address.Address, filter *bitfield.BitField, filterOut bool, tsk types.TipSetKey) ([]*miner.ChainSectorInfo, error) { ts, err := a.Chain.GetTipSetFromKey(tsk) if err != nil { return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) @@ -70,8 +70,8 @@ func (a *StateAPI) StateMinerSectors(ctx context.Context, addr address.Address, return stmgr.GetMinerSectorSet(ctx, a.StateManager, ts, addr, filter, filterOut) } -func (a *StateAPI) StateMinerActiveSectors(ctx context.Context, maddr address.Address, tsk types.TipSetKey) ([]*api.ChainSectorInfo, error) { - var out []*api.ChainSectorInfo +func (a *StateAPI) StateMinerActiveSectors(ctx context.Context, maddr address.Address, tsk types.TipSetKey) ([]*miner.ChainSectorInfo, error) { // TODO: only used in cli + var out []*miner.ChainSectorInfo err := a.StateManager.WithParentStateTsk(tsk, a.StateManager.WithActor(maddr, diff --git a/storage/miner.go b/storage/miner.go index d2af4be2c..47f50ce71 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -3,6 +3,7 @@ package storage import ( "context" "errors" + miner2 "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "time" "github.com/filecoin-project/go-state-types/dline" @@ -67,7 +68,7 @@ type SealingStateEvt struct { type storageMinerApi interface { // Call a read only method on actors (no interaction with the chain required) StateCall(context.Context, *types.Message, types.TipSetKey) (*api.InvocResult, error) - StateMinerSectors(context.Context, address.Address, *bitfield.BitField, bool, types.TipSetKey) ([]*api.ChainSectorInfo, error) + StateMinerSectors(context.Context, address.Address, *bitfield.BitField, bool, types.TipSetKey) ([]*miner2.ChainSectorInfo, error) StateSectorPreCommitInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) StateSectorGetInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorOnChainInfo, error) StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*api.SectorLocation, error) From 441985019597030eff05ac316d1a873432806e6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 15 Sep 2020 15:29:25 +0200 Subject: [PATCH 117/885] state api impl fixes --- api/api_full.go | 2 +- chain/actors/builtin/miner/miner.go | 3 + chain/actors/builtin/miner/v0.go | 5 + chain/actors/builtin/power/power.go | 2 +- chain/actors/builtin/power/v0.go | 11 +- chain/stmgr/utils.go | 12 +- node/impl/full/state.go | 171 +++++++++++++--------------- 7 files changed, 99 insertions(+), 107 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index c76b9038c..6625ae7f5 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -20,11 +20,11 @@ import ( "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/specs-actors/actors/builtin/market" "github.com/filecoin-project/specs-actors/actors/builtin/paych" - "github.com/filecoin-project/specs-actors/actors/builtin/power" "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" "github.com/filecoin-project/specs-actors/actors/runtime/proof" miner2 "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + "github.com/filecoin-project/lotus/chain/actors/builtin/power" "github.com/filecoin-project/lotus/chain/types" marketevents "github.com/filecoin-project/lotus/markets/loggers" "github.com/filecoin-project/lotus/node/modules/dtypes" diff --git a/chain/actors/builtin/miner/miner.go b/chain/actors/builtin/miner/miner.go index 6f3bbe4bb..9824ce61c 100644 --- a/chain/actors/builtin/miner/miner.go +++ b/chain/actors/builtin/miner/miner.go @@ -1,6 +1,7 @@ package miner import ( + "github.com/filecoin-project/go-state-types/dline" "github.com/libp2p/go-libp2p-core/peer" "golang.org/x/xerrors" @@ -41,6 +42,8 @@ type State interface { ForEachDeadline(cb func(idx uint64, dl Deadline) error) error NumDeadlines() (uint64, error) Info() (MinerInfo, error) + + DeadlineInfo(epoch abi.ChainEpoch) *dline.Info } type Deadline interface { diff --git a/chain/actors/builtin/miner/v0.go b/chain/actors/builtin/miner/v0.go index 0fb75795c..a7ed66fbd 100644 --- a/chain/actors/builtin/miner/v0.go +++ b/chain/actors/builtin/miner/v0.go @@ -4,6 +4,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-bitfield" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/dline" "github.com/filecoin-project/lotus/chain/actors/adt" v0adt "github.com/filecoin-project/specs-actors/actors/util/adt" cbor "github.com/ipfs/go-ipld-cbor" @@ -142,6 +143,10 @@ func (s *v0State) Info() (MinerInfo, error) { return mi, nil } +func (s *v0State) DeadlineInfo(epoch abi.ChainEpoch) *dline.Info { + return s.State.DeadlineInfo(epoch) +} + func (d *v0Deadline) LoadPartition(idx uint64) (Partition, error) { p, err := d.Deadline.LoadPartition(d.store, idx) if err != nil { diff --git a/chain/actors/builtin/power/power.go b/chain/actors/builtin/power/power.go index 7588615a6..fa0400f2e 100644 --- a/chain/actors/builtin/power/power.go +++ b/chain/actors/builtin/power/power.go @@ -30,7 +30,7 @@ type State interface { TotalLocked() (abi.TokenAmount, error) TotalPower() (Claim, error) - MinerPower(address.Address) (Claim, error) + MinerPower(address.Address) (Claim, bool, error) MinerNominalPowerMeetsConsensusMinimum(address.Address) (bool, error) } diff --git a/chain/actors/builtin/power/v0.go b/chain/actors/builtin/power/v0.go index f3152eb6a..12eb318e5 100644 --- a/chain/actors/builtin/power/v0.go +++ b/chain/actors/builtin/power/v0.go @@ -23,19 +23,20 @@ func (s *v0State) TotalPower() (Claim, error) { }, nil } -func (s *v0State) MinerPower(addr address.Address) (Claim, error) { +func (s *v0State) MinerPower(addr address.Address) (Claim, bool, error) { claims, err := adt.AsMap(s.store, s.Claims) if err != nil { - return Claim{}, err + return Claim{}, false, err } var claim power.Claim - if _, err := claims.Get(abi.AddrKey(addr), &claim); err != nil { - return Claim{}, err + ok, err := claims.Get(abi.AddrKey(addr), &claim) + if err != nil { + return Claim{}, false, err } return Claim{ RawBytePower: claim.RawBytePower, QualityAdjPower: claim.QualityAdjPower, - }, nil + }, ok, nil } func (s *v0State) MinerNominalPowerMeetsConsensusMinimum(a address.Address) (bool, error) { diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index 0217bd91e..4dc210154 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -26,7 +26,6 @@ import ( "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/builtin/account" "github.com/filecoin-project/specs-actors/actors/builtin/cron" - "github.com/filecoin-project/specs-actors/actors/builtin/market" "github.com/filecoin-project/specs-actors/actors/builtin/multisig" "github.com/filecoin-project/specs-actors/actors/builtin/paych" "github.com/filecoin-project/specs-actors/actors/builtin/reward" @@ -36,6 +35,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init" + "github.com/filecoin-project/lotus/chain/actors/builtin/market" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/beacon" "github.com/filecoin-project/lotus/chain/state" @@ -257,17 +257,11 @@ func GetMinerSlashed(ctx context.Context, sm *StateManager, ts *types.TipSet, ma return false, xerrors.Errorf("failed to load power actor state: %w", err) } - store := sm.cs.Store(ctx) - - claims, err := adt.AsMap(store, spas.Claims) + _, ok, err := spas.MinerPower(maddr) if err != nil { - return false, err + return false, xerrors.Errorf("getting miner power: %w", err) } - ok, err := claims.Get(abi.AddrKey(maddr), nil) - if err != nil { - return false, err - } if !ok { return true, nil } diff --git a/node/impl/full/state.go b/node/impl/full/state.go index f61ebea49..98c9ed2e0 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -71,127 +71,121 @@ func (a *StateAPI) StateMinerSectors(ctx context.Context, addr address.Address, } func (a *StateAPI) StateMinerActiveSectors(ctx context.Context, maddr address.Address, tsk types.TipSetKey) ([]*miner.ChainSectorInfo, error) { // TODO: only used in cli - var out []*miner.ChainSectorInfo - - err := a.StateManager.WithParentStateTsk(tsk, - a.StateManager.WithActor(maddr, - a.StateManager.WithActorState(ctx, func(store adt.Store, mas *miner.State) error { - var allActive []bitfield.BitField - - err := a.StateManager.WithDeadlines( - a.StateManager.WithEachDeadline( - a.StateManager.WithEachPartition(func(store adt.Store, partIdx uint64, partition *miner.Partition) error { - active, err := partition.ActiveSectors() - if err != nil { - return xerrors.Errorf("partition.ActiveSectors: %w", err) - } - - allActive = append(allActive, active) - return nil - })))(store, mas) - if err != nil { - return xerrors.Errorf("with deadlines: %w", err) - } - - active, err := bitfield.MultiMerge(allActive...) - if err != nil { - return xerrors.Errorf("merging active sector bitfields: %w", err) - } - - out, err = stmgr.LoadSectorsFromSet(ctx, a.Chain.Blockstore(), mas.Sectors, &active, false) - return err - }))) + act, err := a.StateManager.LoadActorTsk(ctx, maddr, tsk) if err != nil { - return nil, xerrors.Errorf("getting active sectors from partitions: %w", err) + return nil, xerrors.Errorf("failed to load miner actor: %w", err) } - return out, nil + mas, err := miner.Load(a.StateManager.ChainStore().Store(ctx), act) + if err != nil { + return nil, xerrors.Errorf("failed to load miner actor state: %w", err) + } + + activeSectors, err := miner.AllPartSectors(mas, miner.Partition.ActiveSectors) + if err != nil { + return nil, xerrors.Errorf("merge partition active sets: %w", err) + } + + return mas.LoadSectorsFromSet(&activeSectors, false) } -func (a *StateAPI) StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (api.MinerInfo, error) { - ts, err := a.Chain.GetTipSetFromKey(tsk) +func (a *StateAPI) StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (miner.MinerInfo, error) { + act, err := a.StateManager.LoadActorTsk(ctx, actor, tsk) if err != nil { - return api.MinerInfo{}, xerrors.Errorf("loading tipset %s: %w", tsk, err) + return miner.MinerInfo{}, xerrors.Errorf("failed to load miner actor: %w", err) } - a.StateManager.LoadActorStateRaw(ctx context.Context, addr address.Address, out interface{}, st cid.Cid) - - mi, err := stmgr.StateMinerInfo(ctx, a.StateManager, ts, actor) + mas, err := miner.Load(a.StateManager.ChainStore().Store(ctx), act) if err != nil { - return api.MinerInfo{}, err + return miner.MinerInfo{}, xerrors.Errorf("failed to load miner actor state: %w", err) } - return api.NewApiMinerInfo(mi), nil + + return mas.Info() } func (a *StateAPI) StateMinerDeadlines(ctx context.Context, m address.Address, tsk types.TipSetKey) ([]miner.Deadline, error) { - var out []*miner.Deadline - state, err := a.StateManager.LoadParentStateTsk(tsk) + act, err := a.StateManager.LoadActorTsk(ctx, m, tsk) if err != nil { - return nil, err + return nil, xerrors.Errorf("failed to load miner actor: %w", err) } - act, err := state.GetActor(addr) + + mas, err := miner.Load(a.StateManager.ChainStore().Store(ctx), act) if err != nil { - return nil, err + return nil, xerrors.Errorf("failed to load miner actor state: %w", err) } - mas, err := miner.Load(a.Chain.Store(ctx), act) + + deadlines, err := mas.NumDeadlines() if err != nil { - return nil, err + return nil, xerrors.Errorf("getting deadline count: %w", err) } - var deadlines []miner.Deadline - if err := mas.ForEachDeadline(func(_ uint64, dl miner.Deadline) error { - deadlines = append(deadlines, dl) + + out := make([]miner.Deadline, deadlines) + if err := mas.ForEachDeadline(func(i uint64, dl miner.Deadline) error { + out[i] = dl return nil }); err != nil { - return err + return nil, err } - return deadlines, nil + return out, nil } func (a *StateAPI) StateMinerPartitions(ctx context.Context, m address.Address, dlIdx uint64, tsk types.TipSetKey) ([]*miner.Partition, error) { + act, err := a.StateManager.LoadActorTsk(ctx, m, tsk) + if err != nil { + return nil, xerrors.Errorf("failed to load miner actor: %w", err) + } + + mas, err := miner.Load(a.StateManager.ChainStore().Store(ctx), act) + if err != nil { + return nil, xerrors.Errorf("failed to load miner actor state: %w", err) + } + + dl, err := mas.LoadDeadline(dlIdx) + if err != nil { + return nil, xerrors.Errorf("failed to load the deadline: %w", err) + } + var out []*miner.Partition - return out, a.StateManager.WithParentStateTsk(tsk, - a.StateManager.WithActor(m, - a.StateManager.WithActorState(ctx, - a.StateManager.WithDeadlines( - a.StateManager.WithDeadline(dlIdx, - a.StateManager.WithEachPartition(func(store adt.Store, partIdx uint64, partition *miner.Partition) error { - out = append(out, partition) - return nil - })))))) + err = dl.ForEachPartition(func(_ uint64, part miner.Partition) error { + p := part + out = append(out, &p) + return nil + }) + + return out, err } func (a *StateAPI) StateMinerProvingDeadline(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*dline.Info, error) { - ts, err := a.Chain.GetTipSetFromKey(tsk) + ts, err := a.StateManager.ChainStore().GetTipSetFromKey(tsk) if err != nil { return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - var mas miner.State - _, err = a.StateManager.LoadActorState(ctx, addr, &mas, ts) + act, err := a.StateManager.LoadActor(ctx, addr, ts) if err != nil { - return nil, xerrors.Errorf("(get sset) failed to load miner actor state: %w", err) + return nil, xerrors.Errorf("failed to load miner actor: %w", err) + } + + mas, err := miner.Load(a.StateManager.ChainStore().Store(ctx), act) + if err != nil { + return nil, xerrors.Errorf("failed to load miner actor state: %w", err) } return mas.DeadlineInfo(ts.Height()).NextNotElapsed(), nil } func (a *StateAPI) StateMinerFaults(ctx context.Context, addr address.Address, tsk types.TipSetKey) (bitfield.BitField, error) { - out := bitfield.New() - - err := a.StateManager.WithParentStateTsk(tsk, - a.StateManager.WithActor(addr, - a.StateManager.WithActorState(ctx, - a.StateManager.WithDeadlines( - a.StateManager.WithEachDeadline( - a.StateManager.WithEachPartition(func(store adt.Store, idx uint64, partition *miner.Partition) (err error) { - out, err = bitfield.MergeBitFields(out, partition.Faults) - return err - })))))) + act, err := a.StateManager.LoadActorTsk(ctx, addr, tsk) if err != nil { - return bitfield.BitField{}, err + return bitfield.BitField{}, xerrors.Errorf("failed to load miner actor: %w", err) } - return out, err + mas, err := miner.Load(a.StateManager.ChainStore().Store(ctx), act) + if err != nil { + return bitfield.BitField{}, xerrors.Errorf("failed to load miner actor state: %w", err) + } + + return miner.AllPartSectors(mas, miner.Partition.FaultySectors) } func (a *StateAPI) StateAllMinerFaults(ctx context.Context, lookback abi.ChainEpoch, endTsk types.TipSetKey) ([]*api.Fault, error) { @@ -238,22 +232,17 @@ func (a *StateAPI) StateAllMinerFaults(ctx context.Context, lookback abi.ChainEp } func (a *StateAPI) StateMinerRecoveries(ctx context.Context, addr address.Address, tsk types.TipSetKey) (bitfield.BitField, error) { - out := bitfield.New() - - err := a.StateManager.WithParentStateTsk(tsk, - a.StateManager.WithActor(addr, - a.StateManager.WithActorState(ctx, - a.StateManager.WithDeadlines( - a.StateManager.WithEachDeadline( - a.StateManager.WithEachPartition(func(store adt.Store, idx uint64, partition *miner.Partition) (err error) { - out, err = bitfield.MergeBitFields(out, partition.Recoveries) - return err - })))))) + act, err := a.StateManager.LoadActorTsk(ctx, addr, tsk) if err != nil { - return bitfield.BitField{}, err + return bitfield.BitField{}, xerrors.Errorf("failed to load miner actor: %w", err) } - return out, err + mas, err := miner.Load(a.StateManager.ChainStore().Store(ctx), act) + if err != nil { + return bitfield.BitField{}, xerrors.Errorf("failed to load miner actor state: %w", err) + } + + return miner.AllPartSectors(mas, miner.Partition.RecoveringSectors) } func (a *StateAPI) StateMinerPower(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*api.MinerPower, error) { From 9dad38c9a5a4fb834c0a479486875ee04988ca59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 15 Sep 2020 15:29:39 +0200 Subject: [PATCH 118/885] gofmt --- api/apistruct/struct.go | 50 ++++++++++++++--------------- chain/actors/builtin/miner/utils.go | 2 +- chain/gen/genesis/util.go | 16 ++++----- 3 files changed, 34 insertions(+), 34 deletions(-) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index ea8c311a0..d0bdd14d3 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -174,31 +174,31 @@ type FullNodeStruct struct { StateMinerInitialPledgeCollateral func(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error) `perm:"read"` StateMinerAvailableBalance func(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) `perm:"read"` StateSectorPreCommitInfo func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) `perm:"read"` - StateSectorGetInfo func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorOnChainInfo, error) `perm:"read"` - StateSectorExpiration func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*api.SectorExpiration, error) `perm:"read"` - StateSectorPartition func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*api.SectorLocation, error) `perm:"read"` - StateCall func(context.Context, *types.Message, types.TipSetKey) (*api.InvocResult, error) `perm:"read"` - StateReplay func(context.Context, types.TipSetKey, cid.Cid) (*api.InvocResult, error) `perm:"read"` - StateGetActor func(context.Context, address.Address, types.TipSetKey) (*types.Actor, error) `perm:"read"` - StateReadState func(context.Context, address.Address, types.TipSetKey) (*api.ActorState, error) `perm:"read"` - StateWaitMsg func(ctx context.Context, cid cid.Cid, confidence uint64) (*api.MsgLookup, error) `perm:"read"` - StateSearchMsg func(context.Context, cid.Cid) (*api.MsgLookup, error) `perm:"read"` - StateListMiners func(context.Context, types.TipSetKey) ([]address.Address, error) `perm:"read"` - StateListActors func(context.Context, types.TipSetKey) ([]address.Address, error) `perm:"read"` - StateMarketBalance func(context.Context, address.Address, types.TipSetKey) (api.MarketBalance, error) `perm:"read"` - StateMarketParticipants func(context.Context, types.TipSetKey) (map[string]api.MarketBalance, error) `perm:"read"` - StateMarketDeals func(context.Context, types.TipSetKey) (map[string]api.MarketDeal, error) `perm:"read"` - StateMarketStorageDeal func(context.Context, abi.DealID, types.TipSetKey) (*api.MarketDeal, error) `perm:"read"` - StateLookupID func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) `perm:"read"` - StateAccountKey func(context.Context, address.Address, types.TipSetKey) (address.Address, error) `perm:"read"` - StateChangedActors func(context.Context, cid.Cid, cid.Cid) (map[string]types.Actor, error) `perm:"read"` - StateGetReceipt func(context.Context, cid.Cid, types.TipSetKey) (*types.MessageReceipt, error) `perm:"read"` - StateMinerSectorCount func(context.Context, address.Address, types.TipSetKey) (api.MinerSectors, error) `perm:"read"` - StateListMessages func(ctx context.Context, match *types.Message, tsk types.TipSetKey, toht abi.ChainEpoch) ([]cid.Cid, error) `perm:"read"` - StateCompute func(context.Context, abi.ChainEpoch, []*types.Message, types.TipSetKey) (*api.ComputeStateOutput, error) `perm:"read"` - StateVerifiedClientStatus func(context.Context, address.Address, types.TipSetKey) (*verifreg.DataCap, error) `perm:"read"` - StateDealProviderCollateralBounds func(context.Context, abi.PaddedPieceSize, bool, types.TipSetKey) (api.DealCollateralBounds, error) `perm:"read"` - StateCirculatingSupply func(context.Context, types.TipSetKey) (api.CirculatingSupply, error) `perm:"read"` + StateSectorGetInfo func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorOnChainInfo, error) `perm:"read"` + StateSectorExpiration func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*api.SectorExpiration, error) `perm:"read"` + StateSectorPartition func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*api.SectorLocation, error) `perm:"read"` + StateCall func(context.Context, *types.Message, types.TipSetKey) (*api.InvocResult, error) `perm:"read"` + StateReplay func(context.Context, types.TipSetKey, cid.Cid) (*api.InvocResult, error) `perm:"read"` + StateGetActor func(context.Context, address.Address, types.TipSetKey) (*types.Actor, error) `perm:"read"` + StateReadState func(context.Context, address.Address, types.TipSetKey) (*api.ActorState, error) `perm:"read"` + StateWaitMsg func(ctx context.Context, cid cid.Cid, confidence uint64) (*api.MsgLookup, error) `perm:"read"` + StateSearchMsg func(context.Context, cid.Cid) (*api.MsgLookup, error) `perm:"read"` + StateListMiners func(context.Context, types.TipSetKey) ([]address.Address, error) `perm:"read"` + StateListActors func(context.Context, types.TipSetKey) ([]address.Address, error) `perm:"read"` + StateMarketBalance func(context.Context, address.Address, types.TipSetKey) (api.MarketBalance, error) `perm:"read"` + StateMarketParticipants func(context.Context, types.TipSetKey) (map[string]api.MarketBalance, error) `perm:"read"` + StateMarketDeals func(context.Context, types.TipSetKey) (map[string]api.MarketDeal, error) `perm:"read"` + StateMarketStorageDeal func(context.Context, abi.DealID, types.TipSetKey) (*api.MarketDeal, error) `perm:"read"` + StateLookupID func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) `perm:"read"` + StateAccountKey func(context.Context, address.Address, types.TipSetKey) (address.Address, error) `perm:"read"` + StateChangedActors func(context.Context, cid.Cid, cid.Cid) (map[string]types.Actor, error) `perm:"read"` + StateGetReceipt func(context.Context, cid.Cid, types.TipSetKey) (*types.MessageReceipt, error) `perm:"read"` + StateMinerSectorCount func(context.Context, address.Address, types.TipSetKey) (api.MinerSectors, error) `perm:"read"` + StateListMessages func(ctx context.Context, match *types.Message, tsk types.TipSetKey, toht abi.ChainEpoch) ([]cid.Cid, error) `perm:"read"` + StateCompute func(context.Context, abi.ChainEpoch, []*types.Message, types.TipSetKey) (*api.ComputeStateOutput, error) `perm:"read"` + StateVerifiedClientStatus func(context.Context, address.Address, types.TipSetKey) (*verifreg.DataCap, error) `perm:"read"` + StateDealProviderCollateralBounds func(context.Context, abi.PaddedPieceSize, bool, types.TipSetKey) (api.DealCollateralBounds, error) `perm:"read"` + StateCirculatingSupply func(context.Context, types.TipSetKey) (api.CirculatingSupply, error) `perm:"read"` MsigGetAvailableBalance func(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) `perm:"read"` MsigGetVested func(context.Context, address.Address, types.TipSetKey, types.TipSetKey) (types.BigInt, error) `perm:"read"` diff --git a/chain/actors/builtin/miner/utils.go b/chain/actors/builtin/miner/utils.go index 052d2da60..f9c6b3da3 100644 --- a/chain/actors/builtin/miner/utils.go +++ b/chain/actors/builtin/miner/utils.go @@ -25,4 +25,4 @@ func AllPartSectors(mas State, sget func(Partition) (bitfield.BitField, error)) } return bitfield.MultiMerge(parts...) -} \ No newline at end of file +} diff --git a/chain/gen/genesis/util.go b/chain/gen/genesis/util.go index 6a27f2f29..d87500206 100644 --- a/chain/gen/genesis/util.go +++ b/chain/gen/genesis/util.go @@ -49,13 +49,13 @@ func doExecValue(ctx context.Context, vm *vm.VM, to, from address.Address, value return ret.Return, nil } -var GenesisNetworkVersion = func() network.Version {// TODO: Get from build/ - if build.UseNewestNetwork() {// TODO: Get from build/ - return build.NewestNetworkVersion// TODO: Get from build/ - }// TODO: Get from build/ - return network.Version1// TODO: Get from build/ -}()// TODO: Get from build/ +var GenesisNetworkVersion = func() network.Version { // TODO: Get from build/ + if build.UseNewestNetwork() { // TODO: Get from build/ + return build.NewestNetworkVersion // TODO: Get from build/ + } // TODO: Get from build/ + return network.Version1 // TODO: Get from build/ +}() // TODO: Get from build/ -func genesisNetworkVersion(context.Context, abi.ChainEpoch) network.Version {// TODO: Get from build/ +func genesisNetworkVersion(context.Context, abi.ChainEpoch) network.Version { // TODO: Get from build/ return GenesisNetworkVersion // TODO: Get from build/ -}// TODO: Get from build/ +} // TODO: Get from build/ From ae326ce65de2ad3973ac625226d80dac0f1ccf8f Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 15 Sep 2020 15:43:37 +0100 Subject: [PATCH 119/885] feat: add inspect runtime method to chaos actor --- conformance/chaos/actor.go | 32 ++++++ conformance/chaos/actor_test.go | 26 +++++ conformance/chaos/cbor_gen.go | 198 +++++++++++++++++++++++++++++++- conformance/chaos/gen/gen.go | 1 + 4 files changed, 254 insertions(+), 3 deletions(-) diff --git a/conformance/chaos/actor.go b/conformance/chaos/actor.go index daff26694..526dac64c 100644 --- a/conformance/chaos/actor.go +++ b/conformance/chaos/actor.go @@ -64,6 +64,9 @@ const ( // MethodAbortWith is the identifier for the method that panics optionally with // a passed exit code. MethodAbortWith + // MethodInspectRuntime is the identifier for the method that returns the + // current runtime values. + MethodInspectRuntime ) // Exports defines the methods this actor exposes publicly. @@ -77,6 +80,7 @@ func (a Actor) Exports() []interface{} { MethodSend: a.Send, MethodMutateState: a.MutateState, MethodAbortWith: a.AbortWith, + MethodInspectRuntime: a.InspectRuntime, } } @@ -247,3 +251,31 @@ func (a Actor) AbortWith(rt runtime.Runtime, args *AbortWithArgs) *abi.EmptyValu } return nil } + +// InspectRuntimeReturn is the return value for the Actor.InspectRuntime method. +type InspectRuntimeReturn struct { + NetworkVersion int64 + Caller address.Address + Receiver address.Address + ValueReceived abi.TokenAmount + CurrEpoch abi.ChainEpoch + CurrentBalance abi.TokenAmount + State State + TotalFilCircSupply abi.TokenAmount +} + +// InspectRuntime returns a copy of the serializable values available in the Runtime. +func (a Actor) InspectRuntime(rt runtime.Runtime, _ *abi.EmptyValue) *InspectRuntimeReturn { + rt.ValidateImmediateCallerAcceptAny() + var st State + rt.StateReadonly(&st) + return &InspectRuntimeReturn{ + NetworkVersion: int64(rt.NetworkVersion()), + Caller: rt.Caller(), + Receiver: rt.Receiver(), + ValueReceived: rt.ValueReceived(), + CurrentBalance: rt.CurrentBalance(), + State: st, + TotalFilCircSupply: rt.TotalFilCircSupply(), + } +} diff --git a/conformance/chaos/actor_test.go b/conformance/chaos/actor_test.go index 67ced2899..2e899980d 100644 --- a/conformance/chaos/actor_test.go +++ b/conformance/chaos/actor_test.go @@ -6,6 +6,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/exitcode" + "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/support/mock" atesting "github.com/filecoin-project/specs-actors/support/testing" ) @@ -151,3 +152,28 @@ func TestAbortWithUncontrolled(t *testing.T) { }) rt.Verify() } + +func TestInspectRuntime(t *testing.T) { + caller := atesting.NewIDAddr(t, 100) + receiver := atesting.NewIDAddr(t, 101) + builder := mock.NewBuilder(context.Background(), receiver) + + rt := builder.Build(t) + rt.SetCaller(caller, builtin.AccountActorCodeID) + rt.StateCreate(&State{}) + var a Actor + + rt.ExpectValidateCallerAny() + ret := rt.Call(a.InspectRuntime, abi.Empty) + rtr, ok := ret.(*InspectRuntimeReturn) + if !ok { + t.Fatal("invalid return value") + } + if rtr.Caller != caller { + t.Fatal("unexpected runtime caller") + } + if rtr.Receiver != receiver { + t.Fatal("unexpected runtime receiver") + } + rt.Verify() +} diff --git a/conformance/chaos/cbor_gen.go b/conformance/chaos/cbor_gen.go index 710b84b93..96b707386 100644 --- a/conformance/chaos/cbor_gen.go +++ b/conformance/chaos/cbor_gen.go @@ -6,10 +6,10 @@ import ( "fmt" "io" - abi "github.com/filecoin-project/go-state-types/abi" - exitcode "github.com/filecoin-project/go-state-types/exitcode" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/exitcode" cbg "github.com/whyrusleeping/cbor-gen" - xerrors "golang.org/x/xerrors" + "golang.org/x/xerrors" ) var _ = xerrors.Errorf @@ -730,3 +730,195 @@ func (t *AbortWithArgs) UnmarshalCBOR(r io.Reader) error { } return nil } + +var lengthBufInspectRuntimeReturn = []byte{136} + +func (t *InspectRuntimeReturn) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write(lengthBufInspectRuntimeReturn); err != nil { + return err + } + + scratch := make([]byte, 9) + + // t.NetworkVersion (int64) (int64) + if t.NetworkVersion >= 0 { + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.NetworkVersion)); err != nil { + return err + } + } else { + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.NetworkVersion-1)); err != nil { + return err + } + } + + // t.Caller (address.Address) (struct) + if err := t.Caller.MarshalCBOR(w); err != nil { + return err + } + + // t.Receiver (address.Address) (struct) + if err := t.Receiver.MarshalCBOR(w); err != nil { + return err + } + + // t.ValueReceived (big.Int) (struct) + if err := t.ValueReceived.MarshalCBOR(w); err != nil { + return err + } + + // t.CurrEpoch (abi.ChainEpoch) (int64) + if t.CurrEpoch >= 0 { + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.CurrEpoch)); err != nil { + return err + } + } else { + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.CurrEpoch-1)); err != nil { + return err + } + } + + // t.CurrentBalance (big.Int) (struct) + if err := t.CurrentBalance.MarshalCBOR(w); err != nil { + return err + } + + // t.State (chaos.State) (struct) + if err := t.State.MarshalCBOR(w); err != nil { + return err + } + + // t.TotalFilCircSupply (big.Int) (struct) + if err := t.TotalFilCircSupply.MarshalCBOR(w); err != nil { + return err + } + return nil +} + +func (t *InspectRuntimeReturn) UnmarshalCBOR(r io.Reader) error { + *t = InspectRuntimeReturn{} + + br := cbg.GetPeeker(r) + scratch := make([]byte, 8) + + maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 8 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.NetworkVersion (int64) (int64) + { + maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.NetworkVersion = int64(extraI) + } + // t.Caller (address.Address) (struct) + + { + + if err := t.Caller.UnmarshalCBOR(br); err != nil { + return xerrors.Errorf("unmarshaling t.Caller: %w", err) + } + + } + // t.Receiver (address.Address) (struct) + + { + + if err := t.Receiver.UnmarshalCBOR(br); err != nil { + return xerrors.Errorf("unmarshaling t.Receiver: %w", err) + } + + } + // t.ValueReceived (big.Int) (struct) + + { + + if err := t.ValueReceived.UnmarshalCBOR(br); err != nil { + return xerrors.Errorf("unmarshaling t.ValueReceived: %w", err) + } + + } + // t.CurrEpoch (abi.ChainEpoch) (int64) + { + maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.CurrEpoch = abi.ChainEpoch(extraI) + } + // t.CurrentBalance (big.Int) (struct) + + { + + if err := t.CurrentBalance.UnmarshalCBOR(br); err != nil { + return xerrors.Errorf("unmarshaling t.CurrentBalance: %w", err) + } + + } + // t.State (chaos.State) (struct) + + { + + if err := t.State.UnmarshalCBOR(br); err != nil { + return xerrors.Errorf("unmarshaling t.State: %w", err) + } + + } + // t.TotalFilCircSupply (big.Int) (struct) + + { + + if err := t.TotalFilCircSupply.UnmarshalCBOR(br); err != nil { + return xerrors.Errorf("unmarshaling t.TotalFilCircSupply: %w", err) + } + + } + return nil +} diff --git a/conformance/chaos/gen/gen.go b/conformance/chaos/gen/gen.go index 97ea98dc8..496cc3d35 100644 --- a/conformance/chaos/gen/gen.go +++ b/conformance/chaos/gen/gen.go @@ -15,6 +15,7 @@ func main() { chaos.SendReturn{}, chaos.MutateStateArgs{}, chaos.AbortWithArgs{}, + chaos.InspectRuntimeReturn{}, ); err != nil { panic(err) } From 00c6397ec96170efd2124ef9c3bc9432090750f1 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 15 Sep 2020 10:44:44 -0700 Subject: [PATCH 120/885] more progress --- api/api_full.go | 13 ----- chain/actors/builtin/miner/miner.go | 15 +++++ chain/actors/builtin/miner/v0.go | 83 ++++++++++++++++++++++++++ node/impl/full/state.go | 91 ++++------------------------- 4 files changed, 109 insertions(+), 93 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 6625ae7f5..a5af175d8 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -473,19 +473,6 @@ type MinerSectors struct { Active uint64 } -type SectorExpiration struct { - OnTime abi.ChainEpoch - - // non-zero if sector is faulty, epoch at which it will be permanently - // removed if it doesn't recover - Early abi.ChainEpoch -} - -type SectorLocation struct { - Deadline uint64 - Partition uint64 -} - type ImportRes struct { Root cid.Cid ImportID multistore.StoreID diff --git a/chain/actors/builtin/miner/miner.go b/chain/actors/builtin/miner/miner.go index 9824ce61c..5ad8db39e 100644 --- a/chain/actors/builtin/miner/miner.go +++ b/chain/actors/builtin/miner/miner.go @@ -35,6 +35,8 @@ type State interface { cbor.Marshaler GetSector(abi.SectorNumber) (*SectorOnChainInfo, error) + FindSector(abi.SectorNumber) (*SectorLocation, error) + GetSectorExpiration(abi.SectorNumber) (*SectorExpiration, error) GetPrecommittedSector(abi.SectorNumber) (*SectorPreCommitOnChainInfo, error) LoadSectorsFromSet(filter *bitfield.BitField, filterOut bool) ([]*ChainSectorInfo, error) @@ -80,3 +82,16 @@ type ChainSectorInfo struct { Info SectorOnChainInfo ID abi.SectorNumber } + +type SectorExpiration struct { + OnTime abi.ChainEpoch + + // non-zero if sector is faulty, epoch at which it will be permanently + // removed if it doesn't recover + Early abi.ChainEpoch +} + +type SectorLocation struct { + Deadline uint64 + Partition uint64 +} diff --git a/chain/actors/builtin/miner/v0.go b/chain/actors/builtin/miner/v0.go index a7ed66fbd..1a4bd3800 100644 --- a/chain/actors/builtin/miner/v0.go +++ b/chain/actors/builtin/miner/v0.go @@ -1,6 +1,8 @@ package miner import ( + "errors" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-bitfield" "github.com/filecoin-project/go-state-types/abi" @@ -39,6 +41,87 @@ func (s *v0State) GetSector(num abi.SectorNumber) (*SectorOnChainInfo, error) { return info, nil } +func (s *v0State) FindSector(num abi.SectorNumber) (*SectorLocation, error) { + dlIdx, partIdx, err := s.State.FindSector(s.store, num) + if err != nil { + return nil, err + } + return &SectorLocation{ + Deadline: dlIdx, + Partition: partIdx, + }, nil +} + +// GetSectorExpiration returns the effective expiration of the given sector. +// +// If the sector isn't found or has already been terminated, this method returns +// nil and no error. If the sector does not expire early, the Early expiration +// field is 0. +func (s *v0State) GetSectorExpiration(num abi.SectorNumber) (out *SectorExpiration, err error) { + dls, err := s.State.LoadDeadlines(s.store) + if err != nil { + return nil, err + } + // NOTE: this can be optimized significantly. + // 1. If the sector is non-faulty, it will either expire on-time (can be + // learned from the sector info), or in the next quantized expiration + // epoch (i.e., the first element in the partition's expiration queue. + // 2. If it's faulty, it will expire early within the first 14 entries + // of the expiration queue. + stopErr := errors.New("stop") + err := dls.ForEach(s.store, func(dlIdx uint64, dl *miner.Deadline) error { + partitions, err := dl.PartitionsArray(s.store) + if err != nil { + return err + } + quant := s.State.QuantSpecForDeadline(dlIdx) + var part miner.Partition + return partitions.ForEach(&part, func(partIdx int64) error { + if found, err := part.Sectors.IsSet(uint64(num)); err != nil { + return err + } else if !found { + return nil + } + if found, err := part.Terminated.IsSet(uint64(num)); err != nil { + return err + } else if found { + // already terminated + return stopErr + } + + q, err := miner.LoadExpirationQueue(s.store, part.EarlyTerminated, quant) + if err != nil { + return err + } + var exp miner.ExpirationSet + return q.ForEach(&exp, func(epoch int64) error { + if early, err := exp.EarlySectors.IsSet(uint64(num)); err != nil { + return err + } else if early { + out.Early = abi.ChainEpoch(epoch) + return nil + } + if onTime, err := exp.OnTime.IsSet(uint64(num)); err != nil { + return err + } else if onTime { + out.OnTime = epoch + return stopErr + } + }) + }) + }) + if err == stopErr { + err = nil + } + if err != nil { + return nil, err + } + if out.Early == 0 && out.OnTime == 0 { + return nil, nil + } + return out, nil +} + func (s *v0State) GetPrecommittedSector(num abi.SectorNumber) (*SectorPreCommitOnChainInfo, error) { info, ok, err := s.State.GetPrecommittedSector(s.store, num) if !ok || err != nil { diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 98c9ed2e0..73ab7bc4d 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -698,97 +698,28 @@ func (a *StateAPI) StateSectorGetInfo(ctx context.Context, maddr address.Address return stmgr.MinerSectorInfo(ctx, a.StateManager, maddr, n, ts) } -type sectorPartitionCb func(store adt.Store, mas *miner.State, di uint64, pi uint64, part *miner.Partition) error - -func (a *StateAPI) sectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tsk types.TipSetKey, cb sectorPartitionCb) error { - return a.StateManager.WithParentStateTsk(tsk, - a.StateManager.WithActor(maddr, - a.StateManager.WithActorState(ctx, func(store adt.Store, mas *miner.State) error { - return a.StateManager.WithDeadlines(func(store adt.Store, deadlines *miner.Deadlines) error { - err := a.StateManager.WithEachDeadline(func(store adt.Store, di uint64, deadline *miner.Deadline) error { - return a.StateManager.WithEachPartition(func(store adt.Store, pi uint64, partition *miner.Partition) error { - set, err := partition.Sectors.IsSet(uint64(sectorNumber)) - if err != nil { - return xerrors.Errorf("is set: %w", err) - } - if set { - if err := cb(store, mas, di, pi, partition); err != nil { - return err - } - - return errBreakForeach - } - return nil - })(store, di, deadline) - })(store, deadlines) - if err == errBreakForeach { - err = nil - } - return err - })(store, mas) - }))) -} - func (a *StateAPI) StateSectorExpiration(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tsk types.TipSetKey) (*api.SectorExpiration, error) { - var onTimeEpoch, earlyEpoch abi.ChainEpoch - - err := a.sectorPartition(ctx, maddr, sectorNumber, tsk, func(store adt.Store, mas *miner.State, di uint64, pi uint64, part *miner.Partition) error { - quant := mas.QuantSpecForDeadline(di) - expirations, err := miner.LoadExpirationQueue(store, part.ExpirationsEpochs, quant) - if err != nil { - return xerrors.Errorf("loading expiration queue: %w", err) - } - - var eset miner.ExpirationSet - return expirations.Array.ForEach(&eset, func(epoch int64) error { - set, err := eset.OnTimeSectors.IsSet(uint64(sectorNumber)) - if err != nil { - return xerrors.Errorf("checking if sector is in onTime set: %w", err) - } - if set { - onTimeEpoch = abi.ChainEpoch(epoch) - } - - set, err = eset.EarlySectors.IsSet(uint64(sectorNumber)) - if err != nil { - return xerrors.Errorf("checking if sector is in early set: %w", err) - } - if set { - earlyEpoch = abi.ChainEpoch(epoch) - } - - return nil - }) - }) + act, err := a.StateManager.LoadActorTsk(ctx, maddr, tsk) if err != nil { return nil, err } - - if onTimeEpoch == 0 { - return nil, xerrors.Errorf("expiration for sector %d not found", sectorNumber) + mas, err := miner.Load(a.StateManager.ChainStore().Store(ctx), act) + if err != nil { + return nil, err } - - return &api.SectorExpiration{ - OnTime: onTimeEpoch, - Early: earlyEpoch, - }, nil + return mas.GetSectorExpiration(sectorNumber) } func (a *StateAPI) StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tsk types.TipSetKey) (*api.SectorLocation, error) { - var found *api.SectorLocation - - err := a.sectorPartition(ctx, maddr, sectorNumber, tsk, func(store adt.Store, mas *miner.State, di, pi uint64, partition *miner.Partition) error { - found = &api.SectorLocation{ - Deadline: di, - Partition: pi, - } - return errBreakForeach - }) + act, err := a.StateManager.LoadActorTsk(ctx, maddr, tsk) if err != nil { return nil, err } - - return found, nil + mas, err := miner.Load(a.StateManager.ChainStore().Store(ctx), act) + if err != nil { + return nil, err + } + return mas.FindSector(sectorNumber) } func (a *StateAPI) StateListMessages(ctx context.Context, match *types.Message, tsk types.TipSetKey, toheight abi.ChainEpoch) ([]cid.Cid, error) { From 6bf7976add095e8544e28f67517985dfb5e6591e Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 15 Sep 2020 10:51:18 -0700 Subject: [PATCH 121/885] fix decode --- chain/actors/builtin/miner/v0.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/chain/actors/builtin/miner/v0.go b/chain/actors/builtin/miner/v0.go index 1a4bd3800..c0712a0c2 100644 --- a/chain/actors/builtin/miner/v0.go +++ b/chain/actors/builtin/miner/v0.go @@ -1,6 +1,7 @@ package miner import ( + "bytes" "errors" "github.com/filecoin-project/go-address" @@ -151,7 +152,7 @@ func (s *v0State) LoadSectorsFromSet(filter *bitfield.BitField, filterOut bool) } var oci miner.SectorOnChainInfo - if err := cbor.DecodeInto(v.Raw, &oci); err != nil { + if err := oci.UnmarshalCBOR(bytes.NewReader(v.Raw)); err != nil { return err } sset = append(sset, &ChainSectorInfo{ From 4cd92d8576aeb31044b821d09ef1d333e2a24f0c Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 15 Sep 2020 10:57:32 -0700 Subject: [PATCH 122/885] remove final WithStateTree --- chain/messagepool/provider.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/chain/messagepool/provider.go b/chain/messagepool/provider.go index d67468d9a..347e90044 100644 --- a/chain/messagepool/provider.go +++ b/chain/messagepool/provider.go @@ -47,13 +47,15 @@ func (mpp *mpoolProvider) PubSubPublish(k string, v []byte) error { } func (mpp *mpoolProvider) GetActorAfter(addr address.Address, ts *types.TipSet) (*types.Actor, error) { - var act types.Actor stcid, _, err := mpp.sm.TipSetState(context.TODO(), ts) if err != nil { return nil, xerrors.Errorf("computing tipset state for GetActor: %w", err) } - version := mpp.sm.GetNtwkVersion(context.TODO(), ts.Height()) - return &act, mpp.sm.WithStateTree(stcid, version, mpp.sm.WithActor(addr, stmgr.GetActor(&act))) + st, err := mpp.sm.StateTree(stcid) + if err != nil { + return nil, xerrors.Errorf("failed to load state tree: %w", err) + } + return st.GetActor(addr) } func (mpp *mpoolProvider) StateAccountKey(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) { From 53384e83d7c6d5488c0ec605959666dee49dc248 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 15 Sep 2020 11:16:35 -0700 Subject: [PATCH 123/885] migrate wallet access --- node/impl/full/wallet.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/node/impl/full/wallet.go b/node/impl/full/wallet.go index af786085b..bda8824e7 100644 --- a/node/impl/full/wallet.go +++ b/node/impl/full/wallet.go @@ -36,16 +36,13 @@ func (a *WalletAPI) WalletList(ctx context.Context) ([]address.Address, error) { } func (a *WalletAPI) WalletBalance(ctx context.Context, addr address.Address) (types.BigInt, error) { - var bal types.BigInt - err := a.StateManager.WithParentStateTsk(types.EmptyTSK, a.StateManager.WithActor(addr, func(act *types.Actor) error { - bal = act.Balance - return nil - })) - + act, err := a.StateManager.LoadActorTsk(ctx, addr, types.EmptyTSK) if xerrors.Is(err, types.ErrActorNotFound) { return big.Zero(), nil + } else if err != nil { + return big.Zero(), err } - return bal, err + return act.Balance, nil } func (a *WalletAPI) WalletSign(ctx context.Context, k address.Address, msg []byte) (*crypto.Signature, error) { From 24756ac0568e2d43682a12aa419a9bdeb1cac72e Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 15 Sep 2020 11:55:54 -0700 Subject: [PATCH 124/885] allow specification of repo directory in lotus-bench --- cmd/lotus-bench/import.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index 7fe63eae1..f2845ba20 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -56,6 +56,10 @@ var importBenchCmd = &cli.Command{ Usage: "set the parallelism factor for batch seal verification", Value: runtime.NumCPU(), }, + &cli.StringFlag{ + Name: "repodir", + Usage: "set the repo directory for the lotus bench run (defaults to /tmp)", + }, }, Action: func(cctx *cli.Context) error { vm.BatchSealVerifyParallelism = cctx.Int("batch-seal-verify-threads") @@ -70,9 +74,15 @@ var importBenchCmd = &cli.Command{ } defer cfi.Close() //nolint:errcheck // read only file - tdir, err := ioutil.TempDir("", "lotus-import-bench") - if err != nil { - return err + var tdir string + if rdir := cctx.String("repodir"); rdir != "" { + tdir = rdir + } else { + tmp, err := ioutil.TempDir("", "lotus-import-bench") + if err != nil { + return err + } + tdir = tmp } bds, err := badger.NewDatastore(tdir, nil) From c64f983900f8a0e2b7c1800dd2a4b1b96b201d1b Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 15 Sep 2020 12:09:39 -0700 Subject: [PATCH 125/885] migrate StateMinserSectorCount --- api/api_full.go | 8 ++- api/apistruct/struct.go | 88 ++++++++++++++++----------------- cmd/lotus-storage-miner/info.go | 21 +++----- node/impl/full/state.go | 85 +++++++++++++++---------------- storage/miner.go | 7 ++- 5 files changed, 99 insertions(+), 110 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index a5af175d8..d3b655634 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -469,8 +469,12 @@ type FileRef struct { } type MinerSectors struct { - Sectors uint64 - Active uint64 + // Live sectors that should be proven. + Live uint64 + // Sectors actively contributing to power. + Active uint64 + // Sectors with failed proofs. + Faulty uint64 } type ImportRes struct { diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index d0bdd14d3..a554d2a5f 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -34,7 +34,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" - miner2 "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/modules/dtypes" ) @@ -161,44 +161,44 @@ type FullNodeStruct struct { ClientDataTransferUpdates func(ctx context.Context) (<-chan api.DataTransferChannel, error) `perm:"write"` ClientRetrieveTryRestartInsufficientFunds func(ctx context.Context, paymentChannel address.Address) error `perm:"write"` - StateNetworkName func(context.Context) (dtypes.NetworkName, error) `perm:"read"` - StateMinerSectors func(context.Context, address.Address, *bitfield.BitField, bool, types.TipSetKey) ([]*miner2.ChainSectorInfo, error) `perm:"read"` - StateMinerActiveSectors func(context.Context, address.Address, types.TipSetKey) ([]*miner2.ChainSectorInfo, error) `perm:"read"` - StateMinerProvingDeadline func(context.Context, address.Address, types.TipSetKey) (*dline.Info, error) `perm:"read"` - StateMinerPower func(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error) `perm:"read"` - StateMinerInfo func(context.Context, address.Address, types.TipSetKey) (miner2.MinerInfo, error) `perm:"read"` - StateMinerFaults func(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error) `perm:"read"` - StateAllMinerFaults func(context.Context, abi.ChainEpoch, types.TipSetKey) ([]*api.Fault, error) `perm:"read"` - StateMinerRecoveries func(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error) `perm:"read"` - StateMinerPreCommitDepositForPower func(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error) `perm:"read"` - StateMinerInitialPledgeCollateral func(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error) `perm:"read"` - StateMinerAvailableBalance func(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) `perm:"read"` - StateSectorPreCommitInfo func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) `perm:"read"` - StateSectorGetInfo func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorOnChainInfo, error) `perm:"read"` - StateSectorExpiration func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*api.SectorExpiration, error) `perm:"read"` - StateSectorPartition func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*api.SectorLocation, error) `perm:"read"` - StateCall func(context.Context, *types.Message, types.TipSetKey) (*api.InvocResult, error) `perm:"read"` - StateReplay func(context.Context, types.TipSetKey, cid.Cid) (*api.InvocResult, error) `perm:"read"` - StateGetActor func(context.Context, address.Address, types.TipSetKey) (*types.Actor, error) `perm:"read"` - StateReadState func(context.Context, address.Address, types.TipSetKey) (*api.ActorState, error) `perm:"read"` - StateWaitMsg func(ctx context.Context, cid cid.Cid, confidence uint64) (*api.MsgLookup, error) `perm:"read"` - StateSearchMsg func(context.Context, cid.Cid) (*api.MsgLookup, error) `perm:"read"` - StateListMiners func(context.Context, types.TipSetKey) ([]address.Address, error) `perm:"read"` - StateListActors func(context.Context, types.TipSetKey) ([]address.Address, error) `perm:"read"` - StateMarketBalance func(context.Context, address.Address, types.TipSetKey) (api.MarketBalance, error) `perm:"read"` - StateMarketParticipants func(context.Context, types.TipSetKey) (map[string]api.MarketBalance, error) `perm:"read"` - StateMarketDeals func(context.Context, types.TipSetKey) (map[string]api.MarketDeal, error) `perm:"read"` - StateMarketStorageDeal func(context.Context, abi.DealID, types.TipSetKey) (*api.MarketDeal, error) `perm:"read"` - StateLookupID func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) `perm:"read"` - StateAccountKey func(context.Context, address.Address, types.TipSetKey) (address.Address, error) `perm:"read"` - StateChangedActors func(context.Context, cid.Cid, cid.Cid) (map[string]types.Actor, error) `perm:"read"` - StateGetReceipt func(context.Context, cid.Cid, types.TipSetKey) (*types.MessageReceipt, error) `perm:"read"` - StateMinerSectorCount func(context.Context, address.Address, types.TipSetKey) (api.MinerSectors, error) `perm:"read"` - StateListMessages func(ctx context.Context, match *types.Message, tsk types.TipSetKey, toht abi.ChainEpoch) ([]cid.Cid, error) `perm:"read"` - StateCompute func(context.Context, abi.ChainEpoch, []*types.Message, types.TipSetKey) (*api.ComputeStateOutput, error) `perm:"read"` - StateVerifiedClientStatus func(context.Context, address.Address, types.TipSetKey) (*verifreg.DataCap, error) `perm:"read"` - StateDealProviderCollateralBounds func(context.Context, abi.PaddedPieceSize, bool, types.TipSetKey) (api.DealCollateralBounds, error) `perm:"read"` - StateCirculatingSupply func(context.Context, types.TipSetKey) (api.CirculatingSupply, error) `perm:"read"` + StateNetworkName func(context.Context) (dtypes.NetworkName, error) `perm:"read"` + StateMinerSectors func(context.Context, address.Address, *bitfield.BitField, bool, types.TipSetKey) ([]*miner.ChainSectorInfo, error) `perm:"read"` + StateMinerActiveSectors func(context.Context, address.Address, types.TipSetKey) ([]*miner.ChainSectorInfo, error) `perm:"read"` + StateMinerProvingDeadline func(context.Context, address.Address, types.TipSetKey) (*dline.Info, error) `perm:"read"` + StateMinerPower func(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error) `perm:"read"` + StateMinerInfo func(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error) `perm:"read"` + StateMinerFaults func(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error) `perm:"read"` + StateAllMinerFaults func(context.Context, abi.ChainEpoch, types.TipSetKey) ([]*api.Fault, error) `perm:"read"` + StateMinerRecoveries func(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error) `perm:"read"` + StateMinerPreCommitDepositForPower func(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error) `perm:"read"` + StateMinerInitialPledgeCollateral func(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error) `perm:"read"` + StateMinerAvailableBalance func(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) `perm:"read"` + StateSectorPreCommitInfo func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) `perm:"read"` + StateSectorGetInfo func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorOnChainInfo, error) `perm:"read"` + StateSectorExpiration func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorExpiration, error) `perm:"read"` + StateSectorPartition func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorLocation, error) `perm:"read"` + StateCall func(context.Context, *types.Message, types.TipSetKey) (*api.InvocResult, error) `perm:"read"` + StateReplay func(context.Context, types.TipSetKey, cid.Cid) (*api.InvocResult, error) `perm:"read"` + StateGetActor func(context.Context, address.Address, types.TipSetKey) (*types.Actor, error) `perm:"read"` + StateReadState func(context.Context, address.Address, types.TipSetKey) (*api.ActorState, error) `perm:"read"` + StateWaitMsg func(ctx context.Context, cid cid.Cid, confidence uint64) (*api.MsgLookup, error) `perm:"read"` + StateSearchMsg func(context.Context, cid.Cid) (*api.MsgLookup, error) `perm:"read"` + StateListMiners func(context.Context, types.TipSetKey) ([]address.Address, error) `perm:"read"` + StateListActors func(context.Context, types.TipSetKey) ([]address.Address, error) `perm:"read"` + StateMarketBalance func(context.Context, address.Address, types.TipSetKey) (api.MarketBalance, error) `perm:"read"` + StateMarketParticipants func(context.Context, types.TipSetKey) (map[string]api.MarketBalance, error) `perm:"read"` + StateMarketDeals func(context.Context, types.TipSetKey) (map[string]api.MarketDeal, error) `perm:"read"` + StateMarketStorageDeal func(context.Context, abi.DealID, types.TipSetKey) (*api.MarketDeal, error) `perm:"read"` + StateLookupID func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) `perm:"read"` + StateAccountKey func(context.Context, address.Address, types.TipSetKey) (address.Address, error) `perm:"read"` + StateChangedActors func(context.Context, cid.Cid, cid.Cid) (map[string]types.Actor, error) `perm:"read"` + StateGetReceipt func(context.Context, cid.Cid, types.TipSetKey) (*types.MessageReceipt, error) `perm:"read"` + StateMinerSectorCount func(context.Context, address.Address, types.TipSetKey) (api.MinerSectors, error) `perm:"read"` + StateListMessages func(ctx context.Context, match *types.Message, tsk types.TipSetKey, toht abi.ChainEpoch) ([]cid.Cid, error) `perm:"read"` + StateCompute func(context.Context, abi.ChainEpoch, []*types.Message, types.TipSetKey) (*api.ComputeStateOutput, error) `perm:"read"` + StateVerifiedClientStatus func(context.Context, address.Address, types.TipSetKey) (*verifreg.DataCap, error) `perm:"read"` + StateDealProviderCollateralBounds func(context.Context, abi.PaddedPieceSize, bool, types.TipSetKey) (api.DealCollateralBounds, error) `perm:"read"` + StateCirculatingSupply func(context.Context, types.TipSetKey) (api.CirculatingSupply, error) `perm:"read"` MsigGetAvailableBalance func(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) `perm:"read"` MsigGetVested func(context.Context, address.Address, types.TipSetKey, types.TipSetKey) (types.BigInt, error) `perm:"read"` @@ -730,11 +730,11 @@ func (c *FullNodeStruct) StateNetworkName(ctx context.Context) (dtypes.NetworkNa return c.Internal.StateNetworkName(ctx) } -func (c *FullNodeStruct) StateMinerSectors(ctx context.Context, addr address.Address, filter *bitfield.BitField, filterOut bool, tsk types.TipSetKey) ([]*miner2.ChainSectorInfo, error) { +func (c *FullNodeStruct) StateMinerSectors(ctx context.Context, addr address.Address, filter *bitfield.BitField, filterOut bool, tsk types.TipSetKey) ([]*miner.ChainSectorInfo, error) { return c.Internal.StateMinerSectors(ctx, addr, filter, filterOut, tsk) } -func (c *FullNodeStruct) StateMinerActiveSectors(ctx context.Context, addr address.Address, tsk types.TipSetKey) ([]*miner2.ChainSectorInfo, error) { +func (c *FullNodeStruct) StateMinerActiveSectors(ctx context.Context, addr address.Address, tsk types.TipSetKey) ([]*miner.ChainSectorInfo, error) { return c.Internal.StateMinerActiveSectors(ctx, addr, tsk) } @@ -746,7 +746,7 @@ func (c *FullNodeStruct) StateMinerPower(ctx context.Context, a address.Address, return c.Internal.StateMinerPower(ctx, a, tsk) } -func (c *FullNodeStruct) StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (miner2.MinerInfo, error) { +func (c *FullNodeStruct) StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (miner.MinerInfo, error) { return c.Internal.StateMinerInfo(ctx, actor, tsk) } @@ -782,11 +782,11 @@ func (c *FullNodeStruct) StateSectorGetInfo(ctx context.Context, maddr address.A return c.Internal.StateSectorGetInfo(ctx, maddr, n, tsk) } -func (c *FullNodeStruct) StateSectorExpiration(ctx context.Context, maddr address.Address, n abi.SectorNumber, tsk types.TipSetKey) (*api.SectorExpiration, error) { +func (c *FullNodeStruct) StateSectorExpiration(ctx context.Context, maddr address.Address, n abi.SectorNumber, tsk types.TipSetKey) (*miner.SectorExpiration, error) { return c.Internal.StateSectorExpiration(ctx, maddr, n, tsk) } -func (c *FullNodeStruct) StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*api.SectorLocation, error) { +func (c *FullNodeStruct) StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*miner.SectorLocation, error) { return c.Internal.StateSectorPartition(ctx, maddr, sectorNumber, tok) } diff --git a/cmd/lotus-storage-miner/info.go b/cmd/lotus-storage-miner/info.go index 55ef024f3..c47a22b0e 100644 --- a/cmd/lotus-storage-miner/info.go +++ b/cmd/lotus-storage-miner/info.go @@ -112,26 +112,19 @@ func infoCmdAct(cctx *cli.Context) error { if err != nil { return err } - faults, err := api.StateMinerFaults(ctx, maddr, types.EmptyTSK) - if err != nil { - return err - } - nfaults, err := faults.Count() - if err != nil { - return xerrors.Errorf("counting faults: %w", err) - } - - fmt.Printf("\tCommitted: %s\n", types.SizeStr(types.BigMul(types.NewInt(secCounts.Sectors), types.NewInt(uint64(mi.SectorSize))))) + proving := secCounts.Active + secCounts.Faulty + nfaults := secCounts.Faulty + fmt.Printf("\tCommitted: %s\n", types.SizeStr(types.BigMul(types.NewInt(secCounts.Live), types.NewInt(uint64(mi.SectorSize))))) if nfaults == 0 { - fmt.Printf("\tProving: %s\n", types.SizeStr(types.BigMul(types.NewInt(secCounts.Active), types.NewInt(uint64(mi.SectorSize))))) + fmt.Printf("\tProving: %s\n", types.SizeStr(types.BigMul(types.NewInt(proving), types.NewInt(uint64(mi.SectorSize))))) } else { var faultyPercentage float64 - if secCounts.Sectors != 0 { - faultyPercentage = float64(10000*nfaults/secCounts.Sectors) / 100. + if secCounts.Live != 0 { + faultyPercentage = float64(10000*nfaults/secCounts.Live) / 100. } fmt.Printf("\tProving: %s (%s Faulty, %.2f%%)\n", - types.SizeStr(types.BigMul(types.NewInt(secCounts.Sectors), types.NewInt(uint64(mi.SectorSize)))), + types.SizeStr(types.BigMul(types.NewInt(proving), types.NewInt(uint64(mi.SectorSize)))), types.SizeStr(types.BigMul(types.NewInt(nfaults), types.NewInt(uint64(mi.SectorSize)))), faultyPercentage) } diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 73ab7bc4d..d1490af64 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -635,57 +635,50 @@ func (a *StateAPI) StateChangedActors(ctx context.Context, old cid.Cid, new cid. } func (a *StateAPI) StateMinerSectorCount(ctx context.Context, addr address.Address, tsk types.TipSetKey) (api.MinerSectors, error) { - var out api.MinerSectors - - err := a.StateManager.WithParentStateTsk(tsk, - a.StateManager.WithActor(addr, - a.StateManager.WithActorState(ctx, func(store adt.Store, mas *miner.State) error { - var allActive []bitfield.BitField - - err := a.StateManager.WithDeadlines( - a.StateManager.WithEachDeadline( - a.StateManager.WithEachPartition(func(store adt.Store, partIdx uint64, partition *miner.Partition) error { - active, err := partition.ActiveSectors() - if err != nil { - return xerrors.Errorf("partition.ActiveSectors: %w", err) - } - - allActive = append(allActive, active) - return nil - })))(store, mas) - if err != nil { - return xerrors.Errorf("with deadlines: %w", err) - } - - active, err := bitfield.MultiMerge(allActive...) - if err != nil { - return xerrors.Errorf("merging active sector bitfields: %w", err) - } - - out.Active, err = active.Count() - if err != nil { - return xerrors.Errorf("counting active sectors: %w", err) - } - - sarr, err := adt.AsArray(store, mas.Sectors) - if err != nil { - return err - } - - out.Sectors = sarr.Length() - return nil - }))) + act, err := a.StateManager.LoadActorTsk(ctx, addr, tsk) if err != nil { return api.MinerSectors{}, err } - - return out, nil + mas, err := miner.Load(a.Chain.Store(ctx), act) + if err != nil { + return api.MinerSectors{}, err + } + var activeCount, liveCount, faultyCount uint64 + if err := mas.ForEachDeadline(func(_ uint64, dl miner.Deadline) error { + return dl.ForEachPartition(func(_ uint64, part miner.Partition) error { + if active, err := part.ActiveSectors(); err != nil { + return err + } else if count, err := active.Count(); err != nil { + return err + } else { + activeCount += count + } + if live, err := part.LiveSectors(); err != nil { + return err + } else if count, err := live.Count(); err != nil { + return err + } else { + liveCount += count + } + if faulty, err := part.FaultySectors(); err != nil { + return err + } else if count, err := faulty.Count(); err != nil { + return err + } else { + faultyCount += count + } + return nil + }) + }); err != nil { + return api.MinerSectors{}, err + } + return api.MinerSectors{Live: liveCount, Active: activeCount, Faulty: faultyCount}, nil } -func (a *StateAPI) StateSectorPreCommitInfo(ctx context.Context, maddr address.Address, n abi.SectorNumber, tsk types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) { +func (a *StateAPI) StateSectorPreCommitInfo(ctx context.Context, maddr address.Address, n abi.SectorNumber, tsk types.TipSetKey) (*miner.SectorPreCommitOnChainInfo, error) { ts, err := a.Chain.GetTipSetFromKey(tsk) if err != nil { - return miner.SectorPreCommitOnChainInfo{}, xerrors.Errorf("loading tipset %s: %w", tsk, err) + return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) } return stmgr.PreCommitInfo(ctx, a.StateManager, maddr, n, ts) } @@ -698,7 +691,7 @@ func (a *StateAPI) StateSectorGetInfo(ctx context.Context, maddr address.Address return stmgr.MinerSectorInfo(ctx, a.StateManager, maddr, n, ts) } -func (a *StateAPI) StateSectorExpiration(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tsk types.TipSetKey) (*api.SectorExpiration, error) { +func (a *StateAPI) StateSectorExpiration(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tsk types.TipSetKey) (*miner.SectorExpiration, error) { act, err := a.StateManager.LoadActorTsk(ctx, maddr, tsk) if err != nil { return nil, err @@ -710,7 +703,7 @@ func (a *StateAPI) StateSectorExpiration(ctx context.Context, maddr address.Addr return mas.GetSectorExpiration(sectorNumber) } -func (a *StateAPI) StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tsk types.TipSetKey) (*api.SectorLocation, error) { +func (a *StateAPI) StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tsk types.TipSetKey) (*miner.SectorLocation, error) { act, err := a.StateManager.LoadActorTsk(ctx, maddr, tsk) if err != nil { return nil, err diff --git a/storage/miner.go b/storage/miner.go index 47f50ce71..3485dba03 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -3,7 +3,6 @@ package storage import ( "context" "errors" - miner2 "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "time" "github.com/filecoin-project/go-state-types/dline" @@ -22,11 +21,11 @@ import ( "github.com/filecoin-project/go-state-types/crypto" sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/specs-storage/storage" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/events" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/types" @@ -68,10 +67,10 @@ type SealingStateEvt struct { type storageMinerApi interface { // Call a read only method on actors (no interaction with the chain required) StateCall(context.Context, *types.Message, types.TipSetKey) (*api.InvocResult, error) - StateMinerSectors(context.Context, address.Address, *bitfield.BitField, bool, types.TipSetKey) ([]*miner2.ChainSectorInfo, error) + StateMinerSectors(context.Context, address.Address, *bitfield.BitField, bool, types.TipSetKey) ([]*miner.ChainSectorInfo, error) StateSectorPreCommitInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) StateSectorGetInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorOnChainInfo, error) - StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*api.SectorLocation, error) + StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*miner.SectorLocation, error) StateMinerInfo(context.Context, address.Address, types.TipSetKey) (api.MinerInfo, error) StateMinerProvingDeadline(context.Context, address.Address, types.TipSetKey) (*dline.Info, error) StateMinerPreCommitDepositForPower(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error) From 91e9573863b8161b417910c253fdb994624ba211 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 15 Sep 2020 12:13:13 -0700 Subject: [PATCH 126/885] Compile fixes --- api/api_full.go | 4 ++-- api/apistruct/struct.go | 1 - chain/actors/builtin/miner/v0.go | 8 ++++---- chain/stmgr/utils.go | 6 ++++-- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index d3b655634..88496d669 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -344,9 +344,9 @@ type FullNode interface { // expiration epoch StateSectorGetInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorOnChainInfo, error) // StateSectorExpiration returns epoch at which given sector will expire - StateSectorExpiration(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*SectorExpiration, error) + StateSectorExpiration(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorExpiration, error) // StateSectorPartition finds deadline/partition with the specified sector - StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*SectorLocation, error) + StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*miner.SectorLocation, error) // StateSearchMsg searches for a message in the chain, and returns its receipt and the tipset where it was executed StateSearchMsg(context.Context, cid.Cid) (*MsgLookup, error) // StateWaitMsg looks back in the chain for a message. If not found, it blocks until the diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index a554d2a5f..1fcd5e0f3 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -27,7 +27,6 @@ import ( "github.com/filecoin-project/lotus/extern/sector-storage/stores" "github.com/filecoin-project/lotus/extern/sector-storage/storiface" marketevents "github.com/filecoin-project/lotus/markets/loggers" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/specs-actors/actors/builtin/paych" "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" "github.com/filecoin-project/specs-storage/storage" diff --git a/chain/actors/builtin/miner/v0.go b/chain/actors/builtin/miner/v0.go index c0712a0c2..e1ef9c51f 100644 --- a/chain/actors/builtin/miner/v0.go +++ b/chain/actors/builtin/miner/v0.go @@ -10,7 +10,6 @@ import ( "github.com/filecoin-project/go-state-types/dline" "github.com/filecoin-project/lotus/chain/actors/adt" v0adt "github.com/filecoin-project/specs-actors/actors/util/adt" - cbor "github.com/ipfs/go-ipld-cbor" "github.com/libp2p/go-libp2p-core/peer" cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" @@ -70,7 +69,7 @@ func (s *v0State) GetSectorExpiration(num abi.SectorNumber) (out *SectorExpirati // 2. If it's faulty, it will expire early within the first 14 entries // of the expiration queue. stopErr := errors.New("stop") - err := dls.ForEach(s.store, func(dlIdx uint64, dl *miner.Deadline) error { + err = dls.ForEach(s.store, func(dlIdx uint64, dl *miner.Deadline) error { partitions, err := dl.PartitionsArray(s.store) if err != nil { return err @@ -102,12 +101,13 @@ func (s *v0State) GetSectorExpiration(num abi.SectorNumber) (out *SectorExpirati out.Early = abi.ChainEpoch(epoch) return nil } - if onTime, err := exp.OnTime.IsSet(uint64(num)); err != nil { + if onTime, err := exp.OnTimeSectors.IsSet(uint64(num)); err != nil { return err } else if onTime { - out.OnTime = epoch + out.OnTime = abi.ChainEpoch(epoch) return stopErr } + return nil }) }) }) diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index 4dc210154..88275f019 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -103,8 +103,10 @@ func GetPowerRaw(ctx context.Context, sm *StateManager, st cid.Cid, maddr addres var mpow power.Claim if maddr != address.Undef { - mpow, err = mas.MinerPower(maddr) - if err != nil { + var found bool + mpow, found, err = mas.MinerPower(maddr) + if err != nil || !found { + // TODO: return an error when not found? return power.Claim{}, power.Claim{}, err } } From ed4caac9bff516ffd16544641c5fcc3b17c9c88f Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 15 Sep 2020 22:24:09 +0100 Subject: [PATCH 127/885] fix: remove network version and total fil --- conformance/chaos/actor.go | 27 ++++++++---------- conformance/chaos/cbor_gen.go | 54 ++--------------------------------- 2 files changed, 14 insertions(+), 67 deletions(-) diff --git a/conformance/chaos/actor.go b/conformance/chaos/actor.go index 526dac64c..005a06f0d 100644 --- a/conformance/chaos/actor.go +++ b/conformance/chaos/actor.go @@ -254,14 +254,12 @@ func (a Actor) AbortWith(rt runtime.Runtime, args *AbortWithArgs) *abi.EmptyValu // InspectRuntimeReturn is the return value for the Actor.InspectRuntime method. type InspectRuntimeReturn struct { - NetworkVersion int64 - Caller address.Address - Receiver address.Address - ValueReceived abi.TokenAmount - CurrEpoch abi.ChainEpoch - CurrentBalance abi.TokenAmount - State State - TotalFilCircSupply abi.TokenAmount + Caller address.Address + Receiver address.Address + ValueReceived abi.TokenAmount + CurrEpoch abi.ChainEpoch + CurrentBalance abi.TokenAmount + State State } // InspectRuntime returns a copy of the serializable values available in the Runtime. @@ -270,12 +268,11 @@ func (a Actor) InspectRuntime(rt runtime.Runtime, _ *abi.EmptyValue) *InspectRun var st State rt.StateReadonly(&st) return &InspectRuntimeReturn{ - NetworkVersion: int64(rt.NetworkVersion()), - Caller: rt.Caller(), - Receiver: rt.Receiver(), - ValueReceived: rt.ValueReceived(), - CurrentBalance: rt.CurrentBalance(), - State: st, - TotalFilCircSupply: rt.TotalFilCircSupply(), + Caller: rt.Caller(), + Receiver: rt.Receiver(), + ValueReceived: rt.ValueReceived(), + CurrEpoch: rt.CurrEpoch(), + CurrentBalance: rt.CurrentBalance(), + State: st, } } diff --git a/conformance/chaos/cbor_gen.go b/conformance/chaos/cbor_gen.go index 96b707386..2d9deec93 100644 --- a/conformance/chaos/cbor_gen.go +++ b/conformance/chaos/cbor_gen.go @@ -731,7 +731,7 @@ func (t *AbortWithArgs) UnmarshalCBOR(r io.Reader) error { return nil } -var lengthBufInspectRuntimeReturn = []byte{136} +var lengthBufInspectRuntimeReturn = []byte{134} func (t *InspectRuntimeReturn) MarshalCBOR(w io.Writer) error { if t == nil { @@ -744,17 +744,6 @@ func (t *InspectRuntimeReturn) MarshalCBOR(w io.Writer) error { scratch := make([]byte, 9) - // t.NetworkVersion (int64) (int64) - if t.NetworkVersion >= 0 { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.NetworkVersion)); err != nil { - return err - } - } else { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.NetworkVersion-1)); err != nil { - return err - } - } - // t.Caller (address.Address) (struct) if err := t.Caller.MarshalCBOR(w); err != nil { return err @@ -790,11 +779,6 @@ func (t *InspectRuntimeReturn) MarshalCBOR(w io.Writer) error { if err := t.State.MarshalCBOR(w); err != nil { return err } - - // t.TotalFilCircSupply (big.Int) (struct) - if err := t.TotalFilCircSupply.MarshalCBOR(w); err != nil { - return err - } return nil } @@ -812,35 +796,10 @@ func (t *InspectRuntimeReturn) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type array") } - if extra != 8 { + if extra != 6 { return fmt.Errorf("cbor input had wrong number of fields") } - // t.NetworkVersion (int64) (int64) - { - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) - var extraI int64 - if err != nil { - return err - } - switch maj { - case cbg.MajUnsignedInt: - extraI = int64(extra) - if extraI < 0 { - return fmt.Errorf("int64 positive overflow") - } - case cbg.MajNegativeInt: - extraI = int64(extra) - if extraI < 0 { - return fmt.Errorf("int64 negative oveflow") - } - extraI = -1 - extraI - default: - return fmt.Errorf("wrong type for int64 field: %d", maj) - } - - t.NetworkVersion = int64(extraI) - } // t.Caller (address.Address) (struct) { @@ -910,15 +869,6 @@ func (t *InspectRuntimeReturn) UnmarshalCBOR(r io.Reader) error { return xerrors.Errorf("unmarshaling t.State: %w", err) } - } - // t.TotalFilCircSupply (big.Int) (struct) - - { - - if err := t.TotalFilCircSupply.UnmarshalCBOR(br); err != nil { - return xerrors.Errorf("unmarshaling t.TotalFilCircSupply: %w", err) - } - } return nil } From 4e01fad0d47faafbf799eb7a0e6ee14e065018b2 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 15 Sep 2020 14:44:03 -0700 Subject: [PATCH 128/885] start multisig abstraction --- chain/actors/builtin/multisig/multisig.go | 31 +++++++++++++ chain/actors/builtin/multisig/v0.go | 16 +++++++ node/impl/full/state.go | 55 +++++++++-------------- 3 files changed, 69 insertions(+), 33 deletions(-) create mode 100644 chain/actors/builtin/multisig/multisig.go create mode 100644 chain/actors/builtin/multisig/v0.go diff --git a/chain/actors/builtin/multisig/multisig.go b/chain/actors/builtin/multisig/multisig.go new file mode 100644 index 000000000..676dbe75f --- /dev/null +++ b/chain/actors/builtin/multisig/multisig.go @@ -0,0 +1,31 @@ +package multisig + +import ( + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/cbor" + v0builtin "github.com/filecoin-project/specs-actors/actors/builtin" + + "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/chain/types" +) + +func Load(store adt.Store, act *types.Actor) (State, error) { + switch act.Code { + case v0builtin.MultisigActorCodeID: + out := v0State{store: store} + err := store.Get(store.Context(), act.Head, &out) + if err != nil { + return nil, err + } + return &out, nil + } + return nil, xerrors.Errorf("unknown actor code %s", act.Code) +} + +type State interface { + cbor.Marshaler + + LockedBalance(epoch abi.ChainEpoch) (abi.TokenAmount, error) +} diff --git a/chain/actors/builtin/multisig/v0.go b/chain/actors/builtin/multisig/v0.go new file mode 100644 index 000000000..3bc7e70b2 --- /dev/null +++ b/chain/actors/builtin/multisig/v0.go @@ -0,0 +1,16 @@ +package multisig + +import ( + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/specs-actors/actors/builtin/multisig" +) + +type v0State struct { + multisig.State + store adt.Store +} + +func (s *v0State) LockedBalance(currEpoch abi.ChainEpoch) (abi.TokenAmount, error) { + return s.State.AmountLocked(currEpoch - s.StartEpoch), nil +} diff --git a/node/impl/full/state.go b/node/impl/full/state.go index d1490af64..10b5de82b 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -22,7 +22,6 @@ import ( "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/builtin/market" - samsig "github.com/filecoin-project/specs-actors/actors/builtin/multisig" "github.com/filecoin-project/specs-actors/actors/builtin/power" "github.com/filecoin-project/specs-actors/actors/builtin/reward" "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" @@ -31,6 +30,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + "github.com/filecoin-project/lotus/chain/actors/builtin/multisig" "github.com/filecoin-project/lotus/chain/beacon" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/state" @@ -790,28 +790,19 @@ func (a *StateAPI) MsigGetAvailableBalance(ctx context.Context, addr address.Add return types.EmptyInt, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - var st samsig.State - act, err := a.StateManager.LoadActorState(ctx, addr, &st, ts) + act, err := a.StateManager.LoadActor(ctx, addr, ts) + if err != nil { + return types.EmptyInt, xerrors.Errorf("failed to load multisig actor: %w", err) + } + msas, err := multisig.Load(a.Chain.Store(ctx), act) if err != nil { return types.EmptyInt, xerrors.Errorf("failed to load multisig actor state: %w", err) } - - if act.Code != builtin.MultisigActorCodeID { - return types.EmptyInt, fmt.Errorf("given actor was not a multisig") + locked, err := msas.LockedBalance(ts.Height()) + if err != nil { + return types.EmptyInt, xerrors.Errorf("failed to compute locked multisig balance: %w", err) } - - if st.UnlockDuration == 0 { - return act.Balance, nil - } - - offset := ts.Height() - st.StartEpoch - if offset > st.UnlockDuration { - return act.Balance, nil - } - - minBalance := types.BigDiv(st.InitialBalance, types.NewInt(uint64(st.UnlockDuration))) - minBalance = types.BigMul(minBalance, types.NewInt(uint64(offset))) - return types.BigSub(act.Balance, minBalance), nil + return types.BigSub(act.Balance, locked), nil } func (a *StateAPI) MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error) { @@ -831,29 +822,27 @@ func (a *StateAPI) MsigGetVested(ctx context.Context, addr address.Address, star return big.Zero(), nil } - var mst samsig.State - act, err := a.StateManager.LoadActorState(ctx, addr, &mst, endTs) + act, err := a.StateManager.LoadActor(ctx, addr, endTs) if err != nil { - return types.EmptyInt, xerrors.Errorf("failed to load multisig actor state at end epoch: %w", err) + return types.EmptyInt, xerrors.Errorf("failed to load multisig actor at end epoch: %w", err) } - if act.Code != builtin.MultisigActorCodeID { - return types.EmptyInt, fmt.Errorf("given actor was not a multisig") + msas, err := multisig.Load(a.Chain.Store(ctx), act) + if err != nil { + return types.EmptyInt, xerrors.Errorf("failed to load multisig actor state: %w", err) } - if mst.UnlockDuration == 0 || - mst.InitialBalance.IsZero() || - mst.StartEpoch+mst.UnlockDuration <= startTs.Height() || - mst.StartEpoch >= endTs.Height() { - return big.Zero(), nil + startLk, err := msas.LockedBalance(startTs.Height()) + if err != nil { + return types.EmptyInt, xerrors.Errorf("failed to compute locked balance at start height: %w", err) } - startLk := mst.InitialBalance - if startTs.Height() > mst.StartEpoch { - startLk = mst.AmountLocked(startTs.Height() - mst.StartEpoch) + endLk, err := msas.LockedBalance(endTs.Height()) + if err != nil { + return types.EmptyInt, xerrors.Errorf("failed to compute locked balance at end height: %w", err) } - return big.Sub(startLk, mst.AmountLocked(endTs.Height()-mst.StartEpoch)), nil + return types.BigSub(startLk, endLk), nil } var initialPledgeNum = types.NewInt(110) From e1ba4eb5c449c9686c96d50aacd68d403d1370e0 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 15 Sep 2020 14:55:48 -0700 Subject: [PATCH 129/885] sub reward actor --- chain/actors/builtin/reward/reward.go | 30 +++++++++++++++++++++++++++ chain/actors/builtin/reward/v0.go | 11 ++++++++++ 2 files changed, 41 insertions(+) create mode 100644 chain/actors/builtin/reward/reward.go create mode 100644 chain/actors/builtin/reward/v0.go diff --git a/chain/actors/builtin/reward/reward.go b/chain/actors/builtin/reward/reward.go new file mode 100644 index 000000000..a1a7cac5d --- /dev/null +++ b/chain/actors/builtin/reward/reward.go @@ -0,0 +1,30 @@ +package reward + +import ( + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-state-types/cbor" + v0builtin "github.com/filecoin-project/specs-actors/actors/builtin" + + "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/chain/types" +) + +var Address = v0builtin.RewardActorAddr + +func Load(store adt.Store, act *types.Actor) (st State, err error) { + switch act.Code { + case v0builtin.RewardActorCodeID: + out := v0State{store: store} + err := store.Get(store.Context(), act.Head, &out) + if err != nil { + return nil, err + } + return &out, nil + } + return nil, xerrors.Errorf("unknown actor code %s", act.Code) +} + +type State interface { + cbor.Marshaler +} diff --git a/chain/actors/builtin/reward/v0.go b/chain/actors/builtin/reward/v0.go new file mode 100644 index 000000000..8fef3756c --- /dev/null +++ b/chain/actors/builtin/reward/v0.go @@ -0,0 +1,11 @@ +package reward + +import ( + "github.com/filecoin-project/specs-actors/actors/builtin/reward" + "github.com/filecoin-project/specs-actors/actors/util/adt" +) + +type v0State struct { + reward.State + store adt.Store +} From 02bc5fab268579539224af39e85b4a960209dbc2 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 15 Sep 2020 14:56:00 -0700 Subject: [PATCH 130/885] add power actor address alias --- chain/actors/builtin/power/power.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/chain/actors/builtin/power/power.go b/chain/actors/builtin/power/power.go index fa0400f2e..9e35e52ab 100644 --- a/chain/actors/builtin/power/power.go +++ b/chain/actors/builtin/power/power.go @@ -12,6 +12,8 @@ import ( "github.com/filecoin-project/lotus/chain/types" ) +var Address = v0builtin.StoragePowerActorAddr + func Load(store adt.Store, act *types.Actor) (st State, err error) { switch act.Code { case v0builtin.StoragePowerActorCodeID: From e53aee26a3d1b440729212fde7c77b8c06bf9a14 Mon Sep 17 00:00:00 2001 From: austinabell Date: Tue, 15 Sep 2020 18:36:06 -0400 Subject: [PATCH 131/885] Make state transition in validation async --- chain/sync.go | 45 +++++++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/chain/sync.go b/chain/sync.go index f7530f556..e23284134 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -788,31 +788,35 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) (er b.Header.ParentWeight, pweight) } - // Stuff that needs stateroot / worker address - stateroot, precp, err := syncer.sm.TipSetState(ctx, baseTs) - if err != nil { - return xerrors.Errorf("get tipsetstate(%d, %s) failed: %w", h.Height, h.Parents, err) - } - - if stateroot != h.ParentStateRoot { - msgs, err := syncer.store.MessagesForTipset(baseTs) + stateRootCheck := async.Err(func() error { + stateroot, precp, err := syncer.sm.TipSetState(ctx, baseTs) if err != nil { - log.Error("failed to load messages for tipset during tipset state mismatch error: ", err) - } else { - log.Warn("Messages for tipset with mismatching state:") - for i, m := range msgs { - mm := m.VMMessage() - log.Warnf("Message[%d]: from=%s to=%s method=%d params=%x", i, mm.From, mm.To, mm.Method, mm.Params) - } + return xerrors.Errorf("get tipsetstate(%d, %s) failed: %w", h.Height, h.Parents, err) } - return xerrors.Errorf("parent state root did not match computed state (%s != %s)", stateroot, h.ParentStateRoot) - } + if stateroot != h.ParentStateRoot { + msgs, err := syncer.store.MessagesForTipset(baseTs) + if err != nil { + log.Error("failed to load messages for tipset during tipset state mismatch error: ", err) + } else { + log.Warn("Messages for tipset with mismatching state:") + for i, m := range msgs { + mm := m.VMMessage() + log.Warnf("Message[%d]: from=%s to=%s method=%d params=%x", i, mm.From, mm.To, mm.Method, mm.Params) + } + } - if precp != h.ParentMessageReceipts { - return xerrors.Errorf("parent receipts root did not match computed value (%s != %s)", precp, h.ParentMessageReceipts) - } + return xerrors.Errorf("parent state root did not match computed state (%s != %s)", stateroot, h.ParentStateRoot) + } + if precp != h.ParentMessageReceipts { + return xerrors.Errorf("parent receipts root did not match computed value (%s != %s)", precp, h.ParentMessageReceipts) + } + + return nil + }) + + // Stuff that needs worker address waddr, err := stmgr.GetMinerWorkerRaw(ctx, syncer.sm, lbst, h.Miner) if err != nil { return xerrors.Errorf("GetMinerWorkerRaw failed: %w", err) @@ -933,6 +937,7 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) (er winnerCheck, msgsCheck, baseFeeCheck, + stateRootCheck, } var merr error From c2148ba106d7491f145c793f1c9fd9a20654e33c Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Tue, 15 Sep 2020 19:27:36 -0400 Subject: [PATCH 132/885] Multisig: Add a CLI command to propose locking some balance --- cli/multisig.go | 94 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/cli/multisig.go b/cli/multisig.go index 4596628f4..01e922a07 100644 --- a/cli/multisig.go +++ b/cli/multisig.go @@ -11,6 +11,10 @@ import ( "strconv" "text/tabwriter" + "github.com/filecoin-project/lotus/chain/actors" + + "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/util/adt" @@ -43,6 +47,9 @@ var multisigCmd = &cli.Command{ msigSwapProposeCmd, msigSwapApproveCmd, msigSwapCancelCmd, + msigLockProposeCmd, + msigLockApproveCmd, + msigLockCancelCmd, msigVestedCmd, }, } @@ -971,6 +978,93 @@ var msigSwapCancelCmd = &cli.Command{ }, } +var msigLockProposeCmd = &cli.Command{ + Name: "lock-propose", + Usage: "Propose to lock up some balance", + ArgsUsage: "[multisigAddress startEpoch unlockDuration amount]", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "from", + Usage: "account to send the approve message from", + }, + }, + Action: func(cctx *cli.Context) error { + if cctx.Args().Len() != 4 { + return ShowHelp(cctx, fmt.Errorf("must pass multisig address, start epoch, unlock duration, and amount")) + } + + api, closer, err := GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + ctx := ReqContext(cctx) + + msig, err := address.NewFromString(cctx.Args().Get(0)) + if err != nil { + return err + } + + start, err := strconv.ParseUint(cctx.Args().Get(1), 10, 64) + if err != nil { + return err + } + + duration, err := strconv.ParseUint(cctx.Args().Get(2), 10, 64) + if err != nil { + return err + } + + amount, err := types.ParseFIL(cctx.Args().Get(3)) + if err != nil { + return err + } + + var from address.Address + if cctx.IsSet("from") { + f, err := address.NewFromString(cctx.String("from")) + if err != nil { + return err + } + from = f + } else { + defaddr, err := api.WalletDefaultAddress(ctx) + if err != nil { + return err + } + from = defaddr + } + + params, actErr := actors.SerializeParams(&samsig.LockBalanceParams{ + StartEpoch: abi.ChainEpoch(start), + UnlockDuration: abi.ChainEpoch(duration), + Amount: abi.NewTokenAmount(amount.Int64()), + }) + + if actErr != nil { + return actErr + } + + msgCid, err := api.MsigPropose(ctx, msig, msig, big.Zero(), from, uint64(builtin.MethodsMultisig.LockBalance), params) + if err != nil { + return err + } + + fmt.Println("sent lock proposal in message: ", msgCid) + + wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) + if err != nil { + return err + } + + if wait.Receipt.ExitCode != 0 { + return fmt.Errorf("lock proposal returned exit %d", wait.Receipt.ExitCode) + } + + return nil + }, +} + var msigVestedCmd = &cli.Command{ Name: "vested", Usage: "Gets the amount vested in an msig between two epochs", From 92471d41d6923032eb807b857c02461e66d225b6 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 15 Sep 2020 16:47:58 -0700 Subject: [PATCH 133/885] migrate precommit deposit function --- chain/actors/builtin/builtin.go | 5 ++ chain/actors/builtin/market/market.go | 5 ++ chain/actors/builtin/market/v0.go | 7 ++ chain/actors/builtin/power/power.go | 3 + chain/actors/builtin/power/v0.go | 5 ++ chain/actors/builtin/reward/reward.go | 3 + chain/actors/builtin/reward/v0.go | 5 ++ node/impl/full/state.go | 111 +++++++++++--------------- 8 files changed, 81 insertions(+), 63 deletions(-) diff --git a/chain/actors/builtin/builtin.go b/chain/actors/builtin/builtin.go index accc4e7e6..517f0d70c 100644 --- a/chain/actors/builtin/builtin.go +++ b/chain/actors/builtin/builtin.go @@ -4,6 +4,8 @@ import ( "fmt" "github.com/filecoin-project/go-state-types/network" + + v0smoothing "github.com/filecoin-project/specs-actors/actors/util/smoothing" ) type Version int @@ -21,3 +23,6 @@ func VersionForNetwork(version network.Version) Version { panic(fmt.Sprintf("unsupported network version %d", version)) } } + +// TODO: find some way to abstract over this. +type FilterEstimate = v0smoothing.FilterEstimate diff --git a/chain/actors/builtin/market/market.go b/chain/actors/builtin/market/market.go index 3d12ac9a8..99cca9879 100644 --- a/chain/actors/builtin/market/market.go +++ b/chain/actors/builtin/market/market.go @@ -12,6 +12,8 @@ import ( "github.com/filecoin-project/lotus/chain/types" ) +var Address = v0builtin.StorageMarketActorAddr + func Load(store adt.Store, act *types.Actor) (st State, err error) { switch act.Code { case v0builtin.StorageMarketActorCodeID: @@ -30,6 +32,9 @@ type State interface { EscrowTable() (BalanceTable, error) LockedTable() (BalanceTable, error) TotalLocked() (abi.TokenAmount, error) + VerifyDealsForActivation( + minerAddr address.Address, deals []abi.DealID, currEpoch, sectorExpiry abi.ChainEpoch, + ) (weight, verifiedWeight abi.DealWeight, err error) } type BalanceTable interface { diff --git a/chain/actors/builtin/market/v0.go b/chain/actors/builtin/market/v0.go index da86cda0f..fb67902da 100644 --- a/chain/actors/builtin/market/v0.go +++ b/chain/actors/builtin/market/v0.go @@ -1,6 +1,7 @@ package market import ( + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/specs-actors/actors/builtin/market" @@ -25,3 +26,9 @@ func (s *v0State) EscrowTable() (BalanceTable, error) { func (s *v0State) LockedTable() (BalanceTable, error) { return adt.AsBalanceTable(s.store, s.State.LockedTable) } + +func (s *v0State) VerifyDealsForActivation( + minerAddr address.Address, deals []abi.DealID, currEpoch, sectorExpiry abi.ChainEpoch, +) (weight, verifiedWeight abi.DealWeight, err error) { + return market.ValidateDealsForActivation(&s.State, s.store, deals, minerAddr, sectorExpiry, currEpoch) +} diff --git a/chain/actors/builtin/power/power.go b/chain/actors/builtin/power/power.go index 9e35e52ab..07399e1bf 100644 --- a/chain/actors/builtin/power/power.go +++ b/chain/actors/builtin/power/power.go @@ -9,6 +9,7 @@ import ( v0builtin "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/types" ) @@ -32,6 +33,8 @@ type State interface { TotalLocked() (abi.TokenAmount, error) TotalPower() (Claim, error) + TotalPowerSmoothed() (builtin.FilterEstimate, error) + MinerPower(address.Address) (Claim, bool, error) MinerNominalPowerMeetsConsensusMinimum(address.Address) (bool, error) } diff --git a/chain/actors/builtin/power/v0.go b/chain/actors/builtin/power/v0.go index 12eb318e5..45dd570f6 100644 --- a/chain/actors/builtin/power/v0.go +++ b/chain/actors/builtin/power/v0.go @@ -3,6 +3,7 @@ package power import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/specs-actors/actors/builtin/power" "github.com/filecoin-project/specs-actors/actors/util/adt" ) @@ -42,3 +43,7 @@ func (s *v0State) MinerPower(addr address.Address) (Claim, bool, error) { func (s *v0State) MinerNominalPowerMeetsConsensusMinimum(a address.Address) (bool, error) { return s.State.MinerNominalPowerMeetsConsensusMinimum(s.store, a) } + +func (s *v0State) TotalPowerSmoothed() (builtin.FilterEstimate, error) { + return *s.State.ThisEpochQAPowerSmoothed, nil +} diff --git a/chain/actors/builtin/reward/reward.go b/chain/actors/builtin/reward/reward.go index a1a7cac5d..ba03feced 100644 --- a/chain/actors/builtin/reward/reward.go +++ b/chain/actors/builtin/reward/reward.go @@ -7,6 +7,7 @@ import ( v0builtin "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/types" ) @@ -27,4 +28,6 @@ func Load(store adt.Store, act *types.Actor) (st State, err error) { type State interface { cbor.Marshaler + + RewardSmoothed() (builtin.FilterEstimate, error) } diff --git a/chain/actors/builtin/reward/v0.go b/chain/actors/builtin/reward/v0.go index 8fef3756c..a894fa752 100644 --- a/chain/actors/builtin/reward/v0.go +++ b/chain/actors/builtin/reward/v0.go @@ -1,6 +1,7 @@ package reward import ( + "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/specs-actors/actors/builtin/reward" "github.com/filecoin-project/specs-actors/actors/util/adt" ) @@ -9,3 +10,7 @@ type v0State struct { reward.State store adt.Store } + +func (s *v0State) RewardSmoothed() (builtin.FilterEstimate, error) { + return *s.State.ThisEpochRewardSmoothed, nil +} diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 10b5de82b..3ea40388f 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -21,16 +21,17 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/builtin/market" - "github.com/filecoin-project/specs-actors/actors/builtin/power" - "github.com/filecoin-project/specs-actors/actors/builtin/reward" + v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" "github.com/filecoin-project/specs-actors/actors/util/adt" + "github.com/filecoin-project/specs-actors/actors/util/smoothing" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/actors/builtin/multisig" + "github.com/filecoin-project/lotus/chain/actors/builtin/power" + "github.com/filecoin-project/lotus/chain/actors/builtin/reward" "github.com/filecoin-project/lotus/chain/beacon" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/state" @@ -41,6 +42,7 @@ import ( "github.com/filecoin-project/lotus/chain/wallet" "github.com/filecoin-project/lotus/lib/bufbstore" "github.com/filecoin-project/lotus/node/modules/dtypes" + "github.com/filecoin-project/lotus/chain/actors/builtin/market" ) var errBreakForeach = errors.New("break") @@ -854,74 +856,57 @@ func (a *StateAPI) StateMinerPreCommitDepositForPower(ctx context.Context, maddr return types.EmptyInt, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - var minerState miner.State - var powerState power.State - var rewardState reward.State - - err = a.StateManager.WithParentStateTsk(tsk, func(state *state.StateTree) error { - if err := a.StateManager.WithActor(maddr, a.StateManager.WithActorState(ctx, &minerState))(state); err != nil { - return xerrors.Errorf("getting miner state: %w", err) - } - - if err := a.StateManager.WithActor(builtin.StoragePowerActorAddr, a.StateManager.WithActorState(ctx, &powerState))(state); err != nil { - return xerrors.Errorf("getting power state: %w", err) - } - - if err := a.StateManager.WithActor(builtin.RewardActorAddr, a.StateManager.WithActorState(ctx, &rewardState))(state); err != nil { - return xerrors.Errorf("getting reward state: %w", err) - } - - return nil - }) + state, err := a.StateManager.ParentState(ts) if err != nil { - return types.EmptyInt, err + return types.EmptyInt, xerrors.Errorf("loading state %s: %w", tsk, err) } - dealWeights := market.VerifyDealsForActivationReturn{ - DealWeight: big.Zero(), - VerifiedDealWeight: big.Zero(), - } - - if len(pci.DealIDs) != 0 { - var err error - params, err := actors.SerializeParams(&market.VerifyDealsForActivationParams{ - DealIDs: pci.DealIDs, - SectorExpiry: pci.Expiration, - }) - if err != nil { - return types.EmptyInt, err - } - - ret, err := a.StateManager.Call(ctx, &types.Message{ - From: maddr, - To: builtin.StorageMarketActorAddr, - Method: builtin.MethodsMarket.VerifyDealsForActivation, - Params: params, - }, ts) - if err != nil { - return types.EmptyInt, err - } - - if err := dealWeights.UnmarshalCBOR(bytes.NewReader(ret.MsgRct.Return)); err != nil { - return types.BigInt{}, err - } - } - - mi, err := a.StateMinerInfo(ctx, maddr, tsk) + ssize, err := pci.SealProof.SectorSize() if err != nil { - return types.EmptyInt, err + return types.EmptyInt, xerrors.Errorf("failed to get resolve size: %w", err) } - ssize := mi.SectorSize + store := a.Chain.Store(ctx) - duration := pci.Expiration - ts.Height() // NB: not exactly accurate, but should always lead us to *over* estimate, not under + var sectorWeight abi.StoragePower + if act, err := state.GetActor(market.Address); err != nil { + return types.EmptyInt, xerrors.Errorf("loading miner actor %s: %w", maddr, err) + } else s, err := market.Load(store, act); err != nil { + return types.EmptyInt, xerrors.Errorf("loading market actor state %s: %w", maddr, err) + } else if w, vw, err := s.VerifyDealsForActivation(maddr, pci.DealIDs, ts.Height(), pci.Expiration); err != nil { + return types.EmptyInt, xerrors.Errorf("verifying deals for activation: %w", err) + } else { + // NB: not exactly accurate, but should always lead us to *over* estimate, not under + duration := pci.Expiration - ts.Height() - sectorWeight := miner.QAPowerForWeight(ssize, duration, dealWeights.DealWeight, dealWeights.VerifiedDealWeight) - deposit := miner.PreCommitDepositForPower( - rewardState.ThisEpochRewardSmoothed, - powerState.ThisEpochQAPowerSmoothed, - sectorWeight, - ) + // TODO: handle changes to this function across actor upgrades. + sectorWeight = v0miner.QAPowerForWeight(ssize, duration, w, vw) + } + + var powerSmoothed smoothing.FilterEstimate + if act, err := state.GetActor(power.Address); err != nil { + return types.EmptyInt, xerrors.Errorf("loading miner actor: %w", err) + } else s, err := power.Load(store, act); err != nil { + return types.EmptyInt, xerrors.Errorf("loading power actor state: %w", err) + } else p, err := s.TotalPowerSmoothed(); err != nil { + return types.EmptyInt, xerrors.Errorf("failed to determine total power: %w", err) + } else { + powerSmoothed = p + } + + var rewardSmoothed smoothing.FilterEstimate + if act, err := state.GetActor(reward.Address); err != nil { + return types.EmptyInt, xerrors.Errorf("loading miner actor: %w", err) + } else s, err := reward.Load(store, act); err != nil { + return types.EmptyInt, xerrors.Errorf("loading reward actor state: %w", err) + } else r, err := s.RewardSmoothed(); err != nil { + return types.EmptyInt, xerrors.Errorf("failed to determine total reward: %w", err) + } else { + rewardSmoothed = r + } + + // TODO: abstract over network upgrades. + deposit := v0miner.PreCommitDepositForPower(rewardSmoothed, powerSmoothed, sectorWeight) return types.BigDiv(types.BigMul(deposit, initialPledgeNum), initialPledgeDen), nil } From b20f558abba6790322ec6d206bd20ff6ded0e548 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Tue, 15 Sep 2020 19:51:18 -0400 Subject: [PATCH 134/885] Multisig: Add a CLI command to approve locking some balance --- cli/multisig.go | 100 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/cli/multisig.go b/cli/multisig.go index 01e922a07..9ff1ad18e 100644 --- a/cli/multisig.go +++ b/cli/multisig.go @@ -1065,6 +1065,106 @@ var msigLockProposeCmd = &cli.Command{ }, } +var msigLockApproveCmd = &cli.Command{ + Name: "lock-approve", + Usage: "Approve a message to lock up some balance", + ArgsUsage: "[multisigAddress proposerAddress txId startEpoch unlockDuration amount]", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "from", + Usage: "account to send the approve message from", + }, + }, + Action: func(cctx *cli.Context) error { + if cctx.Args().Len() != 6 { + return ShowHelp(cctx, fmt.Errorf("must pass multisig address, proposer address, tx id, start epoch, unlock duration, and amount")) + } + + api, closer, err := GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + ctx := ReqContext(cctx) + + msig, err := address.NewFromString(cctx.Args().Get(0)) + if err != nil { + return err + } + + prop, err := address.NewFromString(cctx.Args().Get(1)) + if err != nil { + return err + } + + txid, err := strconv.ParseUint(cctx.Args().Get(2), 10, 64) + if err != nil { + return err + } + + start, err := strconv.ParseUint(cctx.Args().Get(3), 10, 64) + if err != nil { + return err + } + + duration, err := strconv.ParseUint(cctx.Args().Get(4), 10, 64) + if err != nil { + return err + } + + amount, err := types.ParseFIL(cctx.Args().Get(5)) + if err != nil { + return err + } + + var from address.Address + if cctx.IsSet("from") { + f, err := address.NewFromString(cctx.String("from")) + if err != nil { + return err + } + from = f + } else { + defaddr, err := api.WalletDefaultAddress(ctx) + if err != nil { + return err + } + from = defaddr + } + + params, actErr := actors.SerializeParams(&samsig.LockBalanceParams{ + StartEpoch: abi.ChainEpoch(start), + UnlockDuration: abi.ChainEpoch(duration), + Amount: abi.NewTokenAmount(amount.Int64()), + }) + + if actErr != nil { + return actErr + } + + // It takes the following params: , , , , , + // , , + + msgCid, err := api.MsigApprove(ctx, msig, txid, prop, msig, big.Zero(), from, uint64(builtin.MethodsMultisig.LockBalance), params) + if err != nil { + return err + } + + fmt.Println("sent lock approval in message: ", msgCid) + + wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) + if err != nil { + return err + } + + if wait.Receipt.ExitCode != 0 { + return fmt.Errorf("lock approval returned exit %d", wait.Receipt.ExitCode) + } + + return nil + }, +} + var msigVestedCmd = &cli.Command{ Name: "vested", Usage: "Gets the amount vested in an msig between two epochs", From 2ca1d111c53b7dbab5da50adc0b6176b3dfed9f2 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Tue, 15 Sep 2020 19:52:58 -0400 Subject: [PATCH 135/885] Multisig: Add a CLI command to cancel locking some balance --- cli/multisig.go | 95 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 92 insertions(+), 3 deletions(-) diff --git a/cli/multisig.go b/cli/multisig.go index 9ff1ad18e..1f626ae7f 100644 --- a/cli/multisig.go +++ b/cli/multisig.go @@ -1142,9 +1142,6 @@ var msigLockApproveCmd = &cli.Command{ return actErr } - // It takes the following params: , , , , , - // , , - msgCid, err := api.MsigApprove(ctx, msig, txid, prop, msig, big.Zero(), from, uint64(builtin.MethodsMultisig.LockBalance), params) if err != nil { return err @@ -1165,6 +1162,98 @@ var msigLockApproveCmd = &cli.Command{ }, } +var msigLockCancelCmd = &cli.Command{ + Name: "lock-cancel", + Usage: "Cancel a message to lock up some balance", + ArgsUsage: "[multisigAddress txId startEpoch unlockDuration amount]", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "from", + Usage: "account to send the approve message from", + }, + }, + Action: func(cctx *cli.Context) error { + if cctx.Args().Len() != 6 { + return ShowHelp(cctx, fmt.Errorf("must pass multisig address, tx id, start epoch, unlock duration, and amount")) + } + + api, closer, err := GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + ctx := ReqContext(cctx) + + msig, err := address.NewFromString(cctx.Args().Get(0)) + if err != nil { + return err + } + + txid, err := strconv.ParseUint(cctx.Args().Get(1), 10, 64) + if err != nil { + return err + } + + start, err := strconv.ParseUint(cctx.Args().Get(2), 10, 64) + if err != nil { + return err + } + + duration, err := strconv.ParseUint(cctx.Args().Get(3), 10, 64) + if err != nil { + return err + } + + amount, err := types.ParseFIL(cctx.Args().Get(4)) + if err != nil { + return err + } + + var from address.Address + if cctx.IsSet("from") { + f, err := address.NewFromString(cctx.String("from")) + if err != nil { + return err + } + from = f + } else { + defaddr, err := api.WalletDefaultAddress(ctx) + if err != nil { + return err + } + from = defaddr + } + + params, actErr := actors.SerializeParams(&samsig.LockBalanceParams{ + StartEpoch: abi.ChainEpoch(start), + UnlockDuration: abi.ChainEpoch(duration), + Amount: abi.NewTokenAmount(amount.Int64()), + }) + + if actErr != nil { + return actErr + } + + msgCid, err := api.MsigCancel(ctx, msig, txid, msig, big.Zero(), from, uint64(builtin.MethodsMultisig.LockBalance), params) + if err != nil { + return err + } + + fmt.Println("sent lock cancellation in message: ", msgCid) + + wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) + if err != nil { + return err + } + + if wait.Receipt.ExitCode != 0 { + return fmt.Errorf("lock cancellation returned exit %d", wait.Receipt.ExitCode) + } + + return nil + }, +} + var msigVestedCmd = &cli.Command{ Name: "vested", Usage: "Gets the amount vested in an msig between two epochs", From 952f2c2f1e2659ebe4fb4f5940832d02e5da6c4e Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Wed, 9 Sep 2020 05:13:51 -0400 Subject: [PATCH 136/885] Sever chainstore's dependence on the state tree --- chain/store/store.go | 25 ++++--------------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/chain/store/store.go b/chain/store/store.go index 20a7e3031..ad30e0324 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -18,7 +18,6 @@ import ( "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/vm" "github.com/filecoin-project/lotus/journal" bstore "github.com/filecoin-project/lotus/lib/blockstore" @@ -767,32 +766,16 @@ type BlockMessages struct { func (cs *ChainStore) BlockMsgsForTipset(ts *types.TipSet) ([]BlockMessages, error) { applied := make(map[address.Address]uint64) - cst := cbor.NewCborStore(cs.bs) - st, err := state.LoadStateTree(cst, ts.Blocks()[0].ParentStateRoot) - if err != nil { - return nil, xerrors.Errorf("failed to load state tree") - } - - preloadAddr := func(a address.Address) error { - if _, ok := applied[a]; !ok { - act, err := st.GetActor(a) - if err != nil { - return err - } - - applied[a] = act.Nonce - } - return nil - } - selectMsg := func(m *types.Message) (bool, error) { - if err := preloadAddr(m.From); err != nil { - return false, err + // The first match for a sender is guaranteed to have correct nonce -- the block isn't valid otherwise + if _, ok := applied[m.From]; !ok { + applied[m.From] = m.Nonce } if applied[m.From] != m.Nonce { return false, nil } + applied[m.From]++ return true, nil From 937ff4e9aef0273c7ec4b9edadd3d3dea4610f82 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Tue, 15 Sep 2020 20:55:11 -0400 Subject: [PATCH 137/885] Add a sync test around blocks with bad nonce messages --- chain/sync_test.go | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/chain/sync_test.go b/chain/sync_test.go index 0b0d1ed00..f4c83341f 100644 --- a/chain/sync_test.go +++ b/chain/sync_test.go @@ -662,6 +662,49 @@ func TestDuplicateNonce(t *testing.T) { require.Equal(t, includedMsg, mft[0].VMMessage().Cid(), "messages for tipset didn't contain expected message") } +// This test asserts that a block that includes a message with bad nonce can't be synced. A nonce is "bad" if it can't +// be applied on the parent state. +func TestBadNonce(t *testing.T) { + H := 10 + tu := prepSyncTest(t, H) + + base := tu.g.CurTipset + + // Produce a message from the banker with a bad nonce + makeBadMsg := func() *types.SignedMessage { + + ba, err := tu.nds[0].StateGetActor(context.TODO(), tu.g.Banker(), base.TipSet().Key()) + require.NoError(t, err) + msg := types.Message{ + To: tu.g.Banker(), + From: tu.g.Banker(), + + Nonce: ba.Nonce + 5, + + Value: types.NewInt(1), + + Method: 0, + + GasLimit: 100_000_000, + GasFeeCap: types.NewInt(0), + GasPremium: types.NewInt(0), + } + + sig, err := tu.g.Wallet().Sign(context.TODO(), tu.g.Banker(), msg.Cid().Bytes()) + require.NoError(t, err) + + return &types.SignedMessage{ + Message: msg, + Signature: *sig, + } + } + + msgs := make([][]*types.SignedMessage, 1) + msgs[0] = []*types.SignedMessage{makeBadMsg()} + + tu.mineOnBlock(base, 0, []int{0}, true, true, msgs) +} + func BenchmarkSyncBasic(b *testing.B) { for i := 0; i < b.N; i++ { runSyncBenchLength(b, 100) From 454c382e7ec024d2de7c7a91f83e6e0d71d78b8b Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 15 Sep 2020 18:05:33 -0700 Subject: [PATCH 138/885] migrate StateMinerInitialPledgeCollateral --- chain/stmgr/utils.go | 2 +- node/impl/full/state.go | 119 ++++++++++++++++++++-------------------- 2 files changed, 61 insertions(+), 60 deletions(-) diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index 88275f019..1bbc9ccc4 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -30,10 +30,10 @@ import ( "github.com/filecoin-project/specs-actors/actors/builtin/paych" "github.com/filecoin-project/specs-actors/actors/builtin/reward" "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" - "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors/adt" init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init" "github.com/filecoin-project/lotus/chain/actors/builtin/market" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 3ea40388f..d69ba0a41 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -912,85 +912,86 @@ func (a *StateAPI) StateMinerPreCommitDepositForPower(ctx context.Context, maddr } func (a *StateAPI) StateMinerInitialPledgeCollateral(ctx context.Context, maddr address.Address, pci miner.SectorPreCommitInfo, tsk types.TipSetKey) (types.BigInt, error) { + // TODO: this repeats a lot of the previous function. Fix that. ts, err := a.Chain.GetTipSetFromKey(tsk) if err != nil { return types.EmptyInt, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - var minerState miner.State - var powerState power.State - var rewardState reward.State - - err = a.StateManager.WithParentStateTsk(tsk, func(state *state.StateTree) error { - if err := a.StateManager.WithActor(maddr, a.StateManager.WithActorState(ctx, &minerState))(state); err != nil { - return xerrors.Errorf("getting miner state: %w", err) - } - - if err := a.StateManager.WithActor(builtin.StoragePowerActorAddr, a.StateManager.WithActorState(ctx, &powerState))(state); err != nil { - return xerrors.Errorf("getting power state: %w", err) - } - - if err := a.StateManager.WithActor(builtin.RewardActorAddr, a.StateManager.WithActorState(ctx, &rewardState))(state); err != nil { - return xerrors.Errorf("getting reward state: %w", err) - } - - return nil - }) + state, err := a.StateManager.ParentState(ts) if err != nil { - return types.EmptyInt, err + return types.EmptyInt, xerrors.Errorf("loading state %s: %w", tsk, err) } - dealWeights := market.VerifyDealsForActivationReturn{ - DealWeight: big.Zero(), - VerifiedDealWeight: big.Zero(), - } - - if len(pci.DealIDs) != 0 { - var err error - params, err := actors.SerializeParams(&market.VerifyDealsForActivationParams{ - DealIDs: pci.DealIDs, - SectorExpiry: pci.Expiration, - }) - if err != nil { - return types.EmptyInt, err - } - - ret, err := a.StateManager.Call(ctx, &types.Message{ - From: maddr, - To: builtin.StorageMarketActorAddr, - Method: builtin.MethodsMarket.VerifyDealsForActivation, - Params: params, - }, ts) - if err != nil { - return types.EmptyInt, err - } - - if err := dealWeights.UnmarshalCBOR(bytes.NewReader(ret.MsgRct.Return)); err != nil { - return types.BigInt{}, err - } - } - - mi, err := a.StateMinerInfo(ctx, maddr, tsk) + ssize, err := pci.SealProof.SectorSize() if err != nil { - return types.EmptyInt, err + return types.EmptyInt, xerrors.Errorf("failed to get resolve size: %w", err) } - ssize := mi.SectorSize + store := a.Chain.Store(ctx) - duration := pci.Expiration - ts.Height() // NB: not exactly accurate, but should always lead us to *over* estimate, not under + var sectorWeight abi.StoragePower + if act, err := state.GetActor(market.Address); err != nil { + return types.EmptyInt, xerrors.Errorf("loading miner actor %s: %w", maddr, err) + } else s, err := market.Load(store, act); err != nil { + return types.EmptyInt, xerrors.Errorf("loading market actor state %s: %w", maddr, err) + } else if w, vw, err := s.VerifyDealsForActivation(maddr, pci.DealIDs, ts.Height(), pci.Expiration); err != nil { + return types.EmptyInt, xerrors.Errorf("verifying deals for activation: %w", err) + } else { + // NB: not exactly accurate, but should always lead us to *over* estimate, not under + duration := pci.Expiration - ts.Height() + + // TODO: handle changes to this function across actor upgrades. + sectorWeight = v0miner.QAPowerForWeight(ssize, duration, w, vw) + } + + var ( + powerSmoothed smoothing.FilterEstimate + pledgeCollerateral abi.TokenAmount + ) + if act, err := state.GetActor(power.Address); err != nil { + return types.EmptyInt, xerrors.Errorf("loading miner actor: %w", err) + } else s, err := power.Load(store, act); err != nil { + return types.EmptyInt, xerrors.Errorf("loading power actor state: %w", err) + } else p, err := s.TotalPowerSmoothed(); err != nil { + return types.EmptyInt, xerrors.Errorf("failed to determine total power: %w", err) + } else c, err := s.TotalLocked(); err != nil { + return types.EmptyInt, xerrors.Errorf("failed to determine pledge collateral: %w", err) + } else { + powerSmoothed = p + pledgeCollateral = c + } + + var ( + rewardSmoothed smoothing.FilterEstimate + baselinePower abi.StoragePower + ) + if act, err := state.GetActor(reward.Address); err != nil { + return types.EmptyInt, xerrors.Errorf("loading miner actor: %w", err) + } else s, err := reward.Load(store, act); err != nil { + return types.EmptyInt, xerrors.Errorf("loading reward actor state: %w", err) + } else r, err := s.RewardSmoothed(); err != nil { + return types.EmptyInt, xerrors.Errorf("failed to determine total reward: %w", err) + } else p, err := s.BaselinePower(); err != nil { + return types.EmptyInt, xerrors.Errorf("failed to determine baseline power: %w", err) + } else { + rewardSmoothed = r + baselinePower = p + } + + // TODO: abstract over network upgrades. circSupply, err := a.StateCirculatingSupply(ctx, ts.Key()) if err != nil { return big.Zero(), xerrors.Errorf("getting circulating supply: %w", err) } - sectorWeight := miner.QAPowerForWeight(ssize, duration, dealWeights.DealWeight, dealWeights.VerifiedDealWeight) initialPledge := miner.InitialPledgeForPower( sectorWeight, - rewardState.ThisEpochBaselinePower, - powerState.ThisEpochPledgeCollateral, - rewardState.ThisEpochRewardSmoothed, - powerState.ThisEpochQAPowerSmoothed, + baselinePower, + pledgeCollateral, + rewardSmoothed, + powerSmoothed, circSupply.FilCirculating, ) From fa9be9a627385da6a6c4a36d94f9b0b9bd811a43 Mon Sep 17 00:00:00 2001 From: zgfzgf <1901989065@qq.com> Date: Wed, 16 Sep 2020 09:14:56 +0800 Subject: [PATCH 139/885] Replace fx.Extract with fx.Populate --- node/builder.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/node/builder.go b/node/builder.go index 8c7674ffd..c37a5db58 100644 --- a/node/builder.go +++ b/node/builder.go @@ -382,7 +382,7 @@ func StorageMiner(out *api.StorageMiner) Option { func(s *Settings) error { resAPI := &impl.StorageMinerAPI{} - s.invokes[ExtractApiKey] = fx.Extract(resAPI) + s.invokes[ExtractApiKey] = fx.Populate(resAPI) *out = resAPI return nil }, @@ -509,7 +509,7 @@ func FullAPI(out *api.FullNode) Option { }, func(s *Settings) error { resAPI := &impl.FullNodeAPI{} - s.invokes[ExtractApiKey] = fx.Extract(resAPI) + s.invokes[ExtractApiKey] = fx.Populate(resAPI) *out = resAPI return nil }, From 5c69249ba36caf130c1fa5aa1c6937c7d86e34ae Mon Sep 17 00:00:00 2001 From: jennijuju Date: Tue, 15 Sep 2020 21:19:27 -0400 Subject: [PATCH 140/885] Use `window post` for window PoSt related log messages. --- chain/stmgr/utils.go | 2 +- chain/sync.go | 2 +- cmd/lotus-bench/main.go | 4 ++-- storage/addresses.go | 2 +- storage/wdpost_run.go | 20 ++++++++++---------- storage/wdpost_sched.go | 14 +++++++------- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index 27b71f358..c09f2b22a 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -554,7 +554,7 @@ func MinerGetBaseInfo(ctx context.Context, sm *StateManager, bcs beacon.Schedule sectors, err := GetSectorsForWinningPoSt(ctx, pv, sm, lbst, maddr, prand) if err != nil { - return nil, xerrors.Errorf("getting wpost proving set: %w", err) + return nil, xerrors.Errorf("getting winning post proving set: %w", err) } if len(sectors) == 0 { diff --git a/chain/sync.go b/chain/sync.go index f7530f556..b5716a343 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -991,7 +991,7 @@ func (syncer *Syncer) VerifyWinningPoStProof(ctx context.Context, h *types.Block rand, err := store.DrawRandomness(rbase.Data, crypto.DomainSeparationTag_WinningPoStChallengeSeed, h.Height, buf.Bytes()) if err != nil { - return xerrors.Errorf("failed to get randomness for verifying winningPost proof: %w", err) + return xerrors.Errorf("failed to get randomness for verifying winning post proof: %w", err) } mid, err := address.IDFromAddress(h.Miner) diff --git a/cmd/lotus-bench/main.go b/cmd/lotus-bench/main.go index 431bfdd44..7da754415 100644 --- a/cmd/lotus-bench/main.go +++ b/cmd/lotus-bench/main.go @@ -387,7 +387,7 @@ var sealBenchCmd = &cli.Command{ return err } if !ok { - log.Error("post verification failed") + log.Error("window post verification failed") } verifyWindowpost1 := time.Now() @@ -403,7 +403,7 @@ var sealBenchCmd = &cli.Command{ return err } if !ok { - log.Error("post verification failed") + log.Error("window post verification failed") } verifyWindowpost2 := time.Now() diff --git a/storage/addresses.go b/storage/addresses.go index bef845367..2388b9fca 100644 --- a/storage/addresses.go +++ b/storage/addresses.go @@ -60,7 +60,7 @@ func AddressFor(ctx context.Context, a addrSelectApi, mi api.MinerInfo, use Addr return addr, nil } - log.Warnw("control address didn't have enough funds for PoSt message", "address", addr, "required", types.FIL(minFunds), "balance", types.FIL(b)) + log.Warnw("control address didn't have enough funds for window post message", "address", addr, "required", types.FIL(minFunds), "balance", types.FIL(b)) } // Try to use the owner account if we can, fallback to worker if we can't diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 7aa90bbba..17b567440 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -78,7 +78,7 @@ func (s *WindowPoStScheduler) doPost(ctx context.Context, deadline *dline.Info, posts, err := s.runPost(ctx, *deadline, ts) if err != nil { - log.Errorf("runPost failed: %+v", err) + log.Errorf("run window post failed: %+v", err) s.failPost(err, deadline) return } @@ -92,7 +92,7 @@ func (s *WindowPoStScheduler) doPost(ctx context.Context, deadline *dline.Info, post := &posts[i] sm, err := s.submitPost(ctx, post) if err != nil { - log.Errorf("submitPost failed: %+v", err) + log.Errorf("submit window post failed: %+v", err) s.failPost(err, deadline) } else { recordProofsEvent(post.Partitions, sm.Cid()) @@ -397,7 +397,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty rand, err := s.api.ChainGetRandomnessFromBeacon(ctx, ts.Key(), crypto.DomainSeparationTag_WindowedPoStChallengeSeed, di.Challenge, buf.Bytes()) if err != nil { - return nil, xerrors.Errorf("failed to get chain randomness for windowPost (ts=%d; deadline=%d): %w", ts.Height(), di, err) + return nil, xerrors.Errorf("failed to get chain randomness for window post (ts=%d; deadline=%d): %w", ts.Height(), di, err) } // Get the partitions for the given deadline @@ -490,7 +490,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty } // Generate proof - log.Infow("running windowPost", + log.Infow("running window post", "chain-random", rand, "deadline", di, "height", ts.Height(), @@ -507,7 +507,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty postOut, ps, err = s.prover.GenerateWindowPoSt(ctx, abi.ActorID(mid), sinfos, abi.PoStRandomness(rand)) elapsed := time.Since(tsStart) - log.Infow("computing window PoSt", "batch", batchIdx, "elapsed", elapsed) + log.Infow("computing window post", "batch", batchIdx, "elapsed", elapsed) if err == nil { // Proof generation successful, stop retrying @@ -517,10 +517,10 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty // Proof generation failed, so retry if len(ps) == 0 { - return nil, xerrors.Errorf("running post failed: %w", err) + return nil, xerrors.Errorf("running window post failed: %w", err) } - log.Warnw("generate window PoSt skipped sectors", "sectors", ps, "error", err, "try", retries) + log.Warnw("generate window post skipped sectors", "sectors", ps, "error", err, "try", retries) skipCount += uint64(len(ps)) for _, sector := range ps { @@ -547,7 +547,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty commEpoch := di.Open commRand, err := s.api.ChainGetRandomnessFromTickets(ctx, ts.Key(), crypto.DomainSeparationTag_PoStChainCommit, commEpoch, nil) if err != nil { - return nil, xerrors.Errorf("failed to get chain randomness for windowPost (ts=%d; deadline=%d): %w", ts.Height(), commEpoch, err) + return nil, xerrors.Errorf("failed to get chain randomness for window post (ts=%d; deadline=%d): %w", ts.Height(), commEpoch, err) } for i := range posts { @@ -644,7 +644,7 @@ func (s *WindowPoStScheduler) submitPost(ctx context.Context, proof *miner.Submi enc, aerr := actors.SerializeParams(proof) if aerr != nil { - return nil, xerrors.Errorf("could not serialize submit post parameters: %w", aerr) + return nil, xerrors.Errorf("could not serialize submit window post parameters: %w", aerr) } msg := &types.Message{ @@ -705,7 +705,7 @@ func (s *WindowPoStScheduler) setSender(ctx context.Context, msg *types.Message, pa, err := AddressFor(ctx, s.api, mi, PoStAddr, minFunds) if err != nil { - log.Errorw("error selecting address for post", "error", err) + log.Errorw("error selecting address for window post", "error", err) msg.From = s.worker return } diff --git a/storage/wdpost_sched.go b/storage/wdpost_sched.go index 037f4e481..d130d801c 100644 --- a/storage/wdpost_sched.go +++ b/storage/wdpost_sched.go @@ -110,7 +110,7 @@ func (s *WindowPoStScheduler) Run(ctx context.Context) { select { case changes, ok := <-notifs: if !ok { - log.Warn("WindowPoStScheduler notifs channel closed") + log.Warn("window post scheduler notifs channel closed") notifs = nil continue } @@ -151,10 +151,10 @@ func (s *WindowPoStScheduler) Run(ctx context.Context) { } if err := s.revert(ctx, lowest); err != nil { - log.Error("handling head reverts in windowPost sched: %+v", err) + log.Error("handling head reverts in window post sched: %+v", err) } if err := s.update(ctx, highest); err != nil { - log.Error("handling head updates in windowPost sched: %+v", err) + log.Error("handling head updates in window post sched: %+v", err) } span.End() @@ -184,7 +184,7 @@ func (s *WindowPoStScheduler) revert(ctx context.Context, newLowest *types.TipSe func (s *WindowPoStScheduler) update(ctx context.Context, new *types.TipSet) error { if new == nil { - return xerrors.Errorf("no new tipset in WindowPoStScheduler.update") + return xerrors.Errorf("no new tipset in window post sched update") } di, err := s.api.StateMinerProvingDeadline(ctx, s.actor, new.Key()) @@ -206,7 +206,7 @@ func (s *WindowPoStScheduler) update(ctx context.Context, new *types.TipSet) err // (Need to get correct deadline above, which is tricky) if di.Open+StartConfidence >= new.Height() { - log.Info("not starting windowPost yet, waiting for startconfidence", di.Open, di.Open+StartConfidence, new.Height()) + log.Info("not starting window post yet, waiting for startconfidence", di.Open, di.Open+StartConfidence, new.Height()) return nil } @@ -216,7 +216,7 @@ func (s *WindowPoStScheduler) update(ctx context.Context, new *types.TipSet) err s.activeEPS = 0 } s.failLk.Unlock()*/ - log.Infof("at %d, doPost for P %d, dd %d", new.Height(), di.PeriodStart, di.Index) + log.Infof("at %d, do window post for P %d, dd %d", new.Height(), di.PeriodStart, di.Index) s.doPost(ctx, di, new) @@ -238,7 +238,7 @@ func (s *WindowPoStScheduler) abortActivePoSt() { } }) - log.Warnf("Aborting Window PoSt (Deadline: %+v)", s.activeDeadline) + log.Warnf("Aborting window post (Deadline: %+v)", s.activeDeadline) } s.activeDeadline = nil From e2cf0c4d4c0659931960982205c349e75e65c9c4 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Tue, 15 Sep 2020 21:24:30 -0400 Subject: [PATCH 141/885] Make interactive deal CLI reject deals shorter than minimum duration --- cli/client.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cli/client.go b/cli/client.go index e68f98791..d9ce9251e 100644 --- a/cli/client.go +++ b/cli/client.go @@ -12,6 +12,8 @@ import ( "text/tabwriter" "time" + "github.com/filecoin-project/specs-actors/actors/builtin" + tm "github.com/buger/goterm" "github.com/docker/go-units" "github.com/fatih/color" @@ -527,6 +529,11 @@ func interactiveDeal(cctx *cli.Context) error { continue } + if days < int(build.MinDealDuration/builtin.EpochsInDay) { + printErr(xerrors.Errorf("minimum duration is %d days", int(build.MinDealDuration/builtin.EpochsInDay))) + continue + } + state = "miner" case "miner": fmt.Print("Miner Address (t0..): ") From 808051d34ed5cdde0cb12e94a4f188952eb7c137 Mon Sep 17 00:00:00 2001 From: Travis Person Date: Wed, 16 Sep 2020 02:06:40 +0000 Subject: [PATCH 142/885] break out of mining loop when stop is called during niceSleep --- miner/miner.go | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/miner/miner.go b/miner/miner.go index bcfdb46c1..555fe2e19 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -139,6 +139,7 @@ func (m *Miner) niceSleep(d time.Duration) bool { case <-build.Clock.After(d): return true case <-m.stop: + log.Infow("recieved interrupt while trying to sleep in mining cycle") return false } } @@ -169,7 +170,9 @@ func (m *Miner) mine(ctx context.Context) { prebase, err := m.GetBestMiningCandidate(ctx) if err != nil { log.Errorf("failed to get best mining candidate: %s", err) - m.niceSleep(time.Second * 5) + if !m.niceSleep(time.Second * 5) { + break + } continue } @@ -199,7 +202,9 @@ func (m *Miner) mine(ctx context.Context) { _, err = m.api.BeaconGetEntry(ctx, prebase.TipSet.Height()+prebase.NullRounds+1) if err != nil { log.Errorf("failed getting beacon entry: %s", err) - m.niceSleep(time.Second) + if !m.niceSleep(time.Second) { + break + } continue } @@ -208,7 +213,9 @@ func (m *Miner) mine(ctx context.Context) { if base.TipSet.Equals(lastBase.TipSet) && lastBase.NullRounds == base.NullRounds { log.Warnf("BestMiningCandidate from the previous round: %s (nulls:%d)", lastBase.TipSet.Cids(), lastBase.NullRounds) - m.niceSleep(time.Duration(build.BlockDelaySecs) * time.Second) + if !m.niceSleep(time.Duration(build.BlockDelaySecs) * time.Second) { + break + } continue } @@ -217,7 +224,9 @@ func (m *Miner) mine(ctx context.Context) { b, err := m.mineOne(ctx, base) if err != nil { log.Errorf("mining block failed: %+v", err) - m.niceSleep(time.Second) + if !m.niceSleep(time.Second) { + break + } onDone(false, 0, err) continue } From 6670d22fb5714f4dfc651323b75a03631b0ddf97 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 15 Sep 2020 20:20:48 -0700 Subject: [PATCH 143/885] add command to (slowly) prune lotus chain datastore --- chain/store/store.go | 46 +++--- cmd/lotus-shed/main.go | 1 + cmd/lotus-shed/pruning.go | 290 ++++++++++++++++++++++++++++++++++++++ go.mod | 1 + 4 files changed, 321 insertions(+), 17 deletions(-) create mode 100644 cmd/lotus-shed/pruning.go diff --git a/chain/store/store.go b/chain/store/store.go index 20a7e3031..404befac7 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -1183,13 +1183,6 @@ func recurseLinks(bs bstore.Blockstore, walked *cid.Set, root cid.Cid, in []cid. } func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, inclRecentRoots abi.ChainEpoch, skipOldMsgs bool, w io.Writer) error { - if ts == nil { - ts = cs.GetHeaviestTipSet() - } - - seen := cid.NewSet() - walked := cid.NewSet() - h := &car.CarHeader{ Roots: ts.Cids(), Version: 1, @@ -1199,6 +1192,28 @@ func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, inclRecentRo return xerrors.Errorf("failed to write car header: %s", err) } + return cs.WalkSnapshot(ctx, ts, inclRecentRoots, skipOldMsgs, func(c cid.Cid) error { + blk, err := cs.bs.Get(c) + if err != nil { + return xerrors.Errorf("writing object to car, bs.Get: %w", err) + } + + if err := carutil.LdWrite(w, c.Bytes(), blk.RawData()); err != nil { + return xerrors.Errorf("failed to write block to car output: %w", err) + } + + return nil + }) +} + +func (cs *ChainStore) WalkSnapshot(ctx context.Context, ts *types.TipSet, inclRecentRoots abi.ChainEpoch, skipOldMsgs bool, cb func(cid.Cid) error) error { + if ts == nil { + ts = cs.GetHeaviestTipSet() + } + + seen := cid.NewSet() + walked := cid.NewSet() + blocksToWalk := ts.Cids() walkChain := func(blk cid.Cid) error { @@ -1206,15 +1221,15 @@ func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, inclRecentRo return nil } + if err := cb(blk); err != nil { + return err + } + data, err := cs.bs.Get(blk) if err != nil { return xerrors.Errorf("getting block: %w", err) } - if err := carutil.LdWrite(w, blk.Bytes(), data.RawData()); err != nil { - return xerrors.Errorf("failed to write block to car output: %w", err) - } - var b types.BlockHeader if err := b.UnmarshalCBOR(bytes.NewBuffer(data.RawData())); err != nil { return xerrors.Errorf("unmarshaling block header (cid=%s): %w", blk, err) @@ -1254,14 +1269,11 @@ func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, inclRecentRo if c.Prefix().Codec != cid.DagCBOR { continue } - data, err := cs.bs.Get(c) - if err != nil { - return xerrors.Errorf("writing object to car (get %s): %w", c, err) + + if err := cb(c); err != nil { + return err } - if err := carutil.LdWrite(w, c.Bytes(), data.RawData()); err != nil { - return xerrors.Errorf("failed to write out car object: %w", err) - } } } diff --git a/cmd/lotus-shed/main.go b/cmd/lotus-shed/main.go index cff3059b6..a64f981a1 100644 --- a/cmd/lotus-shed/main.go +++ b/cmd/lotus-shed/main.go @@ -35,6 +35,7 @@ func main() { mathCmd, mpoolStatsCmd, exportChainCmd, + stateTreePruneCmd, } app := &cli.App{ diff --git a/cmd/lotus-shed/pruning.go b/cmd/lotus-shed/pruning.go new file mode 100644 index 000000000..6f0c20541 --- /dev/null +++ b/cmd/lotus-shed/pruning.go @@ -0,0 +1,290 @@ +package main + +import ( + "context" + "fmt" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/store" + "github.com/filecoin-project/lotus/chain/vm" + "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" + "github.com/filecoin-project/lotus/lib/blockstore" + "github.com/filecoin-project/lotus/node/repo" + "github.com/ipfs/bbloom" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/query" + dshelp "github.com/ipfs/go-ipfs-ds-help" + "github.com/urfave/cli/v2" + "golang.org/x/xerrors" +) + +type cidSet interface { + Add(cid.Cid) + Has(cid.Cid) bool + HasRaw([]byte) bool + Len() int +} + +type bloomSet struct { + bloom *bbloom.Bloom +} + +func newBloomSet(size int64) (*bloomSet, error) { + b, err := bbloom.New(float64(size), 3) + if err != nil { + return nil, err + } + + return &bloomSet{bloom: b}, nil +} + +func (bs *bloomSet) Add(c cid.Cid) { + bs.bloom.Add(c.Hash()) + +} + +func (bs *bloomSet) Has(c cid.Cid) bool { + return bs.bloom.Has(c.Hash()) +} + +func (bs *bloomSet) HasRaw(b []byte) bool { + return bs.bloom.Has(b) +} + +func (bs *bloomSet) Len() int { + return int(bs.bloom.ElementsAdded()) +} + +type mapSet struct { + m map[string]struct{} +} + +func newMapSet() *mapSet { + return &mapSet{m: make(map[string]struct{})} +} + +func (bs *mapSet) Add(c cid.Cid) { + bs.m[string(c.Hash())] = struct{}{} +} + +func (bs *mapSet) Has(c cid.Cid) bool { + _, ok := bs.m[string(c.Hash())] + return ok +} + +func (bs *mapSet) HasRaw(b []byte) bool { + _, ok := bs.m[string(b)] + return ok +} + +func (bs *mapSet) Len() int { + return len(bs.m) +} + +var stateTreePruneCmd = &cli.Command{ + Name: "state-prune", + Description: "Deletes old state root data from local chainstore", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "repo", + Value: "~/.lotus", + }, + &cli.Int64Flag{ + Name: "keep-from-lookback", + Usage: "keep stateroots at or newer than the current height minus this lookback", + Value: 1800, // 2 x finality + }, + &cli.IntFlag{ + Name: "delete-up-to", + Usage: "delete up to the given number of objects (used to run a faster 'partial' sync)", + }, + &cli.BoolFlag{ + Name: "use-bloom-set", + Usage: "use a bloom filter for the 'good' set instead of a map, reduces memory usage but may not clean up as much", + }, + &cli.BoolFlag{ + Name: "dry-run", + Usage: "only enumerate the good set, don't do any deletions", + }, + &cli.BoolFlag{ + Name: "only-ds-gc", + Usage: "Only run datastore GC", + }, + &cli.IntFlag{ + Name: "gc-count", + Usage: "number of times to run gc", + Value: 20, + }, + }, + Action: func(cctx *cli.Context) error { + ctx := context.TODO() + + fsrepo, err := repo.NewFS(cctx.String("repo")) + if err != nil { + return err + } + + lkrepo, err := fsrepo.Lock(repo.FullNode) + if err != nil { + return err + } + + defer lkrepo.Close() //nolint:errcheck + + ds, err := lkrepo.Datastore("/chain") + if err != nil { + return err + } + + defer ds.Close() + + mds, err := lkrepo.Datastore("/metadata") + if err != nil { + return err + } + defer mds.Close() + + if cctx.Bool("only-ds-gc") { + gcds, ok := ds.(datastore.GCDatastore) + if ok { + fmt.Println("running datastore gc....") + for i := 0; i < cctx.Int("gc-count"); i++ { + if err := gcds.CollectGarbage(); err != nil { + return xerrors.Errorf("datastore GC failed: %w", err) + } + } + fmt.Println("gc complete!") + return nil + } else { + return fmt.Errorf("datastore doesnt support gc") + } + } + + bs := blockstore.NewBlockstore(ds) + + cs := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier)) + if err := cs.Load(); err != nil { + return fmt.Errorf("loading chainstore: %w", err) + } + + var goodSet cidSet + if cctx.Bool("use-bloom-set") { + bset, err := newBloomSet(10000000) + if err != nil { + return err + } + goodSet = bset + } else { + goodSet = newMapSet() + } + + ts := cs.GetHeaviestTipSet() + + rrLb := abi.ChainEpoch(cctx.Int64("keep-from-lookback")) + + if err := cs.WalkSnapshot(ctx, ts, rrLb, true, func(c cid.Cid) error { + if goodSet.Len()%20 == 0 { + fmt.Printf("\renumerating keep set: %d ", goodSet.Len()) + } + goodSet.Add(c) + return nil + }); err != nil { + return fmt.Errorf("snapshot walk failed: %w", err) + } + + fmt.Println() + fmt.Printf("Succesfully marked keep set! (%d objects)\n", goodSet.Len()) + + if cctx.Bool("dry-run") { + return nil + } + + var b datastore.Batch + var batchCount int + markForRemoval := func(c cid.Cid) error { + if b == nil { + nb, err := ds.Batch() + if err != nil { + return fmt.Errorf("opening batch: %w", err) + } + + b = nb + } + batchCount++ + + if err := b.Delete(dshelp.MultihashToDsKey(c.Hash())); err != nil { + return err + } + + if batchCount > 100 { + if err := b.Commit(); err != nil { + return xerrors.Errorf("failed to commit batch deletes: %w", err) + } + b = nil + batchCount = 0 + } + return nil + } + + res, err := ds.Query(query.Query{KeysOnly: true}) + if err != nil { + return xerrors.Errorf("failed to query datastore: %w", err) + } + + dupTo := cctx.Int("delete-up-to") + + var deleteCount int + var goodHits int + for { + v, ok := res.NextSync() + if !ok { + break + } + + bk, err := dshelp.BinaryFromDsKey(datastore.RawKey(v.Key[len("/blocks"):])) + if err != nil { + return xerrors.Errorf("failed to parse key: %w", err) + } + + if goodSet.HasRaw(bk) { + goodHits++ + continue + } + + nc := cid.NewCidV1(cid.Raw, bk) + + deleteCount++ + if err := markForRemoval(nc); err != nil { + return fmt.Errorf("failed to remove cid %s: %w", nc, err) + } + + if deleteCount%20 == 0 { + fmt.Printf("\rdeleting %d objects (good hits: %d)... ", deleteCount, goodHits) + } + + if dupTo != 0 && deleteCount > dupTo { + break + } + } + + if b != nil { + if err := b.Commit(); err != nil { + return xerrors.Errorf("failed to commit final batch delete: %w", err) + } + } + + gcds, ok := ds.(datastore.GCDatastore) + if ok { + fmt.Println("running datastore gc....") + for i := 0; i < cctx.Int("gc-count"); i++ { + if err := gcds.CollectGarbage(); err != nil { + return xerrors.Errorf("datastore GC failed: %w", err) + } + } + fmt.Println("gc complete!") + } + + return nil + }, +} diff --git a/go.mod b/go.mod index 65c2addb8..b82f60dc8 100644 --- a/go.mod +++ b/go.mod @@ -49,6 +49,7 @@ require ( github.com/hashicorp/go-multierror v1.1.0 github.com/hashicorp/golang-lru v0.5.4 github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d + github.com/ipfs/bbloom v0.0.4 github.com/ipfs/go-bitswap v0.2.20 github.com/ipfs/go-block-format v0.0.2 github.com/ipfs/go-blockservice v0.1.4-0.20200624145336-a978cec6e834 From 05c11531b1d9cfb2edb3062e22602e9446474fcf Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Tue, 15 Sep 2020 21:06:04 -0700 Subject: [PATCH 144/885] feat(paych): convert paych actor build abstraction for paych actor and switch to using it in payment channel manager and state predicates --- chain/actors/builtin/paych/mock/mock.go | 89 +++++++++++ chain/actors/builtin/paych/paych.go | 56 +++++++ chain/actors/builtin/paych/v0.go | 89 +++++++++++ chain/events/state/predicates.go | 47 +++--- chain/events/state/predicates_test.go | 2 +- chain/stmgr/stmgr.go | 19 +++ cli/paych_test.go | 8 +- paychmgr/manager.go | 37 ++--- paychmgr/mock_test.go | 66 +++------ paychmgr/msglistener_test.go | 2 - paychmgr/paych.go | 97 ++++++------ paychmgr/paych_test.go | 189 +++++++----------------- paychmgr/paychget_test.go | 45 ++---- paychmgr/simple.go | 19 +-- paychmgr/state.go | 41 ++--- 15 files changed, 460 insertions(+), 346 deletions(-) create mode 100644 chain/actors/builtin/paych/mock/mock.go create mode 100644 chain/actors/builtin/paych/paych.go create mode 100644 chain/actors/builtin/paych/v0.go diff --git a/chain/actors/builtin/paych/mock/mock.go b/chain/actors/builtin/paych/mock/mock.go new file mode 100644 index 000000000..31f7fba93 --- /dev/null +++ b/chain/actors/builtin/paych/mock/mock.go @@ -0,0 +1,89 @@ +package mock + +import ( + "io" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/lotus/chain/actors/builtin/paych" +) + +type mockState struct { + from address.Address + to address.Address + settlingAt abi.ChainEpoch + toSend abi.TokenAmount + lanes map[uint64]paych.LaneState +} + +type mockLaneState struct { + redeemed big.Int + nonce uint64 +} + +// NewMockPayChState constructs a state for a payment channel with the set fixed values +// that satisfies the paych.State interface. +func NewMockPayChState(from address.Address, + to address.Address, + settlingAt abi.ChainEpoch, + toSend abi.TokenAmount, + lanes map[uint64]paych.LaneState, +) paych.State { + return &mockState{from, to, settlingAt, toSend, lanes} +} + +// NewMockLaneState constructs a state for a payment channel lane with the set fixed values +// that satisfies the paych.LaneState interface. Useful for populating lanes when +// calling NewMockPayChState +func NewMockLaneState(redeemed big.Int, nonce uint64) paych.LaneState { + return &mockLaneState{redeemed, nonce} +} + +func (ms *mockState) MarshalCBOR(io.Writer) error { + panic("not implemented") +} + +// Channel owner, who has funded the actor +func (ms *mockState) From() address.Address { + return ms.from +} + +// Recipient of payouts from channel +func (ms *mockState) To() address.Address { + return ms.to +} + +// Height at which the channel can be `Collected` +func (ms *mockState) SettlingAt() abi.ChainEpoch { + return ms.settlingAt +} + +// Amount successfully redeemed through the payment channel, paid out on `Collect()` +func (ms *mockState) ToSend() abi.TokenAmount { + return ms.toSend +} + +// Get total number of lanes +func (ms *mockState) LaneCount() (uint64, error) { + return uint64(len(ms.lanes)), nil +} + +// Iterate lane states +func (ms *mockState) ForEachLaneState(cb func(idx uint64, dl paych.LaneState) error) error { + var lastErr error + for lane, state := range ms.lanes { + if err := cb(lane, state); err != nil { + lastErr = err + } + } + return lastErr +} + +func (mls *mockLaneState) Redeemed() big.Int { + return mls.redeemed +} + +func (mls *mockLaneState) Nonce() uint64 { + return mls.nonce +} diff --git a/chain/actors/builtin/paych/paych.go b/chain/actors/builtin/paych/paych.go new file mode 100644 index 000000000..974d64fde --- /dev/null +++ b/chain/actors/builtin/paych/paych.go @@ -0,0 +1,56 @@ +package paych + +import ( + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + big "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/go-state-types/cbor" + v0builtin "github.com/filecoin-project/specs-actors/actors/builtin" + + "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/chain/types" +) + +// Load returns an abstract copy of payment channel state, irregardless of actor version +func Load(store adt.Store, act *types.Actor) (State, error) { + switch act.Code { + case v0builtin.PaymentChannelActorCodeID: + out := v0State{store: store} + err := store.Get(store.Context(), act.Head, &out) + if err != nil { + return nil, err + } + return &out, nil + } + return nil, xerrors.Errorf("unknown actor code %s", act.Code) +} + +// State is an abstract version of payment channel state that works across +// versions +type State interface { + cbor.Marshaler + // Channel owner, who has funded the actor + From() address.Address + // Recipient of payouts from channel + To() address.Address + + // Height at which the channel can be `Collected` + SettlingAt() abi.ChainEpoch + + // Amount successfully redeemed through the payment channel, paid out on `Collect()` + ToSend() abi.TokenAmount + + // Get total number of lanes + LaneCount() (uint64, error) + + // Iterate lane states + ForEachLaneState(cb func(idx uint64, dl LaneState) error) error +} + +// LaneState is an abstract copy of the state of a single lane +type LaneState interface { + Redeemed() big.Int + Nonce() uint64 +} diff --git a/chain/actors/builtin/paych/v0.go b/chain/actors/builtin/paych/v0.go new file mode 100644 index 000000000..16a65bc9b --- /dev/null +++ b/chain/actors/builtin/paych/v0.go @@ -0,0 +1,89 @@ +package paych + +import ( + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + big "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/specs-actors/actors/builtin/paych" + v0adt "github.com/filecoin-project/specs-actors/actors/util/adt" +) + +type v0State struct { + paych.State + store adt.Store + lsAmt *v0adt.Array +} + +// Channel owner, who has funded the actor +func (s *v0State) From() address.Address { + return s.State.From +} + +// Recipient of payouts from channel +func (s *v0State) To() address.Address { + return s.State.From +} + +// Height at which the channel can be `Collected` +func (s *v0State) SettlingAt() abi.ChainEpoch { + return s.State.SettlingAt +} + +// Amount successfully redeemed through the payment channel, paid out on `Collect()` +func (s *v0State) ToSend() abi.TokenAmount { + return s.State.ToSend +} + +func (s *v0State) getOrLoadLsAmt() (*v0adt.Array, error) { + if s.lsAmt != nil { + return s.lsAmt, nil + } + + // Get the lane state from the chain + lsamt, err := v0adt.AsArray(s.store, s.State.LaneStates) + if err != nil { + return nil, err + } + + s.lsAmt = lsamt + return lsamt, nil +} + +// Get total number of lanes +func (s *v0State) LaneCount() (uint64, error) { + lsamt, err := s.getOrLoadLsAmt() + if err != nil { + return 0, err + } + return lsamt.Length(), nil +} + +// Iterate lane states +func (s *v0State) ForEachLaneState(cb func(idx uint64, dl LaneState) error) error { + // Get the lane state from the chain + lsamt, err := s.getOrLoadLsAmt() + if err != nil { + return err + } + + // Note: we use a map instead of an array to store laneStates because the + // client sets the lane ID (the index) and potentially they could use a + // very large index. + var ls paych.LaneState + return lsamt.ForEach(&ls, func(i int64) error { + return cb(uint64(i), &v0LaneState{ls}) + }) +} + +type v0LaneState struct { + paych.LaneState +} + +func (ls *v0LaneState) Redeemed() big.Int { + return ls.LaneState.Redeemed +} + +func (ls *v0LaneState) Nonce() uint64 { + return ls.LaneState.Nonce +} diff --git a/chain/events/state/predicates.go b/chain/events/state/predicates.go index 0858793d8..e5caa41d2 100644 --- a/chain/events/state/predicates.go +++ b/chain/events/state/predicates.go @@ -7,13 +7,12 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/lotus/chain/actors/builtin/paych" "github.com/filecoin-project/specs-actors/actors/builtin" init_ "github.com/filecoin-project/specs-actors/actors/builtin/init" "github.com/filecoin-project/specs-actors/actors/builtin/market" "github.com/filecoin-project/specs-actors/actors/builtin/miner" - "github.com/filecoin-project/specs-actors/actors/builtin/paych" "github.com/filecoin-project/specs-actors/actors/util/adt" - "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" typegen "github.com/whyrusleeping/cbor-gen" @@ -49,7 +48,7 @@ func NewStatePredicates(api ChainAPI) *StatePredicates { // - err type DiffTipSetKeyFunc func(ctx context.Context, oldState, newState types.TipSetKey) (changed bool, user UserData, err error) -type DiffActorStateFunc func(ctx context.Context, oldActorStateHead, newActorStateHead cid.Cid) (changed bool, user UserData, err error) +type DiffActorStateFunc func(ctx context.Context, oldActorState *types.Actor, newActorState *types.Actor) (changed bool, user UserData, err error) // OnActorStateChanged calls diffStateFunc when the state changes for the given actor func (sp *StatePredicates) OnActorStateChanged(addr address.Address, diffStateFunc DiffActorStateFunc) DiffTipSetKeyFunc { @@ -66,7 +65,7 @@ func (sp *StatePredicates) OnActorStateChanged(addr address.Address, diffStateFu if oldActor.Head.Equals(newActor.Head) { return false, nil, nil } - return diffStateFunc(ctx, oldActor.Head, newActor.Head) + return diffStateFunc(ctx, oldActor, newActor) } } @@ -74,13 +73,13 @@ type DiffStorageMarketStateFunc func(ctx context.Context, oldState *market.State // OnStorageMarketActorChanged calls diffStorageMarketState when the state changes for the market actor func (sp *StatePredicates) OnStorageMarketActorChanged(diffStorageMarketState DiffStorageMarketStateFunc) DiffTipSetKeyFunc { - return sp.OnActorStateChanged(builtin.StorageMarketActorAddr, func(ctx context.Context, oldActorStateHead, newActorStateHead cid.Cid) (changed bool, user UserData, err error) { + return sp.OnActorStateChanged(builtin.StorageMarketActorAddr, func(ctx context.Context, oldActorState, newActorState *types.Actor) (changed bool, user UserData, err error) { var oldState market.State - if err := sp.cst.Get(ctx, oldActorStateHead, &oldState); err != nil { + if err := sp.cst.Get(ctx, oldActorState.Head, &oldState); err != nil { return false, nil, err } var newState market.State - if err := sp.cst.Get(ctx, newActorStateHead, &newState); err != nil { + if err := sp.cst.Get(ctx, newActorState.Head, &newState); err != nil { return false, nil, err } return diffStorageMarketState(ctx, &oldState, &newState) @@ -408,13 +407,13 @@ func (sp *StatePredicates) AvailableBalanceChangedForAddresses(getAddrs func() [ type DiffMinerActorStateFunc func(ctx context.Context, oldState *miner.State, newState *miner.State) (changed bool, user UserData, err error) func (sp *StatePredicates) OnInitActorChange(diffInitActorState DiffInitActorStateFunc) DiffTipSetKeyFunc { - return sp.OnActorStateChanged(builtin.InitActorAddr, func(ctx context.Context, oldActorStateHead, newActorStateHead cid.Cid) (changed bool, user UserData, err error) { + return sp.OnActorStateChanged(builtin.InitActorAddr, func(ctx context.Context, oldActorState, newActorState *types.Actor) (changed bool, user UserData, err error) { var oldState init_.State - if err := sp.cst.Get(ctx, oldActorStateHead, &oldState); err != nil { + if err := sp.cst.Get(ctx, oldActorState.Head, &oldState); err != nil { return false, nil, err } var newState init_.State - if err := sp.cst.Get(ctx, newActorStateHead, &newState); err != nil { + if err := sp.cst.Get(ctx, newActorState.Head, &newState); err != nil { return false, nil, err } return diffInitActorState(ctx, &oldState, &newState) @@ -423,13 +422,13 @@ func (sp *StatePredicates) OnInitActorChange(diffInitActorState DiffInitActorSta } func (sp *StatePredicates) OnMinerActorChange(minerAddr address.Address, diffMinerActorState DiffMinerActorStateFunc) DiffTipSetKeyFunc { - return sp.OnActorStateChanged(minerAddr, func(ctx context.Context, oldActorStateHead, newActorStateHead cid.Cid) (changed bool, user UserData, err error) { + return sp.OnActorStateChanged(minerAddr, func(ctx context.Context, oldActorState, newActorState *types.Actor) (changed bool, user UserData, err error) { var oldState miner.State - if err := sp.cst.Get(ctx, oldActorStateHead, &oldState); err != nil { + if err := sp.cst.Get(ctx, oldActorState.Head, &oldState); err != nil { return false, nil, err } var newState miner.State - if err := sp.cst.Get(ctx, newActorStateHead, &newState); err != nil { + if err := sp.cst.Get(ctx, newActorState.Head, &newState); err != nil { return false, nil, err } return diffMinerActorState(ctx, &oldState, &newState) @@ -608,20 +607,20 @@ func (sp *StatePredicates) OnMinerPreCommitChange() DiffMinerActorStateFunc { } // DiffPaymentChannelStateFunc is function that compares two states for the payment channel -type DiffPaymentChannelStateFunc func(ctx context.Context, oldState *paych.State, newState *paych.State) (changed bool, user UserData, err error) +type DiffPaymentChannelStateFunc func(ctx context.Context, oldState paych.State, newState paych.State) (changed bool, user UserData, err error) // OnPaymentChannelActorChanged calls diffPaymentChannelState when the state changes for the the payment channel actor func (sp *StatePredicates) OnPaymentChannelActorChanged(paychAddr address.Address, diffPaymentChannelState DiffPaymentChannelStateFunc) DiffTipSetKeyFunc { - return sp.OnActorStateChanged(paychAddr, func(ctx context.Context, oldActorStateHead, newActorStateHead cid.Cid) (changed bool, user UserData, err error) { - var oldState paych.State - if err := sp.cst.Get(ctx, oldActorStateHead, &oldState); err != nil { + return sp.OnActorStateChanged(paychAddr, func(ctx context.Context, oldActorState, newActorState *types.Actor) (changed bool, user UserData, err error) { + oldState, err := paych.Load(adt.WrapStore(ctx, sp.cst), oldActorState) + if err != nil { return false, nil, err } - var newState paych.State - if err := sp.cst.Get(ctx, newActorStateHead, &newState); err != nil { + newState, err := paych.Load(adt.WrapStore(ctx, sp.cst), newActorState) + if err != nil { return false, nil, err } - return diffPaymentChannelState(ctx, &oldState, &newState) + return diffPaymentChannelState(ctx, oldState, newState) }) } @@ -633,13 +632,13 @@ type PayChToSendChange struct { // OnToSendAmountChanges monitors changes on the total amount to send from one party to the other on a payment channel func (sp *StatePredicates) OnToSendAmountChanges() DiffPaymentChannelStateFunc { - return func(ctx context.Context, oldState *paych.State, newState *paych.State) (changed bool, user UserData, err error) { - if oldState.ToSend.Equals(newState.ToSend) { + return func(ctx context.Context, oldState paych.State, newState paych.State) (changed bool, user UserData, err error) { + if oldState.ToSend().Equals(newState.ToSend()) { return false, nil, nil } return true, &PayChToSendChange{ - OldToSend: oldState.ToSend, - NewToSend: newState.ToSend, + OldToSend: oldState.ToSend(), + NewToSend: newState.ToSend(), }, nil } } diff --git a/chain/events/state/predicates_test.go b/chain/events/state/predicates_test.go index a1dccfadc..7117a96cc 100644 --- a/chain/events/state/predicates_test.go +++ b/chain/events/state/predicates_test.go @@ -221,7 +221,7 @@ func TestMarketPredicates(t *testing.T) { // Test that OnActorStateChanged does not call the callback if the state has not changed mockAddr, err := address.NewFromString("t01") require.NoError(t, err) - actorDiffFn := preds.OnActorStateChanged(mockAddr, func(context.Context, cid.Cid, cid.Cid) (bool, UserData, error) { + actorDiffFn := preds.OnActorStateChanged(mockAddr, func(context.Context, *types.Actor, *types.Actor) (bool, UserData, error) { t.Fatal("No state change so this should not be called") return false, nil, nil }) diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index f71b788c8..ae8b47bce 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -25,6 +25,7 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/builtin/market" + "github.com/filecoin-project/lotus/chain/actors/builtin/paych" "github.com/filecoin-project/lotus/chain/actors/builtin/power" "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/store" @@ -1145,3 +1146,21 @@ func (sm *StateManager) GetNtwkVersion(ctx context.Context, height abi.ChainEpoc return build.NewestNetworkVersion } + +func (sm *StateManager) GetPaychState(ctx context.Context, addr address.Address, ts *types.TipSet) (*types.Actor, paych.State, error) { + st, err := sm.ParentState(ts) + if err != nil { + return nil, nil, err + } + + act, err := st.GetActor(addr) + if err != nil { + return nil, nil, err + } + + actState, err := paych.Load(sm.cs.Store(ctx), act) + if err != nil { + return nil, nil, err + } + return act, actState, nil +} diff --git a/cli/paych_test.go b/cli/paych_test.go index cccc80ff4..1497a54a6 100644 --- a/cli/paych_test.go +++ b/cli/paych_test.go @@ -24,7 +24,8 @@ import ( "github.com/filecoin-project/lotus/chain/events" "github.com/filecoin-project/lotus/api/apibstore" - "github.com/filecoin-project/specs-actors/actors/builtin/paych" + "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/chain/actors/builtin/paych" cbor "github.com/ipfs/go-ipld-cbor" "github.com/filecoin-project/go-address" @@ -88,7 +89,7 @@ func TestPaymentChannels(t *testing.T) { // Wait for the chain to reach the settle height chState := getPaychState(ctx, t, paymentReceiver, chAddr) - waitForHeight(ctx, t, paymentReceiver, chState.SettlingAt) + waitForHeight(ctx, t, paymentReceiver, chState.SettlingAt()) // receiver: paych collect cmd = []string{chAddr.String()} @@ -540,8 +541,7 @@ func getPaychState(ctx context.Context, t *testing.T, node test.TestNode, chAddr require.NoError(t, err) store := cbor.NewCborStore(apibstore.NewAPIBlockstore(node)) - var chState paych.State - err = store.Get(ctx, act.Head, &chState) + chState, err := paych.Load(adt.WrapStore(ctx, store), act) require.NoError(t, err) return chState diff --git a/paychmgr/manager.go b/paychmgr/manager.go index 4b102f062..4556c37be 100644 --- a/paychmgr/manager.go +++ b/paychmgr/manager.go @@ -4,28 +4,23 @@ import ( "context" "sync" - "github.com/filecoin-project/go-state-types/crypto" - - "github.com/filecoin-project/lotus/node/modules/helpers" - - "github.com/ipfs/go-datastore" - - xerrors "golang.org/x/xerrors" - - "github.com/filecoin-project/lotus/api" - - "github.com/filecoin-project/specs-actors/actors/builtin/paych" - "github.com/filecoin-project/specs-actors/actors/util/adt" - "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" logging "github.com/ipfs/go-log/v2" "go.uber.org/fx" + xerrors "golang.org/x/xerrors" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/crypto" + v0paych "github.com/filecoin-project/specs-actors/actors/builtin/paych" + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/chain/actors/builtin/paych" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/impl/full" + "github.com/filecoin-project/lotus/node/modules/helpers" ) var log = logging.Logger("paych") @@ -40,9 +35,9 @@ type PaychAPI struct { // stateManagerAPI defines the methods needed from StateManager type stateManagerAPI interface { - LoadActorState(ctx context.Context, a address.Address, out interface{}, ts *types.TipSet) (*types.Actor, error) + ResolveToKeyAddress(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) + GetPaychState(ctx context.Context, addr address.Address, ts *types.TipSet) (*types.Actor, paych.State, error) Call(ctx context.Context, msg *types.Message, ts *types.TipSet) (*api.InvocResult, error) - AdtStore(ctx context.Context) adt.Store } // paychAPI defines the API methods needed by the payment channel manager @@ -226,7 +221,7 @@ func (pm *Manager) GetChannelInfo(addr address.Address) (*ChannelInfo, error) { return ca.getChannelInfo(addr) } -func (pm *Manager) CreateVoucher(ctx context.Context, ch address.Address, voucher paych.SignedVoucher) (*api.VoucherCreateResult, error) { +func (pm *Manager) CreateVoucher(ctx context.Context, ch address.Address, voucher v0paych.SignedVoucher) (*api.VoucherCreateResult, error) { ca, err := pm.accessorByAddress(ch) if err != nil { return nil, err @@ -238,7 +233,7 @@ func (pm *Manager) CreateVoucher(ctx context.Context, ch address.Address, vouche // CheckVoucherValid checks if the given voucher is valid (is or could become spendable at some point). // If the channel is not in the store, fetches the channel from state (and checks that // the channel To address is owned by the wallet). -func (pm *Manager) CheckVoucherValid(ctx context.Context, ch address.Address, sv *paych.SignedVoucher) error { +func (pm *Manager) CheckVoucherValid(ctx context.Context, ch address.Address, sv *v0paych.SignedVoucher) error { // Get an accessor for the channel, creating it from state if necessary ca, err := pm.inboundChannelAccessor(ctx, ch) if err != nil { @@ -250,7 +245,7 @@ func (pm *Manager) CheckVoucherValid(ctx context.Context, ch address.Address, sv } // CheckVoucherSpendable checks if the given voucher is currently spendable -func (pm *Manager) CheckVoucherSpendable(ctx context.Context, ch address.Address, sv *paych.SignedVoucher, secret []byte, proof []byte) (bool, error) { +func (pm *Manager) CheckVoucherSpendable(ctx context.Context, ch address.Address, sv *v0paych.SignedVoucher, secret []byte, proof []byte) (bool, error) { ca, err := pm.accessorByAddress(ch) if err != nil { return false, err @@ -261,7 +256,7 @@ func (pm *Manager) CheckVoucherSpendable(ctx context.Context, ch address.Address // AddVoucherOutbound adds a voucher for an outbound channel. // Returns an error if the channel is not already in the store. -func (pm *Manager) AddVoucherOutbound(ctx context.Context, ch address.Address, sv *paych.SignedVoucher, proof []byte, minDelta types.BigInt) (types.BigInt, error) { +func (pm *Manager) AddVoucherOutbound(ctx context.Context, ch address.Address, sv *v0paych.SignedVoucher, proof []byte, minDelta types.BigInt) (types.BigInt, error) { ca, err := pm.accessorByAddress(ch) if err != nil { return types.NewInt(0), err @@ -272,7 +267,7 @@ func (pm *Manager) AddVoucherOutbound(ctx context.Context, ch address.Address, s // AddVoucherInbound adds a voucher for an inbound channel. // If the channel is not in the store, fetches the channel from state (and checks that // the channel To address is owned by the wallet). -func (pm *Manager) AddVoucherInbound(ctx context.Context, ch address.Address, sv *paych.SignedVoucher, proof []byte, minDelta types.BigInt) (types.BigInt, error) { +func (pm *Manager) AddVoucherInbound(ctx context.Context, ch address.Address, sv *v0paych.SignedVoucher, proof []byte, minDelta types.BigInt) (types.BigInt, error) { // Get an accessor for the channel, creating it from state if necessary ca, err := pm.inboundChannelAccessor(ctx, ch) if err != nil { @@ -341,7 +336,7 @@ func (pm *Manager) trackInboundChannel(ctx context.Context, ch address.Address) return pm.store.TrackChannel(stateCi) } -func (pm *Manager) SubmitVoucher(ctx context.Context, ch address.Address, sv *paych.SignedVoucher, secret []byte, proof []byte) (cid.Cid, error) { +func (pm *Manager) SubmitVoucher(ctx context.Context, ch address.Address, sv *v0paych.SignedVoucher, secret []byte, proof []byte) (cid.Cid, error) { ca, err := pm.accessorByAddress(ch) if err != nil { return cid.Undef, err diff --git a/paychmgr/mock_test.go b/paychmgr/mock_test.go index bc19de223..c761221d2 100644 --- a/paychmgr/mock_test.go +++ b/paychmgr/mock_test.go @@ -2,23 +2,18 @@ package paychmgr import ( "context" - "fmt" + "errors" "sync" - "github.com/filecoin-project/lotus/lib/sigs" - - "github.com/filecoin-project/go-state-types/crypto" - - cbornode "github.com/ipfs/go-ipld-cbor" - - "github.com/filecoin-project/specs-actors/actors/util/adt" + "github.com/ipfs/go-cid" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/crypto" + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/actors/builtin/paych" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/specs-actors/actors/builtin/account" - "github.com/filecoin-project/specs-actors/actors/builtin/paych" - "github.com/ipfs/go-cid" + "github.com/filecoin-project/lotus/lib/sigs" ) type mockManagerAPI struct { @@ -40,29 +35,23 @@ type mockPchState struct { type mockStateManager struct { lk sync.Mutex - accountState map[address.Address]account.State + accountState map[address.Address]address.Address paychState map[address.Address]mockPchState - store adt.Store response *api.InvocResult lastCall *types.Message } func newMockStateManager() *mockStateManager { return &mockStateManager{ - accountState: make(map[address.Address]account.State), + accountState: make(map[address.Address]address.Address), paychState: make(map[address.Address]mockPchState), - store: adt.WrapStore(context.Background(), cbornode.NewMemCborStore()), } } -func (sm *mockStateManager) AdtStore(ctx context.Context) adt.Store { - return sm.store -} - -func (sm *mockStateManager) setAccountState(a address.Address, state account.State) { +func (sm *mockStateManager) setAccountAddress(a address.Address, lookup address.Address) { sm.lk.Lock() defer sm.lk.Unlock() - sm.accountState[a] = state + sm.accountState[a] = lookup } func (sm *mockStateManager) setPaychState(a address.Address, actor *types.Actor, state paych.State) { @@ -71,31 +60,24 @@ func (sm *mockStateManager) setPaychState(a address.Address, actor *types.Actor, sm.paychState[a] = mockPchState{actor, state} } -func (sm *mockStateManager) storeLaneStates(laneStates map[uint64]paych.LaneState) (cid.Cid, error) { - arr := adt.MakeEmptyArray(sm.store) - for i, ls := range laneStates { - ls := ls - if err := arr.Set(i, &ls); err != nil { - return cid.Undef, err - } - } - return arr.Root() -} - -func (sm *mockStateManager) LoadActorState(ctx context.Context, a address.Address, out interface{}, ts *types.TipSet) (*types.Actor, error) { +func (sm *mockStateManager) ResolveToKeyAddress(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) { sm.lk.Lock() defer sm.lk.Unlock() + keyAddr, ok := sm.accountState[addr] + if !ok { + return address.Undef, errors.New("not found") + } + return keyAddr, nil +} - if outState, ok := out.(*account.State); ok { - *outState = sm.accountState[a] - return nil, nil +func (sm *mockStateManager) GetPaychState(ctx context.Context, addr address.Address, ts *types.TipSet) (*types.Actor, paych.State, error) { + sm.lk.Lock() + defer sm.lk.Unlock() + info, ok := sm.paychState[addr] + if !ok { + return nil, nil, errors.New("not found") } - if outState, ok := out.(*paych.State); ok { - info := sm.paychState[a] - *outState = info.state - return info.actor, nil - } - panic(fmt.Sprintf("unexpected state type %v", out)) + return info.actor, info.state, nil } func (sm *mockStateManager) setCallResponse(response *api.InvocResult) { diff --git a/paychmgr/msglistener_test.go b/paychmgr/msglistener_test.go index 2c3ae16e4..4b8ae6f30 100644 --- a/paychmgr/msglistener_test.go +++ b/paychmgr/msglistener_test.go @@ -4,9 +4,7 @@ import ( "testing" "github.com/ipfs/go-cid" - "github.com/stretchr/testify/require" - "golang.org/x/xerrors" ) diff --git a/paychmgr/paych.go b/paychmgr/paych.go index 20d76b7fd..53f16b4fc 100644 --- a/paychmgr/paych.go +++ b/paychmgr/paych.go @@ -5,22 +5,20 @@ import ( "context" "fmt" - "github.com/filecoin-project/lotus/api" - - "github.com/filecoin-project/specs-actors/actors/util/adt" - "github.com/ipfs/go-cid" + "golang.org/x/xerrors" "github.com/filecoin-project/go-address" cborutil "github.com/filecoin-project/go-cbor-util" "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/specs-actors/actors/builtin" + v0paych "github.com/filecoin-project/specs-actors/actors/builtin/paych" + + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/actors/builtin/paych" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/sigs" - "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/builtin/account" - "github.com/filecoin-project/specs-actors/actors/builtin/paych" - "golang.org/x/xerrors" ) // insufficientFundsErr indicates that there are not enough funds in the @@ -45,6 +43,19 @@ func (e *ErrInsufficientFunds) Shortfall() types.BigInt { return e.shortfall } +type laneState struct { + redeemed big.Int + nonce uint64 +} + +func (ls laneState) Redeemed() big.Int { + return ls.redeemed +} + +func (ls laneState) Nonce() uint64 { + return ls.nonce +} + // channelAccessor is used to simplify locking when accessing a channel type channelAccessor struct { from address.Address @@ -92,7 +103,7 @@ func (ca *channelAccessor) outboundActiveByFromTo(from, to address.Address) (*Ch // nonce, signing the voucher and storing it in the local datastore. // If there are not enough funds in the channel to create the voucher, returns // the shortfall in funds. -func (ca *channelAccessor) createVoucher(ctx context.Context, ch address.Address, voucher paych.SignedVoucher) (*api.VoucherCreateResult, error) { +func (ca *channelAccessor) createVoucher(ctx context.Context, ch address.Address, voucher v0paych.SignedVoucher) (*api.VoucherCreateResult, error) { ca.lk.Lock() defer ca.lk.Unlock() @@ -151,14 +162,14 @@ func (ca *channelAccessor) nextNonceForLane(ci *ChannelInfo, lane uint64) uint64 return maxnonce + 1 } -func (ca *channelAccessor) checkVoucherValid(ctx context.Context, ch address.Address, sv *paych.SignedVoucher) (map[uint64]*paych.LaneState, error) { +func (ca *channelAccessor) checkVoucherValid(ctx context.Context, ch address.Address, sv *v0paych.SignedVoucher) (map[uint64]paych.LaneState, error) { ca.lk.Lock() defer ca.lk.Unlock() return ca.checkVoucherValidUnlocked(ctx, ch, sv) } -func (ca *channelAccessor) checkVoucherValidUnlocked(ctx context.Context, ch address.Address, sv *paych.SignedVoucher) (map[uint64]*paych.LaneState, error) { +func (ca *channelAccessor) checkVoucherValidUnlocked(ctx context.Context, ch address.Address, sv *v0paych.SignedVoucher) (map[uint64]paych.LaneState, error) { if sv.ChannelAddr != ch { return nil, xerrors.Errorf("voucher ChannelAddr doesn't match channel address, got %s, expected %s", sv.ChannelAddr, ch) } @@ -170,12 +181,10 @@ func (ca *channelAccessor) checkVoucherValidUnlocked(ctx context.Context, ch add } // Load channel "From" account actor state - var actState account.State - _, err = ca.api.LoadActorState(ctx, pchState.From, &actState, nil) + from, err := ca.api.ResolveToKeyAddress(ctx, pchState.From(), nil) if err != nil { return nil, err } - from := actState.Address // verify voucher signature vb, err := sv.SigningBytes() @@ -199,12 +208,12 @@ func (ca *channelAccessor) checkVoucherValidUnlocked(ctx context.Context, ch add // If the new voucher nonce value is less than the highest known // nonce for the lane ls, lsExists := laneStates[sv.Lane] - if lsExists && sv.Nonce <= ls.Nonce { + if lsExists && sv.Nonce <= ls.Nonce() { return nil, fmt.Errorf("nonce too low") } // If the voucher amount is less than the highest known voucher amount - if lsExists && sv.Amount.LessThanEqual(ls.Redeemed) { + if lsExists && sv.Amount.LessThanEqual(ls.Redeemed()) { return nil, fmt.Errorf("voucher amount is lower than amount for voucher with lower nonce") } @@ -230,7 +239,7 @@ func (ca *channelAccessor) checkVoucherValidUnlocked(ctx context.Context, ch add // Total required balance = total redeemed + toSend // Must not exceed actor balance - newTotal := types.BigAdd(totalRedeemed, pchState.ToSend) + newTotal := types.BigAdd(totalRedeemed, pchState.ToSend()) if act.Balance.LessThan(newTotal) { return nil, newErrInsufficientFunds(types.BigSub(newTotal, act.Balance)) } @@ -242,7 +251,7 @@ func (ca *channelAccessor) checkVoucherValidUnlocked(ctx context.Context, ch add return laneStates, nil } -func (ca *channelAccessor) checkVoucherSpendable(ctx context.Context, ch address.Address, sv *paych.SignedVoucher, secret []byte, proof []byte) (bool, error) { +func (ca *channelAccessor) checkVoucherSpendable(ctx context.Context, ch address.Address, sv *v0paych.SignedVoucher, secret []byte, proof []byte) (bool, error) { ca.lk.Lock() defer ca.lk.Unlock() @@ -281,7 +290,7 @@ func (ca *channelAccessor) checkVoucherSpendable(ctx context.Context, ch address } } - enc, err := actors.SerializeParams(&paych.UpdateChannelStateParams{ + enc, err := actors.SerializeParams(&v0paych.UpdateChannelStateParams{ Sv: *sv, Secret: secret, Proof: proof, @@ -308,22 +317,22 @@ func (ca *channelAccessor) checkVoucherSpendable(ctx context.Context, ch address } func (ca *channelAccessor) getPaychRecipient(ctx context.Context, ch address.Address) (address.Address, error) { - var state paych.State - if _, err := ca.api.LoadActorState(ctx, ch, &state, nil); err != nil { + _, state, err := ca.api.GetPaychState(ctx, ch, nil) + if err != nil { return address.Address{}, err } - return state.To, nil + return state.To(), nil } -func (ca *channelAccessor) addVoucher(ctx context.Context, ch address.Address, sv *paych.SignedVoucher, proof []byte, minDelta types.BigInt) (types.BigInt, error) { +func (ca *channelAccessor) addVoucher(ctx context.Context, ch address.Address, sv *v0paych.SignedVoucher, proof []byte, minDelta types.BigInt) (types.BigInt, error) { ca.lk.Lock() defer ca.lk.Unlock() return ca.addVoucherUnlocked(ctx, ch, sv, proof, minDelta) } -func (ca *channelAccessor) addVoucherUnlocked(ctx context.Context, ch address.Address, sv *paych.SignedVoucher, proof []byte, minDelta types.BigInt) (types.BigInt, error) { +func (ca *channelAccessor) addVoucherUnlocked(ctx context.Context, ch address.Address, sv *v0paych.SignedVoucher, proof []byte, minDelta types.BigInt) (types.BigInt, error) { ci, err := ca.store.ByAddress(ch) if err != nil { return types.BigInt{}, err @@ -367,7 +376,7 @@ func (ca *channelAccessor) addVoucherUnlocked(ctx context.Context, ch address.Ad laneState, exists := laneStates[sv.Lane] redeemed := big.NewInt(0) if exists { - redeemed = laneState.Redeemed + redeemed = laneState.Redeemed() } delta := types.BigSub(sv.Amount, redeemed) @@ -387,7 +396,7 @@ func (ca *channelAccessor) addVoucherUnlocked(ctx context.Context, ch address.Ad return delta, ca.store.putChannelInfo(ci) } -func (ca *channelAccessor) submitVoucher(ctx context.Context, ch address.Address, sv *paych.SignedVoucher, secret []byte, proof []byte) (cid.Cid, error) { +func (ca *channelAccessor) submitVoucher(ctx context.Context, ch address.Address, sv *v0paych.SignedVoucher, secret []byte, proof []byte) (cid.Cid, error) { ca.lk.Lock() defer ca.lk.Unlock() @@ -428,7 +437,7 @@ func (ca *channelAccessor) submitVoucher(ctx context.Context, ch address.Address } } - enc, err := actors.SerializeParams(&paych.UpdateChannelStateParams{ + enc, err := actors.SerializeParams(&v0paych.UpdateChannelStateParams{ Sv: *sv, Secret: secret, Proof: proof, @@ -487,13 +496,11 @@ func (ca *channelAccessor) listVouchers(ctx context.Context, ch address.Address) // laneState gets the LaneStates from chain, then applies all vouchers in // the data store over the chain state -func (ca *channelAccessor) laneState(ctx context.Context, state *paych.State, ch address.Address) (map[uint64]*paych.LaneState, error) { +func (ca *channelAccessor) laneState(ctx context.Context, state paych.State, ch address.Address) (map[uint64]paych.LaneState, error) { // TODO: we probably want to call UpdateChannelState with all vouchers to be fully correct // (but technically dont't need to) - // Get the lane state from the chain - store := ca.api.AdtStore(ctx) - lsamt, err := adt.AsArray(store, state.LaneStates) + laneCount, err := state.LaneCount() if err != nil { return nil, err } @@ -501,11 +508,9 @@ func (ca *channelAccessor) laneState(ctx context.Context, state *paych.State, ch // Note: we use a map instead of an array to store laneStates because the // client sets the lane ID (the index) and potentially they could use a // very large index. - var ls paych.LaneState - laneStates := make(map[uint64]*paych.LaneState, lsamt.Length()) - err = lsamt.ForEach(&ls, func(i int64) error { - current := ls - laneStates[uint64(i)] = ¤t + laneStates := make(map[uint64]paych.LaneState, laneCount) + err = state.ForEachLaneState(func(idx uint64, ls paych.LaneState) error { + laneStates[idx] = ls return nil }) if err != nil { @@ -526,27 +531,19 @@ func (ca *channelAccessor) laneState(ctx context.Context, state *paych.State, ch // If there's a voucher for a lane that isn't in chain state just // create it ls, ok := laneStates[v.Voucher.Lane] - if !ok { - ls = &paych.LaneState{ - Redeemed: types.NewInt(0), - Nonce: 0, - } - laneStates[v.Voucher.Lane] = ls - } - if v.Voucher.Nonce < ls.Nonce { + if ok && v.Voucher.Nonce < ls.Nonce() { continue } - ls.Nonce = v.Voucher.Nonce - ls.Redeemed = v.Voucher.Amount + laneStates[v.Voucher.Lane] = laneState{v.Voucher.Amount, v.Voucher.Nonce} } return laneStates, nil } // Get the total redeemed amount across all lanes, after applying the voucher -func (ca *channelAccessor) totalRedeemedWithVoucher(laneStates map[uint64]*paych.LaneState, sv *paych.SignedVoucher) (big.Int, error) { +func (ca *channelAccessor) totalRedeemedWithVoucher(laneStates map[uint64]paych.LaneState, sv *v0paych.SignedVoucher) (big.Int, error) { // TODO: merges if len(sv.Merges) != 0 { return big.Int{}, xerrors.Errorf("dont currently support paych lane merges") @@ -554,17 +551,17 @@ func (ca *channelAccessor) totalRedeemedWithVoucher(laneStates map[uint64]*paych total := big.NewInt(0) for _, ls := range laneStates { - total = big.Add(total, ls.Redeemed) + total = big.Add(total, ls.Redeemed()) } lane, ok := laneStates[sv.Lane] if ok { // If the voucher is for an existing lane, and the voucher nonce // is higher than the lane nonce - if sv.Nonce > lane.Nonce { + if sv.Nonce > lane.Nonce() { // Add the delta between the redeemed amount and the voucher // amount to the total - delta := big.Sub(sv.Amount, lane.Redeemed) + delta := big.Sub(sv.Amount, lane.Redeemed()) total = big.Add(total, delta) } } else { diff --git a/paychmgr/paych_test.go b/paychmgr/paych_test.go index 18c6655da..434c83e9c 100644 --- a/paychmgr/paych_test.go +++ b/paychmgr/paych_test.go @@ -5,31 +5,24 @@ import ( "context" "testing" - "github.com/filecoin-project/lotus/api" - - "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/ipfs/go-cid" - - "github.com/filecoin-project/go-state-types/crypto" - "github.com/filecoin-project/lotus/lib/sigs" - - "github.com/stretchr/testify/require" - - "github.com/filecoin-project/go-state-types/big" - - "github.com/filecoin-project/go-state-types/abi" - tutils "github.com/filecoin-project/specs-actors/support/testing" - - "github.com/filecoin-project/specs-actors/actors/builtin/paych" - - "github.com/filecoin-project/specs-actors/actors/builtin/account" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/types" - ds "github.com/ipfs/go-datastore" ds_sync "github.com/ipfs/go-datastore/sync" + "github.com/stretchr/testify/require" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/go-state-types/crypto" + "github.com/filecoin-project/specs-actors/actors/builtin" + v0paych "github.com/filecoin-project/specs-actors/actors/builtin/paych" + tutils "github.com/filecoin-project/specs-actors/support/testing" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/actors/builtin/paych" + paychmock "github.com/filecoin-project/lotus/chain/actors/builtin/paych/mock" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/sigs" ) func TestCheckVoucherValid(t *testing.T) { @@ -46,8 +39,8 @@ func TestCheckVoucherValid(t *testing.T) { toAcct := tutils.NewActorAddr(t, "toAct") mock := newMockManagerAPI() - mock.setAccountState(fromAcct, account.State{Address: from}) - mock.setAccountState(toAcct, account.State{Address: to}) + mock.setAccountAddress(fromAcct, from) + mock.setAccountAddress(toAcct, to) tcases := []struct { name string @@ -96,10 +89,7 @@ func TestCheckVoucherValid(t *testing.T) { voucherLane: 1, voucherNonce: 2, laneStates: map[uint64]paych.LaneState{ - 1: { - Redeemed: big.NewInt(2), - Nonce: 3, - }, + 1: paychmock.NewMockLaneState(big.NewInt(2), 3), }, }, { name: "passes when nonce higher", @@ -110,10 +100,7 @@ func TestCheckVoucherValid(t *testing.T) { voucherLane: 1, voucherNonce: 3, laneStates: map[uint64]paych.LaneState{ - 1: { - Redeemed: big.NewInt(2), - Nonce: 2, - }, + 1: paychmock.NewMockLaneState(big.NewInt(2), 2), }, }, { name: "passes when nonce for different lane", @@ -124,10 +111,7 @@ func TestCheckVoucherValid(t *testing.T) { voucherLane: 2, voucherNonce: 2, laneStates: map[uint64]paych.LaneState{ - 1: { - Redeemed: big.NewInt(2), - Nonce: 3, - }, + 1: paychmock.NewMockLaneState(big.NewInt(2), 3), }, }, { name: "fails when voucher has higher nonce but lower value than lane state", @@ -139,10 +123,7 @@ func TestCheckVoucherValid(t *testing.T) { voucherLane: 1, voucherNonce: 3, laneStates: map[uint64]paych.LaneState{ - 1: { - Redeemed: big.NewInt(6), - Nonce: 2, - }, + 1: paychmock.NewMockLaneState(big.NewInt(6), 2), }, }, { name: "fails when voucher + ToSend > balance", @@ -168,10 +149,7 @@ func TestCheckVoucherValid(t *testing.T) { voucherNonce: 2, laneStates: map[uint64]paych.LaneState{ // Lane 1 (same as voucher lane 1) - 1: { - Redeemed: big.NewInt(4), - Nonce: 1, - }, + 1: paychmock.NewMockLaneState(big.NewInt(4), 1), }, }, { // required balance = toSend + total redeemed @@ -188,10 +166,7 @@ func TestCheckVoucherValid(t *testing.T) { voucherNonce: 1, laneStates: map[uint64]paych.LaneState{ // Lane 2 (different from voucher lane 1) - 2: { - Redeemed: big.NewInt(4), - Nonce: 1, - }, + 2: paychmock.NewMockLaneState(big.NewInt(4), 1), }, }} @@ -208,18 +183,8 @@ func TestCheckVoucherValid(t *testing.T) { Balance: tcase.actorBalance, } - // Set the state of the channel's lanes - laneStates, err := mock.storeLaneStates(tcase.laneStates) - require.NoError(t, err) - - mock.setPaychState(ch, act, paych.State{ - From: fromAcct, - To: toAcct, - ToSend: tcase.toSend, - SettlingAt: abi.ChainEpoch(0), - MinSettleHeight: abi.ChainEpoch(0), - LaneStates: laneStates, - }) + mock.setPaychState(ch, act, paychmock.NewMockPayChState( + fromAcct, toAcct, abi.ChainEpoch(0), tcase.toSend, tcase.laneStates)) // Create a manager mgr, err := newManager(store, mock) @@ -255,22 +220,16 @@ func TestCheckVoucherValidCountingAllLanes(t *testing.T) { minDelta := big.NewInt(0) mock := newMockManagerAPI() - mock.setAccountState(fromAcct, account.State{Address: from}) - mock.setAccountState(toAcct, account.State{Address: to}) + mock.setAccountAddress(fromAcct, from) + mock.setAccountAddress(toAcct, to) store := NewStore(ds_sync.MutexWrap(ds.NewMapDatastore())) actorBalance := big.NewInt(10) toSend := big.NewInt(1) laneStates := map[uint64]paych.LaneState{ - 1: { - Nonce: 1, - Redeemed: big.NewInt(3), - }, - 2: { - Nonce: 1, - Redeemed: big.NewInt(4), - }, + 1: paychmock.NewMockLaneState(big.NewInt(3), 1), + 2: paychmock.NewMockLaneState(big.NewInt(4), 1), } act := &types.Actor{ @@ -280,16 +239,7 @@ func TestCheckVoucherValidCountingAllLanes(t *testing.T) { Balance: actorBalance, } - lsCid, err := mock.storeLaneStates(laneStates) - require.NoError(t, err) - mock.setPaychState(ch, act, paych.State{ - From: fromAcct, - To: toAcct, - ToSend: toSend, - SettlingAt: abi.ChainEpoch(0), - MinSettleHeight: abi.ChainEpoch(0), - LaneStates: lsCid, - }) + mock.setPaychState(ch, act, paychmock.NewMockPayChState(fromAcct, toAcct, abi.ChainEpoch(0), toSend, laneStates)) mgr, err := newManager(store, mock) require.NoError(t, err) @@ -389,7 +339,7 @@ func TestCreateVoucher(t *testing.T) { // Create a voucher in lane 1 voucherLane1Amt := big.NewInt(5) - voucher := paych.SignedVoucher{ + voucher := v0paych.SignedVoucher{ Lane: 1, Amount: voucherLane1Amt, } @@ -404,7 +354,7 @@ func TestCreateVoucher(t *testing.T) { // Create a voucher in lane 1 again, with a higher amount voucherLane1Amt = big.NewInt(8) - voucher = paych.SignedVoucher{ + voucher = v0paych.SignedVoucher{ Lane: 1, Amount: voucherLane1Amt, } @@ -419,7 +369,7 @@ func TestCreateVoucher(t *testing.T) { // Create a voucher in lane 2 that covers all the remaining funds // in the channel voucherLane2Amt := big.Sub(s.amt, voucherLane1Amt) - voucher = paych.SignedVoucher{ + voucher = v0paych.SignedVoucher{ Lane: 2, Amount: voucherLane2Amt, } @@ -433,7 +383,7 @@ func TestCreateVoucher(t *testing.T) { // Create a voucher in lane 2 that exceeds the remaining funds in the // channel voucherLane2Amt = big.Add(voucherLane2Amt, big.NewInt(1)) - voucher = paych.SignedVoucher{ + voucher = v0paych.SignedVoucher{ Lane: 2, Amount: voucherLane2Amt, } @@ -567,8 +517,8 @@ func TestAllocateLaneWithExistingLaneState(t *testing.T) { toAcct := tutils.NewActorAddr(t, "toAct") mock := newMockManagerAPI() - mock.setAccountState(fromAcct, account.State{Address: from}) - mock.setAccountState(toAcct, account.State{Address: to}) + mock.setAccountAddress(fromAcct, from) + mock.setAccountAddress(toAcct, to) mock.addWalletAddress(to) store := NewStore(ds_sync.MutexWrap(ds.NewMapDatastore())) @@ -584,16 +534,7 @@ func TestAllocateLaneWithExistingLaneState(t *testing.T) { Balance: actorBalance, } - arr, err := adt.MakeEmptyArray(mock.store).Root() - require.NoError(t, err) - mock.setPaychState(ch, act, paych.State{ - From: fromAcct, - To: toAcct, - ToSend: toSend, - SettlingAt: abi.ChainEpoch(0), - MinSettleHeight: abi.ChainEpoch(0), - LaneStates: arr, - }) + mock.setPaychState(ch, act, paychmock.NewMockPayChState(fromAcct, toAcct, abi.ChainEpoch(0), toSend, make(map[uint64]paych.LaneState))) mgr, err := newManager(store, mock) require.NoError(t, err) @@ -681,19 +622,10 @@ func TestAddVoucherInboundWalletKey(t *testing.T) { } mock := newMockManagerAPI() - arr, err := adt.MakeEmptyArray(mock.store).Root() - require.NoError(t, err) - mock.setAccountState(fromAcct, account.State{Address: from}) - mock.setAccountState(toAcct, account.State{Address: to}) + mock.setAccountAddress(fromAcct, from) + mock.setAccountAddress(toAcct, to) - mock.setPaychState(ch, act, paych.State{ - From: fromAcct, - To: toAcct, - ToSend: types.NewInt(0), - SettlingAt: abi.ChainEpoch(0), - MinSettleHeight: abi.ChainEpoch(0), - LaneStates: arr, - }) + mock.setPaychState(ch, act, paychmock.NewMockPayChState(fromAcct, toAcct, abi.ChainEpoch(0), types.NewInt(0), make(map[uint64]paych.LaneState))) // Create a manager store := NewStore(ds_sync.MutexWrap(ds.NewMapDatastore())) @@ -840,7 +772,7 @@ func TestCheckSpendable(t *testing.T) { // Check that the secret and proof were passed through correctly lastCall := s.mock.getLastCall() - var p paych.UpdateChannelStateParams + var p v0paych.UpdateChannelStateParams err = p.UnmarshalCBOR(bytes.NewReader(lastCall.Params)) require.NoError(t, err) require.Equal(t, otherProof, p.Proof) @@ -854,7 +786,7 @@ func TestCheckSpendable(t *testing.T) { require.True(t, spendable) lastCall = s.mock.getLastCall() - var p2 paych.UpdateChannelStateParams + var p2 v0paych.UpdateChannelStateParams err = p2.UnmarshalCBOR(bytes.NewReader(lastCall.Params)) require.NoError(t, err) require.Equal(t, proof, p2.Proof) @@ -911,7 +843,7 @@ func TestSubmitVoucher(t *testing.T) { // Check that the secret and proof were passed through correctly msg := s.mock.pushedMessages(submitCid) - var p paych.UpdateChannelStateParams + var p v0paych.UpdateChannelStateParams err = p.UnmarshalCBOR(bytes.NewReader(msg.Message.Params)) require.NoError(t, err) require.Equal(t, submitProof, p.Proof) @@ -931,7 +863,7 @@ func TestSubmitVoucher(t *testing.T) { require.NoError(t, err) msg = s.mock.pushedMessages(submitCid) - var p2 paych.UpdateChannelStateParams + var p2 v0paych.UpdateChannelStateParams err = p2.UnmarshalCBOR(bytes.NewReader(msg.Message.Params)) require.NoError(t, err) require.Equal(t, addVoucherProof2, p2.Proof) @@ -947,7 +879,7 @@ func TestSubmitVoucher(t *testing.T) { require.NoError(t, err) msg = s.mock.pushedMessages(submitCid) - var p3 paych.UpdateChannelStateParams + var p3 v0paych.UpdateChannelStateParams err = p3.UnmarshalCBOR(bytes.NewReader(msg.Message.Params)) require.NoError(t, err) require.Equal(t, proof3, p3.Proof) @@ -986,10 +918,8 @@ func testSetupMgrWithChannel(ctx context.Context, t *testing.T) *testScaffold { toAcct := tutils.NewActorAddr(t, "toAct") mock := newMockManagerAPI() - arr, err := adt.MakeEmptyArray(mock.store).Root() - require.NoError(t, err) - mock.setAccountState(fromAcct, account.State{Address: from}) - mock.setAccountState(toAcct, account.State{Address: to}) + mock.setAccountAddress(fromAcct, from) + mock.setAccountAddress(toAcct, to) // Create channel in state balance := big.NewInt(20) @@ -999,14 +929,7 @@ func testSetupMgrWithChannel(ctx context.Context, t *testing.T) *testScaffold { Nonce: 0, Balance: balance, } - mock.setPaychState(ch, act, paych.State{ - From: fromAcct, - To: toAcct, - ToSend: big.NewInt(0), - SettlingAt: abi.ChainEpoch(0), - MinSettleHeight: abi.ChainEpoch(0), - LaneStates: arr, - }) + mock.setPaychState(ch, act, paychmock.NewMockPayChState(fromAcct, toAcct, abi.ChainEpoch(0), big.NewInt(0), make(map[uint64]paych.LaneState))) store := NewStore(ds_sync.MutexWrap(ds.NewMapDatastore())) mgr, err := newManager(store, mock) @@ -1043,8 +966,8 @@ func testGenerateKeyPair(t *testing.T) ([]byte, []byte) { return priv, pub } -func createTestVoucher(t *testing.T, ch address.Address, voucherLane uint64, nonce uint64, voucherAmount big.Int, key []byte) *paych.SignedVoucher { - sv := &paych.SignedVoucher{ +func createTestVoucher(t *testing.T, ch address.Address, voucherLane uint64, nonce uint64, voucherAmount big.Int, key []byte) *v0paych.SignedVoucher { + sv := &v0paych.SignedVoucher{ ChannelAddr: ch, Lane: voucherLane, Nonce: nonce, @@ -1059,13 +982,13 @@ func createTestVoucher(t *testing.T, ch address.Address, voucherLane uint64, non return sv } -func createTestVoucherWithExtra(t *testing.T, ch address.Address, voucherLane uint64, nonce uint64, voucherAmount big.Int, key []byte) *paych.SignedVoucher { - sv := &paych.SignedVoucher{ +func createTestVoucherWithExtra(t *testing.T, ch address.Address, voucherLane uint64, nonce uint64, voucherAmount big.Int, key []byte) *v0paych.SignedVoucher { + sv := &v0paych.SignedVoucher{ ChannelAddr: ch, Lane: voucherLane, Nonce: nonce, Amount: voucherAmount, - Extra: &paych.ModVerifyParams{ + Extra: &v0paych.ModVerifyParams{ Actor: tutils.NewActorAddr(t, "act"), }, } @@ -1083,13 +1006,13 @@ type mockBestSpendableAPI struct { mgr *Manager } -func (m *mockBestSpendableAPI) PaychVoucherList(ctx context.Context, ch address.Address) ([]*paych.SignedVoucher, error) { +func (m *mockBestSpendableAPI) PaychVoucherList(ctx context.Context, ch address.Address) ([]*v0paych.SignedVoucher, error) { vi, err := m.mgr.ListVouchers(ctx, ch) if err != nil { return nil, err } - out := make([]*paych.SignedVoucher, len(vi)) + out := make([]*v0paych.SignedVoucher, len(vi)) for k, v := range vi { out[k] = v.Voucher } @@ -1097,7 +1020,7 @@ func (m *mockBestSpendableAPI) PaychVoucherList(ctx context.Context, ch address. return out, nil } -func (m *mockBestSpendableAPI) PaychVoucherCheckSpendable(ctx context.Context, ch address.Address, voucher *paych.SignedVoucher, secret []byte, proof []byte) (bool, error) { +func (m *mockBestSpendableAPI) PaychVoucherCheckSpendable(ctx context.Context, ch address.Address, voucher *v0paych.SignedVoucher, secret []byte, proof []byte) (bool, error) { return m.mgr.CheckVoucherSpendable(ctx, ch, voucher, secret, proof) } diff --git a/paychmgr/paychget_test.go b/paychmgr/paychget_test.go index 1f3e4c396..93233c54f 100644 --- a/paychmgr/paychget_test.go +++ b/paychmgr/paychget_test.go @@ -6,29 +6,22 @@ import ( "testing" "time" - "github.com/filecoin-project/specs-actors/actors/builtin/account" - "github.com/filecoin-project/specs-actors/actors/util/adt" - - "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/specs-actors/actors/builtin/paych" - cborrpc "github.com/filecoin-project/go-cbor-util" - - init_ "github.com/filecoin-project/specs-actors/actors/builtin/init" - - "github.com/filecoin-project/specs-actors/actors/builtin" - - "github.com/filecoin-project/lotus/chain/types" - - "github.com/filecoin-project/go-address" - - "github.com/filecoin-project/go-state-types/big" - tutils "github.com/filecoin-project/specs-actors/support/testing" "github.com/ipfs/go-cid" ds "github.com/ipfs/go-datastore" ds_sync "github.com/ipfs/go-datastore/sync" - "github.com/stretchr/testify/require" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/specs-actors/actors/builtin" + init_ "github.com/filecoin-project/specs-actors/actors/builtin/init" + tutils "github.com/filecoin-project/specs-actors/support/testing" + + "github.com/filecoin-project/lotus/chain/actors/builtin/paych" + paychmock "github.com/filecoin-project/lotus/chain/actors/builtin/paych/mock" + "github.com/filecoin-project/lotus/chain/types" ) func testChannelResponse(t *testing.T, ch address.Address) types.MessageReceipt { @@ -976,25 +969,15 @@ func TestPaychAvailableFunds(t *testing.T) { require.EqualValues(t, 0, av.VoucherReedeemedAmt.Int64()) // Create channel in state - arr, err := adt.MakeEmptyArray(mock.store).Root() - require.NoError(t, err) - mock.setAccountState(fromAcct, account.State{Address: from}) - mock.setAccountState(toAcct, account.State{Address: to}) + mock.setAccountAddress(fromAcct, from) + mock.setAccountAddress(toAcct, to) act := &types.Actor{ Code: builtin.AccountActorCodeID, Head: cid.Cid{}, Nonce: 0, Balance: createAmt, } - mock.setPaychState(ch, act, paych.State{ - From: fromAcct, - To: toAcct, - ToSend: big.NewInt(0), - SettlingAt: abi.ChainEpoch(0), - MinSettleHeight: abi.ChainEpoch(0), - LaneStates: arr, - }) - + mock.setPaychState(ch, act, paychmock.NewMockPayChState(fromAcct, toAcct, abi.ChainEpoch(0), big.NewInt(0), make(map[uint64]paych.LaneState))) // Send create channel response response := testChannelResponse(t, ch) mock.receiveMsgResponse(createMsgCid, response) diff --git a/paychmgr/simple.go b/paychmgr/simple.go index 4cf579a47..2f6dc0593 100644 --- a/paychmgr/simple.go +++ b/paychmgr/simple.go @@ -6,20 +6,17 @@ import ( "fmt" "sync" - "github.com/filecoin-project/lotus/api" - - "golang.org/x/sync/errgroup" - - "github.com/filecoin-project/go-state-types/big" - - "github.com/filecoin-project/specs-actors/actors/builtin" - init_ "github.com/filecoin-project/specs-actors/actors/builtin/init" - "github.com/filecoin-project/specs-actors/actors/builtin/paych" "github.com/ipfs/go-cid" + "golang.org/x/sync/errgroup" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/specs-actors/actors/builtin" + init_ "github.com/filecoin-project/specs-actors/actors/builtin/init" + v0paych "github.com/filecoin-project/specs-actors/actors/builtin/paych" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/types" @@ -320,7 +317,7 @@ func (ca *channelAccessor) currentAvailableFunds(channelID string, queuedAmt typ } for _, ls := range laneStates { - totalRedeemed = types.BigAdd(totalRedeemed, ls.Redeemed) + totalRedeemed = types.BigAdd(totalRedeemed, ls.Redeemed()) } } @@ -385,7 +382,7 @@ func (ca *channelAccessor) processTask(ctx context.Context, amt types.BigInt) *p // createPaych sends a message to create the channel and returns the message cid func (ca *channelAccessor) createPaych(ctx context.Context, amt types.BigInt) (cid.Cid, error) { - params, aerr := actors.SerializeParams(&paych.ConstructorParams{From: ca.from, To: ca.to}) + params, aerr := actors.SerializeParams(&v0paych.ConstructorParams{From: ca.from, To: ca.to}) if aerr != nil { return cid.Undef, aerr } diff --git a/paychmgr/state.go b/paychmgr/state.go index 00fe2adce..2571ef73e 100644 --- a/paychmgr/state.go +++ b/paychmgr/state.go @@ -3,13 +3,9 @@ package paychmgr import ( "context" - "github.com/filecoin-project/specs-actors/actors/util/adt" - - "github.com/filecoin-project/specs-actors/actors/builtin/account" - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/specs-actors/actors/builtin/paych" + "github.com/filecoin-project/lotus/chain/actors/builtin/paych" "github.com/filecoin-project/lotus/chain/types" ) @@ -17,14 +13,8 @@ type stateAccessor struct { sm stateManagerAPI } -func (ca *stateAccessor) loadPaychActorState(ctx context.Context, ch address.Address) (*types.Actor, *paych.State, error) { - var pcast paych.State - act, err := ca.sm.LoadActorState(ctx, ch, &pcast, nil) - if err != nil { - return nil, nil, err - } - - return act, &pcast, nil +func (ca *stateAccessor) loadPaychActorState(ctx context.Context, ch address.Address) (*types.Actor, paych.State, error) { + return ca.sm.GetPaychState(ctx, ch, nil) } func (ca *stateAccessor) loadStateChannelInfo(ctx context.Context, ch address.Address, dir uint64) (*ChannelInfo, error) { @@ -33,17 +23,15 @@ func (ca *stateAccessor) loadStateChannelInfo(ctx context.Context, ch address.Ad return nil, err } - var account account.State - _, err = ca.sm.LoadActorState(ctx, st.From, &account, nil) + // Load channel "From" account actor state + from, err := ca.sm.ResolveToKeyAddress(ctx, st.From(), nil) if err != nil { return nil, err } - from := account.Address - _, err = ca.sm.LoadActorState(ctx, st.To, &account, nil) + to, err := ca.sm.ResolveToKeyAddress(ctx, st.To(), nil) if err != nil { return nil, err } - to := account.Address nextLane, err := ca.nextLaneFromState(ctx, st) if err != nil { @@ -67,25 +55,24 @@ func (ca *stateAccessor) loadStateChannelInfo(ctx context.Context, ch address.Ad return ci, nil } -func (ca *stateAccessor) nextLaneFromState(ctx context.Context, st *paych.State) (uint64, error) { - store := ca.sm.AdtStore(ctx) - laneStates, err := adt.AsArray(store, st.LaneStates) +func (ca *stateAccessor) nextLaneFromState(ctx context.Context, st paych.State) (uint64, error) { + laneCount, err := st.LaneCount() if err != nil { return 0, err } - if laneStates.Length() == 0 { + if laneCount == 0 { return 0, nil } - maxID := int64(0) - if err := laneStates.ForEach(nil, func(i int64) error { - if i > maxID { - maxID = i + maxID := uint64(0) + if err := st.ForEachLaneState(func(idx uint64, _ paych.LaneState) error { + if idx > maxID { + maxID = idx } return nil }); err != nil { return 0, err } - return uint64(maxID + 1), nil + return maxID + 1, nil } From b4ee51928233bcc6cacedbc185498e45630cb192 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Wed, 16 Sep 2020 01:00:00 -0400 Subject: [PATCH 145/885] Partial progress towards switching to miner and power interfaces --- build/params_2k.go | 4 +- build/params_testnet.go | 4 +- chain/actors/builtin/power/power.go | 1 + chain/actors/builtin/power/v0.go | 28 ++++++++++++-- chain/gen/gen_test.go | 4 +- chain/gen/genesis/miners.go | 16 ++++---- chain/gen/genesis/t04_power.go | 4 +- chain/gen/genesis/util.go | 1 + chain/stmgr/forks_test.go | 8 ++-- chain/stmgr/stmgr.go | 2 +- chain/stmgr/utils.go | 59 +---------------------------- chain/store/store_test.go | 4 +- chain/store/weight.go | 17 ++++++--- chain/sync.go | 19 +++++----- chain/sync_test.go | 4 +- chain/vectors/gen/main.go | 4 +- chain/vm/invoker.go | 4 +- cli/paych_test.go | 4 +- 18 files changed, 81 insertions(+), 106 deletions(-) diff --git a/build/params_2k.go b/build/params_2k.go index 0ef1d9b34..313ccfa9d 100644 --- a/build/params_2k.go +++ b/build/params_2k.go @@ -6,7 +6,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/specs-actors/actors/builtin/miner" - "github.com/filecoin-project/specs-actors/actors/builtin/power" + v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" ) @@ -20,7 +20,7 @@ var DrandSchedule = map[abi.ChainEpoch]DrandEnum{ } func init() { - power.ConsensusMinerMinPower = big.NewInt(2048) + v0power.ConsensusMinerMinPower = big.NewInt(2048) miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } diff --git a/build/params_testnet.go b/build/params_testnet.go index 932ad7a7d..4cfd8a9b6 100644 --- a/build/params_testnet.go +++ b/build/params_testnet.go @@ -9,7 +9,7 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/builtin/miner" - "github.com/filecoin-project/specs-actors/actors/builtin/power" + v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" ) var DrandSchedule = map[abi.ChainEpoch]DrandEnum{ @@ -23,7 +23,7 @@ const BreezeGasTampingDuration = 120 const UpgradeSmokeHeight = 51000 func init() { - power.ConsensusMinerMinPower = big.NewInt(10 << 40) + v0power.ConsensusMinerMinPower = big.NewInt(10 << 40) miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ abi.RegisteredSealProof_StackedDrg32GiBV1: {}, abi.RegisteredSealProof_StackedDrg64GiBV1: {}, diff --git a/chain/actors/builtin/power/power.go b/chain/actors/builtin/power/power.go index 07399e1bf..65148f0a5 100644 --- a/chain/actors/builtin/power/power.go +++ b/chain/actors/builtin/power/power.go @@ -37,6 +37,7 @@ type State interface { MinerPower(address.Address) (Claim, bool, error) MinerNominalPowerMeetsConsensusMinimum(address.Address) (bool, error) + ListAllMiners() ([]address.Address, error) } type Claim struct { diff --git a/chain/actors/builtin/power/v0.go b/chain/actors/builtin/power/v0.go index 45dd570f6..5cf6920c8 100644 --- a/chain/actors/builtin/power/v0.go +++ b/chain/actors/builtin/power/v0.go @@ -4,12 +4,12 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/builtin/power" + v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" "github.com/filecoin-project/specs-actors/actors/util/adt" ) type v0State struct { - power.State + v0power.State store adt.Store } @@ -29,7 +29,7 @@ func (s *v0State) MinerPower(addr address.Address) (Claim, bool, error) { if err != nil { return Claim{}, false, err } - var claim power.Claim + var claim v0power.Claim ok, err := claims.Get(abi.AddrKey(addr), &claim) if err != nil { return Claim{}, false, err @@ -47,3 +47,25 @@ func (s *v0State) MinerNominalPowerMeetsConsensusMinimum(a address.Address) (boo func (s *v0State) TotalPowerSmoothed() (builtin.FilterEstimate, error) { return *s.State.ThisEpochQAPowerSmoothed, nil } + +func (s *v0State) ListAllMiners() ([]address.Address, error) { + claims, err := adt.AsMap(s.store, s.Claims) + if err != nil { + return nil, err + } + + var miners []address.Address + err = claims.ForEach(nil, func(k string) error { + a, err := address.NewFromBytes([]byte(k)) + if err != nil { + return err + } + miners = append(miners, a) + return nil + }) + if err != nil { + return nil, err + } + + return miners, nil +} diff --git a/chain/gen/gen_test.go b/chain/gen/gen_test.go index ebc28a990..fd6bceb95 100644 --- a/chain/gen/gen_test.go +++ b/chain/gen/gen_test.go @@ -6,7 +6,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/specs-actors/actors/builtin/miner" - "github.com/filecoin-project/specs-actors/actors/builtin/power" + v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" _ "github.com/filecoin-project/lotus/lib/sigs/bls" @@ -17,7 +17,7 @@ func init() { miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } - power.ConsensusMinerMinPower = big.NewInt(2048) + v0power.ConsensusMinerMinPower = big.NewInt(2048) verifreg.MinVerifiedDealSize = big.NewInt(256) } diff --git a/chain/gen/genesis/miners.go b/chain/gen/genesis/miners.go index 3bed1c1a9..88fe57189 100644 --- a/chain/gen/genesis/miners.go +++ b/chain/gen/genesis/miners.go @@ -19,7 +19,7 @@ import ( "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/builtin/market" "github.com/filecoin-project/specs-actors/actors/builtin/miner" - "github.com/filecoin-project/specs-actors/actors/builtin/power" + v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" "github.com/filecoin-project/specs-actors/actors/builtin/reward" "github.com/filecoin-project/specs-actors/actors/runtime" @@ -99,7 +99,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid } { - constructorParams := &power.CreateMinerParams{ + constructorParams := &v0power.CreateMinerParams{ Owner: m.Worker, Worker: m.Worker, Peer: []byte(m.PeerId), @@ -112,7 +112,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid return cid.Undef, xerrors.Errorf("failed to create genesis miner: %w", err) } - var ma power.CreateMinerReturn + var ma v0power.CreateMinerReturn if err := ma.UnmarshalCBOR(bytes.NewReader(rval)); err != nil { return cid.Undef, xerrors.Errorf("unmarshaling CreateMinerReturn: %w", err) } @@ -207,7 +207,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid } } - err = vm.MutateState(ctx, builtin.StoragePowerActorAddr, func(cst cbor.IpldStore, st *power.State) error { + err = vm.MutateState(ctx, builtin.StoragePowerActorAddr, func(cst cbor.IpldStore, st *v0power.State) error { st.TotalQualityAdjPower = qaPow st.TotalRawBytePower = rawPow @@ -249,7 +249,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid sectorWeight := miner.QAPowerForWeight(m.SectorSize, minerInfos[i].presealExp, dweight.DealWeight, dweight.VerifiedDealWeight) // we've added fake power for this sector above, remove it now - err = vm.MutateState(ctx, builtin.StoragePowerActorAddr, func(cst cbor.IpldStore, st *power.State) error { + err = vm.MutateState(ctx, builtin.StoragePowerActorAddr, func(cst cbor.IpldStore, st *v0power.State) error { st.TotalQualityAdjPower = types.BigSub(st.TotalQualityAdjPower, sectorWeight) //nolint:scopelint st.TotalRawBytePower = types.BigSub(st.TotalRawBytePower, types.NewInt(uint64(m.SectorSize))) return nil @@ -301,7 +301,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid } // Sanity-check total network power - err = vm.MutateState(ctx, builtin.StoragePowerActorAddr, func(cst cbor.IpldStore, st *power.State) error { + err = vm.MutateState(ctx, builtin.StoragePowerActorAddr, func(cst cbor.IpldStore, st *v0power.State) error { if !st.TotalRawBytePower.Equals(rawPow) { return xerrors.Errorf("st.TotalRawBytePower doesn't match previously calculated rawPow") } @@ -340,12 +340,12 @@ func (fr *fakeRand) GetBeaconRandomness(ctx context.Context, personalization cry return out, nil } -func currentTotalPower(ctx context.Context, vm *vm.VM, maddr address.Address) (*power.CurrentTotalPowerReturn, error) { +func currentTotalPower(ctx context.Context, vm *vm.VM, maddr address.Address) (*v0power.CurrentTotalPowerReturn, error) { pwret, err := doExecValue(ctx, vm, builtin.StoragePowerActorAddr, maddr, big.Zero(), builtin.MethodsPower.CurrentTotalPower, nil) if err != nil { return nil, err } - var pwr power.CurrentTotalPowerReturn + var pwr v0power.CurrentTotalPowerReturn if err := pwr.UnmarshalCBOR(bytes.NewReader(pwret)); err != nil { return nil, err } diff --git a/chain/gen/genesis/t04_power.go b/chain/gen/genesis/t04_power.go index 86ba684e0..40ea68079 100644 --- a/chain/gen/genesis/t04_power.go +++ b/chain/gen/genesis/t04_power.go @@ -6,7 +6,7 @@ import ( "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/util/adt" - "github.com/filecoin-project/specs-actors/actors/builtin/power" + v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" cbor "github.com/ipfs/go-ipld-cbor" "github.com/filecoin-project/lotus/chain/types" @@ -30,7 +30,7 @@ func SetupStoragePowerActor(bs bstore.Blockstore) (*types.Actor, error) { return nil, err } - sms := power.ConstructState(emptyMap, emptyMultiMap) + sms := v0power.ConstructState(emptyMap, emptyMultiMap) stcid, err := store.Put(store.Context(), sms) if err != nil { diff --git a/chain/gen/genesis/util.go b/chain/gen/genesis/util.go index d87500206..bcafb007e 100644 --- a/chain/gen/genesis/util.go +++ b/chain/gen/genesis/util.go @@ -2,6 +2,7 @@ package genesis import ( "context" + "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/build" diff --git a/chain/stmgr/forks_test.go b/chain/stmgr/forks_test.go index 2fc11b3d7..d9d0af745 100644 --- a/chain/stmgr/forks_test.go +++ b/chain/stmgr/forks_test.go @@ -12,7 +12,7 @@ import ( "github.com/filecoin-project/specs-actors/actors/builtin" init_ "github.com/filecoin-project/specs-actors/actors/builtin/init" "github.com/filecoin-project/specs-actors/actors/builtin/miner" - "github.com/filecoin-project/specs-actors/actors/builtin/power" + v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" "github.com/filecoin-project/specs-actors/actors/runtime" "golang.org/x/xerrors" @@ -36,7 +36,7 @@ func init() { miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } - power.ConsensusMinerMinPower = big.NewInt(2048) + v0power.ConsensusMinerMinPower = big.NewInt(2048) verifreg.MinVerifiedDealSize = big.NewInt(256) } @@ -148,8 +148,8 @@ func TestForkHeightTriggers(t *testing.T) { } inv.Register(builtin.PaymentChannelActorCodeID, &testActor{}, &testActorState{}) - sm.SetVMConstructor(func(vmopt *vm.VMOpts) (*vm.VM, error) { - nvm, err := vm.NewVM(vmopt) + sm.SetVMConstructor(func(ctx context.Context, vmopt *vm.VMOpts) (*vm.VM, error) { + nvm, err := vm.NewVM(ctx, vmopt) if err != nil { return nil, err } diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index f71b788c8..827f23920 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -197,7 +197,7 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp for i := parentEpoch; i < epoch; i++ { // handle state forks - // XXX: The state tre + // XXX: The state tree err = sm.handleStateForks(ctx, vmi.StateTree(), i, ts) if err != nil { return cid.Undef, cid.Undef, xerrors.Errorf("error handling state forks: %w", err) diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index 1bbc9ccc4..e295725f8 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -13,7 +13,6 @@ import ( "github.com/filecoin-project/specs-actors/actors/runtime/proof" cid "github.com/ipfs/go-cid" - cbor "github.com/ipfs/go-ipld-cbor" cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" @@ -42,7 +41,6 @@ import ( "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" - "github.com/filecoin-project/lotus/lib/blockstore" "github.com/filecoin-project/lotus/node/modules/dtypes" ) @@ -326,65 +324,12 @@ func ListMinerActors(ctx context.Context, sm *StateManager, ts *types.TipSet) ([ return nil, xerrors.Errorf("failed to load power actor: %w", err) } - state, err := power.Load(sm.cs.Store(ctx), act) + powState, err := power.Load(sm.cs.Store(ctx), act) if err != nil { return nil, xerrors.Errorf("failed to load power actor state: %w", err) } - m, err := adt.AsMap(sm.cs.Store(ctx), state.Claims) - if err != nil { - return nil, err - } - - var miners []address.Address - err = m.ForEach(nil, func(k string) error { - a, err := address.NewFromBytes([]byte(k)) - if err != nil { - return err - } - miners = append(miners, a) - return nil - }) - if err != nil { - return nil, err - } - - return miners, nil -} - -func LoadSectorsFromSet(ctx context.Context, bs blockstore.Blockstore, ssc cid.Cid, filter *bitfield.BitField, filterOut bool) ([]*miner.ChainSectorInfo, error) { - a, err := adt.AsArray(store.ActorStore(ctx, bs), ssc) - if err != nil { - return nil, err - } - - var sset []*miner.ChainSectorInfo - var v cbg.Deferred - if err := a.ForEach(&v, func(i int64) error { - if filter != nil { - set, err := filter.IsSet(uint64(i)) - if err != nil { - return xerrors.Errorf("filter check error: %w", err) - } - if set == filterOut { - return nil - } - } - - var oci miner.SectorOnChainInfo - if err := cbor.DecodeInto(v.Raw, &oci); err != nil { - return err - } - sset = append(sset, &miner.ChainSectorInfo{ - Info: oci, - ID: abi.SectorNumber(i), - }) - return nil - }); err != nil { - return nil, err - } - - return sset, nil + return powState.ListAllMiners() } func ComputeState(ctx context.Context, sm *StateManager, height abi.ChainEpoch, msgs []*types.Message, ts *types.TipSet) (cid.Cid, []*api.InvocResult, error) { diff --git a/chain/store/store_test.go b/chain/store/store_test.go index e56bab4c9..8662f10ef 100644 --- a/chain/store/store_test.go +++ b/chain/store/store_test.go @@ -11,7 +11,7 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/specs-actors/actors/builtin/miner" - "github.com/filecoin-project/specs-actors/actors/builtin/power" + v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" "github.com/filecoin-project/lotus/chain/gen" @@ -25,7 +25,7 @@ func init() { miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } - power.ConsensusMinerMinPower = big.NewInt(2048) + v0power.ConsensusMinerMinPower = big.NewInt(2048) verifreg.MinVerifiedDealSize = big.NewInt(256) } diff --git a/chain/store/weight.go b/chain/store/weight.go index 5249df011..2d83738c5 100644 --- a/chain/store/weight.go +++ b/chain/store/weight.go @@ -4,12 +4,13 @@ import ( "context" "math/big" + "github.com/filecoin-project/lotus/chain/actors/builtin/power" + big2 "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/builtin/power" cbor "github.com/ipfs/go-ipld-cbor" "golang.org/x/xerrors" ) @@ -39,11 +40,17 @@ func (cs *ChainStore) Weight(ctx context.Context, ts *types.TipSet) (types.BigIn return types.NewInt(0), xerrors.Errorf("get power actor: %w", err) } - var st power.State - if err := cst.Get(ctx, act.Head, &st); err != nil { - return types.NewInt(0), xerrors.Errorf("get power actor head (%s, height=%d): %w", act.Head, ts.Height(), err) + powState, err := power.Load(cs.Store(ctx), act) + if err != nil { + return types.NewInt(0), xerrors.Errorf("failed to load power actor state: %w", err) } - tpow = st.TotalQualityAdjPower // TODO: REVIEW: Is this correct? + + claim, err := powState.TotalPower() + if err != nil { + return types.NewInt(0), xerrors.Errorf("failed to get total power: %w", err) + } + + tpow = claim.QualityAdjPower // TODO: REVIEW: Is this correct? } log2P := int64(0) diff --git a/chain/sync.go b/chain/sync.go index bb3e50bdb..c16821889 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -34,12 +34,12 @@ import ( "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/builtin/power" "github.com/filecoin-project/specs-actors/actors/util/adt" blst "github.com/supranational/blst/bindings/go" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors/builtin/power" "github.com/filecoin-project/lotus/chain/beacon" "github.com/filecoin-project/lotus/chain/exchange" "github.com/filecoin-project/lotus/chain/gen" @@ -635,26 +635,25 @@ func (syncer *Syncer) ValidateTipSet(ctx context.Context, fts *store.FullTipSet) } func (syncer *Syncer) minerIsValid(ctx context.Context, maddr address.Address, baseTs *types.TipSet) error { - var spast power.State - - _, err := syncer.sm.LoadActorState(ctx, builtin.StoragePowerActorAddr, &spast, baseTs) + act, err := syncer.sm.LoadActor(ctx, builtin.StoragePowerActorAddr, baseTs) if err != nil { - return err + return xerrors.Errorf("failed to load power actor: %w", err) } - cm, err := adt.AsMap(syncer.store.Store(ctx), spast.Claims) + powState, err := power.Load(syncer.store.Store(ctx), act) if err != nil { - return err + return xerrors.Errorf("failed to load power actor state: %w", err) } - var claim power.Claim - exist, err := cm.Get(abi.AddrKey(maddr), &claim) + _, exist, err := powState.MinerPower(maddr) if err != nil { - return err + return xerrors.Errorf("failed to look up miner's claim: %w", err) } + if !exist { return xerrors.New("miner isn't valid") } + return nil } diff --git a/chain/sync_test.go b/chain/sync_test.go index 0b0d1ed00..5f972ecc6 100644 --- a/chain/sync_test.go +++ b/chain/sync_test.go @@ -21,7 +21,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/specs-actors/actors/builtin/miner" - "github.com/filecoin-project/specs-actors/actors/builtin/power" + v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" "github.com/filecoin-project/lotus/api" @@ -46,7 +46,7 @@ func init() { miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } - power.ConsensusMinerMinPower = big.NewInt(2048) + v0power.ConsensusMinerMinPower = big.NewInt(2048) verifreg.MinVerifiedDealSize = big.NewInt(256) } diff --git a/chain/vectors/gen/main.go b/chain/vectors/gen/main.go index ecc2498b9..4bf2c420e 100644 --- a/chain/vectors/gen/main.go +++ b/chain/vectors/gen/main.go @@ -6,7 +6,7 @@ import ( "math/rand" "os" - "github.com/filecoin-project/specs-actors/actors/builtin/power" + v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" "github.com/filecoin-project/go-address" "golang.org/x/xerrors" @@ -27,7 +27,7 @@ import ( func init() { verifreg.MinVerifiedDealSize = big.NewInt(2048) - power.ConsensusMinerMinPower = big.NewInt(2048) + v0power.ConsensusMinerMinPower = big.NewInt(2048) } func MakeHeaderVectors() []vectors.HeaderVector { diff --git a/chain/vm/invoker.go b/chain/vm/invoker.go index 9f5f8e2d9..56e3bea84 100644 --- a/chain/vm/invoker.go +++ b/chain/vm/invoker.go @@ -23,7 +23,7 @@ import ( "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/specs-actors/actors/builtin/multisig" "github.com/filecoin-project/specs-actors/actors/builtin/paych" - "github.com/filecoin-project/specs-actors/actors/builtin/power" + v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" "github.com/filecoin-project/specs-actors/actors/builtin/reward" "github.com/filecoin-project/specs-actors/actors/builtin/system" "github.com/filecoin-project/specs-actors/actors/runtime" @@ -50,7 +50,7 @@ func NewInvoker() *Invoker { inv.Register(builtin.InitActorCodeID, init_.Actor{}, init_.State{}) inv.Register(builtin.RewardActorCodeID, reward.Actor{}, reward.State{}) inv.Register(builtin.CronActorCodeID, cron.Actor{}, cron.State{}) - inv.Register(builtin.StoragePowerActorCodeID, power.Actor{}, power.State{}) + inv.Register(builtin.StoragePowerActorCodeID, v0power.Actor{}, v0power.State{}) inv.Register(builtin.StorageMarketActorCodeID, market.Actor{}, market.State{}) inv.Register(builtin.StorageMinerActorCodeID, miner.Actor{}, miner.State{}) inv.Register(builtin.MultisigActorCodeID, multisig.Actor{}, multisig.State{}) diff --git a/cli/paych_test.go b/cli/paych_test.go index cccc80ff4..0e5bc0096 100644 --- a/cli/paych_test.go +++ b/cli/paych_test.go @@ -16,7 +16,7 @@ import ( "github.com/filecoin-project/go-state-types/big" saminer "github.com/filecoin-project/specs-actors/actors/builtin/miner" - "github.com/filecoin-project/specs-actors/actors/builtin/power" + v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" "github.com/multiformats/go-multiaddr" @@ -39,7 +39,7 @@ import ( ) func init() { - power.ConsensusMinerMinPower = big.NewInt(2048) + v0power.ConsensusMinerMinPower = big.NewInt(2048) saminer.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } From 90853e24cfd9e3ef749855fbdf319a821f4403aa Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Wed, 16 Sep 2020 01:47:24 -0400 Subject: [PATCH 146/885] Add a boolean HasMinPower to return of GetPower --- api/api_full.go | 5 +++-- chain/stmgr/utils.go | 32 ++++++++++++++++---------------- chain/sync.go | 2 +- cmd/lotus-shed/balances.go | 3 +-- node/impl/full/state.go | 3 ++- 5 files changed, 23 insertions(+), 22 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 88496d669..fbbf92d8b 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -600,8 +600,9 @@ type VoucherCreateResult struct { } type MinerPower struct { - MinerPower power.Claim - TotalPower power.Claim + MinerPower power.Claim + TotalPower power.Claim + HasMinPower bool } type QueryOffer struct { diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index e295725f8..e58d69156 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -79,37 +79,42 @@ func GetMinerWorkerRaw(ctx context.Context, sm *StateManager, st cid.Cid, maddr return vm.ResolveToKeyAddr(state, sm.cs.Store(ctx), info.Worker) } -func GetPower(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) (power.Claim, power.Claim, error) { +func GetPower(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) (power.Claim, power.Claim, bool, error) { return GetPowerRaw(ctx, sm, ts.ParentState(), maddr) } -func GetPowerRaw(ctx context.Context, sm *StateManager, st cid.Cid, maddr address.Address) (power.Claim, power.Claim, error) { +func GetPowerRaw(ctx context.Context, sm *StateManager, st cid.Cid, maddr address.Address) (power.Claim, power.Claim, bool, error) { act, err := sm.LoadActorRaw(ctx, builtin.StoragePowerActorAddr, st) if err != nil { - return power.Claim{}, power.Claim{}, xerrors.Errorf("(get sset) failed to load power actor state: %w", err) + return power.Claim{}, power.Claim{}, false, xerrors.Errorf("(get sset) failed to load power actor state: %w", err) } - mas, err := power.Load(sm.cs.Store(ctx), act) + pas, err := power.Load(sm.cs.Store(ctx), act) if err != nil { - return power.Claim{}, power.Claim{}, err + return power.Claim{}, power.Claim{}, false, err } - tpow, err := mas.TotalPower() + tpow, err := pas.TotalPower() if err != nil { - return power.Claim{}, power.Claim{}, err + return power.Claim{}, power.Claim{}, false, err } var mpow power.Claim if maddr != address.Undef { var found bool - mpow, found, err = mas.MinerPower(maddr) + mpow, found, err = pas.MinerPower(maddr) if err != nil || !found { // TODO: return an error when not found? - return power.Claim{}, power.Claim{}, err + return power.Claim{}, power.Claim{}, false, err } } - return mpow, tpow, nil + minpow, err := pas.MinerNominalPowerMeetsConsensusMinimum(maddr) + if err != nil { + return power.Claim{}, power.Claim{}, false, err + } + + return mpow, tpow, minpow, nil } func PreCommitInfo(ctx context.Context, sm *StateManager, maddr address.Address, sid abi.SectorNumber, ts *types.TipSet) (*miner.SectorPreCommitOnChainInfo, error) { @@ -470,7 +475,7 @@ func MinerGetBaseInfo(ctx context.Context, sm *StateManager, bcs beacon.Schedule return nil, nil } - mpow, tpow, err := GetPowerRaw(ctx, sm, lbst, maddr) + mpow, tpow, hmp, err := GetPowerRaw(ctx, sm, lbst, maddr) if err != nil { return nil, xerrors.Errorf("failed to get power: %w", err) } @@ -485,11 +490,6 @@ func MinerGetBaseInfo(ctx context.Context, sm *StateManager, bcs beacon.Schedule return nil, xerrors.Errorf("resolving worker address: %w", err) } - hmp, err := MinerHasMinPower(ctx, sm, maddr, lbts) - if err != nil { - return nil, xerrors.Errorf("determining if miner has min power failed: %w", err) - } - return &api.MiningBaseInfo{ MinerPower: mpow.QualityAdjPower, NetworkPower: tpow.QualityAdjPower, diff --git a/chain/sync.go b/chain/sync.go index c16821889..1d4e1f02b 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -853,7 +853,7 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) (er return xerrors.Errorf("received block was from slashed or invalid miner") } - mpow, tpow, err := stmgr.GetPowerRaw(ctx, syncer.sm, lbst, h.Miner) + mpow, tpow, _, err := stmgr.GetPowerRaw(ctx, syncer.sm, lbst, h.Miner) if err != nil { return xerrors.Errorf("failed getting power: %w", err) } diff --git a/cmd/lotus-shed/balances.go b/cmd/lotus-shed/balances.go index 7dbfe2eb7..248f3b26e 100644 --- a/cmd/lotus-shed/balances.go +++ b/cmd/lotus-shed/balances.go @@ -12,7 +12,6 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" - "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/store" @@ -193,7 +192,7 @@ var chainBalanceStateCmd = &cli.Command{ } if act.Code == builtin.StorageMinerActorCodeID && minerInfo { - pow, _, err := stmgr.GetPowerRaw(ctx, sm, sroot, addr) + pow, _, _, err := stmgr.GetPowerRaw(ctx, sm, sroot, addr) if err != nil { return xerrors.Errorf("failed to get power: %w", err) } diff --git a/node/impl/full/state.go b/node/impl/full/state.go index d69ba0a41..a19edb1ca 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -253,7 +253,7 @@ func (a *StateAPI) StateMinerPower(ctx context.Context, addr address.Address, ts return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - m, net, err := stmgr.GetPower(ctx, a.StateManager, ts, addr) + m, net, hmp, err := stmgr.GetPower(ctx, a.StateManager, ts, addr) if err != nil { return nil, err } @@ -261,6 +261,7 @@ func (a *StateAPI) StateMinerPower(ctx context.Context, addr address.Address, ts return &api.MinerPower{ MinerPower: m, TotalPower: net, + HasMinPower: hmp, }, nil } From 72d19f369bf5ed9430c2ee374e1a15fb65ba88a8 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Wed, 16 Sep 2020 01:49:33 -0400 Subject: [PATCH 147/885] Incremental progress towards using new power state interface --- cmd/lotus-storage-miner/info.go | 3 +- cmd/lotus-storage-miner/init.go | 6 ++-- node/node_test.go | 4 +-- tools/stats/metrics.go | 50 +++++---------------------------- 4 files changed, 13 insertions(+), 50 deletions(-) diff --git a/cmd/lotus-storage-miner/info.go b/cmd/lotus-storage-miner/info.go index c47a22b0e..3a7720c52 100644 --- a/cmd/lotus-storage-miner/info.go +++ b/cmd/lotus-storage-miner/info.go @@ -17,7 +17,6 @@ import ( "github.com/filecoin-project/go-state-types/abi" sealing "github.com/filecoin-project/lotus/extern/storage-sealing" "github.com/filecoin-project/specs-actors/actors/builtin/miner" - "github.com/filecoin-project/specs-actors/actors/builtin/power" "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/lotus/api" @@ -129,7 +128,7 @@ func infoCmdAct(cctx *cli.Context) error { faultyPercentage) } - if pow.MinerPower.RawBytePower.LessThan(power.ConsensusMinerMinPower) { + if !pow.HasMinPower { fmt.Print("Below minimum power threshold, no blocks will be won") } else { expWinChance := float64(types.BigMul(qpercI, types.NewInt(build.BlocksPerEpoch)).Int64()) / 1000000 diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index e2a2419f3..8325aae8b 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -34,7 +34,7 @@ import ( "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/builtin/market" miner2 "github.com/filecoin-project/specs-actors/actors/builtin/miner" - "github.com/filecoin-project/specs-actors/actors/builtin/power" + v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" lapi "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" @@ -633,7 +633,7 @@ func createStorageMiner(ctx context.Context, api lapi.FullNode, peerid peer.ID, return address.Undef, err } - params, err := actors.SerializeParams(&power.CreateMinerParams{ + params, err := actors.SerializeParams(&v0power.CreateMinerParams{ Owner: owner, Worker: worker, SealProofType: spt, @@ -681,7 +681,7 @@ func createStorageMiner(ctx context.Context, api lapi.FullNode, peerid peer.ID, return address.Undef, xerrors.Errorf("create miner failed: exit code %d", mw.Receipt.ExitCode) } - var retval power.CreateMinerReturn + var retval v0power.CreateMinerReturn if err := retval.UnmarshalCBOR(bytes.NewReader(mw.Receipt.Return)); err != nil { return address.Undef, err } diff --git a/node/node_test.go b/node/node_test.go index 8cc51f629..0611bca60 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -11,7 +11,7 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/lotus/lib/lotuslog" saminer "github.com/filecoin-project/specs-actors/actors/builtin/miner" - "github.com/filecoin-project/specs-actors/actors/builtin/power" + v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" logging "github.com/ipfs/go-log/v2" @@ -21,7 +21,7 @@ import ( func init() { _ = logging.SetLogLevel("*", "INFO") - power.ConsensusMinerMinPower = big.NewInt(2048) + v0power.ConsensusMinerMinPower = big.NewInt(2048) saminer.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } diff --git a/tools/stats/metrics.go b/tools/stats/metrics.go index ae79d9273..4f11d2476 100644 --- a/tools/stats/metrics.go +++ b/tools/stats/metrics.go @@ -10,14 +10,11 @@ import ( "strings" "time" - "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/builtin/power" - "github.com/filecoin-project/specs-actors/actors/util/adt" "golang.org/x/xerrors" "github.com/ipfs/go-cid" @@ -255,55 +252,22 @@ func RecordTipsetStatePoints(ctx context.Context, api api.FullNode, pl *PointLis p := NewPoint("network.balance", netBalFilFloat) pl.AddPoint(p) - totalPower, err := api.StateMinerPower(ctx, address.Address{}, tipset.Key()) + miners, err := api.StateListMiners(ctx, tipset.Key()) if err != nil { return err } - p = NewPoint("chain.power", totalPower.TotalPower.QualityAdjPower.Int64()) - pl.AddPoint(p) - - powerActor, err := api.StateGetActor(ctx, builtin.StoragePowerActorAddr, tipset.Key()) - if err != nil { - return err - } - - powerRaw, err := api.ChainReadObj(ctx, powerActor.Head) - if err != nil { - return err - } - - var powerActorState power.State - - if err := powerActorState.UnmarshalCBOR(bytes.NewReader(powerRaw)); err != nil { - return fmt.Errorf("failed to unmarshal power actor state: %w", err) - } - - s := &apiIpldStore{ctx, api} - mp, err := adt.AsMap(s, powerActorState.Claims) - if err != nil { - return err - } - - var claim power.Claim - err = mp.ForEach(&claim, func(key string) error { - addr, err := address.NewFromBytes([]byte(key)) + for _, addr := range miners { + mp, err := api.StateMinerPower(ctx, addr, tipset.Key()) if err != nil { return err } - if claim.QualityAdjPower.Int64() == 0 { - return nil + if !mp.MinerPower.QualityAdjPower.IsZero() { + p = NewPoint("chain.miner_power", mp.MinerPower.QualityAdjPower.Int64()) + p.AddTag("miner", addr.String()) + pl.AddPoint(p) } - - p = NewPoint("chain.miner_power", claim.QualityAdjPower.Int64()) - p.AddTag("miner", addr.String()) - pl.AddPoint(p) - - return nil - }) - if err != nil { - return err } return nil From 362fc180ec5ea053047a04ad39984cbfdcc5cc88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Wed, 16 Sep 2020 10:53:15 +0100 Subject: [PATCH 148/885] fix a regression caused by #3800. --- chain/vm/burn.go | 19 ++++++++++++------- chain/vm/vm.go | 30 +++++++++++++++--------------- 2 files changed, 27 insertions(+), 22 deletions(-) diff --git a/chain/vm/burn.go b/chain/vm/burn.go index eb0611349..9f9b95755 100644 --- a/chain/vm/burn.go +++ b/chain/vm/burn.go @@ -22,6 +22,17 @@ type GasOutputs struct { GasBurned int64 } +// ZeroGasOutputs returns a logically zeroed GasOutputs. +func ZeroGasOutputs() GasOutputs { + return GasOutputs{ + BaseFeeBurn: big.Zero(), + OverEstimationBurn: big.Zero(), + MinerPenalty: big.Zero(), + MinerTip: big.Zero(), + Refund: big.Zero(), + } +} + // ComputeGasOverestimationBurn computes amount of gas to be refunded and amount of gas to be burned // Result is (refund, burn) func ComputeGasOverestimationBurn(gasUsed, gasLimit int64) (int64, int64) { @@ -58,13 +69,7 @@ func ComputeGasOverestimationBurn(gasUsed, gasLimit int64) (int64, int64) { func ComputeGasOutputs(gasUsed, gasLimit int64, baseFee, feeCap, gasPremium abi.TokenAmount) GasOutputs { gasUsedBig := big.NewInt(gasUsed) - out := GasOutputs{ - BaseFeeBurn: big.Zero(), - OverEstimationBurn: big.Zero(), - MinerPenalty: big.Zero(), - MinerTip: big.Zero(), - Refund: big.Zero(), - } + out := ZeroGasOutputs() baseFeeToPay := baseFee if baseFee.Cmp(feeCap.Int) > 0 { diff --git a/chain/vm/vm.go b/chain/vm/vm.go index f245bb2b0..a90856a1b 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -355,14 +355,14 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, msgGasCost := msgGas.Total() // this should never happen, but is currently still exercised by some tests if msgGasCost > msg.GasLimit { + gasOutputs := ZeroGasOutputs() + gasOutputs.MinerPenalty = types.BigMul(vm.baseFee, abi.NewTokenAmount(msgGasCost)) return &ApplyRet{ MessageReceipt: types.MessageReceipt{ ExitCode: exitcode.SysErrOutOfGas, GasUsed: 0, }, - GasCosts: GasOutputs{ - MinerPenalty: types.BigMul(vm.baseFee, abi.NewTokenAmount(msgGasCost)), - }, + GasCosts: gasOutputs, Duration: time.Since(start), }, nil } @@ -374,15 +374,15 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, // this should never happen, but is currently still exercised by some tests if err != nil { if xerrors.Is(err, types.ErrActorNotFound) { + gasOutputs := ZeroGasOutputs() + gasOutputs.MinerPenalty = minerPenaltyAmount return &ApplyRet{ MessageReceipt: types.MessageReceipt{ ExitCode: exitcode.SysErrSenderInvalid, GasUsed: 0, }, ActorErr: aerrors.Newf(exitcode.SysErrSenderInvalid, "actor not found: %s", msg.From), - GasCosts: GasOutputs{ - MinerPenalty: minerPenaltyAmount, - }, + GasCosts: gasOutputs, Duration: time.Since(start), }, nil } @@ -391,20 +391,22 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, // this should never happen, but is currently still exercised by some tests if !fromActor.Code.Equals(builtin.AccountActorCodeID) { + gasOutputs := ZeroGasOutputs() + gasOutputs.MinerPenalty = minerPenaltyAmount return &ApplyRet{ MessageReceipt: types.MessageReceipt{ ExitCode: exitcode.SysErrSenderInvalid, GasUsed: 0, }, ActorErr: aerrors.Newf(exitcode.SysErrSenderInvalid, "send from not account actor: %s", fromActor.Code), - GasCosts: GasOutputs{ - MinerPenalty: minerPenaltyAmount, - }, + GasCosts: gasOutputs, Duration: time.Since(start), }, nil } if msg.Nonce != fromActor.Nonce { + gasOutputs := ZeroGasOutputs() + gasOutputs.MinerPenalty = minerPenaltyAmount return &ApplyRet{ MessageReceipt: types.MessageReceipt{ ExitCode: exitcode.SysErrSenderStateInvalid, @@ -413,15 +415,15 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, ActorErr: aerrors.Newf(exitcode.SysErrSenderStateInvalid, "actor nonce invalid: msg:%d != state:%d", msg.Nonce, fromActor.Nonce), - GasCosts: GasOutputs{ - MinerPenalty: minerPenaltyAmount, - }, + GasCosts: gasOutputs, Duration: time.Since(start), }, nil } gascost := types.BigMul(types.NewInt(uint64(msg.GasLimit)), msg.GasFeeCap) if fromActor.Balance.LessThan(gascost) { + gasOutputs := ZeroGasOutputs() + gasOutputs.MinerPenalty = minerPenaltyAmount return &ApplyRet{ MessageReceipt: types.MessageReceipt{ ExitCode: exitcode.SysErrSenderStateInvalid, @@ -429,9 +431,7 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, }, ActorErr: aerrors.Newf(exitcode.SysErrSenderStateInvalid, "actor balance less than needed: %s < %s", types.FIL(fromActor.Balance), types.FIL(gascost)), - GasCosts: GasOutputs{ - MinerPenalty: minerPenaltyAmount, - }, + GasCosts: gasOutputs, Duration: time.Since(start), }, nil } From d870c74a5e22115517f8fb74905001b014f97ea8 Mon Sep 17 00:00:00 2001 From: Frank Date: Wed, 16 Sep 2020 19:08:47 +0800 Subject: [PATCH 149/885] add lotus-pcr to ignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 940f37b3f..05f762d8d 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ /lotus-stats /lotus-bench /lotus-gateway +/lotus-pcr /bench.json /lotuspond/front/node_modules /lotuspond/front/build From b4115021c5063ec5c12a44f3b4d62c055f79d8e5 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Tue, 15 Sep 2020 15:02:14 +0200 Subject: [PATCH 150/885] make sure lotus compiles with +testground build flag --- tools/stats/metrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/stats/metrics.go b/tools/stats/metrics.go index ae79d9273..e50ac953f 100644 --- a/tools/stats/metrics.go +++ b/tools/stats/metrics.go @@ -189,7 +189,7 @@ func RecordTipsetPoints(ctx context.Context, api api.FullNode, pl *PointList, ti pl.AddPoint(p) } { - blks := len(cids) + blks := int64(len(cids)) p = NewPoint("chain.gas_fill_ratio", float64(totalGasLimit)/float64(blks*build.BlockGasTarget)) pl.AddPoint(p) p = NewPoint("chain.gas_capacity_ratio", float64(totalUniqGasLimit)/float64(blks*build.BlockGasTarget)) From ed74091c204c417a1f82ec3f6260e32f8042bb9f Mon Sep 17 00:00:00 2001 From: Frank Date: Wed, 16 Sep 2020 19:49:45 +0800 Subject: [PATCH 151/885] add exist sector state check --- cmd/lotus-storage-miner/sectors.go | 6 +++++ extern/storage-sealing/sector_state.go | 33 ++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/cmd/lotus-storage-miner/sectors.go b/cmd/lotus-storage-miner/sectors.go index 06f09fe20..c08d6ca14 100644 --- a/cmd/lotus-storage-miner/sectors.go +++ b/cmd/lotus-storage-miner/sectors.go @@ -17,6 +17,8 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" + + sealing "github.com/filecoin-project/lotus/extern/storage-sealing" lcli "github.com/filecoin-project/lotus/cli" ) @@ -420,6 +422,10 @@ var sectorsUpdateCmd = &cli.Command{ return xerrors.Errorf("could not parse sector number: %w", err) } + if _, ok := sealing.ExistSectorStateList[sealing.SectorState(cctx.Args().Get(1))]; !ok { + return xerrors.Errorf("Not existing sector state") + } + return nodeApi.SectorsUpdate(ctx, abi.SectorNumber(id), api.SectorState(cctx.Args().Get(1))) }, } diff --git a/extern/storage-sealing/sector_state.go b/extern/storage-sealing/sector_state.go index 4e674603d..3ed891064 100644 --- a/extern/storage-sealing/sector_state.go +++ b/extern/storage-sealing/sector_state.go @@ -2,6 +2,8 @@ package sealing type SectorState string +var ExistSectorStateList = make(map[SectorState]struct{}) + const ( UndefinedSectorState SectorState = "" @@ -39,6 +41,37 @@ const ( RemoveFailed SectorState = "RemoveFailed" Removed SectorState = "Removed" ) +func init() { + ExistSectorStateList[Empty] = struct{}{} + ExistSectorStateList[WaitDeals] = struct{}{} + ExistSectorStateList[Packing] = struct{}{} + ExistSectorStateList[PreCommit1] = struct{}{} + ExistSectorStateList[PreCommit2] = struct{}{} + ExistSectorStateList[PreCommitting] = struct{}{} + ExistSectorStateList[PreCommitWait] = struct{}{} + ExistSectorStateList[WaitSeed] = struct{}{} + ExistSectorStateList[Committing] = struct{}{} + ExistSectorStateList[SubmitCommit] = struct{}{} + ExistSectorStateList[CommitWait] = struct{}{} + ExistSectorStateList[FinalizeSector] = struct{}{} + ExistSectorStateList[Proving] = struct{}{} + ExistSectorStateList[FailedUnrecoverable] = struct{}{} + ExistSectorStateList[SealPreCommit1Failed] = struct{}{} + ExistSectorStateList[SealPreCommit2Failed] = struct{}{} + ExistSectorStateList[PreCommitFailed] = struct{}{} + ExistSectorStateList[ComputeProofFailed] = struct{}{} + ExistSectorStateList[CommitFailed] = struct{}{} + ExistSectorStateList[PackingFailed] = struct{}{} + ExistSectorStateList[FinalizeFailed] = struct{}{} + ExistSectorStateList[DealsExpired] = struct{}{} + ExistSectorStateList[RecoverDealIDs] = struct{}{} + ExistSectorStateList[Faulty] = struct{}{} + ExistSectorStateList[FaultReported] = struct{}{} + ExistSectorStateList[FaultedFinal] = struct{}{} + ExistSectorStateList[Removing] = struct{}{} + ExistSectorStateList[RemoveFailed] = struct{}{} + ExistSectorStateList[Removed] = struct{}{} +} func toStatState(st SectorState) statSectorState { switch st { From 3f67c4524e8bbdae123e1ee50cf603899ce5cde0 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Tue, 15 Sep 2020 11:42:52 +0200 Subject: [PATCH 152/885] add filecoin-project oni as submodule and compile lotus-soup --- .circleci/config.yml | 19 +++++++++++++++++++ .gitmodules | 3 +++ extern/oni | 1 + 3 files changed, 23 insertions(+) create mode 160000 extern/oni diff --git a/.circleci/config.yml b/.circleci/config.yml index 4fdb3e38f..5e2ef6eab 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -254,6 +254,25 @@ jobs: path: /tmp/test-reports - store_artifacts: path: /tmp/test-artifacts/conformance-coverage.html + test-lotus-soup: + description: | + Compile `lotus-soup` Testground test plan using the current version of Lotus. + parameters: + <<: *test-params + executor: << parameters.executor >> + steps: + - install-deps + - prepare + - run: cd extern/oni && git submodule sync + - run: cd extern/oni && git submodule update --init + - run: cd extern/oni/extra/filecoin-ffi && make + - run: + name: "update lotus-soup dependency to lotus" + command: pushd extern/oni/lotus-soup && echo 'replace github.com/filecoin-project/lotus => ../../../' >> go.mod + - run: + name: "build lotus-soup testplan" + command: pushd extern/oni/lotus-soup && go build -tags=testground . + build-macos: description: build darwin lotus binary diff --git a/.gitmodules b/.gitmodules index 4b450aaf3..35f5a3d3f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -11,3 +11,6 @@ [submodule "extern/fil-blst"] path = extern/fil-blst url = https://github.com/filecoin-project/fil-blst.git +[submodule "extern/oni"] + path = extern/oni + url = https://github.com/filecoin-project/oni diff --git a/extern/oni b/extern/oni new file mode 160000 index 000000000..d9bcdfe61 --- /dev/null +++ b/extern/oni @@ -0,0 +1 @@ +Subproject commit d9bcdfe61b89f00bef07f6d3995e36e277930acd From f4f80b25d11d53ca69ceb285dfefb748f72a2b41 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 16 Sep 2020 12:40:08 +0200 Subject: [PATCH 153/885] add test-lotus-soup to workflows --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5e2ef6eab..b2340c836 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -447,6 +447,7 @@ workflows: test-suite-name: conformance-bleeding-edge packages: "./conformance" vectors-branch: master + - test-lotus-soup - build-debug - build-all: requires: From b0677ab38e675f66eb05213aa6a98bb56fc7c45b Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 16 Sep 2020 12:49:45 +0200 Subject: [PATCH 154/885] replace lotus and filecoin-ffi deps --- .circleci/config.yml | 6 +++--- extern/oni | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b2340c836..468812ccc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -265,10 +265,10 @@ jobs: - prepare - run: cd extern/oni && git submodule sync - run: cd extern/oni && git submodule update --init - - run: cd extern/oni/extra/filecoin-ffi && make + - run: cd extern/filecoin-ffi && make - run: - name: "update lotus-soup dependency to lotus" - command: pushd extern/oni/lotus-soup && echo 'replace github.com/filecoin-project/lotus => ../../../' >> go.mod + name: "replace lotus and filecoin-ffi deps" + command: pushd extern/oni/lotus-soup && go mod edit -replace github.com/filecoin-project/lotus=../../../ && go mod edit -replace github.com/filecoin-project/filecoin-ffi=../../filecoin-ffi - run: name: "build lotus-soup testplan" command: pushd extern/oni/lotus-soup && go build -tags=testground . diff --git a/extern/oni b/extern/oni index d9bcdfe61..097d82535 160000 --- a/extern/oni +++ b/extern/oni @@ -1 +1 @@ -Subproject commit d9bcdfe61b89f00bef07f6d3995e36e277930acd +Subproject commit 097d82535de690160fb84e97d0baf1bd04e175c6 From 0ff5a71114622a0109f682d048b81f6f7fcc4461 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 16 Sep 2020 15:13:28 +0200 Subject: [PATCH 155/885] update oni to master --- extern/oni | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/oni b/extern/oni index 097d82535..8b7e7d438 160000 --- a/extern/oni +++ b/extern/oni @@ -1 +1 @@ -Subproject commit 097d82535de690160fb84e97d0baf1bd04e175c6 +Subproject commit 8b7e7d438c4cc38a0d2d671876d4590ad20655b3 From 56f223bfc3ee736212a4125cfd8c36b56af03e09 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 16 Sep 2020 15:16:15 +0200 Subject: [PATCH 156/885] add go mod edit for blst deps --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 468812ccc..fff35d7a8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -267,8 +267,8 @@ jobs: - run: cd extern/oni && git submodule update --init - run: cd extern/filecoin-ffi && make - run: - name: "replace lotus and filecoin-ffi deps" - command: pushd extern/oni/lotus-soup && go mod edit -replace github.com/filecoin-project/lotus=../../../ && go mod edit -replace github.com/filecoin-project/filecoin-ffi=../../filecoin-ffi + name: "replace lotus, filecoin-ffi, blst and fil-blst deps" + command: cd extern/oni/lotus-soup && go mod edit -replace github.com/filecoin-project/lotus=../../../ && go mod edit -replace github.com/filecoin-project/filecoin-ffi=../../filecoin-ffi && go mod edit -replace github.com/supranational/blst=../../fil-blst/blst && go mod edit -replace github.com/filecoin-project/fil-blst=../../fil-blst - run: name: "build lotus-soup testplan" command: pushd extern/oni/lotus-soup && go build -tags=testground . From 6656e5a6c836fdd762fc198772bba99bdd7cff46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 16 Sep 2020 22:47:06 +0200 Subject: [PATCH 157/885] circle: test-lotus-soup -> build-lotus-soup --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fff35d7a8..acd447f69 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -254,7 +254,7 @@ jobs: path: /tmp/test-reports - store_artifacts: path: /tmp/test-artifacts/conformance-coverage.html - test-lotus-soup: + build-lotus-soup: description: | Compile `lotus-soup` Testground test plan using the current version of Lotus. parameters: @@ -447,7 +447,7 @@ workflows: test-suite-name: conformance-bleeding-edge packages: "./conformance" vectors-branch: master - - test-lotus-soup + - build-lotus-soup - build-debug - build-all: requires: From a6b7791d325fec21798540aee420e6511fda2734 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 16 Sep 2020 22:56:04 +0200 Subject: [PATCH 158/885] lint --- miner/miner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/miner/miner.go b/miner/miner.go index 555fe2e19..2d8591992 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -139,7 +139,7 @@ func (m *Miner) niceSleep(d time.Duration) bool { case <-build.Clock.After(d): return true case <-m.stop: - log.Infow("recieved interrupt while trying to sleep in mining cycle") + log.Infow("received interrupt while trying to sleep in mining cycle") return false } } From 665d23cb8e72f661d55beb1f390216e4bcf3c7f4 Mon Sep 17 00:00:00 2001 From: Ingar Shu Date: Tue, 15 Sep 2020 12:10:22 -0700 Subject: [PATCH 159/885] Add set-ask, get-ask retrieval CLI --- cmd/lotus-storage-miner/retrieval-deals.go | 114 +++++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/cmd/lotus-storage-miner/retrieval-deals.go b/cmd/lotus-storage-miner/retrieval-deals.go index df194978d..0f15f19f2 100644 --- a/cmd/lotus-storage-miner/retrieval-deals.go +++ b/cmd/lotus-storage-miner/retrieval-deals.go @@ -5,9 +5,12 @@ import ( "os" "text/tabwriter" + "github.com/docker/go-units" "github.com/filecoin-project/go-fil-markets/retrievalmarket" + "github.com/filecoin-project/go-state-types/abi" "github.com/urfave/cli/v2" + "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" ) @@ -17,6 +20,8 @@ var retrievalDealsCmd = &cli.Command{ Subcommands: []*cli.Command{ retrievalDealSelectionCmd, retrievalDealsListCmd, + retrievalSetAskCmd, + retrievalGetAskCmd, }, } @@ -154,3 +159,112 @@ var retrievalDealsListCmd = &cli.Command{ return w.Flush() }, } + +var retrievalSetAskCmd = &cli.Command{ + Name: "set-ask", + Usage: "Configure the provider's retrieval ask", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "price", + Usage: "Set the price of the ask for retrievals (per byte)", + }, + &cli.StringFlag{ + Name: "unseal-price", + Usage: "Set the price to unseal", + }, + &cli.StringFlag{ + Name: "payment-interval", + Usage: "Set the payment interval (in bytes) for retrieval", + DefaultText: "1Mb", + }, + &cli.StringFlag{ + Name: "payment-interval-increase", + Usage: "Set the payment interval increase (in bytes) for retrieval", + DefaultText: "1Mb", + }, + }, + Action: func(cctx *cli.Context) error { + ctx := lcli.DaemonContext(cctx) + + api, closer, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return err + } + defer closer() + + ask, err := api.MarketGetRetrievalAsk(ctx) + if err != nil { + return err + } + + if cctx.IsSet("price") { + v, err := types.ParseFIL(cctx.String("price")) + if err != nil { + return err + } + ask.PricePerByte = abi.TokenAmount(v) + } + + if cctx.IsSet("unseal-price") { + v, err := types.ParseFIL(cctx.String("unseal-price")) + if err != nil { + return err + } + ask.UnsealPrice = abi.TokenAmount(v) + } + + if cctx.IsSet("payment-interval") { + v, err := units.RAMInBytes(cctx.String("payment-interval")) + if err != nil { + return err + } + ask.PaymentInterval = uint64(v) + } + + if cctx.IsSet("payment-interval-increase") { + v, err := units.RAMInBytes(cctx.String("payment-interval-increase")) + if err != nil { + return err + } + ask.PaymentIntervalIncrease = uint64(v) + } + + return api.MarketSetRetrievalAsk(ctx, ask) + }, +} + +var retrievalGetAskCmd = &cli.Command{ + Name: "get-ask", + Usage: "Get the provider's current retrieval ask", + Flags: []cli.Flag{}, + Action: func(cctx *cli.Context) error { + ctx := lcli.DaemonContext(cctx) + + api, closer, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return err + } + defer closer() + + ask, err := api.MarketGetRetrievalAsk(ctx) + if err != nil { + return err + } + + w := tabwriter.NewWriter(os.Stdout, 2, 4, 2, ' ', 0) + fmt.Fprintf(w, "Price per Byte\tUnseal Price\tPayment Interval\tPayment Interval Increase\n") + if ask == nil { + fmt.Fprintf(w, "\n") + return w.Flush() + } + + fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", + types.FIL(ask.PricePerByte), + types.FIL(ask.UnsealPrice), + units.BytesSize(float64(ask.PaymentInterval)), + units.BytesSize(float64(ask.PaymentIntervalIncrease)), + ) + return w.Flush() + + }, +} From 2aef7f7c07445d64b0b4c7b9bf7b7b10898531f7 Mon Sep 17 00:00:00 2001 From: Ingar Shu Date: Wed, 16 Sep 2020 13:56:34 -0700 Subject: [PATCH 160/885] Clarify help text units --- cmd/lotus-storage-miner/retrieval-deals.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/lotus-storage-miner/retrieval-deals.go b/cmd/lotus-storage-miner/retrieval-deals.go index 0f15f19f2..08a0fea51 100644 --- a/cmd/lotus-storage-miner/retrieval-deals.go +++ b/cmd/lotus-storage-miner/retrieval-deals.go @@ -170,17 +170,17 @@ var retrievalSetAskCmd = &cli.Command{ }, &cli.StringFlag{ Name: "unseal-price", - Usage: "Set the price to unseal", + Usage: "Set the price to unseal (FIL/GiB)", }, &cli.StringFlag{ Name: "payment-interval", Usage: "Set the payment interval (in bytes) for retrieval", - DefaultText: "1Mb", + DefaultText: "1MiB", }, &cli.StringFlag{ Name: "payment-interval-increase", Usage: "Set the payment interval increase (in bytes) for retrieval", - DefaultText: "1Mb", + DefaultText: "1MiB", }, }, Action: func(cctx *cli.Context) error { From b530f25f0933b3cb2a311e401456b00c36dfc206 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Wed, 16 Sep 2020 17:20:25 -0400 Subject: [PATCH 161/885] Migrate miner actor --- api/test/window_post.go | 36 +++++++++++++++++++++-------- chain/actors/builtin/miner/miner.go | 1 + chain/actors/builtin/miner/v0.go | 27 ++++++++++++---------- chain/stmgr/forks_test.go | 4 ++-- node/impl/full/state.go | 1 - 5 files changed, 45 insertions(+), 24 deletions(-) diff --git a/api/test/window_post.go b/api/test/window_post.go index 3f15c754b..bdc390730 100644 --- a/api/test/window_post.go +++ b/api/test/window_post.go @@ -4,6 +4,13 @@ import ( "context" "fmt" + "github.com/filecoin-project/lotus/api/apibstore" + "github.com/filecoin-project/lotus/chain/actors/adt" + lotusminer "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + cbor "github.com/ipfs/go-ipld-cbor" + + v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" + "os" "strings" "testing" @@ -15,7 +22,6 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/extern/sector-storage/mock" sealing "github.com/filecoin-project/lotus/extern/storage-sealing" - miner2 "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" @@ -159,7 +165,7 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector head, err := client.ChainHead(ctx) require.NoError(t, err) - if head.Height() > di.PeriodStart+(miner2.WPoStProvingPeriod)+2 { + if head.Height() > di.PeriodStart+(v0miner.WPoStProvingPeriod)+2 { break } @@ -178,6 +184,14 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector require.Equal(t, p.MinerPower, p.TotalPower) require.Equal(t, p.MinerPower.RawBytePower, types.NewInt(uint64(ssz)*uint64(nSectors+GenesisPreseals))) + store := cbor.NewCborStore(apibstore.NewAPIBlockstore(client)) + + mact, err := client.StateGetActor(ctx, maddr, types.EmptyTSK) + require.NoError(t, err) + + minState, err := lotusminer.Load(adt.WrapStore(ctx, store), mact) + require.NoError(t, err) + fmt.Printf("Drop some sectors\n") // Drop 2 sectors from deadline 2 partition 0 (full partition / deadline) @@ -186,12 +200,14 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector require.NoError(t, err) require.Greater(t, len(parts), 0) - n, err := parts[0].Sectors.Count() + secs, err := parts[0].AllSectors() + require.NoError(t, err) + n, err := secs.Count() require.NoError(t, err) require.Equal(t, uint64(2), n) // Drop the partition - err = parts[0].Sectors.ForEach(func(sid uint64) error { + err = secs.ForEach(func(sid uint64) error { return miner.StorageMiner.(*impl.StorageMinerAPI).IStorageMgr.(*mock.SectorMgr).MarkCorrupted(abi.SectorID{ Miner: abi.ActorID(mid), Number: abi.SectorNumber(sid), @@ -208,15 +224,17 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector require.NoError(t, err) require.Greater(t, len(parts), 0) - n, err := parts[0].Sectors.Count() + secs, err := parts[0].AllSectors() + require.NoError(t, err) + n, err := secs.Count() require.NoError(t, err) require.Equal(t, uint64(2), n) // Drop the sector - sn, err := parts[0].Sectors.First() + sn, err := secs.First() require.NoError(t, err) - all, err := parts[0].Sectors.All(2) + all, err := secs.All(2) require.NoError(t, err) fmt.Println("the sectors", all) @@ -238,7 +256,7 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector head, err := client.ChainHead(ctx) require.NoError(t, err) - if head.Height() > di.PeriodStart+(miner2.WPoStProvingPeriod)+2 { + if head.Height() > di.PeriodStart+(minState.WpostProvingPeriod())+2 { break } @@ -268,7 +286,7 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector head, err := client.ChainHead(ctx) require.NoError(t, err) - if head.Height() > di.PeriodStart+(miner2.WPoStProvingPeriod)+2 { + if head.Height() > di.PeriodStart+(minState.WpostProvingPeriod())+2 { break } diff --git a/chain/actors/builtin/miner/miner.go b/chain/actors/builtin/miner/miner.go index 5ad8db39e..e436fdc69 100644 --- a/chain/actors/builtin/miner/miner.go +++ b/chain/actors/builtin/miner/miner.go @@ -46,6 +46,7 @@ type State interface { Info() (MinerInfo, error) DeadlineInfo(epoch abi.ChainEpoch) *dline.Info + WpostProvingPeriod() abi.ChainEpoch } type Deadline interface { diff --git a/chain/actors/builtin/miner/v0.go b/chain/actors/builtin/miner/v0.go index e1ef9c51f..446c828f1 100644 --- a/chain/actors/builtin/miner/v0.go +++ b/chain/actors/builtin/miner/v0.go @@ -14,21 +14,21 @@ import ( cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" + v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" ) type v0State struct { - miner.State + v0miner.State store adt.Store } type v0Deadline struct { - miner.Deadline + v0miner.Deadline store adt.Store } type v0Partition struct { - miner.Partition + v0miner.Partition store adt.Store } @@ -69,13 +69,13 @@ func (s *v0State) GetSectorExpiration(num abi.SectorNumber) (out *SectorExpirati // 2. If it's faulty, it will expire early within the first 14 entries // of the expiration queue. stopErr := errors.New("stop") - err = dls.ForEach(s.store, func(dlIdx uint64, dl *miner.Deadline) error { + err = dls.ForEach(s.store, func(dlIdx uint64, dl *v0miner.Deadline) error { partitions, err := dl.PartitionsArray(s.store) if err != nil { return err } quant := s.State.QuantSpecForDeadline(dlIdx) - var part miner.Partition + var part v0miner.Partition return partitions.ForEach(&part, func(partIdx int64) error { if found, err := part.Sectors.IsSet(uint64(num)); err != nil { return err @@ -89,11 +89,11 @@ func (s *v0State) GetSectorExpiration(num abi.SectorNumber) (out *SectorExpirati return stopErr } - q, err := miner.LoadExpirationQueue(s.store, part.EarlyTerminated, quant) + q, err := v0miner.LoadExpirationQueue(s.store, part.EarlyTerminated, quant) if err != nil { return err } - var exp miner.ExpirationSet + var exp v0miner.ExpirationSet return q.ForEach(&exp, func(epoch int64) error { if early, err := exp.EarlySectors.IsSet(uint64(num)); err != nil { return err @@ -151,7 +151,7 @@ func (s *v0State) LoadSectorsFromSet(filter *bitfield.BitField, filterOut bool) } } - var oci miner.SectorOnChainInfo + var oci v0miner.SectorOnChainInfo if err := oci.UnmarshalCBOR(bytes.NewReader(v.Raw)); err != nil { return err } @@ -184,13 +184,13 @@ func (s *v0State) ForEachDeadline(cb func(uint64, Deadline) error) error { if err != nil { return err } - return dls.ForEach(s.store, func(i uint64, dl *miner.Deadline) error { + return dls.ForEach(s.store, func(i uint64, dl *v0miner.Deadline) error { return cb(i, &v0Deadline{*dl, s.store}) }) } func (s *v0State) NumDeadlines() (uint64, error) { - return miner.WPoStPeriodDeadlines, nil + return v0miner.WPoStPeriodDeadlines, nil } func (s *v0State) Info() (MinerInfo, error) { @@ -230,6 +230,9 @@ func (s *v0State) Info() (MinerInfo, error) { func (s *v0State) DeadlineInfo(epoch abi.ChainEpoch) *dline.Info { return s.State.DeadlineInfo(epoch) } +func (s *v0State) WpostProvingPeriod() abi.ChainEpoch { + return v0miner.WPoStProvingPeriod +} func (d *v0Deadline) LoadPartition(idx uint64) (Partition, error) { p, err := d.Deadline.LoadPartition(d.store, idx) @@ -244,7 +247,7 @@ func (d *v0Deadline) ForEachPartition(cb func(uint64, Partition) error) error { if err != nil { return err } - var part miner.Partition + var part v0miner.Partition return ps.ForEach(&part, func(i int64) error { return cb(uint64(i), &v0Partition{part, d.store}) }) diff --git a/chain/stmgr/forks_test.go b/chain/stmgr/forks_test.go index d9d0af745..2db11e832 100644 --- a/chain/stmgr/forks_test.go +++ b/chain/stmgr/forks_test.go @@ -11,7 +11,7 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/specs-actors/actors/builtin" init_ "github.com/filecoin-project/specs-actors/actors/builtin/init" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" + v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" "github.com/filecoin-project/specs-actors/actors/runtime" @@ -33,7 +33,7 @@ import ( ) func init() { - miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ + v0miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } v0power.ConsensusMinerMinPower = big.NewInt(2048) diff --git a/node/impl/full/state.go b/node/impl/full/state.go index a19edb1ca..766f924bd 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -27,7 +27,6 @@ import ( "github.com/filecoin-project/specs-actors/actors/util/smoothing" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/actors/builtin/multisig" "github.com/filecoin-project/lotus/chain/actors/builtin/power" From f56602c5887864d88328df4374cb4a894e1c53f9 Mon Sep 17 00:00:00 2001 From: Ingar Shu Date: Wed, 16 Sep 2020 14:30:04 -0700 Subject: [PATCH 162/885] Specify retrieval price in FIL/GiB --- cmd/lotus-storage-miner/retrieval-deals.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/lotus-storage-miner/retrieval-deals.go b/cmd/lotus-storage-miner/retrieval-deals.go index 08a0fea51..03d397852 100644 --- a/cmd/lotus-storage-miner/retrieval-deals.go +++ b/cmd/lotus-storage-miner/retrieval-deals.go @@ -166,11 +166,11 @@ var retrievalSetAskCmd = &cli.Command{ Flags: []cli.Flag{ &cli.StringFlag{ Name: "price", - Usage: "Set the price of the ask for retrievals (per byte)", + Usage: "Set the price of the ask for retrievals (FIL/GiB)", }, &cli.StringFlag{ Name: "unseal-price", - Usage: "Set the price to unseal (FIL/GiB)", + Usage: "Set the price to unseal", }, &cli.StringFlag{ Name: "payment-interval", @@ -202,7 +202,7 @@ var retrievalSetAskCmd = &cli.Command{ if err != nil { return err } - ask.PricePerByte = abi.TokenAmount(v) + ask.PricePerByte = types.BigDiv(types.BigInt(v), types.NewInt(1<<30)) } if cctx.IsSet("unseal-price") { From 7115485b0acd78a79b90eb0b335c63b63e9256fe Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Wed, 16 Sep 2020 21:56:02 -0400 Subject: [PATCH 163/885] Add a getter for network version --- api/api_full.go | 4 ++++ api/apistruct/struct.go | 7 +++++++ node/impl/full/state.go | 10 ++++++++++ 3 files changed, 21 insertions(+) diff --git a/api/api_full.go b/api/api_full.go index fbbf92d8b..b472febe0 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -5,6 +5,8 @@ import ( "fmt" "time" + "github.com/filecoin-project/go-state-types/network" + "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p-core/peer" @@ -388,6 +390,8 @@ type FullNode interface { // StateCirculatingSupply returns the circulating supply of Filecoin at the given tipset StateCirculatingSupply(context.Context, types.TipSetKey) (CirculatingSupply, error) + // StateNetworkVersion returns the network version at the given tipset + StateNetworkVersion(context.Context, types.TipSetKey) (network.Version, error) // MethodGroup: Msig // The Msig methods are used to interact with multisig wallets on the diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 1fcd5e0f3..26f7a8708 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -5,6 +5,8 @@ import ( "io" "time" + stnetwork "github.com/filecoin-project/go-state-types/network" + "github.com/ipfs/go-cid" metrics "github.com/libp2p/go-libp2p-core/metrics" "github.com/libp2p/go-libp2p-core/network" @@ -198,6 +200,7 @@ type FullNodeStruct struct { StateVerifiedClientStatus func(context.Context, address.Address, types.TipSetKey) (*verifreg.DataCap, error) `perm:"read"` StateDealProviderCollateralBounds func(context.Context, abi.PaddedPieceSize, bool, types.TipSetKey) (api.DealCollateralBounds, error) `perm:"read"` StateCirculatingSupply func(context.Context, types.TipSetKey) (api.CirculatingSupply, error) `perm:"read"` + StateNetworkVersion func(context.Context, types.TipSetKey) (stnetwork.Version, error) `perm:"read"` MsigGetAvailableBalance func(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) `perm:"read"` MsigGetVested func(context.Context, address.Address, types.TipSetKey, types.TipSetKey) (types.BigInt, error) `perm:"read"` @@ -873,6 +876,10 @@ func (c *FullNodeStruct) StateCirculatingSupply(ctx context.Context, tsk types.T return c.Internal.StateCirculatingSupply(ctx, tsk) } +func (c *FullNodeStruct) StateNetworkVersion(ctx context.Context, tsk types.TipSetKey) (stnetwork.Version, error) { + return c.Internal.StateNetworkVersion(ctx, tsk) +} + func (c *FullNodeStruct) MsigGetAvailableBalance(ctx context.Context, a address.Address, tsk types.TipSetKey) (types.BigInt, error) { return c.Internal.MsigGetAvailableBalance(ctx, a, tsk) } diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 766f924bd..1b29a93a9 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "github.com/filecoin-project/go-state-types/network" "strconv" "github.com/filecoin-project/go-state-types/dline" @@ -1120,3 +1121,12 @@ func (a *StateAPI) StateCirculatingSupply(ctx context.Context, tsk types.TipSetK } return a.StateManager.GetCirculatingSupplyDetailed(ctx, ts.Height(), sTree) } + +func (a *StateAPI) StateNetworkVersion(ctx context.Context, tsk types.TipSetKey) (network.Version, error) { + ts, err := a.Chain.GetTipSetFromKey(tsk) + if err != nil { + return -1, xerrors.Errorf("loading tipset %s: %w", tsk, err) + } + + return a.StateManager.GetNtwkVersion(ctx, ts.Height()), nil +} From 80b6994fe2cfede39053c452a335fb6dd19c9122 Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Wed, 16 Sep 2020 19:13:12 -0700 Subject: [PATCH 164/885] feat(market): update state diffing for market actor Update to abstract actor for markets state diffing. Also move the diff adt functions inside the abstract actors --- .../{events/state => actors/adt}/diff_adt.go | 7 +- .../state => actors/adt}/diff_adt_test.go | 19 +- chain/actors/builtin/market/market.go | 52 +++++ chain/actors/builtin/market/v0.go | 201 +++++++++++++++- chain/events/state/predicates.go | 221 +++++------------- chain/events/state/predicates_test.go | 81 ++++--- cmd/lotus-chainwatch/processor/market.go | 7 +- 7 files changed, 373 insertions(+), 215 deletions(-) rename chain/{events/state => actors/adt}/diff_adt.go (94%) rename chain/{events/state => actors/adt}/diff_adt_test.go (96%) diff --git a/chain/events/state/diff_adt.go b/chain/actors/adt/diff_adt.go similarity index 94% rename from chain/events/state/diff_adt.go rename to chain/actors/adt/diff_adt.go index 519c61c2d..0784b77a2 100644 --- a/chain/events/state/diff_adt.go +++ b/chain/actors/adt/diff_adt.go @@ -1,10 +1,9 @@ -package state +package adt import ( "bytes" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/specs-actors/actors/util/adt" typegen "github.com/whyrusleeping/cbor-gen" ) @@ -27,7 +26,7 @@ type AdtArrayDiff interface { // - All values that exist in curArr nnd not in prevArr are passed to adtArrayDiff.Add() // - All values that exist in preArr and in curArr are passed to AdtArrayDiff.Modify() // - It is the responsibility of AdtArrayDiff.Modify() to determine if the values it was passed have been modified. -func DiffAdtArray(preArr, curArr *adt.Array, out AdtArrayDiff) error { +func DiffAdtArray(preArr, curArr Array, out AdtArrayDiff) error { prevVal := new(typegen.Deferred) if err := preArr.ForEach(prevVal, func(i int64) error { curVal := new(typegen.Deferred) @@ -76,7 +75,7 @@ type AdtMapDiff interface { Remove(key string, val *typegen.Deferred) error } -func DiffAdtMap(preMap, curMap *adt.Map, out AdtMapDiff) error { +func DiffAdtMap(preMap, curMap Map, out AdtMapDiff) error { prevVal := new(typegen.Deferred) if err := preMap.ForEach(prevVal, func(key string) error { curVal := new(typegen.Deferred) diff --git a/chain/events/state/diff_adt_test.go b/chain/actors/adt/diff_adt_test.go similarity index 96% rename from chain/events/state/diff_adt_test.go rename to chain/actors/adt/diff_adt_test.go index 55926afb9..436e28bbf 100644 --- a/chain/events/state/diff_adt_test.go +++ b/chain/actors/adt/diff_adt_test.go @@ -1,4 +1,4 @@ -package state +package adt import ( "bytes" @@ -13,7 +13,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/specs-actors/actors/runtime" - "github.com/filecoin-project/specs-actors/actors/util/adt" + v0adt "github.com/filecoin-project/specs-actors/actors/util/adt" bstore "github.com/filecoin-project/lotus/lib/blockstore" ) @@ -22,8 +22,8 @@ func TestDiffAdtArray(t *testing.T) { ctxstoreA := newContextStore() ctxstoreB := newContextStore() - arrA := adt.MakeEmptyArray(ctxstoreA) - arrB := adt.MakeEmptyArray(ctxstoreB) + arrA := v0adt.MakeEmptyArray(ctxstoreA) + arrB := v0adt.MakeEmptyArray(ctxstoreB) require.NoError(t, arrA.Set(0, runtime.CBORBytes([]byte{0}))) // delete @@ -76,8 +76,8 @@ func TestDiffAdtMap(t *testing.T) { ctxstoreA := newContextStore() ctxstoreB := newContextStore() - mapA := adt.MakeEmptyMap(ctxstoreA) - mapB := adt.MakeEmptyMap(ctxstoreB) + mapA := v0adt.MakeEmptyMap(ctxstoreA) + mapB := v0adt.MakeEmptyMap(ctxstoreB) require.NoError(t, mapA.Put(abi.UIntKey(0), runtime.CBORBytes([]byte{0}))) // delete @@ -292,12 +292,9 @@ func (t *TestDiffArray) Remove(key uint64, val *typegen.Deferred) error { return nil } -func newContextStore() *contextStore { +func newContextStore() Store { ctx := context.Background() bs := bstore.NewTemporarySync() store := cbornode.NewCborStore(bs) - return &contextStore{ - ctx: ctx, - cst: store, - } + return WrapStore(ctx, store) } diff --git a/chain/actors/builtin/market/market.go b/chain/actors/builtin/market/market.go index 99cca9879..051c46dbe 100644 --- a/chain/actors/builtin/market/market.go +++ b/chain/actors/builtin/market/market.go @@ -29,9 +29,14 @@ func Load(store adt.Store, act *types.Actor) (st State, err error) { type State interface { cbor.Marshaler + BalancesChanged(State) bool EscrowTable() (BalanceTable, error) LockedTable() (BalanceTable, error) TotalLocked() (abi.TokenAmount, error) + StatesChanged(State) bool + States() (DealStates, error) + ProposalsChanged(State) bool + Proposals() (DealProposals, error) VerifyDealsForActivation( minerAddr address.Address, deals []abi.DealID, currEpoch, sectorExpiry abi.ChainEpoch, ) (weight, verifiedWeight abi.DealWeight, err error) @@ -40,3 +45,50 @@ type State interface { type BalanceTable interface { Get(key address.Address) (abi.TokenAmount, error) } + +type DealStates interface { + GetDeal(key abi.DealID) (DealState, error) + Diff(DealStates) (*DealStateChanges, error) +} + +type DealProposals interface { + Diff(DealProposals) (*DealProposalChanges, error) +} + +type DealState interface { + SectorStartEpoch() abi.ChainEpoch + SlashEpoch() abi.ChainEpoch + LastUpdatedEpoch() abi.ChainEpoch + Equals(DealState) bool +} + +type DealProposal interface { +} + +type DealStateChanges struct { + Added []DealIDState + Modified []DealStateChange + Removed []DealIDState +} + +type DealIDState struct { + ID abi.DealID + Deal DealState +} + +// DealStateChange is a change in deal state from -> to +type DealStateChange struct { + ID abi.DealID + From DealState + To DealState +} + +type DealProposalChanges struct { + Added []ProposalIDState + Removed []ProposalIDState +} + +type ProposalIDState struct { + ID abi.DealID + Proposal DealProposal +} diff --git a/chain/actors/builtin/market/v0.go b/chain/actors/builtin/market/v0.go index fb67902da..8f0aa0594 100644 --- a/chain/actors/builtin/market/v0.go +++ b/chain/actors/builtin/market/v0.go @@ -1,11 +1,17 @@ package market import ( + "bytes" + "errors" + "fmt" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/specs-actors/actors/builtin/market" - "github.com/filecoin-project/specs-actors/actors/util/adt" + v0adt "github.com/filecoin-project/specs-actors/actors/util/adt" + typegen "github.com/whyrusleeping/cbor-gen" ) type v0State struct { @@ -19,12 +25,58 @@ func (s *v0State) TotalLocked() (abi.TokenAmount, error) { return fml, nil } +func (s *v0State) BalancesChanged(otherState State) bool { + v0otherState, ok := otherState.(*v0State) + if !ok { + // there's no way to compare differnt versions of the state, so let's + // just say that means the state of balances has changed + return true + } + return !s.State.EscrowTable.Equals(v0otherState.State.EscrowTable) || !s.State.LockedTable.Equals(v0otherState.State.LockedTable) +} + +func (s *v0State) StatesChanged(otherState State) bool { + v0otherState, ok := otherState.(*v0State) + if !ok { + // there's no way to compare differnt versions of the state, so let's + // just say that means the state of balances has changed + return true + } + return !s.State.States.Equals(v0otherState.State.States) +} + +func (s *v0State) States() (DealStates, error) { + stateArray, err := v0adt.AsArray(s.store, s.State.States) + if err != nil { + return nil, err + } + return &v0DealStates{stateArray}, nil +} + +func (s *v0State) ProposalsChanged(otherState State) bool { + v0otherState, ok := otherState.(*v0State) + if !ok { + // there's no way to compare differnt versions of the state, so let's + // just say that means the state of balances has changed + return true + } + return !s.State.Proposals.Equals(v0otherState.State.Proposals) +} + +func (s *v0State) Proposals() (DealProposals, error) { + proposalArray, err := v0adt.AsArray(s.store, s.State.Proposals) + if err != nil { + return nil, err + } + return &v0DealProposals{proposalArray}, nil +} + func (s *v0State) EscrowTable() (BalanceTable, error) { - return adt.AsBalanceTable(s.store, s.State.EscrowTable) + return v0adt.AsBalanceTable(s.store, s.State.EscrowTable) } func (s *v0State) LockedTable() (BalanceTable, error) { - return adt.AsBalanceTable(s.store, s.State.LockedTable) + return v0adt.AsBalanceTable(s.store, s.State.LockedTable) } func (s *v0State) VerifyDealsForActivation( @@ -32,3 +84,146 @@ func (s *v0State) VerifyDealsForActivation( ) (weight, verifiedWeight abi.DealWeight, err error) { return market.ValidateDealsForActivation(&s.State, s.store, deals, minerAddr, sectorExpiry, currEpoch) } + +type v0DealStates struct { + adt.Array +} + +func (s *v0DealStates) GetDeal(dealID abi.DealID) (DealState, error) { + var deal market.DealState + found, err := s.Array.Get(uint64(dealID), &deal) + if err != nil { + return nil, err + } + if !found { + return nil, nil + } + return &v0DealState{deal}, nil +} + +func (s *v0DealStates) Diff(other DealStates) (*DealStateChanges, error) { + v0other, ok := other.(*v0DealStates) + if !ok { + // TODO handle this if possible on a case by case basis but for now, just fail + return nil, errors.New("cannot compare deal states across versions") + } + results := new(DealStateChanges) + if err := adt.DiffAdtArray(s, v0other, &v0MarketStatesDiffer{results}); err != nil { + return nil, fmt.Errorf("diffing deal states: %w", err) + } + + return results, nil +} + +type v0MarketStatesDiffer struct { + Results *DealStateChanges +} + +func (d *v0MarketStatesDiffer) Add(key uint64, val *typegen.Deferred) error { + ds := new(v0DealState) + err := ds.UnmarshalCBOR(bytes.NewReader(val.Raw)) + if err != nil { + return err + } + d.Results.Added = append(d.Results.Added, DealIDState{abi.DealID(key), ds}) + return nil +} + +func (d *v0MarketStatesDiffer) Modify(key uint64, from, to *typegen.Deferred) error { + dsFrom := new(v0DealState) + if err := dsFrom.UnmarshalCBOR(bytes.NewReader(from.Raw)); err != nil { + return err + } + + dsTo := new(v0DealState) + if err := dsTo.UnmarshalCBOR(bytes.NewReader(to.Raw)); err != nil { + return err + } + + if *dsFrom != *dsTo { + d.Results.Modified = append(d.Results.Modified, DealStateChange{abi.DealID(key), dsFrom, dsTo}) + } + return nil +} + +func (d *v0MarketStatesDiffer) Remove(key uint64, val *typegen.Deferred) error { + ds := new(v0DealState) + err := ds.UnmarshalCBOR(bytes.NewReader(val.Raw)) + if err != nil { + return err + } + d.Results.Removed = append(d.Results.Removed, DealIDState{abi.DealID(key), ds}) + return nil +} + +type v0DealState struct { + market.DealState +} + +func (ds *v0DealState) SectorStartEpoch() abi.ChainEpoch { + return ds.DealState.SectorStartEpoch +} + +func (ds *v0DealState) SlashEpoch() abi.ChainEpoch { + return ds.DealState.SlashEpoch +} + +func (ds *v0DealState) LastUpdatedEpoch() abi.ChainEpoch { + return ds.DealState.LastUpdatedEpoch +} + +func (ds *v0DealState) Equals(other DealState) bool { + v0other, ok := other.(*v0DealState) + return ok && *ds == *v0other +} + +type v0DealProposals struct { + adt.Array +} + +func (s *v0DealProposals) Diff(other DealProposals) (*DealProposalChanges, error) { + v0other, ok := other.(*v0DealProposals) + if !ok { + // TODO handle this if possible on a case by case basis but for now, just fail + return nil, errors.New("cannot compare deal proposals across versions") + } + results := new(DealProposalChanges) + if err := adt.DiffAdtArray(s, v0other, &v0MarketProposalsDiffer{results}); err != nil { + return nil, fmt.Errorf("diffing deal proposals: %w", err) + } + + return results, nil +} + +type v0MarketProposalsDiffer struct { + Results *DealProposalChanges +} + +type v0DealProposal struct { + market.DealProposal +} + +func (d *v0MarketProposalsDiffer) Add(key uint64, val *typegen.Deferred) error { + dp := new(v0DealProposal) + err := dp.UnmarshalCBOR(bytes.NewReader(val.Raw)) + if err != nil { + return err + } + d.Results.Added = append(d.Results.Added, ProposalIDState{abi.DealID(key), dp}) + return nil +} + +func (d *v0MarketProposalsDiffer) Modify(key uint64, from, to *typegen.Deferred) error { + // short circuit, DealProposals are static + return nil +} + +func (d *v0MarketProposalsDiffer) Remove(key uint64, val *typegen.Deferred) error { + dp := new(v0DealProposal) + err := dp.UnmarshalCBOR(bytes.NewReader(val.Raw)) + if err != nil { + return err + } + d.Results.Removed = append(d.Results.Removed, ProposalIDState{abi.DealID(key), dp}) + return nil +} diff --git a/chain/events/state/predicates.go b/chain/events/state/predicates.go index e5caa41d2..e1e88e1a1 100644 --- a/chain/events/state/predicates.go +++ b/chain/events/state/predicates.go @@ -7,12 +7,13 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/chain/actors/builtin/market" "github.com/filecoin-project/lotus/chain/actors/builtin/paych" "github.com/filecoin-project/specs-actors/actors/builtin" init_ "github.com/filecoin-project/specs-actors/actors/builtin/init" - "github.com/filecoin-project/specs-actors/actors/builtin/market" "github.com/filecoin-project/specs-actors/actors/builtin/miner" - "github.com/filecoin-project/specs-actors/actors/util/adt" + v0adt "github.com/filecoin-project/specs-actors/actors/util/adt" cbor "github.com/ipfs/go-ipld-cbor" typegen "github.com/whyrusleeping/cbor-gen" @@ -69,26 +70,26 @@ func (sp *StatePredicates) OnActorStateChanged(addr address.Address, diffStateFu } } -type DiffStorageMarketStateFunc func(ctx context.Context, oldState *market.State, newState *market.State) (changed bool, user UserData, err error) +type DiffStorageMarketStateFunc func(ctx context.Context, oldState market.State, newState market.State) (changed bool, user UserData, err error) // OnStorageMarketActorChanged calls diffStorageMarketState when the state changes for the market actor func (sp *StatePredicates) OnStorageMarketActorChanged(diffStorageMarketState DiffStorageMarketStateFunc) DiffTipSetKeyFunc { return sp.OnActorStateChanged(builtin.StorageMarketActorAddr, func(ctx context.Context, oldActorState, newActorState *types.Actor) (changed bool, user UserData, err error) { - var oldState market.State - if err := sp.cst.Get(ctx, oldActorState.Head, &oldState); err != nil { + oldState, err := market.Load(adt.WrapStore(ctx, sp.cst), oldActorState) + if err != nil { return false, nil, err } - var newState market.State - if err := sp.cst.Get(ctx, newActorState.Head, &newState); err != nil { + newState, err := market.Load(adt.WrapStore(ctx, sp.cst), newActorState) + if err != nil { return false, nil, err } - return diffStorageMarketState(ctx, &oldState, &newState) + return diffStorageMarketState(ctx, oldState, newState) }) } type BalanceTables struct { - EscrowTable *adt.BalanceTable - LockedTable *adt.BalanceTable + EscrowTable market.BalanceTable + LockedTable market.BalanceTable } // DiffBalanceTablesFunc compares two balance tables @@ -96,32 +97,27 @@ type DiffBalanceTablesFunc func(ctx context.Context, oldBalanceTable, newBalance // OnBalanceChanged runs when the escrow table for available balances changes func (sp *StatePredicates) OnBalanceChanged(diffBalances DiffBalanceTablesFunc) DiffStorageMarketStateFunc { - return func(ctx context.Context, oldState *market.State, newState *market.State) (changed bool, user UserData, err error) { - if oldState.EscrowTable.Equals(newState.EscrowTable) && oldState.LockedTable.Equals(newState.LockedTable) { + return func(ctx context.Context, oldState market.State, newState market.State) (changed bool, user UserData, err error) { + if !oldState.BalancesChanged(newState) { return false, nil, nil } - ctxStore := &contextStore{ - ctx: ctx, - cst: sp.cst, - } - - oldEscrowRoot, err := adt.AsBalanceTable(ctxStore, oldState.EscrowTable) + oldEscrowRoot, err := oldState.EscrowTable() if err != nil { return false, nil, err } - oldLockedRoot, err := adt.AsBalanceTable(ctxStore, oldState.LockedTable) + oldLockedRoot, err := oldState.LockedTable() if err != nil { return false, nil, err } - newEscrowRoot, err := adt.AsBalanceTable(ctxStore, newState.EscrowTable) + newEscrowRoot, err := newState.EscrowTable() if err != nil { return false, nil, err } - newLockedRoot, err := adt.AsBalanceTable(ctxStore, newState.LockedTable) + newLockedRoot, err := newState.LockedTable() if err != nil { return false, nil, err } @@ -130,25 +126,22 @@ func (sp *StatePredicates) OnBalanceChanged(diffBalances DiffBalanceTablesFunc) } } -type DiffAdtArraysFunc func(ctx context.Context, oldDealStateRoot, newDealStateRoot *adt.Array) (changed bool, user UserData, err error) +type DiffDealStatesFunc func(ctx context.Context, oldDealStateRoot, newDealStateRoot market.DealStates) (changed bool, user UserData, err error) +type DiffDealProposalsFunc func(ctx context.Context, oldDealStateRoot, newDealStateRoot market.DealProposals) (changed bool, user UserData, err error) +type DiffAdtArraysFunc func(ctx context.Context, oldDealStateRoot, newDealStateRoot adt.Array) (changed bool, user UserData, err error) // OnDealStateChanged calls diffDealStates when the market deal state changes -func (sp *StatePredicates) OnDealStateChanged(diffDealStates DiffAdtArraysFunc) DiffStorageMarketStateFunc { - return func(ctx context.Context, oldState *market.State, newState *market.State) (changed bool, user UserData, err error) { - if oldState.States.Equals(newState.States) { +func (sp *StatePredicates) OnDealStateChanged(diffDealStates DiffDealStatesFunc) DiffStorageMarketStateFunc { + return func(ctx context.Context, oldState market.State, newState market.State) (changed bool, user UserData, err error) { + if !oldState.StatesChanged(newState) { return false, nil, nil } - ctxStore := &contextStore{ - ctx: ctx, - cst: sp.cst, - } - - oldRoot, err := adt.AsArray(ctxStore, oldState.States) + oldRoot, err := oldState.States() if err != nil { return false, nil, err } - newRoot, err := adt.AsArray(ctxStore, newState.States) + newRoot, err := newState.States() if err != nil { return false, nil, err } @@ -158,22 +151,17 @@ func (sp *StatePredicates) OnDealStateChanged(diffDealStates DiffAdtArraysFunc) } // OnDealProposalChanged calls diffDealProps when the market proposal state changes -func (sp *StatePredicates) OnDealProposalChanged(diffDealProps DiffAdtArraysFunc) DiffStorageMarketStateFunc { - return func(ctx context.Context, oldState *market.State, newState *market.State) (changed bool, user UserData, err error) { - if oldState.Proposals.Equals(newState.Proposals) { +func (sp *StatePredicates) OnDealProposalChanged(diffDealProps DiffDealProposalsFunc) DiffStorageMarketStateFunc { + return func(ctx context.Context, oldState market.State, newState market.State) (changed bool, user UserData, err error) { + if !oldState.ProposalsChanged(newState) { return false, nil, nil } - ctxStore := &contextStore{ - ctx: ctx, - cst: sp.cst, - } - - oldRoot, err := adt.AsArray(ctxStore, oldState.Proposals) + oldRoot, err := oldState.Proposals() if err != nil { return false, nil, err } - newRoot, err := adt.AsArray(ctxStore, newState.Proposals) + newRoot, err := newState.Proposals() if err != nil { return false, nil, err } @@ -182,51 +170,14 @@ func (sp *StatePredicates) OnDealProposalChanged(diffDealProps DiffAdtArraysFunc } } -var _ AdtArrayDiff = &MarketDealProposalChanges{} - -type MarketDealProposalChanges struct { - Added []ProposalIDState - Removed []ProposalIDState -} - -type ProposalIDState struct { - ID abi.DealID - Proposal market.DealProposal -} - -func (m *MarketDealProposalChanges) Add(key uint64, val *typegen.Deferred) error { - dp := new(market.DealProposal) - err := dp.UnmarshalCBOR(bytes.NewReader(val.Raw)) - if err != nil { - return err - } - m.Added = append(m.Added, ProposalIDState{abi.DealID(key), *dp}) - return nil -} - -func (m *MarketDealProposalChanges) Modify(key uint64, from, to *typegen.Deferred) error { - // short circuit, DealProposals are static - return nil -} - -func (m *MarketDealProposalChanges) Remove(key uint64, val *typegen.Deferred) error { - dp := new(market.DealProposal) - err := dp.UnmarshalCBOR(bytes.NewReader(val.Raw)) - if err != nil { - return err - } - m.Removed = append(m.Removed, ProposalIDState{abi.DealID(key), *dp}) - return nil -} - // OnDealProposalAmtChanged detects changes in the deal proposal AMT for all deal proposals and returns a MarketProposalsChanges structure containing: // - Added Proposals // - Modified Proposals // - Removed Proposals -func (sp *StatePredicates) OnDealProposalAmtChanged() DiffAdtArraysFunc { - return func(ctx context.Context, oldDealProps, newDealProps *adt.Array) (changed bool, user UserData, err error) { - proposalChanges := new(MarketDealProposalChanges) - if err := DiffAdtArray(oldDealProps, newDealProps, proposalChanges); err != nil { +func (sp *StatePredicates) OnDealProposalAmtChanged() DiffDealProposalsFunc { + return func(ctx context.Context, oldDealProps, newDealProps market.DealProposals) (changed bool, user UserData, err error) { + proposalChanges, err := oldDealProps.Diff(newDealProps) + if err != nil { return false, nil, err } @@ -238,64 +189,14 @@ func (sp *StatePredicates) OnDealProposalAmtChanged() DiffAdtArraysFunc { } } -var _ AdtArrayDiff = &MarketDealStateChanges{} - -type MarketDealStateChanges struct { - Added []DealIDState - Modified []DealStateChange - Removed []DealIDState -} - -type DealIDState struct { - ID abi.DealID - Deal market.DealState -} - -func (m *MarketDealStateChanges) Add(key uint64, val *typegen.Deferred) error { - ds := new(market.DealState) - err := ds.UnmarshalCBOR(bytes.NewReader(val.Raw)) - if err != nil { - return err - } - m.Added = append(m.Added, DealIDState{abi.DealID(key), *ds}) - return nil -} - -func (m *MarketDealStateChanges) Modify(key uint64, from, to *typegen.Deferred) error { - dsFrom := new(market.DealState) - if err := dsFrom.UnmarshalCBOR(bytes.NewReader(from.Raw)); err != nil { - return err - } - - dsTo := new(market.DealState) - if err := dsTo.UnmarshalCBOR(bytes.NewReader(to.Raw)); err != nil { - return err - } - - if *dsFrom != *dsTo { - m.Modified = append(m.Modified, DealStateChange{abi.DealID(key), dsFrom, dsTo}) - } - return nil -} - -func (m *MarketDealStateChanges) Remove(key uint64, val *typegen.Deferred) error { - ds := new(market.DealState) - err := ds.UnmarshalCBOR(bytes.NewReader(val.Raw)) - if err != nil { - return err - } - m.Removed = append(m.Removed, DealIDState{abi.DealID(key), *ds}) - return nil -} - // OnDealStateAmtChanged detects changes in the deal state AMT for all deal states and returns a MarketDealStateChanges structure containing: // - Added Deals // - Modified Deals // - Removed Deals -func (sp *StatePredicates) OnDealStateAmtChanged() DiffAdtArraysFunc { - return func(ctx context.Context, oldDealStates, newDealStates *adt.Array) (changed bool, user UserData, err error) { - dealStateChanges := new(MarketDealStateChanges) - if err := DiffAdtArray(oldDealStates, newDealStates, dealStateChanges); err != nil { +func (sp *StatePredicates) OnDealStateAmtChanged() DiffDealStatesFunc { + return func(ctx context.Context, oldDealStates, newDealStates market.DealStates) (changed bool, user UserData, err error) { + dealStateChanges, err := oldDealStates.Diff(newDealStates) + if err != nil { return false, nil, err } @@ -313,37 +214,31 @@ type ChangedDeals map[abi.DealID]DealStateChange // DealStateChange is a change in deal state from -> to type DealStateChange struct { ID abi.DealID - From *market.DealState - To *market.DealState + From market.DealState + To market.DealState } // DealStateChangedForIDs detects changes in the deal state AMT for the given deal IDs -func (sp *StatePredicates) DealStateChangedForIDs(dealIds []abi.DealID) DiffAdtArraysFunc { - return func(ctx context.Context, oldDealStateArray, newDealStateArray *adt.Array) (changed bool, user UserData, err error) { +func (sp *StatePredicates) DealStateChangedForIDs(dealIds []abi.DealID) DiffDealStatesFunc { + return func(ctx context.Context, oldDealStates, newDealStates market.DealStates) (changed bool, user UserData, err error) { changedDeals := make(ChangedDeals) for _, dealID := range dealIds { - var oldDealPtr, newDealPtr *market.DealState - var oldDeal, newDeal market.DealState // If the deal has been removed, we just set it to nil - found, err := oldDealStateArray.Get(uint64(dealID), &oldDeal) + oldDeal, err := oldDealStates.GetDeal(dealID) if err != nil { return false, nil, err } - if found { - oldDealPtr = &oldDeal - } - found, err = newDealStateArray.Get(uint64(dealID), &newDeal) + newDeal, err := newDealStates.GetDeal(dealID) if err != nil { return false, nil, err } - if found { - newDealPtr = &newDeal - } - if oldDeal != newDeal { - changedDeals[dealID] = DealStateChange{dealID, oldDealPtr, newDealPtr} + existenceChanged := (oldDeal == nil) != (newDeal == nil) + valueChanged := (oldDeal != nil && newDeal != nil) && !oldDeal.Equals(newDeal) + if existenceChanged || valueChanged { + changedDeals[dealID] = DealStateChange{dealID, oldDeal, newDeal} } } if len(changedDeals) > 0 { @@ -441,7 +336,7 @@ type MinerSectorChanges struct { Removed []miner.SectorOnChainInfo } -var _ AdtArrayDiff = &MinerSectorChanges{} +var _ adt.AdtArrayDiff = &MinerSectorChanges{} type SectorExtensions struct { From miner.SectorOnChainInfo @@ -508,17 +403,17 @@ func (sp *StatePredicates) OnMinerSectorChange() DiffMinerActorStateFunc { return false, nil, nil } - oldSectors, err := adt.AsArray(ctxStore, oldState.Sectors) + oldSectors, err := v0adt.AsArray(ctxStore, oldState.Sectors) if err != nil { return false, nil, err } - newSectors, err := adt.AsArray(ctxStore, newState.Sectors) + newSectors, err := v0adt.AsArray(ctxStore, newState.Sectors) if err != nil { return false, nil, err } - if err := DiffAdtArray(oldSectors, newSectors, sectorChanges); err != nil { + if err := adt.DiffAdtArray(oldSectors, newSectors, sectorChanges); err != nil { return false, nil, err } @@ -584,17 +479,17 @@ func (sp *StatePredicates) OnMinerPreCommitChange() DiffMinerActorStateFunc { return false, nil, nil } - oldPrecommits, err := adt.AsMap(ctxStore, oldState.PreCommittedSectors) + oldPrecommits, err := v0adt.AsMap(ctxStore, oldState.PreCommittedSectors) if err != nil { return false, nil, err } - newPrecommits, err := adt.AsMap(ctxStore, newState.PreCommittedSectors) + newPrecommits, err := v0adt.AsMap(ctxStore, newState.PreCommittedSectors) if err != nil { return false, nil, err } - if err := DiffAdtMap(oldPrecommits, newPrecommits, precommitChanges); err != nil { + if err := adt.DiffAdtMap(oldPrecommits, newPrecommits, precommitChanges); err != nil { return false, nil, err } @@ -763,17 +658,17 @@ func (sp *StatePredicates) OnAddressMapChange() DiffInitActorStateFunc { return false, nil, nil } - oldAddrs, err := adt.AsMap(ctxStore, oldState.AddressMap) + oldAddrs, err := v0adt.AsMap(ctxStore, oldState.AddressMap) if err != nil { return false, nil, err } - newAddrs, err := adt.AsMap(ctxStore, newState.AddressMap) + newAddrs, err := v0adt.AsMap(ctxStore, newState.AddressMap) if err != nil { return false, nil, err } - if err := DiffAdtMap(oldAddrs, newAddrs, addressChanges); err != nil { + if err := adt.DiffAdtMap(oldAddrs, newAddrs, addressChanges); err != nil { return false, nil, err } diff --git a/chain/events/state/predicates_test.go b/chain/events/state/predicates_test.go index 7117a96cc..783c720ed 100644 --- a/chain/events/state/predicates_test.go +++ b/chain/events/state/predicates_test.go @@ -16,7 +16,10 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" - "github.com/filecoin-project/specs-actors/actors/builtin/market" + "github.com/filecoin-project/lotus/chain/actors/builtin/market" + v0builtin "github.com/filecoin-project/specs-actors/actors/builtin" + v0market "github.com/filecoin-project/specs-actors/actors/builtin/market" + "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/specs-actors/actors/util/adt" tutils "github.com/filecoin-project/specs-actors/support/testing" @@ -69,22 +72,22 @@ func TestMarketPredicates(t *testing.T) { bs := bstore.NewTemporarySync() store := adt.WrapStore(ctx, cbornode.NewCborStore(bs)) - oldDeal1 := &market.DealState{ + oldDeal1 := &v0market.DealState{ SectorStartEpoch: 1, LastUpdatedEpoch: 2, SlashEpoch: 0, } - oldDeal2 := &market.DealState{ + oldDeal2 := &v0market.DealState{ SectorStartEpoch: 4, LastUpdatedEpoch: 5, SlashEpoch: 0, } - oldDeals := map[abi.DealID]*market.DealState{ + oldDeals := map[abi.DealID]*v0market.DealState{ abi.DealID(1): oldDeal1, abi.DealID(2): oldDeal2, } - oldProp1 := &market.DealProposal{ + oldProp1 := &v0market.DealProposal{ PieceCID: dummyCid, PieceSize: 0, VerifiedDeal: false, @@ -96,7 +99,7 @@ func TestMarketPredicates(t *testing.T) { ProviderCollateral: big.Zero(), ClientCollateral: big.Zero(), } - oldProp2 := &market.DealProposal{ + oldProp2 := &v0market.DealProposal{ PieceCID: dummyCid, PieceSize: 0, VerifiedDeal: false, @@ -108,7 +111,7 @@ func TestMarketPredicates(t *testing.T) { ProviderCollateral: big.Zero(), ClientCollateral: big.Zero(), } - oldProps := map[abi.DealID]*market.DealProposal{ + oldProps := map[abi.DealID]*v0market.DealProposal{ abi.DealID(1): oldProp1, abi.DealID(2): oldProp2, } @@ -122,7 +125,7 @@ func TestMarketPredicates(t *testing.T) { oldStateC := createMarketState(ctx, t, store, oldDeals, oldProps, oldBalances) - newDeal1 := &market.DealState{ + newDeal1 := &v0market.DealState{ SectorStartEpoch: 1, LastUpdatedEpoch: 3, SlashEpoch: 0, @@ -131,19 +134,19 @@ func TestMarketPredicates(t *testing.T) { // deal 2 removed // added - newDeal3 := &market.DealState{ + newDeal3 := &v0market.DealState{ SectorStartEpoch: 1, LastUpdatedEpoch: 2, SlashEpoch: 3, } - newDeals := map[abi.DealID]*market.DealState{ + newDeals := map[abi.DealID]*v0market.DealState{ abi.DealID(1): newDeal1, // deal 2 was removed abi.DealID(3): newDeal3, } // added - newProp3 := &market.DealProposal{ + newProp3 := &v0market.DealProposal{ PieceCID: dummyCid, PieceSize: 0, VerifiedDeal: false, @@ -155,7 +158,7 @@ func TestMarketPredicates(t *testing.T) { ProviderCollateral: big.Zero(), ClientCollateral: big.Zero(), } - newProps := map[abi.DealID]*market.DealProposal{ + newProps := map[abi.DealID]*v0market.DealProposal{ abi.DealID(1): oldProp1, // 1 was persisted // prop 2 was removed abi.DealID(3): newProp3, // new @@ -178,8 +181,8 @@ func TestMarketPredicates(t *testing.T) { require.NoError(t, err) api := newMockAPI(bs) - api.setActor(oldState.Key(), &types.Actor{Head: oldStateC}) - api.setActor(newState.Key(), &types.Actor{Head: newStateC}) + api.setActor(oldState.Key(), &types.Actor{Code: v0builtin.StorageMarketActorCodeID, Head: oldStateC}) + api.setActor(newState.Key(), &types.Actor{Code: v0builtin.StorageMarketActorCodeID, Head: newStateC}) t.Run("deal ID predicate", func(t *testing.T) { preds := NewStatePredicates(api) @@ -203,11 +206,11 @@ func TestMarketPredicates(t *testing.T) { require.Contains(t, changedDealIDs, abi.DealID(1)) require.Contains(t, changedDealIDs, abi.DealID(2)) deal1 := changedDealIDs[abi.DealID(1)] - if deal1.From.LastUpdatedEpoch != 2 || deal1.To.LastUpdatedEpoch != 3 { + if deal1.From.LastUpdatedEpoch() != 2 || deal1.To.LastUpdatedEpoch() != 3 { t.Fatal("Unexpected change to LastUpdatedEpoch") } deal2 := changedDealIDs[abi.DealID(2)] - if deal2.From.LastUpdatedEpoch != 5 || deal2.To != nil { + if deal2.From.LastUpdatedEpoch() != 5 || deal2.To != nil { t.Fatal("Expected To to be nil") } @@ -230,11 +233,17 @@ func TestMarketPredicates(t *testing.T) { require.False(t, changed) // Test that OnDealStateChanged does not call the callback if the state has not changed - diffDealStateFn := preds.OnDealStateChanged(func(context.Context, *adt.Array, *adt.Array) (bool, UserData, error) { + diffDealStateFn := preds.OnDealStateChanged(func(context.Context, market.DealStates, market.DealStates) (bool, UserData, error) { t.Fatal("No state change so this should not be called") return false, nil, nil }) - marketState := createEmptyMarketState(t, store) + v0marketState := createEmptyMarketState(t, store) + marketCid, err := store.Put(ctx, v0marketState) + require.NoError(t, err) + marketState, err := market.Load(store, &types.Actor{ + Code: v0builtin.StorageMarketActorCodeID, + Head: marketCid, + }) changed, _, err = diffDealStateFn(ctx, marketState, marketState) require.NoError(t, err) require.False(t, changed) @@ -252,18 +261,18 @@ func TestMarketPredicates(t *testing.T) { require.NoError(t, err) require.True(t, changed) - changedDeals, ok := valArr.(*MarketDealStateChanges) + changedDeals, ok := valArr.(*market.DealStateChanges) require.True(t, ok) require.Len(t, changedDeals.Added, 1) require.Equal(t, abi.DealID(3), changedDeals.Added[0].ID) - require.Equal(t, *newDeal3, changedDeals.Added[0].Deal) + require.True(t, dealEquality(*newDeal3, changedDeals.Added[0].Deal)) require.Len(t, changedDeals.Removed, 1) require.Len(t, changedDeals.Modified, 1) require.Equal(t, abi.DealID(1), changedDeals.Modified[0].ID) - require.Equal(t, newDeal1, changedDeals.Modified[0].To) - require.Equal(t, oldDeal1, changedDeals.Modified[0].From) + require.True(t, dealEquality(*newDeal1, changedDeals.Modified[0].To)) + require.True(t, dealEquality(*oldDeal1, changedDeals.Modified[0].From)) require.Equal(t, abi.DealID(2), changedDeals.Removed[0].ID) }) @@ -279,17 +288,15 @@ func TestMarketPredicates(t *testing.T) { require.NoError(t, err) require.True(t, changed) - changedProps, ok := valArr.(*MarketDealProposalChanges) + changedProps, ok := valArr.(*market.DealProposalChanges) require.True(t, ok) require.Len(t, changedProps.Added, 1) require.Equal(t, abi.DealID(3), changedProps.Added[0].ID) - require.Equal(t, *newProp3, changedProps.Added[0].Proposal) // proposals cannot be modified -- no modified testing require.Len(t, changedProps.Removed, 1) require.Equal(t, abi.DealID(2), changedProps.Removed[0].ID) - require.Equal(t, *oldProp2, changedProps.Removed[0].Proposal) }) t.Run("balances predicate", func(t *testing.T) { @@ -342,7 +349,13 @@ func TestMarketPredicates(t *testing.T) { t.Fatal("No state change so this should not be called") return false, nil, nil }) - marketState := createEmptyMarketState(t, store) + v0marketState := createEmptyMarketState(t, store) + marketCid, err := store.Put(ctx, v0marketState) + require.NoError(t, err) + marketState, err := market.Load(store, &types.Actor{ + Code: v0builtin.StorageMarketActorCodeID, + Head: marketCid, + }) changed, _, err = diffDealBalancesFn(ctx, marketState, marketState) require.NoError(t, err) require.False(t, changed) @@ -450,7 +463,7 @@ type balance struct { locked abi.TokenAmount } -func createMarketState(ctx context.Context, t *testing.T, store adt.Store, deals map[abi.DealID]*market.DealState, props map[abi.DealID]*market.DealProposal, balances map[address.Address]balance) cid.Cid { +func createMarketState(ctx context.Context, t *testing.T, store adt.Store, deals map[abi.DealID]*v0market.DealState, props map[abi.DealID]*v0market.DealProposal, balances map[address.Address]balance) cid.Cid { dealRootCid := createDealAMT(ctx, t, store, deals) propRootCid := createProposalAMT(ctx, t, store, props) balancesCids := createBalanceTable(ctx, t, store, balances) @@ -465,15 +478,15 @@ func createMarketState(ctx context.Context, t *testing.T, store adt.Store, deals return stateC } -func createEmptyMarketState(t *testing.T, store adt.Store) *market.State { +func createEmptyMarketState(t *testing.T, store adt.Store) *v0market.State { emptyArrayCid, err := adt.MakeEmptyArray(store).Root() require.NoError(t, err) emptyMap, err := adt.MakeEmptyMap(store).Root() require.NoError(t, err) - return market.ConstructState(emptyArrayCid, emptyMap, emptyMap) + return v0market.ConstructState(emptyArrayCid, emptyMap, emptyMap) } -func createDealAMT(ctx context.Context, t *testing.T, store adt.Store, deals map[abi.DealID]*market.DealState) cid.Cid { +func createDealAMT(ctx context.Context, t *testing.T, store adt.Store, deals map[abi.DealID]*v0market.DealState) cid.Cid { root := adt.MakeEmptyArray(store) for dealID, dealState := range deals { err := root.Set(uint64(dealID), dealState) @@ -484,7 +497,7 @@ func createDealAMT(ctx context.Context, t *testing.T, store adt.Store, deals map return rootCid } -func createProposalAMT(ctx context.Context, t *testing.T, store adt.Store, props map[abi.DealID]*market.DealProposal) cid.Cid { +func createProposalAMT(ctx context.Context, t *testing.T, store adt.Store, props map[abi.DealID]*v0market.DealProposal) cid.Cid { root := adt.MakeEmptyArray(store) for dealID, prop := range props { err := root.Set(uint64(dealID), prop) @@ -607,3 +620,9 @@ func newSectorPreCommitInfo(sectorNo abi.SectorNumber, sealed cid.Cid, expiratio Expiration: expiration, } } + +func dealEquality(expected v0market.DealState, actual market.DealState) bool { + return expected.LastUpdatedEpoch == actual.LastUpdatedEpoch() && + expected.SectorStartEpoch == actual.SectorStartEpoch() && + expected.SlashEpoch == actual.SlashEpoch() +} diff --git a/cmd/lotus-chainwatch/processor/market.go b/cmd/lotus-chainwatch/processor/market.go index e50ec3076..a4bae4b20 100644 --- a/cmd/lotus-chainwatch/processor/market.go +++ b/cmd/lotus-chainwatch/processor/market.go @@ -8,6 +8,7 @@ import ( "golang.org/x/sync/errgroup" "golang.org/x/xerrors" + "github.com/filecoin-project/lotus/chain/actors/builtin/market" "github.com/filecoin-project/lotus/chain/events/state" ) @@ -293,14 +294,14 @@ func (p *Processor) updateMarketActorDealProposals(ctx context.Context, marketTi if !changed { continue } - changes, ok := val.(*state.MarketDealStateChanges) + changes, ok := val.(*market.DealStateChanges) if !ok { return xerrors.Errorf("Unknown type returned by Deal State AMT predicate: %T", val) } for _, modified := range changes.Modified { - if modified.From.SlashEpoch != modified.To.SlashEpoch { - if _, err := stmt.Exec(modified.To.SlashEpoch, modified.ID); err != nil { + if modified.From.SlashEpoch() != modified.To.SlashEpoch() { + if _, err := stmt.Exec(modified.To.SlashEpoch(), modified.ID); err != nil { return err } } From d5af25b76cba60d0aceb94ab395d5df9ebee1428 Mon Sep 17 00:00:00 2001 From: Frank Date: Thu, 17 Sep 2020 10:38:07 +0800 Subject: [PATCH 165/885] update init sector state list --- cmd/lotus-storage-miner/sectors.go | 4 +- extern/storage-sealing/sector_state.go | 63 +++++++++++++------------- 2 files changed, 33 insertions(+), 34 deletions(-) diff --git a/cmd/lotus-storage-miner/sectors.go b/cmd/lotus-storage-miner/sectors.go index c08d6ca14..1b5cfb676 100644 --- a/cmd/lotus-storage-miner/sectors.go +++ b/cmd/lotus-storage-miner/sectors.go @@ -17,9 +17,9 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" - - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" + lcli "github.com/filecoin-project/lotus/cli" + sealing "github.com/filecoin-project/lotus/extern/storage-sealing" ) var sectorsCmd = &cli.Command{ diff --git a/extern/storage-sealing/sector_state.go b/extern/storage-sealing/sector_state.go index 3ed891064..10b96e504 100644 --- a/extern/storage-sealing/sector_state.go +++ b/extern/storage-sealing/sector_state.go @@ -2,7 +2,37 @@ package sealing type SectorState string -var ExistSectorStateList = make(map[SectorState]struct{}) +var ExistSectorStateList = map[SectorState]struct{}{ + Empty: {}, + WaitDeals: {}, + Packing: {}, + PreCommit1: {}, + PreCommit2: {}, + PreCommitting: {}, + PreCommitWait: {}, + WaitSeed: {}, + Committing: {}, + SubmitCommit: {}, + CommitWait: {}, + FinalizeSector: {}, + Proving: {}, + FailedUnrecoverable: {}, + SealPreCommit1Failed: {}, + SealPreCommit2Failed: {}, + PreCommitFailed: {}, + ComputeProofFailed: {}, + CommitFailed: {}, + PackingFailed: {}, + FinalizeFailed: {}, + DealsExpired: {}, + RecoverDealIDs: {}, + Faulty: {}, + FaultReported: {}, + FaultedFinal: {}, + Removing: {}, + RemoveFailed: {}, + Removed: {}, +} const ( UndefinedSectorState SectorState = "" @@ -41,37 +71,6 @@ const ( RemoveFailed SectorState = "RemoveFailed" Removed SectorState = "Removed" ) -func init() { - ExistSectorStateList[Empty] = struct{}{} - ExistSectorStateList[WaitDeals] = struct{}{} - ExistSectorStateList[Packing] = struct{}{} - ExistSectorStateList[PreCommit1] = struct{}{} - ExistSectorStateList[PreCommit2] = struct{}{} - ExistSectorStateList[PreCommitting] = struct{}{} - ExistSectorStateList[PreCommitWait] = struct{}{} - ExistSectorStateList[WaitSeed] = struct{}{} - ExistSectorStateList[Committing] = struct{}{} - ExistSectorStateList[SubmitCommit] = struct{}{} - ExistSectorStateList[CommitWait] = struct{}{} - ExistSectorStateList[FinalizeSector] = struct{}{} - ExistSectorStateList[Proving] = struct{}{} - ExistSectorStateList[FailedUnrecoverable] = struct{}{} - ExistSectorStateList[SealPreCommit1Failed] = struct{}{} - ExistSectorStateList[SealPreCommit2Failed] = struct{}{} - ExistSectorStateList[PreCommitFailed] = struct{}{} - ExistSectorStateList[ComputeProofFailed] = struct{}{} - ExistSectorStateList[CommitFailed] = struct{}{} - ExistSectorStateList[PackingFailed] = struct{}{} - ExistSectorStateList[FinalizeFailed] = struct{}{} - ExistSectorStateList[DealsExpired] = struct{}{} - ExistSectorStateList[RecoverDealIDs] = struct{}{} - ExistSectorStateList[Faulty] = struct{}{} - ExistSectorStateList[FaultReported] = struct{}{} - ExistSectorStateList[FaultedFinal] = struct{}{} - ExistSectorStateList[Removing] = struct{}{} - ExistSectorStateList[RemoveFailed] = struct{}{} - ExistSectorStateList[Removed] = struct{}{} -} func toStatState(st SectorState) statSectorState { switch st { From bf2e3baa8fd2e192cfcca15c05c66ced1bc12eb9 Mon Sep 17 00:00:00 2001 From: Frank Date: Thu, 17 Sep 2020 11:49:23 +0800 Subject: [PATCH 166/885] fix conformance gen --- conformance/chaos/cbor_gen.go | 6 +++--- conformance/chaos/gen/gen.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/conformance/chaos/cbor_gen.go b/conformance/chaos/cbor_gen.go index 2d9deec93..61e36e661 100644 --- a/conformance/chaos/cbor_gen.go +++ b/conformance/chaos/cbor_gen.go @@ -6,10 +6,10 @@ import ( "fmt" "io" - "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/exitcode" + abi "github.com/filecoin-project/go-state-types/abi" + exitcode "github.com/filecoin-project/go-state-types/exitcode" cbg "github.com/whyrusleeping/cbor-gen" - "golang.org/x/xerrors" + xerrors "golang.org/x/xerrors" ) var _ = xerrors.Errorf diff --git a/conformance/chaos/gen/gen.go b/conformance/chaos/gen/gen.go index 496cc3d35..ea04aee21 100644 --- a/conformance/chaos/gen/gen.go +++ b/conformance/chaos/gen/gen.go @@ -7,7 +7,7 @@ import ( ) func main() { - if err := gen.WriteTupleEncodersToFile("../cbor_gen.go", "chaos", + if err := gen.WriteTupleEncodersToFile("./cbor_gen.go", "chaos", chaos.State{}, chaos.CreateActorArgs{}, chaos.ResolveAddressResponse{}, From b5ba7a0fad639952d22ee102037d4dc5ae53b99c Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Wed, 16 Sep 2020 22:34:13 -0400 Subject: [PATCH 167/885] Miner migration --- build/params_2k.go | 4 +- build/params_shared_funcs.go | 6 +-- build/params_shared_vals.go | 5 +- build/params_testground.go | 4 +- build/params_testnet.go | 4 +- chain/actors/builtin/miner/miner.go | 13 +++++- chain/actors/builtin/miner/v0.go | 33 +++++++------ chain/events/state/predicates.go | 34 ++++---------- chain/events/state/predicates_test.go | 22 +++++---- chain/gen/gen.go | 4 +- chain/gen/gen_test.go | 4 +- chain/gen/genesis/miners.go | 19 ++++---- chain/store/store_test.go | 4 +- chain/sync_test.go | 4 +- chain/vm/invoker.go | 4 +- chain/vm/syscalls.go | 6 +-- cli/paych_test.go | 4 +- cmd/lotus-pcr/main.go | 29 ++++++++++-- extern/storage-sealing/checks.go | 27 +++++++++-- extern/storage-sealing/constants.go | 10 +--- extern/storage-sealing/fsm_events.go | 2 +- extern/storage-sealing/precommit_policy.go | 24 ++++++++-- extern/storage-sealing/sealing.go | 22 ++++++++- extern/storage-sealing/states_failed.go | 3 +- extern/storage-sealing/states_sealing.go | 54 ++++++++++++++++++---- extern/storage-sealing/upgrade_queue.go | 3 +- markets/storageadapter/client.go | 3 +- markets/storageadapter/provider.go | 3 +- node/impl/client/client.go | 4 +- node/node_test.go | 6 +-- node/test/builder.go | 4 +- storage/adapter_storage_miner.go | 32 ++++++------- storage/miner.go | 11 ++++- storage/wdpost_journal.go | 2 +- storage/wdpost_run.go | 37 +++++++++------ 35 files changed, 288 insertions(+), 162 deletions(-) diff --git a/build/params_2k.go b/build/params_2k.go index 313ccfa9d..cf34706e5 100644 --- a/build/params_2k.go +++ b/build/params_2k.go @@ -5,7 +5,7 @@ package build import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" + v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" ) @@ -21,7 +21,7 @@ var DrandSchedule = map[abi.ChainEpoch]DrandEnum{ func init() { v0power.ConsensusMinerMinPower = big.NewInt(2048) - miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ + v0miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } verifreg.MinVerifiedDealSize = big.NewInt(256) diff --git a/build/params_shared_funcs.go b/build/params_shared_funcs.go index 2c585271a..08f16cefd 100644 --- a/build/params_shared_funcs.go +++ b/build/params_shared_funcs.go @@ -6,14 +6,14 @@ import ( "github.com/libp2p/go-libp2p-core/protocol" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" + v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/lotus/node/modules/dtypes" ) func DefaultSectorSize() abi.SectorSize { - szs := make([]abi.SectorSize, 0, len(miner.SupportedProofTypes)) - for spt := range miner.SupportedProofTypes { + szs := make([]abi.SectorSize, 0, len(v0miner.SupportedProofTypes)) + for spt := range v0miner.SupportedProofTypes { ss, err := spt.SectorSize() if err != nil { panic(err) diff --git a/build/params_shared_vals.go b/build/params_shared_vals.go index 4a46b7fd1..ac7796ae7 100644 --- a/build/params_shared_vals.go +++ b/build/params_shared_vals.go @@ -9,7 +9,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" + v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" ) // ///// @@ -23,6 +23,7 @@ const UnixfsLinksPerLevel = 1024 const AllowableClockDriftSecs = uint64(1) const NewestNetworkVersion = network.Version2 +const ActorUpgradeNetworkVersion = network.Version3 // Epochs const ForkLengthThreshold = Finality @@ -31,7 +32,7 @@ const ForkLengthThreshold = Finality var BlocksPerEpoch = uint64(builtin.ExpectedLeadersPerEpoch) // Epochs -const Finality = miner.ChainFinality +const Finality = v0miner.ChainFinality const MessageConfidence = uint64(5) // constants for Weight calculation diff --git a/build/params_testground.go b/build/params_testground.go index 395d2a855..77e312ac2 100644 --- a/build/params_testground.go +++ b/build/params_testground.go @@ -13,7 +13,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" + v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" ) var ( @@ -32,7 +32,7 @@ var ( AllowableClockDriftSecs = uint64(1) - Finality = miner.ChainFinality + Finality = v0miner.ChainFinality ForkLengthThreshold = Finality SlashablePowerDelay = 20 diff --git a/build/params_testnet.go b/build/params_testnet.go index 4cfd8a9b6..a879d3ba7 100644 --- a/build/params_testnet.go +++ b/build/params_testnet.go @@ -8,7 +8,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" + v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" ) @@ -24,7 +24,7 @@ const UpgradeSmokeHeight = 51000 func init() { v0power.ConsensusMinerMinPower = big.NewInt(10 << 40) - miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ + v0miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ abi.RegisteredSealProof_StackedDrg32GiBV1: {}, abi.RegisteredSealProof_StackedDrg64GiBV1: {}, } diff --git a/chain/actors/builtin/miner/miner.go b/chain/actors/builtin/miner/miner.go index e436fdc69..90c44eff3 100644 --- a/chain/actors/builtin/miner/miner.go +++ b/chain/actors/builtin/miner/miner.go @@ -38,7 +38,9 @@ type State interface { FindSector(abi.SectorNumber) (*SectorLocation, error) GetSectorExpiration(abi.SectorNumber) (*SectorExpiration, error) GetPrecommittedSector(abi.SectorNumber) (*SectorPreCommitOnChainInfo, error) - LoadSectorsFromSet(filter *bitfield.BitField, filterOut bool) ([]*ChainSectorInfo, error) + LoadSectorsFromSet(filter *bitfield.BitField, filterOut bool) (adt.Array, error) + LoadPreCommittedSectorsFromSet(filter *bitfield.BitField, filterOut bool) (adt.Map, error) + IsAllocated(abi.SectorNumber) (bool, error) LoadDeadline(idx uint64) (Deadline, error) ForEachDeadline(cb func(idx uint64, dl Deadline) error) error @@ -65,6 +67,15 @@ type Partition interface { type SectorOnChainInfo = v0miner.SectorOnChainInfo type SectorPreCommitInfo = v0miner.SectorPreCommitInfo type SectorPreCommitOnChainInfo = v0miner.SectorPreCommitOnChainInfo +type PoStPartition = v0miner.PoStPartition +type RecoveryDeclaration = v0miner.RecoveryDeclaration +type FaultDeclaration = v0miner.FaultDeclaration + +// Params +type DeclareFaultsParams = v0miner.DeclareFaultsParams +type DeclareFaultsRecoveredParams = v0miner.DeclareFaultsRecoveredParams +type SubmitWindowedPoStParams = v0miner.SubmitWindowedPoStParams +type ProveCommitSectorParams = v0miner.ProveCommitSectorParams type MinerInfo struct { Owner address.Address // Must be an ID-address. diff --git a/chain/actors/builtin/miner/v0.go b/chain/actors/builtin/miner/v0.go index 446c828f1..b56fb1745 100644 --- a/chain/actors/builtin/miner/v0.go +++ b/chain/actors/builtin/miner/v0.go @@ -1,7 +1,6 @@ package miner import ( - "bytes" "errors" "github.com/filecoin-project/go-address" @@ -132,13 +131,12 @@ func (s *v0State) GetPrecommittedSector(num abi.SectorNumber) (*SectorPreCommitO return info, nil } -func (s *v0State) LoadSectorsFromSet(filter *bitfield.BitField, filterOut bool) ([]*ChainSectorInfo, error) { +func (s *v0State) LoadSectorsFromSet(filter *bitfield.BitField, filterOut bool) (adt.Array, error) { a, err := v0adt.AsArray(s.store, s.State.Sectors) if err != nil { return nil, err } - var sset []*ChainSectorInfo var v cbg.Deferred if err := a.ForEach(&v, func(i int64) error { if filter != nil { @@ -147,24 +145,31 @@ func (s *v0State) LoadSectorsFromSet(filter *bitfield.BitField, filterOut bool) return xerrors.Errorf("filter check error: %w", err) } if set == filterOut { - return nil + err = a.Delete(uint64(i)) + if err != nil { + return xerrors.Errorf("filtering error: %w", err) + } } } - - var oci v0miner.SectorOnChainInfo - if err := oci.UnmarshalCBOR(bytes.NewReader(v.Raw)); err != nil { - return err - } - sset = append(sset, &ChainSectorInfo{ - Info: oci, - ID: abi.SectorNumber(i), - }) return nil }); err != nil { return nil, err } - return sset, nil + return a, nil +} + +func (s *v0State) LoadPreCommittedSectors() (adt.Map, error) { + return v0adt.AsMap(s.store, s.State.PreCommittedSectors) +} + +func (s *v0State) IsAllocated(num abi.SectorNumber) (bool, error) { + var allocatedSectors bitfield.BitField + if err := s.store.Get(s.store.Context(), s.State.AllocatedSectors, &allocatedSectors); err != nil { + return false, err + } + + return allocatedSectors.IsSet(uint64(num)) } func (s *v0State) LoadDeadline(idx uint64) (Deadline, error) { diff --git a/chain/events/state/predicates.go b/chain/events/state/predicates.go index e1e88e1a1..5a885504f 100644 --- a/chain/events/state/predicates.go +++ b/chain/events/state/predicates.go @@ -4,6 +4,10 @@ import ( "bytes" "context" + v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" + + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" @@ -12,7 +16,6 @@ import ( "github.com/filecoin-project/lotus/chain/actors/builtin/paych" "github.com/filecoin-project/specs-actors/actors/builtin" init_ "github.com/filecoin-project/specs-actors/actors/builtin/init" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" v0adt "github.com/filecoin-project/specs-actors/actors/util/adt" cbor "github.com/ipfs/go-ipld-cbor" typegen "github.com/whyrusleeping/cbor-gen" @@ -387,28 +390,18 @@ func (m *MinerSectorChanges) Remove(key uint64, val *typegen.Deferred) error { func (sp *StatePredicates) OnMinerSectorChange() DiffMinerActorStateFunc { return func(ctx context.Context, oldState, newState *miner.State) (changed bool, user UserData, err error) { - ctxStore := &contextStore{ - ctx: ctx, - cst: sp.cst, - } - sectorChanges := &MinerSectorChanges{ Added: []miner.SectorOnChainInfo{}, Extended: []SectorExtensions{}, Removed: []miner.SectorOnChainInfo{}, } - // no sector changes - if oldState.Sectors.Equals(newState.Sectors) { - return false, nil, nil - } - - oldSectors, err := v0adt.AsArray(ctxStore, oldState.Sectors) + oldSectors, err := oldState.LoadSectorsFromSet(nil, false) if err != nil { return false, nil, err } - newSectors, err := v0adt.AsArray(ctxStore, newState.Sectors) + newSectors, err := newState.LoadSectorsFromSet(nil, false) if err != nil { return false, nil, err } @@ -436,7 +429,7 @@ func (m *MinerPreCommitChanges) AsKey(key string) (abi.Keyer, error) { if err != nil { return nil, err } - return miner.SectorKey(abi.SectorNumber(sector)), nil + return v0miner.SectorKey(abi.SectorNumber(sector)), nil } func (m *MinerPreCommitChanges) Add(key string, val *typegen.Deferred) error { @@ -465,26 +458,17 @@ func (m *MinerPreCommitChanges) Remove(key string, val *typegen.Deferred) error func (sp *StatePredicates) OnMinerPreCommitChange() DiffMinerActorStateFunc { return func(ctx context.Context, oldState, newState *miner.State) (changed bool, user UserData, err error) { - ctxStore := &contextStore{ - ctx: ctx, - cst: sp.cst, - } - precommitChanges := &MinerPreCommitChanges{ Added: []miner.SectorPreCommitOnChainInfo{}, Removed: []miner.SectorPreCommitOnChainInfo{}, } - if oldState.PreCommittedSectors.Equals(newState.PreCommittedSectors) { - return false, nil, nil - } - - oldPrecommits, err := v0adt.AsMap(ctxStore, oldState.PreCommittedSectors) + oldPrecommits, err := oldState.LoadPreCommittedSectors() if err != nil { return false, nil, err } - newPrecommits, err := v0adt.AsMap(ctxStore, newState.PreCommittedSectors) + newPrecommits, err := newState.LoadPreCommittedSectors() if err != nil { return false, nil, err } diff --git a/chain/events/state/predicates_test.go b/chain/events/state/predicates_test.go index 783c720ed..25f25334d 100644 --- a/chain/events/state/predicates_test.go +++ b/chain/events/state/predicates_test.go @@ -4,6 +4,8 @@ import ( "context" "testing" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + "github.com/filecoin-project/go-bitfield" "github.com/stretchr/testify/require" @@ -20,7 +22,7 @@ import ( v0builtin "github.com/filecoin-project/specs-actors/actors/builtin" v0market "github.com/filecoin-project/specs-actors/actors/builtin/market" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" + v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/specs-actors/actors/util/adt" tutils "github.com/filecoin-project/specs-actors/support/testing" @@ -375,12 +377,12 @@ func TestMinerSectorChange(t *testing.T) { } owner, worker := nextIDAddrF(), nextIDAddrF() - si0 := newSectorOnChainInfo(0, tutils.MakeCID("0", &miner.SealedCIDPrefix), big.NewInt(0), abi.ChainEpoch(0), abi.ChainEpoch(10)) - si1 := newSectorOnChainInfo(1, tutils.MakeCID("1", &miner.SealedCIDPrefix), big.NewInt(1), abi.ChainEpoch(1), abi.ChainEpoch(11)) - si2 := newSectorOnChainInfo(2, tutils.MakeCID("2", &miner.SealedCIDPrefix), big.NewInt(2), abi.ChainEpoch(2), abi.ChainEpoch(11)) + si0 := newSectorOnChainInfo(0, tutils.MakeCID("0", &v0miner.SealedCIDPrefix), big.NewInt(0), abi.ChainEpoch(0), abi.ChainEpoch(10)) + si1 := newSectorOnChainInfo(1, tutils.MakeCID("1", &v0miner.SealedCIDPrefix), big.NewInt(1), abi.ChainEpoch(1), abi.ChainEpoch(11)) + si2 := newSectorOnChainInfo(2, tutils.MakeCID("2", &v0miner.SealedCIDPrefix), big.NewInt(2), abi.ChainEpoch(2), abi.ChainEpoch(11)) oldMinerC := createMinerState(ctx, t, store, owner, worker, []miner.SectorOnChainInfo{si0, si1, si2}) - si3 := newSectorOnChainInfo(3, tutils.MakeCID("3", &miner.SealedCIDPrefix), big.NewInt(3), abi.ChainEpoch(3), abi.ChainEpoch(12)) + si3 := newSectorOnChainInfo(3, tutils.MakeCID("3", &v0miner.SealedCIDPrefix), big.NewInt(3), abi.ChainEpoch(3), abi.ChainEpoch(12)) // 0 delete // 1 extend // 2 same @@ -545,20 +547,20 @@ func createMinerState(ctx context.Context, t *testing.T, store adt.Store, owner, return stateC } -func createEmptyMinerState(ctx context.Context, t *testing.T, store adt.Store, owner, worker address.Address) *miner.State { +func createEmptyMinerState(ctx context.Context, t *testing.T, store adt.Store, owner, worker address.Address) *v0miner.State { emptyArrayCid, err := adt.MakeEmptyArray(store).Root() require.NoError(t, err) emptyMap, err := adt.MakeEmptyMap(store).Root() require.NoError(t, err) - emptyDeadline, err := store.Put(store.Context(), miner.ConstructDeadline(emptyArrayCid)) + emptyDeadline, err := store.Put(store.Context(), v0miner.ConstructDeadline(emptyArrayCid)) require.NoError(t, err) - emptyVestingFunds := miner.ConstructVestingFunds() + emptyVestingFunds := v0miner.ConstructVestingFunds() emptyVestingFundsCid, err := store.Put(store.Context(), emptyVestingFunds) require.NoError(t, err) - emptyDeadlines := miner.ConstructDeadlines(emptyDeadline) + emptyDeadlines := v0miner.ConstructDeadlines(emptyDeadline) emptyDeadlinesCid, err := store.Put(store.Context(), emptyDeadlines) require.NoError(t, err) @@ -568,7 +570,7 @@ func createEmptyMinerState(ctx context.Context, t *testing.T, store adt.Store, o emptyBitfieldCid, err := store.Put(store.Context(), emptyBitfield) require.NoError(t, err) - state, err := miner.ConstructState(minerInfo, 123, emptyBitfieldCid, emptyArrayCid, emptyMap, emptyDeadlinesCid, emptyVestingFundsCid) + state, err := v0miner.ConstructState(minerInfo, 123, emptyBitfieldCid, emptyArrayCid, emptyMap, emptyDeadlinesCid, emptyVestingFundsCid) require.NoError(t, err) return state diff --git a/chain/gen/gen.go b/chain/gen/gen.go index d661411fe..4cae19082 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -14,7 +14,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" - saminer "github.com/filecoin-project/specs-actors/actors/builtin/miner" + v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" block "github.com/ipfs/go-block-format" "github.com/ipfs/go-blockservice" "github.com/ipfs/go-cid" @@ -121,7 +121,7 @@ var DefaultRemainderAccountActor = genesis.Actor{ } func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) { - saminer.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ + v0miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } diff --git a/chain/gen/gen_test.go b/chain/gen/gen_test.go index fd6bceb95..496bb016b 100644 --- a/chain/gen/gen_test.go +++ b/chain/gen/gen_test.go @@ -5,7 +5,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" + v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" @@ -14,7 +14,7 @@ import ( ) func init() { - miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ + v0miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } v0power.ConsensusMinerMinPower = big.NewInt(2048) diff --git a/chain/gen/genesis/miners.go b/chain/gen/genesis/miners.go index 88fe57189..f6256ffbc 100644 --- a/chain/gen/genesis/miners.go +++ b/chain/gen/genesis/miners.go @@ -6,6 +6,8 @@ import ( "fmt" "math/rand" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" cbg "github.com/whyrusleeping/cbor-gen" @@ -18,7 +20,7 @@ import ( "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/builtin/market" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" + v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" "github.com/filecoin-project/specs-actors/actors/builtin/reward" "github.com/filecoin-project/specs-actors/actors/runtime" @@ -123,9 +125,10 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid } minerInfos[i].maddr = ma.IDAddress - err = vm.MutateState(ctx, minerInfos[i].maddr, func(cst cbor.IpldStore, st *miner.State) error { - maxPeriods := miner.MaxSectorExpirationExtension / miner.WPoStProvingPeriod - minerInfos[i].presealExp = (maxPeriods-1)*miner.WPoStProvingPeriod + st.ProvingPeriodStart - 1 + // TODO: ActorUpgrade + err = vm.MutateState(ctx, minerInfos[i].maddr, func(cst cbor.IpldStore, st *v0miner.State) error { + maxPeriods := v0miner.MaxSectorExpirationExtension / v0miner.WPoStProvingPeriod + minerInfos[i].presealExp = (maxPeriods-1)*v0miner.WPoStProvingPeriod + st.ProvingPeriodStart - 1 return nil }) @@ -201,7 +204,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid return cid.Undef, xerrors.Errorf("getting deal weight: %w", err) } - sectorWeight := miner.QAPowerForWeight(m.SectorSize, minerInfos[i].presealExp, dweight.DealWeight, dweight.VerifiedDealWeight) + sectorWeight := v0miner.QAPowerForWeight(m.SectorSize, minerInfos[i].presealExp, dweight.DealWeight, dweight.VerifiedDealWeight) qaPow = types.BigAdd(qaPow, sectorWeight) } @@ -246,7 +249,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid return cid.Undef, xerrors.Errorf("getting deal weight: %w", err) } - sectorWeight := miner.QAPowerForWeight(m.SectorSize, minerInfos[i].presealExp, dweight.DealWeight, dweight.VerifiedDealWeight) + sectorWeight := v0miner.QAPowerForWeight(m.SectorSize, minerInfos[i].presealExp, dweight.DealWeight, dweight.VerifiedDealWeight) // we've added fake power for this sector above, remove it now err = vm.MutateState(ctx, builtin.StoragePowerActorAddr, func(cst cbor.IpldStore, st *v0power.State) error { @@ -268,9 +271,9 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid return cid.Undef, xerrors.Errorf("getting current total power: %w", err) } - pcd := miner.PreCommitDepositForPower(epochReward.ThisEpochRewardSmoothed, tpow.QualityAdjPowerSmoothed, sectorWeight) + pcd := v0miner.PreCommitDepositForPower(epochReward.ThisEpochRewardSmoothed, tpow.QualityAdjPowerSmoothed, sectorWeight) - pledge := miner.InitialPledgeForPower( + pledge := v0miner.InitialPledgeForPower( sectorWeight, epochReward.ThisEpochBaselinePower, tpow.PledgeCollateral, diff --git a/chain/store/store_test.go b/chain/store/store_test.go index 8662f10ef..7da8d219d 100644 --- a/chain/store/store_test.go +++ b/chain/store/store_test.go @@ -10,7 +10,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" + v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" @@ -22,7 +22,7 @@ import ( ) func init() { - miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ + v0miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } v0power.ConsensusMinerMinPower = big.NewInt(2048) diff --git a/chain/sync_test.go b/chain/sync_test.go index 5f972ecc6..e8df32c56 100644 --- a/chain/sync_test.go +++ b/chain/sync_test.go @@ -20,7 +20,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" + v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" @@ -43,7 +43,7 @@ func init() { if err != nil { panic(err) } - miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ + v0miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } v0power.ConsensusMinerMinPower = big.NewInt(2048) diff --git a/chain/vm/invoker.go b/chain/vm/invoker.go index 56e3bea84..5024164a7 100644 --- a/chain/vm/invoker.go +++ b/chain/vm/invoker.go @@ -20,7 +20,7 @@ import ( "github.com/filecoin-project/specs-actors/actors/builtin/cron" init_ "github.com/filecoin-project/specs-actors/actors/builtin/init" "github.com/filecoin-project/specs-actors/actors/builtin/market" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" + v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/specs-actors/actors/builtin/multisig" "github.com/filecoin-project/specs-actors/actors/builtin/paych" v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" @@ -52,7 +52,7 @@ func NewInvoker() *Invoker { inv.Register(builtin.CronActorCodeID, cron.Actor{}, cron.State{}) inv.Register(builtin.StoragePowerActorCodeID, v0power.Actor{}, v0power.State{}) inv.Register(builtin.StorageMarketActorCodeID, market.Actor{}, market.State{}) - inv.Register(builtin.StorageMinerActorCodeID, miner.Actor{}, miner.State{}) + inv.Register(builtin.StorageMinerActorCodeID, v0miner.Actor{}, v0miner.State{}) inv.Register(builtin.MultisigActorCodeID, multisig.Actor{}, multisig.State{}) inv.Register(builtin.PaymentChannelActorCodeID, paych.Actor{}, paych.State{}) inv.Register(builtin.VerifiedRegistryActorCodeID, verifreg.Actor{}, verifreg.State{}) diff --git a/chain/vm/syscalls.go b/chain/vm/syscalls.go index 3e221f61f..aab1812d9 100644 --- a/chain/vm/syscalls.go +++ b/chain/vm/syscalls.go @@ -7,6 +7,8 @@ import ( goruntime "runtime" "sync" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + "github.com/filecoin-project/specs-actors/actors/runtime/proof" "github.com/filecoin-project/go-address" @@ -21,9 +23,7 @@ import ( "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/sigs" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/specs-actors/actors/runtime" - "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" ) @@ -197,7 +197,7 @@ func (ss *syscallShim) VerifyBlockSig(blk *types.BlockHeader) error { return err } - info, err := mas.GetInfo(adt.WrapStore(ss.ctx, ss.cst)) + info, err := mas.Info() if err != nil { return err } diff --git a/cli/paych_test.go b/cli/paych_test.go index c1e52899b..1cf95ef6c 100644 --- a/cli/paych_test.go +++ b/cli/paych_test.go @@ -15,7 +15,7 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/go-state-types/big" - saminer "github.com/filecoin-project/specs-actors/actors/builtin/miner" + v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" @@ -41,7 +41,7 @@ import ( func init() { v0power.ConsensusMinerMinPower = big.NewInt(2048) - saminer.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ + v0miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } verifreg.MinVerifiedDealSize = big.NewInt(256) diff --git a/cmd/lotus-pcr/main.go b/cmd/lotus-pcr/main.go index dc12693ca..d265bdd49 100644 --- a/cmd/lotus-pcr/main.go +++ b/cmd/lotus-pcr/main.go @@ -12,6 +12,11 @@ import ( "strconv" "time" + v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" + + "github.com/filecoin-project/go-state-types/network" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" @@ -24,7 +29,6 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/exitcode" "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/api" @@ -321,6 +325,7 @@ type refunderNodeApi interface { StateMinerInitialPledgeCollateral(ctx context.Context, addr address.Address, precommitInfo miner.SectorPreCommitInfo, tsk types.TipSetKey) (types.BigInt, error) StateSectorPreCommitInfo(ctx context.Context, addr address.Address, sector abi.SectorNumber, tsk types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) + StateNetworkVersion(ctx context.Context, tsk types.TipSetKey) (network.Version, error) MpoolPushMessage(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec) (*types.SignedMessage, error) GasEstimateGasPremium(ctx context.Context, nblocksincl uint64, sender address.Address, gaslimit int64, tsk types.TipSetKey) (types.BigInt, error) WalletBalance(ctx context.Context, addr address.Address) (types.BigInt, error) @@ -389,14 +394,28 @@ func (r *refunder) ProcessTipset(ctx context.Context, tipset *types.TipSet, refu continue } - var proveCommitSector miner.ProveCommitSectorParams - if err := proveCommitSector.UnmarshalCBOR(bytes.NewBuffer(m.Params)); err != nil { - log.Warnw("failed to decode provecommit params", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To) + var sn abi.SectorNumber + + nv, err := r.api.StateNetworkVersion(ctx, tipset.Key()) + if err != nil { + log.Warnw("failed to get network version") continue } + if nv < build.ActorUpgradeNetworkVersion { + var proveCommitSector v0miner.ProveCommitSectorParams + if err := proveCommitSector.UnmarshalCBOR(bytes.NewBuffer(m.Params)); err != nil { + log.Warnw("failed to decode provecommit params", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To) + continue + } + + sn = proveCommitSector.SectorNumber + } else { + // TODO: ActorUpgrade + } + // We use the parent tipset key because precommit information is removed when ProveCommitSector is executed - precommitChainInfo, err := r.api.StateSectorPreCommitInfo(ctx, m.To, proveCommitSector.SectorNumber, tipset.Parents()) + precommitChainInfo, err := r.api.StateSectorPreCommitInfo(ctx, m.To, sn, tipset.Parents()) if err != nil { log.Warnw("failed to get precommit info for sector", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", proveCommitSector.SectorNumber) continue diff --git a/extern/storage-sealing/checks.go b/extern/storage-sealing/checks.go index 074c4cfcf..ae5ce0d33 100644 --- a/extern/storage-sealing/checks.go +++ b/extern/storage-sealing/checks.go @@ -4,6 +4,9 @@ import ( "bytes" "context" + "github.com/filecoin-project/lotus/build" + v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" + v0proof "github.com/filecoin-project/specs-actors/actors/runtime/proof" "golang.org/x/xerrors" @@ -13,7 +16,6 @@ import ( "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/extern/sector-storage/zerocomm" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" ) // TODO: For now we handle this by halting state execution, when we get jsonrpc reconnecting @@ -93,7 +95,19 @@ func checkPrecommit(ctx context.Context, maddr address.Address, si SectorInfo, t return &ErrBadCommD{xerrors.Errorf("on chain CommD differs from sector: %s != %s", commD, si.CommD)} } - if height-(si.TicketEpoch+SealRandomnessLookback) > SealRandomnessLookbackLimit(si.SectorType) { + nv, err := api.StateNetworkVersion(ctx, tok) + if err != nil { + return &ErrApi{xerrors.Errorf("calling StateNetworkVersion: %w", err)} + } + + var msd abi.ChainEpoch + if nv < build.ActorUpgradeNetworkVersion { + msd = v0miner.MaxSealDuration[si.SectorType] + } else { + // TODO: ActorUpgrade + } + + if height-(si.TicketEpoch+SealRandomnessLookback) > msd { return &ErrExpiredTicket{xerrors.Errorf("ticket expired: seal height: %d, head: %d", si.TicketEpoch+SealRandomnessLookback, height)} } @@ -139,8 +153,13 @@ func (m *Sealing) checkCommit(ctx context.Context, si SectorInfo, proof []byte, return &ErrNoPrecommit{xerrors.Errorf("precommit info not found on-chain")} } - if pci.PreCommitEpoch+miner.PreCommitChallengeDelay != si.SeedEpoch { - return &ErrBadSeed{xerrors.Errorf("seed epoch doesn't match on chain info: %d != %d", pci.PreCommitEpoch+miner.PreCommitChallengeDelay, si.SeedEpoch)} + pccd, err := m.getPreCommitChallengeDelay(ctx, tok) + if err != nil { + return xerrors.Errorf("failed to get precommit challenge delay: %w", err) + } + + if pci.PreCommitEpoch+pccd != si.SeedEpoch { + return &ErrBadSeed{xerrors.Errorf("seed epoch doesn't match on chain info: %d != %d", pci.PreCommitEpoch+pccd, si.SeedEpoch)} } buf := new(bytes.Buffer) diff --git a/extern/storage-sealing/constants.go b/extern/storage-sealing/constants.go index ebb3d3347..06c48fa4c 100644 --- a/extern/storage-sealing/constants.go +++ b/extern/storage-sealing/constants.go @@ -1,17 +1,11 @@ package sealing import ( - "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" + v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" ) // Epochs -const SealRandomnessLookback = miner.ChainFinality - -// Epochs -func SealRandomnessLookbackLimit(spt abi.RegisteredSealProof) abi.ChainEpoch { - return miner.MaxSealDuration[spt] -} +const SealRandomnessLookback = v0miner.ChainFinality // Epochs const InteractivePoRepConfidence = 6 diff --git a/extern/storage-sealing/fsm_events.go b/extern/storage-sealing/fsm_events.go index ee95ab1c7..3e597d761 100644 --- a/extern/storage-sealing/fsm_events.go +++ b/extern/storage-sealing/fsm_events.go @@ -1,12 +1,12 @@ package sealing import ( + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/ipfs/go-cid" "golang.org/x/xerrors" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/specs-storage/storage" ) diff --git a/extern/storage-sealing/precommit_policy.go b/extern/storage-sealing/precommit_policy.go index 93a963535..e36b8251a 100644 --- a/extern/storage-sealing/precommit_policy.go +++ b/extern/storage-sealing/precommit_policy.go @@ -3,8 +3,11 @@ package sealing import ( "context" + "github.com/filecoin-project/go-state-types/network" + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" + v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" ) type PreCommitPolicy interface { @@ -13,6 +16,7 @@ type PreCommitPolicy interface { type Chain interface { ChainHead(ctx context.Context) (TipSetToken, abi.ChainEpoch, error) + StateNetworkVersion(ctx context.Context, tok TipSetToken) (network.Version, error) } // BasicPreCommitPolicy satisfies PreCommitPolicy. It has two modes: @@ -48,9 +52,9 @@ func NewBasicPreCommitPolicy(api Chain, duration abi.ChainEpoch, provingBoundary // Expiration produces the pre-commit sector expiration epoch for an encoded // replica containing the provided enumeration of pieces and deals. func (p *BasicPreCommitPolicy) Expiration(ctx context.Context, ps ...Piece) (abi.ChainEpoch, error) { - _, epoch, err := p.api.ChainHead(ctx) + tok, epoch, err := p.api.ChainHead(ctx) if err != nil { - return 0, nil + return 0, err } var end *abi.ChainEpoch @@ -76,7 +80,19 @@ func (p *BasicPreCommitPolicy) Expiration(ctx context.Context, ps ...Piece) (abi end = &tmp } - *end += miner.WPoStProvingPeriod - (*end % miner.WPoStProvingPeriod) + p.provingBoundary - 1 + nv, err := p.api.StateNetworkVersion(ctx, tok) + if err != nil { + return 0, err + } + + var wpp abi.ChainEpoch + if nv < build.ActorUpgradeNetworkVersion { + wpp = v0miner.WPoStProvingPeriod + } else { + // TODO: ActorUpgrade + } + + *end += wpp - (*end % wpp) + p.provingBoundary - 1 return *end, nil } diff --git a/extern/storage-sealing/sealing.go b/extern/storage-sealing/sealing.go index 533333860..0fe754217 100644 --- a/extern/storage-sealing/sealing.go +++ b/extern/storage-sealing/sealing.go @@ -8,6 +8,10 @@ import ( "sync" "time" + "github.com/filecoin-project/lotus/build" + v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" + + "github.com/filecoin-project/go-state-types/network" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" @@ -20,10 +24,10 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" statemachine "github.com/filecoin-project/go-statemachine" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/specs-actors/actors/builtin/market" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" ) const SectorStorePrefix = "/sectors" @@ -53,6 +57,7 @@ type SealingAPI interface { StateMinerPreCommitDepositForPower(context.Context, address.Address, miner.SectorPreCommitInfo, TipSetToken) (big.Int, error) StateMinerInitialPledgeCollateral(context.Context, address.Address, miner.SectorPreCommitInfo, TipSetToken) (big.Int, error) StateMarketStorageDeal(context.Context, abi.DealID, TipSetToken) (market.DealProposal, error) + StateNetworkVersion(ctx context.Context, tok TipSetToken) (network.Version, error) SendMsg(ctx context.Context, from, to address.Address, method abi.MethodNum, value, maxFee abi.TokenAmount, params []byte) (cid.Cid, error) ChainHead(ctx context.Context) (TipSetToken, abi.ChainEpoch, error) ChainGetRandomnessFromBeacon(ctx context.Context, tok TipSetToken, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error) @@ -417,3 +422,18 @@ func getDealPerSectorLimit(size abi.SectorSize) uint64 { } return 512 } + +func (m *Sealing) getPreCommitChallengeDelay(ctx context.Context, tok TipSetToken) (abi.ChainEpoch, error) { + nv, err := m.api.StateNetworkVersion(ctx, tok) + if err != nil { + return -1, xerrors.Errorf("failed to get network version: %w", err) + } + + if nv < build.ActorUpgradeNetworkVersion { + return v0miner.PreCommitChallengeDelay, nil + } else { + // TODO: ActorUpgrade + return -1, nil + } + +} diff --git a/extern/storage-sealing/states_failed.go b/extern/storage-sealing/states_failed.go index 4dd945a81..b026e70f9 100644 --- a/extern/storage-sealing/states_failed.go +++ b/extern/storage-sealing/states_failed.go @@ -4,13 +4,14 @@ import ( "bytes" "time" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + "golang.org/x/xerrors" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/exitcode" "github.com/filecoin-project/go-statemachine" "github.com/filecoin-project/specs-actors/actors/builtin/market" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/lotus/extern/sector-storage/zerocomm" ) diff --git a/extern/storage-sealing/states_sealing.go b/extern/storage-sealing/states_sealing.go index 5e2b72ee1..e7a3d7450 100644 --- a/extern/storage-sealing/states_sealing.go +++ b/extern/storage-sealing/states_sealing.go @@ -4,6 +4,10 @@ import ( "bytes" "context" + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" + "golang.org/x/xerrors" "github.com/filecoin-project/go-state-types/abi" @@ -12,7 +16,6 @@ import ( "github.com/filecoin-project/go-state-types/exitcode" "github.com/filecoin-project/go-statemachine" "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/specs-storage/storage" ) @@ -180,7 +183,21 @@ func (m *Sealing) handlePreCommitting(ctx statemachine.Context, sector SectorInf // Sectors must last _at least_ MinSectorExpiration + MaxSealDuration. // TODO: The "+10" allows the pre-commit to take 10 blocks to be accepted. - if minExpiration := height + miner.MaxSealDuration[sector.SectorType] + miner.MinSectorExpiration + 10; expiration < minExpiration { + nv, err := m.api.StateNetworkVersion(ctx.Context(), tok) + if err != nil { + return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("failed to get network version: %w", err)}) + } + + var msd abi.ChainEpoch + var mse abi.ChainEpoch + if nv < build.ActorUpgradeNetworkVersion { + msd = v0miner.MaxSealDuration[sector.SectorType] + mse = v0miner.MinSectorExpiration + } else { + // TODO: ActorUpgrade + } + + if minExpiration := height + msd + mse + 10; expiration < minExpiration { expiration = minExpiration } // TODO: enforce a reasonable _maximum_ sector lifetime? @@ -253,7 +270,7 @@ func (m *Sealing) handlePreCommitWait(ctx statemachine.Context, sector SectorInf func (m *Sealing) handleWaitSeed(ctx statemachine.Context, sector SectorInfo) error { tok, _, err := m.api.ChainHead(ctx.Context()) if err != nil { - log.Errorf("handleCommitting: api error, not proceeding: %+v", err) + log.Errorf("handleWaitSeed: api error, not proceeding: %+v", err) return nil } @@ -265,7 +282,17 @@ func (m *Sealing) handleWaitSeed(ctx statemachine.Context, sector SectorInfo) er return ctx.Send(SectorChainPreCommitFailed{error: xerrors.Errorf("precommit info not found on chain")}) } - randHeight := pci.PreCommitEpoch + miner.PreCommitChallengeDelay + nv, err := m.api.StateNetworkVersion(ctx.Context(), tok) + if err != nil { + return ctx.Send(SectorChainPreCommitFailed{xerrors.Errorf("failed to get network version: %w", err)}) + } + + pccd, err := m.getPreCommitChallengeDelay(ctx.Context(), tok) + if err != nil { + return ctx.Send(SectorChainPreCommitFailed{xerrors.Errorf("failed to get precommit challenge delay: %w", err)}) + } + + randHeight := pci.PreCommitEpoch + pccd err = m.events.ChainAt(func(ectx context.Context, _ TipSetToken, curH abi.ChainEpoch) error { // in case of null blocks the randomness can land after the tipset we @@ -356,14 +383,23 @@ func (m *Sealing) handleSubmitCommit(ctx statemachine.Context, sector SectorInfo return ctx.Send(SectorCommitFailed{xerrors.Errorf("commit check error: %w", err)}) } - params := &miner.ProveCommitSectorParams{ - SectorNumber: sector.SectorNumber, - Proof: sector.Proof, + nv, err := m.api.StateNetworkVersion(ctx.Context(), tok) + if err != nil { + return ctx.Send(SectorCommitFailed{xerrors.Errorf("failed to get network version: %w", err)}) } enc := new(bytes.Buffer) - if err := params.MarshalCBOR(enc); err != nil { - return ctx.Send(SectorCommitFailed{xerrors.Errorf("could not serialize commit sector parameters: %w", err)}) + if nv < build.ActorUpgradeNetworkVersion { + params := &v0miner.ProveCommitSectorParams{ + SectorNumber: sector.SectorNumber, + Proof: sector.Proof, + } + + if err := params.MarshalCBOR(enc); err != nil { + return ctx.Send(SectorCommitFailed{xerrors.Errorf("could not serialize commit sector parameters: %w", err)}) + } + } else { + // TODO: ActorUpgrade } waddr, err := m.api.StateMinerWorkerAddress(ctx.Context(), m.maddr, tok) diff --git a/extern/storage-sealing/upgrade_queue.go b/extern/storage-sealing/upgrade_queue.go index 650fdc83d..371a5c862 100644 --- a/extern/storage-sealing/upgrade_queue.go +++ b/extern/storage-sealing/upgrade_queue.go @@ -3,11 +3,12 @@ package sealing import ( "context" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + "golang.org/x/xerrors" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" ) func (m *Sealing) IsMarkedForUpgrade(id abi.SectorNumber) bool { diff --git a/markets/storageadapter/client.go b/markets/storageadapter/client.go index 4168792da..0c9e66eda 100644 --- a/markets/storageadapter/client.go +++ b/markets/storageadapter/client.go @@ -6,6 +6,8 @@ import ( "bytes" "context" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + "github.com/filecoin-project/go-state-types/big" "golang.org/x/xerrors" @@ -29,7 +31,6 @@ import ( "github.com/filecoin-project/lotus/node/impl/full" "github.com/filecoin-project/specs-actors/actors/builtin" samarket "github.com/filecoin-project/specs-actors/actors/builtin/market" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/ipfs/go-cid" ) diff --git a/markets/storageadapter/provider.go b/markets/storageadapter/provider.go index 7af1808c1..22d6c1e1d 100644 --- a/markets/storageadapter/provider.go +++ b/markets/storageadapter/provider.go @@ -8,6 +8,8 @@ import ( "io" "time" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" "golang.org/x/xerrors" @@ -20,7 +22,6 @@ import ( "github.com/filecoin-project/go-state-types/exitcode" "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/builtin/market" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" "github.com/filecoin-project/lotus/api" diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 7a107c2fd..6c60269ab 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -41,7 +41,7 @@ import ( "github.com/filecoin-project/go-multistore" "github.com/filecoin-project/go-padreader" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" + v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" marketevents "github.com/filecoin-project/lotus/markets/loggers" @@ -87,7 +87,7 @@ func calcDealExpiration(minDuration uint64, md *dline.Info, startEpoch abi.Chain minExp := startEpoch + abi.ChainEpoch(minDuration) // Align on miners ProvingPeriodBoundary - return minExp + miner.WPoStProvingPeriod - (minExp % miner.WPoStProvingPeriod) + (md.PeriodStart % miner.WPoStProvingPeriod) - 1 + return minExp + v0miner.WPoStProvingPeriod - (minExp % v0miner.WPoStProvingPeriod) + (md.PeriodStart % v0miner.WPoStProvingPeriod) - 1 } func (a *API) imgr() *importmgr.Mgr { diff --git a/node/node_test.go b/node/node_test.go index 0611bca60..92b741f73 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -10,7 +10,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/lotus/lib/lotuslog" - saminer "github.com/filecoin-project/specs-actors/actors/builtin/miner" + v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" logging "github.com/ipfs/go-log/v2" @@ -22,7 +22,7 @@ func init() { _ = logging.SetLogLevel("*", "INFO") v0power.ConsensusMinerMinPower = big.NewInt(2048) - saminer.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ + v0miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } verifreg.MinVerifiedDealSize = big.NewInt(256) @@ -68,7 +68,7 @@ func TestAPIDealFlowReal(t *testing.T) { logging.SetLogLevel("sub", "ERROR") logging.SetLogLevel("storageminer", "ERROR") - saminer.PreCommitChallengeDelay = 5 + v0miner.PreCommitChallengeDelay = 5 t.Run("basic", func(t *testing.T) { test.TestDealFlow(t, builder.Builder, time.Second, false, false) diff --git a/node/test/builder.go b/node/test/builder.go index de2071e7a..2b26c13fc 100644 --- a/node/test/builder.go +++ b/node/test/builder.go @@ -37,7 +37,7 @@ import ( "github.com/filecoin-project/lotus/node/repo" "github.com/filecoin-project/lotus/storage/mockstorage" "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" + v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/ipfs/go-datastore" "github.com/libp2p/go-libp2p-core/crypto" "github.com/libp2p/go-libp2p-core/peer" @@ -83,7 +83,7 @@ func CreateTestStorageNode(ctx context.Context, t *testing.T, waddr address.Addr peerid, err := peer.IDFromPrivateKey(pk) require.NoError(t, err) - enc, err := actors.SerializeParams(&miner.ChangePeerIDParams{NewID: abi.PeerID(peerid)}) + enc, err := actors.SerializeParams(&v0miner.ChangePeerIDParams{NewID: abi.PeerID(peerid)}) require.NoError(t, err) msg := &types.Message{ diff --git a/storage/adapter_storage_miner.go b/storage/adapter_storage_miner.go index ef917b758..8b64789ad 100644 --- a/storage/adapter_storage_miner.go +++ b/storage/adapter_storage_miner.go @@ -4,7 +4,7 @@ import ( "bytes" "context" - "github.com/filecoin-project/go-bitfield" + "github.com/filecoin-project/go-state-types/network" "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" @@ -16,7 +16,6 @@ import ( "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/builtin/market" - "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/apibstore" @@ -84,7 +83,7 @@ func (s SealingAPIAdapter) StateMinerWorkerAddress(ctx context.Context, maddr ad return mi.Worker, nil } -func (s SealingAPIAdapter) StateMinerDeadlines(ctx context.Context, maddr address.Address, tok sealing.TipSetToken) ([]*miner.Deadline, error) { +func (s SealingAPIAdapter) StateMinerDeadlines(ctx context.Context, maddr address.Address, tok sealing.TipSetToken) ([]miner.Deadline, error) { tsk, err := types.TipSetKeyFromBytes(tok) if err != nil { return nil, xerrors.Errorf("failed to unmarshal TipSetToken to TipSetKey: %w", err) @@ -185,23 +184,13 @@ func (s SealingAPIAdapter) StateSectorPreCommitInfo(ctx context.Context, maddr a if err != nil { return nil, xerrors.Errorf("handleSealFailed(%d): temp error: loading miner state: %+v", sectorNumber, err) } - stor := store.ActorStore(ctx, apibstore.NewAPIBlockstore(s.delegate)) - precommits, err := adt.AsMap(stor, state.PreCommittedSectors) - if err != nil { - return nil, err - } - var pci miner.SectorPreCommitOnChainInfo - ok, err := precommits.Get(abi.UIntKey(uint64(sectorNumber)), &pci) + pci, err := state.GetPrecommittedSector(sectorNumber) if err != nil { return nil, err } - if !ok { - var allocated bitfield.BitField - if err := stor.Get(ctx, state.AllocatedSectors, &allocated); err != nil { - return nil, xerrors.Errorf("loading allocated sector bitfield: %w", err) - } - set, err := allocated.IsSet(uint64(sectorNumber)) + if pci != nil { + set, err := state.IsAllocated(sectorNumber) if err != nil { return nil, xerrors.Errorf("checking if sector is allocated: %w", err) } @@ -212,7 +201,7 @@ func (s SealingAPIAdapter) StateSectorPreCommitInfo(ctx context.Context, maddr a return nil, nil } - return &pci, nil + return pci, nil } func (s SealingAPIAdapter) StateSectorGetInfo(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok sealing.TipSetToken) (*miner.SectorOnChainInfo, error) { @@ -258,6 +247,15 @@ func (s SealingAPIAdapter) StateMarketStorageDeal(ctx context.Context, dealID ab return deal.Proposal, nil } +func (s SealingAPIAdapter) StateNetworkVersion(ctx context.Context, tok sealing.TipSetToken) (network.Version, error) { + tsk, err := types.TipSetKeyFromBytes(tok) + if err != nil { + return -1, err + } + + return s.delegate.StateNetworkVersion(ctx, tsk) +} + func (s SealingAPIAdapter) SendMsg(ctx context.Context, from, to address.Address, method abi.MethodNum, value, maxFee abi.TokenAmount, params []byte) (cid.Cid, error) { msg := types.Message{ To: to, diff --git a/storage/miner.go b/storage/miner.go index 3485dba03..693ad7c05 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -5,6 +5,10 @@ import ( "errors" "time" + v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" + + "github.com/filecoin-project/go-state-types/network" + "github.com/filecoin-project/go-state-types/dline" "github.com/filecoin-project/go-bitfield" @@ -71,7 +75,8 @@ type storageMinerApi interface { StateSectorPreCommitInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) StateSectorGetInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorOnChainInfo, error) StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*miner.SectorLocation, error) - StateMinerInfo(context.Context, address.Address, types.TipSetKey) (api.MinerInfo, error) + StateMinerInfo(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error) + StateMinerDeadlines(context.Context, address.Address, types.TipSetKey) ([]miner.Deadline, error) StateMinerProvingDeadline(context.Context, address.Address, types.TipSetKey) (*dline.Info, error) StateMinerPreCommitDepositForPower(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error) StateMinerInitialPledgeCollateral(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error) @@ -83,6 +88,7 @@ type storageMinerApi interface { StateMinerFaults(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error) StateMinerRecoveries(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error) StateAccountKey(context.Context, address.Address, types.TipSetKey) (address.Address, error) + StateNetworkVersion(context.Context, types.TipSetKey) (network.Version, error) MpoolPushMessage(context.Context, *types.Message, *api.MessageSendSpec) (*types.SignedMessage, error) @@ -139,7 +145,8 @@ func (m *Miner) Run(ctx context.Context) error { evts := events.NewEvents(ctx, m.api) adaptedAPI := NewSealingAPIAdapter(m.api) - pcp := sealing.NewBasicPreCommitPolicy(adaptedAPI, miner.MaxSectorExpirationExtension-(miner.WPoStProvingPeriod*2), md.PeriodStart%miner.WPoStProvingPeriod) + // TODO: Maybe we update this policy after actor upgrades? + pcp := sealing.NewBasicPreCommitPolicy(adaptedAPI, v0miner.MaxSectorExpirationExtension-(v0miner.WPoStProvingPeriod*2), md.PeriodStart%v0miner.WPoStProvingPeriod) m.sealing = sealing.New(adaptedAPI, fc, NewEventsAdapter(evts), m.maddr, m.ds, m.sealer, m.sc, m.verif, &pcp, sealing.GetSealingConfigFunc(m.getSealConfig), m.handleSealingNotifications) go m.sealing.Run(ctx) //nolint:errcheck // logged intside the function diff --git a/storage/wdpost_journal.go b/storage/wdpost_journal.go index c1a4d4817..48eb2f2b1 100644 --- a/storage/wdpost_journal.go +++ b/storage/wdpost_journal.go @@ -3,7 +3,7 @@ package storage import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/dline" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/ipfs/go-cid" ) diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index aa2706455..bfc2bc95e 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -14,20 +14,18 @@ import ( "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/go-state-types/dline" "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/ipfs/go-cid" "go.opencensus.io/trace" "golang.org/x/xerrors" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/specs-actors/actors/runtime/proof" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/apibstore" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" - iminer "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/journal" @@ -155,7 +153,7 @@ func (s *WindowPoStScheduler) checkSectors(ctx context.Context, check bitfield.B return sbf, nil } -func (s *WindowPoStScheduler) checkNextRecoveries(ctx context.Context, dlIdx uint64, partitions []iminer.Partition) ([]miner.RecoveryDeclaration, *types.SignedMessage, error) { +func (s *WindowPoStScheduler) checkNextRecoveries(ctx context.Context, dlIdx uint64, partitions []miner.Partition) ([]miner.RecoveryDeclaration, *types.SignedMessage, error) { ctx, span := trace.StartSpan(ctx, "storage.checkNextRecoveries") defer span.End() @@ -167,11 +165,11 @@ func (s *WindowPoStScheduler) checkNextRecoveries(ctx context.Context, dlIdx uin for partIdx, partition := range partitions { faults, err := partition.FaultySectors() if err != nil { - return xerrors.Errorf("getting faults: %w", err) + return nil, nil, xerrors.Errorf("getting faults: %w", err) } recovering, err := partition.RecoveringSectors() if err != nil { - return xerrors.Errorf("getting recovering: %w", err) + return nil, nil, xerrors.Errorf("getting recovering: %w", err) } unrecovered, err := bitfield.SubtractBitField(faults, recovering) if err != nil { @@ -254,7 +252,7 @@ func (s *WindowPoStScheduler) checkNextRecoveries(ctx context.Context, dlIdx uin return recoveries, sm, nil } -func (s *WindowPoStScheduler) checkNextFaults(ctx context.Context, dlIdx uint64, partitions []iminer.Partition) ([]miner.FaultDeclaration, *types.SignedMessage, error) { +func (s *WindowPoStScheduler) checkNextFaults(ctx context.Context, dlIdx uint64, partitions []miner.Partition) ([]miner.FaultDeclaration, *types.SignedMessage, error) { ctx, span := trace.StartSpan(ctx, "storage.checkNextFaults") defer span.End() @@ -348,7 +346,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty return nil, xerrors.Errorf("resolving actor: %w", err) } - mas, err := iminer.Load(stor, act) + mas, err := miner.Load(stor, act) if err != nil { return nil, xerrors.Errorf("getting miner state: %w", err) } @@ -365,8 +363,8 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty log.Errorf("loading deadline: %v", err) return } - var partitions []iminer.Partition - err = dl.ForEachPartition(func(_ uint64, part iminer.Partition) error { + var partitions []miner.Partition + err = dl.ForEachPartition(func(_ uint64, part miner.Partition) error { partitions = append(partitions, part) return nil }) @@ -435,9 +433,9 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty return nil, xerrors.Errorf("loading deadline: %w", err) } - var partitions []iminer.Partitions - err = dl.ForEachPartition(func(_ uint64, part iminer.Partition) error { - partitions = apppend(partitions, part) + var partitions []miner.Partition + err = dl.ForEachPartition(func(_ uint64, part miner.Partition) error { + partitions = append(partitions, part) return nil }) if err != nil { @@ -465,7 +463,11 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty return nil, xerrors.Errorf("getting active sectors: %w", err) } - toProve, err = bitfield.MergeBitFields(toProve, partition.Recoveries) + recs, err := partition.RecoveringSectors() + if err != nil { + return nil, xerrors.Errorf("getting recovering sectors: %w", err) + } + toProve, err = bitfield.MergeBitFields(toProve, recs) if err != nil { return nil, xerrors.Errorf("adding recoveries to set of sectors to prove: %w", err) } @@ -492,7 +494,12 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty skipCount += sc - ssi, err := s.sectorsForProof(ctx, good, partition.Sectors, ts) + partitionSectors, err := partition.AllSectors() + if err != nil { + return nil, xerrors.Errorf("getting partition sectors: %w", err) + } + + ssi, err := s.sectorsForProof(ctx, good, partitionSectors, ts) if err != nil { return nil, xerrors.Errorf("getting sorted sector info: %w", err) } From 9e48dd211ac1e6de6a6be5eef434fea7dd5fc1ef Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Thu, 17 Sep 2020 02:34:15 -0400 Subject: [PATCH 168/885] Fixups --- chain/actors/builtin/miner/miner.go | 2 +- chain/events/state/predicates.go | 8 ++--- chain/stmgr/utils.go | 41 +++++++++++++++++++++--- extern/storage-sealing/states_sealing.go | 5 --- 4 files changed, 42 insertions(+), 14 deletions(-) diff --git a/chain/actors/builtin/miner/miner.go b/chain/actors/builtin/miner/miner.go index 90c44eff3..4acb41dae 100644 --- a/chain/actors/builtin/miner/miner.go +++ b/chain/actors/builtin/miner/miner.go @@ -39,7 +39,7 @@ type State interface { GetSectorExpiration(abi.SectorNumber) (*SectorExpiration, error) GetPrecommittedSector(abi.SectorNumber) (*SectorPreCommitOnChainInfo, error) LoadSectorsFromSet(filter *bitfield.BitField, filterOut bool) (adt.Array, error) - LoadPreCommittedSectorsFromSet(filter *bitfield.BitField, filterOut bool) (adt.Map, error) + LoadPreCommittedSectors() (adt.Map, error) IsAllocated(abi.SectorNumber) (bool, error) LoadDeadline(idx uint64) (Deadline, error) diff --git a/chain/events/state/predicates.go b/chain/events/state/predicates.go index 5a885504f..3e515e6a4 100644 --- a/chain/events/state/predicates.go +++ b/chain/events/state/predicates.go @@ -302,7 +302,7 @@ func (sp *StatePredicates) AvailableBalanceChangedForAddresses(getAddrs func() [ } } -type DiffMinerActorStateFunc func(ctx context.Context, oldState *miner.State, newState *miner.State) (changed bool, user UserData, err error) +type DiffMinerActorStateFunc func(ctx context.Context, oldState miner.State, newState miner.State) (changed bool, user UserData, err error) func (sp *StatePredicates) OnInitActorChange(diffInitActorState DiffInitActorStateFunc) DiffTipSetKeyFunc { return sp.OnActorStateChanged(builtin.InitActorAddr, func(ctx context.Context, oldActorState, newActorState *types.Actor) (changed bool, user UserData, err error) { @@ -329,7 +329,7 @@ func (sp *StatePredicates) OnMinerActorChange(minerAddr address.Address, diffMin if err := sp.cst.Get(ctx, newActorState.Head, &newState); err != nil { return false, nil, err } - return diffMinerActorState(ctx, &oldState, &newState) + return diffMinerActorState(ctx, oldState, newState) }) } @@ -389,7 +389,7 @@ func (m *MinerSectorChanges) Remove(key uint64, val *typegen.Deferred) error { } func (sp *StatePredicates) OnMinerSectorChange() DiffMinerActorStateFunc { - return func(ctx context.Context, oldState, newState *miner.State) (changed bool, user UserData, err error) { + return func(ctx context.Context, oldState, newState miner.State) (changed bool, user UserData, err error) { sectorChanges := &MinerSectorChanges{ Added: []miner.SectorOnChainInfo{}, Extended: []SectorExtensions{}, @@ -457,7 +457,7 @@ func (m *MinerPreCommitChanges) Remove(key string, val *typegen.Deferred) error } func (sp *StatePredicates) OnMinerPreCommitChange() DiffMinerActorStateFunc { - return func(ctx context.Context, oldState, newState *miner.State) (changed bool, user UserData, err error) { + return func(ctx context.Context, oldState, newState miner.State) (changed bool, user UserData, err error) { precommitChanges := &MinerPreCommitChanges{ Added: []miner.SectorPreCommitOnChainInfo{}, Removed: []miner.SectorPreCommitOnChainInfo{}, diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index e58d69156..16167af75 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -9,6 +9,8 @@ import ( "runtime" "strings" + v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" + saruntime "github.com/filecoin-project/specs-actors/actors/runtime" "github.com/filecoin-project/specs-actors/actors/runtime/proof" @@ -156,7 +158,29 @@ func GetMinerSectorSet(ctx context.Context, sm *StateManager, ts *types.TipSet, return nil, xerrors.Errorf("(get sset) failed to load miner actor state: %w", err) } - return mas.LoadSectorsFromSet(filter, filterOut) + sectors, err := mas.LoadSectorsFromSet(filter, filterOut) + if err != nil { + return nil, xerrors.Errorf("(get sset) failed to load sectors: %w", err) + } + + var sset []*miner.ChainSectorInfo + var v cbg.Deferred + if err := sectors.ForEach(&v, func(i int64) error { + var oci v0miner.SectorOnChainInfo + if err := oci.UnmarshalCBOR(bytes.NewReader(v.Raw)); err != nil { + return err + } + sset = append(sset, &miner.ChainSectorInfo{ + Info: oci, + ID: abi.SectorNumber(i), + }) + + return nil + }); err != nil { + return nil, err + } + + return sset, nil } func GetSectorsForWinningPoSt(ctx context.Context, pv ffiwrapper.Verifier, sm *StateManager, st cid.Cid, maddr address.Address, rand abi.PoStRandomness) ([]proof.SectorInfo, error) { @@ -220,12 +244,21 @@ func GetSectorsForWinningPoSt(ctx context.Context, pv ffiwrapper.Verifier, sm *S out := make([]proof.SectorInfo, len(ids)) for i, n := range ids { - s := sectors[n] + var sinfo miner.SectorOnChainInfo + found, err := sectors.Get(n, &sinfo) + + if err != nil { + return nil, xerrors.Errorf("loading sector info: %w", err) + } + + if !found { + return nil, xerrors.Errorf("didn't find sector info for sector %d", n) + } out[i] = proof.SectorInfo{ SealProof: spt, - SectorNumber: s.ID, - SealedCID: s.Info.SealedCID, + SectorNumber: sinfo.SectorNumber, + SealedCID: sinfo.SealedCID, } } diff --git a/extern/storage-sealing/states_sealing.go b/extern/storage-sealing/states_sealing.go index e7a3d7450..f0ff4025d 100644 --- a/extern/storage-sealing/states_sealing.go +++ b/extern/storage-sealing/states_sealing.go @@ -282,11 +282,6 @@ func (m *Sealing) handleWaitSeed(ctx statemachine.Context, sector SectorInfo) er return ctx.Send(SectorChainPreCommitFailed{error: xerrors.Errorf("precommit info not found on chain")}) } - nv, err := m.api.StateNetworkVersion(ctx.Context(), tok) - if err != nil { - return ctx.Send(SectorChainPreCommitFailed{xerrors.Errorf("failed to get network version: %w", err)}) - } - pccd, err := m.getPreCommitChallengeDelay(ctx.Context(), tok) if err != nil { return ctx.Send(SectorChainPreCommitFailed{xerrors.Errorf("failed to get precommit challenge delay: %w", err)}) From b60614982ea91a51e128c175da43d1d4e425a0bb Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Thu, 17 Sep 2020 02:42:39 -0400 Subject: [PATCH 169/885] Migrate reward actor --- chain/actors/builtin/reward/reward.go | 2 ++ chain/actors/builtin/reward/v0.go | 5 +++++ chain/gen/genesis/miners.go | 10 ++++----- chain/gen/genesis/t02_reward.go | 4 ++-- chain/stmgr/stmgr.go | 30 +++++++++++++++++---------- chain/stmgr/utils.go | 4 ++-- chain/vm/invoker.go | 4 ++-- 7 files changed, 37 insertions(+), 22 deletions(-) diff --git a/chain/actors/builtin/reward/reward.go b/chain/actors/builtin/reward/reward.go index ba03feced..d400258df 100644 --- a/chain/actors/builtin/reward/reward.go +++ b/chain/actors/builtin/reward/reward.go @@ -1,6 +1,7 @@ package reward import ( + "github.com/filecoin-project/go-state-types/abi" "golang.org/x/xerrors" "github.com/filecoin-project/go-state-types/cbor" @@ -30,4 +31,5 @@ type State interface { cbor.Marshaler RewardSmoothed() (builtin.FilterEstimate, error) + TotalStoragePowerReward() abi.TokenAmount } diff --git a/chain/actors/builtin/reward/v0.go b/chain/actors/builtin/reward/v0.go index a894fa752..b0558f0ae 100644 --- a/chain/actors/builtin/reward/v0.go +++ b/chain/actors/builtin/reward/v0.go @@ -1,6 +1,7 @@ package reward import ( + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/specs-actors/actors/builtin/reward" "github.com/filecoin-project/specs-actors/actors/util/adt" @@ -14,3 +15,7 @@ type v0State struct { func (s *v0State) RewardSmoothed() (builtin.FilterEstimate, error) { return *s.State.ThisEpochRewardSmoothed, nil } + +func (s *v0State) TotalStoragePowerReward() abi.TokenAmount { + return s.State.TotalMined +} diff --git a/chain/gen/genesis/miners.go b/chain/gen/genesis/miners.go index f6256ffbc..aff226882 100644 --- a/chain/gen/genesis/miners.go +++ b/chain/gen/genesis/miners.go @@ -22,7 +22,7 @@ import ( "github.com/filecoin-project/specs-actors/actors/builtin/market" v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" - "github.com/filecoin-project/specs-actors/actors/builtin/reward" + v0reward "github.com/filecoin-project/specs-actors/actors/builtin/reward" "github.com/filecoin-project/specs-actors/actors/runtime" "github.com/filecoin-project/lotus/chain/state" @@ -222,8 +222,8 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid return cid.Undef, xerrors.Errorf("mutating state: %w", err) } - err = vm.MutateState(ctx, builtin.RewardActorAddr, func(sct cbor.IpldStore, st *reward.State) error { - *st = *reward.ConstructState(qaPow) + err = vm.MutateState(ctx, builtin.RewardActorAddr, func(sct cbor.IpldStore, st *v0reward.State) error { + *st = *v0reward.ConstructState(qaPow) return nil }) if err != nil { @@ -381,13 +381,13 @@ func dealWeight(ctx context.Context, vm *vm.VM, maddr address.Address, dealIDs [ return dealWeights, nil } -func currentEpochBlockReward(ctx context.Context, vm *vm.VM, maddr address.Address) (*reward.ThisEpochRewardReturn, error) { +func currentEpochBlockReward(ctx context.Context, vm *vm.VM, maddr address.Address) (*v0reward.ThisEpochRewardReturn, error) { rwret, err := doExecValue(ctx, vm, builtin.RewardActorAddr, maddr, big.Zero(), builtin.MethodsReward.ThisEpochReward, nil) if err != nil { return nil, err } - var epochReward reward.ThisEpochRewardReturn + var epochReward v0reward.ThisEpochRewardReturn if err := epochReward.UnmarshalCBOR(bytes.NewReader(rwret)); err != nil { return nil, err } diff --git a/chain/gen/genesis/t02_reward.go b/chain/gen/genesis/t02_reward.go index d499b24d0..e29e390f9 100644 --- a/chain/gen/genesis/t02_reward.go +++ b/chain/gen/genesis/t02_reward.go @@ -6,7 +6,7 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/builtin/reward" + v0reward "github.com/filecoin-project/specs-actors/actors/builtin/reward" cbor "github.com/ipfs/go-ipld-cbor" "github.com/filecoin-project/lotus/build" @@ -17,7 +17,7 @@ import ( func SetupRewardActor(bs bstore.Blockstore, qaPower big.Int) (*types.Actor, error) { cst := cbor.NewCborStore(bs) - st := reward.ConstructState(qaPower) + st := v0reward.ConstructState(qaPower) hcid, err := cst.Put(context.TODO(), st) if err != nil { diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index 8cb1329d2..8a72b4ba4 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -5,6 +5,8 @@ import ( "fmt" "sync" + "github.com/filecoin-project/lotus/chain/actors/builtin/reward" + "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" logging "github.com/ipfs/go-log/v2" @@ -18,7 +20,7 @@ import ( "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/builtin/multisig" - "github.com/filecoin-project/specs-actors/actors/builtin/reward" + v0reward "github.com/filecoin-project/specs-actors/actors/builtin/reward" "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/lotus/api" @@ -242,15 +244,21 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp processedMsgs[m.Cid()] = true } - var err error - params, err := actors.SerializeParams(&reward.AwardBlockRewardParams{ - Miner: b.Miner, - Penalty: penalty, - GasReward: gasReward, - WinCount: b.WinCount, - }) - if err != nil { - return cid.Undef, cid.Undef, xerrors.Errorf("failed to serialize award params: %w", err) + var params []byte + + nv := sm.GetNtwkVersion(ctx, epoch) + if nv < build.ActorUpgradeNetworkVersion { + params, err = actors.SerializeParams(&v0reward.AwardBlockRewardParams{ + Miner: b.Miner, + Penalty: penalty, + GasReward: gasReward, + WinCount: b.WinCount, + }) + if err != nil { + return cid.Undef, cid.Undef, xerrors.Errorf("failed to serialize award params: %w", err) + } + } else { + // TODO: ActorUpgrade } sysAct, err := vmi.StateTree().GetActor(builtin.SystemActorAddr) @@ -1018,7 +1026,7 @@ func GetFilMined(ctx context.Context, st *state.StateTree) (abi.TokenAmount, err return big.Zero(), xerrors.Errorf("failed to load reward state: %w", err) } - return rst.TotalMined, nil + return rst.TotalStoragePowerReward(), nil } func getFilMarketLocked(ctx context.Context, st *state.StateTree) (abi.TokenAmount, error) { diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index 16167af75..b000a62e1 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -29,7 +29,7 @@ import ( "github.com/filecoin-project/specs-actors/actors/builtin/cron" "github.com/filecoin-project/specs-actors/actors/builtin/multisig" "github.com/filecoin-project/specs-actors/actors/builtin/paych" - "github.com/filecoin-project/specs-actors/actors/builtin/reward" + v0reward "github.com/filecoin-project/specs-actors/actors/builtin/reward" "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" "github.com/filecoin-project/lotus/api" @@ -555,7 +555,7 @@ func init() { builtin.StorageMarketActorCodeID: {builtin.MethodsMarket, market.Actor{}}, builtin.PaymentChannelActorCodeID: {builtin.MethodsPaych, paych.Actor{}}, builtin.MultisigActorCodeID: {builtin.MethodsMultisig, multisig.Actor{}}, - builtin.RewardActorCodeID: {builtin.MethodsReward, reward.Actor{}}, + builtin.RewardActorCodeID: {builtin.MethodsReward, v0reward.Actor{}}, builtin.VerifiedRegistryActorCodeID: {builtin.MethodsVerifiedRegistry, verifreg.Actor{}}, } diff --git a/chain/vm/invoker.go b/chain/vm/invoker.go index 5024164a7..a86a0f03c 100644 --- a/chain/vm/invoker.go +++ b/chain/vm/invoker.go @@ -24,7 +24,7 @@ import ( "github.com/filecoin-project/specs-actors/actors/builtin/multisig" "github.com/filecoin-project/specs-actors/actors/builtin/paych" v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" - "github.com/filecoin-project/specs-actors/actors/builtin/reward" + v0reward "github.com/filecoin-project/specs-actors/actors/builtin/reward" "github.com/filecoin-project/specs-actors/actors/builtin/system" "github.com/filecoin-project/specs-actors/actors/runtime" vmr "github.com/filecoin-project/specs-actors/actors/runtime" @@ -48,7 +48,7 @@ func NewInvoker() *Invoker { // NETUPGRADE: register code IDs for v2, etc. inv.Register(builtin.SystemActorCodeID, system.Actor{}, abi.EmptyValue{}) inv.Register(builtin.InitActorCodeID, init_.Actor{}, init_.State{}) - inv.Register(builtin.RewardActorCodeID, reward.Actor{}, reward.State{}) + inv.Register(builtin.RewardActorCodeID, v0reward.Actor{}, v0reward.State{}) inv.Register(builtin.CronActorCodeID, cron.Actor{}, cron.State{}) inv.Register(builtin.StoragePowerActorCodeID, v0power.Actor{}, v0power.State{}) inv.Register(builtin.StorageMarketActorCodeID, market.Actor{}, market.State{}) From e2295c372ab2ed0504bda3bb96a502cc3222b8ab Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Thu, 17 Sep 2020 02:50:59 -0400 Subject: [PATCH 170/885] Migrate multisig actor --- chain/actors/builtin/multisig/multisig.go | 3 +++ chain/actors/builtin/multisig/v0.go | 14 +++++++++++++- chain/stmgr/stmgr.go | 23 +++++++++++++---------- chain/stmgr/utils.go | 4 ++-- chain/vm/invoker.go | 4 ++-- cli/multisig.go | 14 +++++++------- 6 files changed, 40 insertions(+), 22 deletions(-) diff --git a/chain/actors/builtin/multisig/multisig.go b/chain/actors/builtin/multisig/multisig.go index 676dbe75f..fc58599a9 100644 --- a/chain/actors/builtin/multisig/multisig.go +++ b/chain/actors/builtin/multisig/multisig.go @@ -28,4 +28,7 @@ type State interface { cbor.Marshaler LockedBalance(epoch abi.ChainEpoch) (abi.TokenAmount, error) + StartEpoch() abi.ChainEpoch + UnlockDuration() abi.ChainEpoch + InitialBalance() abi.TokenAmount } diff --git a/chain/actors/builtin/multisig/v0.go b/chain/actors/builtin/multisig/v0.go index 3bc7e70b2..dc464d9af 100644 --- a/chain/actors/builtin/multisig/v0.go +++ b/chain/actors/builtin/multisig/v0.go @@ -12,5 +12,17 @@ type v0State struct { } func (s *v0State) LockedBalance(currEpoch abi.ChainEpoch) (abi.TokenAmount, error) { - return s.State.AmountLocked(currEpoch - s.StartEpoch), nil + return s.State.AmountLocked(currEpoch - s.State.StartEpoch), nil +} + +func (s *v0State) StartEpoch() abi.ChainEpoch { + return s.State.StartEpoch +} + +func (s *v0State) UnlockDuration() abi.ChainEpoch { + return s.State.UnlockDuration +} + +func (s *v0State) InitialBalance() abi.TokenAmount { + return s.State.InitialBalance } diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index 8a72b4ba4..18c67b5f0 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -5,6 +5,10 @@ import ( "fmt" "sync" + v0msig "github.com/filecoin-project/specs-actors/actors/builtin/multisig" + + "github.com/filecoin-project/lotus/chain/actors/builtin/multisig" + "github.com/filecoin-project/lotus/chain/actors/builtin/reward" "github.com/ipfs/go-cid" @@ -19,7 +23,6 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/builtin/multisig" v0reward "github.com/filecoin-project/specs-actors/actors/builtin/reward" "github.com/filecoin-project/specs-actors/actors/util/adt" @@ -793,7 +796,7 @@ func (sm *StateManager) SetVMConstructor(nvm func(context.Context, *vm.VMOpts) ( } type genesisInfo struct { - genesisMsigs []multisig.State + genesisMsigs []v0msig.State // info about the Accounts in the genesis state genesisActors []genesisActor genesisPledge abi.TokenAmount @@ -856,15 +859,15 @@ func (sm *StateManager) setupGenesisActors(ctx context.Context) error { return err } - if s.StartEpoch != 0 { + if s.StartEpoch() != 0 { return xerrors.New("genesis multisig doesn't start vesting at epoch 0!") } - ot, f := totalsByEpoch[s.UnlockDuration] + ot, f := totalsByEpoch[s.UnlockDuration()] if f { - totalsByEpoch[s.UnlockDuration] = big.Add(ot, s.InitialBalance) + totalsByEpoch[s.UnlockDuration()] = big.Add(ot, s.InitialBalance()) } else { - totalsByEpoch[s.UnlockDuration] = s.InitialBalance + totalsByEpoch[s.UnlockDuration()] = s.InitialBalance() } } else if act.Code == builtin.AccountActorCodeID { @@ -894,9 +897,9 @@ func (sm *StateManager) setupGenesisActors(ctx context.Context) error { return xerrors.Errorf("error setting up genesis infos: %w", err) } - gi.genesisMsigs = make([]multisig.State, 0, len(totalsByEpoch)) + gi.genesisMsigs = make([]v0msig.State, 0, len(totalsByEpoch)) for k, v := range totalsByEpoch { - ns := multisig.State{ + ns := v0msig.State{ InitialBalance: v, UnlockDuration: k, PendingTxns: cid.Undef, @@ -971,9 +974,9 @@ func (sm *StateManager) setupGenesisActorsTestnet(ctx context.Context) error { totalsByEpoch[sixYears] = big.NewInt(100_000_000) totalsByEpoch[sixYears] = big.Add(totalsByEpoch[sixYears], big.NewInt(300_000_000)) - gi.genesisMsigs = make([]multisig.State, 0, len(totalsByEpoch)) + gi.genesisMsigs = make([]v0msig.State, 0, len(totalsByEpoch)) for k, v := range totalsByEpoch { - ns := multisig.State{ + ns := v0msig.State{ InitialBalance: v, UnlockDuration: k, PendingTxns: cid.Undef, diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index b000a62e1..afd790a74 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -27,7 +27,7 @@ import ( "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/builtin/account" "github.com/filecoin-project/specs-actors/actors/builtin/cron" - "github.com/filecoin-project/specs-actors/actors/builtin/multisig" + v0msig "github.com/filecoin-project/specs-actors/actors/builtin/multisig" "github.com/filecoin-project/specs-actors/actors/builtin/paych" v0reward "github.com/filecoin-project/specs-actors/actors/builtin/reward" "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" @@ -554,7 +554,7 @@ func init() { builtin.StorageMinerActorCodeID: {builtin.MethodsMiner, miner.Actor{}}, builtin.StorageMarketActorCodeID: {builtin.MethodsMarket, market.Actor{}}, builtin.PaymentChannelActorCodeID: {builtin.MethodsPaych, paych.Actor{}}, - builtin.MultisigActorCodeID: {builtin.MethodsMultisig, multisig.Actor{}}, + builtin.MultisigActorCodeID: {builtin.MethodsMultisig, v0msig.Actor{}}, builtin.RewardActorCodeID: {builtin.MethodsReward, v0reward.Actor{}}, builtin.VerifiedRegistryActorCodeID: {builtin.MethodsVerifiedRegistry, verifreg.Actor{}}, } diff --git a/chain/vm/invoker.go b/chain/vm/invoker.go index a86a0f03c..4cda32c44 100644 --- a/chain/vm/invoker.go +++ b/chain/vm/invoker.go @@ -21,7 +21,7 @@ import ( init_ "github.com/filecoin-project/specs-actors/actors/builtin/init" "github.com/filecoin-project/specs-actors/actors/builtin/market" v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" - "github.com/filecoin-project/specs-actors/actors/builtin/multisig" + v0msig "github.com/filecoin-project/specs-actors/actors/builtin/multisig" "github.com/filecoin-project/specs-actors/actors/builtin/paych" v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" v0reward "github.com/filecoin-project/specs-actors/actors/builtin/reward" @@ -53,7 +53,7 @@ func NewInvoker() *Invoker { inv.Register(builtin.StoragePowerActorCodeID, v0power.Actor{}, v0power.State{}) inv.Register(builtin.StorageMarketActorCodeID, market.Actor{}, market.State{}) inv.Register(builtin.StorageMinerActorCodeID, v0miner.Actor{}, v0miner.State{}) - inv.Register(builtin.MultisigActorCodeID, multisig.Actor{}, multisig.State{}) + inv.Register(builtin.MultisigActorCodeID, v0msig.Actor{}, v0msig.State{}) inv.Register(builtin.PaymentChannelActorCodeID, paych.Actor{}, paych.State{}) inv.Register(builtin.VerifiedRegistryActorCodeID, verifreg.Actor{}, verifreg.State{}) inv.Register(builtin.AccountActorCodeID, account.Actor{}, account.State{}) diff --git a/cli/multisig.go b/cli/multisig.go index 4596628f4..ffbbd733b 100644 --- a/cli/multisig.go +++ b/cli/multisig.go @@ -17,7 +17,7 @@ import ( "github.com/filecoin-project/go-address" init_ "github.com/filecoin-project/specs-actors/actors/builtin/init" - samsig "github.com/filecoin-project/specs-actors/actors/builtin/multisig" + v0msig "github.com/filecoin-project/specs-actors/actors/builtin/multisig" cid "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" "github.com/urfave/cli/v2" @@ -199,7 +199,7 @@ var msigInspectCmd = &cli.Command{ return err } - var mstate samsig.State + var mstate v0msig.State if err := mstate.UnmarshalCBOR(bytes.NewReader(obj)); err != nil { return err } @@ -251,7 +251,7 @@ var msigInspectCmd = &cli.Command{ }, } -func GetMultisigPending(ctx context.Context, lapi api.FullNode, hroot cid.Cid) (map[int64]*samsig.Transaction, error) { +func GetMultisigPending(ctx context.Context, lapi api.FullNode, hroot cid.Cid) (map[int64]*v0msig.Transaction, error) { bs := apibstore.NewAPIBlockstore(lapi) store := adt.WrapStore(ctx, cbor.NewCborStore(bs)) @@ -260,8 +260,8 @@ func GetMultisigPending(ctx context.Context, lapi api.FullNode, hroot cid.Cid) ( return nil, err } - txs := make(map[int64]*samsig.Transaction) - var tx samsig.Transaction + txs := make(map[int64]*v0msig.Transaction) + var tx v0msig.Transaction err = nd.ForEach(&tx, func(k string) error { txid, _ := binary.Varint([]byte(k)) @@ -276,7 +276,7 @@ func GetMultisigPending(ctx context.Context, lapi api.FullNode, hroot cid.Cid) ( return txs, nil } -func state(tx *samsig.Transaction) string { +func state(tx *v0msig.Transaction) string { /* // TODO(why): I strongly disagree with not having these... but i need to move forward if tx.Complete { return "done" @@ -385,7 +385,7 @@ var msigProposeCmd = &cli.Command{ return fmt.Errorf("proposal returned exit %d", wait.Receipt.ExitCode) } - var retval samsig.ProposeReturn + var retval v0msig.ProposeReturn if err := retval.UnmarshalCBOR(bytes.NewReader(wait.Receipt.Return)); err != nil { return fmt.Errorf("failed to unmarshal propose return value: %w", err) } From 691bd9f442c80bbf0cce178e265706a90c2520ce Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Thu, 17 Sep 2020 00:32:10 -0700 Subject: [PATCH 171/885] feat(markets): complete markets conversion complete markets conversion to using chain/actors types, also replacing DealProposal/DealState interfaces with structs --- api/api_full.go | 2 +- api/api_storage.go | 4 +- api/apistruct/struct.go | 8 +- chain/actors/builtin/market/market.go | 40 +++++-- chain/actors/builtin/market/v0.go | 144 +++++++++++++++++--------- chain/events/state/predicates.go | 19 ++-- chain/events/state/predicates_test.go | 14 +-- chain/gen/gen.go | 9 +- chain/stmgr/stmgr.go | 20 ++++ chain/stmgr/utils.go | 22 ++-- cli/client.go | 8 +- extern/storage-sealing/sealing.go | 2 +- go.mod | 2 +- go.sum | 4 + markets/storageadapter/client.go | 23 ---- markets/storageadapter/provider.go | 22 ---- markets/utils/converters.go | 8 -- node/impl/full/gas.go | 10 +- node/impl/full/state.go | 95 ++++++++--------- node/impl/storminer.go | 30 +++++- storage/adapter_storage_miner.go | 5 +- 21 files changed, 269 insertions(+), 222 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index b472febe0..f565ad543 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -19,8 +19,8 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/go-state-types/dline" + "github.com/filecoin-project/lotus/chain/actors/builtin/market" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" - "github.com/filecoin-project/specs-actors/actors/builtin/market" "github.com/filecoin-project/specs-actors/actors/builtin/paych" "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" "github.com/filecoin-project/specs-actors/actors/runtime/proof" diff --git a/api/api_storage.go b/api/api_storage.go index 37bef2d6c..824772181 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -71,7 +71,7 @@ type StorageMiner interface { stores.SectorIndex MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error - MarketListDeals(ctx context.Context) ([]storagemarket.StorageDeal, error) + MarketListDeals(ctx context.Context) ([]MarketDeal, error) MarketListRetrievalDeals(ctx context.Context) ([]retrievalmarket.ProviderDealState, error) MarketGetDealUpdates(ctx context.Context) (<-chan storagemarket.MinerDeal, error) MarketListIncompleteDeals(ctx context.Context) ([]storagemarket.MinerDeal, error) @@ -83,7 +83,7 @@ type StorageMiner interface { MarketDataTransferUpdates(ctx context.Context) (<-chan DataTransferChannel, error) DealsImportData(ctx context.Context, dealPropCid cid.Cid, file string) error - DealsList(ctx context.Context) ([]storagemarket.StorageDeal, error) + DealsList(ctx context.Context) ([]MarketDeal, error) DealsConsiderOnlineStorageDeals(context.Context) (bool, error) DealsSetConsiderOnlineStorageDeals(context.Context, bool) error DealsConsiderOnlineRetrievalDeals(context.Context) (bool, error) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 26f7a8708..8bf8f1997 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -251,7 +251,7 @@ type StorageMinerStruct struct { MiningBase func(context.Context) (*types.TipSet, error) `perm:"read"` MarketImportDealData func(context.Context, cid.Cid, string) error `perm:"write"` - MarketListDeals func(ctx context.Context) ([]storagemarket.StorageDeal, error) `perm:"read"` + MarketListDeals func(ctx context.Context) ([]api.MarketDeal, error) `perm:"read"` MarketListRetrievalDeals func(ctx context.Context) ([]retrievalmarket.ProviderDealState, error) `perm:"read"` MarketGetDealUpdates func(ctx context.Context) (<-chan storagemarket.MinerDeal, error) `perm:"read"` MarketListIncompleteDeals func(ctx context.Context) ([]storagemarket.MinerDeal, error) `perm:"read"` @@ -296,7 +296,7 @@ type StorageMinerStruct struct { StorageTryLock func(ctx context.Context, sector abi.SectorID, read stores.SectorFileType, write stores.SectorFileType) (bool, error) `perm:"admin"` DealsImportData func(ctx context.Context, dealPropCid cid.Cid, file string) error `perm:"write"` - DealsList func(ctx context.Context) ([]storagemarket.StorageDeal, error) `perm:"read"` + DealsList func(ctx context.Context) ([]api.MarketDeal, error) `perm:"read"` DealsConsiderOnlineStorageDeals func(context.Context) (bool, error) `perm:"read"` DealsSetConsiderOnlineStorageDeals func(context.Context, bool) error `perm:"admin"` DealsConsiderOnlineRetrievalDeals func(context.Context) (bool, error) `perm:"read"` @@ -1128,7 +1128,7 @@ func (c *StorageMinerStruct) MarketImportDealData(ctx context.Context, propcid c return c.Internal.MarketImportDealData(ctx, propcid, path) } -func (c *StorageMinerStruct) MarketListDeals(ctx context.Context) ([]storagemarket.StorageDeal, error) { +func (c *StorageMinerStruct) MarketListDeals(ctx context.Context) ([]api.MarketDeal, error) { return c.Internal.MarketListDeals(ctx) } @@ -1172,7 +1172,7 @@ func (c *StorageMinerStruct) DealsImportData(ctx context.Context, dealPropCid ci return c.Internal.DealsImportData(ctx, dealPropCid, file) } -func (c *StorageMinerStruct) DealsList(ctx context.Context) ([]storagemarket.StorageDeal, error) { +func (c *StorageMinerStruct) DealsList(ctx context.Context) ([]api.MarketDeal, error) { return c.Internal.DealsList(ctx) } diff --git a/chain/actors/builtin/market/market.go b/chain/actors/builtin/market/market.go index 051c46dbe..73c2528d0 100644 --- a/chain/actors/builtin/market/market.go +++ b/chain/actors/builtin/market/market.go @@ -7,6 +7,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/cbor" v0builtin "github.com/filecoin-project/specs-actors/actors/builtin" + "github.com/ipfs/go-cid" "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/types" @@ -43,26 +44,39 @@ type State interface { } type BalanceTable interface { + ForEach(cb func(address.Address, abi.TokenAmount) error) error Get(key address.Address) (abi.TokenAmount, error) } type DealStates interface { - GetDeal(key abi.DealID) (DealState, error) + Get(id abi.DealID) (*DealState, bool, error) Diff(DealStates) (*DealStateChanges, error) } type DealProposals interface { + ForEach(cb func(id abi.DealID, dp DealProposal) error) error + Get(id abi.DealID) (*DealProposal, bool, error) Diff(DealProposals) (*DealProposalChanges, error) } -type DealState interface { - SectorStartEpoch() abi.ChainEpoch - SlashEpoch() abi.ChainEpoch - LastUpdatedEpoch() abi.ChainEpoch - Equals(DealState) bool +type DealState struct { + SectorStartEpoch abi.ChainEpoch // -1 if not yet included in proven sector + LastUpdatedEpoch abi.ChainEpoch // -1 if deal state never updated + SlashEpoch abi.ChainEpoch // -1 if deal never slashed } -type DealProposal interface { +type DealProposal struct { + PieceCID cid.Cid + PieceSize abi.PaddedPieceSize + VerifiedDeal bool + Client address.Address + Provider address.Address + Label string + StartEpoch abi.ChainEpoch + EndEpoch abi.ChainEpoch + StoragePricePerEpoch abi.TokenAmount + ProviderCollateral abi.TokenAmount + ClientCollateral abi.TokenAmount } type DealStateChanges struct { @@ -79,8 +93,8 @@ type DealIDState struct { // DealStateChange is a change in deal state from -> to type DealStateChange struct { ID abi.DealID - From DealState - To DealState + From *DealState + To *DealState } type DealProposalChanges struct { @@ -92,3 +106,11 @@ type ProposalIDState struct { ID abi.DealID Proposal DealProposal } + +func EmptyDealState() *DealState { + return &DealState{ + SectorStartEpoch: -1, + SlashEpoch: -1, + LastUpdatedEpoch: -1, + } +} diff --git a/chain/actors/builtin/market/v0.go b/chain/actors/builtin/market/v0.go index 8f0aa0594..92ebb59ba 100644 --- a/chain/actors/builtin/market/v0.go +++ b/chain/actors/builtin/market/v0.go @@ -72,11 +72,19 @@ func (s *v0State) Proposals() (DealProposals, error) { } func (s *v0State) EscrowTable() (BalanceTable, error) { - return v0adt.AsBalanceTable(s.store, s.State.EscrowTable) + bt, err := v0adt.AsBalanceTable(s.store, s.State.EscrowTable) + if err != nil { + return nil, err + } + return &v0BalanceTable{bt}, nil } func (s *v0State) LockedTable() (BalanceTable, error) { - return v0adt.AsBalanceTable(s.store, s.State.LockedTable) + bt, err := v0adt.AsBalanceTable(s.store, s.State.LockedTable) + if err != nil { + return nil, err + } + return &v0BalanceTable{bt}, nil } func (s *v0State) VerifyDealsForActivation( @@ -85,20 +93,37 @@ func (s *v0State) VerifyDealsForActivation( return market.ValidateDealsForActivation(&s.State, s.store, deals, minerAddr, sectorExpiry, currEpoch) } +type v0BalanceTable struct { + *v0adt.BalanceTable +} + +func (bt *v0BalanceTable) ForEach(cb func(address.Address, abi.TokenAmount) error) error { + asMap := (*v0adt.Map)(bt.BalanceTable) + var ta abi.TokenAmount + return asMap.ForEach(&ta, func(key string) error { + a, err := address.NewFromBytes([]byte(key)) + if err != nil { + return err + } + return cb(a, ta) + }) +} + type v0DealStates struct { adt.Array } -func (s *v0DealStates) GetDeal(dealID abi.DealID) (DealState, error) { - var deal market.DealState - found, err := s.Array.Get(uint64(dealID), &deal) +func (s *v0DealStates) Get(dealID abi.DealID) (*DealState, bool, error) { + var v0deal market.DealState + found, err := s.Array.Get(uint64(dealID), &v0deal) if err != nil { - return nil, err + return nil, false, err } if !found { - return nil, nil + return nil, false, nil } - return &v0DealState{deal}, nil + deal := fromV0DealState(v0deal) + return &deal, true, nil } func (s *v0DealStates) Diff(other DealStates) (*DealStateChanges, error) { @@ -108,7 +133,7 @@ func (s *v0DealStates) Diff(other DealStates) (*DealStateChanges, error) { return nil, errors.New("cannot compare deal states across versions") } results := new(DealStateChanges) - if err := adt.DiffAdtArray(s, v0other, &v0MarketStatesDiffer{results}); err != nil { + if err := adt.DiffAdtArray(s.Array, v0other.Array, &v0MarketStatesDiffer{results}); err != nil { return nil, fmt.Errorf("diffing deal states: %w", err) } @@ -120,61 +145,50 @@ type v0MarketStatesDiffer struct { } func (d *v0MarketStatesDiffer) Add(key uint64, val *typegen.Deferred) error { - ds := new(v0DealState) - err := ds.UnmarshalCBOR(bytes.NewReader(val.Raw)) + v0ds := new(market.DealState) + err := v0ds.UnmarshalCBOR(bytes.NewReader(val.Raw)) if err != nil { return err } - d.Results.Added = append(d.Results.Added, DealIDState{abi.DealID(key), ds}) + d.Results.Added = append(d.Results.Added, DealIDState{abi.DealID(key), fromV0DealState(*v0ds)}) return nil } func (d *v0MarketStatesDiffer) Modify(key uint64, from, to *typegen.Deferred) error { - dsFrom := new(v0DealState) - if err := dsFrom.UnmarshalCBOR(bytes.NewReader(from.Raw)); err != nil { + v0dsFrom := new(market.DealState) + if err := v0dsFrom.UnmarshalCBOR(bytes.NewReader(from.Raw)); err != nil { return err } - dsTo := new(v0DealState) - if err := dsTo.UnmarshalCBOR(bytes.NewReader(to.Raw)); err != nil { + v0dsTo := new(market.DealState) + if err := v0dsTo.UnmarshalCBOR(bytes.NewReader(to.Raw)); err != nil { return err } - if *dsFrom != *dsTo { - d.Results.Modified = append(d.Results.Modified, DealStateChange{abi.DealID(key), dsFrom, dsTo}) + if *v0dsFrom != *v0dsTo { + dsFrom := fromV0DealState(*v0dsFrom) + dsTo := fromV0DealState(*v0dsTo) + d.Results.Modified = append(d.Results.Modified, DealStateChange{abi.DealID(key), &dsFrom, &dsTo}) } return nil } func (d *v0MarketStatesDiffer) Remove(key uint64, val *typegen.Deferred) error { - ds := new(v0DealState) - err := ds.UnmarshalCBOR(bytes.NewReader(val.Raw)) + v0ds := new(market.DealState) + err := v0ds.UnmarshalCBOR(bytes.NewReader(val.Raw)) if err != nil { return err } - d.Results.Removed = append(d.Results.Removed, DealIDState{abi.DealID(key), ds}) + d.Results.Removed = append(d.Results.Removed, DealIDState{abi.DealID(key), fromV0DealState(*v0ds)}) return nil } -type v0DealState struct { - market.DealState -} - -func (ds *v0DealState) SectorStartEpoch() abi.ChainEpoch { - return ds.DealState.SectorStartEpoch -} - -func (ds *v0DealState) SlashEpoch() abi.ChainEpoch { - return ds.DealState.SlashEpoch -} - -func (ds *v0DealState) LastUpdatedEpoch() abi.ChainEpoch { - return ds.DealState.LastUpdatedEpoch -} - -func (ds *v0DealState) Equals(other DealState) bool { - v0other, ok := other.(*v0DealState) - return ok && *ds == *v0other +func fromV0DealState(v0 market.DealState) DealState { + return DealState{ + SectorStartEpoch: v0.SectorStartEpoch, + SlashEpoch: v0.SlashEpoch, + LastUpdatedEpoch: v0.LastUpdatedEpoch, + } } type v0DealProposals struct { @@ -188,28 +202,60 @@ func (s *v0DealProposals) Diff(other DealProposals) (*DealProposalChanges, error return nil, errors.New("cannot compare deal proposals across versions") } results := new(DealProposalChanges) - if err := adt.DiffAdtArray(s, v0other, &v0MarketProposalsDiffer{results}); err != nil { + if err := adt.DiffAdtArray(s.Array, v0other.Array, &v0MarketProposalsDiffer{results}); err != nil { return nil, fmt.Errorf("diffing deal proposals: %w", err) } return results, nil } +func (s *v0DealProposals) Get(dealID abi.DealID) (*DealProposal, bool, error) { + var v0proposal market.DealProposal + found, err := s.Array.Get(uint64(dealID), &v0proposal) + if err != nil { + return nil, false, err + } + if !found { + return nil, false, nil + } + proposal := fromV0DealProposal(v0proposal) + return &proposal, true, nil +} + +func (s *v0DealProposals) ForEach(cb func(dealID abi.DealID, dp DealProposal) error) error { + var v0dp market.DealProposal + return s.Array.ForEach(&v0dp, func(idx int64) error { + return cb(abi.DealID(idx), fromV0DealProposal(v0dp)) + }) +} + type v0MarketProposalsDiffer struct { Results *DealProposalChanges } -type v0DealProposal struct { - market.DealProposal +func fromV0DealProposal(v0 market.DealProposal) DealProposal { + return DealProposal{ + PieceCID: v0.PieceCID, + PieceSize: v0.PieceSize, + VerifiedDeal: v0.VerifiedDeal, + Client: v0.Client, + Provider: v0.Provider, + Label: v0.Label, + StartEpoch: v0.StartEpoch, + EndEpoch: v0.EndEpoch, + StoragePricePerEpoch: v0.StoragePricePerEpoch, + ProviderCollateral: v0.ProviderCollateral, + ClientCollateral: v0.ClientCollateral, + } } func (d *v0MarketProposalsDiffer) Add(key uint64, val *typegen.Deferred) error { - dp := new(v0DealProposal) - err := dp.UnmarshalCBOR(bytes.NewReader(val.Raw)) + v0dp := new(market.DealProposal) + err := v0dp.UnmarshalCBOR(bytes.NewReader(val.Raw)) if err != nil { return err } - d.Results.Added = append(d.Results.Added, ProposalIDState{abi.DealID(key), dp}) + d.Results.Added = append(d.Results.Added, ProposalIDState{abi.DealID(key), fromV0DealProposal(*v0dp)}) return nil } @@ -219,11 +265,11 @@ func (d *v0MarketProposalsDiffer) Modify(key uint64, from, to *typegen.Deferred) } func (d *v0MarketProposalsDiffer) Remove(key uint64, val *typegen.Deferred) error { - dp := new(v0DealProposal) - err := dp.UnmarshalCBOR(bytes.NewReader(val.Raw)) + v0dp := new(market.DealProposal) + err := v0dp.UnmarshalCBOR(bytes.NewReader(val.Raw)) if err != nil { return err } - d.Results.Removed = append(d.Results.Removed, ProposalIDState{abi.DealID(key), dp}) + d.Results.Removed = append(d.Results.Removed, ProposalIDState{abi.DealID(key), fromV0DealProposal(*v0dp)}) return nil } diff --git a/chain/events/state/predicates.go b/chain/events/state/predicates.go index 3e515e6a4..7a7609823 100644 --- a/chain/events/state/predicates.go +++ b/chain/events/state/predicates.go @@ -212,14 +212,7 @@ func (sp *StatePredicates) OnDealStateAmtChanged() DiffDealStatesFunc { } // ChangedDeals is a set of changes to deal state -type ChangedDeals map[abi.DealID]DealStateChange - -// DealStateChange is a change in deal state from -> to -type DealStateChange struct { - ID abi.DealID - From market.DealState - To market.DealState -} +type ChangedDeals map[abi.DealID]market.DealStateChange // DealStateChangedForIDs detects changes in the deal state AMT for the given deal IDs func (sp *StatePredicates) DealStateChangedForIDs(dealIds []abi.DealID) DiffDealStatesFunc { @@ -228,20 +221,20 @@ func (sp *StatePredicates) DealStateChangedForIDs(dealIds []abi.DealID) DiffDeal for _, dealID := range dealIds { // If the deal has been removed, we just set it to nil - oldDeal, err := oldDealStates.GetDeal(dealID) + oldDeal, oldFound, err := oldDealStates.Get(dealID) if err != nil { return false, nil, err } - newDeal, err := newDealStates.GetDeal(dealID) + newDeal, newFound, err := newDealStates.Get(dealID) if err != nil { return false, nil, err } - existenceChanged := (oldDeal == nil) != (newDeal == nil) - valueChanged := (oldDeal != nil && newDeal != nil) && !oldDeal.Equals(newDeal) + existenceChanged := oldFound != newFound + valueChanged := (oldFound && newFound) && *oldDeal != *newDeal if existenceChanged || valueChanged { - changedDeals[dealID] = DealStateChange{dealID, oldDeal, newDeal} + changedDeals[dealID] = market.DealStateChange{ID: dealID, From: oldDeal, To: newDeal} } } if len(changedDeals) > 0 { diff --git a/chain/events/state/predicates_test.go b/chain/events/state/predicates_test.go index 25f25334d..6541aa3b4 100644 --- a/chain/events/state/predicates_test.go +++ b/chain/events/state/predicates_test.go @@ -208,11 +208,11 @@ func TestMarketPredicates(t *testing.T) { require.Contains(t, changedDealIDs, abi.DealID(1)) require.Contains(t, changedDealIDs, abi.DealID(2)) deal1 := changedDealIDs[abi.DealID(1)] - if deal1.From.LastUpdatedEpoch() != 2 || deal1.To.LastUpdatedEpoch() != 3 { + if deal1.From.LastUpdatedEpoch != 2 || deal1.To.LastUpdatedEpoch != 3 { t.Fatal("Unexpected change to LastUpdatedEpoch") } deal2 := changedDealIDs[abi.DealID(2)] - if deal2.From.LastUpdatedEpoch() != 5 || deal2.To != nil { + if deal2.From.LastUpdatedEpoch != 5 || deal2.To != nil { t.Fatal("Expected To to be nil") } @@ -273,8 +273,8 @@ func TestMarketPredicates(t *testing.T) { require.Len(t, changedDeals.Modified, 1) require.Equal(t, abi.DealID(1), changedDeals.Modified[0].ID) - require.True(t, dealEquality(*newDeal1, changedDeals.Modified[0].To)) - require.True(t, dealEquality(*oldDeal1, changedDeals.Modified[0].From)) + require.True(t, dealEquality(*newDeal1, *changedDeals.Modified[0].To)) + require.True(t, dealEquality(*oldDeal1, *changedDeals.Modified[0].From)) require.Equal(t, abi.DealID(2), changedDeals.Removed[0].ID) }) @@ -624,7 +624,7 @@ func newSectorPreCommitInfo(sectorNo abi.SectorNumber, sealed cid.Cid, expiratio } func dealEquality(expected v0market.DealState, actual market.DealState) bool { - return expected.LastUpdatedEpoch == actual.LastUpdatedEpoch() && - expected.SectorStartEpoch == actual.SectorStartEpoch() && - expected.SlashEpoch == actual.SlashEpoch() + return expected.LastUpdatedEpoch == actual.LastUpdatedEpoch && + expected.SectorStartEpoch == actual.SectorStartEpoch && + expected.SlashEpoch == actual.SlashEpoch } diff --git a/chain/gen/gen.go b/chain/gen/gen.go index 4cae19082..4163f0b2d 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -489,13 +489,16 @@ func (cg *ChainGen) makeBlock(parents *types.TipSet, m address.Address, vrfticke // ResyncBankerNonce is used for dealing with messages made when // simulating forks func (cg *ChainGen) ResyncBankerNonce(ts *types.TipSet) error { - var act types.Actor - err := cg.sm.WithParentState(ts, cg.sm.WithActor(cg.banker, stmgr.GetActor(&act))) + st, err := cg.sm.ParentState(ts) + if err != nil { + return err + } + act, err := st.GetActor(cg.banker) if err != nil { return err } - cg.bankerNonce = act.Nonce + return nil } diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index 18c67b5f0..2a6737e2c 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -1175,3 +1175,23 @@ func (sm *StateManager) GetPaychState(ctx context.Context, addr address.Address, } return act, actState, nil } + +func (sm *StateManager) GetMarketState(ctx context.Context, ts *types.TipSet) (market.State, error) { + st, err := sm.ParentState(ts) + if err != nil { + return nil, err + } + + // TODO maybe there needs to be code here to differentiate address based on ts height? + addr := builtin.StorageMarketActorAddr + act, err := st.GetActor(addr) + if err != nil { + return nil, err + } + + actState, err := market.Load(sm.cs.Store(ctx), act) + if err != nil { + return nil, err + } + return actState, nil +} diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index afd790a74..38b66c512 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -34,7 +34,6 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/actors/adt" init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init" "github.com/filecoin-project/lotus/chain/actors/builtin/market" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" @@ -318,40 +317,35 @@ func GetStorageDeal(ctx context.Context, sm *StateManager, dealID abi.DealID, ts return nil, xerrors.Errorf("failed to load market actor state: %w", err) } - store := sm.ChainStore().Store(ctx) - - da, err := adt.AsArray(store, state.Proposals) + proposals, err := state.Proposals() if err != nil { return nil, err } - var dp market.DealProposal - if found, err := da.Get(uint64(dealID), &dp); err != nil { + proposal, found, err := proposals.Get(dealID) + + if err != nil { return nil, err } else if !found { return nil, xerrors.Errorf("deal %d not found", dealID) } - sa, err := market.AsDealStateArray(store, state.States) + states, err := state.States() if err != nil { return nil, err } - st, found, err := sa.Get(dealID) + st, found, err := states.Get(dealID) if err != nil { return nil, err } if !found { - st = &market.DealState{ - SectorStartEpoch: -1, - LastUpdatedEpoch: -1, - SlashEpoch: -1, - } + st = market.EmptyDealState() } return &api.MarketDeal{ - Proposal: dp, + Proposal: *proposal, State: *st, }, nil } diff --git a/cli/client.go b/cli/client.go index e68f98791..eda5ffae8 100644 --- a/cli/client.go +++ b/cli/client.go @@ -29,13 +29,13 @@ import ( "github.com/filecoin-project/go-multistore" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" - "github.com/filecoin-project/specs-actors/actors/builtin/market" "github.com/filecoin-project/lotus/api" lapi "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/tablewriter" + "github.com/filecoin-project/lotus/chain/actors/builtin/market" ) var CidBaseFlag = cli.StringFlag{ @@ -1045,11 +1045,7 @@ func dealFromDealInfo(ctx context.Context, full api.FullNode, head *types.TipSet if v.DealID == 0 { return deal{ LocalDeal: v, - OnChainDealState: market.DealState{ - SectorStartEpoch: -1, - LastUpdatedEpoch: -1, - SlashEpoch: -1, - }, + OnChainDealState: *market.EmptyDealState() } } diff --git a/extern/storage-sealing/sealing.go b/extern/storage-sealing/sealing.go index 0fe754217..e9a98fec9 100644 --- a/extern/storage-sealing/sealing.go +++ b/extern/storage-sealing/sealing.go @@ -25,9 +25,9 @@ import ( "github.com/filecoin-project/go-state-types/crypto" statemachine "github.com/filecoin-project/go-statemachine" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + "github.com/filecoin-project/lotus/chain/actors/builtin/market" sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" - "github.com/filecoin-project/specs-actors/actors/builtin/market" ) const SectorStorePrefix = "/sectors" diff --git a/go.mod b/go.mod index 65c2addb8..0397ff275 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 github.com/filecoin-project/go-data-transfer v0.6.3 github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f - github.com/filecoin-project/go-fil-markets v0.6.1-0.20200911011457-2959ccca6a3c + github.com/filecoin-project/go-fil-markets v0.6.1-0.20200917052354-ee0af754c6e9 github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200822201400-474f4fdccc52 github.com/filecoin-project/go-multistore v0.0.3 github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 diff --git a/go.sum b/go.sum index b6b724767..3d012da92 100644 --- a/go.sum +++ b/go.sum @@ -228,6 +228,10 @@ github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f h1 github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= github.com/filecoin-project/go-fil-markets v0.6.1-0.20200911011457-2959ccca6a3c h1:YGoyYmELQ0LHwDj/WcOvY3oYt+3iM0wdrAhqJQUAIy4= github.com/filecoin-project/go-fil-markets v0.6.1-0.20200911011457-2959ccca6a3c/go.mod h1:PLr9svZxsnHkae1Ky7+66g7fP9AlneVxIVu+oSMq56A= +github.com/filecoin-project/go-fil-markets v0.6.1-0.20200917050751-2af52e9606c6 h1:k97Z2JP3WpDVGU/7Bz3RtnqrYtn9X428Ps8OkoFq61I= +github.com/filecoin-project/go-fil-markets v0.6.1-0.20200917050751-2af52e9606c6/go.mod h1:PLr9svZxsnHkae1Ky7+66g7fP9AlneVxIVu+oSMq56A= +github.com/filecoin-project/go-fil-markets v0.6.1-0.20200917052354-ee0af754c6e9 h1:SnCUC9wHDId9TtV8PsQp8q1OOsi+NOLOwitIDnAgUa4= +github.com/filecoin-project/go-fil-markets v0.6.1-0.20200917052354-ee0af754c6e9/go.mod h1:PLr9svZxsnHkae1Ky7+66g7fP9AlneVxIVu+oSMq56A= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24= diff --git a/markets/storageadapter/client.go b/markets/storageadapter/client.go index 0c9e66eda..781715903 100644 --- a/markets/storageadapter/client.go +++ b/markets/storageadapter/client.go @@ -98,29 +98,6 @@ func (c *ClientNodeAdapter) VerifySignature(ctx context.Context, sig crypto.Sign return err == nil, err } -func (c *ClientNodeAdapter) ListClientDeals(ctx context.Context, addr address.Address, encodedTs shared.TipSetToken) ([]storagemarket.StorageDeal, error) { - tsk, err := types.TipSetKeyFromBytes(encodedTs) - if err != nil { - return nil, err - } - - allDeals, err := c.StateMarketDeals(ctx, tsk) - if err != nil { - return nil, err - } - - var out []storagemarket.StorageDeal - - for _, deal := range allDeals { - storageDeal := utils.FromOnChainDeal(deal.Proposal, deal.State) - if storageDeal.Client == addr { - out = append(out, storageDeal) - } - } - - return out, nil -} - // Adds funds with the StorageMinerActor for a storage participant. Used by both providers and clients. func (c *ClientNodeAdapter) AddFunds(ctx context.Context, addr address.Address, amount abi.TokenAmount) (cid.Cid, error) { // (Provider Node API) diff --git a/markets/storageadapter/provider.go b/markets/storageadapter/provider.go index 22d6c1e1d..98935b30a 100644 --- a/markets/storageadapter/provider.go +++ b/markets/storageadapter/provider.go @@ -142,28 +142,6 @@ func (n *ProviderNodeAdapter) VerifySignature(ctx context.Context, sig crypto.Si return err == nil, err } -func (n *ProviderNodeAdapter) ListProviderDeals(ctx context.Context, addr address.Address, encodedTs shared.TipSetToken) ([]storagemarket.StorageDeal, error) { - tsk, err := types.TipSetKeyFromBytes(encodedTs) - if err != nil { - return nil, err - } - allDeals, err := n.StateMarketDeals(ctx, tsk) - if err != nil { - return nil, err - } - - var out []storagemarket.StorageDeal - - for _, deal := range allDeals { - sharedDeal := utils.FromOnChainDeal(deal.Proposal, deal.State) - if sharedDeal.Provider == addr { - out = append(out, sharedDeal) - } - } - - return out, nil -} - func (n *ProviderNodeAdapter) GetMinerWorkerAddress(ctx context.Context, miner address.Address, tok shared.TipSetToken) (address.Address, error) { tsk, err := types.TipSetKeyFromBytes(tok) if err != nil { diff --git a/markets/utils/converters.go b/markets/utils/converters.go index 05472801d..4a3d21140 100644 --- a/markets/utils/converters.go +++ b/markets/utils/converters.go @@ -4,7 +4,6 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/specs-actors/actors/builtin/market" peer "github.com/libp2p/go-libp2p-core/peer" "github.com/multiformats/go-multiaddr" @@ -31,13 +30,6 @@ func NewStorageProviderInfo(address address.Address, miner address.Address, sect } } -func FromOnChainDeal(prop market.DealProposal, state market.DealState) storagemarket.StorageDeal { - return storagemarket.StorageDeal{ - DealProposal: prop, - DealState: state, - } -} - func ToSharedBalance(bal api.MarketBalance) storagemarket.Balance { return storagemarket.Balance{ Locked: bal.Locked, diff --git a/node/impl/full/gas.go b/node/impl/full/gas.go index 536bef360..b207a1f6d 100644 --- a/node/impl/full/gas.go +++ b/node/impl/full/gas.go @@ -166,8 +166,14 @@ func (a *GasAPI) GasEstimateGasLimit(ctx context.Context, msgIn *types.Message, } // Special case for PaymentChannel collect, which is deleting actor - var act types.Actor - err = a.Stmgr.WithParentState(ts, a.Stmgr.WithActor(msg.To, stmgr.GetActor(&act))) + st, err := a.Stmgr.ParentState(ts) + if err != nil { + _ = err + // somewhat ignore it as it can happen and we just want to detect + // an existing PaymentChannel actor + return res.MsgRct.GasUsed, nil + } + act, err := st.GetActor(msg.To) if err != nil { _ = err // somewhat ignore it as it can happen and we just want to detect diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 1b29a93a9..241561a9f 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -4,10 +4,10 @@ import ( "bytes" "context" "errors" - "fmt" - "github.com/filecoin-project/go-state-types/network" "strconv" + "github.com/filecoin-project/go-state-types/network" + "github.com/filecoin-project/go-state-types/dline" cid "github.com/ipfs/go-cid" @@ -28,6 +28,7 @@ import ( "github.com/filecoin-project/specs-actors/actors/util/smoothing" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/actors/builtin/market" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/actors/builtin/multisig" "github.com/filecoin-project/lotus/chain/actors/builtin/power" @@ -42,7 +43,6 @@ import ( "github.com/filecoin-project/lotus/chain/wallet" "github.com/filecoin-project/lotus/lib/bufbstore" "github.com/filecoin-project/lotus/node/modules/dtypes" - "github.com/filecoin-project/lotus/chain/actors/builtin/market" ) var errBreakForeach = errors.New("break") @@ -259,8 +259,8 @@ func (a *StateAPI) StateMinerPower(ctx context.Context, addr address.Address, ts } return &api.MinerPower{ - MinerPower: m, - TotalPower: net, + MinerPower: m, + TotalPower: net, HasMinPower: hmp, }, nil } @@ -489,37 +489,31 @@ func (a *StateAPI) StateMarketBalance(ctx context.Context, addr address.Address, func (a *StateAPI) StateMarketParticipants(ctx context.Context, tsk types.TipSetKey) (map[string]api.MarketBalance, error) { out := map[string]api.MarketBalance{} - var state market.State ts, err := a.Chain.GetTipSetFromKey(tsk) if err != nil { return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - if _, err := a.StateManager.LoadActorState(ctx, builtin.StorageMarketActorAddr, &state, ts); err != nil { - return nil, err - } - store := a.StateManager.ChainStore().Store(ctx) - escrow, err := adt.AsMap(store, state.EscrowTable) + + state, err := a.StateManager.GetMarketState(ctx, ts) if err != nil { return nil, err } - locked, err := adt.AsMap(store, state.LockedTable) + escrow, err := state.EscrowTable() + if err != nil { + return nil, err + } + locked, err := state.LockedTable() if err != nil { return nil, err } - var es, lk abi.TokenAmount - err = escrow.ForEach(&es, func(k string) error { - a, err := address.NewFromBytes([]byte(k)) + err = escrow.ForEach(func(a address.Address, es abi.TokenAmount) error { + + lk, err := locked.Get(a) if err != nil { return err } - if found, err := locked.Get(abi.AddrKey(a), &lk); err != nil { - return err - } else if !found { - return fmt.Errorf("locked funds not found") - } - out[a.String()] = api.MarketBalance{ Escrow: es, Locked: lk, @@ -535,37 +529,36 @@ func (a *StateAPI) StateMarketParticipants(ctx context.Context, tsk types.TipSet func (a *StateAPI) StateMarketDeals(ctx context.Context, tsk types.TipSetKey) (map[string]api.MarketDeal, error) { out := map[string]api.MarketDeal{} - var state market.State ts, err := a.Chain.GetTipSetFromKey(tsk) if err != nil { return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - if _, err := a.StateManager.LoadActorState(ctx, builtin.StorageMarketActorAddr, &state, ts); err != nil { - return nil, err - } - store := a.StateManager.ChainStore().Store(ctx) - da, err := adt.AsArray(store, state.Proposals) + state, err := a.StateManager.GetMarketState(ctx, ts) if err != nil { return nil, err } - sa, err := adt.AsArray(store, state.States) + da, err := state.Proposals() if err != nil { return nil, err } - var d market.DealProposal - if err := da.ForEach(&d, func(i int64) error { - var s market.DealState - if found, err := sa.Get(uint64(i), &s); err != nil { + sa, err := state.States() + if err != nil { + return nil, err + } + + if err := da.ForEach(func(dealID abi.DealID, d market.DealProposal) error { + s, found, err := sa.Get(dealID) + if err != nil { return xerrors.Errorf("failed to get state for deal in proposals array: %w", err) } else if !found { - s.SectorStartEpoch = -1 + s = market.EmptyDealState() } - out[strconv.FormatInt(i, 10)] = api.MarketDeal{ + out[strconv.FormatInt(int64(dealID), 10)] = api.MarketDeal{ Proposal: d, - State: s, + State: *s, } return nil }); err != nil { @@ -872,10 +865,10 @@ func (a *StateAPI) StateMinerPreCommitDepositForPower(ctx context.Context, maddr var sectorWeight abi.StoragePower if act, err := state.GetActor(market.Address); err != nil { return types.EmptyInt, xerrors.Errorf("loading miner actor %s: %w", maddr, err) - } else s, err := market.Load(store, act); err != nil { + } else if s, err := market.Load(store, act); err != nil { return types.EmptyInt, xerrors.Errorf("loading market actor state %s: %w", maddr, err) } else if w, vw, err := s.VerifyDealsForActivation(maddr, pci.DealIDs, ts.Height(), pci.Expiration); err != nil { - return types.EmptyInt, xerrors.Errorf("verifying deals for activation: %w", err) + return types.EmptyInt, xerrors.Errorf("verifying deals for activation: %w", err) } else { // NB: not exactly accurate, but should always lead us to *over* estimate, not under duration := pci.Expiration - ts.Height() @@ -887,9 +880,9 @@ func (a *StateAPI) StateMinerPreCommitDepositForPower(ctx context.Context, maddr var powerSmoothed smoothing.FilterEstimate if act, err := state.GetActor(power.Address); err != nil { return types.EmptyInt, xerrors.Errorf("loading miner actor: %w", err) - } else s, err := power.Load(store, act); err != nil { + } else if s, err := power.Load(store, act); err != nil { return types.EmptyInt, xerrors.Errorf("loading power actor state: %w", err) - } else p, err := s.TotalPowerSmoothed(); err != nil { + } else if p, err := s.TotalPowerSmoothed(); err != nil { return types.EmptyInt, xerrors.Errorf("failed to determine total power: %w", err) } else { powerSmoothed = p @@ -898,9 +891,9 @@ func (a *StateAPI) StateMinerPreCommitDepositForPower(ctx context.Context, maddr var rewardSmoothed smoothing.FilterEstimate if act, err := state.GetActor(reward.Address); err != nil { return types.EmptyInt, xerrors.Errorf("loading miner actor: %w", err) - } else s, err := reward.Load(store, act); err != nil { + } else if s, err := reward.Load(store, act); err != nil { return types.EmptyInt, xerrors.Errorf("loading reward actor state: %w", err) - } else r, err := s.RewardSmoothed(); err != nil { + } else if r, err := s.RewardSmoothed(); err != nil { return types.EmptyInt, xerrors.Errorf("failed to determine total reward: %w", err) } else { rewardSmoothed = r @@ -934,10 +927,10 @@ func (a *StateAPI) StateMinerInitialPledgeCollateral(ctx context.Context, maddr var sectorWeight abi.StoragePower if act, err := state.GetActor(market.Address); err != nil { return types.EmptyInt, xerrors.Errorf("loading miner actor %s: %w", maddr, err) - } else s, err := market.Load(store, act); err != nil { + } else if s, err := market.Load(store, act); err != nil { return types.EmptyInt, xerrors.Errorf("loading market actor state %s: %w", maddr, err) } else if w, vw, err := s.VerifyDealsForActivation(maddr, pci.DealIDs, ts.Height(), pci.Expiration); err != nil { - return types.EmptyInt, xerrors.Errorf("verifying deals for activation: %w", err) + return types.EmptyInt, xerrors.Errorf("verifying deals for activation: %w", err) } else { // NB: not exactly accurate, but should always lead us to *over* estimate, not under duration := pci.Expiration - ts.Height() @@ -947,16 +940,16 @@ func (a *StateAPI) StateMinerInitialPledgeCollateral(ctx context.Context, maddr } var ( - powerSmoothed smoothing.FilterEstimate + powerSmoothed smoothing.FilterEstimate pledgeCollerateral abi.TokenAmount ) if act, err := state.GetActor(power.Address); err != nil { return types.EmptyInt, xerrors.Errorf("loading miner actor: %w", err) - } else s, err := power.Load(store, act); err != nil { + } else if s, err := power.Load(store, act); err != nil { return types.EmptyInt, xerrors.Errorf("loading power actor state: %w", err) - } else p, err := s.TotalPowerSmoothed(); err != nil { + } else if p, err := s.TotalPowerSmoothed(); err != nil { return types.EmptyInt, xerrors.Errorf("failed to determine total power: %w", err) - } else c, err := s.TotalLocked(); err != nil { + } else if c, err := s.TotalLocked(); err != nil { return types.EmptyInt, xerrors.Errorf("failed to determine pledge collateral: %w", err) } else { powerSmoothed = p @@ -965,15 +958,15 @@ func (a *StateAPI) StateMinerInitialPledgeCollateral(ctx context.Context, maddr var ( rewardSmoothed smoothing.FilterEstimate - baselinePower abi.StoragePower + baselinePower abi.StoragePower ) if act, err := state.GetActor(reward.Address); err != nil { return types.EmptyInt, xerrors.Errorf("loading miner actor: %w", err) - } else s, err := reward.Load(store, act); err != nil { + } else if s, err := reward.Load(store, act); err != nil { return types.EmptyInt, xerrors.Errorf("loading reward actor state: %w", err) - } else r, err := s.RewardSmoothed(); err != nil { + } else if r, err := s.RewardSmoothed(); err != nil { return types.EmptyInt, xerrors.Errorf("failed to determine total reward: %w", err) - } else p, err := s.BaselinePower(); err != nil { + } else if p, err := s.BaselinePower(); err != nil { return types.EmptyInt, xerrors.Errorf("failed to determine baseline power: %w", err) } else { rewardSmoothed = r diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 6eedc9f54..89dd7c2c2 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -305,8 +305,30 @@ func (sm *StorageMinerAPI) MarketImportDealData(ctx context.Context, propCid cid return sm.StorageProvider.ImportDataForDeal(ctx, propCid, fi) } -func (sm *StorageMinerAPI) MarketListDeals(ctx context.Context) ([]storagemarket.StorageDeal, error) { - return sm.StorageProvider.ListDeals(ctx) +func (sm *StorageMinerAPI) listDeals(ctx context.Context) ([]api.MarketDeal, error) { + ts, err := sm.Full.ChainHead(ctx) + if err != nil { + return nil, err + } + tsk := ts.Key() + allDeals, err := sm.Full.StateMarketDeals(ctx, tsk) + if err != nil { + return nil, err + } + + var out []api.MarketDeal + + for _, deal := range allDeals { + if deal.Proposal.Provider == sm.Miner.Address() { + out = append(out, deal) + } + } + + return out, nil +} + +func (sm *StorageMinerAPI) MarketListDeals(ctx context.Context) ([]api.MarketDeal, error) { + return sm.StorageProvider.listDeals(ctx) } func (sm *StorageMinerAPI) MarketListRetrievalDeals(ctx context.Context) ([]retrievalmarket.ProviderDealState, error) { @@ -395,8 +417,8 @@ func (sm *StorageMinerAPI) MarketDataTransferUpdates(ctx context.Context) (<-cha return channels, nil } -func (sm *StorageMinerAPI) DealsList(ctx context.Context) ([]storagemarket.StorageDeal, error) { - return sm.StorageProvider.ListDeals(ctx) +func (sm *StorageMinerAPI) DealsList(ctx context.Context) ([]api.MarketDeal, error) { + return sm.listDeals(ctx) } func (sm *StorageMinerAPI) RetrievalDealsList(ctx context.Context) (map[retrievalmarket.ProviderDealIdentifier]retrievalmarket.ProviderDealState, error) { diff --git a/storage/adapter_storage_miner.go b/storage/adapter_storage_miner.go index 8b64789ad..7dced4331 100644 --- a/storage/adapter_storage_miner.go +++ b/storage/adapter_storage_miner.go @@ -15,12 +15,13 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/builtin/market" + v0market "github.com/filecoin-project/specs-actors/actors/builtin/market" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/apibstore" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/actors/builtin/market" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" @@ -136,7 +137,7 @@ func (s SealingAPIAdapter) StateComputeDataCommitment(ctx context.Context, maddr return cid.Undef, xerrors.Errorf("failed to unmarshal TipSetToken to TipSetKey: %w", err) } - ccparams, err := actors.SerializeParams(&market.ComputeDataCommitmentParams{ + ccparams, err := actors.SerializeParams(&v0market.ComputeDataCommitmentParams{ DealIDs: deals, SectorType: sectorType, }) From 7bf165c73a37025307619ea49ae8fc6aaa36b2b9 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Thu, 17 Sep 2020 03:07:01 -0400 Subject: [PATCH 172/885] Remove miner-related specs actors types from API --- api/api_full.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index f565ad543..e575c5bdd 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -25,7 +25,6 @@ import ( "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" "github.com/filecoin-project/specs-actors/actors/runtime/proof" - miner2 "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/actors/builtin/power" "github.com/filecoin-project/lotus/chain/types" marketevents "github.com/filecoin-project/lotus/markets/loggers" @@ -317,16 +316,16 @@ type FullNode interface { // StateMinerSectors returns info about the given miner's sectors. If the filter bitfield is nil, all sectors are included. // If the filterOut boolean is set to true, any sectors in the filter are excluded. // If false, only those sectors in the filter are included. - StateMinerSectors(context.Context, address.Address, *bitfield.BitField, bool, types.TipSetKey) ([]*miner2.ChainSectorInfo, error) + StateMinerSectors(context.Context, address.Address, *bitfield.BitField, bool, types.TipSetKey) ([]*miner.ChainSectorInfo, error) // StateMinerActiveSectors returns info about sectors that a given miner is actively proving. - StateMinerActiveSectors(context.Context, address.Address, types.TipSetKey) ([]*miner2.ChainSectorInfo, error) + StateMinerActiveSectors(context.Context, address.Address, types.TipSetKey) ([]*miner.ChainSectorInfo, error) // StateMinerProvingDeadline calculates the deadline at some epoch for a proving period // and returns the deadline-related calculations. StateMinerProvingDeadline(context.Context, address.Address, types.TipSetKey) (*dline.Info, error) // StateMinerPower returns the power of the indicated miner StateMinerPower(context.Context, address.Address, types.TipSetKey) (*MinerPower, error) // StateMinerInfo returns info about the indicated miner - StateMinerInfo(context.Context, address.Address, types.TipSetKey) (miner2.MinerInfo, error) + StateMinerInfo(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error) // StateMinerFaults returns a bitfield indicating the faulty sectors of the given miner StateMinerFaults(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error) // StateAllMinerFaults returns all non-expired Faults that occur within lookback epochs of the given tipset From 053cfc1cc7f5935d616d19813862c3c38bde6cb8 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Thu, 17 Sep 2020 03:48:39 -0400 Subject: [PATCH 173/885] Migrate verifreg actor --- api/api_full.go | 3 +-- api/apistruct/struct.go | 5 ++--- build/params_2k.go | 4 ++-- chain/gen/gen_test.go | 4 ++-- chain/gen/genesis/t06_vreg.go | 4 ++-- chain/stmgr/forks_test.go | 4 ++-- chain/stmgr/utils.go | 4 ++-- chain/store/store_test.go | 4 ++-- chain/sync_test.go | 4 ++-- chain/vectors/gen/main.go | 4 ++-- chain/vm/invoker.go | 4 ++-- cli/paych_test.go | 4 ++-- markets/storageadapter/provider.go | 7 ++++--- node/impl/full/state.go | 33 +++++++++++------------------- node/node_test.go | 4 ++-- 15 files changed, 41 insertions(+), 51 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index e575c5bdd..d570da7de 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -22,7 +22,6 @@ import ( "github.com/filecoin-project/lotus/chain/actors/builtin/market" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/specs-actors/actors/builtin/paych" - "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" "github.com/filecoin-project/specs-actors/actors/runtime/proof" "github.com/filecoin-project/lotus/chain/actors/builtin/power" @@ -382,7 +381,7 @@ type FullNode interface { // StateVerifiedClientStatus returns the data cap for the given address. // Returns nil if there is no entry in the data cap table for the // address. - StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*verifreg.DataCap, error) + StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (abi.StoragePower, error) // StateDealProviderCollateralBounds returns the min and max collateral a storage provider // can issue. It takes the deal size and verified status as parameters. StateDealProviderCollateralBounds(context.Context, abi.PaddedPieceSize, bool, types.TipSetKey) (DealCollateralBounds, error) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 8bf8f1997..837400b65 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -30,7 +30,6 @@ import ( "github.com/filecoin-project/lotus/extern/sector-storage/storiface" marketevents "github.com/filecoin-project/lotus/markets/loggers" "github.com/filecoin-project/specs-actors/actors/builtin/paych" - "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" "github.com/filecoin-project/specs-storage/storage" "github.com/filecoin-project/lotus/api" @@ -197,7 +196,7 @@ type FullNodeStruct struct { StateMinerSectorCount func(context.Context, address.Address, types.TipSetKey) (api.MinerSectors, error) `perm:"read"` StateListMessages func(ctx context.Context, match *types.Message, tsk types.TipSetKey, toht abi.ChainEpoch) ([]cid.Cid, error) `perm:"read"` StateCompute func(context.Context, abi.ChainEpoch, []*types.Message, types.TipSetKey) (*api.ComputeStateOutput, error) `perm:"read"` - StateVerifiedClientStatus func(context.Context, address.Address, types.TipSetKey) (*verifreg.DataCap, error) `perm:"read"` + StateVerifiedClientStatus func(context.Context, address.Address, types.TipSetKey) (abi.StoragePower, error) `perm:"read"` StateDealProviderCollateralBounds func(context.Context, abi.PaddedPieceSize, bool, types.TipSetKey) (api.DealCollateralBounds, error) `perm:"read"` StateCirculatingSupply func(context.Context, types.TipSetKey) (api.CirculatingSupply, error) `perm:"read"` StateNetworkVersion func(context.Context, types.TipSetKey) (stnetwork.Version, error) `perm:"read"` @@ -864,7 +863,7 @@ func (c *FullNodeStruct) StateCompute(ctx context.Context, height abi.ChainEpoch return c.Internal.StateCompute(ctx, height, msgs, tsk) } -func (c *FullNodeStruct) StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*verifreg.DataCap, error) { +func (c *FullNodeStruct) StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (abi.StoragePower, error) { return c.Internal.StateVerifiedClientStatus(ctx, addr, tsk) } diff --git a/build/params_2k.go b/build/params_2k.go index cf34706e5..9c8a591d1 100644 --- a/build/params_2k.go +++ b/build/params_2k.go @@ -7,7 +7,7 @@ import ( "github.com/filecoin-project/go-state-types/big" v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" - "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" + v0verifreg "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" ) const UpgradeBreezeHeight = -1 @@ -24,7 +24,7 @@ func init() { v0miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } - verifreg.MinVerifiedDealSize = big.NewInt(256) + v0verifreg.MinVerifiedDealSize = big.NewInt(256) BuildType |= Build2k } diff --git a/chain/gen/gen_test.go b/chain/gen/gen_test.go index 496bb016b..9d1262f77 100644 --- a/chain/gen/gen_test.go +++ b/chain/gen/gen_test.go @@ -7,7 +7,7 @@ import ( "github.com/filecoin-project/go-state-types/big" v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" - "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" + v0verifreg "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" _ "github.com/filecoin-project/lotus/lib/sigs/bls" _ "github.com/filecoin-project/lotus/lib/sigs/secp" @@ -18,7 +18,7 @@ func init() { abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } v0power.ConsensusMinerMinPower = big.NewInt(2048) - verifreg.MinVerifiedDealSize = big.NewInt(256) + v0verifreg.MinVerifiedDealSize = big.NewInt(256) } func testGeneration(t testing.TB, n int, msgs int, sectors int) { diff --git a/chain/gen/genesis/t06_vreg.go b/chain/gen/genesis/t06_vreg.go index 6636fa05f..d91cdf7b1 100644 --- a/chain/gen/genesis/t06_vreg.go +++ b/chain/gen/genesis/t06_vreg.go @@ -7,7 +7,7 @@ import ( cbor "github.com/ipfs/go-ipld-cbor" "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" + v0verifreg "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/lotus/chain/types" @@ -34,7 +34,7 @@ func SetupVerifiedRegistryActor(bs bstore.Blockstore) (*types.Actor, error) { return nil, err } - sms := verifreg.ConstructState(h, RootVerifierID) + sms := v0verifreg.ConstructState(h, RootVerifierID) stcid, err := store.Put(store.Context(), sms) if err != nil { diff --git a/chain/stmgr/forks_test.go b/chain/stmgr/forks_test.go index 2db11e832..c4fb1b3be 100644 --- a/chain/stmgr/forks_test.go +++ b/chain/stmgr/forks_test.go @@ -13,7 +13,7 @@ import ( init_ "github.com/filecoin-project/specs-actors/actors/builtin/init" v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" - "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" + v0verifreg "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" "github.com/filecoin-project/specs-actors/actors/runtime" "golang.org/x/xerrors" @@ -37,7 +37,7 @@ func init() { abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } v0power.ConsensusMinerMinPower = big.NewInt(2048) - verifreg.MinVerifiedDealSize = big.NewInt(256) + v0verifreg.MinVerifiedDealSize = big.NewInt(256) } const testForkHeight = 40 diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index 38b66c512..b7c888a36 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -30,7 +30,7 @@ import ( v0msig "github.com/filecoin-project/specs-actors/actors/builtin/multisig" "github.com/filecoin-project/specs-actors/actors/builtin/paych" v0reward "github.com/filecoin-project/specs-actors/actors/builtin/reward" - "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" + v0verifreg "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" @@ -550,7 +550,7 @@ func init() { builtin.PaymentChannelActorCodeID: {builtin.MethodsPaych, paych.Actor{}}, builtin.MultisigActorCodeID: {builtin.MethodsMultisig, v0msig.Actor{}}, builtin.RewardActorCodeID: {builtin.MethodsReward, v0reward.Actor{}}, - builtin.VerifiedRegistryActorCodeID: {builtin.MethodsVerifiedRegistry, verifreg.Actor{}}, + builtin.VerifiedRegistryActorCodeID: {builtin.MethodsVerifiedRegistry, v0verifreg.Actor{}}, } for c, m := range cidToMethods { diff --git a/chain/store/store_test.go b/chain/store/store_test.go index 7da8d219d..83238a67d 100644 --- a/chain/store/store_test.go +++ b/chain/store/store_test.go @@ -12,7 +12,7 @@ import ( "github.com/filecoin-project/go-state-types/crypto" v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" - "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" + v0verifreg "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/store" @@ -26,7 +26,7 @@ func init() { abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } v0power.ConsensusMinerMinPower = big.NewInt(2048) - verifreg.MinVerifiedDealSize = big.NewInt(256) + v0verifreg.MinVerifiedDealSize = big.NewInt(256) } func BenchmarkGetRandomness(b *testing.B) { diff --git a/chain/sync_test.go b/chain/sync_test.go index e8df32c56..d98663fb1 100644 --- a/chain/sync_test.go +++ b/chain/sync_test.go @@ -22,7 +22,7 @@ import ( "github.com/filecoin-project/go-state-types/big" v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" - "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" + v0verifreg "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" @@ -47,7 +47,7 @@ func init() { abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } v0power.ConsensusMinerMinPower = big.NewInt(2048) - verifreg.MinVerifiedDealSize = big.NewInt(256) + v0verifreg.MinVerifiedDealSize = big.NewInt(256) } const source = 0 diff --git a/chain/vectors/gen/main.go b/chain/vectors/gen/main.go index 4bf2c420e..36b770f03 100644 --- a/chain/vectors/gen/main.go +++ b/chain/vectors/gen/main.go @@ -19,14 +19,14 @@ import ( "github.com/filecoin-project/lotus/chain/types/mock" "github.com/filecoin-project/lotus/chain/vectors" "github.com/filecoin-project/lotus/chain/wallet" - "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" + v0verifreg "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" _ "github.com/filecoin-project/lotus/lib/sigs/bls" _ "github.com/filecoin-project/lotus/lib/sigs/secp" ) func init() { - verifreg.MinVerifiedDealSize = big.NewInt(2048) + v0verifreg.MinVerifiedDealSize = big.NewInt(2048) v0power.ConsensusMinerMinPower = big.NewInt(2048) } diff --git a/chain/vm/invoker.go b/chain/vm/invoker.go index 4cda32c44..b31b45767 100644 --- a/chain/vm/invoker.go +++ b/chain/vm/invoker.go @@ -8,7 +8,7 @@ import ( "github.com/filecoin-project/go-state-types/exitcode" "github.com/filecoin-project/specs-actors/actors/builtin/account" - "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" + v0verifreg "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" @@ -55,7 +55,7 @@ func NewInvoker() *Invoker { inv.Register(builtin.StorageMinerActorCodeID, v0miner.Actor{}, v0miner.State{}) inv.Register(builtin.MultisigActorCodeID, v0msig.Actor{}, v0msig.State{}) inv.Register(builtin.PaymentChannelActorCodeID, paych.Actor{}, paych.State{}) - inv.Register(builtin.VerifiedRegistryActorCodeID, verifreg.Actor{}, verifreg.State{}) + inv.Register(builtin.VerifiedRegistryActorCodeID, v0verifreg.Actor{}, v0verifreg.State{}) inv.Register(builtin.AccountActorCodeID, account.Actor{}, account.State{}) return inv diff --git a/cli/paych_test.go b/cli/paych_test.go index 1cf95ef6c..598c178eb 100644 --- a/cli/paych_test.go +++ b/cli/paych_test.go @@ -17,7 +17,7 @@ import ( "github.com/filecoin-project/go-state-types/big" v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" - "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" + v0verifreg "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" "github.com/multiformats/go-multiaddr" @@ -44,7 +44,7 @@ func init() { v0miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } - verifreg.MinVerifiedDealSize = big.NewInt(256) + v0verifreg.MinVerifiedDealSize = big.NewInt(256) } // TestPaymentChannels does a basic test to exercise the payment channel CLI diff --git a/markets/storageadapter/provider.go b/markets/storageadapter/provider.go index 98935b30a..fb93a1e72 100644 --- a/markets/storageadapter/provider.go +++ b/markets/storageadapter/provider.go @@ -8,6 +8,8 @@ import ( "io" "time" + "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/ipfs/go-cid" @@ -22,7 +24,6 @@ import ( "github.com/filecoin-project/go-state-types/exitcode" "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/builtin/market" - "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" @@ -361,10 +362,10 @@ func (n *ProviderNodeAdapter) WaitForMessage(ctx context.Context, mcid cid.Cid, return cb(receipt.Receipt.ExitCode, receipt.Receipt.Return, receipt.Message, nil) } -func (n *ProviderNodeAdapter) GetDataCap(ctx context.Context, addr address.Address, encodedTs shared.TipSetToken) (*verifreg.DataCap, error) { +func (n *ProviderNodeAdapter) GetDataCap(ctx context.Context, addr address.Address, encodedTs shared.TipSetToken) (abi.StoragePower, error) { tsk, err := types.TipSetKeyFromBytes(encodedTs) if err != nil { - return nil, err + return big.Zero(), err } return n.StateVerifiedClientStatus(ctx, addr, tsk) } diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 241561a9f..a21f0cf80 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -6,9 +6,10 @@ import ( "errors" "strconv" - "github.com/filecoin-project/go-state-types/network" + "github.com/filecoin-project/lotus/chain/actors/builtin/verifreg" "github.com/filecoin-project/go-state-types/dline" + "github.com/filecoin-project/go-state-types/network" cid "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" @@ -23,7 +24,6 @@ import ( "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/specs-actors/actors/builtin" v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" - "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/specs-actors/actors/util/smoothing" @@ -1018,40 +1018,31 @@ func (a *StateAPI) StateMinerAvailableBalance(ctx context.Context, maddr address } // StateVerifiedClientStatus returns the data cap for the given address. -// Returns nil if there is no entry in the data cap table for the +// Returns zero if there is no entry in the data cap table for the // address. -func (a *StateAPI) StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*verifreg.DataCap, error) { +func (a *StateAPI) StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (abi.StoragePower, error) { act, err := a.StateGetActor(ctx, builtin.VerifiedRegistryActorAddr, tsk) if err != nil { - return nil, err + return big.Zero(), err } aid, err := a.StateLookupID(ctx, addr, tsk) if err != nil { log.Warnf("lookup failure %v", err) - return nil, err + return big.Zero(), err } - store := a.StateManager.ChainStore().Store(ctx) - - var st verifreg.State - if err := store.Get(ctx, act.Head, &st); err != nil { - return nil, err - } - - vh, err := adt.AsMap(store, st.VerifiedClients) + vrs, err := verifreg.Load(a.StateManager.ChainStore().Store(ctx), act) if err != nil { - return nil, err + return big.Zero(), xerrors.Errorf("failed to load verified registry state: %w", err) } - var dcap verifreg.DataCap - if found, err := vh.Get(abi.AddrKey(aid), &dcap); err != nil { - return nil, err - } else if !found { - return nil, nil + _, dcap, err := vrs.VerifiedClientDataCap(aid) + if err != nil { + return big.Zero(), xerrors.Errorf("looking up verified client: %w", err) } - return &dcap, nil + return dcap, nil } var dealProviderCollateralNum = types.NewInt(110) diff --git a/node/node_test.go b/node/node_test.go index 92b741f73..68ee178fc 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -12,7 +12,7 @@ import ( "github.com/filecoin-project/lotus/lib/lotuslog" v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" - "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" + v0verifreg "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" logging "github.com/ipfs/go-log/v2" "github.com/filecoin-project/lotus/api/test" @@ -25,7 +25,7 @@ func init() { v0miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } - verifreg.MinVerifiedDealSize = big.NewInt(256) + v0verifreg.MinVerifiedDealSize = big.NewInt(256) } func TestAPI(t *testing.T) { From 31ff5230bbf3506bc69f86c0f30daa53e01c7867 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Thu, 17 Sep 2020 04:17:14 -0400 Subject: [PATCH 174/885] Get State API almost working --- api/api_full.go | 2 + api/apistruct/struct.go | 5 + chain/actors/builtin/miner/miner.go | 3 + chain/actors/builtin/miner/v0.go | 8 ++ chain/actors/builtin/reward/reward.go | 4 +- chain/actors/builtin/reward/v0.go | 12 +- chain/actors/builtin/verifreg/v0.go | 37 ++++++ chain/actors/builtin/verifreg/verifreg.go | 34 ++++++ chain/stmgr/read.go | 1 + chain/stmgr/stmgr.go | 2 +- chain/stmgr/utils.go | 12 +- chain/sub/incoming.go | 1 - cli/client.go | 8 +- cmd/lotus-storage-miner/proving.go | 10 +- markets/storageadapter/provider.go | 10 +- node/impl/full/state.go | 136 +++++++++++++--------- node/impl/storminer.go | 2 +- storage/adapter_storage_miner.go | 4 +- storage/addresses.go | 5 +- storage/miner.go | 2 +- storage/wdpost_sched.go | 2 - 21 files changed, 215 insertions(+), 85 deletions(-) create mode 100644 chain/actors/builtin/verifreg/v0.go create mode 100644 chain/actors/builtin/verifreg/verifreg.go diff --git a/api/api_full.go b/api/api_full.go index d570da7de..1adcc5c19 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -325,6 +325,8 @@ type FullNode interface { StateMinerPower(context.Context, address.Address, types.TipSetKey) (*MinerPower, error) // StateMinerInfo returns info about the indicated miner StateMinerInfo(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error) + // StateMinerDeadlines returns all the proving deadlines for the given miner + StateMinerDeadlines(context.Context, address.Address, types.TipSetKey) ([]*miner.Deadline, error) // StateMinerFaults returns a bitfield indicating the faulty sectors of the given miner StateMinerFaults(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error) // StateAllMinerFaults returns all non-expired Faults that occur within lookback epochs of the given tipset diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 837400b65..ab86d7819 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -167,6 +167,7 @@ type FullNodeStruct struct { StateMinerProvingDeadline func(context.Context, address.Address, types.TipSetKey) (*dline.Info, error) `perm:"read"` StateMinerPower func(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error) `perm:"read"` StateMinerInfo func(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error) `perm:"read"` + StateMinerDeadlines func(context.Context, address.Address, types.TipSetKey) ([]*miner.Deadline, error) `perm:"read"` StateMinerFaults func(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error) `perm:"read"` StateAllMinerFaults func(context.Context, abi.ChainEpoch, types.TipSetKey) ([]*api.Fault, error) `perm:"read"` StateMinerRecoveries func(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error) `perm:"read"` @@ -751,6 +752,10 @@ func (c *FullNodeStruct) StateMinerInfo(ctx context.Context, actor address.Addre return c.Internal.StateMinerInfo(ctx, actor, tsk) } +func (c *FullNodeStruct) StateMinerDeadlines(ctx context.Context, actor address.Address, tsk types.TipSetKey) ([]*miner.Deadline, error) { + return c.Internal.StateMinerDeadlines(ctx, actor, tsk) +} + func (c *FullNodeStruct) StateMinerFaults(ctx context.Context, actor address.Address, tsk types.TipSetKey) (bitfield.BitField, error) { return c.Internal.StateMinerFaults(ctx, actor, tsk) } diff --git a/chain/actors/builtin/miner/miner.go b/chain/actors/builtin/miner/miner.go index 4acb41dae..725c5f2ff 100644 --- a/chain/actors/builtin/miner/miner.go +++ b/chain/actors/builtin/miner/miner.go @@ -34,6 +34,9 @@ func Load(store adt.Store, act *types.Actor) (st State, err error) { type State interface { cbor.Marshaler + AvailableBalance(abi.TokenAmount) (abi.TokenAmount, error) + VestedFunds(abi.ChainEpoch) (abi.TokenAmount, error) + GetSector(abi.SectorNumber) (*SectorOnChainInfo, error) FindSector(abi.SectorNumber) (*SectorLocation, error) GetSectorExpiration(abi.SectorNumber) (*SectorExpiration, error) diff --git a/chain/actors/builtin/miner/v0.go b/chain/actors/builtin/miner/v0.go index b56fb1745..cbe42b3da 100644 --- a/chain/actors/builtin/miner/v0.go +++ b/chain/actors/builtin/miner/v0.go @@ -31,6 +31,14 @@ type v0Partition struct { store adt.Store } +func (s *v0State) AvailableBalance(bal abi.TokenAmount) (abi.TokenAmount, error) { + return s.GetAvailableBalance(bal), nil +} + +func (s *v0State) VestedFunds(epoch abi.ChainEpoch) (abi.TokenAmount, error) { + return s.CheckVestedFunds(s.store, epoch) +} + func (s *v0State) GetSector(num abi.SectorNumber) (*SectorOnChainInfo, error) { info, ok, err := s.State.GetSector(s.store, num) if !ok || err != nil { diff --git a/chain/actors/builtin/reward/reward.go b/chain/actors/builtin/reward/reward.go index d400258df..9093b33c6 100644 --- a/chain/actors/builtin/reward/reward.go +++ b/chain/actors/builtin/reward/reward.go @@ -31,5 +31,7 @@ type State interface { cbor.Marshaler RewardSmoothed() (builtin.FilterEstimate, error) - TotalStoragePowerReward() abi.TokenAmount + EffectiveBaselinePower() (abi.StoragePower, error) + ThisEpochBaselinePower() (abi.StoragePower, error) + TotalStoragePowerReward() (abi.TokenAmount, error) } diff --git a/chain/actors/builtin/reward/v0.go b/chain/actors/builtin/reward/v0.go index b0558f0ae..16ac2b071 100644 --- a/chain/actors/builtin/reward/v0.go +++ b/chain/actors/builtin/reward/v0.go @@ -16,6 +16,14 @@ func (s *v0State) RewardSmoothed() (builtin.FilterEstimate, error) { return *s.State.ThisEpochRewardSmoothed, nil } -func (s *v0State) TotalStoragePowerReward() abi.TokenAmount { - return s.State.TotalMined +func (s *v0State) TotalStoragePowerReward() (abi.TokenAmount, error) { + return s.State.TotalMined, nil +} + +func (s *v0State) EffectiveBaselinePower() (abi.StoragePower, error) { + return s.State.EffectiveBaselinePower, nil +} + +func (s *v0State) ThisEpochBaselinePower() (abi.StoragePower, error) { + return s.State.ThisEpochBaselinePower, nil } diff --git a/chain/actors/builtin/verifreg/v0.go b/chain/actors/builtin/verifreg/v0.go new file mode 100644 index 000000000..cbaf4d236 --- /dev/null +++ b/chain/actors/builtin/verifreg/v0.go @@ -0,0 +1,37 @@ +package verifreg + +import ( + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" + v0verifreg "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" + v0adt "github.com/filecoin-project/specs-actors/actors/util/adt" + "golang.org/x/xerrors" + + "github.com/filecoin-project/lotus/chain/actors/adt" +) + +type v0State struct { + v0verifreg.State + store adt.Store +} + +func (s *v0State) VerifiedClientDataCap(addr address.Address) (bool, abi.StoragePower, error) { + if addr.Protocol() != address.ID { + return false, big.Zero(), xerrors.Errorf("can only look up ID addresses") + } + + vh, err := v0adt.AsMap(s.store, s.VerifiedClients) + if err != nil { + return false, big.Zero(), xerrors.Errorf("loading verified clients: %w", err) + } + + var dcap abi.StoragePower + if found, err := vh.Get(abi.AddrKey(addr), &dcap); err != nil { + return false, big.Zero(), xerrors.Errorf("looking up verified clients: %w", err) + } else if !found { + return false, big.Zero(), nil + } + + return true, dcap, nil +} diff --git a/chain/actors/builtin/verifreg/verifreg.go b/chain/actors/builtin/verifreg/verifreg.go new file mode 100644 index 000000000..4cb5bb55b --- /dev/null +++ b/chain/actors/builtin/verifreg/verifreg.go @@ -0,0 +1,34 @@ +package verifreg + +import ( + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-state-types/cbor" + v0builtin "github.com/filecoin-project/specs-actors/actors/builtin" + + "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/chain/types" +) + +var Address = v0builtin.VerifiedRegistryActorAddr + +func Load(store adt.Store, act *types.Actor) (State, error) { + switch act.Code { + case v0builtin.VerifiedRegistryActorCodeID: + out := v0State{store: store} + err := store.Get(store.Context(), act.Head, &out) + if err != nil { + return nil, err + } + return &out, nil + } + return nil, xerrors.Errorf("unknown actor code %s", act.Code) +} + +type State interface { + cbor.Marshaler + + VerifiedClientDataCap(address.Address) (bool, abi.StoragePower, error) +} diff --git a/chain/stmgr/read.go b/chain/stmgr/read.go index 53eb8d384..9a9b80265 100644 --- a/chain/stmgr/read.go +++ b/chain/stmgr/read.go @@ -2,6 +2,7 @@ package stmgr import ( "context" + "golang.org/x/xerrors" "github.com/ipfs/go-cid" diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index 2a6737e2c..89ad6edd9 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -1029,7 +1029,7 @@ func GetFilMined(ctx context.Context, st *state.StateTree) (abi.TokenAmount, err return big.Zero(), xerrors.Errorf("failed to load reward state: %w", err) } - return rst.TotalStoragePowerReward(), nil + return rst.TotalStoragePowerReward() } func getFilMarketLocked(ctx context.Context, st *state.StateTree) (abi.TokenAmount, error) { diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index b7c888a36..4fabefbff 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -9,6 +9,10 @@ import ( "runtime" "strings" + v0init "github.com/filecoin-project/specs-actors/actors/builtin/init" + v0market "github.com/filecoin-project/specs-actors/actors/builtin/market" + v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" + v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" saruntime "github.com/filecoin-project/specs-actors/actors/runtime" @@ -541,12 +545,12 @@ var MethodsMap = map[cid.Cid]map[abi.MethodNum]MethodMeta{} func init() { cidToMethods := map[cid.Cid][2]interface{}{ // builtin.SystemActorCodeID: {builtin.MethodsSystem, system.Actor{} }- apparently it doesn't have methods - builtin.InitActorCodeID: {builtin.MethodsInit, init_.Actor{}}, + builtin.InitActorCodeID: {builtin.MethodsInit, v0init.Actor{}}, builtin.CronActorCodeID: {builtin.MethodsCron, cron.Actor{}}, builtin.AccountActorCodeID: {builtin.MethodsAccount, account.Actor{}}, - builtin.StoragePowerActorCodeID: {builtin.MethodsPower, power.Actor{}}, - builtin.StorageMinerActorCodeID: {builtin.MethodsMiner, miner.Actor{}}, - builtin.StorageMarketActorCodeID: {builtin.MethodsMarket, market.Actor{}}, + builtin.StoragePowerActorCodeID: {builtin.MethodsPower, v0power.Actor{}}, + builtin.StorageMinerActorCodeID: {builtin.MethodsMiner, v0miner.Actor{}}, + builtin.StorageMarketActorCodeID: {builtin.MethodsMarket, v0market.Actor{}}, builtin.PaymentChannelActorCodeID: {builtin.MethodsPaych, paych.Actor{}}, builtin.MultisigActorCodeID: {builtin.MethodsMultisig, v0msig.Actor{}}, builtin.RewardActorCodeID: {builtin.MethodsReward, v0reward.Actor{}}, diff --git a/chain/sub/incoming.go b/chain/sub/incoming.go index 1af5d8188..c6e0c8b80 100644 --- a/chain/sub/incoming.go +++ b/chain/sub/incoming.go @@ -1,7 +1,6 @@ package sub import ( - "bytes" "context" "errors" "fmt" diff --git a/cli/client.go b/cli/client.go index eda5ffae8..f6d529943 100644 --- a/cli/client.go +++ b/cli/client.go @@ -33,9 +33,9 @@ import ( "github.com/filecoin-project/lotus/api" lapi "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors/builtin/market" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/tablewriter" - "github.com/filecoin-project/lotus/chain/actors/builtin/market" ) var CidBaseFlag = cli.StringFlag{ @@ -417,7 +417,7 @@ var clientDealCmd = &cli.Command{ return err } - isVerified := dcap != nil + isVerified := dcap != types.EmptyInt // If the user has explicitly set the --verified-deal flag if cctx.IsSet("verified-deal") { @@ -1044,8 +1044,8 @@ var clientListDeals = &cli.Command{ func dealFromDealInfo(ctx context.Context, full api.FullNode, head *types.TipSet, v api.DealInfo) deal { if v.DealID == 0 { return deal{ - LocalDeal: v, - OnChainDealState: *market.EmptyDealState() + LocalDeal: v, + OnChainDealState: *market.EmptyDealState(), } } diff --git a/cmd/lotus-storage-miner/proving.go b/cmd/lotus-storage-miner/proving.go index d9bf81376..d3e213752 100644 --- a/cmd/lotus-storage-miner/proving.go +++ b/cmd/lotus-storage-miner/proving.go @@ -74,7 +74,7 @@ var provingFaultsCmd = &cli.Command{ _, _ = fmt.Fprintln(tw, "deadline\tpartition\tsectors") err = mas.ForEachDeadline(func(dlIdx uint64, dl miner.Deadline) error { dl.ForEachPartition(func(partIdx uint64, part miner.Partition) error { - faults, err := part.Faults() + faults, err := part.FaultySectors() if err != nil { return err } @@ -158,7 +158,7 @@ var provingInfoCmd = &cli.Command{ } } - if bf, err := part.Faults(); err != nil { + if bf, err := part.FaultySectors(); err != nil { return err } else if count, err := bf.Count(); err != nil { return err @@ -166,7 +166,7 @@ var provingInfoCmd = &cli.Command{ faults += count } - if bf, err := part.Recovering(); err != nil { + if bf, err := part.RecoveringSectors(); err != nil { return err } else if count, err := bf.Count(); err != nil { return err @@ -286,14 +286,14 @@ var provingDeadlinesCmd = &cli.Command{ faults := uint64(0) for _, partition := range partitions { - sc, err := partition.Sectors.Count() + sc, err := partition.AllSectors.Count() if err != nil { return err } sectors += sc - fc, err := partition.Faults.Count() + fc, err := partition.FaultySectors.Count() if err != nil { return err } diff --git a/markets/storageadapter/provider.go b/markets/storageadapter/provider.go index fb93a1e72..90bf28b41 100644 --- a/markets/storageadapter/provider.go +++ b/markets/storageadapter/provider.go @@ -8,8 +8,6 @@ import ( "io" "time" - "github.com/filecoin-project/go-state-types/big" - "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/ipfs/go-cid" @@ -362,12 +360,14 @@ func (n *ProviderNodeAdapter) WaitForMessage(ctx context.Context, mcid cid.Cid, return cb(receipt.Receipt.ExitCode, receipt.Receipt.Return, receipt.Message, nil) } -func (n *ProviderNodeAdapter) GetDataCap(ctx context.Context, addr address.Address, encodedTs shared.TipSetToken) (abi.StoragePower, error) { +func (n *ProviderNodeAdapter) GetDataCap(ctx context.Context, addr address.Address, encodedTs shared.TipSetToken) (*abi.StoragePower, error) { tsk, err := types.TipSetKeyFromBytes(encodedTs) if err != nil { - return big.Zero(), err + return nil, err } - return n.StateVerifiedClientStatus(ctx, addr, tsk) + + sp, err := n.StateVerifiedClientStatus(ctx, addr, tsk) + return &sp, err } func (n *ProviderNodeAdapter) OnDealExpiredOrSlashed(ctx context.Context, dealID abi.DealID, onDealExpired storagemarket.DealExpiredCallback, onDealSlashed storagemarket.DealSlashedCallback) error { diff --git a/node/impl/full/state.go b/node/impl/full/state.go index a21f0cf80..48f9d503c 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -6,6 +6,8 @@ import ( "errors" "strconv" + v0market "github.com/filecoin-project/specs-actors/actors/builtin/market" + "github.com/filecoin-project/lotus/chain/actors/builtin/verifreg" "github.com/filecoin-project/go-state-types/dline" @@ -73,6 +75,11 @@ func (a *StateAPI) StateMinerSectors(ctx context.Context, addr address.Address, } func (a *StateAPI) StateMinerActiveSectors(ctx context.Context, maddr address.Address, tsk types.TipSetKey) ([]*miner.ChainSectorInfo, error) { // TODO: only used in cli + ts, err := a.Chain.GetTipSetFromKey(tsk) + if err != nil { + return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) + } + act, err := a.StateManager.LoadActorTsk(ctx, maddr, tsk) if err != nil { return nil, xerrors.Errorf("failed to load miner actor: %w", err) @@ -88,7 +95,7 @@ func (a *StateAPI) StateMinerActiveSectors(ctx context.Context, maddr address.Ad return nil, xerrors.Errorf("merge partition active sets: %w", err) } - return mas.LoadSectorsFromSet(&activeSectors, false) + return stmgr.GetMinerSectorSet(ctx, a.StateManager, ts, maddr, &activeSectors, false) } func (a *StateAPI) StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (miner.MinerInfo, error) { @@ -105,7 +112,7 @@ func (a *StateAPI) StateMinerInfo(ctx context.Context, actor address.Address, ts return mas.Info() } -func (a *StateAPI) StateMinerDeadlines(ctx context.Context, m address.Address, tsk types.TipSetKey) ([]miner.Deadline, error) { +func (a *StateAPI) StateMinerDeadlines(ctx context.Context, m address.Address, tsk types.TipSetKey) ([]*miner.Deadline, error) { act, err := a.StateManager.LoadActorTsk(ctx, m, tsk) if err != nil { return nil, xerrors.Errorf("failed to load miner actor: %w", err) @@ -121,9 +128,9 @@ func (a *StateAPI) StateMinerDeadlines(ctx context.Context, m address.Address, t return nil, xerrors.Errorf("getting deadline count: %w", err) } - out := make([]miner.Deadline, deadlines) + out := make([]*miner.Deadline, deadlines) if err := mas.ForEachDeadline(func(i uint64, dl miner.Deadline) error { - out[i] = dl + out[i] = &dl return nil }); err != nil { return nil, err @@ -671,12 +678,18 @@ func (a *StateAPI) StateMinerSectorCount(ctx context.Context, addr address.Addre return api.MinerSectors{Live: liveCount, Active: activeCount, Faulty: faultyCount}, nil } -func (a *StateAPI) StateSectorPreCommitInfo(ctx context.Context, maddr address.Address, n abi.SectorNumber, tsk types.TipSetKey) (*miner.SectorPreCommitOnChainInfo, error) { +func (a *StateAPI) StateSectorPreCommitInfo(ctx context.Context, maddr address.Address, n abi.SectorNumber, tsk types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) { ts, err := a.Chain.GetTipSetFromKey(tsk) if err != nil { - return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) + return miner.SectorPreCommitOnChainInfo{}, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - return stmgr.PreCommitInfo(ctx, a.StateManager, maddr, n, ts) + + pci, err := stmgr.PreCommitInfo(ctx, a.StateManager, maddr, n, ts) + if err != nil { + return miner.SectorPreCommitOnChainInfo{}, err + } + + return *pci, err } func (a *StateAPI) StateSectorGetInfo(ctx context.Context, maddr address.Address, n abi.SectorNumber, tsk types.TipSetKey) (*miner.SectorOnChainInfo, error) { @@ -873,7 +886,7 @@ func (a *StateAPI) StateMinerPreCommitDepositForPower(ctx context.Context, maddr // NB: not exactly accurate, but should always lead us to *over* estimate, not under duration := pci.Expiration - ts.Height() - // TODO: handle changes to this function across actor upgrades. + // TODO: ActorUpgrade sectorWeight = v0miner.QAPowerForWeight(ssize, duration, w, vw) } @@ -899,8 +912,8 @@ func (a *StateAPI) StateMinerPreCommitDepositForPower(ctx context.Context, maddr rewardSmoothed = r } - // TODO: abstract over network upgrades. - deposit := v0miner.PreCommitDepositForPower(rewardSmoothed, powerSmoothed, sectorWeight) + // TODO: ActorUpgrade + deposit := v0miner.PreCommitDepositForPower(&rewardSmoothed, &powerSmoothed, sectorWeight) return types.BigDiv(types.BigMul(deposit, initialPledgeNum), initialPledgeDen), nil } @@ -940,8 +953,8 @@ func (a *StateAPI) StateMinerInitialPledgeCollateral(ctx context.Context, maddr } var ( - powerSmoothed smoothing.FilterEstimate - pledgeCollerateral abi.TokenAmount + powerSmoothed smoothing.FilterEstimate + pledgeCollateral abi.TokenAmount ) if act, err := state.GetActor(power.Address); err != nil { return types.EmptyInt, xerrors.Errorf("loading miner actor: %w", err) @@ -966,26 +979,26 @@ func (a *StateAPI) StateMinerInitialPledgeCollateral(ctx context.Context, maddr return types.EmptyInt, xerrors.Errorf("loading reward actor state: %w", err) } else if r, err := s.RewardSmoothed(); err != nil { return types.EmptyInt, xerrors.Errorf("failed to determine total reward: %w", err) - } else if p, err := s.BaselinePower(); err != nil { + } else if p, err := s.ThisEpochBaselinePower(); err != nil { return types.EmptyInt, xerrors.Errorf("failed to determine baseline power: %w", err) } else { rewardSmoothed = r baselinePower = p } - // TODO: abstract over network upgrades. - circSupply, err := a.StateCirculatingSupply(ctx, ts.Key()) if err != nil { return big.Zero(), xerrors.Errorf("getting circulating supply: %w", err) } - initialPledge := miner.InitialPledgeForPower( + // TODO: ActorUpgrade + + initialPledge := v0miner.InitialPledgeForPower( sectorWeight, baselinePower, pledgeCollateral, - rewardSmoothed, - powerSmoothed, + &rewardSmoothed, + &powerSmoothed, circSupply.FilCirculating, ) @@ -998,23 +1011,27 @@ func (a *StateAPI) StateMinerAvailableBalance(ctx context.Context, maddr address return types.EmptyInt, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - var act *types.Actor - var mas miner.State - - if err := a.StateManager.WithParentState(ts, a.StateManager.WithActor(maddr, func(actor *types.Actor) error { - act = actor - return a.StateManager.WithActorState(ctx, &mas)(actor) - })); err != nil { - return types.BigInt{}, xerrors.Errorf("getting miner state: %w", err) + act, err := a.StateManager.LoadActor(ctx, maddr, ts) + if err != nil { + return types.EmptyInt, xerrors.Errorf("failed to load miner actor: %w", err) } - as := store.ActorStore(ctx, a.Chain.Blockstore()) - vested, err := mas.CheckVestedFunds(as, ts.Height()) + mas, err := miner.Load(a.StateManager.ChainStore().Store(ctx), act) + if err != nil { + return types.EmptyInt, xerrors.Errorf("failed to load miner actor state: %w", err) + } + + vested, err := mas.VestedFunds(ts.Height()) if err != nil { return types.EmptyInt, err } - return types.BigAdd(mas.GetAvailableBalance(act.Balance), vested), nil + abal, err := mas.AvailableBalance(act.Balance) + if err != nil { + return types.EmptyInt, err + } + + return types.BigAdd(abal, vested), nil } // StateVerifiedClientStatus returns the data cap for the given address. @@ -1023,23 +1040,23 @@ func (a *StateAPI) StateMinerAvailableBalance(ctx context.Context, maddr address func (a *StateAPI) StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (abi.StoragePower, error) { act, err := a.StateGetActor(ctx, builtin.VerifiedRegistryActorAddr, tsk) if err != nil { - return big.Zero(), err + return types.EmptyInt, err } aid, err := a.StateLookupID(ctx, addr, tsk) if err != nil { log.Warnf("lookup failure %v", err) - return big.Zero(), err + return types.EmptyInt, err } vrs, err := verifreg.Load(a.StateManager.ChainStore().Store(ctx), act) if err != nil { - return big.Zero(), xerrors.Errorf("failed to load verified registry state: %w", err) + return types.EmptyInt, xerrors.Errorf("failed to load verified registry state: %w", err) } _, dcap, err := vrs.VerifiedClientDataCap(aid) if err != nil { - return big.Zero(), xerrors.Errorf("looking up verified client: %w", err) + return types.EmptyInt, xerrors.Errorf("looking up verified client: %w", err) } return dcap, nil @@ -1056,23 +1073,24 @@ func (a *StateAPI) StateDealProviderCollateralBounds(ctx context.Context, size a return api.DealCollateralBounds{}, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - var powerState power.State - var rewardState reward.State - - err = a.StateManager.WithParentStateTsk(ts.Key(), func(state *state.StateTree) error { - if err := a.StateManager.WithActor(builtin.StoragePowerActorAddr, a.StateManager.WithActorState(ctx, &powerState))(state); err != nil { - return xerrors.Errorf("getting power state: %w", err) - } - - if err := a.StateManager.WithActor(builtin.RewardActorAddr, a.StateManager.WithActorState(ctx, &rewardState))(state); err != nil { - return xerrors.Errorf("getting reward state: %w", err) - } - - return nil - }) - + pact, err := a.StateGetActor(ctx, builtin.StoragePowerActorAddr, tsk) if err != nil { - return api.DealCollateralBounds{}, xerrors.Errorf("getting power and reward actor states: %w", err) + return api.DealCollateralBounds{}, xerrors.Errorf("failed to load power actor: %w", err) + } + + ract, err := a.StateGetActor(ctx, builtin.RewardActorAddr, tsk) + if err != nil { + return api.DealCollateralBounds{}, xerrors.Errorf("failed to load reward actor: %w", err) + } + + pst, err := power.Load(a.StateManager.ChainStore().Store(ctx), pact) + if err != nil { + return api.DealCollateralBounds{}, xerrors.Errorf("failed to load power actor state: %w", err) + } + + rst, err := reward.Load(a.StateManager.ChainStore().Store(ctx), ract) + if err != nil { + return api.DealCollateralBounds{}, xerrors.Errorf("failed to load reward actor state: %w", err) } circ, err := a.StateCirculatingSupply(ctx, ts.Key()) @@ -1080,11 +1098,21 @@ func (a *StateAPI) StateDealProviderCollateralBounds(ctx context.Context, size a return api.DealCollateralBounds{}, xerrors.Errorf("getting total circulating supply: %w", err) } - min, max := market.DealProviderCollateralBounds(size, + powClaim, err := pst.TotalPower() + if err != nil { + return api.DealCollateralBounds{}, xerrors.Errorf("getting total power: %w", err) + } + + rewPow, err := rst.ThisEpochBaselinePower() + if err != nil { + return api.DealCollateralBounds{}, xerrors.Errorf("getting reward baseline power: %w", err) + } + + min, max := v0market.DealProviderCollateralBounds(size, verified, - powerState.TotalRawBytePower, - powerState.ThisEpochQualityAdjPower, - rewardState.ThisEpochBaselinePower, + powClaim.RawBytePower, + powClaim.QualityAdjPower, + rewPow, circ.FilCirculating, a.StateManager.GetNtwkVersion(ctx, ts.Height())) return api.DealCollateralBounds{ @@ -1109,7 +1137,7 @@ func (a *StateAPI) StateCirculatingSupply(ctx context.Context, tsk types.TipSetK func (a *StateAPI) StateNetworkVersion(ctx context.Context, tsk types.TipSetKey) (network.Version, error) { ts, err := a.Chain.GetTipSetFromKey(tsk) if err != nil { - return -1, xerrors.Errorf("loading tipset %s: %w", tsk, err) + return network.VersionMax, xerrors.Errorf("loading tipset %s: %w", tsk, err) } return a.StateManager.GetNtwkVersion(ctx, ts.Height()), nil diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 89dd7c2c2..5634c140b 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -328,7 +328,7 @@ func (sm *StorageMinerAPI) listDeals(ctx context.Context) ([]api.MarketDeal, err } func (sm *StorageMinerAPI) MarketListDeals(ctx context.Context) ([]api.MarketDeal, error) { - return sm.StorageProvider.listDeals(ctx) + return sm.listDeals(ctx) } func (sm *StorageMinerAPI) MarketListRetrievalDeals(ctx context.Context) ([]retrievalmarket.ProviderDealState, error) { diff --git a/storage/adapter_storage_miner.go b/storage/adapter_storage_miner.go index 7dced4331..8db977d2c 100644 --- a/storage/adapter_storage_miner.go +++ b/storage/adapter_storage_miner.go @@ -84,7 +84,7 @@ func (s SealingAPIAdapter) StateMinerWorkerAddress(ctx context.Context, maddr ad return mi.Worker, nil } -func (s SealingAPIAdapter) StateMinerDeadlines(ctx context.Context, maddr address.Address, tok sealing.TipSetToken) ([]miner.Deadline, error) { +func (s SealingAPIAdapter) StateMinerDeadlines(ctx context.Context, maddr address.Address, tok sealing.TipSetToken) ([]*miner.Deadline, error) { tsk, err := types.TipSetKeyFromBytes(tok) if err != nil { return nil, xerrors.Errorf("failed to unmarshal TipSetToken to TipSetKey: %w", err) @@ -251,7 +251,7 @@ func (s SealingAPIAdapter) StateMarketStorageDeal(ctx context.Context, dealID ab func (s SealingAPIAdapter) StateNetworkVersion(ctx context.Context, tok sealing.TipSetToken) (network.Version, error) { tsk, err := types.TipSetKeyFromBytes(tok) if err != nil { - return -1, err + return network.VersionMax, err } return s.delegate.StateNetworkVersion(ctx, tsk) diff --git a/storage/addresses.go b/storage/addresses.go index bef845367..639e996e3 100644 --- a/storage/addresses.go +++ b/storage/addresses.go @@ -3,12 +3,13 @@ package storage import ( "context" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + "golang.org/x/xerrors" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" ) @@ -27,7 +28,7 @@ type addrSelectApi interface { StateAccountKey(context.Context, address.Address, types.TipSetKey) (address.Address, error) } -func AddressFor(ctx context.Context, a addrSelectApi, mi api.MinerInfo, use AddrUse, minFunds abi.TokenAmount) (address.Address, error) { +func AddressFor(ctx context.Context, a addrSelectApi, mi miner.MinerInfo, use AddrUse, minFunds abi.TokenAmount) (address.Address, error) { switch use { case PreCommitAddr, CommitAddr: // always use worker, at least for now diff --git a/storage/miner.go b/storage/miner.go index 693ad7c05..227c51961 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -76,7 +76,7 @@ type storageMinerApi interface { StateSectorGetInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorOnChainInfo, error) StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*miner.SectorLocation, error) StateMinerInfo(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error) - StateMinerDeadlines(context.Context, address.Address, types.TipSetKey) ([]miner.Deadline, error) + StateMinerDeadlines(context.Context, address.Address, types.TipSetKey) ([]*miner.Deadline, error) StateMinerProvingDeadline(context.Context, address.Address, types.TipSetKey) (*dline.Info, error) StateMinerPreCommitDepositForPower(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error) StateMinerInitialPledgeCollateral(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error) diff --git a/storage/wdpost_sched.go b/storage/wdpost_sched.go index c927a1547..112210baf 100644 --- a/storage/wdpost_sched.go +++ b/storage/wdpost_sched.go @@ -4,8 +4,6 @@ import ( "context" "time" - "github.com/filecoin-project/go-state-types/dline" - "golang.org/x/xerrors" "github.com/filecoin-project/go-address" From 12e76bc1afd9a33a8e10be62c7d332e83d4852bc Mon Sep 17 00:00:00 2001 From: Marius Darila <3396463+kenshyx@users.noreply.github.com> Date: Thu, 17 Sep 2020 12:17:18 +0200 Subject: [PATCH 175/885] replace Requires with Wants Prevent lotus-miner shutdown when daemon restarts. --- scripts/lotus-miner.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/lotus-miner.service b/scripts/lotus-miner.service index c079f44a9..54c48d411 100644 --- a/scripts/lotus-miner.service +++ b/scripts/lotus-miner.service @@ -2,7 +2,7 @@ Description=Lotus Miner After=network.target After=lotus-daemon.service -Requires=lotus-daemon.service +Wants=lotus-daemon.service [Service] ExecStart=/usr/local/bin/lotus-miner run From e6326438013bfe0c30cc1f86c04f398b3c681dd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 17 Sep 2020 12:22:56 +0200 Subject: [PATCH 176/885] api: Test return types --- api/api_full.go | 2 +- api/api_test.go | 69 +++++++++++++++++++ api/apistruct/struct.go | 4 +- cli/wallet.go | 6 +- .../sector-storage/ffiwrapper/sealer_test.go | 12 ++++ node/impl/full/wallet.go | 4 +- 6 files changed, 91 insertions(+), 6 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 1adcc5c19..778b458a8 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -230,7 +230,7 @@ type FullNode interface { WalletSignMessage(context.Context, address.Address, *types.Message) (*types.SignedMessage, error) // WalletVerify takes an address, a signature, and some bytes, and indicates whether the signature is valid. // The address does not have to be in the wallet. - WalletVerify(context.Context, address.Address, []byte, *crypto.Signature) bool + WalletVerify(context.Context, address.Address, []byte, *crypto.Signature) (bool, error) // WalletDefaultAddress returns the address marked as default in the wallet. WalletDefaultAddress(context.Context) (address.Address, error) // WalletSetDefault marks the given address as as the default one. diff --git a/api/api_test.go b/api/api_test.go index 1b438258a..065141426 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -1,12 +1,16 @@ package api import ( + "encoding/json" "os" "os/exec" "path/filepath" + "reflect" "runtime" "strings" "testing" + + "github.com/stretchr/testify/require" ) func goCmd() string { @@ -32,3 +36,68 @@ func TestDoesntDependOnFFI(t *testing.T) { } } } + +func TestReturnTypes(t *testing.T) { + errType := reflect.TypeOf(new(error)).Elem() + bareIface := reflect.TypeOf(new(interface{})).Elem() + jmarsh := reflect.TypeOf(new(json.Marshaler)).Elem() + + tst := func(api interface{}) func(t *testing.T) { + return func(t *testing.T) { + ra := reflect.TypeOf(api).Elem() + for i := 0; i < ra.NumMethod(); i++ { + m := ra.Method(i) + switch m.Type.NumOut() { + case 1: // if 1 return value, it must be an error + require.Equal(t, errType, m.Type.Out(0), m.Name) + + case 2: // if 2 return values, first cant be an interface/function, second must be an error + seen := map[reflect.Type]struct{}{} + todo := []reflect.Type{m.Type.Out(0)} + for len(todo) > 0 { + typ := todo[len(todo) - 1] + todo = todo[:len(todo)-1] + + if _, ok := seen[typ]; ok { + continue + } + seen[typ] = struct{}{} + + if typ.Kind() == reflect.Interface && typ != bareIface && !typ.Implements(jmarsh) { + t.Error("methods can't return interfaces", m.Name) + } + + switch typ.Kind() { + case reflect.Ptr: + fallthrough + case reflect.Array: + fallthrough + case reflect.Slice: + fallthrough + case reflect.Chan: + todo = append(todo, typ.Elem()) + case reflect.Map: + todo = append(todo, typ.Elem()) + todo = append(todo, typ.Key()) + case reflect.Struct: + for i := 0; i < typ.NumField(); i++ { + todo = append(todo, typ.Field(i).Type) + } + } + } + + require.NotEqual(t, reflect.Func.String(), m.Type.Out(0).Kind().String(), m.Name) + require.Equal(t, errType, m.Type.Out(1), m.Name) + + default: + t.Error("methods can only have 1 or 2 return values", m.Name) + } + } + } + } + + t.Run("common", tst(new(Common))) + t.Run("full", tst(new(FullNode))) + t.Run("miner", tst(new(StorageMiner))) + t.Run("worker", tst(new(WorkerAPI))) +} diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index ab86d7819..c8a2b91ac 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -134,7 +134,7 @@ type FullNodeStruct struct { WalletBalance func(context.Context, address.Address) (types.BigInt, error) `perm:"read"` WalletSign func(context.Context, address.Address, []byte) (*crypto.Signature, error) `perm:"sign"` WalletSignMessage func(context.Context, address.Address, *types.Message) (*types.SignedMessage, error) `perm:"sign"` - WalletVerify func(context.Context, address.Address, []byte, *crypto.Signature) bool `perm:"read"` + WalletVerify func(context.Context, address.Address, []byte, *crypto.Signature) (bool, error) `perm:"read"` WalletDefaultAddress func(context.Context) (address.Address, error) `perm:"write"` WalletSetDefault func(context.Context, address.Address) error `perm:"admin"` WalletExport func(context.Context, address.Address) (*types.KeyInfo, error) `perm:"admin"` @@ -604,7 +604,7 @@ func (c *FullNodeStruct) WalletSignMessage(ctx context.Context, k address.Addres return c.Internal.WalletSignMessage(ctx, k, msg) } -func (c *FullNodeStruct) WalletVerify(ctx context.Context, k address.Address, msg []byte, sig *crypto.Signature) bool { +func (c *FullNodeStruct) WalletVerify(ctx context.Context, k address.Address, msg []byte, sig *crypto.Signature) (bool, error) { return c.Internal.WalletVerify(ctx, k, msg, sig) } diff --git a/cli/wallet.go b/cli/wallet.go index 4339a1fb6..27993a1ba 100644 --- a/cli/wallet.go +++ b/cli/wallet.go @@ -382,7 +382,11 @@ var walletVerify = &cli.Command{ return err } - if api.WalletVerify(ctx, addr, msg, &sig) { + ok, err := api.WalletVerify(ctx, addr, msg, &sig) + if err != nil { + return err + } + if ok { fmt.Println("valid") return nil } diff --git a/extern/sector-storage/ffiwrapper/sealer_test.go b/extern/sector-storage/ffiwrapper/sealer_test.go index d59de2cab..bb26adb77 100644 --- a/extern/sector-storage/ffiwrapper/sealer_test.go +++ b/extern/sector-storage/ffiwrapper/sealer_test.go @@ -245,6 +245,10 @@ func TestDownloadParams(t *testing.T) { } func TestSealAndVerify(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode") + } + defer requireFDsClosed(t, openFDs(t)) if runtime.NumCPU() < 10 && os.Getenv("CI") == "" { // don't bother on slow hardware @@ -314,6 +318,10 @@ func TestSealAndVerify(t *testing.T) { } func TestSealPoStNoCommit(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode") + } + defer requireFDsClosed(t, openFDs(t)) if runtime.NumCPU() < 10 && os.Getenv("CI") == "" { // don't bother on slow hardware @@ -375,6 +383,10 @@ func TestSealPoStNoCommit(t *testing.T) { } func TestSealAndVerify3(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode") + } + defer requireFDsClosed(t, openFDs(t)) if runtime.NumCPU() < 10 && os.Getenv("CI") == "" { // don't bother on slow hardware diff --git a/node/impl/full/wallet.go b/node/impl/full/wallet.go index bda8824e7..64231b74e 100644 --- a/node/impl/full/wallet.go +++ b/node/impl/full/wallet.go @@ -67,8 +67,8 @@ func (a *WalletAPI) WalletSignMessage(ctx context.Context, k address.Address, ms }, nil } -func (a *WalletAPI) WalletVerify(ctx context.Context, k address.Address, msg []byte, sig *crypto.Signature) bool { - return sigs.Verify(sig, k, msg) == nil +func (a *WalletAPI) WalletVerify(ctx context.Context, k address.Address, msg []byte, sig *crypto.Signature) (bool, error) { + return sigs.Verify(sig, k, msg) == nil, nil } func (a *WalletAPI) WalletDefaultAddress(ctx context.Context) (address.Address, error) { From 61c0b8c3db61ccb1d97fe05fd856c74fda0d0451 Mon Sep 17 00:00:00 2001 From: vyzo Date: Wed, 16 Sep 2020 10:14:28 +0300 Subject: [PATCH 177/885] properly close streams in blocksync we were leaking streams right and left... --- chain/exchange/client.go | 7 +++++++ chain/exchange/server.go | 5 ++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/chain/exchange/client.go b/chain/exchange/client.go index 22f7a9457..57563d5b2 100644 --- a/chain/exchange/client.go +++ b/chain/exchange/client.go @@ -7,6 +7,7 @@ import ( "math/rand" "time" + "github.com/libp2p/go-libp2p-core/helpers" "github.com/libp2p/go-libp2p-core/host" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" @@ -357,6 +358,12 @@ func (c *client) sendRequestToPeer(ctx context.Context, peer peer.ID, req *Reque return nil, xerrors.Errorf("failed to open stream to peer: %w", err) } + defer func() { + // Note: this will become just stream.Close once we've completed the go-libp2p migration to + // go-libp2p-core 0.7.0 + go helpers.FullClose(stream) //nolint:errcheck + }() + // Write request. _ = stream.SetWriteDeadline(time.Now().Add(WriteReqDeadline)) if err := cborutil.WriteCborRPC(stream, req); err != nil { diff --git a/chain/exchange/server.go b/chain/exchange/server.go index 54e169b3f..dcdb5b3a5 100644 --- a/chain/exchange/server.go +++ b/chain/exchange/server.go @@ -15,6 +15,7 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/ipfs/go-cid" + "github.com/libp2p/go-libp2p-core/helpers" inet "github.com/libp2p/go-libp2p-core/network" ) @@ -39,7 +40,9 @@ func (s *server) HandleStream(stream inet.Stream) { ctx, span := trace.StartSpan(context.Background(), "chainxchg.HandleStream") defer span.End() - defer stream.Close() //nolint:errcheck + // Note: this will become just stream.Close once we've completed the go-libp2p migration to + // go-libp2p-core 0.7.0 + defer helpers.FullClose(stream) //nolint:errcheck var req Request if err := cborutil.ReadCborRPC(bufio.NewReader(stream), &req); err != nil { From 35f6e1064620f05f29f8ec775f453b2e56c3e9ae Mon Sep 17 00:00:00 2001 From: vyzo Date: Wed, 16 Sep 2020 21:04:44 +0300 Subject: [PATCH 178/885] parallel chain sync --- chain/sync.go | 114 +++++++++++++++++++++++++++----------------------- 1 file changed, 62 insertions(+), 52 deletions(-) diff --git a/chain/sync.go b/chain/sync.go index b5716a343..882fb1d95 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -7,7 +7,6 @@ import ( "fmt" "os" "sort" - "strconv" "strings" "sync" "time" @@ -63,20 +62,12 @@ var ( // where the Syncer publishes candidate chain heads to be synced. LocalIncoming = "incoming" - log = logging.Logger("chain") - defaultMessageFetchWindowSize = 200 -) + log = logging.Logger("chain") -func init() { - if s := os.Getenv("LOTUS_BSYNC_MSG_WINDOW"); s != "" { - val, err := strconv.Atoi(s) - if err != nil { - log.Errorf("failed to parse LOTUS_BSYNC_MSG_WINDOW: %s", err) - return - } - defaultMessageFetchWindowSize = val - } -} + concurrentSyncRequests = 16 + syncRequestBatchSize = 4 + syncRequestRetries = 5 +) // Syncer is in charge of running the chain synchronization logic. As such, it // is tasked with these functions, amongst others: @@ -132,8 +123,6 @@ type Syncer struct { verifier ffiwrapper.Verifier - windowSize int - tickerCtxCancel context.CancelFunc checkptLk sync.Mutex @@ -175,7 +164,6 @@ func NewSyncer(ds dtypes.MetadataDS, sm *stmgr.StateManager, exchange exchange.C receiptTracker: newBlockReceiptTracker(), connmgr: connmgr, verifier: verifier, - windowSize: defaultMessageFetchWindowSize, incoming: pubsub.New(50), } @@ -1483,8 +1471,6 @@ func (syncer *Syncer) iterFullTipsets(ctx context.Context, headers []*types.TipS span.AddAttributes(trace.Int64Attribute("num_headers", int64(len(headers)))) - windowSize := syncer.windowSize -mainLoop: for i := len(headers) - 1; i >= 0; { fts, err := syncer.store.TryFillTipSet(headers[i]) if err != nil { @@ -1498,35 +1484,73 @@ mainLoop: continue } - batchSize := windowSize + batchSize := concurrentSyncRequests * syncRequestBatchSize if i < batchSize { - batchSize = i + if i == 0 { + batchSize = 1 + } else { + batchSize = i + } } - nextI := (i + 1) - batchSize // want to fetch batchSize values, 'i' points to last one we want to fetch, so its 'inclusive' of our request, thus we need to add one to our request start index - ss.SetStage(api.StageFetchingMessages) - var bstout []*exchange.CompactedMessages - for len(bstout) < batchSize { - next := headers[nextI] + bstout := make([]*exchange.CompactedMessages, batchSize) + var wg sync.WaitGroup + var mx sync.Mutex + var batchErr error + for j := 0; j < batchSize; j += syncRequestBatchSize { + wg.Add(1) + go func(j int) { + defer wg.Done() - nreq := batchSize - len(bstout) - bstips, err := syncer.Exchange.GetChainMessages(ctx, next, uint64(nreq)) - if err != nil { - // TODO check errors for temporary nature - if windowSize > 1 { - windowSize /= 2 - log.Infof("error fetching messages: %s; reducing window size to %d and trying again", err, windowSize) - continue mainLoop + nreq := syncRequestBatchSize + if j*syncRequestBatchSize+nreq > batchSize { + nreq = batchSize - j*syncRequestBatchSize } - return xerrors.Errorf("message processing failed: %w", err) - } - bstout = append(bstout, bstips...) - nextI += len(bstips) + failed := false + for offset := 0; !failed && offset < nreq; { + nextI := (i + 1) - batchSize + j*syncRequestBatchSize + offset + nextHeader := headers[nextI] + + var requestErr error + var requestResult []*exchange.CompactedMessages + for retry := 0; requestResult == nil && retry < syncRequestRetries; retry++ { + if retry > 0 { + log.Infof("fetching messages at %d (retry %d)", nextI, retry) + } else { + log.Infof("fetching messages at %d", nextI) + } + + result, err := syncer.Exchange.GetChainMessages(ctx, nextHeader, uint64(nreq-offset)) + if err != nil { + requestErr = multierror.Append(requestErr, err) + } else { + requestResult = result + } + } + + mx.Lock() + if requestResult == nil { + // we failed! + log.Errorf("error fetching messages at %d: %s", nextI, requestErr) + batchErr = multierror.Append(batchErr, requestErr) + failed = true + } else { + copy(bstout[j*syncRequestBatchSize+offset:], requestResult) + offset += len(requestResult) + } + mx.Unlock() + } + }(j) } + wg.Wait() ss.SetStage(api.StageMessages) + if batchErr != nil { + return xerrors.Errorf("failed to fetch messages: %w", err) + } + for bsi := 0; bsi < len(bstout); bsi++ { // temp storage so we don't persist data we dont want to bs := bstore.NewTemporary() @@ -1555,23 +1579,9 @@ mainLoop: } } - if i >= windowSize { - newWindowSize := windowSize + 10 - if newWindowSize > int(exchange.MaxRequestLength) { - newWindowSize = int(exchange.MaxRequestLength) - } - if newWindowSize > windowSize { - windowSize = newWindowSize - log.Infof("successfully fetched %d messages; increasing window size to %d", len(bstout), windowSize) - } - } - i -= batchSize } - // remember our window size - syncer.windowSize = windowSize - return nil } From 05a233f84d6de0916b1135f46eef3a61c7aa02ae Mon Sep 17 00:00:00 2001 From: vyzo Date: Wed, 16 Sep 2020 21:43:35 +0300 Subject: [PATCH 179/885] add some more logging --- chain/sync.go | 1 + 1 file changed, 1 insertion(+) diff --git a/chain/sync.go b/chain/sync.go index 882fb1d95..87d2cf6f4 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -1537,6 +1537,7 @@ func (syncer *Syncer) iterFullTipsets(ctx context.Context, headers []*types.TipS batchErr = multierror.Append(batchErr, requestErr) failed = true } else { + log.Infof("fetched messages for %d tipsets", len(requestResult)) copy(bstout[j*syncRequestBatchSize+offset:], requestResult) offset += len(requestResult) } From b984e94a87237b14ef1eb90df95d5979b4c665bf Mon Sep 17 00:00:00 2001 From: vyzo Date: Wed, 16 Sep 2020 21:52:08 +0300 Subject: [PATCH 180/885] fix bug --- chain/sync.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/chain/sync.go b/chain/sync.go index 87d2cf6f4..80a5ad423 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -1504,13 +1504,13 @@ func (syncer *Syncer) iterFullTipsets(ctx context.Context, headers []*types.TipS defer wg.Done() nreq := syncRequestBatchSize - if j*syncRequestBatchSize+nreq > batchSize { - nreq = batchSize - j*syncRequestBatchSize + if j+nreq > batchSize { + nreq = batchSize - j } failed := false for offset := 0; !failed && offset < nreq; { - nextI := (i + 1) - batchSize + j*syncRequestBatchSize + offset + nextI := (i + 1) - batchSize + j + offset nextHeader := headers[nextI] var requestErr error @@ -1537,8 +1537,7 @@ func (syncer *Syncer) iterFullTipsets(ctx context.Context, headers []*types.TipS batchErr = multierror.Append(batchErr, requestErr) failed = true } else { - log.Infof("fetched messages for %d tipsets", len(requestResult)) - copy(bstout[j*syncRequestBatchSize+offset:], requestResult) + copy(bstout[j+offset:], requestResult) offset += len(requestResult) } mx.Unlock() From 8a4b629f407a9b0ff1e39e9eeb5ba7d541c4adfc Mon Sep 17 00:00:00 2001 From: vyzo Date: Wed, 16 Sep 2020 21:55:51 +0300 Subject: [PATCH 181/885] increase sync request batch size to 8 --- chain/sync.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chain/sync.go b/chain/sync.go index 80a5ad423..554f81ee9 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -65,7 +65,7 @@ var ( log = logging.Logger("chain") concurrentSyncRequests = 16 - syncRequestBatchSize = 4 + syncRequestBatchSize = 8 syncRequestRetries = 5 ) From 2a428f09e67f01bdd927f09e9e40f758b2bd60e1 Mon Sep 17 00:00:00 2001 From: vyzo Date: Wed, 16 Sep 2020 22:09:36 +0300 Subject: [PATCH 182/885] increase exchange ShufflePeersPrefix to 16, use that as the value of concurrent sync requests --- chain/exchange/protocol.go | 2 +- chain/sync.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/chain/exchange/protocol.go b/chain/exchange/protocol.go index ca4b61836..ac02cf60f 100644 --- a/chain/exchange/protocol.go +++ b/chain/exchange/protocol.go @@ -40,7 +40,7 @@ const ( WriteReqDeadline = 5 * time.Second ReadResDeadline = WriteReqDeadline ReadResMinSpeed = 50 << 10 - ShufflePeersPrefix = 5 + ShufflePeersPrefix = 16 WriteResDeadline = 60 * time.Second ) diff --git a/chain/sync.go b/chain/sync.go index 554f81ee9..0ab8ac183 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -64,7 +64,7 @@ var ( log = logging.Logger("chain") - concurrentSyncRequests = 16 + concurrentSyncRequests = exchange.ShufflePeersPrefix syncRequestBatchSize = 8 syncRequestRetries = 5 ) From fb605f6d7fedea4704882a631d3155b0c280c599 Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 17 Sep 2020 17:21:26 +0300 Subject: [PATCH 183/885] refactor parallel fetch logic into a separate function --- chain/sync.go | 118 ++++++++++++++++++++++++++++---------------------- 1 file changed, 67 insertions(+), 51 deletions(-) diff --git a/chain/sync.go b/chain/sync.go index 0ab8ac183..95c2e2e84 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -1494,57 +1494,8 @@ func (syncer *Syncer) iterFullTipsets(ctx context.Context, headers []*types.TipS } ss.SetStage(api.StageFetchingMessages) - bstout := make([]*exchange.CompactedMessages, batchSize) - var wg sync.WaitGroup - var mx sync.Mutex - var batchErr error - for j := 0; j < batchSize; j += syncRequestBatchSize { - wg.Add(1) - go func(j int) { - defer wg.Done() - - nreq := syncRequestBatchSize - if j+nreq > batchSize { - nreq = batchSize - j - } - - failed := false - for offset := 0; !failed && offset < nreq; { - nextI := (i + 1) - batchSize + j + offset - nextHeader := headers[nextI] - - var requestErr error - var requestResult []*exchange.CompactedMessages - for retry := 0; requestResult == nil && retry < syncRequestRetries; retry++ { - if retry > 0 { - log.Infof("fetching messages at %d (retry %d)", nextI, retry) - } else { - log.Infof("fetching messages at %d", nextI) - } - - result, err := syncer.Exchange.GetChainMessages(ctx, nextHeader, uint64(nreq-offset)) - if err != nil { - requestErr = multierror.Append(requestErr, err) - } else { - requestResult = result - } - } - - mx.Lock() - if requestResult == nil { - // we failed! - log.Errorf("error fetching messages at %d: %s", nextI, requestErr) - batchErr = multierror.Append(batchErr, requestErr) - failed = true - } else { - copy(bstout[j+offset:], requestResult) - offset += len(requestResult) - } - mx.Unlock() - } - }(j) - } - wg.Wait() + startOffset := i + 1 - batchSize + bstout, batchErr := syncer.fetchMessages(ctx, headers[startOffset:startOffset+batchSize], startOffset) ss.SetStage(api.StageMessages) if batchErr != nil { @@ -1585,6 +1536,71 @@ func (syncer *Syncer) iterFullTipsets(ctx context.Context, headers []*types.TipS return nil } +func (syncer *Syncer) fetchMessages(ctx context.Context, headers []*types.TipSet, startOffset int) ([]*exchange.CompactedMessages, error) { + batchSize := len(headers) + batch := make([]*exchange.CompactedMessages, batchSize) + + var wg sync.WaitGroup + var mx sync.Mutex + var batchErr error + + start := build.Clock.Now() + + for j := 0; j < batchSize; j += syncRequestBatchSize { + wg.Add(1) + go func(j int) { + defer wg.Done() + + nreq := syncRequestBatchSize + if j+nreq > batchSize { + nreq = batchSize - j + } + + failed := false + for offset := 0; !failed && offset < nreq; { + nextI := j + offset + nextHeader := headers[nextI] + + var requestErr error + var requestResult []*exchange.CompactedMessages + for retry := 0; requestResult == nil && retry < syncRequestRetries; retry++ { + if retry > 0 { + log.Infof("fetching messages at %d (retry %d)", startOffset+nextI, retry) + } else { + log.Infof("fetching messages at %d", startOffset+nextI) + } + + result, err := syncer.Exchange.GetChainMessages(ctx, nextHeader, uint64(nreq-offset)) + if err != nil { + requestErr = multierror.Append(requestErr, err) + } else { + requestResult = result + } + } + + mx.Lock() + if requestResult != nil { + copy(batch[j+offset:], requestResult) + offset += len(requestResult) + } else { + log.Errorf("error fetching messages at %d: %s", nextI, requestErr) + batchErr = multierror.Append(batchErr, requestErr) + failed = true + } + mx.Unlock() + } + }(j) + } + wg.Wait() + + if batchErr != nil { + dt := build.Clock.Since(start) + log.Infof("fetching messages for %d tipsets at %d done; took %s", batchSize, startOffset, dt) + } + + return batch, batchErr +} + func persistMessages(bs bstore.Blockstore, bst *exchange.CompactedMessages) error { for _, m := range bst.Bls { //log.Infof("putting BLS message: %s", m.Cid()) From d7948fcbcd687961f746d88ed55a2afc13024aeb Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 17 Sep 2020 17:33:52 +0300 Subject: [PATCH 184/885] fix log; we want to log time when we succeed! --- chain/sync.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chain/sync.go b/chain/sync.go index 95c2e2e84..d09ba84de 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -1593,7 +1593,7 @@ func (syncer *Syncer) fetchMessages(ctx context.Context, headers []*types.TipSet } wg.Wait() - if batchErr != nil { + if batchErr == nil { dt := build.Clock.Since(start) log.Infof("fetching messages for %d tipsets at %d done; took %s", batchSize, startOffset, dt) } From 2946561decdb7bd7b9056cc256c6e32294a13279 Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 17 Sep 2020 17:35:40 +0300 Subject: [PATCH 185/885] clean up return code --- chain/sync.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/chain/sync.go b/chain/sync.go index d09ba84de..9c9714447 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -1593,11 +1593,12 @@ func (syncer *Syncer) fetchMessages(ctx context.Context, headers []*types.TipSet } wg.Wait() - if batchErr == nil { - dt := build.Clock.Since(start) - log.Infof("fetching messages for %d tipsets at %d done; took %s", batchSize, startOffset, dt) + if batchErr != nil { + return nil, batchErr } + log.Infof("fetching messages for %d tipsets at %d done; took %s", batchSize, startOffset, build.Clock.Since(start)) + return batch, batchErr } From 6dfc40abc1c9152d3fb59061f2008b54b0b63872 Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 17 Sep 2020 18:23:50 +0300 Subject: [PATCH 186/885] error is nil at end, so return batch, nil --- chain/sync.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chain/sync.go b/chain/sync.go index 9c9714447..03ae1cd4f 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -1599,7 +1599,7 @@ func (syncer *Syncer) fetchMessages(ctx context.Context, headers []*types.TipSet log.Infof("fetching messages for %d tipsets at %d done; took %s", batchSize, startOffset, build.Clock.Since(start)) - return batch, batchErr + return batch, nil } func persistMessages(bs bstore.Blockstore, bst *exchange.CompactedMessages) error { From 6eda53565f62f804b72522594b82396cb7aa2038 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 17 Sep 2020 17:30:15 +0200 Subject: [PATCH 187/885] Most tests passing --- api/api_full.go | 18 ++++- api/api_test.go | 2 +- api/apistruct/struct.go | 13 ++-- api/test/window_post.go | 4 +- chain/actors/builtin/builtin.go | 2 +- chain/actors/builtin/miner/miner.go | 1 + chain/actors/builtin/miner/v0.go | 4 ++ chain/actors/builtin/reward/reward.go | 2 +- chain/events/state/predicates.go | 26 +++---- chain/messagepool/selection_test.go | 3 + chain/state/statetree_test.go | 23 ++++--- chain/stmgr/stmgr.go | 6 +- chain/stmgr/utils.go | 12 +++- cli/client.go | 2 +- cmd/lotus-chainwatch/processor/market.go | 4 +- cmd/lotus-pcr/main.go | 8 +-- cmd/lotus-storage-miner/proving.go | 23 ++----- conformance/driver.go | 2 +- .../storage-sealing/precommit_policy_test.go | 6 ++ extern/storage-sealing/sealing.go | 2 +- markets/storageadapter/provider.go | 6 +- node/impl/full/state.go | 68 +++++++++++++++---- storage/adapter_storage_miner.go | 4 +- storage/miner.go | 2 +- 24 files changed, 159 insertions(+), 84 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 778b458a8..6454966a4 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -326,7 +326,9 @@ type FullNode interface { // StateMinerInfo returns info about the indicated miner StateMinerInfo(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error) // StateMinerDeadlines returns all the proving deadlines for the given miner - StateMinerDeadlines(context.Context, address.Address, types.TipSetKey) ([]*miner.Deadline, error) + StateMinerDeadlines(context.Context, address.Address, types.TipSetKey) ([]Deadline, error) + // StateMinerPartitions returns all partitions in the specified deadline + StateMinerPartitions(ctx context.Context, m address.Address, dlIdx uint64, tsk types.TipSetKey) ([]Partition, error) // StateMinerFaults returns a bitfield indicating the faulty sectors of the given miner StateMinerFaults(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error) // StateAllMinerFaults returns all non-expired Faults that occur within lookback epochs of the given tipset @@ -383,7 +385,7 @@ type FullNode interface { // StateVerifiedClientStatus returns the data cap for the given address. // Returns nil if there is no entry in the data cap table for the // address. - StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (abi.StoragePower, error) + StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) // StateDealProviderCollateralBounds returns the min and max collateral a storage provider // can issue. It takes the deal size and verified status as parameters. StateDealProviderCollateralBounds(context.Context, abi.PaddedPieceSize, bool, types.TipSetKey) (DealCollateralBounds, error) @@ -816,6 +818,18 @@ const ( MsigCancel ) +type Deadline struct { + PostSubmissions bitfield.BitField +} + +type Partition struct { + AllSectors bitfield.BitField + FaultySectors bitfield.BitField + RecoveringSectors bitfield.BitField + LiveSectors bitfield.BitField + ActiveSectors bitfield.BitField +} + type Fault struct { Miner address.Address Epoch abi.ChainEpoch diff --git a/api/api_test.go b/api/api_test.go index 065141426..34c47f432 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -55,7 +55,7 @@ func TestReturnTypes(t *testing.T) { seen := map[reflect.Type]struct{}{} todo := []reflect.Type{m.Type.Out(0)} for len(todo) > 0 { - typ := todo[len(todo) - 1] + typ := todo[len(todo)-1] todo = todo[:len(todo)-1] if _, ok := seen[typ]; ok { diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index c8a2b91ac..397a0d60a 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -167,7 +167,8 @@ type FullNodeStruct struct { StateMinerProvingDeadline func(context.Context, address.Address, types.TipSetKey) (*dline.Info, error) `perm:"read"` StateMinerPower func(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error) `perm:"read"` StateMinerInfo func(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error) `perm:"read"` - StateMinerDeadlines func(context.Context, address.Address, types.TipSetKey) ([]*miner.Deadline, error) `perm:"read"` + StateMinerDeadlines func(context.Context, address.Address, types.TipSetKey) ([]api.Deadline, error) `perm:"read"` + StateMinerPartitions func(ctx context.Context, m address.Address, dlIdx uint64, tsk types.TipSetKey) ([]api.Partition, error) `perm:"read"` StateMinerFaults func(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error) `perm:"read"` StateAllMinerFaults func(context.Context, abi.ChainEpoch, types.TipSetKey) ([]*api.Fault, error) `perm:"read"` StateMinerRecoveries func(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error) `perm:"read"` @@ -197,7 +198,7 @@ type FullNodeStruct struct { StateMinerSectorCount func(context.Context, address.Address, types.TipSetKey) (api.MinerSectors, error) `perm:"read"` StateListMessages func(ctx context.Context, match *types.Message, tsk types.TipSetKey, toht abi.ChainEpoch) ([]cid.Cid, error) `perm:"read"` StateCompute func(context.Context, abi.ChainEpoch, []*types.Message, types.TipSetKey) (*api.ComputeStateOutput, error) `perm:"read"` - StateVerifiedClientStatus func(context.Context, address.Address, types.TipSetKey) (abi.StoragePower, error) `perm:"read"` + StateVerifiedClientStatus func(context.Context, address.Address, types.TipSetKey) (*abi.StoragePower, error) `perm:"read"` StateDealProviderCollateralBounds func(context.Context, abi.PaddedPieceSize, bool, types.TipSetKey) (api.DealCollateralBounds, error) `perm:"read"` StateCirculatingSupply func(context.Context, types.TipSetKey) (api.CirculatingSupply, error) `perm:"read"` StateNetworkVersion func(context.Context, types.TipSetKey) (stnetwork.Version, error) `perm:"read"` @@ -752,10 +753,14 @@ func (c *FullNodeStruct) StateMinerInfo(ctx context.Context, actor address.Addre return c.Internal.StateMinerInfo(ctx, actor, tsk) } -func (c *FullNodeStruct) StateMinerDeadlines(ctx context.Context, actor address.Address, tsk types.TipSetKey) ([]*miner.Deadline, error) { +func (c *FullNodeStruct) StateMinerDeadlines(ctx context.Context, actor address.Address, tsk types.TipSetKey) ([]api.Deadline, error) { return c.Internal.StateMinerDeadlines(ctx, actor, tsk) } +func (c *FullNodeStruct) StateMinerPartitions(ctx context.Context, m address.Address, dlIdx uint64, tsk types.TipSetKey) ([]api.Partition, error) { + return c.Internal.StateMinerPartitions(ctx, m, dlIdx, tsk) +} + func (c *FullNodeStruct) StateMinerFaults(ctx context.Context, actor address.Address, tsk types.TipSetKey) (bitfield.BitField, error) { return c.Internal.StateMinerFaults(ctx, actor, tsk) } @@ -868,7 +873,7 @@ func (c *FullNodeStruct) StateCompute(ctx context.Context, height abi.ChainEpoch return c.Internal.StateCompute(ctx, height, msgs, tsk) } -func (c *FullNodeStruct) StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (abi.StoragePower, error) { +func (c *FullNodeStruct) StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) { return c.Internal.StateVerifiedClientStatus(ctx, addr, tsk) } diff --git a/api/test/window_post.go b/api/test/window_post.go index bdc390730..e2bf5f36e 100644 --- a/api/test/window_post.go +++ b/api/test/window_post.go @@ -200,7 +200,7 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector require.NoError(t, err) require.Greater(t, len(parts), 0) - secs, err := parts[0].AllSectors() + secs := parts[0].AllSectors require.NoError(t, err) n, err := secs.Count() require.NoError(t, err) @@ -224,7 +224,7 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector require.NoError(t, err) require.Greater(t, len(parts), 0) - secs, err := parts[0].AllSectors() + secs := parts[0].AllSectors require.NoError(t, err) n, err := secs.Count() require.NoError(t, err) diff --git a/chain/actors/builtin/builtin.go b/chain/actors/builtin/builtin.go index 517f0d70c..bee8e59d6 100644 --- a/chain/actors/builtin/builtin.go +++ b/chain/actors/builtin/builtin.go @@ -17,7 +17,7 @@ const ( // Converts a network version into a specs-actors version. func VersionForNetwork(version network.Version) Version { switch version { - case network.Version0, network.Version1: + case network.Version0, network.Version1, network.Version2: return Version0 default: panic(fmt.Sprintf("unsupported network version %d", version)) diff --git a/chain/actors/builtin/miner/miner.go b/chain/actors/builtin/miner/miner.go index 725c5f2ff..d754d0b74 100644 --- a/chain/actors/builtin/miner/miner.go +++ b/chain/actors/builtin/miner/miner.go @@ -57,6 +57,7 @@ type State interface { type Deadline interface { LoadPartition(idx uint64) (Partition, error) ForEachPartition(cb func(idx uint64, part Partition) error) error + PostSubmissions() (bitfield.BitField, error) } type Partition interface { diff --git a/chain/actors/builtin/miner/v0.go b/chain/actors/builtin/miner/v0.go index cbe42b3da..b3fe594d1 100644 --- a/chain/actors/builtin/miner/v0.go +++ b/chain/actors/builtin/miner/v0.go @@ -266,6 +266,10 @@ func (d *v0Deadline) ForEachPartition(cb func(uint64, Partition) error) error { }) } +func (d *v0Deadline) PostSubmissions() (bitfield.BitField, error) { + return d.Deadline.PostSubmissions, nil +} + func (p *v0Partition) AllSectors() (bitfield.BitField, error) { return p.Partition.Sectors, nil } diff --git a/chain/actors/builtin/reward/reward.go b/chain/actors/builtin/reward/reward.go index 9093b33c6..52a26ab15 100644 --- a/chain/actors/builtin/reward/reward.go +++ b/chain/actors/builtin/reward/reward.go @@ -28,7 +28,7 @@ func Load(store adt.Store, act *types.Actor) (st State, err error) { } type State interface { - cbor.Marshaler + cbor.Er RewardSmoothed() (builtin.FilterEstimate, error) EffectiveBaselinePower() (abi.StoragePower, error) diff --git a/chain/events/state/predicates.go b/chain/events/state/predicates.go index 7a7609823..f27b5dc8f 100644 --- a/chain/events/state/predicates.go +++ b/chain/events/state/predicates.go @@ -12,11 +12,10 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/lotus/chain/actors/adt" + init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init" "github.com/filecoin-project/lotus/chain/actors/builtin/market" "github.com/filecoin-project/lotus/chain/actors/builtin/paych" "github.com/filecoin-project/specs-actors/actors/builtin" - init_ "github.com/filecoin-project/specs-actors/actors/builtin/init" - v0adt "github.com/filecoin-project/specs-actors/actors/util/adt" cbor "github.com/ipfs/go-ipld-cbor" typegen "github.com/whyrusleeping/cbor-gen" @@ -299,12 +298,12 @@ type DiffMinerActorStateFunc func(ctx context.Context, oldState miner.State, new func (sp *StatePredicates) OnInitActorChange(diffInitActorState DiffInitActorStateFunc) DiffTipSetKeyFunc { return sp.OnActorStateChanged(builtin.InitActorAddr, func(ctx context.Context, oldActorState, newActorState *types.Actor) (changed bool, user UserData, err error) { - var oldState init_.State - if err := sp.cst.Get(ctx, oldActorState.Head, &oldState); err != nil { + oldState, err := init_.Load(adt.WrapStore(ctx, sp.cst), oldActorState) + if err != nil { return false, nil, err } - var newState init_.State - if err := sp.cst.Get(ctx, newActorState.Head, &newState); err != nil { + newState, err := init_.Load(adt.WrapStore(ctx, sp.cst), newActorState) + if err != nil { return false, nil, err } return diffInitActorState(ctx, &oldState, &newState) @@ -314,12 +313,12 @@ func (sp *StatePredicates) OnInitActorChange(diffInitActorState DiffInitActorSta func (sp *StatePredicates) OnMinerActorChange(minerAddr address.Address, diffMinerActorState DiffMinerActorStateFunc) DiffTipSetKeyFunc { return sp.OnActorStateChanged(minerAddr, func(ctx context.Context, oldActorState, newActorState *types.Actor) (changed bool, user UserData, err error) { - var oldState miner.State - if err := sp.cst.Get(ctx, oldActorState.Head, &oldState); err != nil { + oldState, err := miner.Load(adt.WrapStore(ctx, sp.cst), oldActorState) + if err != nil { return false, nil, err } - var newState miner.State - if err := sp.cst.Get(ctx, newActorState.Head, &newState); err != nil { + newState, err := miner.Load(adt.WrapStore(ctx, sp.cst), newActorState) + if err != nil { return false, nil, err } return diffMinerActorState(ctx, oldState, newState) @@ -620,7 +619,7 @@ func (i *InitActorAddressChanges) Remove(key string, val *typegen.Deferred) erro func (sp *StatePredicates) OnAddressMapChange() DiffInitActorStateFunc { return func(ctx context.Context, oldState, newState *init_.State) (changed bool, user UserData, err error) { - ctxStore := &contextStore{ + /*ctxStore := &contextStore{ ctx: ctx, cst: sp.cst, } @@ -653,6 +652,9 @@ func (sp *StatePredicates) OnAddressMapChange() DiffInitActorStateFunc { return false, nil, nil } - return true, addressChanges, nil + return true, addressChanges, nil*/ + + panic("TODO") + return false, nil, nil } } diff --git a/chain/messagepool/selection_test.go b/chain/messagepool/selection_test.go index 5e372fc85..ea19dad9c 100644 --- a/chain/messagepool/selection_test.go +++ b/chain/messagepool/selection_test.go @@ -1216,6 +1216,9 @@ func makeZipfPremiumDistribution(rng *rand.Rand) func() uint64 { } func TestCompetitiveMessageSelectionExp(t *testing.T) { + if testing.Short() { + t.Skip("skipping in short mode") + } var capacityBoost, rewardBoost, tqReward float64 seeds := []int64{1947, 1976, 2020, 2100, 10000, 143324, 432432, 131, 32, 45} for _, seed := range seeds { diff --git a/chain/state/statetree_test.go b/chain/state/statetree_test.go index e45090d1a..79ab20606 100644 --- a/chain/state/statetree_test.go +++ b/chain/state/statetree_test.go @@ -5,17 +5,20 @@ import ( "fmt" "testing" - "github.com/filecoin-project/specs-actors/actors/builtin" - - address "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/types" "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" + + address "github.com/filecoin-project/go-address" + "github.com/filecoin-project/specs-actors/actors/builtin" + + "github.com/filecoin-project/lotus/build" + builtin2 "github.com/filecoin-project/lotus/chain/actors/builtin" + "github.com/filecoin-project/lotus/chain/types" ) func BenchmarkStateTreeSet(b *testing.B) { cst := cbor.NewMemCborStore() - st, err := NewStateTree(cst) + st, err := NewStateTree(cst, builtin2.VersionForNetwork(build.NewestNetworkVersion)) if err != nil { b.Fatal(err) } @@ -42,7 +45,7 @@ func BenchmarkStateTreeSet(b *testing.B) { func BenchmarkStateTreeSetFlush(b *testing.B) { cst := cbor.NewMemCborStore() - st, err := NewStateTree(cst) + st, err := NewStateTree(cst, builtin2.VersionForNetwork(build.NewestNetworkVersion)) if err != nil { b.Fatal(err) } @@ -72,7 +75,7 @@ func BenchmarkStateTreeSetFlush(b *testing.B) { func BenchmarkStateTree10kGetActor(b *testing.B) { cst := cbor.NewMemCborStore() - st, err := NewStateTree(cst) + st, err := NewStateTree(cst, builtin2.VersionForNetwork(build.NewestNetworkVersion)) if err != nil { b.Fatal(err) } @@ -114,7 +117,7 @@ func BenchmarkStateTree10kGetActor(b *testing.B) { func TestSetCache(t *testing.T) { cst := cbor.NewMemCborStore() - st, err := NewStateTree(cst) + st, err := NewStateTree(cst, builtin2.VersionForNetwork(build.NewestNetworkVersion)) if err != nil { t.Fatal(err) } @@ -151,7 +154,7 @@ func TestSetCache(t *testing.T) { func TestSnapshots(t *testing.T) { ctx := context.Background() cst := cbor.NewMemCborStore() - st, err := NewStateTree(cst) + st, err := NewStateTree(cst, builtin2.VersionForNetwork(build.NewestNetworkVersion)) if err != nil { t.Fatal(err) } @@ -234,7 +237,7 @@ func assertNotHas(t *testing.T, st *StateTree, addr address.Address) { func TestStateTreeConsistency(t *testing.T) { cst := cbor.NewMemCborStore() - st, err := NewStateTree(cst) + st, err := NewStateTree(cst, builtin2.VersionForNetwork(build.NewestNetworkVersion)) if err != nil { t.Fatal(err) } diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index 89ad6edd9..90154868e 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -1024,9 +1024,9 @@ func GetFilMined(ctx context.Context, st *state.StateTree) (abi.TokenAmount, err return big.Zero(), xerrors.Errorf("failed to load reward actor state: %w", err) } - var rst reward.State - if err := st.Store.Get(ctx, ractor.Head, &rst); err != nil { - return big.Zero(), xerrors.Errorf("failed to load reward state: %w", err) + rst, err := reward.Load(adt.WrapStore(ctx, st.Store), ractor) + if err != nil { + return big.Zero(), err } return rst.TotalStoragePowerReward() diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index 4fabefbff..8f39216c6 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -247,8 +247,18 @@ func GetSectorsForWinningPoSt(ctx context.Context, pv ffiwrapper.Verifier, sm *S out := make([]proof.SectorInfo, len(ids)) for i, n := range ids { + sb, err := provingSectors.Slice(n, 1) + if err != nil { + return nil, err + } + + sid, err := sb.First() + if err != nil { + return nil, err + } + var sinfo miner.SectorOnChainInfo - found, err := sectors.Get(n, &sinfo) + found, err := sectors.Get(sid, &sinfo) if err != nil { return nil, xerrors.Errorf("loading sector info: %w", err) diff --git a/cli/client.go b/cli/client.go index f6d529943..333938790 100644 --- a/cli/client.go +++ b/cli/client.go @@ -417,7 +417,7 @@ var clientDealCmd = &cli.Command{ return err } - isVerified := dcap != types.EmptyInt + isVerified := dcap != nil // If the user has explicitly set the --verified-deal flag if cctx.IsSet("verified-deal") { diff --git a/cmd/lotus-chainwatch/processor/market.go b/cmd/lotus-chainwatch/processor/market.go index a4bae4b20..17aa1c37b 100644 --- a/cmd/lotus-chainwatch/processor/market.go +++ b/cmd/lotus-chainwatch/processor/market.go @@ -300,8 +300,8 @@ func (p *Processor) updateMarketActorDealProposals(ctx context.Context, marketTi } for _, modified := range changes.Modified { - if modified.From.SlashEpoch() != modified.To.SlashEpoch() { - if _, err := stmt.Exec(modified.To.SlashEpoch(), modified.ID); err != nil { + if modified.From.SlashEpoch != modified.To.SlashEpoch { + if _, err := stmt.Exec(modified.To.SlashEpoch, modified.ID); err != nil { return err } } diff --git a/cmd/lotus-pcr/main.go b/cmd/lotus-pcr/main.go index d265bdd49..aa467e38a 100644 --- a/cmd/lotus-pcr/main.go +++ b/cmd/lotus-pcr/main.go @@ -417,24 +417,24 @@ func (r *refunder) ProcessTipset(ctx context.Context, tipset *types.TipSet, refu // We use the parent tipset key because precommit information is removed when ProveCommitSector is executed precommitChainInfo, err := r.api.StateSectorPreCommitInfo(ctx, m.To, sn, tipset.Parents()) if err != nil { - log.Warnw("failed to get precommit info for sector", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", proveCommitSector.SectorNumber) + log.Warnw("failed to get precommit info for sector", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", sn) continue } precommitTipset, err := r.api.ChainGetTipSetByHeight(ctx, precommitChainInfo.PreCommitEpoch, tipset.Key()) if err != nil { - log.Warnf("failed to lookup precommit epoch", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", proveCommitSector.SectorNumber) + log.Warnf("failed to lookup precommit epoch", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", sn) continue } collateral, err := r.api.StateMinerInitialPledgeCollateral(ctx, m.To, precommitChainInfo.Info, precommitTipset.Key()) if err != nil { - log.Warnw("failed to get initial pledge collateral", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", proveCommitSector.SectorNumber) + log.Warnw("failed to get initial pledge collateral", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", sn) } collateral = big.Sub(collateral, precommitChainInfo.PreCommitDeposit) if collateral.LessThan(big.Zero()) { - log.Debugw("skipping zero pledge collateral difference", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", proveCommitSector.SectorNumber) + log.Debugw("skipping zero pledge collateral difference", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", sn) continue } diff --git a/cmd/lotus-storage-miner/proving.go b/cmd/lotus-storage-miner/proving.go index d3e213752..ffc0946e3 100644 --- a/cmd/lotus-storage-miner/proving.go +++ b/cmd/lotus-storage-miner/proving.go @@ -1,7 +1,6 @@ package main import ( - "bytes" "fmt" "os" "text/tabwriter" @@ -73,12 +72,12 @@ var provingFaultsCmd = &cli.Command{ tw := tabwriter.NewWriter(os.Stdout, 2, 4, 2, ' ', 0) _, _ = fmt.Fprintln(tw, "deadline\tpartition\tsectors") err = mas.ForEachDeadline(func(dlIdx uint64, dl miner.Deadline) error { - dl.ForEachPartition(func(partIdx uint64, part miner.Partition) error { + return dl.ForEachPartition(func(partIdx uint64, part miner.Partition) error { faults, err := part.FaultySectors() if err != nil { return err } - faults.ForEach(func(num uint64) error { + return faults.ForEach(func(num uint64) error { _, _ = fmt.Fprintf(tw, "%d\t%d\t%d\n", dlIdx, partIdx, num) return nil }) @@ -173,6 +172,8 @@ var provingInfoCmd = &cli.Command{ } else { recovering += count } + + return nil }) }); err != nil { return xerrors.Errorf("walking miner deadlines and partitions: %w", err) @@ -250,22 +251,6 @@ var provingDeadlinesCmd = &cli.Command{ return xerrors.Errorf("getting deadlines: %w", err) } - var mas miner.State - { - mact, err := api.StateGetActor(ctx, maddr, types.EmptyTSK) - if err != nil { - return err - } - miner.Load - rmas, err := api.ChainReadObj(ctx, mact.Head) - if err != nil { - return err - } - if err := mas.UnmarshalCBOR(bytes.NewReader(rmas)); err != nil { - return err - } - } - fmt.Printf("Miner: %s\n", color.BlueString("%s", maddr)) tw := tabwriter.NewWriter(os.Stdout, 2, 4, 2, ' ', 0) diff --git a/conformance/driver.go b/conformance/driver.go index a33637837..66b6d0f8a 100644 --- a/conformance/driver.go +++ b/conformance/driver.go @@ -132,7 +132,7 @@ func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, preroot cid.Cid, epoch BaseFee: BaseFee, } - lvm, err := vm.NewVM(vmOpts) + lvm, err := vm.NewVM(context.TODO(), vmOpts) if err != nil { return nil, cid.Undef, err } diff --git a/extern/storage-sealing/precommit_policy_test.go b/extern/storage-sealing/precommit_policy_test.go index b9c3ec49b..30b538a88 100644 --- a/extern/storage-sealing/precommit_policy_test.go +++ b/extern/storage-sealing/precommit_policy_test.go @@ -2,6 +2,8 @@ package sealing_test import ( "context" + "github.com/filecoin-project/go-state-types/network" + "github.com/filecoin-project/lotus/build" "testing" "github.com/ipfs/go-cid" @@ -18,6 +20,10 @@ type fakeChain struct { h abi.ChainEpoch } +func (f *fakeChain) StateNetworkVersion(ctx context.Context, tok sealing.TipSetToken) (network.Version, error) { + return build.NewestNetworkVersion, nil +} + func (f *fakeChain) ChainHead(ctx context.Context) (sealing.TipSetToken, abi.ChainEpoch, error) { return []byte{1, 2, 3}, f.h, nil } diff --git a/extern/storage-sealing/sealing.go b/extern/storage-sealing/sealing.go index e9a98fec9..6d60e7a6e 100644 --- a/extern/storage-sealing/sealing.go +++ b/extern/storage-sealing/sealing.go @@ -24,8 +24,8 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" statemachine "github.com/filecoin-project/go-statemachine" - "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/actors/builtin/market" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" ) diff --git a/markets/storageadapter/provider.go b/markets/storageadapter/provider.go index 90bf28b41..04c1055df 100644 --- a/markets/storageadapter/provider.go +++ b/markets/storageadapter/provider.go @@ -108,7 +108,9 @@ func (n *ProviderNodeAdapter) OnDealComplete(ctx context.Context, deal storagema curTime := time.Now() for time.Since(curTime) < addPieceRetryTimeout { if !xerrors.Is(err, sealing.ErrTooManySectorsSealing) { - log.Errorf("failed to addPiece for deal %d, err: %w", deal.DealID, err) + if err != nil { + log.Errorf("failed to addPiece for deal %d, err: %w", deal.DealID, err) + } break } select { @@ -367,7 +369,7 @@ func (n *ProviderNodeAdapter) GetDataCap(ctx context.Context, addr address.Addre } sp, err := n.StateVerifiedClientStatus(ctx, addr, tsk) - return &sp, err + return sp, err } func (n *ProviderNodeAdapter) OnDealExpiredOrSlashed(ctx context.Context, dealID abi.DealID, onDealExpired storagemarket.DealExpiredCallback, onDealSlashed storagemarket.DealSlashedCallback) error { diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 48f9d503c..04055043a 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -112,7 +112,7 @@ func (a *StateAPI) StateMinerInfo(ctx context.Context, actor address.Address, ts return mas.Info() } -func (a *StateAPI) StateMinerDeadlines(ctx context.Context, m address.Address, tsk types.TipSetKey) ([]*miner.Deadline, error) { +func (a *StateAPI) StateMinerDeadlines(ctx context.Context, m address.Address, tsk types.TipSetKey) ([]api.Deadline, error) { act, err := a.StateManager.LoadActorTsk(ctx, m, tsk) if err != nil { return nil, xerrors.Errorf("failed to load miner actor: %w", err) @@ -128,9 +128,16 @@ func (a *StateAPI) StateMinerDeadlines(ctx context.Context, m address.Address, t return nil, xerrors.Errorf("getting deadline count: %w", err) } - out := make([]*miner.Deadline, deadlines) + out := make([]api.Deadline, deadlines) if err := mas.ForEachDeadline(func(i uint64, dl miner.Deadline) error { - out[i] = &dl + ps, err := dl.PostSubmissions() + if err != nil { + return err + } + + out[i] = api.Deadline{ + PostSubmissions: ps, + } return nil }); err != nil { return nil, err @@ -138,7 +145,7 @@ func (a *StateAPI) StateMinerDeadlines(ctx context.Context, m address.Address, t return out, nil } -func (a *StateAPI) StateMinerPartitions(ctx context.Context, m address.Address, dlIdx uint64, tsk types.TipSetKey) ([]*miner.Partition, error) { +func (a *StateAPI) StateMinerPartitions(ctx context.Context, m address.Address, dlIdx uint64, tsk types.TipSetKey) ([]api.Partition, error) { act, err := a.StateManager.LoadActorTsk(ctx, m, tsk) if err != nil { return nil, xerrors.Errorf("failed to load miner actor: %w", err) @@ -154,10 +161,40 @@ func (a *StateAPI) StateMinerPartitions(ctx context.Context, m address.Address, return nil, xerrors.Errorf("failed to load the deadline: %w", err) } - var out []*miner.Partition + var out []api.Partition err = dl.ForEachPartition(func(_ uint64, part miner.Partition) error { - p := part - out = append(out, &p) + allSectors, err := part.AllSectors() + if err != nil { + return xerrors.Errorf("getting AllSectors: %w", err) + } + + faultySectors, err := part.FaultySectors() + if err != nil { + return xerrors.Errorf("getting FaultySectors: %w", err) + } + + recoveringSectors, err := part.RecoveringSectors() + if err != nil { + return xerrors.Errorf("getting RecoveringSectors: %w", err) + } + + liveSectors, err := part.LiveSectors() + if err != nil { + return xerrors.Errorf("getting LiveSectors: %w", err) + } + + activeSectors, err := part.ActiveSectors() + if err != nil { + return xerrors.Errorf("getting ActiveSectors: %w", err) + } + + out = append(out, api.Partition{ + AllSectors: allSectors, + FaultySectors: faultySectors, + RecoveringSectors: recoveringSectors, + LiveSectors: liveSectors, + ActiveSectors: activeSectors, + }) return nil }) @@ -1037,29 +1074,32 @@ func (a *StateAPI) StateMinerAvailableBalance(ctx context.Context, maddr address // StateVerifiedClientStatus returns the data cap for the given address. // Returns zero if there is no entry in the data cap table for the // address. -func (a *StateAPI) StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (abi.StoragePower, error) { +func (a *StateAPI) StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) { act, err := a.StateGetActor(ctx, builtin.VerifiedRegistryActorAddr, tsk) if err != nil { - return types.EmptyInt, err + return nil, err } aid, err := a.StateLookupID(ctx, addr, tsk) if err != nil { log.Warnf("lookup failure %v", err) - return types.EmptyInt, err + return nil, err } vrs, err := verifreg.Load(a.StateManager.ChainStore().Store(ctx), act) if err != nil { - return types.EmptyInt, xerrors.Errorf("failed to load verified registry state: %w", err) + return nil, xerrors.Errorf("failed to load verified registry state: %w", err) } - _, dcap, err := vrs.VerifiedClientDataCap(aid) + verified, dcap, err := vrs.VerifiedClientDataCap(aid) if err != nil { - return types.EmptyInt, xerrors.Errorf("looking up verified client: %w", err) + return nil, xerrors.Errorf("looking up verified client: %w", err) + } + if !verified { + return nil, nil } - return dcap, nil + return &dcap, nil } var dealProviderCollateralNum = types.NewInt(110) diff --git a/storage/adapter_storage_miner.go b/storage/adapter_storage_miner.go index 8db977d2c..efbd95817 100644 --- a/storage/adapter_storage_miner.go +++ b/storage/adapter_storage_miner.go @@ -84,7 +84,7 @@ func (s SealingAPIAdapter) StateMinerWorkerAddress(ctx context.Context, maddr ad return mi.Worker, nil } -func (s SealingAPIAdapter) StateMinerDeadlines(ctx context.Context, maddr address.Address, tok sealing.TipSetToken) ([]*miner.Deadline, error) { +func (s SealingAPIAdapter) StateMinerDeadlines(ctx context.Context, maddr address.Address, tok sealing.TipSetToken) ([]api.Deadline, error) { tsk, err := types.TipSetKeyFromBytes(tok) if err != nil { return nil, xerrors.Errorf("failed to unmarshal TipSetToken to TipSetKey: %w", err) @@ -190,7 +190,7 @@ func (s SealingAPIAdapter) StateSectorPreCommitInfo(ctx context.Context, maddr a if err != nil { return nil, err } - if pci != nil { + if pci == nil { set, err := state.IsAllocated(sectorNumber) if err != nil { return nil, xerrors.Errorf("checking if sector is allocated: %w", err) diff --git a/storage/miner.go b/storage/miner.go index 227c51961..d7780898d 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -76,7 +76,7 @@ type storageMinerApi interface { StateSectorGetInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorOnChainInfo, error) StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*miner.SectorLocation, error) StateMinerInfo(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error) - StateMinerDeadlines(context.Context, address.Address, types.TipSetKey) ([]*miner.Deadline, error) + StateMinerDeadlines(context.Context, address.Address, types.TipSetKey) ([]api.Deadline, error) StateMinerProvingDeadline(context.Context, address.Address, types.TipSetKey) (*dline.Info, error) StateMinerPreCommitDepositForPower(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error) StateMinerInitialPledgeCollateral(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error) From 68e884ee441f2ea3b02103536c0947788630612c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 17 Sep 2020 17:31:09 +0200 Subject: [PATCH 188/885] docsgen --- api/docgen/docgen.go | 1 + documentation/en/api-methods.md | 35 ++++++++++++++++++++++++++++----- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/api/docgen/docgen.go b/api/docgen/docgen.go index d00643a02..ced536cc3 100644 --- a/api/docgen/docgen.go +++ b/api/docgen/docgen.go @@ -114,6 +114,7 @@ func init() { addExample(retrievalmarket.ClientEventDealAccepted) addExample(retrievalmarket.DealStatusNew) addExample(network.ReachabilityPublic) + addExample(build.NewestNetworkVersion) addExample(&types.ExecutionTrace{ Msg: exampleValue(reflect.TypeOf(&types.Message{}), nil).(*types.Message), MsgRct: exampleValue(reflect.TypeOf(&types.MessageReceipt{}), nil).(*types.MessageReceipt), diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index 27875eca1..364a0f1be 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -149,6 +149,7 @@ * [StateMinerSectorCount](#StateMinerSectorCount) * [StateMinerSectors](#StateMinerSectors) * [StateNetworkName](#StateNetworkName) + * [StateNetworkVersion](#StateNetworkVersion) * [StateReadState](#StateReadState) * [StateReplay](#StateReplay) * [StateSearchMsg](#StateSearchMsg) @@ -211,7 +212,7 @@ Response: ```json { "Version": "string value", - "APIVersion": 3584, + "APIVersion": 3840, "BlockDelay": 42 } ``` @@ -3509,7 +3510,7 @@ Inputs: Response: `"0"` ### StateMinerPartitions -StateMinerPartitions loads miner partitions for the specified miner/deadline +StateMinerPartitions returns all partitions in the specified deadline Perms: read @@ -3563,7 +3564,8 @@ Response: "TotalPower": { "RawBytePower": "0", "QualityAdjPower": "0" - } + }, + "HasMinPower": true } ``` @@ -3697,8 +3699,9 @@ Inputs: Response: ```json { - "Sectors": 42, - "Active": 42 + "Live": 42, + "Active": 42, + "Faulty": 42 } ``` @@ -3741,6 +3744,28 @@ Inputs: `null` Response: `"lotus"` +### StateNetworkVersion +StateNetworkVersion returns the network version at the given tipset + + +Perms: read + +Inputs: +```json +[ + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `2` + ### StateReadState StateReadState returns the indicated actor's state. From 82b95e34b790d300c273ce765fdd4f3704f610cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 17 Sep 2020 17:37:16 +0200 Subject: [PATCH 189/885] cbor gen --- chain/exchange/cbor_gen.go | 12 +-- chain/types/cbor_gen.go | 128 +++++++++++++++++++++++++ lotuspond/front/src/chain/methods.json | 3 +- 3 files changed, 136 insertions(+), 7 deletions(-) diff --git a/chain/exchange/cbor_gen.go b/chain/exchange/cbor_gen.go index dc91babe3..29b258081 100644 --- a/chain/exchange/cbor_gen.go +++ b/chain/exchange/cbor_gen.go @@ -146,7 +146,7 @@ func (t *Response) MarshalCBOR(w io.Writer) error { scratch := make([]byte, 9) - // t.Status (blocksync.status) (uint64) + // t.Status (exchange.status) (uint64) if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Status)); err != nil { return err @@ -164,7 +164,7 @@ func (t *Response) MarshalCBOR(w io.Writer) error { return err } - // t.Chain ([]*blocksync.BSTipSet) (slice) + // t.Chain ([]*exchange.BSTipSet) (slice) if len(t.Chain) > cbg.MaxLength { return xerrors.Errorf("Slice value in field t.Chain was too long") } @@ -198,7 +198,7 @@ func (t *Response) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.Status (blocksync.status) (uint64) + // t.Status (exchange.status) (uint64) { @@ -222,7 +222,7 @@ func (t *Response) UnmarshalCBOR(r io.Reader) error { t.ErrorMessage = string(sval) } - // t.Chain ([]*blocksync.BSTipSet) (slice) + // t.Chain ([]*exchange.BSTipSet) (slice) maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) if err != nil { @@ -567,7 +567,7 @@ func (t *BSTipSet) MarshalCBOR(w io.Writer) error { } } - // t.Messages (blocksync.CompactedMessages) (struct) + // t.Messages (exchange.CompactedMessages) (struct) if err := t.Messages.MarshalCBOR(w); err != nil { return err } @@ -621,7 +621,7 @@ func (t *BSTipSet) UnmarshalCBOR(r io.Reader) error { t.Blocks[i] = &v } - // t.Messages (blocksync.CompactedMessages) (struct) + // t.Messages (exchange.CompactedMessages) (struct) { diff --git a/chain/types/cbor_gen.go b/chain/types/cbor_gen.go index 676ae7054..f95df33bc 100644 --- a/chain/types/cbor_gen.go +++ b/chain/types/cbor_gen.go @@ -1634,3 +1634,131 @@ func (t *BeaconEntry) UnmarshalCBOR(r io.Reader) error { } return nil } + +var lengthBufStateRoot = []byte{131} + +func (t *StateRoot) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write(lengthBufStateRoot); err != nil { + return err + } + + scratch := make([]byte, 9) + + // t.Version (uint64) (uint64) + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Version)); err != nil { + return err + } + + // t.Actors (cid.Cid) (struct) + + if err := cbg.WriteCidBuf(scratch, w, t.Actors); err != nil { + return xerrors.Errorf("failed to write cid field t.Actors: %w", err) + } + + // t.Info (cid.Cid) (struct) + + if err := cbg.WriteCidBuf(scratch, w, t.Info); err != nil { + return xerrors.Errorf("failed to write cid field t.Info: %w", err) + } + + return nil +} + +func (t *StateRoot) UnmarshalCBOR(r io.Reader) error { + *t = StateRoot{} + + br := cbg.GetPeeker(r) + scratch := make([]byte, 8) + + maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 3 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.Version (uint64) (uint64) + + { + + maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Version = uint64(extra) + + } + // t.Actors (cid.Cid) (struct) + + { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Actors: %w", err) + } + + t.Actors = c + + } + // t.Info (cid.Cid) (struct) + + { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Info: %w", err) + } + + t.Info = c + + } + return nil +} + +var lengthBufStateInfo = []byte{128} + +func (t *StateInfo) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write(lengthBufStateInfo); err != nil { + return err + } + + return nil +} + +func (t *StateInfo) UnmarshalCBOR(r io.Reader) error { + *t = StateInfo{} + + br := cbg.GetPeeker(r) + scratch := make([]byte, 8) + + maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 0 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + return nil +} diff --git a/lotuspond/front/src/chain/methods.json b/lotuspond/front/src/chain/methods.json index ad1076c84..ce4919cc4 100644 --- a/lotuspond/front/src/chain/methods.json +++ b/lotuspond/front/src/chain/methods.json @@ -23,7 +23,8 @@ "AddSigner", "RemoveSigner", "SwapSigner", - "ChangeNumApprovalsThreshold" + "ChangeNumApprovalsThreshold", + "LockBalance" ], "fil/1/paymentchannel": [ "Send", From c40c1361f0307f888d3812d78aff70883aa550c4 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Thu, 17 Sep 2020 18:14:07 +0200 Subject: [PATCH 190/885] fix: paych To() --- chain/actors/builtin/paych/v0.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chain/actors/builtin/paych/v0.go b/chain/actors/builtin/paych/v0.go index 16a65bc9b..7d63b8913 100644 --- a/chain/actors/builtin/paych/v0.go +++ b/chain/actors/builtin/paych/v0.go @@ -22,7 +22,7 @@ func (s *v0State) From() address.Address { // Recipient of payouts from channel func (s *v0State) To() address.Address { - return s.State.From + return s.State.To } // Height at which the channel can be `Collected` From 14a4acec8cefd4e64ba7919b7b0539f49c0e9ef7 Mon Sep 17 00:00:00 2001 From: jennijuju Date: Wed, 16 Sep 2020 17:43:55 -0400 Subject: [PATCH 191/885] Add an option to hide sectors in Removed for `sectors list`. --- cmd/lotus-storage-miner/sectors.go | 33 +++++++++++++++++++----------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/cmd/lotus-storage-miner/sectors.go b/cmd/lotus-storage-miner/sectors.go index 06f09fe20..0e6fe8435 100644 --- a/cmd/lotus-storage-miner/sectors.go +++ b/cmd/lotus-storage-miner/sectors.go @@ -2,6 +2,7 @@ package main import ( "fmt" + sealing "github.com/filecoin-project/lotus/extern/storage-sealing" "os" "sort" "strconv" @@ -136,6 +137,12 @@ var sectorsStatusCmd = &cli.Command{ var sectorsListCmd = &cli.Command{ Name: "list", Usage: "List sectors", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "hide-removed", + Usage: "to hide Removed sectors", + }, + }, Action: func(cctx *cli.Context) error { nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx) if err != nil { @@ -192,19 +199,21 @@ var sectorsListCmd = &cli.Command{ continue } - _, inSSet := commitedIDs[s] - _, inASet := activeIDs[s] + if !cctx.Bool("hide-removed") || st.State != api.SectorState(sealing.Removed) { + _, inSSet := commitedIDs[s] + _, inASet := activeIDs[s] - fmt.Fprintf(w, "%d: %s\tsSet: %s\tactive: %s\ttktH: %d\tseedH: %d\tdeals: %v\t toUpgrade:%t\n", - s, - st.State, - yesno(inSSet), - yesno(inASet), - st.Ticket.Epoch, - st.Seed.Epoch, - st.Deals, - st.ToUpgrade, - ) + fmt.Fprintf(w, "%d: %s\tsSet: %s\tactive: %s\ttktH: %d\tseedH: %d\tdeals: %v\t toUpgrade:%t\n", + s, + st.State, + yesno(inSSet), + yesno(inASet), + st.Ticket.Epoch, + st.Seed.Epoch, + st.Deals, + st.ToUpgrade, + ) + } } return w.Flush() From a4fd356fcbc20f38657c128ceb291c2db42f150a Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 17 Sep 2020 20:35:19 +0200 Subject: [PATCH 192/885] Delete most docs. Update about page with links to docs.filecoin.io --- documentation/en/.glossary.json | 147 +- documentation/en/.library.json | 202 +- documentation/en/about.md | 19 +- documentation/en/building/api-methods.md | 4567 ----------------- .../en/building/api-troubleshooting.md | 36 - documentation/en/building/api.md | 38 - documentation/en/building/building.md | 5 - documentation/en/building/jaeger-tracing.md | 26 - documentation/en/building/local-devnet.md | 54 - documentation/en/building/payment-channels.md | 111 - documentation/en/building/remote-api.md | 69 - .../en/getting-started/getting-started.md | 3 - .../getting-started/setup-troubleshooting.md | 57 - documentation/en/getting-started/setup.md | 169 - documentation/en/getting-started/wallet.md | 58 - .../en/installation/install-linux.md | 129 - .../en/installation/install-macos.md | 62 - documentation/en/installation/installation.md | 39 - documentation/en/installation/update.md | 72 - documentation/en/mining/gpus.md | 17 - documentation/en/mining/lotus-seal-worker.md | 99 - documentation/en/mining/managing-deals.md | 19 - documentation/en/mining/miner-setup.md | 241 - .../en/mining/mining-troubleshooting.md | 59 - documentation/en/mining/mining.md | 8 - documentation/en/store/adding-from-ipfs.md | 20 - documentation/en/store/making-deals.md | 71 - documentation/en/store/retrieve.md | 27 - .../en/store/storage-troubleshooting.md | 30 - documentation/en/store/store.md | 11 - 30 files changed, 14 insertions(+), 6451 deletions(-) delete mode 100644 documentation/en/building/api-methods.md delete mode 100644 documentation/en/building/api-troubleshooting.md delete mode 100644 documentation/en/building/api.md delete mode 100644 documentation/en/building/building.md delete mode 100644 documentation/en/building/jaeger-tracing.md delete mode 100644 documentation/en/building/local-devnet.md delete mode 100644 documentation/en/building/payment-channels.md delete mode 100644 documentation/en/building/remote-api.md delete mode 100644 documentation/en/getting-started/getting-started.md delete mode 100644 documentation/en/getting-started/setup-troubleshooting.md delete mode 100644 documentation/en/getting-started/setup.md delete mode 100644 documentation/en/getting-started/wallet.md delete mode 100644 documentation/en/installation/install-linux.md delete mode 100644 documentation/en/installation/install-macos.md delete mode 100644 documentation/en/installation/installation.md delete mode 100644 documentation/en/installation/update.md delete mode 100644 documentation/en/mining/gpus.md delete mode 100644 documentation/en/mining/lotus-seal-worker.md delete mode 100644 documentation/en/mining/managing-deals.md delete mode 100644 documentation/en/mining/miner-setup.md delete mode 100644 documentation/en/mining/mining-troubleshooting.md delete mode 100644 documentation/en/mining/mining.md delete mode 100644 documentation/en/store/adding-from-ipfs.md delete mode 100644 documentation/en/store/making-deals.md delete mode 100644 documentation/en/store/retrieve.md delete mode 100644 documentation/en/store/storage-troubleshooting.md delete mode 100644 documentation/en/store/store.md diff --git a/documentation/en/.glossary.json b/documentation/en/.glossary.json index e8a9e0846..0967ef424 100644 --- a/documentation/en/.glossary.json +++ b/documentation/en/.glossary.json @@ -1,146 +1 @@ -{ - "bellman": { - "title": "Bellman", - "value": "Bellman is a rust crate for building zk-SNARK circuits. It provides circuit traits and primitive structures, as well as basic gadget implementations such as booleans and number abstractions." - }, - "nvme": { - "title": "NVMe", - "value": "(non-volatile memory express) is a host controller interface and storage protocol created to accelerate the transfer of data between enterprise and client systems and solid-state drives (SSDs) over a computer's high-speed Peripheral Component Interconnect Express (PCIe) bus." - }, - "multiaddr": { - "title": "Multiaddr", - "value": "Multiaddr is a format for encoding addresses from various well-established network protocols. It is useful to write applications that future-proof their use of addresses, and allow multiple transport protocols and addresses to coexist." - }, - "attofil": { - "title": "attoFIL", - "value": "AttoFIL is a word used to describe 10^-18 FIL. The word atto comes from the Norwegian and Danish term: atten eighteen." - }, - "fil": { - "title": "FIL", - "value": "A ticker symbol is an abbreviation used to uniquely identify Filecoin when it is used in a wallet exchange or a cryptocurrency exchange." - }, - "epost": { - "title": "Election Proof-of-Spacetime", - "value": "Election Proof-of-Spacetime couples the Proof-of-Spacetime process with block production, meaning that in order to produce a block, the miner must produce a valid Proof-of-Spacetime proof (snark output)." - }, - "jwt": { - "title": "JWT", - "value": "JSON Web Tokens are an open, industry standard RFC 7519 method for representing claims securely between two parties." - }, - "json-rpc": { - "title": "JSON-RPC", - "value": "JSON-RPC is a remote procedure call protocol encoded in JSON. It is a very simple protocol (and very similar to XML-RPC), defining only a few data types and commands." - }, - "bls-address": { - "title": "BLS Signature (Address)", - "value": "A Boneh–Lynn–Shacham (BLS) signature is a digital signature scheme that allows a user to determine the authenticity of a signer, and is a commonly used signature scheme in the Filecoin Distributed Storage Network." - }, - "faucet": { - "title": "Filecoin Test Faucet", - "value": "A webpage where you can get free test Filecoin to participate in the Testnet." - }, - "chain": { - "title": "Chain", - "value": "The Filecoin Blockchain is a distributed virtual machine that achieves consensus, processes messages, accounts for storage, and maintains security in the Filecoin Protocol. It is the main interface linking various actors in the Filecoin system." - }, - "miner-power": { - "title": "Miner Power", - "value": "Miner storage in relation to network storage, tracked in the power table." - }, - "sector": { - "title": "Sector", - "value": "A fixed-size block of data of SECTOR_SIZE bytes which generally contains client's data." - }, - "sealing": { - "title": "Sealing", - "value": "A slow encoding process that returns commitments and proofs for data being stored in a sector." - }, - "seal": { - "title": "Seal", - "value": "A slow encoding process that returns commitments and proofs for data being stored in a sector." - }, - "posts": { - "title": "Proof-of-Spacetime(s)", - "value": "Filecoin is a protocol token whose blockchain runs on a novel proof, called Proof-of-Spacetime, where blocks are created by miners that are storing data." - }, - "filecoin-testnet": { - "title": "Filecoin Testnet", - "value": "Until we launch, we are making lots of changes to Lotus. The Testnet is expected to bring a few significant fixes/improvements. During Testnet, you can retrieve test filecoin from our network faucet to use as collateral to start mining. Test filecoin do not have any value – the official filecoin tokens will not be released until Mainnet launch." - }, - "filecoin-decentralized-storage-market": { - "title": "Filecoin Decentralized Storage Market", - "value": "Storage Market subsystem is the data entry point into the network. Miners only earn power from data stored in a storage deal and all deals live on the Filecoin network." - }, - "filecoin-proof-parameters": { - "title": "Filecoin Proof Parameters", - "value": "The proving algorithms rely on a large binary parameter file." - }, - "lotus-devnet": { - "title": "DevNet", - "value": "On the DevNets, you can store data as a storage client and also try how Filecoin mining works. The devnets are an important development tool for those who anticipate building applications on top of the Filecoin protocol or storing data on the decentralized storage market. " - }, - "filecoin-distributed-storage-network": { - "title": "Filecoin Distributed Storage Network", - "value": "Filecoin is a distributed storage network based on a blockchain mechanism. Filecoin miners can elect to provide storage capacity for the network, and thereby earn units of the Filecoin cryptocurrency (FIL) by periodically producing cryptographic proofs that certify that they are providing the capacity specified." - }, - "lotus-node": { - "title": "Lotus Node", - "value": "The Lotus Node is full of capabilities. It runs the Blockchain system, makes retrieval deals, does data transfer, supports block producer logic, and syncs and validates the chain." - }, - "block-rewards": { - "title": "Block Reward", - "value": "Over the entire lifetime of the protocol, 1,400,000,000 FIL (TotalIssuance) will be given out to miners. The rate at which the funds are given out is set to halve every six years, smoothly (not a fixed jump like in Bitcoin)." - }, - "block-producer-miner": { - "title": "Miner (Block Producer)", - "value": "The Block Producer Miner's logic. It currently shares an interface and process with the Lotus Node. A Block Producer chooses which messages to include in a block and is rewarded according to each message’s gas price and consumption, forming a market." - }, - "lotus-miner": { - "title": "Miner (lotus-miner)", - "value": "The Miner's logic. It has its own dedicated process. Contributes to the network through Sector commitments and Proofs of Spacetime to prove that it is storing the sectors it has commited to." - }, - "swarm-port": { - "title": "Swarm Port (Libp2p)", - "value": "The LibP2P Swarm manages groups of connections to peers, handles incoming and outgoing streams, and is part of the miners implementation. The port value is part of the Host interface." - }, - "daemon": { - "title": "Lotus Daemon", - "value": "A Daemon is a program that runs as a background process. A Daemon in the context of the Filecoin Distributed Storage Network may enable applications to communicate with peers, handle protocols, participate in pubsub, and interact with a distributed hash table (DHT)." - }, - "storage-deal": { - "title": "Storage deal", - "value": "One of the two types of deals in Filecoin markets. Storage deals are recorded on the blockchain and enforced by the protocol." - }, - "retrieval-deal": { - "title": "Retrieval deal", - "value": "One of the two types of deals in Filecoin markets. Retrieval deals are off chain and enabled by micropayment channel by transacting parties." - }, - "deal-cid": { - "title": "Deal CID", - "value": "CID is a format for referencing content in distributed information systems, it is a way to store information so it can be retrieved based on its content, not its location. DealCID specifically is used in storage deals." - }, - "data-cid": { - "title": "Data CID", - "value": "CID is a format for referencing content in distributed information systems, it is a way to store information so it can be retrieved based on its content, not its location. DataCID specifically is used to represent the file that is stored in the Filecoin Distributed Storage Network." - }, - "cid": { - "title": "CID", - "value": "A CID is a self-describing content-addressed identifier. It uses cryptographic hashes to achieve content addressing. It uses several multiformats to achieve flexible self-description, namely multihash for hashes, multicodec for data content types, and multibase to encode the CID itself into strings." - }, - "total-network-power": { - "title": "Total Network Power", - "value": "A reference to all the Power Tables for every subchain, accounting for each Lotus Miner on chain." - }, - "chain-block-height": { - "title": "Chain Block Height", - "value": "Chain block height is defined as the number of blocks in the chain between any given block and the very first block in the blockchain." - }, - "block-height": { - "title": "Block Height", - "value": "Height of the Merkle Tree of a sector. A sector is a contiguous array of bytes that a miner puts together, seals, and performs Proofs of Spacetime on." - }, - "blocktime": { - "title": "Blocktime", - "value": "The time it takes for a Block to propagate to the whole network." - } -} +{} diff --git a/documentation/en/.library.json b/documentation/en/.library.json index 59cc01e29..e31f09950 100644 --- a/documentation/en/.library.json +++ b/documentation/en/.library.json @@ -2,194 +2,11 @@ "posts": [ { "title": "About Lotus", - "slug": "en+lotus", + "slug": "", "github": "en/about.md", "value": null, "posts": [] }, - { - "title": "Installation", - "slug": "en+install", - "github": "en/installation/installation.md", - "value": null, - "posts": [ - { - "title": "Linux installation", - "slug": "en+install-linux", - "github": "en/installation/install-linux.md", - "value": null - }, - { - "title": "MacOS installation", - "slug": "en+install-macos", - "github": "en/installation/install-macos.md", - "value": null - }, - { - "title": "Updating Lotus", - "slug": "en+update", - "github": "en/installation/update.md", - "value": null - } - ] - }, - { - "title": "Getting started", - "slug": "en+getting-started", - "github": "en/getting-started/getting-started.md", - "value": null, - "posts": [ - { - "title": "Setting up Lotus", - "slug": "en+setup", - "github": "en/getting-started/setup.md", - "value": null - }, - { - - "title": "Obtaining and sending FIL", - "slug": "en+wallet", - "github": "en/getting-started/wallet.md", - "value": null - }, - { - "title": "Setup troubleshooting", - "slug": "en+setup-troubleshooting", - "github": "en/getting-started/setup-troubleshooting.md", - "value": null - } - ] - }, - { - "title": "Storing and retrieving data", - "slug": "en+store", - "github": "en/store/store.md", - "value": null, - "posts": [ - { - "title": "Making storage deals", - "slug": "en+making-deals", - "github": "en/store/making-deals.md", - "value": null - }, - { - "title": "Adding data from IPFS", - "slug": "en+adding-from-ipfs", - "github": "en/store/adding-from-ipfs.md", - "value": null - }, - { - "title": "Retrieving data", - "slug": "en+retriving", - "github": "en/store/retrieve.md", - "value": null - }, - { - "title": "Storage Troubleshooting", - "slug": "en+storage-troubleshooting", - "github": "en/store/storage-troubleshooting.md", - "value": null - } - ] - }, - { - "title": "Storage mining", - "slug": "en+mining", - "github": "en/mining/mining.md", - "value": null, - "posts": [ - { - "title": "Miner setup", - "slug": "en+miner-setup", - "github": "en/mining/miner-setup.md", - "value": null - }, - { - "title": "Managing deals", - "slug": "en+managing-deals", - "github": "en/mining/managing-deals.md", - "value": null - }, - { - "title": "Lotus Worker", - "slug": "en+lotus-worker", - "github": "en/mining/lotus-seal-worker.md", - "value": null - }, - { - "title": "Benchmarking GPUs", - "slug": "en+gpus", - "github": "en/mining/gpus.md", - "value": null - }, - { - "title": "Mining Troubleshooting", - "slug": "en+mining-troubleshooting", - "github": "en/mining/mining-troubleshooting.md", - "value": null - } - ] - }, - { - "title": "Building", - "slug": "en+building", - "github": "en/building/building.md", - "value": null, - "posts": [ - { - "title": "Setting up remote API access", - "slug": "en+remote-api", - "github": "en/building/remote-api.md", - "value": null, - "posts": [] - }, - { - "title": "API endpoints and methods", - "slug": "en+api", - "github": "en/building/api.md", - "value": null, - "posts": [] - }, - { - "title": "API Reference", - "slug": "en+api-methods", - "github": "en/building/api-methods.md", - "value": null, - "posts": [] - }, - - { - "title": "Payment Channels", - "slug": "en+payment-channels", - "github": "en/building/payment-channels.md", - "value": null, - "posts": [] - }, - - { - "title": "Running a local devnet", - "slug": "en+local-devnet", - "github": "en/building/local-devnet.md", - "value": null, - "posts": [] - }, - { - "title": "Jaeger Tracing", - "slug": "en+jaeger-tracing", - "github": "en/building/jaeger-tracing.md", - "value": null, - "posts": [] - }, - - { - "title": "API Troubleshooting", - "slug": "en+api-troubleshooting", - "github": "en/building/api-troubleshooting.md", - "value": null, - "posts": [] - } - ] - }, { "title": "Lotus Architecture (WIP)", "slug": "en+arch", @@ -203,23 +20,6 @@ "value": null } ] - }, - { - "title": "FAQs", - "slug": "en+faqs", - "github": "en/faqs.md", - "value": null, - "posts": [] - }, - { - "title": "Glossary", - "slug": "en+glossary", - "github": "en/.glossary.json", - "value": null, - "custom": { - "glossary": true - }, - "posts": [] } ] } diff --git a/documentation/en/about.md b/documentation/en/about.md index ee8536ac9..f2051e00b 100644 --- a/documentation/en/about.md +++ b/documentation/en/about.md @@ -2,13 +2,18 @@ Lotus is an implementation of the **Filecoin Distributed Storage Network**. -The **Lotus Node** (and the mining applications) can be built to join any of the [Filecoin networks](https://docs.filecoin.io/how-to/networks/). +It is written in Go and provides a suite of command-line applications: -For more details about Filecoin, check out the [Filecoin Docs](https://docs.filecoin.io) and [Filecoin Spec](https://filecoin-project.github.io/specs/). +- Lotus Node (`lotus`): a Filecoin Node: validates network transactions, manages a FIL wallet, can perform storage and retrieval deals. +- Lotus Miner (`lotus-miner`): a Filecoin miner. See the the respective Lotus Miner section in the Mine documentation. +- Lotus Worker (`lotus-worker`): a worker that assists miners to perform mining-related tasks. See its respective guide for more information. -## What can I learn here? +The [Lotus user documentation](https://docs.filecoin.io/get-started/lotus) is part of the [Filecoin documentation site](https://docs.filecoin.io): + +* To install and get started with Lotus, visit the [Get Started section](https://docs.filecoin.io/get-started/lotus). +* Information about how to perform deals on the Filecoin network using Lotus can be found in the [Store section](https://docs.filecoin.io/store/lotus). +* Miners looking to provide storage to the Network can find the latest guides in the [Mine section](https://docs.filecoin.io/mine/lotus). +* Developers and integrators that wish to use the Lotus APIs can start in the [Build section](https://docs.filecoin.io/mine/lotus). + +For more details about Filecoin, check out the [Filecoin Docs](https://docs.filecoin.io) and [Filecoin Spec](https://spec.filecoin.io/). -* How to [install](en+installation) and [setup](en+setup) the Lotus software -* How to [store data on the Filecoin network](en+store) -* How to [setup a high performance FIL miner](en+miner-setup) -* How to [configure and access Lotus APIs](en+remote-api) diff --git a/documentation/en/building/api-methods.md b/documentation/en/building/api-methods.md deleted file mode 100644 index 2f3164bb7..000000000 --- a/documentation/en/building/api-methods.md +++ /dev/null @@ -1,4567 +0,0 @@ -# Groups -* [](#) - * [Closing](#Closing) - * [Shutdown](#Shutdown) - * [Version](#Version) -* [Auth](#Auth) - * [AuthNew](#AuthNew) - * [AuthVerify](#AuthVerify) -* [Beacon](#Beacon) - * [BeaconGetEntry](#BeaconGetEntry) -* [Chain](#Chain) - * [ChainExport](#ChainExport) - * [ChainGetBlock](#ChainGetBlock) - * [ChainGetBlockMessages](#ChainGetBlockMessages) - * [ChainGetGenesis](#ChainGetGenesis) - * [ChainGetMessage](#ChainGetMessage) - * [ChainGetNode](#ChainGetNode) - * [ChainGetParentMessages](#ChainGetParentMessages) - * [ChainGetParentReceipts](#ChainGetParentReceipts) - * [ChainGetPath](#ChainGetPath) - * [ChainGetRandomnessFromBeacon](#ChainGetRandomnessFromBeacon) - * [ChainGetRandomnessFromTickets](#ChainGetRandomnessFromTickets) - * [ChainGetTipSet](#ChainGetTipSet) - * [ChainGetTipSetByHeight](#ChainGetTipSetByHeight) - * [ChainHasObj](#ChainHasObj) - * [ChainHead](#ChainHead) - * [ChainNotify](#ChainNotify) - * [ChainReadObj](#ChainReadObj) - * [ChainSetHead](#ChainSetHead) - * [ChainStatObj](#ChainStatObj) - * [ChainTipSetWeight](#ChainTipSetWeight) -* [Client](#Client) - * [ClientCalcCommP](#ClientCalcCommP) - * [ClientDataTransferUpdates](#ClientDataTransferUpdates) - * [ClientDealSize](#ClientDealSize) - * [ClientFindData](#ClientFindData) - * [ClientGenCar](#ClientGenCar) - * [ClientGetDealInfo](#ClientGetDealInfo) - * [ClientGetDealUpdates](#ClientGetDealUpdates) - * [ClientHasLocal](#ClientHasLocal) - * [ClientImport](#ClientImport) - * [ClientListDataTransfers](#ClientListDataTransfers) - * [ClientListDeals](#ClientListDeals) - * [ClientListImports](#ClientListImports) - * [ClientMinerQueryOffer](#ClientMinerQueryOffer) - * [ClientQueryAsk](#ClientQueryAsk) - * [ClientRemoveImport](#ClientRemoveImport) - * [ClientRetrieve](#ClientRetrieve) - * [ClientRetrieveTryRestartInsufficientFunds](#ClientRetrieveTryRestartInsufficientFunds) - * [ClientRetrieveWithEvents](#ClientRetrieveWithEvents) - * [ClientStartDeal](#ClientStartDeal) -* [Gas](#Gas) - * [GasEstimateFeeCap](#GasEstimateFeeCap) - * [GasEstimateGasLimit](#GasEstimateGasLimit) - * [GasEstimateGasPremium](#GasEstimateGasPremium) - * [GasEstimateMessageGas](#GasEstimateMessageGas) -* [I](#I) - * [ID](#ID) -* [Log](#Log) - * [LogList](#LogList) - * [LogSetLevel](#LogSetLevel) -* [Market](#Market) - * [MarketEnsureAvailable](#MarketEnsureAvailable) -* [Miner](#Miner) - * [MinerCreateBlock](#MinerCreateBlock) - * [MinerGetBaseInfo](#MinerGetBaseInfo) -* [Mpool](#Mpool) - * [MpoolClear](#MpoolClear) - * [MpoolGetConfig](#MpoolGetConfig) - * [MpoolGetNonce](#MpoolGetNonce) - * [MpoolPending](#MpoolPending) - * [MpoolPush](#MpoolPush) - * [MpoolPushMessage](#MpoolPushMessage) - * [MpoolSelect](#MpoolSelect) - * [MpoolSetConfig](#MpoolSetConfig) - * [MpoolSub](#MpoolSub) -* [Msig](#Msig) - * [MsigAddApprove](#MsigAddApprove) - * [MsigAddCancel](#MsigAddCancel) - * [MsigAddPropose](#MsigAddPropose) - * [MsigApprove](#MsigApprove) - * [MsigCancel](#MsigCancel) - * [MsigCreate](#MsigCreate) - * [MsigGetAvailableBalance](#MsigGetAvailableBalance) - * [MsigGetVested](#MsigGetVested) - * [MsigPropose](#MsigPropose) - * [MsigSwapApprove](#MsigSwapApprove) - * [MsigSwapCancel](#MsigSwapCancel) - * [MsigSwapPropose](#MsigSwapPropose) -* [Net](#Net) - * [NetAddrsListen](#NetAddrsListen) - * [NetAgentVersion](#NetAgentVersion) - * [NetAutoNatStatus](#NetAutoNatStatus) - * [NetBandwidthStats](#NetBandwidthStats) - * [NetBandwidthStatsByPeer](#NetBandwidthStatsByPeer) - * [NetBandwidthStatsByProtocol](#NetBandwidthStatsByProtocol) - * [NetConnect](#NetConnect) - * [NetConnectedness](#NetConnectedness) - * [NetDisconnect](#NetDisconnect) - * [NetFindPeer](#NetFindPeer) - * [NetPeers](#NetPeers) - * [NetPubsubScores](#NetPubsubScores) -* [Paych](#Paych) - * [PaychAllocateLane](#PaychAllocateLane) - * [PaychAvailableFunds](#PaychAvailableFunds) - * [PaychAvailableFundsByFromTo](#PaychAvailableFundsByFromTo) - * [PaychCollect](#PaychCollect) - * [PaychGet](#PaychGet) - * [PaychGetWaitReady](#PaychGetWaitReady) - * [PaychList](#PaychList) - * [PaychNewPayment](#PaychNewPayment) - * [PaychSettle](#PaychSettle) - * [PaychStatus](#PaychStatus) - * [PaychVoucherAdd](#PaychVoucherAdd) - * [PaychVoucherCheckSpendable](#PaychVoucherCheckSpendable) - * [PaychVoucherCheckValid](#PaychVoucherCheckValid) - * [PaychVoucherCreate](#PaychVoucherCreate) - * [PaychVoucherList](#PaychVoucherList) - * [PaychVoucherSubmit](#PaychVoucherSubmit) -* [State](#State) - * [StateAccountKey](#StateAccountKey) - * [StateAllMinerFaults](#StateAllMinerFaults) - * [StateCall](#StateCall) - * [StateChangedActors](#StateChangedActors) - * [StateCirculatingSupply](#StateCirculatingSupply) - * [StateCompute](#StateCompute) - * [StateDealProviderCollateralBounds](#StateDealProviderCollateralBounds) - * [StateGetActor](#StateGetActor) - * [StateGetReceipt](#StateGetReceipt) - * [StateListActors](#StateListActors) - * [StateListMessages](#StateListMessages) - * [StateListMiners](#StateListMiners) - * [StateLookupID](#StateLookupID) - * [StateMarketBalance](#StateMarketBalance) - * [StateMarketDeals](#StateMarketDeals) - * [StateMarketParticipants](#StateMarketParticipants) - * [StateMarketStorageDeal](#StateMarketStorageDeal) - * [StateMinerActiveSectors](#StateMinerActiveSectors) - * [StateMinerAvailableBalance](#StateMinerAvailableBalance) - * [StateMinerDeadlines](#StateMinerDeadlines) - * [StateMinerFaults](#StateMinerFaults) - * [StateMinerInfo](#StateMinerInfo) - * [StateMinerInitialPledgeCollateral](#StateMinerInitialPledgeCollateral) - * [StateMinerPartitions](#StateMinerPartitions) - * [StateMinerPower](#StateMinerPower) - * [StateMinerPreCommitDepositForPower](#StateMinerPreCommitDepositForPower) - * [StateMinerProvingDeadline](#StateMinerProvingDeadline) - * [StateMinerRecoveries](#StateMinerRecoveries) - * [StateMinerSectorCount](#StateMinerSectorCount) - * [StateMinerSectors](#StateMinerSectors) - * [StateMsgGasCost](#StateMsgGasCost) - * [StateNetworkName](#StateNetworkName) - * [StateReadState](#StateReadState) - * [StateReplay](#StateReplay) - * [StateSearchMsg](#StateSearchMsg) - * [StateSectorExpiration](#StateSectorExpiration) - * [StateSectorGetInfo](#StateSectorGetInfo) - * [StateSectorPartition](#StateSectorPartition) - * [StateSectorPreCommitInfo](#StateSectorPreCommitInfo) - * [StateVerifiedClientStatus](#StateVerifiedClientStatus) - * [StateWaitMsg](#StateWaitMsg) -* [Sync](#Sync) - * [SyncCheckBad](#SyncCheckBad) - * [SyncCheckpoint](#SyncCheckpoint) - * [SyncIncomingBlocks](#SyncIncomingBlocks) - * [SyncMarkBad](#SyncMarkBad) - * [SyncState](#SyncState) - * [SyncSubmitBlock](#SyncSubmitBlock) - * [SyncUnmarkBad](#SyncUnmarkBad) -* [Wallet](#Wallet) - * [WalletBalance](#WalletBalance) - * [WalletDefaultAddress](#WalletDefaultAddress) - * [WalletDelete](#WalletDelete) - * [WalletExport](#WalletExport) - * [WalletHas](#WalletHas) - * [WalletImport](#WalletImport) - * [WalletList](#WalletList) - * [WalletNew](#WalletNew) - * [WalletSetDefault](#WalletSetDefault) - * [WalletSign](#WalletSign) - * [WalletSignMessage](#WalletSignMessage) - * [WalletVerify](#WalletVerify) -## - - -### Closing - - -Perms: read - -Inputs: `null` - -Response: `{}` - -### Shutdown - - -Perms: admin - -Inputs: `null` - -Response: `{}` - -### Version - - -Perms: read - -Inputs: `null` - -Response: -```json -{ - "Version": "string value", - "APIVersion": 3584, - "BlockDelay": 42 -} -``` - -## Auth - - -### AuthNew - - -Perms: admin - -Inputs: -```json -[ - null -] -``` - -Response: `"Ynl0ZSBhcnJheQ=="` - -### AuthVerify - - -Perms: read - -Inputs: -```json -[ - "string value" -] -``` - -Response: `null` - -## Beacon -The Beacon method group contains methods for interacting with the random beacon (DRAND) - - -### BeaconGetEntry -BeaconGetEntry returns the beacon entry for the given filecoin epoch. If -the entry has not yet been produced, the call will block until the entry -becomes available - - -Perms: read - -Inputs: -```json -[ - 10101 -] -``` - -Response: -```json -{ - "Round": 42, - "Data": "Ynl0ZSBhcnJheQ==" -} -``` - -## Chain -The Chain method group contains methods for interacting with the -blockchain, but that do not require any form of state computation. - - -### ChainExport -ChainExport returns a stream of bytes with CAR dump of chain data. -The exported chain data includes the header chain from the given tipset -back to genesis, the entire genesis state, and the most recent 'nroots' -state trees. -If oldmsgskip is set, messages from before the requested roots are also not included. - - -Perms: read - -Inputs: -```json -[ - 10101, - true, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `"Ynl0ZSBhcnJheQ=="` - -### ChainGetBlock -ChainGetBlock returns the block specified by the given CID. - - -Perms: read - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } -] -``` - -Response: -```json -{ - "Miner": "t01234", - "Ticket": { - "VRFProof": "Ynl0ZSBhcnJheQ==" - }, - "ElectionProof": { - "WinCount": 9, - "VRFProof": "Ynl0ZSBhcnJheQ==" - }, - "BeaconEntries": null, - "WinPoStProof": null, - "Parents": null, - "ParentWeight": "0", - "Height": 10101, - "ParentStateRoot": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "ParentMessageReceipts": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "Messages": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "BLSAggregate": { - "Type": 2, - "Data": "Ynl0ZSBhcnJheQ==" - }, - "Timestamp": 42, - "BlockSig": { - "Type": 2, - "Data": "Ynl0ZSBhcnJheQ==" - }, - "ForkSignaling": 42, - "ParentBaseFee": "0" -} -``` - -### ChainGetBlockMessages -ChainGetBlockMessages returns messages stored in the specified block. - - -Perms: read - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } -] -``` - -Response: -```json -{ - "BlsMessages": null, - "SecpkMessages": null, - "Cids": null -} -``` - -### ChainGetGenesis -ChainGetGenesis returns the genesis tipset. - - -Perms: read - -Inputs: `null` - -Response: -```json -{ - "Cids": null, - "Blocks": null, - "Height": 0 -} -``` - -### ChainGetMessage -ChainGetMessage reads a message referenced by the specified CID from the -chain blockstore. - - -Perms: read - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } -] -``` - -Response: -```json -{ - "Version": 42, - "To": "t01234", - "From": "t01234", - "Nonce": 42, - "Value": "0", - "GasLimit": 9, - "GasFeeCap": "0", - "GasPremium": "0", - "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" -} -``` - -### ChainGetNode -There are not yet any comments for this method. - -Perms: read - -Inputs: -```json -[ - "string value" -] -``` - -Response: -```json -{ - "Cid": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "Obj": {} -} -``` - -### ChainGetParentMessages -ChainGetParentMessages returns messages stored in parent tipset of the -specified block. - - -Perms: read - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } -] -``` - -Response: `null` - -### ChainGetParentReceipts -ChainGetParentReceipts returns receipts for messages in parent tipset of -the specified block. - - -Perms: read - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } -] -``` - -Response: `null` - -### ChainGetPath -ChainGetPath returns a set of revert/apply operations needed to get from -one tipset to another, for example: -``` - to - ^ -from tAA - ^ ^ -tBA tAB - ^---*--^ - ^ - tRR -``` -Would return `[revert(tBA), apply(tAB), apply(tAA)]` - - -Perms: read - -Inputs: -```json -[ - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ], - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `null` - -### ChainGetRandomnessFromBeacon -ChainGetRandomnessFromBeacon is used to sample the beacon for randomness. - - -Perms: read - -Inputs: -```json -[ - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ], - 2, - 10101, - "Ynl0ZSBhcnJheQ==" -] -``` - -Response: `null` - -### ChainGetRandomnessFromTickets -ChainGetRandomnessFromTickets is used to sample the chain for randomness. - - -Perms: read - -Inputs: -```json -[ - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ], - 2, - 10101, - "Ynl0ZSBhcnJheQ==" -] -``` - -Response: `null` - -### ChainGetTipSet -ChainGetTipSet returns the tipset specified by the given TipSetKey. - - -Perms: read - -Inputs: -```json -[ - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "Cids": null, - "Blocks": null, - "Height": 0 -} -``` - -### ChainGetTipSetByHeight -ChainGetTipSetByHeight looks back for a tipset at the specified epoch. -If there are no blocks at the specified epoch, a tipset at an earlier epoch -will be returned. - - -Perms: read - -Inputs: -```json -[ - 10101, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "Cids": null, - "Blocks": null, - "Height": 0 -} -``` - -### ChainHasObj -ChainHasObj checks if a given CID exists in the chain blockstore. - - -Perms: read - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } -] -``` - -Response: `true` - -### ChainHead -ChainHead returns the current head of the chain. - - -Perms: read - -Inputs: `null` - -Response: -```json -{ - "Cids": null, - "Blocks": null, - "Height": 0 -} -``` - -### ChainNotify -ChainNotify returns channel with chain head updates. -First message is guaranteed to be of len == 1, and type == 'current'. - - -Perms: read - -Inputs: `null` - -Response: `null` - -### ChainReadObj -ChainReadObj reads ipld nodes referenced by the specified CID from chain -blockstore and returns raw bytes. - - -Perms: read - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } -] -``` - -Response: `"Ynl0ZSBhcnJheQ=="` - -### ChainSetHead -ChainSetHead forcefully sets current chain head. Use with caution. - - -Perms: admin - -Inputs: -```json -[ - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `{}` - -### ChainStatObj -ChainStatObj returns statistics about the graph referenced by 'obj'. -If 'base' is also specified, then the returned stat will be a diff -between the two objects. - - -Perms: read - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } -] -``` - -Response: -```json -{ - "Size": 42, - "Links": 42 -} -``` - -### ChainTipSetWeight -ChainTipSetWeight computes weight for the specified tipset. - - -Perms: read - -Inputs: -```json -[ - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `"0"` - -## Client -The Client methods all have to do with interacting with the storage and -retrieval markets as a client - - -### ClientCalcCommP -ClientCalcCommP calculates the CommP for a specified file - - -Perms: read - -Inputs: -```json -[ - "string value" -] -``` - -Response: -```json -{ - "Root": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "Size": 1024 -} -``` - -### ClientDataTransferUpdates -There are not yet any comments for this method. - -Perms: write - -Inputs: `null` - -Response: -```json -{ - "TransferID": 3, - "Status": 1, - "BaseCID": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "IsInitiator": true, - "IsSender": true, - "Voucher": "string value", - "Message": "string value", - "OtherPeer": "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf", - "Transferred": 42 -} -``` - -### ClientDealSize -ClientDealSize calculates real deal data size - - -Perms: read - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } -] -``` - -Response: -```json -{ - "PayloadSize": 9, - "PieceSize": 1032 -} -``` - -### ClientFindData -ClientFindData identifies peers that have a certain file, and returns QueryOffers (one per peer). - - -Perms: read - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - null -] -``` - -Response: `null` - -### ClientGenCar -ClientGenCar generates a CAR file for the specified file. - - -Perms: write - -Inputs: -```json -[ - { - "Path": "string value", - "IsCAR": true - }, - "string value" -] -``` - -Response: `{}` - -### ClientGetDealInfo -ClientGetDealInfo returns the latest information about a given deal. - - -Perms: read - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } -] -``` - -Response: -```json -{ - "ProposalCid": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "State": 42, - "Message": "string value", - "Provider": "t01234", - "DataRef": { - "TransferType": "string value", - "Root": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "PieceCid": null, - "PieceSize": 1024 - }, - "PieceCID": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "Size": 42, - "PricePerEpoch": "0", - "Duration": 42, - "DealID": 5432, - "CreationTime": "0001-01-01T00:00:00Z" -} -``` - -### ClientGetDealUpdates -ClientGetDealUpdates returns the status of updated deals - - -Perms: read - -Inputs: `null` - -Response: -```json -{ - "ProposalCid": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "State": 42, - "Message": "string value", - "Provider": "t01234", - "DataRef": { - "TransferType": "string value", - "Root": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "PieceCid": null, - "PieceSize": 1024 - }, - "PieceCID": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "Size": 42, - "PricePerEpoch": "0", - "Duration": 42, - "DealID": 5432, - "CreationTime": "0001-01-01T00:00:00Z" -} -``` - -### ClientHasLocal -ClientHasLocal indicates whether a certain CID is locally stored. - - -Perms: write - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } -] -``` - -Response: `true` - -### ClientImport -ClientImport imports file under the specified path into filestore. - - -Perms: admin - -Inputs: -```json -[ - { - "Path": "string value", - "IsCAR": true - } -] -``` - -Response: -```json -{ - "Root": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "ImportID": 50 -} -``` - -### ClientListDataTransfers -ClientListTransfers returns the status of all ongoing transfers of data - - -Perms: write - -Inputs: `null` - -Response: `null` - -### ClientListDeals -ClientListDeals returns information about the deals made by the local client. - - -Perms: write - -Inputs: `null` - -Response: `null` - -### ClientListImports -ClientListImports lists imported files and their root CIDs - - -Perms: write - -Inputs: `null` - -Response: `null` - -### ClientMinerQueryOffer -ClientMinerQueryOffer returns a QueryOffer for the specific miner and file. - - -Perms: read - -Inputs: -```json -[ - "t01234", - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - null -] -``` - -Response: -```json -{ - "Err": "string value", - "Root": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "Piece": null, - "Size": 42, - "MinPrice": "0", - "UnsealPrice": "0", - "PaymentInterval": 42, - "PaymentIntervalIncrease": 42, - "Miner": "t01234", - "MinerPeer": { - "Address": "t01234", - "ID": "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf", - "PieceCID": null - } -} -``` - -### ClientQueryAsk -ClientQueryAsk returns a signed StorageAsk from the specified miner. - - -Perms: read - -Inputs: -```json -[ - "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf", - "t01234" -] -``` - -Response: -```json -{ - "Ask": { - "Price": "0", - "VerifiedPrice": "0", - "MinPieceSize": 1032, - "MaxPieceSize": 1032, - "Miner": "t01234", - "Timestamp": 10101, - "Expiry": 10101, - "SeqNo": 42 - }, - "Signature": { - "Type": 2, - "Data": "Ynl0ZSBhcnJheQ==" - } -} -``` - -### ClientRemoveImport -ClientRemoveImport removes file import - - -Perms: admin - -Inputs: -```json -[ - 50 -] -``` - -Response: `{}` - -### ClientRetrieve -ClientRetrieve initiates the retrieval of a file, as specified in the order. - - -Perms: admin - -Inputs: -```json -[ - { - "Root": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "Piece": null, - "Size": 42, - "Total": "0", - "UnsealPrice": "0", - "PaymentInterval": 42, - "PaymentIntervalIncrease": 42, - "Client": "t01234", - "Miner": "t01234", - "MinerPeer": { - "Address": "t01234", - "ID": "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf", - "PieceCID": null - } - }, - { - "Path": "string value", - "IsCAR": true - } -] -``` - -Response: `{}` - -### ClientRetrieveTryRestartInsufficientFunds -ClientRetrieveTryRestartInsufficientFunds attempts to restart stalled retrievals on a given payment channel -which are stuck due to insufficient funds - - -Perms: write - -Inputs: -```json -[ - "t01234" -] -``` - -Response: `{}` - -### ClientRetrieveWithEvents -ClientRetrieveWithEvents initiates the retrieval of a file, as specified in the order, and provides a channel -of status updates. - - -Perms: admin - -Inputs: -```json -[ - { - "Root": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "Piece": null, - "Size": 42, - "Total": "0", - "UnsealPrice": "0", - "PaymentInterval": 42, - "PaymentIntervalIncrease": 42, - "Client": "t01234", - "Miner": "t01234", - "MinerPeer": { - "Address": "t01234", - "ID": "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf", - "PieceCID": null - } - }, - { - "Path": "string value", - "IsCAR": true - } -] -``` - -Response: -```json -{ - "Event": 5, - "Status": 0, - "BytesReceived": 42, - "FundsSpent": "0", - "Err": "string value" -} -``` - -### ClientStartDeal -ClientStartDeal proposes a deal with a miner. - - -Perms: admin - -Inputs: -```json -[ - { - "Data": { - "TransferType": "string value", - "Root": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "PieceCid": null, - "PieceSize": 1024 - }, - "Wallet": "t01234", - "Miner": "t01234", - "EpochPrice": "0", - "MinBlocksDuration": 42, - "ProviderCollateral": "0", - "DealStartEpoch": 10101, - "FastRetrieval": true, - "VerifiedDeal": true - } -] -``` - -Response: `null` - -## Gas - - -### GasEstimateFeeCap -GasEstimateFeeCap estimates gas fee cap - - -Perms: read - -Inputs: -```json -[ - { - "Version": 42, - "To": "t01234", - "From": "t01234", - "Nonce": 42, - "Value": "0", - "GasLimit": 9, - "GasFeeCap": "0", - "GasPremium": "0", - "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" - }, - 9, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `"0"` - -### GasEstimateGasLimit -GasEstimateGasLimit estimates gas used by the message and returns it. -It fails if message fails to execute. - - -Perms: read - -Inputs: -```json -[ - { - "Version": 42, - "To": "t01234", - "From": "t01234", - "Nonce": 42, - "Value": "0", - "GasLimit": 9, - "GasFeeCap": "0", - "GasPremium": "0", - "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" - }, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `9` - -### GasEstimateGasPremium -GasEstimateGasPremium estimates what gas price should be used for a -message to have high likelihood of inclusion in `nblocksincl` epochs. - - -Perms: read - -Inputs: -```json -[ - 42, - "t01234", - 9, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `"0"` - -### GasEstimateMessageGas -GasEstimateMessageGas estimates gas values for unset message gas fields - - -Perms: read - -Inputs: -```json -[ - { - "Version": 42, - "To": "t01234", - "From": "t01234", - "Nonce": 42, - "Value": "0", - "GasLimit": 9, - "GasFeeCap": "0", - "GasPremium": "0", - "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" - }, - { - "MaxFee": "0" - }, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "Version": 42, - "To": "t01234", - "From": "t01234", - "Nonce": 42, - "Value": "0", - "GasLimit": 9, - "GasFeeCap": "0", - "GasPremium": "0", - "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" -} -``` - -## I - - -### ID - - -Perms: read - -Inputs: `null` - -Response: `"12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf"` - -## Log - - -### LogList - - -Perms: write - -Inputs: `null` - -Response: `null` - -### LogSetLevel - - -Perms: write - -Inputs: -```json -[ - "string value", - "string value" -] -``` - -Response: `{}` - -## Market - - -### MarketEnsureAvailable -MarketFreeBalance - - -Perms: sign - -Inputs: -```json -[ - "t01234", - "t01234", - "0" -] -``` - -Response: -```json -{ - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" -} -``` - -## Miner - - -### MinerCreateBlock -There are not yet any comments for this method. - -Perms: write - -Inputs: -```json -[ - { - "Miner": "t01234", - "Parents": [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ], - "Ticket": { - "VRFProof": "Ynl0ZSBhcnJheQ==" - }, - "Eproof": { - "WinCount": 9, - "VRFProof": "Ynl0ZSBhcnJheQ==" - }, - "BeaconValues": null, - "Messages": null, - "Epoch": 10101, - "Timestamp": 42, - "WinningPoStProof": null - } -] -``` - -Response: -```json -{ - "Header": { - "Miner": "t01234", - "Ticket": { - "VRFProof": "Ynl0ZSBhcnJheQ==" - }, - "ElectionProof": { - "WinCount": 9, - "VRFProof": "Ynl0ZSBhcnJheQ==" - }, - "BeaconEntries": null, - "WinPoStProof": null, - "Parents": null, - "ParentWeight": "0", - "Height": 10101, - "ParentStateRoot": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "ParentMessageReceipts": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "Messages": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "BLSAggregate": { - "Type": 2, - "Data": "Ynl0ZSBhcnJheQ==" - }, - "Timestamp": 42, - "BlockSig": { - "Type": 2, - "Data": "Ynl0ZSBhcnJheQ==" - }, - "ForkSignaling": 42, - "ParentBaseFee": "0" - }, - "BlsMessages": null, - "SecpkMessages": null -} -``` - -### MinerGetBaseInfo -There are not yet any comments for this method. - -Perms: read - -Inputs: -```json -[ - "t01234", - 10101, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "MinerPower": "0", - "NetworkPower": "0", - "Sectors": null, - "WorkerKey": "t01234", - "SectorSize": 34359738368, - "PrevBeaconEntry": { - "Round": 42, - "Data": "Ynl0ZSBhcnJheQ==" - }, - "BeaconEntries": null, - "HasMinPower": true -} -``` - -## Mpool -The Mpool methods are for interacting with the message pool. The message pool -manages all incoming and outgoing 'messages' going over the network. - - -### MpoolClear -MpoolClear clears pending messages from the mpool - - -Perms: write - -Inputs: -```json -[ - true -] -``` - -Response: `{}` - -### MpoolGetConfig -MpoolGetConfig returns (a copy of) the current mpool config - - -Perms: read - -Inputs: `null` - -Response: -```json -{ - "PriorityAddrs": null, - "SizeLimitHigh": 123, - "SizeLimitLow": 123, - "ReplaceByFeeRatio": 12.3, - "PruneCooldown": 60000000000, - "GasLimitOverestimation": 12.3 -} -``` - -### MpoolGetNonce -MpoolGetNonce gets next nonce for the specified sender. -Note that this method may not be atomic. Use MpoolPushMessage instead. - - -Perms: read - -Inputs: -```json -[ - "t01234" -] -``` - -Response: `42` - -### MpoolPending -MpoolPending returns pending mempool messages. - - -Perms: read - -Inputs: -```json -[ - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `null` - -### MpoolPush -MpoolPush pushes a signed message to mempool. - - -Perms: write - -Inputs: -```json -[ - { - "Message": { - "Version": 42, - "To": "t01234", - "From": "t01234", - "Nonce": 42, - "Value": "0", - "GasLimit": 9, - "GasFeeCap": "0", - "GasPremium": "0", - "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" - }, - "Signature": { - "Type": 2, - "Data": "Ynl0ZSBhcnJheQ==" - } - } -] -``` - -Response: -```json -{ - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" -} -``` - -### MpoolPushMessage -MpoolPushMessage atomically assigns a nonce, signs, and pushes a message -to mempool. -maxFee is only used when GasFeeCap/GasPremium fields aren't specified - -When maxFee is set to 0, MpoolPushMessage will guess appropriate fee -based on current chain conditions - - -Perms: sign - -Inputs: -```json -[ - { - "Version": 42, - "To": "t01234", - "From": "t01234", - "Nonce": 42, - "Value": "0", - "GasLimit": 9, - "GasFeeCap": "0", - "GasPremium": "0", - "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" - }, - { - "MaxFee": "0" - } -] -``` - -Response: -```json -{ - "Message": { - "Version": 42, - "To": "t01234", - "From": "t01234", - "Nonce": 42, - "Value": "0", - "GasLimit": 9, - "GasFeeCap": "0", - "GasPremium": "0", - "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" - }, - "Signature": { - "Type": 2, - "Data": "Ynl0ZSBhcnJheQ==" - } -} -``` - -### MpoolSelect -MpoolSelect returns a list of pending messages for inclusion in the next block - - -Perms: read - -Inputs: -```json -[ - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ], - 12.3 -] -``` - -Response: `null` - -### MpoolSetConfig -MpoolSetConfig sets the mpool config to (a copy of) the supplied config - - -Perms: write - -Inputs: -```json -[ - { - "PriorityAddrs": null, - "SizeLimitHigh": 123, - "SizeLimitLow": 123, - "ReplaceByFeeRatio": 12.3, - "PruneCooldown": 60000000000, - "GasLimitOverestimation": 12.3 - } -] -``` - -Response: `{}` - -### MpoolSub -There are not yet any comments for this method. - -Perms: read - -Inputs: `null` - -Response: -```json -{ - "Type": 0, - "Message": { - "Message": { - "Version": 42, - "To": "t01234", - "From": "t01234", - "Nonce": 42, - "Value": "0", - "GasLimit": 9, - "GasFeeCap": "0", - "GasPremium": "0", - "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" - }, - "Signature": { - "Type": 2, - "Data": "Ynl0ZSBhcnJheQ==" - } - } -} -``` - -## Msig -The Msig methods are used to interact with multisig wallets on the -filecoin network - - -### MsigAddApprove -MsigAddApprove approves a previously proposed AddSigner message -It takes the following params: , , , -, , - - -Perms: sign - -Inputs: -```json -[ - "t01234", - "t01234", - 42, - "t01234", - "t01234", - true -] -``` - -Response: -```json -{ - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" -} -``` - -### MsigAddCancel -MsigAddCancel cancels a previously proposed AddSigner message -It takes the following params: , , , -, - - -Perms: sign - -Inputs: -```json -[ - "t01234", - "t01234", - 42, - "t01234", - true -] -``` - -Response: -```json -{ - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" -} -``` - -### MsigAddPropose -MsigAddPropose proposes adding a signer in the multisig -It takes the following params: , , -, - - -Perms: sign - -Inputs: -```json -[ - "t01234", - "t01234", - "t01234", - true -] -``` - -Response: -```json -{ - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" -} -``` - -### MsigApprove -MsigApprove approves a previously-proposed multisig message -It takes the following params: , , , , , -, , - - -Perms: sign - -Inputs: -```json -[ - "t01234", - 42, - "t01234", - "t01234", - "0", - "t01234", - 42, - "Ynl0ZSBhcnJheQ==" -] -``` - -Response: -```json -{ - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" -} -``` - -### MsigCancel -MsigCancel cancels a previously-proposed multisig message -It takes the following params: , , , , -, , - - -Perms: sign - -Inputs: -```json -[ - "t01234", - 42, - "t01234", - "0", - "t01234", - 42, - "Ynl0ZSBhcnJheQ==" -] -``` - -Response: -```json -{ - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" -} -``` - -### MsigCreate -MsigCreate creates a multisig wallet -It takes the following params: , , -, , - - -Perms: sign - -Inputs: -```json -[ - 42, - null, - 10101, - "0", - "t01234", - "0" -] -``` - -Response: -```json -{ - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" -} -``` - -### MsigGetAvailableBalance -MsigGetAvailableBalance returns the portion of a multisig's balance that can be withdrawn or spent - - -Perms: read - -Inputs: -```json -[ - "t01234", - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `"0"` - -### MsigGetVested -MsigGetVested returns the amount of FIL that vested in a multisig in a certain period. -It takes the following params: , , - - -Perms: read - -Inputs: -```json -[ - "t01234", - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ], - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `"0"` - -### MsigPropose -MsigPropose proposes a multisig message -It takes the following params: , , , -, , - - -Perms: sign - -Inputs: -```json -[ - "t01234", - "t01234", - "0", - "t01234", - 42, - "Ynl0ZSBhcnJheQ==" -] -``` - -Response: -```json -{ - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" -} -``` - -### MsigSwapApprove -MsigSwapApprove approves a previously proposed SwapSigner -It takes the following params: , , , -, , - - -Perms: sign - -Inputs: -```json -[ - "t01234", - "t01234", - 42, - "t01234", - "t01234", - "t01234" -] -``` - -Response: -```json -{ - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" -} -``` - -### MsigSwapCancel -MsigSwapCancel cancels a previously proposed SwapSigner message -It takes the following params: , , , -, - - -Perms: sign - -Inputs: -```json -[ - "t01234", - "t01234", - 42, - "t01234", - "t01234" -] -``` - -Response: -```json -{ - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" -} -``` - -### MsigSwapPropose -MsigSwapPropose proposes swapping 2 signers in the multisig -It takes the following params: , , -, - - -Perms: sign - -Inputs: -```json -[ - "t01234", - "t01234", - "t01234", - "t01234" -] -``` - -Response: -```json -{ - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" -} -``` - -## Net - - -### NetAddrsListen - - -Perms: read - -Inputs: `null` - -Response: -```json -{ - "Addrs": null, - "ID": "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf" -} -``` - -### NetAgentVersion - - -Perms: read - -Inputs: -```json -[ - "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf" -] -``` - -Response: `"string value"` - -### NetAutoNatStatus - - -Perms: read - -Inputs: `null` - -Response: -```json -{ - "Reachability": 1, - "PublicAddr": "string value" -} -``` - -### NetBandwidthStats - - -Perms: read - -Inputs: `null` - -Response: -```json -{ - "TotalIn": 9, - "TotalOut": 9, - "RateIn": 12.3, - "RateOut": 12.3 -} -``` - -### NetBandwidthStatsByPeer - - -Perms: read - -Inputs: `null` - -Response: -```json -{ - "12D3KooWSXmXLJmBR1M7i9RW9GQPNUhZSzXKzxDHWtAgNuJAbyEJ": { - "TotalIn": 174000, - "TotalOut": 12500, - "RateIn": 100, - "RateOut": 50 - } -} -``` - -### NetBandwidthStatsByProtocol - - -Perms: read - -Inputs: `null` - -Response: -```json -{ - "/fil/hello/1.0.0": { - "TotalIn": 174000, - "TotalOut": 12500, - "RateIn": 100, - "RateOut": 50 - } -} -``` - -### NetConnect - - -Perms: write - -Inputs: -```json -[ - { - "Addrs": null, - "ID": "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf" - } -] -``` - -Response: `{}` - -### NetConnectedness - - -Perms: read - -Inputs: -```json -[ - "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf" -] -``` - -Response: `1` - -### NetDisconnect - - -Perms: write - -Inputs: -```json -[ - "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf" -] -``` - -Response: `{}` - -### NetFindPeer - - -Perms: read - -Inputs: -```json -[ - "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf" -] -``` - -Response: -```json -{ - "Addrs": null, - "ID": "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf" -} -``` - -### NetPeers - - -Perms: read - -Inputs: `null` - -Response: `null` - -### NetPubsubScores - - -Perms: read - -Inputs: `null` - -Response: `null` - -## Paych -The Paych methods are for interacting with and managing payment channels - - -### PaychAllocateLane -There are not yet any comments for this method. - -Perms: sign - -Inputs: -```json -[ - "t01234" -] -``` - -Response: `42` - -### PaychAvailableFunds -There are not yet any comments for this method. - -Perms: sign - -Inputs: -```json -[ - "t01234" -] -``` - -Response: -```json -{ - "Channel": "\u003cempty\u003e", - "From": "t01234", - "To": "t01234", - "ConfirmedAmt": "0", - "PendingAmt": "0", - "PendingWaitSentinel": null, - "QueuedAmt": "0", - "VoucherReedeemedAmt": "0" -} -``` - -### PaychAvailableFundsByFromTo -There are not yet any comments for this method. - -Perms: sign - -Inputs: -```json -[ - "t01234", - "t01234" -] -``` - -Response: -```json -{ - "Channel": "\u003cempty\u003e", - "From": "t01234", - "To": "t01234", - "ConfirmedAmt": "0", - "PendingAmt": "0", - "PendingWaitSentinel": null, - "QueuedAmt": "0", - "VoucherReedeemedAmt": "0" -} -``` - -### PaychCollect -There are not yet any comments for this method. - -Perms: sign - -Inputs: -```json -[ - "t01234" -] -``` - -Response: -```json -{ - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" -} -``` - -### PaychGet -There are not yet any comments for this method. - -Perms: sign - -Inputs: -```json -[ - "t01234", - "t01234", - "0" -] -``` - -Response: -```json -{ - "Channel": "t01234", - "WaitSentinel": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } -} -``` - -### PaychGetWaitReady -There are not yet any comments for this method. - -Perms: sign - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } -] -``` - -Response: `"t01234"` - -### PaychList -There are not yet any comments for this method. - -Perms: read - -Inputs: `null` - -Response: `null` - -### PaychNewPayment -There are not yet any comments for this method. - -Perms: sign - -Inputs: -```json -[ - "t01234", - "t01234", - null -] -``` - -Response: -```json -{ - "Channel": "t01234", - "WaitSentinel": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "Vouchers": null -} -``` - -### PaychSettle -There are not yet any comments for this method. - -Perms: sign - -Inputs: -```json -[ - "t01234" -] -``` - -Response: -```json -{ - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" -} -``` - -### PaychStatus -There are not yet any comments for this method. - -Perms: read - -Inputs: -```json -[ - "t01234" -] -``` - -Response: -```json -{ - "ControlAddr": "t01234", - "Direction": 1 -} -``` - -### PaychVoucherAdd -There are not yet any comments for this method. - -Perms: write - -Inputs: -```json -[ - "t01234", - { - "ChannelAddr": "t01234", - "TimeLockMin": 10101, - "TimeLockMax": 10101, - "SecretPreimage": "Ynl0ZSBhcnJheQ==", - "Extra": { - "Actor": "t01234", - "Method": 1, - "Data": "Ynl0ZSBhcnJheQ==" - }, - "Lane": 42, - "Nonce": 42, - "Amount": "0", - "MinSettleHeight": 10101, - "Merges": null, - "Signature": { - "Type": 2, - "Data": "Ynl0ZSBhcnJheQ==" - } - }, - "Ynl0ZSBhcnJheQ==", - "0" -] -``` - -Response: `"0"` - -### PaychVoucherCheckSpendable -There are not yet any comments for this method. - -Perms: read - -Inputs: -```json -[ - "t01234", - { - "ChannelAddr": "t01234", - "TimeLockMin": 10101, - "TimeLockMax": 10101, - "SecretPreimage": "Ynl0ZSBhcnJheQ==", - "Extra": { - "Actor": "t01234", - "Method": 1, - "Data": "Ynl0ZSBhcnJheQ==" - }, - "Lane": 42, - "Nonce": 42, - "Amount": "0", - "MinSettleHeight": 10101, - "Merges": null, - "Signature": { - "Type": 2, - "Data": "Ynl0ZSBhcnJheQ==" - } - }, - "Ynl0ZSBhcnJheQ==", - "Ynl0ZSBhcnJheQ==" -] -``` - -Response: `true` - -### PaychVoucherCheckValid -There are not yet any comments for this method. - -Perms: read - -Inputs: -```json -[ - "t01234", - { - "ChannelAddr": "t01234", - "TimeLockMin": 10101, - "TimeLockMax": 10101, - "SecretPreimage": "Ynl0ZSBhcnJheQ==", - "Extra": { - "Actor": "t01234", - "Method": 1, - "Data": "Ynl0ZSBhcnJheQ==" - }, - "Lane": 42, - "Nonce": 42, - "Amount": "0", - "MinSettleHeight": 10101, - "Merges": null, - "Signature": { - "Type": 2, - "Data": "Ynl0ZSBhcnJheQ==" - } - } -] -``` - -Response: `{}` - -### PaychVoucherCreate -There are not yet any comments for this method. - -Perms: sign - -Inputs: -```json -[ - "t01234", - "0", - 42 -] -``` - -Response: -```json -{ - "Voucher": { - "ChannelAddr": "t01234", - "TimeLockMin": 10101, - "TimeLockMax": 10101, - "SecretPreimage": "Ynl0ZSBhcnJheQ==", - "Extra": { - "Actor": "t01234", - "Method": 1, - "Data": "Ynl0ZSBhcnJheQ==" - }, - "Lane": 42, - "Nonce": 42, - "Amount": "0", - "MinSettleHeight": 10101, - "Merges": null, - "Signature": { - "Type": 2, - "Data": "Ynl0ZSBhcnJheQ==" - } - }, - "Shortfall": "0" -} -``` - -### PaychVoucherList -There are not yet any comments for this method. - -Perms: write - -Inputs: -```json -[ - "t01234" -] -``` - -Response: `null` - -### PaychVoucherSubmit -There are not yet any comments for this method. - -Perms: sign - -Inputs: -```json -[ - "t01234", - { - "ChannelAddr": "t01234", - "TimeLockMin": 10101, - "TimeLockMax": 10101, - "SecretPreimage": "Ynl0ZSBhcnJheQ==", - "Extra": { - "Actor": "t01234", - "Method": 1, - "Data": "Ynl0ZSBhcnJheQ==" - }, - "Lane": 42, - "Nonce": 42, - "Amount": "0", - "MinSettleHeight": 10101, - "Merges": null, - "Signature": { - "Type": 2, - "Data": "Ynl0ZSBhcnJheQ==" - } - }, - "Ynl0ZSBhcnJheQ==", - "Ynl0ZSBhcnJheQ==" -] -``` - -Response: -```json -{ - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" -} -``` - -## State -The State methods are used to query, inspect, and interact with chain state. -All methods take a TipSetKey as a parameter. The state looked up is the state at that tipset. -A nil TipSetKey can be provided as a param, this will cause the heaviest tipset in the chain to be used. - - -### StateAccountKey -StateAccountKey returns the public key address of the given ID address - - -Perms: read - -Inputs: -```json -[ - "t01234", - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `"t01234"` - -### StateAllMinerFaults -StateAllMinerFaults returns all non-expired Faults that occur within lookback epochs of the given tipset - - -Perms: read - -Inputs: -```json -[ - 10101, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `null` - -### StateCall -StateCall runs the given message and returns its result without any persisted changes. - - -Perms: read - -Inputs: -```json -[ - { - "Version": 42, - "To": "t01234", - "From": "t01234", - "Nonce": 42, - "Value": "0", - "GasLimit": 9, - "GasFeeCap": "0", - "GasPremium": "0", - "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" - }, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "Msg": { - "Version": 42, - "To": "t01234", - "From": "t01234", - "Nonce": 42, - "Value": "0", - "GasLimit": 9, - "GasFeeCap": "0", - "GasPremium": "0", - "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" - }, - "MsgRct": { - "ExitCode": 0, - "Return": "Ynl0ZSBhcnJheQ==", - "GasUsed": 9 - }, - "ExecutionTrace": { - "Msg": { - "Version": 42, - "To": "t01234", - "From": "t01234", - "Nonce": 42, - "Value": "0", - "GasLimit": 9, - "GasFeeCap": "0", - "GasPremium": "0", - "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" - }, - "MsgRct": { - "ExitCode": 0, - "Return": "Ynl0ZSBhcnJheQ==", - "GasUsed": 9 - }, - "Error": "string value", - "Duration": 60000000000, - "GasCharges": null, - "Subcalls": null - }, - "Error": "string value", - "Duration": 60000000000 -} -``` - -### StateChangedActors -StateChangedActors returns all the actors whose states change between the two given state CIDs -TODO: Should this take tipset keys instead? - - -Perms: read - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } -] -``` - -Response: -```json -{ - "t01236": { - "Code": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "Head": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "Nonce": 42, - "Balance": "0" - } -} -``` - -### StateCirculatingSupply -StateCirculatingSupply returns the circulating supply of Filecoin at the given tipset - - -Perms: read - -Inputs: -```json -[ - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "FilVested": "0", - "FilMined": "0", - "FilBurnt": "0", - "FilLocked": "0", - "FilCirculating": "0" -} -``` - -### StateCompute -StateCompute is a flexible command that applies the given messages on the given tipset. -The messages are run as though the VM were at the provided height. - - -Perms: read - -Inputs: -```json -[ - 10101, - null, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "Root": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "Trace": null -} -``` - -### StateDealProviderCollateralBounds -StateDealProviderCollateralBounds returns the min and max collateral a storage provider -can issue. It takes the deal size and verified status as parameters. - - -Perms: read - -Inputs: -```json -[ - 1032, - true, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "Min": "0", - "Max": "0" -} -``` - -### StateGetActor -StateGetActor returns the indicated actor's nonce and balance. - - -Perms: read - -Inputs: -```json -[ - "t01234", - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "Code": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "Head": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "Nonce": 42, - "Balance": "0" -} -``` - -### StateGetReceipt -StateGetReceipt returns the message receipt for the given message - - -Perms: read - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "ExitCode": 0, - "Return": "Ynl0ZSBhcnJheQ==", - "GasUsed": 9 -} -``` - -### StateListActors -StateListActors returns the addresses of every actor in the state - - -Perms: read - -Inputs: -```json -[ - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `null` - -### StateListMessages -StateListMessages looks back and returns all messages with a matching to or from address, stopping at the given height. - - -Perms: read - -Inputs: -```json -[ - { - "Version": 42, - "To": "t01234", - "From": "t01234", - "Nonce": 42, - "Value": "0", - "GasLimit": 9, - "GasFeeCap": "0", - "GasPremium": "0", - "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" - }, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ], - 10101 -] -``` - -Response: `null` - -### StateListMiners -StateListMiners returns the addresses of every miner that has claimed power in the Power Actor - - -Perms: read - -Inputs: -```json -[ - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `null` - -### StateLookupID -StateLookupID retrieves the ID address of the given address - - -Perms: read - -Inputs: -```json -[ - "t01234", - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `"t01234"` - -### StateMarketBalance -StateMarketBalance looks up the Escrow and Locked balances of the given address in the Storage Market - - -Perms: read - -Inputs: -```json -[ - "t01234", - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "Escrow": "0", - "Locked": "0" -} -``` - -### StateMarketDeals -StateMarketDeals returns information about every deal in the Storage Market - - -Perms: read - -Inputs: -```json -[ - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "t026363": { - "Proposal": { - "PieceCID": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "PieceSize": 1032, - "VerifiedDeal": true, - "Client": "t01234", - "Provider": "t01234", - "Label": "string value", - "StartEpoch": 10101, - "EndEpoch": 10101, - "StoragePricePerEpoch": "0", - "ProviderCollateral": "0", - "ClientCollateral": "0" - }, - "State": { - "SectorStartEpoch": 10101, - "LastUpdatedEpoch": 10101, - "SlashEpoch": 10101 - } - } -} -``` - -### StateMarketParticipants -StateMarketParticipants returns the Escrow and Locked balances of every participant in the Storage Market - - -Perms: read - -Inputs: -```json -[ - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "t026363": { - "Escrow": "0", - "Locked": "0" - } -} -``` - -### StateMarketStorageDeal -StateMarketStorageDeal returns information about the indicated deal - - -Perms: read - -Inputs: -```json -[ - 5432, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "Proposal": { - "PieceCID": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "PieceSize": 1032, - "VerifiedDeal": true, - "Client": "t01234", - "Provider": "t01234", - "Label": "string value", - "StartEpoch": 10101, - "EndEpoch": 10101, - "StoragePricePerEpoch": "0", - "ProviderCollateral": "0", - "ClientCollateral": "0" - }, - "State": { - "SectorStartEpoch": 10101, - "LastUpdatedEpoch": 10101, - "SlashEpoch": 10101 - } -} -``` - -### StateMinerActiveSectors -StateMinerActiveSectors returns info about sectors that a given miner is actively proving. - - -Perms: read - -Inputs: -```json -[ - "t01234", - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `null` - -### StateMinerAvailableBalance -StateMinerAvailableBalance returns the portion of a miner's balance that can be withdrawn or spent - - -Perms: read - -Inputs: -```json -[ - "t01234", - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `"0"` - -### StateMinerDeadlines -StateMinerDeadlines returns all the proving deadlines for the given miner - - -Perms: read - -Inputs: -```json -[ - "t01234", - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `null` - -### StateMinerFaults -StateMinerFaults returns a bitfield indicating the faulty sectors of the given miner - - -Perms: read - -Inputs: -```json -[ - "t01234", - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -[ - 5, - 1 -] -``` - -### StateMinerInfo -StateMinerInfo returns info about the indicated miner - - -Perms: read - -Inputs: -```json -[ - "t01234", - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "Owner": "t01234", - "Worker": "t01234", - "NewWorker": "t01234", - "ControlAddresses": null, - "WorkerChangeEpoch": 10101, - "PeerId": "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf", - "Multiaddrs": null, - "SealProofType": 3, - "SectorSize": 34359738368, - "WindowPoStPartitionSectors": 42 -} -``` - -### StateMinerInitialPledgeCollateral -StateMinerInitialPledgeCollateral returns the initial pledge collateral for the specified miner's sector - - -Perms: read - -Inputs: -```json -[ - "t01234", - { - "SealProof": 3, - "SectorNumber": 9, - "SealedCID": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "SealRandEpoch": 10101, - "DealIDs": null, - "Expiration": 10101, - "ReplaceCapacity": true, - "ReplaceSectorDeadline": 42, - "ReplaceSectorPartition": 42, - "ReplaceSectorNumber": 9 - }, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `"0"` - -### StateMinerPartitions -StateMinerPartitions loads miner partitions for the specified miner/deadline - - -Perms: read - -Inputs: -```json -[ - "t01234", - 42, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `null` - -### StateMinerPower -StateMinerPower returns the power of the indicated miner - - -Perms: read - -Inputs: -```json -[ - "t01234", - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "MinerPower": { - "RawBytePower": "0", - "QualityAdjPower": "0" - }, - "TotalPower": { - "RawBytePower": "0", - "QualityAdjPower": "0" - } -} -``` - -### StateMinerPreCommitDepositForPower -StateMinerInitialPledgeCollateral returns the precommit deposit for the specified miner's sector - - -Perms: read - -Inputs: -```json -[ - "t01234", - { - "SealProof": 3, - "SectorNumber": 9, - "SealedCID": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "SealRandEpoch": 10101, - "DealIDs": null, - "Expiration": 10101, - "ReplaceCapacity": true, - "ReplaceSectorDeadline": 42, - "ReplaceSectorPartition": 42, - "ReplaceSectorNumber": 9 - }, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `"0"` - -### StateMinerProvingDeadline -StateMinerProvingDeadline calculates the deadline at some epoch for a proving period -and returns the deadline-related calculations. - - -Perms: read - -Inputs: -```json -[ - "t01234", - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "CurrentEpoch": 10101, - "PeriodStart": 10101, - "Index": 42, - "Open": 10101, - "Close": 10101, - "Challenge": 10101, - "FaultCutoff": 10101, - "WPoStPeriodDeadlines": 42, - "WPoStProvingPeriod": 10101, - "WPoStChallengeWindow": 10101, - "WPoStChallengeLookback": 10101, - "FaultDeclarationCutoff": 10101 -} -``` - -### StateMinerRecoveries -StateMinerRecoveries returns a bitfield indicating the recovering sectors of the given miner - - -Perms: read - -Inputs: -```json -[ - "t01234", - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -[ - 5, - 1 -] -``` - -### StateMinerSectorCount -StateMinerSectorCount returns the number of sectors in a miner's sector set and proving set - - -Perms: read - -Inputs: -```json -[ - "t01234", - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "Sectors": 42, - "Active": 42 -} -``` - -### StateMinerSectors -StateMinerSectors returns info about the given miner's sectors. If the filter bitfield is nil, all sectors are included. -If the filterOut boolean is set to true, any sectors in the filter are excluded. -If false, only those sectors in the filter are included. - - -Perms: read - -Inputs: -```json -[ - "t01234", - [ - 0 - ], - true, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `null` - -### StateMsgGasCost -StateMsgGasCost searches for a message in the chain, and returns details of the messages gas costs, including the penalty and miner tip - - -Perms: read - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "Message": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "GasUsed": "0", - "BaseFeeBurn": "0", - "OverEstimationBurn": "0", - "MinerPenalty": "0", - "MinerTip": "0", - "Refund": "0", - "TotalCost": "0" -} -``` - -### StateNetworkName -StateNetworkName returns the name of the network the node is synced to - - -Perms: read - -Inputs: `null` - -Response: `"lotus"` - -### StateReadState -StateReadState returns the indicated actor's state. - - -Perms: read - -Inputs: -```json -[ - "t01234", - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "Balance": "0", - "State": {} -} -``` - -### StateReplay -StateReplay returns the result of executing the indicated message, assuming it was executed in the indicated tipset. - - -Perms: read - -Inputs: -```json -[ - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ], - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } -] -``` - -Response: -```json -{ - "Msg": { - "Version": 42, - "To": "t01234", - "From": "t01234", - "Nonce": 42, - "Value": "0", - "GasLimit": 9, - "GasFeeCap": "0", - "GasPremium": "0", - "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" - }, - "MsgRct": { - "ExitCode": 0, - "Return": "Ynl0ZSBhcnJheQ==", - "GasUsed": 9 - }, - "ExecutionTrace": { - "Msg": { - "Version": 42, - "To": "t01234", - "From": "t01234", - "Nonce": 42, - "Value": "0", - "GasLimit": 9, - "GasFeeCap": "0", - "GasPremium": "0", - "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" - }, - "MsgRct": { - "ExitCode": 0, - "Return": "Ynl0ZSBhcnJheQ==", - "GasUsed": 9 - }, - "Error": "string value", - "Duration": 60000000000, - "GasCharges": null, - "Subcalls": null - }, - "Error": "string value", - "Duration": 60000000000 -} -``` - -### StateSearchMsg -StateSearchMsg searches for a message in the chain, and returns its receipt and the tipset where it was executed - - -Perms: read - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } -] -``` - -Response: -```json -{ - "Message": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "Receipt": { - "ExitCode": 0, - "Return": "Ynl0ZSBhcnJheQ==", - "GasUsed": 9 - }, - "ReturnDec": {}, - "TipSet": [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ], - "Height": 10101 -} -``` - -### StateSectorExpiration -StateSectorExpiration returns epoch at which given sector will expire - - -Perms: read - -Inputs: -```json -[ - "t01234", - 9, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "OnTime": 10101, - "Early": 10101 -} -``` - -### StateSectorGetInfo -StateSectorGetInfo returns the on-chain info for the specified miner's sector. Returns null in case the sector info isn't found -NOTE: returned info.Expiration may not be accurate in some cases, use StateSectorExpiration to get accurate -expiration epoch - - -Perms: read - -Inputs: -```json -[ - "t01234", - 9, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "SectorNumber": 9, - "SealProof": 3, - "SealedCID": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "DealIDs": null, - "Activation": 10101, - "Expiration": 10101, - "DealWeight": "0", - "VerifiedDealWeight": "0", - "InitialPledge": "0", - "ExpectedDayReward": "0", - "ExpectedStoragePledge": "0" -} -``` - -### StateSectorPartition -StateSectorPartition finds deadline/partition with the specified sector - - -Perms: read - -Inputs: -```json -[ - "t01234", - 9, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "Deadline": 42, - "Partition": 42 -} -``` - -### StateSectorPreCommitInfo -StateSectorPreCommitInfo returns the PreCommit info for the specified miner's sector - - -Perms: read - -Inputs: -```json -[ - "t01234", - 9, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "Info": { - "SealProof": 3, - "SectorNumber": 9, - "SealedCID": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "SealRandEpoch": 10101, - "DealIDs": null, - "Expiration": 10101, - "ReplaceCapacity": true, - "ReplaceSectorDeadline": 42, - "ReplaceSectorPartition": 42, - "ReplaceSectorNumber": 9 - }, - "PreCommitDeposit": "0", - "PreCommitEpoch": 10101, - "DealWeight": "0", - "VerifiedDealWeight": "0" -} -``` - -### StateVerifiedClientStatus -StateVerifiedClientStatus returns the data cap for the given address. -Returns nil if there is no entry in the data cap table for the -address. - - -Perms: read - -Inputs: -```json -[ - "t01234", - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `"0"` - -### StateWaitMsg -StateWaitMsg looks back in the chain for a message. If not found, it blocks until the -message arrives on chain, and gets to the indicated confidence depth. - - -Perms: read - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - 42 -] -``` - -Response: -```json -{ - "Message": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "Receipt": { - "ExitCode": 0, - "Return": "Ynl0ZSBhcnJheQ==", - "GasUsed": 9 - }, - "ReturnDec": {}, - "TipSet": [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ], - "Height": 10101 -} -``` - -## Sync -The Sync method group contains methods for interacting with and -observing the lotus sync service. - - -### SyncCheckBad -SyncCheckBad checks if a block was marked as bad, and if it was, returns -the reason. - - -Perms: read - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } -] -``` - -Response: `"string value"` - -### SyncCheckpoint -SyncCheckpoint marks a blocks as checkpointed, meaning that it won't ever fork away from it. - - -Perms: admin - -Inputs: -```json -[ - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: `{}` - -### SyncIncomingBlocks -SyncIncomingBlocks returns a channel streaming incoming, potentially not -yet synced block headers. - - -Perms: read - -Inputs: `null` - -Response: -```json -{ - "Miner": "t01234", - "Ticket": { - "VRFProof": "Ynl0ZSBhcnJheQ==" - }, - "ElectionProof": { - "WinCount": 9, - "VRFProof": "Ynl0ZSBhcnJheQ==" - }, - "BeaconEntries": null, - "WinPoStProof": null, - "Parents": null, - "ParentWeight": "0", - "Height": 10101, - "ParentStateRoot": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "ParentMessageReceipts": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "Messages": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "BLSAggregate": { - "Type": 2, - "Data": "Ynl0ZSBhcnJheQ==" - }, - "Timestamp": 42, - "BlockSig": { - "Type": 2, - "Data": "Ynl0ZSBhcnJheQ==" - }, - "ForkSignaling": 42, - "ParentBaseFee": "0" -} -``` - -### SyncMarkBad -SyncMarkBad marks a blocks as bad, meaning that it won't ever by synced. -Use with extreme caution. - - -Perms: admin - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } -] -``` - -Response: `{}` - -### SyncState -SyncState returns the current status of the lotus sync system. - - -Perms: read - -Inputs: `null` - -Response: -```json -{ - "ActiveSyncs": null -} -``` - -### SyncSubmitBlock -SyncSubmitBlock can be used to submit a newly created block to the. -network through this node - - -Perms: write - -Inputs: -```json -[ - { - "Header": { - "Miner": "t01234", - "Ticket": { - "VRFProof": "Ynl0ZSBhcnJheQ==" - }, - "ElectionProof": { - "WinCount": 9, - "VRFProof": "Ynl0ZSBhcnJheQ==" - }, - "BeaconEntries": null, - "WinPoStProof": null, - "Parents": null, - "ParentWeight": "0", - "Height": 10101, - "ParentStateRoot": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "ParentMessageReceipts": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "Messages": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "BLSAggregate": { - "Type": 2, - "Data": "Ynl0ZSBhcnJheQ==" - }, - "Timestamp": 42, - "BlockSig": { - "Type": 2, - "Data": "Ynl0ZSBhcnJheQ==" - }, - "ForkSignaling": 42, - "ParentBaseFee": "0" - }, - "BlsMessages": null, - "SecpkMessages": null - } -] -``` - -Response: `{}` - -### SyncUnmarkBad -SyncUnmarkBad unmarks a blocks as bad, making it possible to be validated and synced again. - - -Perms: admin - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } -] -``` - -Response: `{}` - -## Wallet - - -### WalletBalance -WalletBalance returns the balance of the given address at the current head of the chain. - - -Perms: read - -Inputs: -```json -[ - "t01234" -] -``` - -Response: `"0"` - -### WalletDefaultAddress -WalletDefaultAddress returns the address marked as default in the wallet. - - -Perms: write - -Inputs: `null` - -Response: `"t01234"` - -### WalletDelete -WalletDelete deletes an address from the wallet. - - -Perms: write - -Inputs: -```json -[ - "t01234" -] -``` - -Response: `{}` - -### WalletExport -WalletExport returns the private key of an address in the wallet. - - -Perms: admin - -Inputs: -```json -[ - "t01234" -] -``` - -Response: -```json -{ - "Type": "string value", - "PrivateKey": "Ynl0ZSBhcnJheQ==" -} -``` - -### WalletHas -WalletHas indicates whether the given address is in the wallet. - - -Perms: write - -Inputs: -```json -[ - "t01234" -] -``` - -Response: `true` - -### WalletImport -WalletImport receives a KeyInfo, which includes a private key, and imports it into the wallet. - - -Perms: admin - -Inputs: -```json -[ - { - "Type": "string value", - "PrivateKey": "Ynl0ZSBhcnJheQ==" - } -] -``` - -Response: `"t01234"` - -### WalletList -WalletList lists all the addresses in the wallet. - - -Perms: write - -Inputs: `null` - -Response: `null` - -### WalletNew -WalletNew creates a new address in the wallet with the given sigType. - - -Perms: write - -Inputs: -```json -[ - 2 -] -``` - -Response: `"t01234"` - -### WalletSetDefault -WalletSetDefault marks the given address as as the default one. - - -Perms: admin - -Inputs: -```json -[ - "t01234" -] -``` - -Response: `{}` - -### WalletSign -WalletSign signs the given bytes using the given address. - - -Perms: sign - -Inputs: -```json -[ - "t01234", - "Ynl0ZSBhcnJheQ==" -] -``` - -Response: -```json -{ - "Type": 2, - "Data": "Ynl0ZSBhcnJheQ==" -} -``` - -### WalletSignMessage -WalletSignMessage signs the given message using the given address. - - -Perms: sign - -Inputs: -```json -[ - "t01234", - { - "Version": 42, - "To": "t01234", - "From": "t01234", - "Nonce": 42, - "Value": "0", - "GasLimit": 9, - "GasFeeCap": "0", - "GasPremium": "0", - "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" - } -] -``` - -Response: -```json -{ - "Message": { - "Version": 42, - "To": "t01234", - "From": "t01234", - "Nonce": 42, - "Value": "0", - "GasLimit": 9, - "GasFeeCap": "0", - "GasPremium": "0", - "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" - }, - "Signature": { - "Type": 2, - "Data": "Ynl0ZSBhcnJheQ==" - } -} -``` - -### WalletVerify -WalletVerify takes an address, a signature, and some bytes, and indicates whether the signature is valid. -The address does not have to be in the wallet. - - -Perms: read - -Inputs: -```json -[ - "t01234", - "Ynl0ZSBhcnJheQ==", - { - "Type": 2, - "Data": "Ynl0ZSBhcnJheQ==" - } -] -``` - -Response: `true` - diff --git a/documentation/en/building/api-troubleshooting.md b/documentation/en/building/api-troubleshooting.md deleted file mode 100644 index 0cb3a6800..000000000 --- a/documentation/en/building/api-troubleshooting.md +++ /dev/null @@ -1,36 +0,0 @@ -# API Troubleshooting - -## Types: params - -`params` must be an array. If there are no `params` you should still pass an empty array. - -## Types: TipSet - -For methods such as `Filecoin.StateMinerPower`, where the method accepts the argument of the type `TipSet`, you can pass `null` to use the current chain head. - -```sh -curl -X POST \ - -H "Content-Type: application/json" \ - --data '{ "jsonrpc": "2.0", "method": "Filecoin.StateMinerPower", "params": ["t0101", null], "id": 3 }' \ - 'http://127.0.0.1:1234/rpc/v0' -``` - -## Types: Sending a CID - -If you do not serialize the CID as a [JSON IPLD link](https://did-ipid.github.io/ipid-did-method/#txref), you will receive an error. Here is an example of a broken CURL request: - -```sh -curl -X POST \ - -H "Content-Type: application/json" \ - --data '{ "jsonrpc": "2.0", "method":"Filecoin.ClientGetDealInfo", "params": ["bafyreiaxl446wlnu6t6dpq4ivrjf4gda4gvsoi4rr6mpxau7z25xvk5pl4"], "id": 0 }' \ - 'http://127.0.0.1:1234/rpc/v0' -``` - -To fix it, change the `params` property to: - -```sh -curl -X POST \ - -H "Content-Type: application/json" \ - --data '{ "jsonrpc": "2.0", "method":"Filecoin.ClientGetDealInfo", "params": [{"/": "bafyreiaxl446wlnu6t6dpq4ivrjf4gda4gvsoi4rr6mpxau7z25xvk5pl4"}], "id": 0 }' \ - 'http://127.0.0.1:1234/rpc/v0' -``` diff --git a/documentation/en/building/api.md b/documentation/en/building/api.md deleted file mode 100644 index 3a2c2902b..000000000 --- a/documentation/en/building/api.md +++ /dev/null @@ -1,38 +0,0 @@ -# API endpoints and methods - -The API can be accessed on: - -- `http://[api:port]/rpc/v0` - HTTP RPC-API endpoint -- `ws://[api:port]/rpc/v0` - Websocket RPC-API endpoint -- `PUT http://[api:port]/rest/v0/import` - REST endpoint for file import (multipart upload). It requires write permissions. - -The RPC methods can be found in the [Reference](en+api-methods) and directly in the source code: - -- [Both Lotus node + miner APIs](https://github.com/filecoin-project/lotus/blob/master/api/api_common.go) -- [Lotus node API](https://github.com/filecoin-project/lotus/blob/master/api/api_full.go) -- [Lotus miner API](https://github.com/filecoin-project/lotus/blob/master/api/api_storage.go) - - -## JSON-RPC client - -Lotus uses its own Go library implementation of [JSON-RPC](https://github.com/filecoin-project/go-jsonrpc). - -## cURL example - -To demonstrate making an API request, we will take the method `ChainHead` from [api/api_full.go](https://github.com/filecoin-project/lotus/blob/master/api/api_full.go). - -```go -ChainHead(context.Context) (*types.TipSet, error) -``` - -And create a CURL command. In this command, `ChainHead` is included as `{ "method": "Filecoin.ChainHead" }`: - -```sh -curl -X POST \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer $(cat ~/.lotusminer/token)" \ - --data '{ "jsonrpc": "2.0", "method": "Filecoin.ChainHead", "params": [], "id": 3 }' \ - 'http://127.0.0.1:1234/rpc/v0' -``` - -(See [this section](en+remote-api) to learn how to generate authorization tokens). diff --git a/documentation/en/building/building.md b/documentation/en/building/building.md deleted file mode 100644 index 5194f8314..000000000 --- a/documentation/en/building/building.md +++ /dev/null @@ -1,5 +0,0 @@ -# Building with Lotus - -Lotus applications provide HTTP (JSON-RPC) APIs that allow developers to control Lotus programatically. - -This section dives into how to setup and use these APIs, additionally providing information on advanced Lotus features and workflows, like Payment Channels or how to setup a fully local Lotus development network. diff --git a/documentation/en/building/jaeger-tracing.md b/documentation/en/building/jaeger-tracing.md deleted file mode 100644 index bbe4d3052..000000000 --- a/documentation/en/building/jaeger-tracing.md +++ /dev/null @@ -1,26 +0,0 @@ -# Jaeger Tracing - -Lotus has tracing built into many of its internals. To view the traces, first download [Jaeger](https://www.jaegertracing.io/download/) (Choose the 'all-in-one' binary). Then run it somewhere, start up the lotus daemon, and open up localhost:16686 in your browser. - -## Open Census - -Lotus uses [OpenCensus](https://opencensus.io/) for tracing application flow. This generates spans through the execution of annotated code paths. - -Currently it is set up to use Jaeger, though other tracing backends should be fairly easy to swap in. - -## Running Locally - -To easily run and view tracing locally, first, install jaeger. The easiest way to do this is to [download the binaries](https://www.jaegertracing.io/download/) and then run the `jaeger-all-in-one` binary. This will start up jaeger, listen for spans on `localhost:6831`, and expose a web UI for viewing traces on `http://localhost:16686/`. - -Now, to start sending traces from Lotus to Jaeger, set the environment variable `LOTUS_JAEGER` to `localhost:6831`, and start the `lotus daemon`. - -Now, to view any generated traces, open up `http://localhost:16686/` in your browser. - -## Adding Spans - -To annotate a new codepath with spans, add the following lines to the top of the function you wish to trace: - -```go -ctx, span := trace.StartSpan(ctx, "put function name here") -defer span.End() -``` diff --git a/documentation/en/building/local-devnet.md b/documentation/en/building/local-devnet.md deleted file mode 100644 index 3382b6471..000000000 --- a/documentation/en/building/local-devnet.md +++ /dev/null @@ -1,54 +0,0 @@ -# Setup Local Devnet - -Build the Lotus Binaries in debug mode, This enables the use of 2048 byte sectors. - -```sh -make 2k -``` - -Set the `LOTUS_SKIP_GENESIS_CHECK` environment variable to `_yes_`. This tells your -Lotus node that it's okay if the genesis being used doesn't match any baked-in -genesis. - -```sh -export LOTUS_SKIP_GENESIS_CHECK=_yes_ -``` - -Download the 2048 byte parameters: -```sh -./lotus fetch-params 2048 -``` - -Pre-seal some sectors: - -```sh -./lotus-seed pre-seal --sector-size 2KiB --num-sectors 2 -``` - -Create the genesis block and start up the first node: - -```sh -./lotus-seed genesis new localnet.json -./lotus-seed genesis add-miner localnet.json ~/.genesis-sectors/pre-seal-t01000.json -./lotus daemon --lotus-make-genesis=devgen.car --genesis-template=localnet.json --bootstrap=false -``` - -Then, in another console, import the genesis miner key: - -```sh -./lotus wallet import --as-default ~/.genesis-sectors/pre-seal-t01000.key -``` - -Set up the genesis miner: - -```sh -./lotus-miner init --genesis-miner --actor=t01000 --sector-size=2KiB --pre-sealed-sectors=~/.genesis-sectors --pre-sealed-metadata=~/.genesis-sectors/pre-seal-t01000.json --nosync -``` - -Now, finally, start up the miner: - -```sh -./lotus-miner run --nosync -``` - -If all went well, you will have your own local Lotus Devnet running. diff --git a/documentation/en/building/payment-channels.md b/documentation/en/building/payment-channels.md deleted file mode 100644 index afddcdc40..000000000 --- a/documentation/en/building/payment-channels.md +++ /dev/null @@ -1,111 +0,0 @@ -# Payment Channels - -Payment channels are used to transfer funds between two actors. - -For example in lotus a payment channel is created when a client wants to fetch data from a provider. -The client sends vouchers for the payment channel, and the provider sends data in response. - -The payment channel is created on-chain with an initial amount. -Vouchers allow the client and the provider to exchange funds incrementally off-chain. -The provider can submit vouchers to chain at any stage. -Either party to the payment channel can settle the payment channel on chain. -After a settlement period (currently 12 hours) either party to the payment channel can call collect on chain. -Collect sends the value of submitted vouchers to the channel recipient (the provider), and refunds the remaining channel balance to the channel creator (the client). - -Vouchers have a lane, a nonce and a value, where vouchers with a higher nonce supersede vouchers with a lower nonce in the same lane. -Each deal is created on a different lane. - -Note that payment channels and vouchers can be used for any situation in which two parties need to incrementally transfer value between each other off-chain. - -## Using the CLI - -For example a client creates a payment channel to a provider with value 10 FIL. - -```sh -$ lotus paych add-funds 10 - -``` - -The client creates a voucher in lane 0 (implied) with nonce 1 (implied) and value 2. - -```sh -$ lotus paych voucher create 2 - -``` - -The client sends the voucher to the provider and the provider adds the voucher to their local store. - -```sh -$ lotus paych voucher add -``` - -The provider sends some data to the client. - -The client creates a voucher in lane 0 (implied) with nonce 2 (implied) and value 4. - -```sh -$ lotus paych voucher create 4 - -``` - -The client sends the voucher to the provider and the provider adds the voucher and sends back more data. -etc. - -The client can add value to the channel after it has been created by calling `paych add-funds` with the same client and provider addresses. - -```sh -$ lotus paych add-funds 5 - # Same address as above. Channel now has 15 -``` - -Once the client has received all their data, they may settle the channel. -Note that settlement doesn't have to be done immediately. -For example the client may keep the channel open as long as it wants to continue making deals with the provider. - -```sh -$ lotus paych settle -``` - -The provider can submit vouchers to chain (note that lotus does this automatically when it sees a settle message appear on chain). -The provider may have received many vouchers with incrementally higher values. -The provider should submit the best vouchers. Note that there will be one best voucher for each lane. - -```sh -$ lotus paych voucher best-spendable - - - - -$ lotus paych voucher submit -``` - -Once the settlement period is over, either the client or provider can call collect to disburse funds. - -```sh -$ lotus paych collect -``` - -Check the status of a channel that is still being created using `lotus paych status-by-from-to`. - -```sh -$ lotus paych status-by-from-to -Creating channel - From: t3sb6xzvs6rhlziatagevxpp3dwapdolurtkpn4kyh3kgoo4tn5o7lutjqlsnvpceztlhxu3lzzfe34rvpsjgq - To: t1zip4sblhyrn4oxygzsm6nafbsynp2avmk3xafea - Pending Amt: 10000 - Wait Sentinel: bafy2bzacedk2jidsyxcynusted35t5ipkhu2kpiodtwyjr3pimrhke6f5pqbm -``` - -Check the status of a channel that has been created using `lotus paych status`. - -```sh -$ lotus paych status -Channel exists - Channel: t2nydpzhmeqkmid5smtqnowlr2mr5az6rexpmyv6i - From: t3sb6xzvs6rhlziatagevxpp3dwapdolurtkpn4kyh3kgoo4tn5o7lutjqlsnvpceztlhxu3lzzfe34rvpsjgq - To: t1zip4sblhyrn4oxygzsm6nafbsynp2avmk3xafea - Confirmed Amt: 10000 - Pending Amt: 6000 - Queued Amt: 3000 - Voucher Redeemed Amt: 2000 -``` diff --git a/documentation/en/building/remote-api.md b/documentation/en/building/remote-api.md deleted file mode 100644 index d0fedb51b..000000000 --- a/documentation/en/building/remote-api.md +++ /dev/null @@ -1,69 +0,0 @@ -# Setting up remote API access - -The **Lotus Miner** and the **Lotus Node** applications come with their own local API endpoints setup by default when they are running. - -These endpoints are used by `lotus` and `lotus-miner` to interact with the running process. In this section we will explain how to enable remote access to the Lotus APIs. - -Note that instructions are the same for `lotus` and `lotus-miner`. For simplicity, we will just show how to do it with `lotus`. - -## Setting the listening interface for the API endpoint - -By default, the API listens on the local "loopback" interface (`127.0.0.1`). This is configured in the `config.toml` file: - -```toml -[API] -# ListenAddress = "/ip4/127.0.0.1/tcp/1234/http" -# RemoteListenAddress = "" -# Timeout = "30s" -``` - -To access the API remotely, Lotus needs to listen on the right IP/interface. The IP associated to each interface can be usually found with the command `ip a`. Once the right IP is known, it can be set in the configuration: - -```toml -[API] -ListenAddress = "/ip4//tcp/3453/http" # port is an example - -# Only relevant for lotus-miner -# This should be the IP:Port pair where the miner is reachable from anyone trying to dial to it. -# If you have placed a reverse proxy or a NAT'ing device in front of it, this may be different from -# the EXTERNAL_INTERFACE_IP. -RemoteListenAddress = "" -``` - -> `0.0.0.0` can be used too. This is a wildcard that means "all interfaces". Depending on the network setup, this may affect security (listening on the wrong, exposed interface). - -After making these changes, please restart the affected process. - -## Issuing tokens - -Any client wishing to talk to the API endpoints will need a token. Tokens can be generated with: - -```sh -lotus auth create-token --perm -``` - -(similarly for the Lotus Miner). - -The permissions work as follows: - -- `read` - Read node state, no private data. -- `write` - Write to local store / chain, and `read` permissions. -- `sign` - Use private keys stored in wallet for signing, `read` and `write` permissions. -- `admin` - Manage permissions, `read`, `write`, and `sign` permissions. - - -Tokens can then be used in applications by setting an Authorization header as: - -``` -Authorization: Bearer -``` - - -## Environment variables - -`lotus`, `lotus-miner` and `lotus-worker` can actually interact with their respective applications running on a different node. All is needed to configure them are the following the *environment variables*: - -```sh -FULLNODE_API_INFO="TOKEN:/ip4//tcp//http" -MINER_API_INFO="TOKEN:/ip4//tcp//http" -``` diff --git a/documentation/en/getting-started/getting-started.md b/documentation/en/getting-started/getting-started.md deleted file mode 100644 index 99b4095d4..000000000 --- a/documentation/en/getting-started/getting-started.md +++ /dev/null @@ -1,3 +0,0 @@ -# Getting started - -This section will get you started with Lotus. We will setup the Lotus daemon (that should already be [installed](en+install)), start it, create a wallet and use it to send and receive some Filecoin. diff --git a/documentation/en/getting-started/setup-troubleshooting.md b/documentation/en/getting-started/setup-troubleshooting.md deleted file mode 100644 index f27a3faa5..000000000 --- a/documentation/en/getting-started/setup-troubleshooting.md +++ /dev/null @@ -1,57 +0,0 @@ -# Setup Troubleshooting - - -## Error: initializing node error: cbor input had wrong number of fields - -This happens when you are starting Lotus which has been compiled for one network, but it encounters data in the Lotus data folder which is for a different network, or for an older incompatible version. - -The solution is to clear the data folder (see below). - -## Config: Clearing data - -Here is a command that will delete your chain data, stored wallets, stored data and any miners you have set up: - -```sh -rm -rf ~/.lotus ~/.lotusminer -``` - -Note you do not always need to clear your data for [updating](en+update). - -## Error: Failed to connect bootstrap peer - -```sh -WARN peermgr peermgr/peermgr.go:131 failed to connect to bootstrap peer: failed to dial : all dials failed - * [/ip4/147.75.80.17/tcp/1347] failed to negotiate security protocol: connected to wrong peer -``` - -- Try running the build steps again and make sure that you have the latest code from GitHub. - -```sh -ERROR hello hello/hello.go:81 other peer has different genesis! -``` - -- Try deleting your file system's `~/.lotus` directory. Check that it exists with `ls ~/.lotus`. - -```sh -- repo is already locked -``` - -- You already have another lotus daemon running. - -## Config: Open files limit - -Lotus will attempt to set up the file descriptor (FD) limit automatically. If that does not work, you can still configure your system to allow higher than the default values. - -On most systems you can check the open files limit with: - -```sh -ulimit -n -``` - -You can also modify this number by using the `ulimit` command. It gives you the ability to control the resources available for the shell or process started by it. If the number is below 10000, you can change it with the following command prior to starting the Lotus daemon: - -```sh -ulimit -n 10000 -``` - -Note that this is not persisted and that systemd manages its own FD limits for services. Please use your favourite search engine to find instructions on how to persist and configure FD limits for your system. diff --git a/documentation/en/getting-started/setup.md b/documentation/en/getting-started/setup.md deleted file mode 100644 index e751da80b..000000000 --- a/documentation/en/getting-started/setup.md +++ /dev/null @@ -1,169 +0,0 @@ -# Setting up Lotus - -Your Lotus binaries have been installed and you are ready to start participating in the Filecoin network. - -## Selecting the right network - -You should have built the Lotus binaries from the right Github branch and Lotus will be fully setup to join the matching [Filecoin network](https://docs.filecoin.io/how-to/networks/). For more information on switching networks, check the [updating Lotus section](en+update). - -## Starting the daemon - -To start the daemon simply run: - -```sh -lotus daemon -``` - -or if you are using the provided systemd service files, do: - -```sh -systemctl start lotus-daemon -``` - -__If you are using Lotus from China__, make sure you set the following environment variable before running Lotus: - -``` -export IPFS_GATEWAY="https://proof-parameters.s3.cn-south-1.jdcloud-oss.com/ipfs/" -``` - - -During the first start, Lotus: - -* Will setup its data folder at `~/.lotus` -* Will download the necessary parameters -* Start syncing the Lotus chain - -If you started lotus using systemd, the logs will appear in `/var/log/lotus/daemon.log` (not in journalctl as usual), otherwise you will see them in your screen. - -Do not be appalled by the amount of warnings and sometimes errors showing in the logs, there are usually part of the usual functioning of the daemon as part of a distributed network. - -## Waiting to sync - -After the first start, the chain will start syncing until it has reached the tip. You can check how far the syncing process is with: - -```sh -lotus sync status -``` - -You can also interactively wait for the chain to be fully synced with: - -```sh -lotus sync wait -``` - -## Interacting with the Lotus daemon - -As shown above, the `lotus` command allows to interact with the running daemon. You will see it getting used in many of the documentation examples. - -This command-line-interface is self-documenting: - -```sh -# Show general help -lotus --help -# Show specific help for the "client" subcommand -lotus client --help -``` - -For example, after your Lotus daemon has been running for a few minutes, use `lotus` to check the number of other peers that it is connected to in the Filecoin network: - -```sh -lotus net peers -``` - -## Controlling the logging level - -```sh -lotus log set-level -``` -This command can be used to toggle the logging levels of the different -systems of a Lotus node. In decreasing order -of logging detail, the levels are `debug`, `info`, `warn`, and `error`. - -As an example, -to set the `chain` and `blocksync` to log at the `debug` level, run -`lotus log set-level --system chain --system blocksync debug`. - -To see the various logging system, run `lotus log list`. - - -## Configuration - -### Configuration file - -The Lotus daemon stores a configuration file in `~/.lotus/config.toml`. Note that by default all settings are commented. Here is an example configuration: - -```toml -[API] - # Binding address for the Lotus API - ListenAddress = "/ip4/127.0.0.1/tcp/1234/http" - # Not used by lotus daemon - RemoteListenAddress = "" - # General network timeout value - Timeout = "30s" - -# Libp2p provides connectivity to other Filecoin network nodes -[Libp2p] - # Binding address swarm - 0 means random port. - ListenAddresses = ["/ip4/0.0.0.0/tcp/0", "/ip6/::/tcp/0"] - # Insert any addresses you want to explicitally - # announce to other peers here. Otherwise, they are - # guessed. - AnnounceAddresses = [] - # Insert any addresses to avoid announcing here. - NoAnnounceAddresses = [] - # Connection manager settings, decrease if your - # machine is overwhelmed by connections. - ConnMgrLow = 150 - ConnMgrHigh = 180 - ConnMgrGrace = "20s" - -# Pubsub is used to broadcast information in the network -[Pubsub] - Bootstrapper = false - RemoteTracer = "/dns4/pubsub-tracer.filecoin.io/tcp/4001/p2p/QmTd6UvR47vUidRNZ1ZKXHrAFhqTJAD27rKL9XYghEKgKX" - -# This section can be used to enable adding and retriving files from IPFS -[Client] - UseIpfs = false - IpfsMAddr = "" - IpfsUseForRetrieval = false - -# Metrics configuration -[Metrics] - Nickname = "" - HeadNotifs = false -``` - -### Ensuring connectivity to your Lotus daemon - -Usually your lotus daemon will establish connectivity with others in the network and try to make itself diallable using uPnP. If you wish to manually ensure that your daemon is reachable: - -* Set a fixed port of your choice in the `ListenAddresses` in the Libp2p section (i.e. 6665). -* Open a port in your router that is forwarded to this port. This is usually called featured as "Port forwarding" and the instructions differ from router model to model but there are many guides online. -* Add your public IP/port to `AnnounceAddresses`. i.e. `/ip4//tcp/6665/`. - -Note that it is not a requirement to use Lotus as a client to the network to be fully reachable, as your node already connects to others directly. - - -### Environment variables - -Common to most Lotus binaries: - -* `LOTUS_FD_MAX`: Sets the file descriptor limit for the process -* `LOTUS_JAEGER`: Sets the Jaeger URL to send traces. See TODO. -* `LOTUS_DEV`: Any non-empty value will enable more verbose logging, useful only for developers. - -Specific to the *Lotus daemon*: - -* `LOTUS_PATH`: Location to store Lotus data (defaults to `~/.lotus`). -* `LOTUS_SKIP_GENESIS_CHECK=_yes_`: Set only if you wish to run a lotus network with a different genesis block. -* `LOTUS_CHAIN_TIPSET_CACHE`: Sets the size for the chainstore tipset cache. Defaults to `8192`. Increase if you perform frequent arbitrary tipset lookups. -* `LOTUS_CHAIN_INDEX_CACHE`: Sets the size for the epoch index cache. Defaults to `32768`. Increase if you perform frequent deep chain lookups for block heights far from the latest height. -* `LOTUS_BSYNC_MSG_WINDOW`: Set the initial maximum window size for message fetching blocksync request. Set to 10-20 if you have an internet connection with low bandwidth. - -Specific to the *Lotus miner*: - -* `LOTUS_MINER_PATH`: Location for the miner's on-disk repo. Defaults to `./lotusminer`. -* A number of environment variables are respected for configuring the behaviour of the Filecoin proving subsystem. [See here](en+miner-setup). - - diff --git a/documentation/en/getting-started/wallet.md b/documentation/en/getting-started/wallet.md deleted file mode 100644 index 25a67fb09..000000000 --- a/documentation/en/getting-started/wallet.md +++ /dev/null @@ -1,58 +0,0 @@ -# Obtaining and sending FIL - -In order to receive and send FIL with Lotus you will need to have installed the program and be running the Lotus daemon. - -## Creating a wallet - - -```sh -lotus wallet new bls -``` - -This will print your Filecoin address. - -Your wallet information is stored in the `~/.lotus/keystore` (or `$LOTUS_PATH/keystore`). For instructions on export/import, see below. - -You can create multiple wallets and list them with: - -```sh -lotus wallet list -``` - -## Obtaining FIL - -FIL can be obtained either by using one of the Faucets (available for the test networks) or by buying it from an exchange supporting FIL trading (once mainnet has launched). - -Once you have received some FIL you can check your balance with: - -```sh -lotus wallet balance -``` - -Remember that your will only see the latest balance when your daemon is fully synced to the chain. - -## Sending FIL - -Sending some FIL can be achieved by running: - -```sh -lotus wallet send
-``` - -Make sure to check `lotus wallet send --help` for additional options. - -## Exporting and importing a wallet - -You can export and re-import a wallet with: - -```sh -lotus wallet export
> wallet.private -``` - -and: - -```sh -lotus wallet import wallet.private -``` - -Keep your wallet's private key safe! diff --git a/documentation/en/installation/install-linux.md b/documentation/en/installation/install-linux.md deleted file mode 100644 index 6fe12996e..000000000 --- a/documentation/en/installation/install-linux.md +++ /dev/null @@ -1,129 +0,0 @@ -# Linux installation - -This page will show you the steps to build and install Lotus in your Linux computer. - -## Dependencies - -### System dependencies - -First of all, building Lotus will require installing some system dependencies, usually provided by your distribution. - -For Arch Linux: - -```sh -sudo pacman -Syu opencl-icd-loader gcc git bzr jq pkg-config opencl-icd-loader opencl-headers -``` - -For Ubuntu: - -```sh -sudo apt update -sudo apt install mesa-opencl-icd ocl-icd-opencl-dev gcc git bzr jq pkg-config curl -sudo apt upgrade -``` - -For Fedora: - -```sh -sudo dnf -y update -sudo dnf -y install gcc git bzr jq pkgconfig mesa-libOpenCL mesa-libOpenCL-devel opencl-headers ocl-icd ocl-icd-devel clang llvm -``` - -For OpenSUSE: - -```sh -sudo zypper in gcc git jq make libOpenCL1 opencl-headers ocl-icd-devel clang llvm -sudo ln -s /usr/lib64/libOpenCL.so.1 /usr/lib64/libOpenCL.so -``` - -### Rustup - -Lotus needs [rustup](https://rustup.rs/): - -```sh -curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -``` - -Please make sure your `$PATH` variable is correctly configured after the rustup installation so that `cargo` and `rustc` are found in their rustup-configured locations. - -### Go - -To build lotus you will need a working installation of **[Go1.14](https://golang.org/dl/)**. Follow the [installation instructions](https://golang.org/doc/install), which generally amount to: - -```sh -# Example! Check the installation instructions. -wget -c https://dl.google.com/go/go1.14.7.linux-amd64.tar.gz -O - | sudo tar -xz -C /usr/local -``` - -## Build and install Lotus - -With all the above, you are ready to build and install the Lotus suite (`lotus`, `lotus-miner` and `lotus-worker`): - -```sh -git clone https://github.com/filecoin-project/lotus.git -cd lotus/ -``` - -__IF YOU ARE IN CHINA__, set `export GOPROXY=https://goproxy.cn` before building - -Now, choose the network that you will be joining: - -* For `testnet`: `git checkout master` -* For `nerpa`: `git checkout ntwk-nerpa` -* For `butterfly`: `git checkout ntwk-butterfly` - -Once on the right branch, do: - -```sh -make clean install -sudo make install -``` - -This will put `lotus`, `lotus-miner` and `lotus-worker` in `/usr/local/bin`. `lotus` will use the `$HOME/.lotus` folder by default for storage (configuration, chain data, wallets...). `lotus-miner` will use `$HOME/.lotusminer` respectively. See the *environment variables* section below for how to customize these. - -> Remeber to [move your Lotus folder](en+update) if you are switching between different networks, or there has been a network reset. - - -### Native Filecoin FFI - -Some newer processors (AMD Zen (and later), Intel Ice Lake) have support SHA extensions. To make full use of your processor's capabilities, make sure you set the following variables BEFORE building from source (as described above): - -```sh -export RUSTFLAGS="-C target-cpu=native -g" -export FFI_BUILD_FROM_SOURCE=1 -``` - -> __NOTE__: This method of building does not produce portable binaries! Make sure you run the binary in the same machine as you built it. - -### systemd service files - -Lotus provides Systemd service files. They can be installed with: - -```sh -make install-daemon-service -make install-miner-service -``` - -After that, you should be able to control Lotus using `systemctl`. - -## Troubleshooting - -This section mentions some of the common pitfalls for building Lotus. Check the [getting started](en+getting-started) section for more tips on issues when running the lotus daemon. - -### Build errors - -Please check the build logs closely. If you have a dirty state in your git branch make sure to: - -```sh -git checkout -git reset origin/ --hard -make clean -``` - -### Slow builds from China - -Users from China can speed up their builds by setting: - -```sh -export GOPROXY=https://goproxy.cn -``` diff --git a/documentation/en/installation/install-macos.md b/documentation/en/installation/install-macos.md deleted file mode 100644 index ea9ecb8ca..000000000 --- a/documentation/en/installation/install-macos.md +++ /dev/null @@ -1,62 +0,0 @@ -# MacOS Instructions - -## Get XCode Command Line Tools - -To check if you already have the XCode Command Line Tools installed via the CLI, run: - -```sh -xcode-select -p -``` - -If this command returns a path, you can move on to the next step. Otherwise, to install via the CLI, run: - -```sh -xcode-select --install -``` - -To update, run: - -```sh -sudo rm -rf /Library/Developer/CommandLineTools -xcode-select --install -``` - -## Get HomeBrew - -We recommend that MacOS users use [HomeBrew](https://brew.sh) to install each the necessary packages. - -Check if you have HomeBrew: - -```sh -brew -v -``` - -This command returns a version number if you have HomeBrew installed and nothing otherwise. - -In your terminal, enter this command to install Homebrew: - -```sh -/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" -``` - -Use the command `brew install` to install the following packages: - -```sh -brew install go bzr jq pkg-config rustup -``` - -Clone - -```sh -git clone https://github.com/filecoin-project/lotus.git -cd lotus/ -``` - -Build - -```sh -make clean && make all -sudo make install -``` - -After intalling Lotus you will be ready to [setup and run the daemon](en+setup.md). diff --git a/documentation/en/installation/installation.md b/documentation/en/installation/installation.md deleted file mode 100644 index 98534da92..000000000 --- a/documentation/en/installation/installation.md +++ /dev/null @@ -1,39 +0,0 @@ -# Installation - -Lotus can be installed in [Linux](en-install-linux) and [MacOS](en-install-macos) machines by building it from source. Windows is not supported yet. - -This section contains guides to install Lotus in the supported platforms. - -Lotus is made of 3 binaries: - -* `lotus`: the main [Lotus node](en+setup) (Lotus client) -* `lotus-miner`: an application specifically for [Filecoin mining](en+miner-setup) -* `lotus-worker`: an additional [application to offload some heavy-processing tasks](en+lotus-worker) from the Lotus Miner. - -These applications are written in Go, but also import several Rust libraries. Lotus does not distribute -pre-compiled builds. - -## Hardware requirements - -### For client nodes - -* 8GiB of RAM -* Recommended for syncing speed: CPU with support for *Intel SHA Extensions* (AMD since Zen microarchitecture, Intel since Ice Lake). -* Recommended for speed: SSD hard drive (the bigger the better) - -### For miners - -The following correspond to the latest testing configuration: - -* 2 TB of hard drive space -* 8 core CPU -* 128 GiB of RAM with 256 GiB of NVMe SSD storage for swap (or simply, more RAM). -* Recommended for speed: CPU with support for *Intel SHA Extensions* (AMD since Zen microarchitecture, Intel since Ice Lake). -* GPU for block mining. The following have been [confirmed to be fast enough](en+gpus): - -- GeForce RTX 2080 Ti -- GeForce RTX 2080 SUPER -- GeForce RTX 2080 -- GeForce GTX 1080 Ti -- GeForce GTX 1080 -- GeForce GTX 1060 diff --git a/documentation/en/installation/update.md b/documentation/en/installation/update.md deleted file mode 100644 index 5d76592c9..000000000 --- a/documentation/en/installation/update.md +++ /dev/null @@ -1,72 +0,0 @@ -# Updating and restarting Lotus - -Updating Lotus is as simple as rebuilding and re-installing the software as explained in the previous sections. - -You can verify which version of Lotus you are running with: - -```sh -lotus version -``` - -Make sure that you `git pull` the branch that corresponds to the network that your Lotus daemon is using: - -```sh -git pull origin -make clean -make all -sudo make install # if necessary -``` - -Finally, restart the Lotus Node and/or Lotus Miner(s). - -__CAVEAT__: If you are running miners: check if your miner is safe to shut down and restart: `lotus-miner proving info`. If any deadline shows a block height in the past, do not restart: - -In the following example, Deadline Open is 454 which is earlier than Current Epoch of 500. This miner should **not** be shut down or restarted. - -``` -$ sudo lotus-miner proving info -Miner: t01001 -Current Epoch: 500 -Proving Period Boundary: 154 -Proving Period Start: 154 (2h53m0s ago) -Next Period Start: 3034 (in 21h7m0s) -Faults: 768 (100.00%) -Recovering: 768 -Deadline Index: 5 -Deadline Sectors: 0 -Deadline Open: 454 (23m0s ago) -Deadline Close: 514 (in 7m0s) -Deadline Challenge: 434 (33m0s ago) -Deadline FaultCutoff: 384 (58m0s ago) -``` - -In this next example, the miner can be safely restarted because no Deadlines are earlier than Current Epoch of 497. You have ~45 minutes before the miner must be back online to declare faults (FaultCutoff). If the miner has no faults, you have about an hour. - -``` -$ sudo lotus-miner proving info -Miner: t01000 -Current Epoch: 497 -Proving Period Boundary: 658 -Proving Period Start: 658 (in 1h20m30s) -Next Period Start: 3538 (in 25h20m30s) -Faults: 0 (0.00%) -Recovering: 0 -Deadline Index: 0 -Deadline Sectors: 768 -Deadline Open: 658 (in 1h20m30s) -Deadline Close: 718 (in 1h50m30s) -Deadline Challenge: 638 (in 1h10m30s) -Deadline FaultCutoff: 588 (in 45m30s) -``` - -## Switching networks and network resets - -If you wish to switch to a different lotus network or there has been a network reset, you will need to: - -* Checkout the appropiate repository branch and rebuild -* Ensure you do not mix Lotus data (`LOTUS_PATH`, usually `~/.lotus`) from a previous or different network. For this, either: - * Rename the folder to something else or, - * Set a different `LOTUS_PATH` for the new network. -* Same for `~/.lotusminer` if you are running a miner. - -Note that deleting the Lotus data folder will wipe all the chain data, wallets and configuration, so think twice before taking any non-reversible action. diff --git a/documentation/en/mining/gpus.md b/documentation/en/mining/gpus.md deleted file mode 100644 index ad0ed4f66..000000000 --- a/documentation/en/mining/gpus.md +++ /dev/null @@ -1,17 +0,0 @@ -# Benchmarking additional GPUs - -If you want to test a GPU that is not explicitly supported, set the following *environment variable*: - -```sh -BELLMAN_CUSTOM_GPU=":" -``` - -Here is an example of trying a GeForce GTX 1660 Ti with 1536 cores. - -```sh -BELLMAN_CUSTOM_GPU="GeForce GTX 1660 Ti:1536" -``` - -To get the number of cores for your GPU, you will need to check your card’s specifications. - -To perform the benchmark you can use Lotus' [benchmarking tool](https://github.com/filecoin-project/lotus/tree/master/cmd/lotus-bench). Results and discussion are tracked in a [GitHub issue thread](https://github.com/filecoin-project/lotus/issues/694). diff --git a/documentation/en/mining/lotus-seal-worker.md b/documentation/en/mining/lotus-seal-worker.md deleted file mode 100644 index 47e201ca5..000000000 --- a/documentation/en/mining/lotus-seal-worker.md +++ /dev/null @@ -1,99 +0,0 @@ -# Lotus Worker - -The **Lotus Worker** is an extra process that can offload heavy processing tasks from your **Lotus Miner**. The sealing process automatically runs in the **Lotus Miner** process, but you can use the Worker on another machine communicating over a fast network to free up resources on the machine running the mining process. - -## Installation - -The `lotus-worker` application is installed along with the others when running `sudo make install` as shown in the [Installation section](en+install-linux). For simplicity, we recommend following the same procedure in the machines that will run the Lotus Workers (even if the Lotus miner and the Lotus daemon are not used there). - -## Setting up the Miner - -### Allow external connections to the miner API - -First, you will need to ensure your `lotus-miner`'s API is accessible over the network. - -To do this, open up `~/.lotusminer/config.toml` (Or if you manually set `LOTUS_MINER_PATH`, look under that directory) and look for the API field. - -Default config: - -```toml -[API] -ListenAddress = "/ip4/127.0.0.1/tcp/2345/http" -RemoteListenAddress = "127.0.0.1:2345" -``` - -To make your node accessible over the local area network, you will need to determine your machine's IP on the LAN (`ip a`), and change the `127.0.0.1` in the file to that address. - -A more permissive and less secure option is to change it to `0.0.0.0`. This will allow anyone who can connect to your computer on that port to access the miner's API, though they will still need an auth token. - -`RemoteListenAddress` must be set to an address which other nodes on your network will be able to reach. - -### Create an authentication token - -Write down the output of: - -```sh -lotus-miner auth api-info --perm admin -``` - -The Lotus Workers will need this token to connect to the miner. - -## Connecting the Lotus Workers - -On each machine that will run the `lotus-worker` application you will need to define the following *environment variable*: - -```sh -export MINER_API_INFO::/ip4//tcp/2345` -``` - -If you are trying to use `lotus-worker` from China. You should additionally set: - -```sh -export IPFS_GATEWAY="https://proof-parameters.s3.cn-south-1.jdcloud-oss.com/ipfs/" -``` - - -Once that is done, you can run the Worker with: - -```sh -lotus-worker run -``` - -> If you are running multiple workers on the same host, you will need to specify the `--listen` flag and ensure each worker is on a different port. - -On your Lotus miner, check that the workers are correctly connected: - -```sh -lotus-miner sealing workers -Worker 0, host computer - CPU: [ ] 0 core(s) in use - RAM: [|||||||||||||||||| ] 28% 18.1 GiB/62.7 GiB - VMEM: [|||||||||||||||||| ] 28% 18.1 GiB/62.7 GiB - GPU: GeForce RTX 2080, not used - -Worker 1, host othercomputer - CPU: [ ] 0 core(s) in use - RAM: [|||||||||||||| ] 23% 14 GiB/62.7 GiB - VMEM: [|||||||||||||| ] 23% 14 GiB/62.7 GiB - GPU: GeForce RTX 2080, not used -``` - -## Running locally for manually managing process priority - -You can also run the **Lotus Worker** on the same machine as your **Lotus Miner**, so you can manually manage the process priority. - -To do so you have to first __disable all seal task types__ in the miner config. This is important to prevent conflicts between the two processes: - -```toml -[Storage] - AllowPreCommit1 = false - AllowPreCommit2 = false - AllowCommit = false - AllowUnseal = false -``` - -You can then run the miner on your local-loopback interface; - -```sh -lotus-worker run -``` diff --git a/documentation/en/mining/managing-deals.md b/documentation/en/mining/managing-deals.md deleted file mode 100644 index 5f73a6a2d..000000000 --- a/documentation/en/mining/managing-deals.md +++ /dev/null @@ -1,19 +0,0 @@ -# Managing deals - - -While the Lotus Miner is running as a daemon, the `lotus-miner` application can be used to manage and configure the miner: - - -```sh -lotus-miner storage-deals --help -``` - -Running the above command will show the different options related to deals. For example, `lotus-miner storage-deals set-ask` allows to set the price for storage that your miner uses to respond ask requests from clients. - -If deals are ongoing, you can check the data transfers with: - -```sh -lotus-miner data-transfers list -``` - -Make sure you explore the `lotus-miner` CLI. Every command is self-documented and takes a `--help` flag that offers specific information about it. diff --git a/documentation/en/mining/miner-setup.md b/documentation/en/mining/miner-setup.md deleted file mode 100644 index cafa1e7b1..000000000 --- a/documentation/en/mining/miner-setup.md +++ /dev/null @@ -1,241 +0,0 @@ -# Miner setup - -This page will guide you through all you need to know to sucessfully run a **Lotus Miner**. Before proceeding, remember that you should be running the Lotus daemon on a fully synced chain. - -## Performance tweaks - -This is a list of performance tweaks to consider before starting the miner: - -### Building - -As [explained already](en+install-linux#native-filecoin-ffi-10) should have exported the following variables before building the Lotus applications: - -```sh -export RUSTFLAGS="-C target-cpu=native -g" -export FFI_BUILD_FROM_SOURCE=1 -``` - -### Environment - -For high performance mining, we recommend setting the following variables in your environment so that they are available when running any of the Lotus applications: - -```sh -# See https://github.com/filecoin-project/bellman -export BELLMAN_CPU_UTILIZATION=0.875 - -# See https://github.com/filecoin-project/rust-fil-proofs/ -export FIL_PROOFS_MAXIMIZE_CACHING=1 # More speed at RAM cost (1x sector-size of RAM - 32 GB). -export FIL_PROOFS_USE_GPU_COLUMN_BUILDER=1 # precommit2 GPU acceleration -export FIL_PROOFS_USE_GPU_TREE_BUILDER=1 -``` - -IF YOU ARE RUNNING FROM CHINA: - -```sh -export IPFS_GATEWAY="https://proof-parameters.s3.cn-south-1.jdcloud-oss.com/ipfs/" -``` - -IF YOUR MINER RUNS IN A DIFFERENT MACHINE AS THE LOTUS DAEMON: - -```sh -export FULLNODE_API_INFO=:/ip4//tcp//http -``` - -If you will be using systemd service files to run the Lotus daemon and miner, make sure you include these variables manually in the service files. - -### Adding swap - -If you have only 128GiB of RAM, you will need to make sure your system provides at least an extra 256GiB of fast swap (preferably NVMe SSD): - -```sh -sudo fallocate -l 256G /swapfile -sudo chmod 600 /swapfile -sudo mkswap /swapfile -sudo swapon /swapfile -# show current swap spaces and take note of the current highest priority -swapon --show -# append the following line to /etc/fstab (ensure highest priority) and then reboot -# /swapfile swap swap pri=50 0 0 -sudo reboot -# check a 256GB swap file is automatically mounted and has the highest priority -swapon --show -``` - -## Creating a new BLS wallet - -You will need a BLS wallet (`t3...`) for mining. To create it, if you don't have one already, run: - -```sh -lotus wallet new bls -``` - -Next make sure to [send some funds](en+wallet) to this address so that the miner setup can be completed. - -## Initializing the miner - -> SPACE RACE: -> To participate in the Space race, please register your miner: -> -> - Visit the [faucet](http://spacerace.faucet.glif.io/) -> - Paste the address you created under REQUEST. -> - Press the Request button. - -Now that you have a miner address you can initialize the Lotus Miner: - -```sh -lotus-miner init --owner= --no-local-storage -``` - -* The `--no-local-storage` flag is used so that we configure specific locations for storage later below. -* The init process will download over 100GiB of initialization parameters to /var/tmp/filecoin-proof-parameters. Make sure there is space or set `FIL_PROOFS_PARAMETER_CACHE` to somewhere else. -* The Lotus Miner configuration folder is created at `~/.lotusminer/` or `$LOTUS_MINER_PATH` if set. - -## Reachability - -Before you start your miner, it is __very important__ to configure it so that it is reachable from any peer in the Filecoin network. For this you will need a stable public IP and edit your `~/.lotusminer/config.toml` as follows: - -```toml -... -[Libp2p] - ListenAddresses = ["/ip4/0.0.0.0/tcp/24001"] # choose a fixed port - AnnounceAddresses = ["/ip4//tcp/24001"] # important! -... -``` - -Once you start your miner, make sure you can connect to its public IP/port (you can use `telnet`, `nc` for the task...). If you have an active firewall or some sort, you may need to additionally open ports in it. - - -## Starting the miner - -You are now ready to start your Lotus miner: - -```sh -lotus-miner run -``` - -or if you are using the systemd service file: - -```sh -systemctl start lotus-miner -``` - -> __Do not proceed__ from here until you have verified that your miner not only is running, but also __reachable on its public IP address__. - -## Publishing the miner addresses - -Once the miner is up and running, publish your miner address (which you configured above) on the chain (please ensure it is dialable): - -```sh -lotus-miner actor set-addrs /ip4//tcp/24001 -``` - -## Setting locations for sealing and long-term storage - -If you used the `--no-local-storage` flag during initialization, you can now specify the disk locations for sealing (SSD recommended) and long-term storage (otherwise you can skip this): - -``` -lotus-miner storage attach --init --seal -lotus-miner storage attach --init --store -lotus-miner storage list -``` - -## Pledging sectors - -If you would like to compete for block rewards by increasing your power in the network as soon as possible, you can optionally pledge one or several sectors, depending on your storage. It can also be used to test that the sealing process works correctly. Pledging is equivalent to storing random data instead of real data obtained through storage deals. - -> Note that pledging sectors to the mainnet network makes most sense when trying to obtain a reasonable amount of total power in the network, thus obtaining real chances to mine new blocks. Otherwise it is only useful for testing purposes. - -If you decide to go ahead, then do: - -```sh -lotus-miner sectors pledge -``` - -This will write data to `$TMPDIR` so make sure that there is enough space available. - -You shoud check that your sealing job has started with: - -```sh -lotus-miner sealing jobs -``` - -This will be accommpanied by a file in `/unsealed`. - -After some minutes, you can check the sealing progress with: - -```sh -lotus-miner sectors list -# and -lotus-miner sealing workers -``` - -When sealing for the new is complete, `pSet: NO` will become `pSet: YES`. - -Once the sealing is finished, you will want to configure how long it took your miner to seal this sector and configure the miner accordingly. To find out how long it took use: - -``` -lotus-miner sectors status --log 0 -``` - -Once you know, you can edit the Miner's `~/.lotusminer/config.toml` accordingly: - -``` -... -[Dealmaking] -... - ExpectedSealDuration = "12h0m0s" # The time it took your miner -``` - -You can also take the chance to edit other values, such as `WaitForDealsDelay` which specifies the delay between accepting the first deal and sealing, allowing to place multiple deals in the same sector. - -Once you are done editing the configuration, [restart your miner](en+update). - -If you wish to be able to re-use a pledged sector for real storage deals before the pledged period of 6 months ends, you will need to mark them for upgrade: - -```sh -lotus-miner sectors mark-for-upgrade -``` - -The sector should become inactive within 24 hours. From that point, the pledged storage can be re-used to store real data associated with real storage deals. - -## Separate address for windowPoSt messages - -WindowPoSt is the mechanism through which storage is verified in Filecoin. It requires miners to submit proofs for all sectors every 24h, which require sending messages to the chain. - -Because many other mining related actions require sending messages to the chain, and not all of those are "high value", it may be desirable to use a separate account to send PoSt messages from. This allows for setting lower GasFeeCaps on the lower value messages without creating head-of-line blocking problems for the PoSt messages in congested chain conditions - -To set this up, first create a new account, and send it some funds for gas fees: - -```sh -lotus wallet new bls -t3defg... - -lotus send t3defg... 100 -``` - -Next add the control address: - -```sh -lotus-miner actor control set --really-do-it t3defg... -Add t3defg... -Message CID: bafy2.. -``` - -Wait for the message to land on chain: - -```sh -lotus state wait-msg bafy2.. -... -Exit Code: 0 -... -``` - -Finally, check the miner control address list to make sure the address was correctly setup: - -```sh -lotus-miner actor control list -name ID key use balance -owner t01111 t3abcd... other 300 FIL -worker t01111 t3abcd... other 300 FIL -control-0 t02222 t3defg... post 100 FIL -``` diff --git a/documentation/en/mining/mining-troubleshooting.md b/documentation/en/mining/mining-troubleshooting.md deleted file mode 100644 index a9972c2bd..000000000 --- a/documentation/en/mining/mining-troubleshooting.md +++ /dev/null @@ -1,59 +0,0 @@ -# Mining Troubleshooting - -## Config: Filecoin Proof Parameters directory - -If you want to put the **Filecoin Proof Parameters** in a different directory, use the following environment variable: - -```sh -FIL_PROOFS_PARAMETER_CACHE -``` - -## Error: Can't acquire bellman.lock - -The **Bellman** lockfile is created to lock a GPU for a process. This bug can occur when this file isn't properly cleaned up: - -```sh -mining block failed: computing election proof: github.com/filecoin-project/lotus/miner.(*Miner).mineOne -``` - -This bug occurs when the miner can't acquire the `bellman.lock`. To fix it you need to stop the `lotus-miner` and remove `/tmp/bellman.lock`. - -## Error: Failed to get api endpoint - -```sh -lotus-miner info -# WARN main lotus-miner/main.go:73 failed to get api endpoint: (/Users/myrmidon/.lotusminer) %!w(*errors.errorString=&{API not running (no endpoint)}): -``` - -If you see this, that means your **Lotus Miner** isn't ready yet. You need to finish [syncing the chain](en+setup#waiting-to-sync-370). - -## Error: Your computer may not be fast enough - -```sh -CAUTION: block production took longer than the block delay. Your computer may not be fast enough to keep up -``` - -If you see this, that means your computer is too slow and your blocks are not included in the chain, and you will not receive any rewards. - -## Error: No space left on device - -```sh -lotus-miner sectors pledge -# No space left on device (os error 28) -``` - -If you see this, that means `pledge-sector` wrote too much data to `$TMPDIR` which by default is the root partition (This is common for Linux setups). Usually your root partition does not get the largest partition of storage so you will need to change the environment variable to something else. - -## Error: GPU unused - -If you suspect that your GPU is not being used, first make sure it is properly configured as described in the [testing configuration page](hardware-mining.md). Once you've done that (and set the `BELLMAN_CUSTOM_GPU` as appropriate if necessary) you can verify your GPU is being used by running a quick lotus-bench benchmark. - -First, to watch GPU utilization run `nvtop` in one terminal, then in a separate terminal, run: - -```sh -make bench -./bench sealing --sector-size=2KiB -``` - -This process uses a fair amount of GPU, and generally takes ~4 minutes to complete. If you do not see any activity in nvtop from lotus during the entire process, it is likely something is misconfigured with your GPU. - diff --git a/documentation/en/mining/mining.md b/documentation/en/mining/mining.md deleted file mode 100644 index b1b944c6e..000000000 --- a/documentation/en/mining/mining.md +++ /dev/null @@ -1,8 +0,0 @@ -# Storage Mining - -This section of the documentation explains how to do storage mining with Lotus. Please note that not everyone can do storage mining, and that you should not attempt it on on networks where sector sizes are 32GB+ unless you meet the [hardware requirements](en+install#hardware-requirements-1). - -From this point we assume that you have setup and are running the [Lotus Node](en+setup), that it has fully synced the Filecoin chain and that you are familiar with how to interact with it using the `lotus` command-line interface. - -In order to perform storage mining, apart from the Lotus daemon, you will be additionally interacting with the `lotus-miner` and potentially the `lotus-worker` applications (which you should have [installed](en+install-linux) along the `lotus` application already). - diff --git a/documentation/en/store/adding-from-ipfs.md b/documentation/en/store/adding-from-ipfs.md deleted file mode 100644 index 2f6b097cc..000000000 --- a/documentation/en/store/adding-from-ipfs.md +++ /dev/null @@ -1,20 +0,0 @@ -# Adding data from IPFS - -Lotus supports making deals with data stored in IPFS, without having to re-import it into lotus. - -To enable this integration, you need to have an IPFS daemon running in the background. - -Then, open up `~/.lotus/config.toml` (or if you manually set `LOTUS_PATH`, look under that directory) and look for the Client field, and set `UseIpfs` to `true`. - -```toml -[Client] -UseIpfs = true -``` - -After restarting the lotus daemon, you should be able to make deals with data in your IPFS node: - -```sh -$ ipfs add -r SomeData -QmSomeData -$ ./lotus client deal QmSomeData t01000 0.0000000001 80000 -``` diff --git a/documentation/en/store/making-deals.md b/documentation/en/store/making-deals.md deleted file mode 100644 index ca3a47182..000000000 --- a/documentation/en/store/making-deals.md +++ /dev/null @@ -1,71 +0,0 @@ -# Making storage deals - -## Adding a file to Lotus - -Before sending data to a Filecoin miner for storage, the data needs to be correctly formatted and packed. This can be achieved by locally importing the data into Lotus with: - -```sh -lotus client import ./your-example-file.txt -``` - -Upon success, this command will return a **Data CID**. This is a very important piece of information, as it will be used to make deals to both store and retrieve the data in the future. - -You can list the data CIDs of the files you locally imported with: - -```sh -lotus client local -``` - -## Storing data in the network - -To store data in the network you will need to: - -* Find a Filecoin miner willing to store it -* Make a deal with the miner agreeing on the price to pay and the duration for which the data should be stored. - -You can obtain a list of all miners in the network with: - -```sh -lotus state list-miners -t0xxxx -t0xxxy -t0xxxz -... -``` - -This will print a list of miner IDs. In order to ask for the terms offered by a particular miner, you can then run: - -```sh -lotus client query-ask -``` - -If you are satisfied with the terms, you can proceed to propose a deal to the miner, using the **Data CID** that you obtained during the import step: - - -```sh -lotus client deal -``` - -This command will interactively ask you for the CID, miner ID and duration in days for the deal. You can also call it with arguments: - -```sh -lotus client deal -``` - -where the `duration` is expressed in blocks (1 block is equivalent to 30s). - -## Checking the status of the deals - -You can list deals with: - -```sh -lotus client list-deals -``` - -Among other things, this will give you information about the current state on your deals, whether they have been published on chain (by the miners) and whether the miners have been slashed for not honoring them. - -For a deal to succeed, the miner needs to be correctly configured and running, accept the deal and *seal* the file correctly. Otherwise, the deal will appear in error state. - -You can make deals with multiple miners for the same data. - -Once a deal is sucessful and the data is *sealed*, it can be [retrieved](en+retrieving). diff --git a/documentation/en/store/retrieve.md b/documentation/en/store/retrieve.md deleted file mode 100644 index 1e8db65af..000000000 --- a/documentation/en/store/retrieve.md +++ /dev/null @@ -1,27 +0,0 @@ -# Retrieving Data - -Once data has been succesfully [stored](en+making-deals) and sealed by a Filecoin miner, it can be retrieved. - -In order to do this we will need to create a **retrieval deal**. - -## Finding data by CID - -In order to retrieve some data you will need the **Data CID** that was used to create the storage deal. - -You can find who is storing the data by running: - -```sh -lotus client find -``` - -## Making a retrieval deal - -You can then make a retrieval deal with: - -```sh -lotus client retrieve -``` - -This commands take other optional flags (check `--help`). - -If the outfile does not exist it will be created in the Lotus repository directory. This process may take 2 to 10 minutes. diff --git a/documentation/en/store/storage-troubleshooting.md b/documentation/en/store/storage-troubleshooting.md deleted file mode 100644 index 7087ec3d0..000000000 --- a/documentation/en/store/storage-troubleshooting.md +++ /dev/null @@ -1,30 +0,0 @@ -# Storage Troubleshooting - -## Error: Routing: not found - -``` -WARN main lotus/main.go:72 routing: not found -``` - -This error means that the miner is offline. - -## Error: Failed to start deal - -```sh -WARN main lotus/main.go:72 failed to start deal: computing commP failed: generating CommP: Piece must be at least 127 bytes -``` - -This error means that there is a minimum file size of 127 bytes. - -## Error: 0kb file response during retrieval - -This means that the file to be retrieved may have not yet been sealed and is thus, not retrievable yet. - -Miners can check sealing progress with this command: - -```sh -lotus-miner sectors list -``` - -When sealing is complete, `pSet: NO` will become `pSet: YES`. - diff --git a/documentation/en/store/store.md b/documentation/en/store/store.md deleted file mode 100644 index 205bd0e23..000000000 --- a/documentation/en/store/store.md +++ /dev/null @@ -1,11 +0,0 @@ -# Storing and retrieving data - -Lotus enables you to store any data on the Filecoin network and retrieve it later. This is achieved by making *deals* with miners. - -A *storage deal* specifies that a miner should store ceratin data for a previously agreed period and price. - -Once a deal is made, the data is then sent to the miners, which regularly proves that it is storing it. If they fail to do so, the miner is penalized (slashed). - -The data can be retrieved with a *retrieval deal*. - -This section explains how to use Lotus to [store](en+making-deals) and [retrieve](en+retrieving) data from the Filecoin network. From b85b50afbcc8ad24fda710369fbcd3ad4a001c5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 17 Sep 2020 19:57:42 +0200 Subject: [PATCH 193/885] shed: bitfield intersect command --- cmd/lotus-shed/bitfield.go | 115 +++++++++++++++++++++++++++---------- 1 file changed, 84 insertions(+), 31 deletions(-) diff --git a/cmd/lotus-shed/bitfield.go b/cmd/lotus-shed/bitfield.go index 79ce214ee..78f6f02fe 100644 --- a/cmd/lotus-shed/bitfield.go +++ b/cmd/lotus-shed/bitfield.go @@ -28,6 +28,7 @@ var bitFieldCmd = &cli.Command{ bitFieldRunsCmd, bitFieldStatCmd, bitFieldDecodeCmd, + bitFieldIntersectCmd, }, } @@ -200,38 +201,9 @@ var bitFieldDecodeCmd = &cli.Command{ }, }, Action: func(cctx *cli.Context) error { - var val string - if cctx.Args().Present() { - val = cctx.Args().Get(0) - } else { - b, err := ioutil.ReadAll(os.Stdin) - if err != nil { - return err - } - val = string(b) - } - - var dec []byte - switch cctx.String("enc") { - case "base64": - d, err := base64.StdEncoding.DecodeString(val) - if err != nil { - return fmt.Errorf("decoding base64 value: %w", err) - } - dec = d - case "hex": - d, err := hex.DecodeString(val) - if err != nil { - return fmt.Errorf("decoding hex value: %w", err) - } - dec = d - default: - return fmt.Errorf("unrecognized encoding: %s", cctx.String("enc")) - } - - rle, err := bitfield.NewFromBytes(dec) + rle, err := decode(cctx, 0) if err != nil { - return xerrors.Errorf("failed to parse bitfield: %w", err) + return err } vals, err := rle.All(100000000000) @@ -243,3 +215,84 @@ var bitFieldDecodeCmd = &cli.Command{ return nil }, } + +var bitFieldIntersectCmd = &cli.Command{ + Name: "intersect", + Description: "intersect 2 bitfields and print the resulting bitfield as base64", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "enc", + Value: "base64", + Usage: "specify input encoding to parse", + }, + }, + Action: func(cctx *cli.Context) error { + b, err := decode(cctx, 1) + if err != nil { + return err + } + + a, err := decode(cctx, 0) + if err != nil { + return err + } + + o, err := bitfield.IntersectBitField(a, b) + if err != nil { + return xerrors.Errorf("intersect: %w", err) + } + + s, err := o.RunIterator() + if err != nil { + return err + } + + bytes, err := rlepluslazy.EncodeRuns(s, []byte{}) + if err != nil { + return err + } + + fmt.Println(base64.StdEncoding.EncodeToString(bytes)) + + return nil + }, +} + +func decode(cctx *cli.Context, a int) (bitfield.BitField, error) { + var val string + if cctx.Args().Present() { + if a >= cctx.NArg() { + return bitfield.BitField{}, xerrors.Errorf("need more than %d args", a) + } + val = cctx.Args().Get(a) + } else { + if a > 0 { + return bitfield.BitField{}, xerrors.Errorf("need more than %d args", a) + } + b, err := ioutil.ReadAll(os.Stdin) + if err != nil { + return bitfield.BitField{}, err + } + val = string(b) + } + + var dec []byte + switch cctx.String("enc") { + case "base64": + d, err := base64.StdEncoding.DecodeString(val) + if err != nil { + return bitfield.BitField{}, fmt.Errorf("decoding base64 value: %w", err) + } + dec = d + case "hex": + d, err := hex.DecodeString(val) + if err != nil { + return bitfield.BitField{}, fmt.Errorf("decoding hex value: %w", err) + } + dec = d + default: + return bitfield.BitField{}, fmt.Errorf("unrecognized encoding: %s", cctx.String("enc")) + } + + return bitfield.NewFromBytes(dec) +} From 591e32af487204f58d092015a8ce25439b884718 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 17 Sep 2020 20:41:46 +0200 Subject: [PATCH 194/885] Remove circle_ci docs-check. --- .circleci/config.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index acd447f69..d8f149889 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -346,15 +346,6 @@ jobs: - run: git --no-pager diff - run: git --no-pager diff --quiet - docs-check: - executor: golang - steps: - - install-deps - - prepare - - run: make docsgen - - run: git --no-pager diff - - run: git --no-pager diff --quiet - lint: &lint description: | Run golangci-lint. @@ -424,7 +415,6 @@ workflows: - mod-tidy-check - gofmt - cbor-gen-check - - docs-check - test: codecov-upload: true test-suite-name: full From bc9232544c7f5d68c737b9d7887ae05f0232ff34 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 17 Sep 2020 20:48:09 +0200 Subject: [PATCH 195/885] Remove faqs too --- documentation/en/faqs.md | 132 --------------------------------------- 1 file changed, 132 deletions(-) delete mode 100644 documentation/en/faqs.md diff --git a/documentation/en/faqs.md b/documentation/en/faqs.md deleted file mode 100644 index 74119a5b6..000000000 --- a/documentation/en/faqs.md +++ /dev/null @@ -1,132 +0,0 @@ -# Frequently Asked Questions - -Here are some FAQs concerning the Lotus implementation and participation in -Testnet. -For questions concerning the broader Filecoin project, please -go [here](https://filecoin.io/faqs/). - -## Introduction to Lotus - -### What is Lotus? - -Lotus is an implementation of the **Filecoin Distributed Storage Network**, written in Go. -It is designed to be modular and interoperable with any other implementation of the Filecoin Protocol. - -### What are the components of Lotus? - -Lotus is composed of two separate pieces that can talk to each other: - -The Lotus Node can sync the blockchain, validating all blocks, transfers, and deals -along the way. It can also facilitate the creation of new storage deals. If you are not -interested in providing your own storage to the network, and do not want to produce blocks -yourself, then the Lotus Node is all you need! - -The Lotus Miner does everything you need for the registration of storage, and the -production of new blocks. The Lotus Miner communicates with the network by talking -to a Lotus Node over the JSON-RPC API. - -## Setting up a Lotus Node - -### How do I set up a Lotus Node? - -Follow the instructions found [here](en+install) and [here](en+setup). - -### Where can I get the latest version of Lotus? - -Download the binary tagged as the `Latest Release` from the [Lotus Github repo](https://github.com/filecoin-project/lotus/releases) or checkout the `master` branch of the source repository. - -### What operating systems can Lotus run on? - -Lotus can build and run on most Linux and MacOS systems with [at least 8GB of RAM](en+install#hardware-requirements-1). Windows is not yet supported. - -### How can I update to the latest version of Lotus? - -To update Lotus, follow the instructions [here](en+update). - -### How do I prepare a fresh installation of Lotus? - -Stop the Lotus daemon, and delete all related files, including sealed and chain data by -running `rm ~/.lotus ~/.lotusminer`. - -Then, install Lotus afresh by following the instructions -found [here](en+install). - -### Can I configure where the node's config and data goes? - -Yes! The `LOTUS_PATH` variable sets the path for where the Lotus node's data is written. -The `LOTUS_MINER_PATH` variable does the same for miner-specific information. - -## Interacting with a Lotus Node - -### How can I communicate with a Lotus Node? - -Lotus Nodes have a command-line interface, as well as a JSON-RPC API. - -### What are the commands I can send using the command-line interface? - -The command-line interface is self-documenting, try running `lotus --help` from the `lotus` home -directory for more. - -### How can I send a request over the JSON-RPC API? - -Information on how to send a `cURL` request to the JSON-RPC API can be found -[here](en+api). - -### What are the requests I can send over the JSON-RPC API? - -Please have a look [here](en+api). - - -## The Test Network - -### What is Testnet? - -Testnet is a live network of Lotus Nodes run by the -community for testing purposes. - -### Is FIL on the Testnet worth anything? - -Nothing at all! - -### How can I see the status of Testnet? - -The [dashboard](https://stats.testnet.filecoin.io/) displays the status of the network as -well as a ton of other metrics you might find interesting. - -## Mining with a Lotus Node on Testnet - -### How do I get started mining with Lotus? - -Follow the instructions found [here](en+mining). - -### What are the minimum hardware requirements? - -An example test configuration, and minimum hardware requirements can be found -[here](en+install#hardware-requirements-8). - -Note that these might NOT be the minimum requirements for mining on Mainnet. - -### What are some GPUs that have been tested? - -See previous question. - -### Why is my GPU not being used when sealing a sector? - -Sealing a sector does not involve constant GPU operations. It's possible -that your GPU simply isn't necessary at the moment you checked. - -## Advanced questions - -### Is there a Docker image for lotus? - -Community-contributed Docker and Docker Compose examples are available -[here](https://github.com/filecoin-project/lotus/tree/master/tools/dockers/docker-examples). - -### How can I run two miners on the same machine? - -You can do so by changing the storage path variable for the second miner, e.g., -`LOTUS_MINER_PATH=~/.lotusminer2`. You will also need to make sure that no ports collide. - -### How do I setup my own local devnet? - -Follow the instructions found [here](en+local-devnet). From 784399738a49f282bc8002771e080d2eba5a33f9 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 17 Sep 2020 20:50:37 +0200 Subject: [PATCH 196/885] Keep api-methods.md even if not part of the docs --- Makefile | 2 +- documentation/en/api-methods.md | 4567 +++++++++++++++++++++++++++++++ 2 files changed, 4568 insertions(+), 1 deletion(-) create mode 100644 documentation/en/api-methods.md diff --git a/Makefile b/Makefile index 245d8a9a0..56ab361ec 100644 --- a/Makefile +++ b/Makefile @@ -286,7 +286,7 @@ method-gen: gen: type-gen method-gen docsgen: - go run ./api/docgen > documentation/en/building/api-methods.md + go run ./api/docgen > documentation/en/api-methods.md print-%: @echo $*=$($*) diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md new file mode 100644 index 000000000..2f3164bb7 --- /dev/null +++ b/documentation/en/api-methods.md @@ -0,0 +1,4567 @@ +# Groups +* [](#) + * [Closing](#Closing) + * [Shutdown](#Shutdown) + * [Version](#Version) +* [Auth](#Auth) + * [AuthNew](#AuthNew) + * [AuthVerify](#AuthVerify) +* [Beacon](#Beacon) + * [BeaconGetEntry](#BeaconGetEntry) +* [Chain](#Chain) + * [ChainExport](#ChainExport) + * [ChainGetBlock](#ChainGetBlock) + * [ChainGetBlockMessages](#ChainGetBlockMessages) + * [ChainGetGenesis](#ChainGetGenesis) + * [ChainGetMessage](#ChainGetMessage) + * [ChainGetNode](#ChainGetNode) + * [ChainGetParentMessages](#ChainGetParentMessages) + * [ChainGetParentReceipts](#ChainGetParentReceipts) + * [ChainGetPath](#ChainGetPath) + * [ChainGetRandomnessFromBeacon](#ChainGetRandomnessFromBeacon) + * [ChainGetRandomnessFromTickets](#ChainGetRandomnessFromTickets) + * [ChainGetTipSet](#ChainGetTipSet) + * [ChainGetTipSetByHeight](#ChainGetTipSetByHeight) + * [ChainHasObj](#ChainHasObj) + * [ChainHead](#ChainHead) + * [ChainNotify](#ChainNotify) + * [ChainReadObj](#ChainReadObj) + * [ChainSetHead](#ChainSetHead) + * [ChainStatObj](#ChainStatObj) + * [ChainTipSetWeight](#ChainTipSetWeight) +* [Client](#Client) + * [ClientCalcCommP](#ClientCalcCommP) + * [ClientDataTransferUpdates](#ClientDataTransferUpdates) + * [ClientDealSize](#ClientDealSize) + * [ClientFindData](#ClientFindData) + * [ClientGenCar](#ClientGenCar) + * [ClientGetDealInfo](#ClientGetDealInfo) + * [ClientGetDealUpdates](#ClientGetDealUpdates) + * [ClientHasLocal](#ClientHasLocal) + * [ClientImport](#ClientImport) + * [ClientListDataTransfers](#ClientListDataTransfers) + * [ClientListDeals](#ClientListDeals) + * [ClientListImports](#ClientListImports) + * [ClientMinerQueryOffer](#ClientMinerQueryOffer) + * [ClientQueryAsk](#ClientQueryAsk) + * [ClientRemoveImport](#ClientRemoveImport) + * [ClientRetrieve](#ClientRetrieve) + * [ClientRetrieveTryRestartInsufficientFunds](#ClientRetrieveTryRestartInsufficientFunds) + * [ClientRetrieveWithEvents](#ClientRetrieveWithEvents) + * [ClientStartDeal](#ClientStartDeal) +* [Gas](#Gas) + * [GasEstimateFeeCap](#GasEstimateFeeCap) + * [GasEstimateGasLimit](#GasEstimateGasLimit) + * [GasEstimateGasPremium](#GasEstimateGasPremium) + * [GasEstimateMessageGas](#GasEstimateMessageGas) +* [I](#I) + * [ID](#ID) +* [Log](#Log) + * [LogList](#LogList) + * [LogSetLevel](#LogSetLevel) +* [Market](#Market) + * [MarketEnsureAvailable](#MarketEnsureAvailable) +* [Miner](#Miner) + * [MinerCreateBlock](#MinerCreateBlock) + * [MinerGetBaseInfo](#MinerGetBaseInfo) +* [Mpool](#Mpool) + * [MpoolClear](#MpoolClear) + * [MpoolGetConfig](#MpoolGetConfig) + * [MpoolGetNonce](#MpoolGetNonce) + * [MpoolPending](#MpoolPending) + * [MpoolPush](#MpoolPush) + * [MpoolPushMessage](#MpoolPushMessage) + * [MpoolSelect](#MpoolSelect) + * [MpoolSetConfig](#MpoolSetConfig) + * [MpoolSub](#MpoolSub) +* [Msig](#Msig) + * [MsigAddApprove](#MsigAddApprove) + * [MsigAddCancel](#MsigAddCancel) + * [MsigAddPropose](#MsigAddPropose) + * [MsigApprove](#MsigApprove) + * [MsigCancel](#MsigCancel) + * [MsigCreate](#MsigCreate) + * [MsigGetAvailableBalance](#MsigGetAvailableBalance) + * [MsigGetVested](#MsigGetVested) + * [MsigPropose](#MsigPropose) + * [MsigSwapApprove](#MsigSwapApprove) + * [MsigSwapCancel](#MsigSwapCancel) + * [MsigSwapPropose](#MsigSwapPropose) +* [Net](#Net) + * [NetAddrsListen](#NetAddrsListen) + * [NetAgentVersion](#NetAgentVersion) + * [NetAutoNatStatus](#NetAutoNatStatus) + * [NetBandwidthStats](#NetBandwidthStats) + * [NetBandwidthStatsByPeer](#NetBandwidthStatsByPeer) + * [NetBandwidthStatsByProtocol](#NetBandwidthStatsByProtocol) + * [NetConnect](#NetConnect) + * [NetConnectedness](#NetConnectedness) + * [NetDisconnect](#NetDisconnect) + * [NetFindPeer](#NetFindPeer) + * [NetPeers](#NetPeers) + * [NetPubsubScores](#NetPubsubScores) +* [Paych](#Paych) + * [PaychAllocateLane](#PaychAllocateLane) + * [PaychAvailableFunds](#PaychAvailableFunds) + * [PaychAvailableFundsByFromTo](#PaychAvailableFundsByFromTo) + * [PaychCollect](#PaychCollect) + * [PaychGet](#PaychGet) + * [PaychGetWaitReady](#PaychGetWaitReady) + * [PaychList](#PaychList) + * [PaychNewPayment](#PaychNewPayment) + * [PaychSettle](#PaychSettle) + * [PaychStatus](#PaychStatus) + * [PaychVoucherAdd](#PaychVoucherAdd) + * [PaychVoucherCheckSpendable](#PaychVoucherCheckSpendable) + * [PaychVoucherCheckValid](#PaychVoucherCheckValid) + * [PaychVoucherCreate](#PaychVoucherCreate) + * [PaychVoucherList](#PaychVoucherList) + * [PaychVoucherSubmit](#PaychVoucherSubmit) +* [State](#State) + * [StateAccountKey](#StateAccountKey) + * [StateAllMinerFaults](#StateAllMinerFaults) + * [StateCall](#StateCall) + * [StateChangedActors](#StateChangedActors) + * [StateCirculatingSupply](#StateCirculatingSupply) + * [StateCompute](#StateCompute) + * [StateDealProviderCollateralBounds](#StateDealProviderCollateralBounds) + * [StateGetActor](#StateGetActor) + * [StateGetReceipt](#StateGetReceipt) + * [StateListActors](#StateListActors) + * [StateListMessages](#StateListMessages) + * [StateListMiners](#StateListMiners) + * [StateLookupID](#StateLookupID) + * [StateMarketBalance](#StateMarketBalance) + * [StateMarketDeals](#StateMarketDeals) + * [StateMarketParticipants](#StateMarketParticipants) + * [StateMarketStorageDeal](#StateMarketStorageDeal) + * [StateMinerActiveSectors](#StateMinerActiveSectors) + * [StateMinerAvailableBalance](#StateMinerAvailableBalance) + * [StateMinerDeadlines](#StateMinerDeadlines) + * [StateMinerFaults](#StateMinerFaults) + * [StateMinerInfo](#StateMinerInfo) + * [StateMinerInitialPledgeCollateral](#StateMinerInitialPledgeCollateral) + * [StateMinerPartitions](#StateMinerPartitions) + * [StateMinerPower](#StateMinerPower) + * [StateMinerPreCommitDepositForPower](#StateMinerPreCommitDepositForPower) + * [StateMinerProvingDeadline](#StateMinerProvingDeadline) + * [StateMinerRecoveries](#StateMinerRecoveries) + * [StateMinerSectorCount](#StateMinerSectorCount) + * [StateMinerSectors](#StateMinerSectors) + * [StateMsgGasCost](#StateMsgGasCost) + * [StateNetworkName](#StateNetworkName) + * [StateReadState](#StateReadState) + * [StateReplay](#StateReplay) + * [StateSearchMsg](#StateSearchMsg) + * [StateSectorExpiration](#StateSectorExpiration) + * [StateSectorGetInfo](#StateSectorGetInfo) + * [StateSectorPartition](#StateSectorPartition) + * [StateSectorPreCommitInfo](#StateSectorPreCommitInfo) + * [StateVerifiedClientStatus](#StateVerifiedClientStatus) + * [StateWaitMsg](#StateWaitMsg) +* [Sync](#Sync) + * [SyncCheckBad](#SyncCheckBad) + * [SyncCheckpoint](#SyncCheckpoint) + * [SyncIncomingBlocks](#SyncIncomingBlocks) + * [SyncMarkBad](#SyncMarkBad) + * [SyncState](#SyncState) + * [SyncSubmitBlock](#SyncSubmitBlock) + * [SyncUnmarkBad](#SyncUnmarkBad) +* [Wallet](#Wallet) + * [WalletBalance](#WalletBalance) + * [WalletDefaultAddress](#WalletDefaultAddress) + * [WalletDelete](#WalletDelete) + * [WalletExport](#WalletExport) + * [WalletHas](#WalletHas) + * [WalletImport](#WalletImport) + * [WalletList](#WalletList) + * [WalletNew](#WalletNew) + * [WalletSetDefault](#WalletSetDefault) + * [WalletSign](#WalletSign) + * [WalletSignMessage](#WalletSignMessage) + * [WalletVerify](#WalletVerify) +## + + +### Closing + + +Perms: read + +Inputs: `null` + +Response: `{}` + +### Shutdown + + +Perms: admin + +Inputs: `null` + +Response: `{}` + +### Version + + +Perms: read + +Inputs: `null` + +Response: +```json +{ + "Version": "string value", + "APIVersion": 3584, + "BlockDelay": 42 +} +``` + +## Auth + + +### AuthNew + + +Perms: admin + +Inputs: +```json +[ + null +] +``` + +Response: `"Ynl0ZSBhcnJheQ=="` + +### AuthVerify + + +Perms: read + +Inputs: +```json +[ + "string value" +] +``` + +Response: `null` + +## Beacon +The Beacon method group contains methods for interacting with the random beacon (DRAND) + + +### BeaconGetEntry +BeaconGetEntry returns the beacon entry for the given filecoin epoch. If +the entry has not yet been produced, the call will block until the entry +becomes available + + +Perms: read + +Inputs: +```json +[ + 10101 +] +``` + +Response: +```json +{ + "Round": 42, + "Data": "Ynl0ZSBhcnJheQ==" +} +``` + +## Chain +The Chain method group contains methods for interacting with the +blockchain, but that do not require any form of state computation. + + +### ChainExport +ChainExport returns a stream of bytes with CAR dump of chain data. +The exported chain data includes the header chain from the given tipset +back to genesis, the entire genesis state, and the most recent 'nroots' +state trees. +If oldmsgskip is set, messages from before the requested roots are also not included. + + +Perms: read + +Inputs: +```json +[ + 10101, + true, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `"Ynl0ZSBhcnJheQ=="` + +### ChainGetBlock +ChainGetBlock returns the block specified by the given CID. + + +Perms: read + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: +```json +{ + "Miner": "t01234", + "Ticket": { + "VRFProof": "Ynl0ZSBhcnJheQ==" + }, + "ElectionProof": { + "WinCount": 9, + "VRFProof": "Ynl0ZSBhcnJheQ==" + }, + "BeaconEntries": null, + "WinPoStProof": null, + "Parents": null, + "ParentWeight": "0", + "Height": 10101, + "ParentStateRoot": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "ParentMessageReceipts": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Messages": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "BLSAggregate": { + "Type": 2, + "Data": "Ynl0ZSBhcnJheQ==" + }, + "Timestamp": 42, + "BlockSig": { + "Type": 2, + "Data": "Ynl0ZSBhcnJheQ==" + }, + "ForkSignaling": 42, + "ParentBaseFee": "0" +} +``` + +### ChainGetBlockMessages +ChainGetBlockMessages returns messages stored in the specified block. + + +Perms: read + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: +```json +{ + "BlsMessages": null, + "SecpkMessages": null, + "Cids": null +} +``` + +### ChainGetGenesis +ChainGetGenesis returns the genesis tipset. + + +Perms: read + +Inputs: `null` + +Response: +```json +{ + "Cids": null, + "Blocks": null, + "Height": 0 +} +``` + +### ChainGetMessage +ChainGetMessage reads a message referenced by the specified CID from the +chain blockstore. + + +Perms: read + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: +```json +{ + "Version": 42, + "To": "t01234", + "From": "t01234", + "Nonce": 42, + "Value": "0", + "GasLimit": 9, + "GasFeeCap": "0", + "GasPremium": "0", + "Method": 1, + "Params": "Ynl0ZSBhcnJheQ==" +} +``` + +### ChainGetNode +There are not yet any comments for this method. + +Perms: read + +Inputs: +```json +[ + "string value" +] +``` + +Response: +```json +{ + "Cid": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Obj": {} +} +``` + +### ChainGetParentMessages +ChainGetParentMessages returns messages stored in parent tipset of the +specified block. + + +Perms: read + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: `null` + +### ChainGetParentReceipts +ChainGetParentReceipts returns receipts for messages in parent tipset of +the specified block. + + +Perms: read + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: `null` + +### ChainGetPath +ChainGetPath returns a set of revert/apply operations needed to get from +one tipset to another, for example: +``` + to + ^ +from tAA + ^ ^ +tBA tAB + ^---*--^ + ^ + tRR +``` +Would return `[revert(tBA), apply(tAB), apply(tAA)]` + + +Perms: read + +Inputs: +```json +[ + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ], + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `null` + +### ChainGetRandomnessFromBeacon +ChainGetRandomnessFromBeacon is used to sample the beacon for randomness. + + +Perms: read + +Inputs: +```json +[ + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ], + 2, + 10101, + "Ynl0ZSBhcnJheQ==" +] +``` + +Response: `null` + +### ChainGetRandomnessFromTickets +ChainGetRandomnessFromTickets is used to sample the chain for randomness. + + +Perms: read + +Inputs: +```json +[ + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ], + 2, + 10101, + "Ynl0ZSBhcnJheQ==" +] +``` + +Response: `null` + +### ChainGetTipSet +ChainGetTipSet returns the tipset specified by the given TipSetKey. + + +Perms: read + +Inputs: +```json +[ + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "Cids": null, + "Blocks": null, + "Height": 0 +} +``` + +### ChainGetTipSetByHeight +ChainGetTipSetByHeight looks back for a tipset at the specified epoch. +If there are no blocks at the specified epoch, a tipset at an earlier epoch +will be returned. + + +Perms: read + +Inputs: +```json +[ + 10101, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "Cids": null, + "Blocks": null, + "Height": 0 +} +``` + +### ChainHasObj +ChainHasObj checks if a given CID exists in the chain blockstore. + + +Perms: read + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: `true` + +### ChainHead +ChainHead returns the current head of the chain. + + +Perms: read + +Inputs: `null` + +Response: +```json +{ + "Cids": null, + "Blocks": null, + "Height": 0 +} +``` + +### ChainNotify +ChainNotify returns channel with chain head updates. +First message is guaranteed to be of len == 1, and type == 'current'. + + +Perms: read + +Inputs: `null` + +Response: `null` + +### ChainReadObj +ChainReadObj reads ipld nodes referenced by the specified CID from chain +blockstore and returns raw bytes. + + +Perms: read + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: `"Ynl0ZSBhcnJheQ=="` + +### ChainSetHead +ChainSetHead forcefully sets current chain head. Use with caution. + + +Perms: admin + +Inputs: +```json +[ + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `{}` + +### ChainStatObj +ChainStatObj returns statistics about the graph referenced by 'obj'. +If 'base' is also specified, then the returned stat will be a diff +between the two objects. + + +Perms: read + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: +```json +{ + "Size": 42, + "Links": 42 +} +``` + +### ChainTipSetWeight +ChainTipSetWeight computes weight for the specified tipset. + + +Perms: read + +Inputs: +```json +[ + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `"0"` + +## Client +The Client methods all have to do with interacting with the storage and +retrieval markets as a client + + +### ClientCalcCommP +ClientCalcCommP calculates the CommP for a specified file + + +Perms: read + +Inputs: +```json +[ + "string value" +] +``` + +Response: +```json +{ + "Root": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Size": 1024 +} +``` + +### ClientDataTransferUpdates +There are not yet any comments for this method. + +Perms: write + +Inputs: `null` + +Response: +```json +{ + "TransferID": 3, + "Status": 1, + "BaseCID": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "IsInitiator": true, + "IsSender": true, + "Voucher": "string value", + "Message": "string value", + "OtherPeer": "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf", + "Transferred": 42 +} +``` + +### ClientDealSize +ClientDealSize calculates real deal data size + + +Perms: read + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: +```json +{ + "PayloadSize": 9, + "PieceSize": 1032 +} +``` + +### ClientFindData +ClientFindData identifies peers that have a certain file, and returns QueryOffers (one per peer). + + +Perms: read + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + null +] +``` + +Response: `null` + +### ClientGenCar +ClientGenCar generates a CAR file for the specified file. + + +Perms: write + +Inputs: +```json +[ + { + "Path": "string value", + "IsCAR": true + }, + "string value" +] +``` + +Response: `{}` + +### ClientGetDealInfo +ClientGetDealInfo returns the latest information about a given deal. + + +Perms: read + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: +```json +{ + "ProposalCid": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "State": 42, + "Message": "string value", + "Provider": "t01234", + "DataRef": { + "TransferType": "string value", + "Root": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "PieceCid": null, + "PieceSize": 1024 + }, + "PieceCID": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Size": 42, + "PricePerEpoch": "0", + "Duration": 42, + "DealID": 5432, + "CreationTime": "0001-01-01T00:00:00Z" +} +``` + +### ClientGetDealUpdates +ClientGetDealUpdates returns the status of updated deals + + +Perms: read + +Inputs: `null` + +Response: +```json +{ + "ProposalCid": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "State": 42, + "Message": "string value", + "Provider": "t01234", + "DataRef": { + "TransferType": "string value", + "Root": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "PieceCid": null, + "PieceSize": 1024 + }, + "PieceCID": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Size": 42, + "PricePerEpoch": "0", + "Duration": 42, + "DealID": 5432, + "CreationTime": "0001-01-01T00:00:00Z" +} +``` + +### ClientHasLocal +ClientHasLocal indicates whether a certain CID is locally stored. + + +Perms: write + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: `true` + +### ClientImport +ClientImport imports file under the specified path into filestore. + + +Perms: admin + +Inputs: +```json +[ + { + "Path": "string value", + "IsCAR": true + } +] +``` + +Response: +```json +{ + "Root": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "ImportID": 50 +} +``` + +### ClientListDataTransfers +ClientListTransfers returns the status of all ongoing transfers of data + + +Perms: write + +Inputs: `null` + +Response: `null` + +### ClientListDeals +ClientListDeals returns information about the deals made by the local client. + + +Perms: write + +Inputs: `null` + +Response: `null` + +### ClientListImports +ClientListImports lists imported files and their root CIDs + + +Perms: write + +Inputs: `null` + +Response: `null` + +### ClientMinerQueryOffer +ClientMinerQueryOffer returns a QueryOffer for the specific miner and file. + + +Perms: read + +Inputs: +```json +[ + "t01234", + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + null +] +``` + +Response: +```json +{ + "Err": "string value", + "Root": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Piece": null, + "Size": 42, + "MinPrice": "0", + "UnsealPrice": "0", + "PaymentInterval": 42, + "PaymentIntervalIncrease": 42, + "Miner": "t01234", + "MinerPeer": { + "Address": "t01234", + "ID": "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf", + "PieceCID": null + } +} +``` + +### ClientQueryAsk +ClientQueryAsk returns a signed StorageAsk from the specified miner. + + +Perms: read + +Inputs: +```json +[ + "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf", + "t01234" +] +``` + +Response: +```json +{ + "Ask": { + "Price": "0", + "VerifiedPrice": "0", + "MinPieceSize": 1032, + "MaxPieceSize": 1032, + "Miner": "t01234", + "Timestamp": 10101, + "Expiry": 10101, + "SeqNo": 42 + }, + "Signature": { + "Type": 2, + "Data": "Ynl0ZSBhcnJheQ==" + } +} +``` + +### ClientRemoveImport +ClientRemoveImport removes file import + + +Perms: admin + +Inputs: +```json +[ + 50 +] +``` + +Response: `{}` + +### ClientRetrieve +ClientRetrieve initiates the retrieval of a file, as specified in the order. + + +Perms: admin + +Inputs: +```json +[ + { + "Root": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Piece": null, + "Size": 42, + "Total": "0", + "UnsealPrice": "0", + "PaymentInterval": 42, + "PaymentIntervalIncrease": 42, + "Client": "t01234", + "Miner": "t01234", + "MinerPeer": { + "Address": "t01234", + "ID": "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf", + "PieceCID": null + } + }, + { + "Path": "string value", + "IsCAR": true + } +] +``` + +Response: `{}` + +### ClientRetrieveTryRestartInsufficientFunds +ClientRetrieveTryRestartInsufficientFunds attempts to restart stalled retrievals on a given payment channel +which are stuck due to insufficient funds + + +Perms: write + +Inputs: +```json +[ + "t01234" +] +``` + +Response: `{}` + +### ClientRetrieveWithEvents +ClientRetrieveWithEvents initiates the retrieval of a file, as specified in the order, and provides a channel +of status updates. + + +Perms: admin + +Inputs: +```json +[ + { + "Root": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Piece": null, + "Size": 42, + "Total": "0", + "UnsealPrice": "0", + "PaymentInterval": 42, + "PaymentIntervalIncrease": 42, + "Client": "t01234", + "Miner": "t01234", + "MinerPeer": { + "Address": "t01234", + "ID": "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf", + "PieceCID": null + } + }, + { + "Path": "string value", + "IsCAR": true + } +] +``` + +Response: +```json +{ + "Event": 5, + "Status": 0, + "BytesReceived": 42, + "FundsSpent": "0", + "Err": "string value" +} +``` + +### ClientStartDeal +ClientStartDeal proposes a deal with a miner. + + +Perms: admin + +Inputs: +```json +[ + { + "Data": { + "TransferType": "string value", + "Root": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "PieceCid": null, + "PieceSize": 1024 + }, + "Wallet": "t01234", + "Miner": "t01234", + "EpochPrice": "0", + "MinBlocksDuration": 42, + "ProviderCollateral": "0", + "DealStartEpoch": 10101, + "FastRetrieval": true, + "VerifiedDeal": true + } +] +``` + +Response: `null` + +## Gas + + +### GasEstimateFeeCap +GasEstimateFeeCap estimates gas fee cap + + +Perms: read + +Inputs: +```json +[ + { + "Version": 42, + "To": "t01234", + "From": "t01234", + "Nonce": 42, + "Value": "0", + "GasLimit": 9, + "GasFeeCap": "0", + "GasPremium": "0", + "Method": 1, + "Params": "Ynl0ZSBhcnJheQ==" + }, + 9, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `"0"` + +### GasEstimateGasLimit +GasEstimateGasLimit estimates gas used by the message and returns it. +It fails if message fails to execute. + + +Perms: read + +Inputs: +```json +[ + { + "Version": 42, + "To": "t01234", + "From": "t01234", + "Nonce": 42, + "Value": "0", + "GasLimit": 9, + "GasFeeCap": "0", + "GasPremium": "0", + "Method": 1, + "Params": "Ynl0ZSBhcnJheQ==" + }, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `9` + +### GasEstimateGasPremium +GasEstimateGasPremium estimates what gas price should be used for a +message to have high likelihood of inclusion in `nblocksincl` epochs. + + +Perms: read + +Inputs: +```json +[ + 42, + "t01234", + 9, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `"0"` + +### GasEstimateMessageGas +GasEstimateMessageGas estimates gas values for unset message gas fields + + +Perms: read + +Inputs: +```json +[ + { + "Version": 42, + "To": "t01234", + "From": "t01234", + "Nonce": 42, + "Value": "0", + "GasLimit": 9, + "GasFeeCap": "0", + "GasPremium": "0", + "Method": 1, + "Params": "Ynl0ZSBhcnJheQ==" + }, + { + "MaxFee": "0" + }, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "Version": 42, + "To": "t01234", + "From": "t01234", + "Nonce": 42, + "Value": "0", + "GasLimit": 9, + "GasFeeCap": "0", + "GasPremium": "0", + "Method": 1, + "Params": "Ynl0ZSBhcnJheQ==" +} +``` + +## I + + +### ID + + +Perms: read + +Inputs: `null` + +Response: `"12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf"` + +## Log + + +### LogList + + +Perms: write + +Inputs: `null` + +Response: `null` + +### LogSetLevel + + +Perms: write + +Inputs: +```json +[ + "string value", + "string value" +] +``` + +Response: `{}` + +## Market + + +### MarketEnsureAvailable +MarketFreeBalance + + +Perms: sign + +Inputs: +```json +[ + "t01234", + "t01234", + "0" +] +``` + +Response: +```json +{ + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" +} +``` + +## Miner + + +### MinerCreateBlock +There are not yet any comments for this method. + +Perms: write + +Inputs: +```json +[ + { + "Miner": "t01234", + "Parents": [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ], + "Ticket": { + "VRFProof": "Ynl0ZSBhcnJheQ==" + }, + "Eproof": { + "WinCount": 9, + "VRFProof": "Ynl0ZSBhcnJheQ==" + }, + "BeaconValues": null, + "Messages": null, + "Epoch": 10101, + "Timestamp": 42, + "WinningPoStProof": null + } +] +``` + +Response: +```json +{ + "Header": { + "Miner": "t01234", + "Ticket": { + "VRFProof": "Ynl0ZSBhcnJheQ==" + }, + "ElectionProof": { + "WinCount": 9, + "VRFProof": "Ynl0ZSBhcnJheQ==" + }, + "BeaconEntries": null, + "WinPoStProof": null, + "Parents": null, + "ParentWeight": "0", + "Height": 10101, + "ParentStateRoot": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "ParentMessageReceipts": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Messages": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "BLSAggregate": { + "Type": 2, + "Data": "Ynl0ZSBhcnJheQ==" + }, + "Timestamp": 42, + "BlockSig": { + "Type": 2, + "Data": "Ynl0ZSBhcnJheQ==" + }, + "ForkSignaling": 42, + "ParentBaseFee": "0" + }, + "BlsMessages": null, + "SecpkMessages": null +} +``` + +### MinerGetBaseInfo +There are not yet any comments for this method. + +Perms: read + +Inputs: +```json +[ + "t01234", + 10101, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "MinerPower": "0", + "NetworkPower": "0", + "Sectors": null, + "WorkerKey": "t01234", + "SectorSize": 34359738368, + "PrevBeaconEntry": { + "Round": 42, + "Data": "Ynl0ZSBhcnJheQ==" + }, + "BeaconEntries": null, + "HasMinPower": true +} +``` + +## Mpool +The Mpool methods are for interacting with the message pool. The message pool +manages all incoming and outgoing 'messages' going over the network. + + +### MpoolClear +MpoolClear clears pending messages from the mpool + + +Perms: write + +Inputs: +```json +[ + true +] +``` + +Response: `{}` + +### MpoolGetConfig +MpoolGetConfig returns (a copy of) the current mpool config + + +Perms: read + +Inputs: `null` + +Response: +```json +{ + "PriorityAddrs": null, + "SizeLimitHigh": 123, + "SizeLimitLow": 123, + "ReplaceByFeeRatio": 12.3, + "PruneCooldown": 60000000000, + "GasLimitOverestimation": 12.3 +} +``` + +### MpoolGetNonce +MpoolGetNonce gets next nonce for the specified sender. +Note that this method may not be atomic. Use MpoolPushMessage instead. + + +Perms: read + +Inputs: +```json +[ + "t01234" +] +``` + +Response: `42` + +### MpoolPending +MpoolPending returns pending mempool messages. + + +Perms: read + +Inputs: +```json +[ + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `null` + +### MpoolPush +MpoolPush pushes a signed message to mempool. + + +Perms: write + +Inputs: +```json +[ + { + "Message": { + "Version": 42, + "To": "t01234", + "From": "t01234", + "Nonce": 42, + "Value": "0", + "GasLimit": 9, + "GasFeeCap": "0", + "GasPremium": "0", + "Method": 1, + "Params": "Ynl0ZSBhcnJheQ==" + }, + "Signature": { + "Type": 2, + "Data": "Ynl0ZSBhcnJheQ==" + } + } +] +``` + +Response: +```json +{ + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" +} +``` + +### MpoolPushMessage +MpoolPushMessage atomically assigns a nonce, signs, and pushes a message +to mempool. +maxFee is only used when GasFeeCap/GasPremium fields aren't specified + +When maxFee is set to 0, MpoolPushMessage will guess appropriate fee +based on current chain conditions + + +Perms: sign + +Inputs: +```json +[ + { + "Version": 42, + "To": "t01234", + "From": "t01234", + "Nonce": 42, + "Value": "0", + "GasLimit": 9, + "GasFeeCap": "0", + "GasPremium": "0", + "Method": 1, + "Params": "Ynl0ZSBhcnJheQ==" + }, + { + "MaxFee": "0" + } +] +``` + +Response: +```json +{ + "Message": { + "Version": 42, + "To": "t01234", + "From": "t01234", + "Nonce": 42, + "Value": "0", + "GasLimit": 9, + "GasFeeCap": "0", + "GasPremium": "0", + "Method": 1, + "Params": "Ynl0ZSBhcnJheQ==" + }, + "Signature": { + "Type": 2, + "Data": "Ynl0ZSBhcnJheQ==" + } +} +``` + +### MpoolSelect +MpoolSelect returns a list of pending messages for inclusion in the next block + + +Perms: read + +Inputs: +```json +[ + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ], + 12.3 +] +``` + +Response: `null` + +### MpoolSetConfig +MpoolSetConfig sets the mpool config to (a copy of) the supplied config + + +Perms: write + +Inputs: +```json +[ + { + "PriorityAddrs": null, + "SizeLimitHigh": 123, + "SizeLimitLow": 123, + "ReplaceByFeeRatio": 12.3, + "PruneCooldown": 60000000000, + "GasLimitOverestimation": 12.3 + } +] +``` + +Response: `{}` + +### MpoolSub +There are not yet any comments for this method. + +Perms: read + +Inputs: `null` + +Response: +```json +{ + "Type": 0, + "Message": { + "Message": { + "Version": 42, + "To": "t01234", + "From": "t01234", + "Nonce": 42, + "Value": "0", + "GasLimit": 9, + "GasFeeCap": "0", + "GasPremium": "0", + "Method": 1, + "Params": "Ynl0ZSBhcnJheQ==" + }, + "Signature": { + "Type": 2, + "Data": "Ynl0ZSBhcnJheQ==" + } + } +} +``` + +## Msig +The Msig methods are used to interact with multisig wallets on the +filecoin network + + +### MsigAddApprove +MsigAddApprove approves a previously proposed AddSigner message +It takes the following params: , , , +, , + + +Perms: sign + +Inputs: +```json +[ + "t01234", + "t01234", + 42, + "t01234", + "t01234", + true +] +``` + +Response: +```json +{ + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" +} +``` + +### MsigAddCancel +MsigAddCancel cancels a previously proposed AddSigner message +It takes the following params: , , , +, + + +Perms: sign + +Inputs: +```json +[ + "t01234", + "t01234", + 42, + "t01234", + true +] +``` + +Response: +```json +{ + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" +} +``` + +### MsigAddPropose +MsigAddPropose proposes adding a signer in the multisig +It takes the following params: , , +, + + +Perms: sign + +Inputs: +```json +[ + "t01234", + "t01234", + "t01234", + true +] +``` + +Response: +```json +{ + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" +} +``` + +### MsigApprove +MsigApprove approves a previously-proposed multisig message +It takes the following params: , , , , , +, , + + +Perms: sign + +Inputs: +```json +[ + "t01234", + 42, + "t01234", + "t01234", + "0", + "t01234", + 42, + "Ynl0ZSBhcnJheQ==" +] +``` + +Response: +```json +{ + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" +} +``` + +### MsigCancel +MsigCancel cancels a previously-proposed multisig message +It takes the following params: , , , , +, , + + +Perms: sign + +Inputs: +```json +[ + "t01234", + 42, + "t01234", + "0", + "t01234", + 42, + "Ynl0ZSBhcnJheQ==" +] +``` + +Response: +```json +{ + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" +} +``` + +### MsigCreate +MsigCreate creates a multisig wallet +It takes the following params: , , +, , + + +Perms: sign + +Inputs: +```json +[ + 42, + null, + 10101, + "0", + "t01234", + "0" +] +``` + +Response: +```json +{ + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" +} +``` + +### MsigGetAvailableBalance +MsigGetAvailableBalance returns the portion of a multisig's balance that can be withdrawn or spent + + +Perms: read + +Inputs: +```json +[ + "t01234", + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `"0"` + +### MsigGetVested +MsigGetVested returns the amount of FIL that vested in a multisig in a certain period. +It takes the following params: , , + + +Perms: read + +Inputs: +```json +[ + "t01234", + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ], + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `"0"` + +### MsigPropose +MsigPropose proposes a multisig message +It takes the following params: , , , +, , + + +Perms: sign + +Inputs: +```json +[ + "t01234", + "t01234", + "0", + "t01234", + 42, + "Ynl0ZSBhcnJheQ==" +] +``` + +Response: +```json +{ + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" +} +``` + +### MsigSwapApprove +MsigSwapApprove approves a previously proposed SwapSigner +It takes the following params: , , , +, , + + +Perms: sign + +Inputs: +```json +[ + "t01234", + "t01234", + 42, + "t01234", + "t01234", + "t01234" +] +``` + +Response: +```json +{ + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" +} +``` + +### MsigSwapCancel +MsigSwapCancel cancels a previously proposed SwapSigner message +It takes the following params: , , , +, + + +Perms: sign + +Inputs: +```json +[ + "t01234", + "t01234", + 42, + "t01234", + "t01234" +] +``` + +Response: +```json +{ + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" +} +``` + +### MsigSwapPropose +MsigSwapPropose proposes swapping 2 signers in the multisig +It takes the following params: , , +, + + +Perms: sign + +Inputs: +```json +[ + "t01234", + "t01234", + "t01234", + "t01234" +] +``` + +Response: +```json +{ + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" +} +``` + +## Net + + +### NetAddrsListen + + +Perms: read + +Inputs: `null` + +Response: +```json +{ + "Addrs": null, + "ID": "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf" +} +``` + +### NetAgentVersion + + +Perms: read + +Inputs: +```json +[ + "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf" +] +``` + +Response: `"string value"` + +### NetAutoNatStatus + + +Perms: read + +Inputs: `null` + +Response: +```json +{ + "Reachability": 1, + "PublicAddr": "string value" +} +``` + +### NetBandwidthStats + + +Perms: read + +Inputs: `null` + +Response: +```json +{ + "TotalIn": 9, + "TotalOut": 9, + "RateIn": 12.3, + "RateOut": 12.3 +} +``` + +### NetBandwidthStatsByPeer + + +Perms: read + +Inputs: `null` + +Response: +```json +{ + "12D3KooWSXmXLJmBR1M7i9RW9GQPNUhZSzXKzxDHWtAgNuJAbyEJ": { + "TotalIn": 174000, + "TotalOut": 12500, + "RateIn": 100, + "RateOut": 50 + } +} +``` + +### NetBandwidthStatsByProtocol + + +Perms: read + +Inputs: `null` + +Response: +```json +{ + "/fil/hello/1.0.0": { + "TotalIn": 174000, + "TotalOut": 12500, + "RateIn": 100, + "RateOut": 50 + } +} +``` + +### NetConnect + + +Perms: write + +Inputs: +```json +[ + { + "Addrs": null, + "ID": "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf" + } +] +``` + +Response: `{}` + +### NetConnectedness + + +Perms: read + +Inputs: +```json +[ + "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf" +] +``` + +Response: `1` + +### NetDisconnect + + +Perms: write + +Inputs: +```json +[ + "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf" +] +``` + +Response: `{}` + +### NetFindPeer + + +Perms: read + +Inputs: +```json +[ + "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf" +] +``` + +Response: +```json +{ + "Addrs": null, + "ID": "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf" +} +``` + +### NetPeers + + +Perms: read + +Inputs: `null` + +Response: `null` + +### NetPubsubScores + + +Perms: read + +Inputs: `null` + +Response: `null` + +## Paych +The Paych methods are for interacting with and managing payment channels + + +### PaychAllocateLane +There are not yet any comments for this method. + +Perms: sign + +Inputs: +```json +[ + "t01234" +] +``` + +Response: `42` + +### PaychAvailableFunds +There are not yet any comments for this method. + +Perms: sign + +Inputs: +```json +[ + "t01234" +] +``` + +Response: +```json +{ + "Channel": "\u003cempty\u003e", + "From": "t01234", + "To": "t01234", + "ConfirmedAmt": "0", + "PendingAmt": "0", + "PendingWaitSentinel": null, + "QueuedAmt": "0", + "VoucherReedeemedAmt": "0" +} +``` + +### PaychAvailableFundsByFromTo +There are not yet any comments for this method. + +Perms: sign + +Inputs: +```json +[ + "t01234", + "t01234" +] +``` + +Response: +```json +{ + "Channel": "\u003cempty\u003e", + "From": "t01234", + "To": "t01234", + "ConfirmedAmt": "0", + "PendingAmt": "0", + "PendingWaitSentinel": null, + "QueuedAmt": "0", + "VoucherReedeemedAmt": "0" +} +``` + +### PaychCollect +There are not yet any comments for this method. + +Perms: sign + +Inputs: +```json +[ + "t01234" +] +``` + +Response: +```json +{ + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" +} +``` + +### PaychGet +There are not yet any comments for this method. + +Perms: sign + +Inputs: +```json +[ + "t01234", + "t01234", + "0" +] +``` + +Response: +```json +{ + "Channel": "t01234", + "WaitSentinel": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +} +``` + +### PaychGetWaitReady +There are not yet any comments for this method. + +Perms: sign + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: `"t01234"` + +### PaychList +There are not yet any comments for this method. + +Perms: read + +Inputs: `null` + +Response: `null` + +### PaychNewPayment +There are not yet any comments for this method. + +Perms: sign + +Inputs: +```json +[ + "t01234", + "t01234", + null +] +``` + +Response: +```json +{ + "Channel": "t01234", + "WaitSentinel": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Vouchers": null +} +``` + +### PaychSettle +There are not yet any comments for this method. + +Perms: sign + +Inputs: +```json +[ + "t01234" +] +``` + +Response: +```json +{ + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" +} +``` + +### PaychStatus +There are not yet any comments for this method. + +Perms: read + +Inputs: +```json +[ + "t01234" +] +``` + +Response: +```json +{ + "ControlAddr": "t01234", + "Direction": 1 +} +``` + +### PaychVoucherAdd +There are not yet any comments for this method. + +Perms: write + +Inputs: +```json +[ + "t01234", + { + "ChannelAddr": "t01234", + "TimeLockMin": 10101, + "TimeLockMax": 10101, + "SecretPreimage": "Ynl0ZSBhcnJheQ==", + "Extra": { + "Actor": "t01234", + "Method": 1, + "Data": "Ynl0ZSBhcnJheQ==" + }, + "Lane": 42, + "Nonce": 42, + "Amount": "0", + "MinSettleHeight": 10101, + "Merges": null, + "Signature": { + "Type": 2, + "Data": "Ynl0ZSBhcnJheQ==" + } + }, + "Ynl0ZSBhcnJheQ==", + "0" +] +``` + +Response: `"0"` + +### PaychVoucherCheckSpendable +There are not yet any comments for this method. + +Perms: read + +Inputs: +```json +[ + "t01234", + { + "ChannelAddr": "t01234", + "TimeLockMin": 10101, + "TimeLockMax": 10101, + "SecretPreimage": "Ynl0ZSBhcnJheQ==", + "Extra": { + "Actor": "t01234", + "Method": 1, + "Data": "Ynl0ZSBhcnJheQ==" + }, + "Lane": 42, + "Nonce": 42, + "Amount": "0", + "MinSettleHeight": 10101, + "Merges": null, + "Signature": { + "Type": 2, + "Data": "Ynl0ZSBhcnJheQ==" + } + }, + "Ynl0ZSBhcnJheQ==", + "Ynl0ZSBhcnJheQ==" +] +``` + +Response: `true` + +### PaychVoucherCheckValid +There are not yet any comments for this method. + +Perms: read + +Inputs: +```json +[ + "t01234", + { + "ChannelAddr": "t01234", + "TimeLockMin": 10101, + "TimeLockMax": 10101, + "SecretPreimage": "Ynl0ZSBhcnJheQ==", + "Extra": { + "Actor": "t01234", + "Method": 1, + "Data": "Ynl0ZSBhcnJheQ==" + }, + "Lane": 42, + "Nonce": 42, + "Amount": "0", + "MinSettleHeight": 10101, + "Merges": null, + "Signature": { + "Type": 2, + "Data": "Ynl0ZSBhcnJheQ==" + } + } +] +``` + +Response: `{}` + +### PaychVoucherCreate +There are not yet any comments for this method. + +Perms: sign + +Inputs: +```json +[ + "t01234", + "0", + 42 +] +``` + +Response: +```json +{ + "Voucher": { + "ChannelAddr": "t01234", + "TimeLockMin": 10101, + "TimeLockMax": 10101, + "SecretPreimage": "Ynl0ZSBhcnJheQ==", + "Extra": { + "Actor": "t01234", + "Method": 1, + "Data": "Ynl0ZSBhcnJheQ==" + }, + "Lane": 42, + "Nonce": 42, + "Amount": "0", + "MinSettleHeight": 10101, + "Merges": null, + "Signature": { + "Type": 2, + "Data": "Ynl0ZSBhcnJheQ==" + } + }, + "Shortfall": "0" +} +``` + +### PaychVoucherList +There are not yet any comments for this method. + +Perms: write + +Inputs: +```json +[ + "t01234" +] +``` + +Response: `null` + +### PaychVoucherSubmit +There are not yet any comments for this method. + +Perms: sign + +Inputs: +```json +[ + "t01234", + { + "ChannelAddr": "t01234", + "TimeLockMin": 10101, + "TimeLockMax": 10101, + "SecretPreimage": "Ynl0ZSBhcnJheQ==", + "Extra": { + "Actor": "t01234", + "Method": 1, + "Data": "Ynl0ZSBhcnJheQ==" + }, + "Lane": 42, + "Nonce": 42, + "Amount": "0", + "MinSettleHeight": 10101, + "Merges": null, + "Signature": { + "Type": 2, + "Data": "Ynl0ZSBhcnJheQ==" + } + }, + "Ynl0ZSBhcnJheQ==", + "Ynl0ZSBhcnJheQ==" +] +``` + +Response: +```json +{ + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" +} +``` + +## State +The State methods are used to query, inspect, and interact with chain state. +All methods take a TipSetKey as a parameter. The state looked up is the state at that tipset. +A nil TipSetKey can be provided as a param, this will cause the heaviest tipset in the chain to be used. + + +### StateAccountKey +StateAccountKey returns the public key address of the given ID address + + +Perms: read + +Inputs: +```json +[ + "t01234", + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `"t01234"` + +### StateAllMinerFaults +StateAllMinerFaults returns all non-expired Faults that occur within lookback epochs of the given tipset + + +Perms: read + +Inputs: +```json +[ + 10101, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `null` + +### StateCall +StateCall runs the given message and returns its result without any persisted changes. + + +Perms: read + +Inputs: +```json +[ + { + "Version": 42, + "To": "t01234", + "From": "t01234", + "Nonce": 42, + "Value": "0", + "GasLimit": 9, + "GasFeeCap": "0", + "GasPremium": "0", + "Method": 1, + "Params": "Ynl0ZSBhcnJheQ==" + }, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "Msg": { + "Version": 42, + "To": "t01234", + "From": "t01234", + "Nonce": 42, + "Value": "0", + "GasLimit": 9, + "GasFeeCap": "0", + "GasPremium": "0", + "Method": 1, + "Params": "Ynl0ZSBhcnJheQ==" + }, + "MsgRct": { + "ExitCode": 0, + "Return": "Ynl0ZSBhcnJheQ==", + "GasUsed": 9 + }, + "ExecutionTrace": { + "Msg": { + "Version": 42, + "To": "t01234", + "From": "t01234", + "Nonce": 42, + "Value": "0", + "GasLimit": 9, + "GasFeeCap": "0", + "GasPremium": "0", + "Method": 1, + "Params": "Ynl0ZSBhcnJheQ==" + }, + "MsgRct": { + "ExitCode": 0, + "Return": "Ynl0ZSBhcnJheQ==", + "GasUsed": 9 + }, + "Error": "string value", + "Duration": 60000000000, + "GasCharges": null, + "Subcalls": null + }, + "Error": "string value", + "Duration": 60000000000 +} +``` + +### StateChangedActors +StateChangedActors returns all the actors whose states change between the two given state CIDs +TODO: Should this take tipset keys instead? + + +Perms: read + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: +```json +{ + "t01236": { + "Code": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Head": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Nonce": 42, + "Balance": "0" + } +} +``` + +### StateCirculatingSupply +StateCirculatingSupply returns the circulating supply of Filecoin at the given tipset + + +Perms: read + +Inputs: +```json +[ + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "FilVested": "0", + "FilMined": "0", + "FilBurnt": "0", + "FilLocked": "0", + "FilCirculating": "0" +} +``` + +### StateCompute +StateCompute is a flexible command that applies the given messages on the given tipset. +The messages are run as though the VM were at the provided height. + + +Perms: read + +Inputs: +```json +[ + 10101, + null, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "Root": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Trace": null +} +``` + +### StateDealProviderCollateralBounds +StateDealProviderCollateralBounds returns the min and max collateral a storage provider +can issue. It takes the deal size and verified status as parameters. + + +Perms: read + +Inputs: +```json +[ + 1032, + true, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "Min": "0", + "Max": "0" +} +``` + +### StateGetActor +StateGetActor returns the indicated actor's nonce and balance. + + +Perms: read + +Inputs: +```json +[ + "t01234", + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "Code": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Head": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Nonce": 42, + "Balance": "0" +} +``` + +### StateGetReceipt +StateGetReceipt returns the message receipt for the given message + + +Perms: read + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "ExitCode": 0, + "Return": "Ynl0ZSBhcnJheQ==", + "GasUsed": 9 +} +``` + +### StateListActors +StateListActors returns the addresses of every actor in the state + + +Perms: read + +Inputs: +```json +[ + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `null` + +### StateListMessages +StateListMessages looks back and returns all messages with a matching to or from address, stopping at the given height. + + +Perms: read + +Inputs: +```json +[ + { + "Version": 42, + "To": "t01234", + "From": "t01234", + "Nonce": 42, + "Value": "0", + "GasLimit": 9, + "GasFeeCap": "0", + "GasPremium": "0", + "Method": 1, + "Params": "Ynl0ZSBhcnJheQ==" + }, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ], + 10101 +] +``` + +Response: `null` + +### StateListMiners +StateListMiners returns the addresses of every miner that has claimed power in the Power Actor + + +Perms: read + +Inputs: +```json +[ + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `null` + +### StateLookupID +StateLookupID retrieves the ID address of the given address + + +Perms: read + +Inputs: +```json +[ + "t01234", + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `"t01234"` + +### StateMarketBalance +StateMarketBalance looks up the Escrow and Locked balances of the given address in the Storage Market + + +Perms: read + +Inputs: +```json +[ + "t01234", + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "Escrow": "0", + "Locked": "0" +} +``` + +### StateMarketDeals +StateMarketDeals returns information about every deal in the Storage Market + + +Perms: read + +Inputs: +```json +[ + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "t026363": { + "Proposal": { + "PieceCID": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "PieceSize": 1032, + "VerifiedDeal": true, + "Client": "t01234", + "Provider": "t01234", + "Label": "string value", + "StartEpoch": 10101, + "EndEpoch": 10101, + "StoragePricePerEpoch": "0", + "ProviderCollateral": "0", + "ClientCollateral": "0" + }, + "State": { + "SectorStartEpoch": 10101, + "LastUpdatedEpoch": 10101, + "SlashEpoch": 10101 + } + } +} +``` + +### StateMarketParticipants +StateMarketParticipants returns the Escrow and Locked balances of every participant in the Storage Market + + +Perms: read + +Inputs: +```json +[ + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "t026363": { + "Escrow": "0", + "Locked": "0" + } +} +``` + +### StateMarketStorageDeal +StateMarketStorageDeal returns information about the indicated deal + + +Perms: read + +Inputs: +```json +[ + 5432, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "Proposal": { + "PieceCID": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "PieceSize": 1032, + "VerifiedDeal": true, + "Client": "t01234", + "Provider": "t01234", + "Label": "string value", + "StartEpoch": 10101, + "EndEpoch": 10101, + "StoragePricePerEpoch": "0", + "ProviderCollateral": "0", + "ClientCollateral": "0" + }, + "State": { + "SectorStartEpoch": 10101, + "LastUpdatedEpoch": 10101, + "SlashEpoch": 10101 + } +} +``` + +### StateMinerActiveSectors +StateMinerActiveSectors returns info about sectors that a given miner is actively proving. + + +Perms: read + +Inputs: +```json +[ + "t01234", + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `null` + +### StateMinerAvailableBalance +StateMinerAvailableBalance returns the portion of a miner's balance that can be withdrawn or spent + + +Perms: read + +Inputs: +```json +[ + "t01234", + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `"0"` + +### StateMinerDeadlines +StateMinerDeadlines returns all the proving deadlines for the given miner + + +Perms: read + +Inputs: +```json +[ + "t01234", + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `null` + +### StateMinerFaults +StateMinerFaults returns a bitfield indicating the faulty sectors of the given miner + + +Perms: read + +Inputs: +```json +[ + "t01234", + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +[ + 5, + 1 +] +``` + +### StateMinerInfo +StateMinerInfo returns info about the indicated miner + + +Perms: read + +Inputs: +```json +[ + "t01234", + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "Owner": "t01234", + "Worker": "t01234", + "NewWorker": "t01234", + "ControlAddresses": null, + "WorkerChangeEpoch": 10101, + "PeerId": "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf", + "Multiaddrs": null, + "SealProofType": 3, + "SectorSize": 34359738368, + "WindowPoStPartitionSectors": 42 +} +``` + +### StateMinerInitialPledgeCollateral +StateMinerInitialPledgeCollateral returns the initial pledge collateral for the specified miner's sector + + +Perms: read + +Inputs: +```json +[ + "t01234", + { + "SealProof": 3, + "SectorNumber": 9, + "SealedCID": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "SealRandEpoch": 10101, + "DealIDs": null, + "Expiration": 10101, + "ReplaceCapacity": true, + "ReplaceSectorDeadline": 42, + "ReplaceSectorPartition": 42, + "ReplaceSectorNumber": 9 + }, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `"0"` + +### StateMinerPartitions +StateMinerPartitions loads miner partitions for the specified miner/deadline + + +Perms: read + +Inputs: +```json +[ + "t01234", + 42, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `null` + +### StateMinerPower +StateMinerPower returns the power of the indicated miner + + +Perms: read + +Inputs: +```json +[ + "t01234", + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "MinerPower": { + "RawBytePower": "0", + "QualityAdjPower": "0" + }, + "TotalPower": { + "RawBytePower": "0", + "QualityAdjPower": "0" + } +} +``` + +### StateMinerPreCommitDepositForPower +StateMinerInitialPledgeCollateral returns the precommit deposit for the specified miner's sector + + +Perms: read + +Inputs: +```json +[ + "t01234", + { + "SealProof": 3, + "SectorNumber": 9, + "SealedCID": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "SealRandEpoch": 10101, + "DealIDs": null, + "Expiration": 10101, + "ReplaceCapacity": true, + "ReplaceSectorDeadline": 42, + "ReplaceSectorPartition": 42, + "ReplaceSectorNumber": 9 + }, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `"0"` + +### StateMinerProvingDeadline +StateMinerProvingDeadline calculates the deadline at some epoch for a proving period +and returns the deadline-related calculations. + + +Perms: read + +Inputs: +```json +[ + "t01234", + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "CurrentEpoch": 10101, + "PeriodStart": 10101, + "Index": 42, + "Open": 10101, + "Close": 10101, + "Challenge": 10101, + "FaultCutoff": 10101, + "WPoStPeriodDeadlines": 42, + "WPoStProvingPeriod": 10101, + "WPoStChallengeWindow": 10101, + "WPoStChallengeLookback": 10101, + "FaultDeclarationCutoff": 10101 +} +``` + +### StateMinerRecoveries +StateMinerRecoveries returns a bitfield indicating the recovering sectors of the given miner + + +Perms: read + +Inputs: +```json +[ + "t01234", + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +[ + 5, + 1 +] +``` + +### StateMinerSectorCount +StateMinerSectorCount returns the number of sectors in a miner's sector set and proving set + + +Perms: read + +Inputs: +```json +[ + "t01234", + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "Sectors": 42, + "Active": 42 +} +``` + +### StateMinerSectors +StateMinerSectors returns info about the given miner's sectors. If the filter bitfield is nil, all sectors are included. +If the filterOut boolean is set to true, any sectors in the filter are excluded. +If false, only those sectors in the filter are included. + + +Perms: read + +Inputs: +```json +[ + "t01234", + [ + 0 + ], + true, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `null` + +### StateMsgGasCost +StateMsgGasCost searches for a message in the chain, and returns details of the messages gas costs, including the penalty and miner tip + + +Perms: read + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "Message": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "GasUsed": "0", + "BaseFeeBurn": "0", + "OverEstimationBurn": "0", + "MinerPenalty": "0", + "MinerTip": "0", + "Refund": "0", + "TotalCost": "0" +} +``` + +### StateNetworkName +StateNetworkName returns the name of the network the node is synced to + + +Perms: read + +Inputs: `null` + +Response: `"lotus"` + +### StateReadState +StateReadState returns the indicated actor's state. + + +Perms: read + +Inputs: +```json +[ + "t01234", + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "Balance": "0", + "State": {} +} +``` + +### StateReplay +StateReplay returns the result of executing the indicated message, assuming it was executed in the indicated tipset. + + +Perms: read + +Inputs: +```json +[ + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ], + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: +```json +{ + "Msg": { + "Version": 42, + "To": "t01234", + "From": "t01234", + "Nonce": 42, + "Value": "0", + "GasLimit": 9, + "GasFeeCap": "0", + "GasPremium": "0", + "Method": 1, + "Params": "Ynl0ZSBhcnJheQ==" + }, + "MsgRct": { + "ExitCode": 0, + "Return": "Ynl0ZSBhcnJheQ==", + "GasUsed": 9 + }, + "ExecutionTrace": { + "Msg": { + "Version": 42, + "To": "t01234", + "From": "t01234", + "Nonce": 42, + "Value": "0", + "GasLimit": 9, + "GasFeeCap": "0", + "GasPremium": "0", + "Method": 1, + "Params": "Ynl0ZSBhcnJheQ==" + }, + "MsgRct": { + "ExitCode": 0, + "Return": "Ynl0ZSBhcnJheQ==", + "GasUsed": 9 + }, + "Error": "string value", + "Duration": 60000000000, + "GasCharges": null, + "Subcalls": null + }, + "Error": "string value", + "Duration": 60000000000 +} +``` + +### StateSearchMsg +StateSearchMsg searches for a message in the chain, and returns its receipt and the tipset where it was executed + + +Perms: read + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: +```json +{ + "Message": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Receipt": { + "ExitCode": 0, + "Return": "Ynl0ZSBhcnJheQ==", + "GasUsed": 9 + }, + "ReturnDec": {}, + "TipSet": [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ], + "Height": 10101 +} +``` + +### StateSectorExpiration +StateSectorExpiration returns epoch at which given sector will expire + + +Perms: read + +Inputs: +```json +[ + "t01234", + 9, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "OnTime": 10101, + "Early": 10101 +} +``` + +### StateSectorGetInfo +StateSectorGetInfo returns the on-chain info for the specified miner's sector. Returns null in case the sector info isn't found +NOTE: returned info.Expiration may not be accurate in some cases, use StateSectorExpiration to get accurate +expiration epoch + + +Perms: read + +Inputs: +```json +[ + "t01234", + 9, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "SectorNumber": 9, + "SealProof": 3, + "SealedCID": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "DealIDs": null, + "Activation": 10101, + "Expiration": 10101, + "DealWeight": "0", + "VerifiedDealWeight": "0", + "InitialPledge": "0", + "ExpectedDayReward": "0", + "ExpectedStoragePledge": "0" +} +``` + +### StateSectorPartition +StateSectorPartition finds deadline/partition with the specified sector + + +Perms: read + +Inputs: +```json +[ + "t01234", + 9, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "Deadline": 42, + "Partition": 42 +} +``` + +### StateSectorPreCommitInfo +StateSectorPreCommitInfo returns the PreCommit info for the specified miner's sector + + +Perms: read + +Inputs: +```json +[ + "t01234", + 9, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "Info": { + "SealProof": 3, + "SectorNumber": 9, + "SealedCID": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "SealRandEpoch": 10101, + "DealIDs": null, + "Expiration": 10101, + "ReplaceCapacity": true, + "ReplaceSectorDeadline": 42, + "ReplaceSectorPartition": 42, + "ReplaceSectorNumber": 9 + }, + "PreCommitDeposit": "0", + "PreCommitEpoch": 10101, + "DealWeight": "0", + "VerifiedDealWeight": "0" +} +``` + +### StateVerifiedClientStatus +StateVerifiedClientStatus returns the data cap for the given address. +Returns nil if there is no entry in the data cap table for the +address. + + +Perms: read + +Inputs: +```json +[ + "t01234", + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `"0"` + +### StateWaitMsg +StateWaitMsg looks back in the chain for a message. If not found, it blocks until the +message arrives on chain, and gets to the indicated confidence depth. + + +Perms: read + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + 42 +] +``` + +Response: +```json +{ + "Message": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Receipt": { + "ExitCode": 0, + "Return": "Ynl0ZSBhcnJheQ==", + "GasUsed": 9 + }, + "ReturnDec": {}, + "TipSet": [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ], + "Height": 10101 +} +``` + +## Sync +The Sync method group contains methods for interacting with and +observing the lotus sync service. + + +### SyncCheckBad +SyncCheckBad checks if a block was marked as bad, and if it was, returns +the reason. + + +Perms: read + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: `"string value"` + +### SyncCheckpoint +SyncCheckpoint marks a blocks as checkpointed, meaning that it won't ever fork away from it. + + +Perms: admin + +Inputs: +```json +[ + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `{}` + +### SyncIncomingBlocks +SyncIncomingBlocks returns a channel streaming incoming, potentially not +yet synced block headers. + + +Perms: read + +Inputs: `null` + +Response: +```json +{ + "Miner": "t01234", + "Ticket": { + "VRFProof": "Ynl0ZSBhcnJheQ==" + }, + "ElectionProof": { + "WinCount": 9, + "VRFProof": "Ynl0ZSBhcnJheQ==" + }, + "BeaconEntries": null, + "WinPoStProof": null, + "Parents": null, + "ParentWeight": "0", + "Height": 10101, + "ParentStateRoot": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "ParentMessageReceipts": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Messages": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "BLSAggregate": { + "Type": 2, + "Data": "Ynl0ZSBhcnJheQ==" + }, + "Timestamp": 42, + "BlockSig": { + "Type": 2, + "Data": "Ynl0ZSBhcnJheQ==" + }, + "ForkSignaling": 42, + "ParentBaseFee": "0" +} +``` + +### SyncMarkBad +SyncMarkBad marks a blocks as bad, meaning that it won't ever by synced. +Use with extreme caution. + + +Perms: admin + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: `{}` + +### SyncState +SyncState returns the current status of the lotus sync system. + + +Perms: read + +Inputs: `null` + +Response: +```json +{ + "ActiveSyncs": null +} +``` + +### SyncSubmitBlock +SyncSubmitBlock can be used to submit a newly created block to the. +network through this node + + +Perms: write + +Inputs: +```json +[ + { + "Header": { + "Miner": "t01234", + "Ticket": { + "VRFProof": "Ynl0ZSBhcnJheQ==" + }, + "ElectionProof": { + "WinCount": 9, + "VRFProof": "Ynl0ZSBhcnJheQ==" + }, + "BeaconEntries": null, + "WinPoStProof": null, + "Parents": null, + "ParentWeight": "0", + "Height": 10101, + "ParentStateRoot": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "ParentMessageReceipts": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Messages": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "BLSAggregate": { + "Type": 2, + "Data": "Ynl0ZSBhcnJheQ==" + }, + "Timestamp": 42, + "BlockSig": { + "Type": 2, + "Data": "Ynl0ZSBhcnJheQ==" + }, + "ForkSignaling": 42, + "ParentBaseFee": "0" + }, + "BlsMessages": null, + "SecpkMessages": null + } +] +``` + +Response: `{}` + +### SyncUnmarkBad +SyncUnmarkBad unmarks a blocks as bad, making it possible to be validated and synced again. + + +Perms: admin + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: `{}` + +## Wallet + + +### WalletBalance +WalletBalance returns the balance of the given address at the current head of the chain. + + +Perms: read + +Inputs: +```json +[ + "t01234" +] +``` + +Response: `"0"` + +### WalletDefaultAddress +WalletDefaultAddress returns the address marked as default in the wallet. + + +Perms: write + +Inputs: `null` + +Response: `"t01234"` + +### WalletDelete +WalletDelete deletes an address from the wallet. + + +Perms: write + +Inputs: +```json +[ + "t01234" +] +``` + +Response: `{}` + +### WalletExport +WalletExport returns the private key of an address in the wallet. + + +Perms: admin + +Inputs: +```json +[ + "t01234" +] +``` + +Response: +```json +{ + "Type": "string value", + "PrivateKey": "Ynl0ZSBhcnJheQ==" +} +``` + +### WalletHas +WalletHas indicates whether the given address is in the wallet. + + +Perms: write + +Inputs: +```json +[ + "t01234" +] +``` + +Response: `true` + +### WalletImport +WalletImport receives a KeyInfo, which includes a private key, and imports it into the wallet. + + +Perms: admin + +Inputs: +```json +[ + { + "Type": "string value", + "PrivateKey": "Ynl0ZSBhcnJheQ==" + } +] +``` + +Response: `"t01234"` + +### WalletList +WalletList lists all the addresses in the wallet. + + +Perms: write + +Inputs: `null` + +Response: `null` + +### WalletNew +WalletNew creates a new address in the wallet with the given sigType. + + +Perms: write + +Inputs: +```json +[ + 2 +] +``` + +Response: `"t01234"` + +### WalletSetDefault +WalletSetDefault marks the given address as as the default one. + + +Perms: admin + +Inputs: +```json +[ + "t01234" +] +``` + +Response: `{}` + +### WalletSign +WalletSign signs the given bytes using the given address. + + +Perms: sign + +Inputs: +```json +[ + "t01234", + "Ynl0ZSBhcnJheQ==" +] +``` + +Response: +```json +{ + "Type": 2, + "Data": "Ynl0ZSBhcnJheQ==" +} +``` + +### WalletSignMessage +WalletSignMessage signs the given message using the given address. + + +Perms: sign + +Inputs: +```json +[ + "t01234", + { + "Version": 42, + "To": "t01234", + "From": "t01234", + "Nonce": 42, + "Value": "0", + "GasLimit": 9, + "GasFeeCap": "0", + "GasPremium": "0", + "Method": 1, + "Params": "Ynl0ZSBhcnJheQ==" + } +] +``` + +Response: +```json +{ + "Message": { + "Version": 42, + "To": "t01234", + "From": "t01234", + "Nonce": 42, + "Value": "0", + "GasLimit": 9, + "GasFeeCap": "0", + "GasPremium": "0", + "Method": 1, + "Params": "Ynl0ZSBhcnJheQ==" + }, + "Signature": { + "Type": 2, + "Data": "Ynl0ZSBhcnJheQ==" + } +} +``` + +### WalletVerify +WalletVerify takes an address, a signature, and some bytes, and indicates whether the signature is valid. +The address does not have to be in the wallet. + + +Perms: read + +Inputs: +```json +[ + "t01234", + "Ynl0ZSBhcnJheQ==", + { + "Type": 2, + "Data": "Ynl0ZSBhcnJheQ==" + } +] +``` + +Response: `true` + From dfd28ab0c44d7f032d6dff4e994b38491268ff0e Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 17 Sep 2020 20:56:43 +0200 Subject: [PATCH 197/885] Fix links in READMEs. --- README.md | 4 ++-- tools/dockers/docker-examples/README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6c1e23efa..fa432bf7d 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@

- + Project Lotus Logo

@@ -18,7 +18,7 @@ Lotus is an implementation of the Filecoin Distributed Storage Network. For more ## Building & Documentation -For instructions on how to build lotus from source, please visit [https://lotu.sh](https://lotu.sh) or read the source [here](https://github.com/filecoin-project/lotus/tree/master/documentation). +For instructions on how to build, install and setup lotus, please visit [https://docs.filecoin.io/get-started/lotus](https://docs.filecoin.io/get-started/lotus/). ## Reporting a Vulnerability diff --git a/tools/dockers/docker-examples/README.md b/tools/dockers/docker-examples/README.md index 28553653c..3b8c34480 100644 --- a/tools/dockers/docker-examples/README.md +++ b/tools/dockers/docker-examples/README.md @@ -11,7 +11,7 @@ In this `docker-examples/` directory are community-contributed Docker and Docker - local node for a developer (`api-local-`) - hosted endpoint for apps / multiple developers (`api-hosted-`) - **For a local devnet or shared devnet** - - basic local devnet (also see [lotus docs on setting up a local devnet](https://lotu.sh/en+setup-local-dev-net)) + - basic local devnet (also see [lotus docs on setting up a local devnet](https://docs.filecoin.io/build/local-devnet/)) - shared devnet From a894c1e8e626ca933b72a3864026ca05bef6a74c Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Thu, 17 Sep 2020 12:30:26 -0700 Subject: [PATCH 198/885] feat(markets): upgrade markets v0.6.1 --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 65c2addb8..41bc727b3 100644 --- a/go.mod +++ b/go.mod @@ -25,9 +25,9 @@ require ( github.com/filecoin-project/go-bitfield v0.2.0 github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 - github.com/filecoin-project/go-data-transfer v0.6.3 + github.com/filecoin-project/go-data-transfer v0.6.4 github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f - github.com/filecoin-project/go-fil-markets v0.6.1-0.20200911011457-2959ccca6a3c + github.com/filecoin-project/go-fil-markets v0.6.1 github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200822201400-474f4fdccc52 github.com/filecoin-project/go-multistore v0.0.3 github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 diff --git a/go.sum b/go.sum index 77b556fb2..11d607034 100644 --- a/go.sum +++ b/go.sum @@ -222,12 +222,12 @@ github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:a github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= -github.com/filecoin-project/go-data-transfer v0.6.3 h1:7TLwm8nuodHYD/uiwJjKc/PGRR+LwqM8jmlZqgWuUfY= -github.com/filecoin-project/go-data-transfer v0.6.3/go.mod h1:PmBKVXkhh67/tnEdJXQwDHl5mT+7Tbcwe1NPninqhnM= +github.com/filecoin-project/go-data-transfer v0.6.4 h1:Q08ABa+cOTOLoAyHeA94fPLcwu53p6eeAaxMxQb0m0A= +github.com/filecoin-project/go-data-transfer v0.6.4/go.mod h1:PmBKVXkhh67/tnEdJXQwDHl5mT+7Tbcwe1NPninqhnM= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f h1:GxJzR3oRIMTPtpZ0b7QF8FKPK6/iPAc7trhlL5k/g+s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= -github.com/filecoin-project/go-fil-markets v0.6.1-0.20200911011457-2959ccca6a3c h1:YGoyYmELQ0LHwDj/WcOvY3oYt+3iM0wdrAhqJQUAIy4= -github.com/filecoin-project/go-fil-markets v0.6.1-0.20200911011457-2959ccca6a3c/go.mod h1:PLr9svZxsnHkae1Ky7+66g7fP9AlneVxIVu+oSMq56A= +github.com/filecoin-project/go-fil-markets v0.6.1 h1:qCFLcVkUCbxwEfH/6EcqTuQvibXt/TXZr+vh8tWv/BQ= +github.com/filecoin-project/go-fil-markets v0.6.1/go.mod h1:dBJl59dAyl8+cGVb/ONPlEQW4+YzhjI3d6bxLfHVpX0= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24= github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200822201400-474f4fdccc52 h1:FXtCp0ybqdQL9knb3OGDpkNTaBbPxgkqPeWKotUwkH0= From 18bc59b93befe07fa4fe9f76652616f743c71fa2 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 17 Sep 2020 13:52:16 -0700 Subject: [PATCH 199/885] uncruft --- api/types.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/api/types.go b/api/types.go index 53c0fe203..a69aa28d9 100644 --- a/api/types.go +++ b/api/types.go @@ -107,5 +107,3 @@ func NewDataTransferChannel(hostID peer.ID, channelState datatransfer.ChannelSta } return channel } - -// type TODO (stebalien): this was here, why was this here? From 708aa368e1d6867647c2424fc4b1290ce3226d81 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 17 Sep 2020 13:53:18 -0700 Subject: [PATCH 200/885] remove dep on v0 miner --- api/test/window_post.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/api/test/window_post.go b/api/test/window_post.go index e2bf5f36e..6b41ba6c6 100644 --- a/api/test/window_post.go +++ b/api/test/window_post.go @@ -9,8 +9,6 @@ import ( lotusminer "github.com/filecoin-project/lotus/chain/actors/builtin/miner" cbor "github.com/ipfs/go-ipld-cbor" - v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" - "os" "strings" "testing" @@ -165,7 +163,7 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector head, err := client.ChainHead(ctx) require.NoError(t, err) - if head.Height() > di.PeriodStart+(v0miner.WPoStProvingPeriod)+2 { + if head.Height() > di.PeriodStart+(di.WPoStProvingPeriod)+2 { break } From ae38970526b3ea65713333a74ae89f0697ef7e90 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 17 Sep 2020 14:53:57 -0700 Subject: [PATCH 201/885] remove WpostProvignPeriod function --- api/test/window_post.go | 17 ++--------------- chain/actors/builtin/miner/miner.go | 1 - chain/actors/builtin/miner/v0.go | 3 --- 3 files changed, 2 insertions(+), 19 deletions(-) diff --git a/api/test/window_post.go b/api/test/window_post.go index 6b41ba6c6..683489a91 100644 --- a/api/test/window_post.go +++ b/api/test/window_post.go @@ -4,11 +4,6 @@ import ( "context" "fmt" - "github.com/filecoin-project/lotus/api/apibstore" - "github.com/filecoin-project/lotus/chain/actors/adt" - lotusminer "github.com/filecoin-project/lotus/chain/actors/builtin/miner" - cbor "github.com/ipfs/go-ipld-cbor" - "os" "strings" "testing" @@ -182,14 +177,6 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector require.Equal(t, p.MinerPower, p.TotalPower) require.Equal(t, p.MinerPower.RawBytePower, types.NewInt(uint64(ssz)*uint64(nSectors+GenesisPreseals))) - store := cbor.NewCborStore(apibstore.NewAPIBlockstore(client)) - - mact, err := client.StateGetActor(ctx, maddr, types.EmptyTSK) - require.NoError(t, err) - - minState, err := lotusminer.Load(adt.WrapStore(ctx, store), mact) - require.NoError(t, err) - fmt.Printf("Drop some sectors\n") // Drop 2 sectors from deadline 2 partition 0 (full partition / deadline) @@ -254,7 +241,7 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector head, err := client.ChainHead(ctx) require.NoError(t, err) - if head.Height() > di.PeriodStart+(minState.WpostProvingPeriod())+2 { + if head.Height() > di.PeriodStart+(di.WPoStProvingPeriod)+2 { break } @@ -284,7 +271,7 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector head, err := client.ChainHead(ctx) require.NoError(t, err) - if head.Height() > di.PeriodStart+(minState.WpostProvingPeriod())+2 { + if head.Height() > di.PeriodStart+di.WPoStProvingPeriod+2 { break } diff --git a/chain/actors/builtin/miner/miner.go b/chain/actors/builtin/miner/miner.go index d754d0b74..cb34dd557 100644 --- a/chain/actors/builtin/miner/miner.go +++ b/chain/actors/builtin/miner/miner.go @@ -51,7 +51,6 @@ type State interface { Info() (MinerInfo, error) DeadlineInfo(epoch abi.ChainEpoch) *dline.Info - WpostProvingPeriod() abi.ChainEpoch } type Deadline interface { diff --git a/chain/actors/builtin/miner/v0.go b/chain/actors/builtin/miner/v0.go index b3fe594d1..74812e85a 100644 --- a/chain/actors/builtin/miner/v0.go +++ b/chain/actors/builtin/miner/v0.go @@ -243,9 +243,6 @@ func (s *v0State) Info() (MinerInfo, error) { func (s *v0State) DeadlineInfo(epoch abi.ChainEpoch) *dline.Info { return s.State.DeadlineInfo(epoch) } -func (s *v0State) WpostProvingPeriod() abi.ChainEpoch { - return v0miner.WPoStProvingPeriod -} func (d *v0Deadline) LoadPartition(idx uint64) (Partition, error) { p, err := d.Deadline.LoadPartition(d.store, idx) From dc58f71604b6dbf63c21da966a2d4e146b36abeb Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 17 Sep 2020 14:59:10 -0700 Subject: [PATCH 202/885] remove unnecessary code Go does have some nice features, once in a while. --- chain/actors/builtin/market/v0.go | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/chain/actors/builtin/market/v0.go b/chain/actors/builtin/market/v0.go index 92ebb59ba..d2e77fa92 100644 --- a/chain/actors/builtin/market/v0.go +++ b/chain/actors/builtin/market/v0.go @@ -184,11 +184,7 @@ func (d *v0MarketStatesDiffer) Remove(key uint64, val *typegen.Deferred) error { } func fromV0DealState(v0 market.DealState) DealState { - return DealState{ - SectorStartEpoch: v0.SectorStartEpoch, - SlashEpoch: v0.SlashEpoch, - LastUpdatedEpoch: v0.LastUpdatedEpoch, - } + return (DealState)(v0) } type v0DealProposals struct { @@ -234,19 +230,7 @@ type v0MarketProposalsDiffer struct { } func fromV0DealProposal(v0 market.DealProposal) DealProposal { - return DealProposal{ - PieceCID: v0.PieceCID, - PieceSize: v0.PieceSize, - VerifiedDeal: v0.VerifiedDeal, - Client: v0.Client, - Provider: v0.Provider, - Label: v0.Label, - StartEpoch: v0.StartEpoch, - EndEpoch: v0.EndEpoch, - StoragePricePerEpoch: v0.StoragePricePerEpoch, - ProviderCollateral: v0.ProviderCollateral, - ClientCollateral: v0.ClientCollateral, - } + return (DealProposal)(v0) } func (d *v0MarketProposalsDiffer) Add(key uint64, val *typegen.Deferred) error { From 5bcfee00425c28988db1ef95c5633788bde7dea1 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 17 Sep 2020 16:08:54 -0700 Subject: [PATCH 203/885] make market diffs work across version upgrades --- chain/actors/builtin/market/market.go | 7 ++ chain/actors/builtin/market/util.go | 91 ++++++++++++++++++++ chain/actors/builtin/market/v0.go | 114 ++++++-------------------- 3 files changed, 121 insertions(+), 91 deletions(-) create mode 100644 chain/actors/builtin/market/util.go diff --git a/chain/actors/builtin/market/market.go b/chain/actors/builtin/market/market.go index 73c2528d0..1c11c027e 100644 --- a/chain/actors/builtin/market/market.go +++ b/chain/actors/builtin/market/market.go @@ -8,6 +8,7 @@ import ( "github.com/filecoin-project/go-state-types/cbor" v0builtin "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/types" @@ -51,12 +52,18 @@ type BalanceTable interface { type DealStates interface { Get(id abi.DealID) (*DealState, bool, error) Diff(DealStates) (*DealStateChanges, error) + + array() adt.Array + decode(*cbg.Deferred) (*DealState, error) } type DealProposals interface { ForEach(cb func(id abi.DealID, dp DealProposal) error) error Get(id abi.DealID) (*DealProposal, bool, error) Diff(DealProposals) (*DealProposalChanges, error) + + array() adt.Array + decode(*cbg.Deferred) (*DealProposal, error) } type DealState struct { diff --git a/chain/actors/builtin/market/util.go b/chain/actors/builtin/market/util.go new file mode 100644 index 000000000..92fdd6823 --- /dev/null +++ b/chain/actors/builtin/market/util.go @@ -0,0 +1,91 @@ +package market + +import ( + "fmt" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/actors/adt" + cbg "github.com/whyrusleeping/cbor-gen" +) + +func diffDealProposals(pre, cur DealProposals) (*DealProposalChanges, error) { + results := new(DealProposalChanges) + if err := adt.DiffAdtArray(pre.array(), cur.array(), &marketProposalsDiffer{results, pre, cur}); err != nil { + return nil, fmt.Errorf("diffing deal states: %w", err) + } + return results, nil +} + +type marketProposalsDiffer struct { + Results *DealProposalChanges + pre, cur DealProposals +} + +func (d *marketProposalsDiffer) Add(key uint64, val *cbg.Deferred) error { + dp, err := d.cur.decode(val) + if err != nil { + return err + } + d.Results.Added = append(d.Results.Added, ProposalIDState{abi.DealID(key), *dp}) + return nil +} + +func (d *marketProposalsDiffer) Modify(key uint64, from, to *cbg.Deferred) error { + // short circuit, DealProposals are static + return nil +} + +func (d *marketProposalsDiffer) Remove(key uint64, val *cbg.Deferred) error { + dp, err := d.pre.decode(val) + if err != nil { + return err + } + d.Results.Removed = append(d.Results.Removed, ProposalIDState{abi.DealID(key), *dp}) + return nil +} + +func diffDealStates(pre, cur DealStates) (*DealStateChanges, error) { + results := new(DealStateChanges) + if err := adt.DiffAdtArray(pre.array(), cur.array(), &marketStatesDiffer{results, pre, cur}); err != nil { + return nil, fmt.Errorf("diffing deal states: %w", err) + } + return results, nil +} + +type marketStatesDiffer struct { + Results *DealStateChanges + pre, cur DealStates +} + +func (d *marketStatesDiffer) Add(key uint64, val *cbg.Deferred) error { + ds, err := d.cur.decode(val) + if err != nil { + return err + } + d.Results.Added = append(d.Results.Added, DealIDState{abi.DealID(key), *ds}) + return nil +} + +func (d *marketStatesDiffer) Modify(key uint64, from, to *cbg.Deferred) error { + dsFrom, err := d.pre.decode(from) + if err != nil { + return err + } + dsTo, err := d.cur.decode(to) + if err != nil { + return err + } + if *dsFrom != *dsTo { + d.Results.Modified = append(d.Results.Modified, DealStateChange{abi.DealID(key), dsFrom, dsTo}) + } + return nil +} + +func (d *marketStatesDiffer) Remove(key uint64, val *cbg.Deferred) error { + ds, err := d.pre.decode(val) + if err != nil { + return err + } + d.Results.Removed = append(d.Results.Removed, DealIDState{abi.DealID(key), *ds}) + return nil +} diff --git a/chain/actors/builtin/market/v0.go b/chain/actors/builtin/market/v0.go index d2e77fa92..587446749 100644 --- a/chain/actors/builtin/market/v0.go +++ b/chain/actors/builtin/market/v0.go @@ -2,8 +2,6 @@ package market import ( "bytes" - "errors" - "fmt" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" @@ -11,7 +9,7 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/specs-actors/actors/builtin/market" v0adt "github.com/filecoin-project/specs-actors/actors/util/adt" - typegen "github.com/whyrusleeping/cbor-gen" + cbg "github.com/whyrusleeping/cbor-gen" ) type v0State struct { @@ -127,60 +125,20 @@ func (s *v0DealStates) Get(dealID abi.DealID) (*DealState, bool, error) { } func (s *v0DealStates) Diff(other DealStates) (*DealStateChanges, error) { - v0other, ok := other.(*v0DealStates) - if !ok { - // TODO handle this if possible on a case by case basis but for now, just fail - return nil, errors.New("cannot compare deal states across versions") - } - results := new(DealStateChanges) - if err := adt.DiffAdtArray(s.Array, v0other.Array, &v0MarketStatesDiffer{results}); err != nil { - return nil, fmt.Errorf("diffing deal states: %w", err) - } - - return results, nil + return diffDealStates(s, other) } -type v0MarketStatesDiffer struct { - Results *DealStateChanges +func (s *v0DealStates) decode(val *cbg.Deferred) (*DealState, error) { + var v0ds market.DealState + if err := v0ds.UnmarshalCBOR(bytes.NewReader(val.Raw)); err != nil { + return nil, err + } + ds := fromV0DealState(v0ds) + return &ds, nil } -func (d *v0MarketStatesDiffer) Add(key uint64, val *typegen.Deferred) error { - v0ds := new(market.DealState) - err := v0ds.UnmarshalCBOR(bytes.NewReader(val.Raw)) - if err != nil { - return err - } - d.Results.Added = append(d.Results.Added, DealIDState{abi.DealID(key), fromV0DealState(*v0ds)}) - return nil -} - -func (d *v0MarketStatesDiffer) Modify(key uint64, from, to *typegen.Deferred) error { - v0dsFrom := new(market.DealState) - if err := v0dsFrom.UnmarshalCBOR(bytes.NewReader(from.Raw)); err != nil { - return err - } - - v0dsTo := new(market.DealState) - if err := v0dsTo.UnmarshalCBOR(bytes.NewReader(to.Raw)); err != nil { - return err - } - - if *v0dsFrom != *v0dsTo { - dsFrom := fromV0DealState(*v0dsFrom) - dsTo := fromV0DealState(*v0dsTo) - d.Results.Modified = append(d.Results.Modified, DealStateChange{abi.DealID(key), &dsFrom, &dsTo}) - } - return nil -} - -func (d *v0MarketStatesDiffer) Remove(key uint64, val *typegen.Deferred) error { - v0ds := new(market.DealState) - err := v0ds.UnmarshalCBOR(bytes.NewReader(val.Raw)) - if err != nil { - return err - } - d.Results.Removed = append(d.Results.Removed, DealIDState{abi.DealID(key), fromV0DealState(*v0ds)}) - return nil +func (s *v0DealStates) array() adt.Array { + return s.Array } func fromV0DealState(v0 market.DealState) DealState { @@ -192,17 +150,7 @@ type v0DealProposals struct { } func (s *v0DealProposals) Diff(other DealProposals) (*DealProposalChanges, error) { - v0other, ok := other.(*v0DealProposals) - if !ok { - // TODO handle this if possible on a case by case basis but for now, just fail - return nil, errors.New("cannot compare deal proposals across versions") - } - results := new(DealProposalChanges) - if err := adt.DiffAdtArray(s.Array, v0other.Array, &v0MarketProposalsDiffer{results}); err != nil { - return nil, fmt.Errorf("diffing deal proposals: %w", err) - } - - return results, nil + return diffDealProposals(s, other) } func (s *v0DealProposals) Get(dealID abi.DealID) (*DealProposal, bool, error) { @@ -225,35 +173,19 @@ func (s *v0DealProposals) ForEach(cb func(dealID abi.DealID, dp DealProposal) er }) } -type v0MarketProposalsDiffer struct { - Results *DealProposalChanges +func (s *v0DealProposals) decode(val *cbg.Deferred) (*DealProposal, error) { + var v0dp market.DealProposal + if err := v0dp.UnmarshalCBOR(bytes.NewReader(val.Raw)); err != nil { + return nil, err + } + dp := fromV0DealProposal(v0dp) + return &dp, nil +} + +func (s *v0DealProposals) array() adt.Array { + return s.Array } func fromV0DealProposal(v0 market.DealProposal) DealProposal { return (DealProposal)(v0) } - -func (d *v0MarketProposalsDiffer) Add(key uint64, val *typegen.Deferred) error { - v0dp := new(market.DealProposal) - err := v0dp.UnmarshalCBOR(bytes.NewReader(val.Raw)) - if err != nil { - return err - } - d.Results.Added = append(d.Results.Added, ProposalIDState{abi.DealID(key), fromV0DealProposal(*v0dp)}) - return nil -} - -func (d *v0MarketProposalsDiffer) Modify(key uint64, from, to *typegen.Deferred) error { - // short circuit, DealProposals are static - return nil -} - -func (d *v0MarketProposalsDiffer) Remove(key uint64, val *typegen.Deferred) error { - v0dp := new(market.DealProposal) - err := v0dp.UnmarshalCBOR(bytes.NewReader(val.Raw)) - if err != nil { - return err - } - d.Results.Removed = append(d.Results.Removed, ProposalIDState{abi.DealID(key), fromV0DealProposal(*v0dp)}) - return nil -} From dcc56434732783a400a679c3adcd59b295586a04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 18 Sep 2020 02:38:30 +0200 Subject: [PATCH 204/885] shed: Even more bitfield utils --- cmd/lotus-shed/bitfield.go | 90 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/cmd/lotus-shed/bitfield.go b/cmd/lotus-shed/bitfield.go index 78f6f02fe..e1e06573c 100644 --- a/cmd/lotus-shed/bitfield.go +++ b/cmd/lotus-shed/bitfield.go @@ -4,6 +4,7 @@ import ( "encoding/base64" "encoding/hex" "fmt" + "io" "io/ioutil" "os" @@ -29,6 +30,8 @@ var bitFieldCmd = &cli.Command{ bitFieldStatCmd, bitFieldDecodeCmd, bitFieldIntersectCmd, + bitFieldEncodeCmd, + bitFieldSubCmd, }, } @@ -258,6 +261,93 @@ var bitFieldIntersectCmd = &cli.Command{ }, } + +var bitFieldSubCmd = &cli.Command{ + Name: "sub", + Description: "subtract 2 bitfields and print the resulting bitfield as base64", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "enc", + Value: "base64", + Usage: "specify input encoding to parse", + }, + }, + Action: func(cctx *cli.Context) error { + b, err := decode(cctx, 1) + if err != nil { + return err + } + + a, err := decode(cctx, 0) + if err != nil { + return err + } + + o, err := bitfield.SubtractBitField(a, b) + if err != nil { + return xerrors.Errorf("intersect: %w", err) + } + + s, err := o.RunIterator() + if err != nil { + return err + } + + bytes, err := rlepluslazy.EncodeRuns(s, []byte{}) + if err != nil { + return err + } + + fmt.Println(base64.StdEncoding.EncodeToString(bytes)) + + return nil + }, +} + +var bitFieldEncodeCmd = &cli.Command{ + Name: "encode", + Description: "encode a series of decimal numbers into a bitfield", + ArgsUsage: "[infile]", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "enc", + Value: "base64", + Usage: "specify input encoding to parse", + }, + }, + Action: func(cctx *cli.Context) error { + f, err := os.Open(cctx.Args().First()) + if err != nil { + return err + } + defer f.Close() + + out := bitfield.New() + for { + var i uint64 + _, err := fmt.Fscan(f, &i) + if err == io.EOF { + break + } + out.Set(i) + } + + s, err := out.RunIterator() + if err != nil { + return err + } + + bytes, err := rlepluslazy.EncodeRuns(s, []byte{}) + if err != nil { + return err + } + + fmt.Println(base64.StdEncoding.EncodeToString(bytes)) + + return nil + }, +} + func decode(cctx *cli.Context, a int) (bitfield.BitField, error) { var val string if cctx.Args().Present() { From 8419dda84b1cbfa194a15a50529a5cc4cf862546 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 18 Sep 2020 02:39:08 +0200 Subject: [PATCH 205/885] wdpost: Only add proven partitions to message params --- storage/wdpost_run.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 17b567440..532d7144b 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -431,7 +431,9 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty postSkipped := bitfield.New() var postOut []proof.PoStProof somethingToProve := true + for retries := 0; retries < 5; retries++ { + var partitions []miner.PoStPartition var sinfos []proof.SectorInfo for partIdx, partition := range batch { // TODO: Can do this in parallel @@ -477,7 +479,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty } sinfos = append(sinfos, ssi...) - params.Partitions = append(params.Partitions, miner.PoStPartition{ + partitions = append(params.Partitions, miner.PoStPartition{ Index: uint64(batchPartitionStartIdx + partIdx), Skipped: skipped, }) @@ -511,6 +513,8 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty if err == nil { // Proof generation successful, stop retrying + params.Partitions = append(params.Partitions, partitions...) + break } From 57c1eac37a29a1c39b85e4d245fdd1b0632f5069 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 18 Sep 2020 02:40:01 +0200 Subject: [PATCH 206/885] gofmt --- cmd/lotus-shed/bitfield.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/lotus-shed/bitfield.go b/cmd/lotus-shed/bitfield.go index e1e06573c..9cc632283 100644 --- a/cmd/lotus-shed/bitfield.go +++ b/cmd/lotus-shed/bitfield.go @@ -261,7 +261,6 @@ var bitFieldIntersectCmd = &cli.Command{ }, } - var bitFieldSubCmd = &cli.Command{ Name: "sub", Description: "subtract 2 bitfields and print the resulting bitfield as base64", @@ -307,7 +306,7 @@ var bitFieldSubCmd = &cli.Command{ var bitFieldEncodeCmd = &cli.Command{ Name: "encode", Description: "encode a series of decimal numbers into a bitfield", - ArgsUsage: "[infile]", + ArgsUsage: "[infile]", Flags: []cli.Flag{ &cli.StringFlag{ Name: "enc", From 26bbd9fb9d13207ed6f32d769bfb8ebfb548f795 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 18 Sep 2020 03:00:31 +0200 Subject: [PATCH 207/885] wdpost: append is hard --- cmd/lotus-shed/bitfield.go | 2 +- storage/wdpost_run.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/lotus-shed/bitfield.go b/cmd/lotus-shed/bitfield.go index 9cc632283..442cbef48 100644 --- a/cmd/lotus-shed/bitfield.go +++ b/cmd/lotus-shed/bitfield.go @@ -319,7 +319,7 @@ var bitFieldEncodeCmd = &cli.Command{ if err != nil { return err } - defer f.Close() + defer f.Close() // nolint out := bitfield.New() for { diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 532d7144b..463c7a3c5 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -479,7 +479,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty } sinfos = append(sinfos, ssi...) - partitions = append(params.Partitions, miner.PoStPartition{ + partitions = append(partitions, miner.PoStPartition{ Index: uint64(batchPartitionStartIdx + partIdx), Skipped: skipped, }) From aa081dbc6d018f3bbfcaf25c2374b4917fb2de1c Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Thu, 17 Sep 2020 15:17:25 -0400 Subject: [PATCH 208/885] Lotus version 0.7.1 --- CHANGELOG.md | 24 ++++++++++++++++++++++++ build/version.go | 4 ++-- documentation/en/api-methods.md | 2 +- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8617554a9..06c0959b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,29 @@ # Lotus changelog +# 0.7.1 / 2020-09-17 + +This optional release of Lotus introduces some critical fixes to the window PoSt process. It also upgrades some core dependencies, and introduces many improvements to the mining process, deal-making cycle, and overall User Experience. + +## Changes + +#### Some notable improvements: + +- Correctly construct params for `SubmitWindowedPoSt` messages (https://github.com/filecoin-project/lotus/pull/3909) +- Skip sectors correctly for Window PoSt (https://github.com/filecoin-project/lotus/pull/3839) +- Split window PoST submission into multiple messages (https://github.com/filecoin-project/lotus/pull/3689) +- Improve journal coverage (https://github.com/filecoin-project/lotus/pull/2455) +- Allow retrievals while sealing (https://github.com/filecoin-project/lotus/pull/3778) +- Don't prune locally published messages (https://github.com/filecoin-project/lotus/pull/3772) +- Add get-ask, set-ask retrieval commands (https://github.com/filecoin-project/lotus/pull/3886) +- Consistently name winning and window post in logs (https://github.com/filecoin-project/lotus/pull/3873)) +- Add auto flag to mpool replace (https://github.com/filecoin-project/lotus/pull/3752)) + +#### Dependencies + +- Upgrade markets to `v0.6.1` (https://github.com/filecoin-project/lotus/pull/3906) +- Upgrade specs-actors to `v0.9.10` (https://github.com/filecoin-project/lotus/pull/3846) +- Upgrade badger (https://github.com/filecoin-project/lotus/pull/3739) + # 0.7.0 / 2020-09-10 This consensus-breaking release of Lotus is designed to test a network upgrade on the space race testnet. The changes that break consensus are: diff --git a/build/version.go b/build/version.go index 2466d8023..933cfca70 100644 --- a/build/version.go +++ b/build/version.go @@ -29,7 +29,7 @@ func buildType() string { } // BuildVersion is the local build version, set by build system -const BuildVersion = "0.7.0" +const BuildVersion = "0.7.1" func UserVersion() string { return BuildVersion + buildType() + CurrentCommit @@ -83,7 +83,7 @@ func VersionForType(nodeType NodeType) (Version, error) { // semver versions of the rpc api exposed var ( - FullAPIVersion = newVer(0, 14, 0) + FullAPIVersion = newVer(0, 15, 0) MinerAPIVersion = newVer(0, 14, 0) WorkerAPIVersion = newVer(0, 14, 0) ) diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index 2f3164bb7..84d72ab67 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -212,7 +212,7 @@ Response: ```json { "Version": "string value", - "APIVersion": 3584, + "APIVersion": 3840, "BlockDelay": 42 } ``` From db2a20da6cb8343a2a8c7a45f3d6e91b48928c01 Mon Sep 17 00:00:00 2001 From: Travis Person Date: Thu, 17 Sep 2020 23:09:12 +0000 Subject: [PATCH 209/885] lotus-shed: add keyinfo verify --- cmd/lotus-shed/keyinfo.go | 90 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/cmd/lotus-shed/keyinfo.go b/cmd/lotus-shed/keyinfo.go index f397bd4c7..6090b7a0f 100644 --- a/cmd/lotus-shed/keyinfo.go +++ b/cmd/lotus-shed/keyinfo.go @@ -9,16 +9,22 @@ import ( "io" "io/ioutil" "os" + "path" "strings" "text/template" "github.com/urfave/cli/v2" + "golang.org/x/xerrors" + + "github.com/multiformats/go-base32" + "github.com/libp2p/go-libp2p-core/crypto" "github.com/libp2p/go-libp2p-core/peer" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/wallet" + "github.com/filecoin-project/lotus/node/modules" "github.com/filecoin-project/lotus/node/modules/lp2p" "github.com/filecoin-project/lotus/node/repo" @@ -43,6 +49,90 @@ var keyinfoCmd = &cli.Command{ keyinfoNewCmd, keyinfoInfoCmd, keyinfoImportCmd, + keyinfoVerifyCmd, + }, +} + +var keyinfoVerifyCmd = &cli.Command{ + Name: "verify", + Usage: "verify the filename of a keystore object on disk with it's contents", + Description: `Keystore objects are base32 enocded strings, with wallets being dynamically named via + the wallet address. This command can ensure that the naming of these keystore objects are correct`, + Action: func(cctx *cli.Context) error { + filePath := cctx.Args().First() + fileName := path.Base(filePath) + + inputFile, err := os.Open(filePath) + if err != nil { + return err + } + defer inputFile.Close() //nolint:errcheck + input := bufio.NewReader(inputFile) + + keyContent, err := ioutil.ReadAll(input) + if err != nil { + return err + } + + var keyInfo types.KeyInfo + if err := json.Unmarshal(keyContent, &keyInfo); err != nil { + return err + } + + switch keyInfo.Type { + case lp2p.KTLibp2pHost: + name, err := base32.RawStdEncoding.DecodeString(fileName) + if err != nil { + return xerrors.Errorf("decoding key: '%s': %w", fileName, err) + } + + if string(name) != keyInfo.Type { + return fmt.Errorf("%s of type %s is incorrect", fileName, keyInfo.Type) + } + case modules.KTJwtHmacSecret: + name, err := base32.RawStdEncoding.DecodeString(fileName) + if err != nil { + return xerrors.Errorf("decoding key: '%s': %w", fileName, err) + } + + if string(name) != modules.JWTSecretName { + return fmt.Errorf("%s of type %s is incorrect", fileName, keyInfo.Type) + } + case wallet.KTSecp256k1, wallet.KTBLS: + keystore := wallet.NewMemKeyStore() + w, err := wallet.NewWallet(keystore) + if err != nil { + return err + } + + if _, err := w.Import(&keyInfo); err != nil { + return err + } + + list, err := keystore.List() + if err != nil { + return err + } + + if len(list) != 1 { + return fmt.Errorf("Unexpected number of keys, expected 1, found %d", len(list)) + } + + name, err := base32.RawStdEncoding.DecodeString(fileName) + if err != nil { + return xerrors.Errorf("decoding key: '%s': %w", fileName, err) + } + + if string(name) != list[0] { + return fmt.Errorf("%s of type %s; file is named for %s, but key is actually %s", fileName, keyInfo.Type, string(name), list[0]) + } + + break + default: + return fmt.Errorf("Unknown keytype %s", keyInfo.Type) + } + + return nil }, } From ac7007d1d010d31a970b5f4706045976e43ab796 Mon Sep 17 00:00:00 2001 From: Travis Person Date: Fri, 18 Sep 2020 00:40:46 +0000 Subject: [PATCH 210/885] lotus-shed: add jwt token command --- cmd/lotus-shed/jwt.go | 99 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/cmd/lotus-shed/jwt.go b/cmd/lotus-shed/jwt.go index d37359f71..7fa1a18dd 100644 --- a/cmd/lotus-shed/jwt.go +++ b/cmd/lotus-shed/jwt.go @@ -1,6 +1,7 @@ package main import ( + "bufio" "crypto/rand" "encoding/hex" "encoding/json" @@ -8,10 +9,12 @@ import ( "io" "io/ioutil" "os" + "strings" "github.com/gbrlsnchs/jwt/v3" "github.com/urfave/cli/v2" + "github.com/filecoin-project/go-jsonrpc/auth" "github.com/filecoin-project/lotus/api/apistruct" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/modules" @@ -24,6 +27,102 @@ var jwtCmd = &cli.Command{ having to run the lotus daemon.`, Subcommands: []*cli.Command{ jwtNewCmd, + jwtTokenCmd, + }, +} + +var jwtTokenCmd = &cli.Command{ + Name: "token", + Usage: "create a token for a given jwt secret", + ArgsUsage: "", + Description: `The jwt tokens have four different levels of permissions that provide some ability + to control access to what methods can be invoked by the holder of the token. + + This command only works on jwt secrets that are base16 encoded files, such as those produced by the + sibling 'new' command. + `, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "output", + Value: "token", + Usage: "specify a name", + }, + &cli.BoolFlag{ + Name: "read", + Value: false, + Usage: "add read permissions to the token", + }, + &cli.BoolFlag{ + Name: "write", + Value: false, + Usage: "add write permissions to the token", + }, + &cli.BoolFlag{ + Name: "sign", + Value: false, + Usage: "add sign permissions to the token", + }, + &cli.BoolFlag{ + Name: "admin", + Value: false, + Usage: "add admin permissions to the token", + }, + }, + Action: func(cctx *cli.Context) error { + if !cctx.Args().Present() { + return fmt.Errorf("please specify a name") + } + + inputFile, err := os.Open(cctx.Args().First()) + if err != nil { + return err + } + defer inputFile.Close() //nolint:errcheck + input := bufio.NewReader(inputFile) + + encoded, err := ioutil.ReadAll(input) + if err != nil { + return err + } + + decoded, err := hex.DecodeString(strings.TrimSpace(string(encoded))) + if err != nil { + return err + } + + var keyInfo types.KeyInfo + if err := json.Unmarshal(decoded, &keyInfo); err != nil { + return err + } + + perms := []auth.Permission{} + + if cctx.Bool("read") { + perms = append(perms, apistruct.PermRead) + } + + if cctx.Bool("write") { + perms = append(perms, apistruct.PermWrite) + } + + if cctx.Bool("sign") { + perms = append(perms, apistruct.PermSign) + } + + if cctx.Bool("admin") { + perms = append(perms, apistruct.PermAdmin) + } + + p := modules.JwtPayload{ + Allow: perms, + } + + token, err := jwt.Sign(&p, jwt.NewHS256(keyInfo.PrivateKey)) + if err != nil { + return err + } + + return ioutil.WriteFile(cctx.String("output"), token, 0600) }, } From b2ee59024f935721f6f277f8ae381afb05a70d91 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 17 Sep 2020 21:39:34 -0700 Subject: [PATCH 211/885] improve diff logic * Make diffing work across versions. * Start porting more chainwatch logic. --- chain/actors/builtin/miner/diff.go | 127 ++++++++++ chain/actors/builtin/miner/miner.go | 28 +- chain/actors/builtin/miner/v0.go | 41 +++ chain/events/state/predicates.go | 128 +--------- chain/events/state/predicates_test.go | 4 +- cmd/lotus-chainwatch/processor/miner.go | 323 ++++++++++++------------ 6 files changed, 364 insertions(+), 287 deletions(-) create mode 100644 chain/actors/builtin/miner/diff.go diff --git a/chain/actors/builtin/miner/diff.go b/chain/actors/builtin/miner/diff.go new file mode 100644 index 000000000..dde4db890 --- /dev/null +++ b/chain/actors/builtin/miner/diff.go @@ -0,0 +1,127 @@ +package miner + +import ( + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/actors/adt" + cbg "github.com/whyrusleeping/cbor-gen" +) + +func DiffPreCommits(pre, cur State) (*PreCommitChanges, error) { + results := new(PreCommitChanges) + + prep, err := pre.precommits() + if err != nil { + return nil, err + } + + curp, err := cur.precommits() + if err != nil { + return nil, err + } + + err = adt.DiffAdtMap(prep, curp, &preCommitDiffer{results, pre, cur}) + if err != nil { + return nil, err + } + + return results, nil +} + +type preCommitDiffer struct { + Results *PreCommitChanges + pre, after State +} + +func (m *preCommitDiffer) AsKey(key string) (abi.Keyer, error) { + sector, err := abi.ParseUIntKey(key) + if err != nil { + return nil, err + } + return abi.UIntKey(sector), nil +} + +func (m *preCommitDiffer) Add(key string, val *cbg.Deferred) error { + sp, err := m.after.decodeSectorPreCommitOnChainInfo(val) + if err != nil { + return err + } + m.Results.Added = append(m.Results.Added, sp) + return nil +} + +func (m *preCommitDiffer) Modify(key string, from, to *cbg.Deferred) error { + return nil +} + +func (m *preCommitDiffer) Remove(key string, val *cbg.Deferred) error { + sp, err := m.pre.decodeSectorPreCommitOnChainInfo(val) + if err != nil { + return err + } + m.Results.Removed = append(m.Results.Removed, sp) + return nil +} + +func DiffSectors(pre, cur State) (*SectorChanges, error) { + results := new(SectorChanges) + + pres, err := pre.sectors() + if err != nil { + return nil, err + } + + curs, err := cur.sectors() + if err != nil { + return nil, err + } + + err = adt.DiffAdtArray(pres, curs, §orDiffer{results, pre, cur}) + if err != nil { + return nil, err + } + + return results, nil +} + +type sectorDiffer struct { + Results *SectorChanges + pre, after State +} + +func (m *sectorDiffer) Add(key uint64, val *cbg.Deferred) error { + si, err := m.after.decodeSectorOnChainInfo(val) + if err != nil { + return err + } + m.Results.Added = append(m.Results.Added, si) + return nil +} + +func (m *sectorDiffer) Modify(key uint64, from, to *cbg.Deferred) error { + siFrom, err := m.pre.decodeSectorOnChainInfo(from) + if err != nil { + return err + } + + siTo, err := m.after.decodeSectorOnChainInfo(to) + if err != nil { + return err + } + + if siFrom.Expiration != siTo.Expiration { + m.Results.Extended = append(m.Results.Extended, SectorExtensions{ + From: siFrom, + To: siTo, + }) + } + return nil +} + +func (m *sectorDiffer) Remove(key uint64, val *cbg.Deferred) error { + si, err := m.pre.decodeSectorOnChainInfo(val) + if err != nil { + return err + } + m.Results.Removed = append(m.Results.Removed, si) + return nil +} diff --git a/chain/actors/builtin/miner/miner.go b/chain/actors/builtin/miner/miner.go index cb34dd557..793c4badc 100644 --- a/chain/actors/builtin/miner/miner.go +++ b/chain/actors/builtin/miner/miner.go @@ -3,6 +3,7 @@ package miner import ( "github.com/filecoin-project/go-state-types/dline" "github.com/libp2p/go-libp2p-core/peer" + cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" @@ -42,21 +43,30 @@ type State interface { GetSectorExpiration(abi.SectorNumber) (*SectorExpiration, error) GetPrecommittedSector(abi.SectorNumber) (*SectorPreCommitOnChainInfo, error) LoadSectorsFromSet(filter *bitfield.BitField, filterOut bool) (adt.Array, error) - LoadPreCommittedSectors() (adt.Map, error) IsAllocated(abi.SectorNumber) (bool, error) LoadDeadline(idx uint64) (Deadline, error) ForEachDeadline(cb func(idx uint64, dl Deadline) error) error NumDeadlines() (uint64, error) + DeadlinesChanged(State) bool + Info() (MinerInfo, error) DeadlineInfo(epoch abi.ChainEpoch) *dline.Info + + // Diff helpers. Used by Diff* functions internally. + sectors() (adt.Array, error) + decodeSectorOnChainInfo(*cbg.Deferred) (SectorOnChainInfo, error) + precommits() (adt.Map, error) + decodeSectorPreCommitOnChainInfo(*cbg.Deferred) (SectorPreCommitOnChainInfo, error) } type Deadline interface { LoadPartition(idx uint64) (Partition, error) ForEachPartition(cb func(idx uint64, part Partition) error) error PostSubmissions() (bitfield.BitField, error) + + PartitionsChanged(Deadline) bool } type Partition interface { @@ -110,3 +120,19 @@ type SectorLocation struct { Deadline uint64 Partition uint64 } + +type SectorChanges struct { + Added []SectorOnChainInfo + Extended []SectorExtensions + Removed []SectorOnChainInfo +} + +type SectorExtensions struct { + From SectorOnChainInfo + To SectorOnChainInfo +} + +type PreCommitChanges struct { + Added []SectorPreCommitOnChainInfo + Removed []SectorPreCommitOnChainInfo +} diff --git a/chain/actors/builtin/miner/v0.go b/chain/actors/builtin/miner/v0.go index 74812e85a..9f4cea6ce 100644 --- a/chain/actors/builtin/miner/v0.go +++ b/chain/actors/builtin/miner/v0.go @@ -1,6 +1,7 @@ package miner import ( + "bytes" "errors" "github.com/filecoin-project/go-address" @@ -206,6 +207,16 @@ func (s *v0State) NumDeadlines() (uint64, error) { return v0miner.WPoStPeriodDeadlines, nil } +func (s *v0State) DeadlinesChanged(other State) bool { + v0other, ok := other.(*v0State) + if !ok { + // treat an upgrade as a change, always + return true + } + + return s.State.Deadlines.Equals(v0other.Deadlines) +} + func (s *v0State) Info() (MinerInfo, error) { info, err := s.State.GetInfo(s.store) if err != nil { @@ -244,6 +255,26 @@ func (s *v0State) DeadlineInfo(epoch abi.ChainEpoch) *dline.Info { return s.State.DeadlineInfo(epoch) } +func (s *v0State) sectors() (adt.Array, error) { + return v0adt.AsArray(s.store, s.Sectors) +} + +func (s *v0State) decodeSectorOnChainInfo(val *cbg.Deferred) (SectorOnChainInfo, error) { + var si v0miner.SectorOnChainInfo + err := si.UnmarshalCBOR(bytes.NewReader(val.Raw)) + return si, err +} + +func (s *v0State) precommits() (adt.Map, error) { + return v0adt.AsMap(s.store, s.PreCommittedSectors) +} + +func (s *v0State) decodeSectorPreCommitOnChainInfo(val *cbg.Deferred) (SectorPreCommitOnChainInfo, error) { + var sp v0miner.SectorPreCommitOnChainInfo + err := sp.UnmarshalCBOR(bytes.NewReader(val.Raw)) + return sp, err +} + func (d *v0Deadline) LoadPartition(idx uint64) (Partition, error) { p, err := d.Deadline.LoadPartition(d.store, idx) if err != nil { @@ -263,6 +294,16 @@ func (d *v0Deadline) ForEachPartition(cb func(uint64, Partition) error) error { }) } +func (s *v0Deadline) PartitionsChanged(other Deadline) bool { + v0other, ok := other.(*v0Deadline) + if !ok { + // treat an upgrade as a change, always + return true + } + + return s.Deadline.Partitions.Equals(v0other.Deadline.Partitions) +} + func (d *v0Deadline) PostSubmissions() (bitfield.BitField, error) { return d.Deadline.PostSubmissions, nil } diff --git a/chain/events/state/predicates.go b/chain/events/state/predicates.go index f27b5dc8f..721de9dd2 100644 --- a/chain/events/state/predicates.go +++ b/chain/events/state/predicates.go @@ -4,8 +4,6 @@ import ( "bytes" "context" - v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" - "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/go-address" @@ -325,83 +323,12 @@ func (sp *StatePredicates) OnMinerActorChange(minerAddr address.Address, diffMin }) } -type MinerSectorChanges struct { - Added []miner.SectorOnChainInfo - Extended []SectorExtensions - Removed []miner.SectorOnChainInfo -} - -var _ adt.AdtArrayDiff = &MinerSectorChanges{} - -type SectorExtensions struct { - From miner.SectorOnChainInfo - To miner.SectorOnChainInfo -} - -func (m *MinerSectorChanges) Add(key uint64, val *typegen.Deferred) error { - si := new(miner.SectorOnChainInfo) - err := si.UnmarshalCBOR(bytes.NewReader(val.Raw)) - if err != nil { - return err - } - m.Added = append(m.Added, *si) - return nil -} - -func (m *MinerSectorChanges) Modify(key uint64, from, to *typegen.Deferred) error { - siFrom := new(miner.SectorOnChainInfo) - err := siFrom.UnmarshalCBOR(bytes.NewReader(from.Raw)) - if err != nil { - return err - } - - siTo := new(miner.SectorOnChainInfo) - err = siTo.UnmarshalCBOR(bytes.NewReader(to.Raw)) - if err != nil { - return err - } - - if siFrom.Expiration != siTo.Expiration { - m.Extended = append(m.Extended, SectorExtensions{ - From: *siFrom, - To: *siTo, - }) - } - return nil -} - -func (m *MinerSectorChanges) Remove(key uint64, val *typegen.Deferred) error { - si := new(miner.SectorOnChainInfo) - err := si.UnmarshalCBOR(bytes.NewReader(val.Raw)) - if err != nil { - return err - } - m.Removed = append(m.Removed, *si) - return nil -} - func (sp *StatePredicates) OnMinerSectorChange() DiffMinerActorStateFunc { return func(ctx context.Context, oldState, newState miner.State) (changed bool, user UserData, err error) { - sectorChanges := &MinerSectorChanges{ - Added: []miner.SectorOnChainInfo{}, - Extended: []SectorExtensions{}, - Removed: []miner.SectorOnChainInfo{}, - } - - oldSectors, err := oldState.LoadSectorsFromSet(nil, false) + sectorChanges, err := miner.DiffSectors(oldState, newState) if err != nil { return false, nil, err } - - newSectors, err := newState.LoadSectorsFromSet(nil, false) - if err != nil { - return false, nil, err - } - - if err := adt.DiffAdtArray(oldSectors, newSectors, sectorChanges); err != nil { - return false, nil, err - } - // nothing changed if len(sectorChanges.Added)+len(sectorChanges.Extended)+len(sectorChanges.Removed) == 0 { return false, nil, nil @@ -411,64 +338,13 @@ func (sp *StatePredicates) OnMinerSectorChange() DiffMinerActorStateFunc { } } -type MinerPreCommitChanges struct { - Added []miner.SectorPreCommitOnChainInfo - Removed []miner.SectorPreCommitOnChainInfo -} - -func (m *MinerPreCommitChanges) AsKey(key string) (abi.Keyer, error) { - sector, err := abi.ParseUIntKey(key) - if err != nil { - return nil, err - } - return v0miner.SectorKey(abi.SectorNumber(sector)), nil -} - -func (m *MinerPreCommitChanges) Add(key string, val *typegen.Deferred) error { - sp := new(miner.SectorPreCommitOnChainInfo) - err := sp.UnmarshalCBOR(bytes.NewReader(val.Raw)) - if err != nil { - return err - } - m.Added = append(m.Added, *sp) - return nil -} - -func (m *MinerPreCommitChanges) Modify(key string, from, to *typegen.Deferred) error { - return nil -} - -func (m *MinerPreCommitChanges) Remove(key string, val *typegen.Deferred) error { - sp := new(miner.SectorPreCommitOnChainInfo) - err := sp.UnmarshalCBOR(bytes.NewReader(val.Raw)) - if err != nil { - return err - } - m.Removed = append(m.Removed, *sp) - return nil -} - func (sp *StatePredicates) OnMinerPreCommitChange() DiffMinerActorStateFunc { return func(ctx context.Context, oldState, newState miner.State) (changed bool, user UserData, err error) { - precommitChanges := &MinerPreCommitChanges{ - Added: []miner.SectorPreCommitOnChainInfo{}, - Removed: []miner.SectorPreCommitOnChainInfo{}, - } - - oldPrecommits, err := oldState.LoadPreCommittedSectors() + precommitChanges, err := miner.DiffPreCommits(oldState, newState) if err != nil { return false, nil, err } - newPrecommits, err := newState.LoadPreCommittedSectors() - if err != nil { - return false, nil, err - } - - if err := adt.DiffAdtMap(oldPrecommits, newPrecommits, precommitChanges); err != nil { - return false, nil, err - } - if len(precommitChanges.Added)+len(precommitChanges.Removed) == 0 { return false, nil, nil } diff --git a/chain/events/state/predicates_test.go b/chain/events/state/predicates_test.go index 6541aa3b4..243d8814e 100644 --- a/chain/events/state/predicates_test.go +++ b/chain/events/state/predicates_test.go @@ -409,7 +409,7 @@ func TestMinerSectorChange(t *testing.T) { require.True(t, change) require.NotNil(t, val) - sectorChanges, ok := val.(*MinerSectorChanges) + sectorChanges, ok := val.(*miner.SectorChanges) require.True(t, ok) require.Equal(t, len(sectorChanges.Added), 1) @@ -433,7 +433,7 @@ func TestMinerSectorChange(t *testing.T) { require.True(t, change) require.NotNil(t, val) - sectorChanges, ok = val.(*MinerSectorChanges) + sectorChanges, ok = val.(*miner.SectorChanges) require.True(t, ok) require.Equal(t, 1, len(sectorChanges.Added)) diff --git a/cmd/lotus-chainwatch/processor/miner.go b/cmd/lotus-chainwatch/processor/miner.go index 71d881927..1fd2a119f 100644 --- a/cmd/lotus-chainwatch/processor/miner.go +++ b/cmd/lotus-chainwatch/processor/miner.go @@ -16,12 +16,14 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/specs-actors/actors/builtin/power" "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/api/apibstore" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/events/state" + "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" cw_util "github.com/filecoin-project/lotus/cmd/lotus-chainwatch/util" ) @@ -204,6 +206,8 @@ func (p *Processor) processMiners(ctx context.Context, minerTips map[types.TipSe log.Debugw("Processed Miners", "duration", time.Since(start).String()) }() + stor := store.ActorStore(ctx, apibstore.NewAPIBlockstore(p.node)) + var out []minerActorInfo // TODO add parallel calls if this becomes slow for tipset, miners := range minerTips { @@ -230,15 +234,13 @@ func (p *Processor) processMiners(ctx context.Context, minerTips map[types.TipSe mi.rawPower = claim.RawBytePower } - // Get the miner state info - astb, err := p.node.ChainReadObj(ctx, act.act.Head) + // Get the miner state + mas, err := miner.Load(stor, &act.act) if err != nil { log.Warnw("failed to find miner actor state", "address", act.addr, "error", err) continue } - if err := mi.state.UnmarshalCBOR(bytes.NewReader(astb)); err != nil { - return nil, err - } + mi.state = mas out = append(out, mi) } } @@ -322,11 +324,6 @@ func (p *Processor) storeMinerPreCommitInfo(ctx context.Context, miners []minerA for _, m := range miners { m := m grp.Go(func() error { - minerSectors, err := adt.AsArray(p.ctxStore, m.state.Sectors) - if err != nil { - return err - } - changes, err := p.getMinerPreCommitChanges(ctx, m) if err != nil { if strings.Contains(err.Error(), types.ErrActorNotFound.Error()) { @@ -399,10 +396,12 @@ func (p *Processor) storeMinerPreCommitInfo(ctx context.Context, miners []minerA } var preCommitExpired []uint64 for _, removed := range changes.Removed { - var sector miner.SectorOnChainInfo - if found, err := minerSectors.Get(uint64(removed.Info.SectorNumber), §or); err != nil { + // TODO: we can optimize this to not load the AMT every time, if necessary. + si, err := m.state.GetSector(removed.Info.SectorNumber) + if err != nil { return err - } else if !found { + } + if si == nil { preCommitExpired = append(preCommitExpired, uint64(removed.Info.SectorNumber)) } } @@ -653,21 +652,12 @@ func (p *Processor) storeMinerSectorEvents(ctx context.Context, sectorEvents, pr func (p *Processor) getMinerStateAt(ctx context.Context, maddr address.Address, tskey types.TipSetKey) (miner.State, error) { prevActor, err := p.node.StateGetActor(ctx, maddr, tskey) if err != nil { - return miner.State{}, err + return nil, err } - var out miner.State - // Get the miner state info - astb, err := p.node.ChainReadObj(ctx, prevActor.Head) - if err != nil { - return miner.State{}, err - } - if err := out.UnmarshalCBOR(bytes.NewReader(astb)); err != nil { - return miner.State{}, err - } - return out, nil + return miner.Load(store.ActorStore(ctx, apibstore.NewAPIBlockstore(p.node)), prevActor) } -func (p *Processor) getMinerPreCommitChanges(ctx context.Context, m minerActorInfo) (*state.MinerPreCommitChanges, error) { +func (p *Processor) getMinerPreCommitChanges(ctx context.Context, m minerActorInfo) (*miner.PreCommitChanges, error) { pred := state.NewStatePredicates(p.node) changed, val, err := pred.OnMinerActorChange(m.common.addr, pred.OnMinerPreCommitChange())(ctx, m.common.parentTsKey, m.common.tsKey) if err != nil { @@ -676,11 +666,11 @@ func (p *Processor) getMinerPreCommitChanges(ctx context.Context, m minerActorIn if !changed { return nil, nil } - out := val.(*state.MinerPreCommitChanges) + out := val.(*miner.PreCommitChanges) return out, nil } -func (p *Processor) getMinerSectorChanges(ctx context.Context, m minerActorInfo) (*state.MinerSectorChanges, error) { +func (p *Processor) getMinerSectorChanges(ctx context.Context, m minerActorInfo) (*miner.SectorChanges, error) { pred := state.NewStatePredicates(p.node) changed, val, err := pred.OnMinerActorChange(m.common.addr, pred.OnMinerSectorChange())(ctx, m.common.parentTsKey, m.common.tsKey) if err != nil { @@ -689,7 +679,7 @@ func (p *Processor) getMinerSectorChanges(ctx context.Context, m minerActorInfo) if !changed { return nil, nil } - out := val.(*state.MinerSectorChanges) + out := val.(*miner.SectorChanges) return out, nil } @@ -698,179 +688,196 @@ func (p *Processor) diffMinerPartitions(ctx context.Context, m minerActorInfo, e if err != nil { return err } - dlIdx := prevMiner.CurrentDeadline curMiner := m.state - - // load the old deadline - prevDls, err := prevMiner.LoadDeadlines(p.ctxStore) - if err != nil { - return err - } - var prevDl miner.Deadline - if err := p.ctxStore.Get(ctx, prevDls.Due[dlIdx], &prevDl); err != nil { - return err + if !prevMiner.DeadlinesChanged(curMiner) { + return nil } + panic("TODO") - prevPartitions, err := prevDl.PartitionsArray(p.ctxStore) - if err != nil { - return err - } + // FIXME: This code doesn't work. + // 1. We need to diff all deadlines, not just the "current" deadline. + // 2. We need to handle the case where we _add_ a partition. (i.e., + // where len(newPartitions) != len(oldPartitions). + /* - // load the new deadline - curDls, err := curMiner.LoadDeadlines(p.ctxStore) - if err != nil { - return err - } + // NOTE: If we change the number of deadlines in an upgrade, this will + // break. - var curDl miner.Deadline - if err := p.ctxStore.Get(ctx, curDls.Due[dlIdx], &curDl); err != nil { - return err - } + // load the old deadline + prevDls, err := prevMiner.LoadDeadlines(p.ctxStore) + if err != nil { + return err + } + var prevDl miner.Deadline + if err := p.ctxStore.Get(ctx, prevDls.Due[dlIdx], &prevDl); err != nil { + return err + } - curPartitions, err := curDl.PartitionsArray(p.ctxStore) - if err != nil { - return err - } + prevPartitions, err := prevDl.PartitionsArray(p.ctxStore) + if err != nil { + return err + } - // TODO this can be optimized by inspecting the miner state for partitions that have changed and only inspecting those. - var prevPart miner.Partition - if err := prevPartitions.ForEach(&prevPart, func(i int64) error { - var curPart miner.Partition - if found, err := curPartitions.Get(uint64(i), &curPart); err != nil { - return err - } else if !found { - log.Fatal("I don't know what this means, are partitions ever removed?") - } - partitionDiff, err := p.diffPartition(prevPart, curPart) - if err != nil { - return err - } + // load the new deadline + curDls, err := curMiner.LoadDeadlines(p.ctxStore) + if err != nil { + return err + } - recovered, err := partitionDiff.Recovered.All(miner.SectorsMax) - if err != nil { - return err - } - events <- &MinerSectorsEvent{ - MinerID: m.common.addr, - StateRoot: m.common.stateroot, - SectorIDs: recovered, - Event: SectorRecovered, - } - inRecovery, err := partitionDiff.InRecovery.All(miner.SectorsMax) - if err != nil { - return err - } - events <- &MinerSectorsEvent{ - MinerID: m.common.addr, - StateRoot: m.common.stateroot, - SectorIDs: inRecovery, - Event: SectorRecovering, - } - faulted, err := partitionDiff.Faulted.All(miner.SectorsMax) - if err != nil { - return err - } - events <- &MinerSectorsEvent{ - MinerID: m.common.addr, - StateRoot: m.common.stateroot, - SectorIDs: faulted, - Event: SectorFaulted, - } - terminated, err := partitionDiff.Terminated.All(miner.SectorsMax) - if err != nil { - return err - } - events <- &MinerSectorsEvent{ - MinerID: m.common.addr, - StateRoot: m.common.stateroot, - SectorIDs: terminated, - Event: SectorTerminated, - } - expired, err := partitionDiff.Expired.All(miner.SectorsMax) - if err != nil { - return err - } - events <- &MinerSectorsEvent{ - MinerID: m.common.addr, - StateRoot: m.common.stateroot, - SectorIDs: expired, - Event: SectorExpired, - } + var curDl miner.Deadline + if err := p.ctxStore.Get(ctx, curDls.Due[dlIdx], &curDl); err != nil { + return err + } + + curPartitions, err := curDl.PartitionsArray(p.ctxStore) + if err != nil { + return err + } + + // TODO this can be optimized by inspecting the miner state for partitions that have changed and only inspecting those. + var prevPart miner.Partition + if err := prevPartitions.ForEach(&prevPart, func(i int64) error { + var curPart miner.Partition + if found, err := curPartitions.Get(uint64(i), &curPart); err != nil { + return err + } else if !found { + log.Fatal("I don't know what this means, are partitions ever removed?") + } + partitionDiff, err := p.diffPartition(prevPart, curPart) + if err != nil { + return err + } + + recovered, err := partitionDiff.Recovered.All(miner.SectorsMax) + if err != nil { + return err + } + events <- &MinerSectorsEvent{ + MinerID: m.common.addr, + StateRoot: m.common.stateroot, + SectorIDs: recovered, + Event: SectorRecovered, + } + inRecovery, err := partitionDiff.InRecovery.All(miner.SectorsMax) + if err != nil { + return err + } + events <- &MinerSectorsEvent{ + MinerID: m.common.addr, + StateRoot: m.common.stateroot, + SectorIDs: inRecovery, + Event: SectorRecovering, + } + faulted, err := partitionDiff.Faulted.All(miner.SectorsMax) + if err != nil { + return err + } + events <- &MinerSectorsEvent{ + MinerID: m.common.addr, + StateRoot: m.common.stateroot, + SectorIDs: faulted, + Event: SectorFaulted, + } + terminated, err := partitionDiff.Terminated.All(miner.SectorsMax) + if err != nil { + return err + } + events <- &MinerSectorsEvent{ + MinerID: m.common.addr, + StateRoot: m.common.stateroot, + SectorIDs: terminated, + Event: SectorTerminated, + } + expired, err := partitionDiff.Expired.All(miner.SectorsMax) + if err != nil { + return err + } + events <- &MinerSectorsEvent{ + MinerID: m.common.addr, + StateRoot: m.common.stateroot, + SectorIDs: expired, + Event: SectorExpired, + } + + return nil + }); err != nil { + return err + } return nil - }); err != nil { - return err - } - - return nil + */ } func (p *Processor) diffPartition(prevPart, curPart miner.Partition) (*PartitionStatus, error) { - // all the sectors that were in previous but not in current - allRemovedSectors, err := bitfield.SubtractBitField(prevPart.Sectors, curPart.Sectors) + prevLiveSectors, err := prevPart.LiveSectors() + if err != nil { + return nil, err + } + curLiveSectors, err := curPart.LiveSectors() if err != nil { return nil, err } - // list of sectors that were terminated before their expiration. - terminatedEarlyArr, err := adt.AsArray(p.ctxStore, curPart.EarlyTerminated) + removedSectors, err := bitfield.SubtractBitField(prevLiveSectors, curLiveSectors) if err != nil { return nil, err } - expired := bitfield.New() - var bf bitfield.BitField - if err := terminatedEarlyArr.ForEach(&bf, func(i int64) error { - // expired = all removals - termination - expirations, err := bitfield.SubtractBitField(allRemovedSectors, bf) - if err != nil { - return err - } - // merge with expired sectors from other epochs - expired, err = bitfield.MergeBitFields(expirations, expired) - if err != nil { - return nil - } - return nil - }); err != nil { - return nil, err - } - - // terminated = all removals - expired - terminated, err := bitfield.SubtractBitField(allRemovedSectors, expired) + prevRecoveries, err := prevPart.RecoveringSectors() if err != nil { return nil, err } - // faults in current but not previous - faults, err := bitfield.SubtractBitField(curPart.Recoveries, prevPart.Recoveries) + curRecoveries, err := curPart.RecoveringSectors() if err != nil { return nil, err } - // recoveries in current but not previous - inRecovery, err := bitfield.SubtractBitField(curPart.Recoveries, prevPart.Recoveries) + newRecoveries, err := bitfield.SubtractBitField(curRecoveries, prevRecoveries) + if err != nil { + return nil, err + } + + prevFaults, err := prevPart.FaultySectors() + if err != nil { + return nil, err + } + + curFaults, err := curPart.FaultySectors() + if err != nil { + return nil, err + } + + newFaults, err := bitfield.SubtractBitField(curFaults, prevFaults) if err != nil { return nil, err } // all current good sectors - newActiveSectors, err := curPart.ActiveSectors() + curActiveSectors, err := curPart.ActiveSectors() if err != nil { return nil, err } // sectors that were previously fault and are now currently active are considered recovered. - recovered, err := bitfield.IntersectBitField(prevPart.Faults, newActiveSectors) + recovered, err := bitfield.IntersectBitField(prevFaults, curActiveSectors) if err != nil { return nil, err } + // TODO: distinguish between "terminated" and "expired" sectors. The + // previous code here never had a chance of working in the first place, + // so I'm not going to try to replicate it right now. + // + // How? If the sector expires before it should (according to sector + // info) and it wasn't replaced by a pre-commit deleted in this change + // set, it was "early terminated". + return &PartitionStatus{ - Terminated: terminated, - Expired: expired, - Faulted: faults, - InRecovery: inRecovery, + Terminated: bitfield.New(), + Expired: removedSectors, + Faulted: newFaults, + InRecovery: newRecoveries, Recovered: recovered, }, nil } From f2a0779bb9a6c4e5f9768c3d6ef3da30027c24cb Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 17 Sep 2020 21:41:36 -0700 Subject: [PATCH 212/885] remove specs-actors import --- chain/events/state/predicates.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/chain/events/state/predicates.go b/chain/events/state/predicates.go index 721de9dd2..e0e977772 100644 --- a/chain/events/state/predicates.go +++ b/chain/events/state/predicates.go @@ -9,15 +9,14 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" - "github.com/filecoin-project/lotus/chain/actors/adt" - init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init" - "github.com/filecoin-project/lotus/chain/actors/builtin/market" - "github.com/filecoin-project/lotus/chain/actors/builtin/paych" - "github.com/filecoin-project/specs-actors/actors/builtin" cbor "github.com/ipfs/go-ipld-cbor" typegen "github.com/whyrusleeping/cbor-gen" "github.com/filecoin-project/lotus/api/apibstore" + "github.com/filecoin-project/lotus/chain/actors/adt" + init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init" + "github.com/filecoin-project/lotus/chain/actors/builtin/market" + "github.com/filecoin-project/lotus/chain/actors/builtin/paych" "github.com/filecoin-project/lotus/chain/types" ) @@ -74,7 +73,7 @@ type DiffStorageMarketStateFunc func(ctx context.Context, oldState market.State, // OnStorageMarketActorChanged calls diffStorageMarketState when the state changes for the market actor func (sp *StatePredicates) OnStorageMarketActorChanged(diffStorageMarketState DiffStorageMarketStateFunc) DiffTipSetKeyFunc { - return sp.OnActorStateChanged(builtin.StorageMarketActorAddr, func(ctx context.Context, oldActorState, newActorState *types.Actor) (changed bool, user UserData, err error) { + return sp.OnActorStateChanged(market.Address, func(ctx context.Context, oldActorState, newActorState *types.Actor) (changed bool, user UserData, err error) { oldState, err := market.Load(adt.WrapStore(ctx, sp.cst), oldActorState) if err != nil { return false, nil, err @@ -295,7 +294,7 @@ func (sp *StatePredicates) AvailableBalanceChangedForAddresses(getAddrs func() [ type DiffMinerActorStateFunc func(ctx context.Context, oldState miner.State, newState miner.State) (changed bool, user UserData, err error) func (sp *StatePredicates) OnInitActorChange(diffInitActorState DiffInitActorStateFunc) DiffTipSetKeyFunc { - return sp.OnActorStateChanged(builtin.InitActorAddr, func(ctx context.Context, oldActorState, newActorState *types.Actor) (changed bool, user UserData, err error) { + return sp.OnActorStateChanged(init_.Address, func(ctx context.Context, oldActorState, newActorState *types.Actor) (changed bool, user UserData, err error) { oldState, err := init_.Load(adt.WrapStore(ctx, sp.cst), oldActorState) if err != nil { return false, nil, err From 8bd6791e58d8420ce4434e8a92ff87768895e274 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 17 Sep 2020 21:44:10 -0700 Subject: [PATCH 213/885] more fixups --- tools/stats/metrics.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/stats/metrics.go b/tools/stats/metrics.go index 4f11d2476..86dbe6780 100644 --- a/tools/stats/metrics.go +++ b/tools/stats/metrics.go @@ -12,13 +12,13 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors/builtin/reward" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/specs-actors/actors/builtin" - "golang.org/x/xerrors" "github.com/ipfs/go-cid" "github.com/multiformats/go-multihash" + "golang.org/x/xerrors" cbg "github.com/whyrusleeping/cbor-gen" @@ -242,7 +242,7 @@ func RecordTipsetStatePoints(ctx context.Context, api api.FullNode, pl *PointLis //p := NewPoint("chain.pledge_collateral", pcFilFloat) //pl.AddPoint(p) - netBal, err := api.WalletBalance(ctx, builtin.RewardActorAddr) + netBal, err := api.WalletBalance(ctx, reward.Address) if err != nil { return err } From daa441b989514f549a1ea5d5bc28bf51e9d3c901 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 17 Sep 2020 21:48:50 -0700 Subject: [PATCH 214/885] simplify market diff --- chain/actors/builtin/market/{util.go => diff.go} | 4 ++-- chain/actors/builtin/market/market.go | 2 -- chain/actors/builtin/market/v0.go | 8 -------- chain/events/state/predicates.go | 4 ++-- 4 files changed, 4 insertions(+), 14 deletions(-) rename chain/actors/builtin/market/{util.go => diff.go} (94%) diff --git a/chain/actors/builtin/market/util.go b/chain/actors/builtin/market/diff.go similarity index 94% rename from chain/actors/builtin/market/util.go rename to chain/actors/builtin/market/diff.go index 92fdd6823..d0b4a2fd3 100644 --- a/chain/actors/builtin/market/util.go +++ b/chain/actors/builtin/market/diff.go @@ -8,7 +8,7 @@ import ( cbg "github.com/whyrusleeping/cbor-gen" ) -func diffDealProposals(pre, cur DealProposals) (*DealProposalChanges, error) { +func DiffDealProposals(pre, cur DealProposals) (*DealProposalChanges, error) { results := new(DealProposalChanges) if err := adt.DiffAdtArray(pre.array(), cur.array(), &marketProposalsDiffer{results, pre, cur}); err != nil { return nil, fmt.Errorf("diffing deal states: %w", err) @@ -44,7 +44,7 @@ func (d *marketProposalsDiffer) Remove(key uint64, val *cbg.Deferred) error { return nil } -func diffDealStates(pre, cur DealStates) (*DealStateChanges, error) { +func DiffDealStates(pre, cur DealStates) (*DealStateChanges, error) { results := new(DealStateChanges) if err := adt.DiffAdtArray(pre.array(), cur.array(), &marketStatesDiffer{results, pre, cur}); err != nil { return nil, fmt.Errorf("diffing deal states: %w", err) diff --git a/chain/actors/builtin/market/market.go b/chain/actors/builtin/market/market.go index 1c11c027e..c65fa093d 100644 --- a/chain/actors/builtin/market/market.go +++ b/chain/actors/builtin/market/market.go @@ -51,7 +51,6 @@ type BalanceTable interface { type DealStates interface { Get(id abi.DealID) (*DealState, bool, error) - Diff(DealStates) (*DealStateChanges, error) array() adt.Array decode(*cbg.Deferred) (*DealState, error) @@ -60,7 +59,6 @@ type DealStates interface { type DealProposals interface { ForEach(cb func(id abi.DealID, dp DealProposal) error) error Get(id abi.DealID) (*DealProposal, bool, error) - Diff(DealProposals) (*DealProposalChanges, error) array() adt.Array decode(*cbg.Deferred) (*DealProposal, error) diff --git a/chain/actors/builtin/market/v0.go b/chain/actors/builtin/market/v0.go index 587446749..671907ad4 100644 --- a/chain/actors/builtin/market/v0.go +++ b/chain/actors/builtin/market/v0.go @@ -124,10 +124,6 @@ func (s *v0DealStates) Get(dealID abi.DealID) (*DealState, bool, error) { return &deal, true, nil } -func (s *v0DealStates) Diff(other DealStates) (*DealStateChanges, error) { - return diffDealStates(s, other) -} - func (s *v0DealStates) decode(val *cbg.Deferred) (*DealState, error) { var v0ds market.DealState if err := v0ds.UnmarshalCBOR(bytes.NewReader(val.Raw)); err != nil { @@ -149,10 +145,6 @@ type v0DealProposals struct { adt.Array } -func (s *v0DealProposals) Diff(other DealProposals) (*DealProposalChanges, error) { - return diffDealProposals(s, other) -} - func (s *v0DealProposals) Get(dealID abi.DealID) (*DealProposal, bool, error) { var v0proposal market.DealProposal found, err := s.Array.Get(uint64(dealID), &v0proposal) diff --git a/chain/events/state/predicates.go b/chain/events/state/predicates.go index e0e977772..4e821633e 100644 --- a/chain/events/state/predicates.go +++ b/chain/events/state/predicates.go @@ -175,7 +175,7 @@ func (sp *StatePredicates) OnDealProposalChanged(diffDealProps DiffDealProposals // - Removed Proposals func (sp *StatePredicates) OnDealProposalAmtChanged() DiffDealProposalsFunc { return func(ctx context.Context, oldDealProps, newDealProps market.DealProposals) (changed bool, user UserData, err error) { - proposalChanges, err := oldDealProps.Diff(newDealProps) + proposalChanges, err := market.DiffDealProposals(oldDealProps, newDealProps) if err != nil { return false, nil, err } @@ -194,7 +194,7 @@ func (sp *StatePredicates) OnDealProposalAmtChanged() DiffDealProposalsFunc { // - Removed Deals func (sp *StatePredicates) OnDealStateAmtChanged() DiffDealStatesFunc { return func(ctx context.Context, oldDealStates, newDealStates market.DealStates) (changed bool, user UserData, err error) { - dealStateChanges, err := oldDealStates.Diff(newDealStates) + dealStateChanges, err := market.DiffDealStates(oldDealStates, newDealStates) if err != nil { return false, nil, err } From 37de154a7cefe4eac620273464fb53b12dc0d190 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Fri, 18 Sep 2020 01:12:45 -0400 Subject: [PATCH 215/885] Fixup tests --- api/test/ccupgrade.go | 1 + chain/actors/builtin/miner/v0.go | 11 ++++++----- chain/events/state/predicates_test.go | 4 ++-- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/api/test/ccupgrade.go b/api/test/ccupgrade.go index 130b55cb8..b281f30a0 100644 --- a/api/test/ccupgrade.go +++ b/api/test/ccupgrade.go @@ -85,6 +85,7 @@ func TestCCUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration) { { exp, err := client.StateSectorExpiration(ctx, maddr, CC, types.EmptyTSK) require.NoError(t, err) + require.NotNil(t, exp) require.Greater(t, 50000, int(exp.OnTime)) } { diff --git a/chain/actors/builtin/miner/v0.go b/chain/actors/builtin/miner/v0.go index 9f4cea6ce..e0ae2594a 100644 --- a/chain/actors/builtin/miner/v0.go +++ b/chain/actors/builtin/miner/v0.go @@ -65,7 +65,7 @@ func (s *v0State) FindSector(num abi.SectorNumber) (*SectorLocation, error) { // If the sector isn't found or has already been terminated, this method returns // nil and no error. If the sector does not expire early, the Early expiration // field is 0. -func (s *v0State) GetSectorExpiration(num abi.SectorNumber) (out *SectorExpiration, err error) { +func (s *v0State) GetSectorExpiration(num abi.SectorNumber) (*SectorExpiration, error) { dls, err := s.State.LoadDeadlines(s.store) if err != nil { return nil, err @@ -77,6 +77,7 @@ func (s *v0State) GetSectorExpiration(num abi.SectorNumber) (out *SectorExpirati // 2. If it's faulty, it will expire early within the first 14 entries // of the expiration queue. stopErr := errors.New("stop") + out := SectorExpiration{} err = dls.ForEach(s.store, func(dlIdx uint64, dl *v0miner.Deadline) error { partitions, err := dl.PartitionsArray(s.store) if err != nil { @@ -97,7 +98,7 @@ func (s *v0State) GetSectorExpiration(num abi.SectorNumber) (out *SectorExpirati return stopErr } - q, err := v0miner.LoadExpirationQueue(s.store, part.EarlyTerminated, quant) + q, err := v0miner.LoadExpirationQueue(s.store, part.ExpirationsEpochs, quant) if err != nil { return err } @@ -128,7 +129,7 @@ func (s *v0State) GetSectorExpiration(num abi.SectorNumber) (out *SectorExpirati if out.Early == 0 && out.OnTime == 0 { return nil, nil } - return out, nil + return &out, nil } func (s *v0State) GetPrecommittedSector(num abi.SectorNumber) (*SectorPreCommitOnChainInfo, error) { @@ -294,14 +295,14 @@ func (d *v0Deadline) ForEachPartition(cb func(uint64, Partition) error) error { }) } -func (s *v0Deadline) PartitionsChanged(other Deadline) bool { +func (d *v0Deadline) PartitionsChanged(other Deadline) bool { v0other, ok := other.(*v0Deadline) if !ok { // treat an upgrade as a change, always return true } - return s.Deadline.Partitions.Equals(v0other.Deadline.Partitions) + return d.Deadline.Partitions.Equals(v0other.Deadline.Partitions) } func (d *v0Deadline) PostSubmissions() (bitfield.BitField, error) { diff --git a/chain/events/state/predicates_test.go b/chain/events/state/predicates_test.go index 243d8814e..0412c6421 100644 --- a/chain/events/state/predicates_test.go +++ b/chain/events/state/predicates_test.go @@ -398,8 +398,8 @@ func TestMinerSectorChange(t *testing.T) { require.NoError(t, err) api := newMockAPI(bs) - api.setActor(oldState.Key(), &types.Actor{Head: oldMinerC}) - api.setActor(newState.Key(), &types.Actor{Head: newMinerC}) + api.setActor(oldState.Key(), &types.Actor{Head: oldMinerC, Code: v0builtin.StorageMinerActorCodeID}) + api.setActor(newState.Key(), &types.Actor{Head: newMinerC, Code: v0builtin.StorageMinerActorCodeID}) preds := NewStatePredicates(api) From fce423c7431d2d48cea876ffdbe95a5747f320dd Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Fri, 18 Sep 2020 02:14:18 -0400 Subject: [PATCH 216/885] Appease the linter and get everything building --- build/params_testground.go | 3 ++- extern/storage-sealing/checks.go | 1 + extern/storage-sealing/precommit_policy.go | 1 + extern/storage-sealing/precommit_policy_test.go | 3 ++- extern/storage-sealing/sealing.go | 5 ++--- extern/storage-sealing/states_sealing.go | 3 +++ go.sum | 4 ---- node/impl/full/state.go | 3 --- 8 files changed, 11 insertions(+), 12 deletions(-) diff --git a/build/params_testground.go b/build/params_testground.go index 77e312ac2..1b30ae2e9 100644 --- a/build/params_testground.go +++ b/build/params_testground.go @@ -80,5 +80,6 @@ var ( 0: DrandMainnet, } - NewestNetworkVersion = network.Version2 + NewestNetworkVersion = network.Version2 + ActorUpgradeNetworkVersion = network.Version3 ) diff --git a/extern/storage-sealing/checks.go b/extern/storage-sealing/checks.go index ae5ce0d33..677cef1e9 100644 --- a/extern/storage-sealing/checks.go +++ b/extern/storage-sealing/checks.go @@ -105,6 +105,7 @@ func checkPrecommit(ctx context.Context, maddr address.Address, si SectorInfo, t msd = v0miner.MaxSealDuration[si.SectorType] } else { // TODO: ActorUpgrade + msd = 0 } if height-(si.TicketEpoch+SealRandomnessLookback) > msd { diff --git a/extern/storage-sealing/precommit_policy.go b/extern/storage-sealing/precommit_policy.go index e36b8251a..76d867144 100644 --- a/extern/storage-sealing/precommit_policy.go +++ b/extern/storage-sealing/precommit_policy.go @@ -90,6 +90,7 @@ func (p *BasicPreCommitPolicy) Expiration(ctx context.Context, ps ...Piece) (abi wpp = v0miner.WPoStProvingPeriod } else { // TODO: ActorUpgrade + wpp = 0 } *end += wpp - (*end % wpp) + p.provingBoundary - 1 diff --git a/extern/storage-sealing/precommit_policy_test.go b/extern/storage-sealing/precommit_policy_test.go index 30b538a88..52814167a 100644 --- a/extern/storage-sealing/precommit_policy_test.go +++ b/extern/storage-sealing/precommit_policy_test.go @@ -2,9 +2,10 @@ package sealing_test import ( "context" + "testing" + "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/build" - "testing" "github.com/ipfs/go-cid" "github.com/stretchr/testify/assert" diff --git a/extern/storage-sealing/sealing.go b/extern/storage-sealing/sealing.go index 6d60e7a6e..01551e6d7 100644 --- a/extern/storage-sealing/sealing.go +++ b/extern/storage-sealing/sealing.go @@ -431,9 +431,8 @@ func (m *Sealing) getPreCommitChallengeDelay(ctx context.Context, tok TipSetToke if nv < build.ActorUpgradeNetworkVersion { return v0miner.PreCommitChallengeDelay, nil - } else { - // TODO: ActorUpgrade - return -1, nil } + // TODO: ActorUpgrade + return -1, nil } diff --git a/extern/storage-sealing/states_sealing.go b/extern/storage-sealing/states_sealing.go index f0ff4025d..6ae42a91f 100644 --- a/extern/storage-sealing/states_sealing.go +++ b/extern/storage-sealing/states_sealing.go @@ -195,6 +195,8 @@ func (m *Sealing) handlePreCommitting(ctx statemachine.Context, sector SectorInf mse = v0miner.MinSectorExpiration } else { // TODO: ActorUpgrade + msd = 0 + mse = 0 } if minExpiration := height + msd + mse + 10; expiration < minExpiration { @@ -395,6 +397,7 @@ func (m *Sealing) handleSubmitCommit(ctx statemachine.Context, sector SectorInfo } } else { // TODO: ActorUpgrade + enc = nil } waddr, err := m.api.StateMinerWorkerAddress(ctx.Context(), m.maddr, tok) diff --git a/go.sum b/go.sum index 3d012da92..5ebec6cbf 100644 --- a/go.sum +++ b/go.sum @@ -226,10 +226,6 @@ github.com/filecoin-project/go-data-transfer v0.6.3 h1:7TLwm8nuodHYD/uiwJjKc/PGR github.com/filecoin-project/go-data-transfer v0.6.3/go.mod h1:PmBKVXkhh67/tnEdJXQwDHl5mT+7Tbcwe1NPninqhnM= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f h1:GxJzR3oRIMTPtpZ0b7QF8FKPK6/iPAc7trhlL5k/g+s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= -github.com/filecoin-project/go-fil-markets v0.6.1-0.20200911011457-2959ccca6a3c h1:YGoyYmELQ0LHwDj/WcOvY3oYt+3iM0wdrAhqJQUAIy4= -github.com/filecoin-project/go-fil-markets v0.6.1-0.20200911011457-2959ccca6a3c/go.mod h1:PLr9svZxsnHkae1Ky7+66g7fP9AlneVxIVu+oSMq56A= -github.com/filecoin-project/go-fil-markets v0.6.1-0.20200917050751-2af52e9606c6 h1:k97Z2JP3WpDVGU/7Bz3RtnqrYtn9X428Ps8OkoFq61I= -github.com/filecoin-project/go-fil-markets v0.6.1-0.20200917050751-2af52e9606c6/go.mod h1:PLr9svZxsnHkae1Ky7+66g7fP9AlneVxIVu+oSMq56A= github.com/filecoin-project/go-fil-markets v0.6.1-0.20200917052354-ee0af754c6e9 h1:SnCUC9wHDId9TtV8PsQp8q1OOsi+NOLOwitIDnAgUa4= github.com/filecoin-project/go-fil-markets v0.6.1-0.20200917052354-ee0af754c6e9/go.mod h1:PLr9svZxsnHkae1Ky7+66g7fP9AlneVxIVu+oSMq56A= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 04055043a..729baef2c 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -3,7 +3,6 @@ package full import ( "bytes" "context" - "errors" "strconv" v0market "github.com/filecoin-project/specs-actors/actors/builtin/market" @@ -47,8 +46,6 @@ import ( "github.com/filecoin-project/lotus/node/modules/dtypes" ) -var errBreakForeach = errors.New("break") - type StateAPI struct { fx.In From 3c7246196933063fe6c01deb36cb650e96f2dead Mon Sep 17 00:00:00 2001 From: vyzo Date: Fri, 18 Sep 2020 09:40:43 +0300 Subject: [PATCH 217/885] MpoolPushUntrusted API for gateway --- api/api_full.go | 3 ++ api/apistruct/struct.go | 8 +++- chain/messagepool/messagepool.go | 77 +++++++++++++++++++++++++++----- cmd/lotus-gateway/api.go | 2 +- node/impl/full/mpool.go | 4 ++ 5 files changed, 80 insertions(+), 14 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index dace85ed3..c60656e4d 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -187,6 +187,9 @@ type FullNode interface { // MpoolPush pushes a signed message to mempool. MpoolPush(context.Context, *types.SignedMessage) (cid.Cid, error) + // MpoolPushUntrusted pushes a signed message to mempool from untrusted sources. + MpoolPushUntrusted(context.Context, *types.SignedMessage) (cid.Cid, error) + // MpoolPushMessage atomically assigns a nonce, signs, and pushes a message // to mempool. // maxFee is only used when GasFeeCap/GasPremium fields aren't specified diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index b37c667e9..f225ac5de 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -120,7 +120,9 @@ type FullNodeStruct struct { MpoolPending func(context.Context, types.TipSetKey) ([]*types.SignedMessage, error) `perm:"read"` MpoolClear func(context.Context, bool) error `perm:"write"` - MpoolPush func(context.Context, *types.SignedMessage) (cid.Cid, error) `perm:"write"` + MpoolPush func(context.Context, *types.SignedMessage) (cid.Cid, error) `perm:"write"` + MpoolPushUntrusted func(context.Context, *types.SignedMessage) (cid.Cid, error) `perm:"write"` + MpoolPushMessage func(context.Context, *types.Message, *api.MessageSendSpec) (*types.SignedMessage, error) `perm:"sign"` MpoolGetNonce func(context.Context, address.Address) (uint64, error) `perm:"read"` MpoolSub func(context.Context) (<-chan api.MpoolUpdate, error) `perm:"read"` @@ -549,6 +551,10 @@ func (c *FullNodeStruct) MpoolPush(ctx context.Context, smsg *types.SignedMessag return c.Internal.MpoolPush(ctx, smsg) } +func (c *FullNodeStruct) MpoolPushUntrusted(ctx context.Context, smsg *types.SignedMessage) (cid.Cid, error) { + return c.Internal.MpoolPushUntrusted(ctx, smsg) +} + func (c *FullNodeStruct) MpoolPushMessage(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec) (*types.SignedMessage, error) { return c.Internal.MpoolPushMessage(ctx, msg, spec) } diff --git a/chain/messagepool/messagepool.go b/chain/messagepool/messagepool.go index 96900925f..6ee86b732 100644 --- a/chain/messagepool/messagepool.go +++ b/chain/messagepool/messagepool.go @@ -55,6 +55,7 @@ var baseFeeLowerBoundFactor = types.NewInt(10) var baseFeeLowerBoundFactorConservative = types.NewInt(100) var MaxActorPendingMessages = 1000 +var MaxUntrustedActorPendingMessages = 10 var MaxNonceGap = uint64(4) @@ -197,9 +198,17 @@ func CapGasFee(msg *types.Message, maxFee abi.TokenAmount) { msg.GasPremium = big.Min(msg.GasFeeCap, msg.GasPremium) // cap premium at FeeCap } -func (ms *msgSet) add(m *types.SignedMessage, mp *MessagePool, strict bool) (bool, error) { +func (ms *msgSet) add(m *types.SignedMessage, mp *MessagePool, strict, untrusted bool) (bool, error) { nextNonce := ms.nextNonce nonceGap := false + + maxNonceGap := MaxNonceGap + maxActorPendingMessages := MaxActorPendingMessages + if untrusted { + maxNonceGap = 0 + maxActorPendingMessages = MaxUntrustedActorPendingMessages + } + switch { case m.Message.Nonce == nextNonce: nextNonce++ @@ -208,7 +217,7 @@ func (ms *msgSet) add(m *types.SignedMessage, mp *MessagePool, strict bool) (boo nextNonce++ } - case strict && m.Message.Nonce > nextNonce+MaxNonceGap: + case strict && m.Message.Nonce > nextNonce+maxNonceGap: return false, xerrors.Errorf("message nonce has too big a gap from expected nonce (Nonce: %d, nextNonce: %d): %w", m.Message.Nonce, nextNonce, ErrNonceGap) case m.Message.Nonce > nextNonce: @@ -244,7 +253,7 @@ func (ms *msgSet) add(m *types.SignedMessage, mp *MessagePool, strict bool) (boo //ms.requiredFunds.Sub(ms.requiredFunds, exms.Message.Value.Int) } - if !has && strict && len(ms.msgs) > MaxActorPendingMessages { + if !has && strict && len(ms.msgs) > maxActorPendingMessages { log.Errorf("too many pending messages from actor %s", m.Message.From) return false, ErrTooManyPendingMessages } @@ -486,7 +495,7 @@ func (mp *MessagePool) Push(m *types.SignedMessage) (cid.Cid, error) { } mp.curTsLk.Lock() - publish, err := mp.addTs(m, mp.curTs, true) + publish, err := mp.addTs(m, mp.curTs, true, false) if err != nil { mp.curTsLk.Unlock() return cid.Undef, err @@ -553,7 +562,7 @@ func (mp *MessagePool) Add(m *types.SignedMessage) error { mp.curTsLk.Lock() defer mp.curTsLk.Unlock() - _, err = mp.addTs(m, mp.curTs, false) + _, err = mp.addTs(m, mp.curTs, false, false) return err } @@ -621,7 +630,7 @@ func (mp *MessagePool) checkBalance(m *types.SignedMessage, curTs *types.TipSet) return nil } -func (mp *MessagePool) addTs(m *types.SignedMessage, curTs *types.TipSet, local bool) (bool, error) { +func (mp *MessagePool) addTs(m *types.SignedMessage, curTs *types.TipSet, local, untrusted bool) (bool, error) { snonce, err := mp.getStateNonce(m.Message.From, curTs) if err != nil { return false, xerrors.Errorf("failed to look up actor state nonce: %s: %w", err, ErrSoftValidationFailure) @@ -643,7 +652,7 @@ func (mp *MessagePool) addTs(m *types.SignedMessage, curTs *types.TipSet, local return false, err } - return publish, mp.addLocked(m, !local) + return publish, mp.addLocked(m, !local, untrusted) } func (mp *MessagePool) addLoaded(m *types.SignedMessage) error { @@ -678,17 +687,17 @@ func (mp *MessagePool) addLoaded(m *types.SignedMessage) error { return err } - return mp.addLocked(m, false) + return mp.addLocked(m, false, false) } func (mp *MessagePool) addSkipChecks(m *types.SignedMessage) error { mp.lk.Lock() defer mp.lk.Unlock() - return mp.addLocked(m, false) + return mp.addLocked(m, false, false) } -func (mp *MessagePool) addLocked(m *types.SignedMessage, strict bool) error { +func (mp *MessagePool) addLocked(m *types.SignedMessage, strict, untrusted bool) error { log.Debugf("mpooladd: %s %d", m.Message.From, m.Message.Nonce) if m.Signature.Type == crypto.SigTypeBLS { mp.blsSigCache.Add(m.Cid(), m.Signature) @@ -715,7 +724,7 @@ func (mp *MessagePool) addLocked(m *types.SignedMessage, strict bool) error { mp.pending[m.Message.From] = mset } - incr, err := mset.add(m, mp, strict) + incr, err := mset.add(m, mp, strict, untrusted) if err != nil { log.Debug(err) return err @@ -873,7 +882,7 @@ func (mp *MessagePool) PushWithNonce(ctx context.Context, addr address.Address, return nil, err } - if err := mp.addLocked(msg, false); err != nil { + if err := mp.addLocked(msg, false, false); err != nil { return nil, xerrors.Errorf("add locked failed: %w", err) } if err := mp.addLocal(msg, msgb); err != nil { @@ -887,6 +896,50 @@ func (mp *MessagePool) PushWithNonce(ctx context.Context, addr address.Address, return msg, err } +// this method is provided for the gateway to push messages. +// differences from Push: +// - strict checks are enabled +// - extra strict add checks are used when adding the messages to the msgSet +// that means: no nonce gaps, at most 10 pending messages for the actor +func (mp *MessagePool) PushUntrusted(m *types.SignedMessage) (cid.Cid, error) { + err := mp.checkMessage(m) + if err != nil { + return cid.Undef, err + } + + // serialize push access to reduce lock contention + mp.addSema <- struct{}{} + defer func() { + <-mp.addSema + }() + + msgb, err := m.Serialize() + if err != nil { + return cid.Undef, err + } + + mp.curTsLk.Lock() + publish, err := mp.addTs(m, mp.curTs, false, true) + if err != nil { + mp.curTsLk.Unlock() + return cid.Undef, err + } + mp.curTsLk.Unlock() + + mp.lk.Lock() + if err := mp.addLocal(m, msgb); err != nil { + mp.lk.Unlock() + return cid.Undef, err + } + mp.lk.Unlock() + + if publish { + err = mp.api.PubSubPublish(build.MessagesTopic(mp.netName), msgb) + } + + return m.Cid(), err +} + func (mp *MessagePool) Remove(from address.Address, nonce uint64, applied bool) { mp.lk.Lock() defer mp.lk.Unlock() diff --git a/cmd/lotus-gateway/api.go b/cmd/lotus-gateway/api.go index 42e9e4829..0a6365dbd 100644 --- a/cmd/lotus-gateway/api.go +++ b/cmd/lotus-gateway/api.go @@ -86,5 +86,5 @@ func (a *GatewayAPI) MpoolPush(ctx context.Context, sm *types.SignedMessage) (ci // TODO: additional anti-spam checks - return a.api.MpoolPush(ctx, sm) + return a.api.MpoolPushUntrusted(ctx, sm) } diff --git a/node/impl/full/mpool.go b/node/impl/full/mpool.go index 6acb17990..a847d8d8e 100644 --- a/node/impl/full/mpool.go +++ b/node/impl/full/mpool.go @@ -113,6 +113,10 @@ func (a *MpoolAPI) MpoolPush(ctx context.Context, smsg *types.SignedMessage) (ci return a.Mpool.Push(smsg) } +func (a *MpoolAPI) MpoolPushUntrusted(ctx context.Context, smsg *types.SignedMessage) (cid.Cid, error) { + return a.Mpool.PushUntrusted(smsg) +} + func (a *MpoolAPI) MpoolPushMessage(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec) (*types.SignedMessage, error) { inMsg := *msg { From 694463ffbe63f2f48985c391bdf98f388155fabc Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Fri, 18 Sep 2020 03:06:28 -0400 Subject: [PATCH 218/885] More lint fixes --- chain/events/state/predicates.go | 1 - chain/events/state/predicates_test.go | 2 ++ chain/stmgr/stmgr.go | 1 + cmd/lotus-pcr/main.go | 1 + 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/chain/events/state/predicates.go b/chain/events/state/predicates.go index 4e821633e..21e1720e6 100644 --- a/chain/events/state/predicates.go +++ b/chain/events/state/predicates.go @@ -530,6 +530,5 @@ func (sp *StatePredicates) OnAddressMapChange() DiffInitActorStateFunc { return true, addressChanges, nil*/ panic("TODO") - return false, nil, nil } } diff --git a/chain/events/state/predicates_test.go b/chain/events/state/predicates_test.go index 0412c6421..604eec75d 100644 --- a/chain/events/state/predicates_test.go +++ b/chain/events/state/predicates_test.go @@ -246,6 +246,7 @@ func TestMarketPredicates(t *testing.T) { Code: v0builtin.StorageMarketActorCodeID, Head: marketCid, }) + require.NoError(t, err) changed, _, err = diffDealStateFn(ctx, marketState, marketState) require.NoError(t, err) require.False(t, changed) @@ -358,6 +359,7 @@ func TestMarketPredicates(t *testing.T) { Code: v0builtin.StorageMarketActorCodeID, Head: marketCid, }) + require.NoError(t, err) changed, _, err = diffDealBalancesFn(ctx, marketState, marketState) require.NoError(t, err) require.False(t, changed) diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index 90154868e..d1adbbc62 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -262,6 +262,7 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp } } else { // TODO: ActorUpgrade + params = nil } sysAct, err := vmi.StateTree().GetActor(builtin.SystemActorAddr) diff --git a/cmd/lotus-pcr/main.go b/cmd/lotus-pcr/main.go index aa467e38a..3dedb24fc 100644 --- a/cmd/lotus-pcr/main.go +++ b/cmd/lotus-pcr/main.go @@ -412,6 +412,7 @@ func (r *refunder) ProcessTipset(ctx context.Context, tipset *types.TipSet, refu sn = proveCommitSector.SectorNumber } else { // TODO: ActorUpgrade + sn = -1 } // We use the parent tipset key because precommit information is removed when ProveCommitSector is executed From 37108ebf8fa9f5fc6f78e9fe4b50ee1d72b4e17a Mon Sep 17 00:00:00 2001 From: Jennifer <42981373+jennijuju@users.noreply.github.com> Date: Fri, 18 Sep 2020 04:36:27 -0400 Subject: [PATCH 219/885] Update build and setup instruction link Update the link since we deprecated lotus.sh and migrates it to doc.Filecoin --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6c1e23efa..766317e4f 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ Lotus is an implementation of the Filecoin Distributed Storage Network. For more ## Building & Documentation -For instructions on how to build lotus from source, please visit [https://lotu.sh](https://lotu.sh) or read the source [here](https://github.com/filecoin-project/lotus/tree/master/documentation). +For instructions on how to build lotus from source, please visit [Lotus build and setup instruction](https://docs.filecoin.io/get-started/lotus/installation/#minimal-requirements) or read the source [here](https://github.com/filecoin-project/lotus/tree/master/documentation). ## Reporting a Vulnerability From eadc61c37a81ca999ff1c2a021352a82f4df067a Mon Sep 17 00:00:00 2001 From: Travis Person Date: Fri, 18 Sep 2020 18:07:58 +0000 Subject: [PATCH 220/885] log shutdown method for lotus daemon and miner --- cmd/lotus-storage-miner/run.go | 4 +++- cmd/lotus/rpc.go | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/cmd/lotus-storage-miner/run.go b/cmd/lotus-storage-miner/run.go index 83d78172e..a5d996f78 100644 --- a/cmd/lotus-storage-miner/run.go +++ b/cmd/lotus-storage-miner/run.go @@ -163,8 +163,10 @@ var runCmd = &cli.Command{ sigChan := make(chan os.Signal, 2) go func() { select { - case <-sigChan: + case sig := <-sigChan: + log.Warnw("received shutdown", "signal", sig) case <-shutdownChan: + log.Warn("received shutdown") } log.Warn("Shutting down...") diff --git a/cmd/lotus/rpc.go b/cmd/lotus/rpc.go index fbe05e938..9718deb3a 100644 --- a/cmd/lotus/rpc.go +++ b/cmd/lotus/rpc.go @@ -66,8 +66,10 @@ func serveRPC(a api.FullNode, stop node.StopFunc, addr multiaddr.Multiaddr, shut shutdownDone := make(chan struct{}) go func() { select { - case <-sigCh: + case sig := <-sigCh: + log.Warnw("received shutdown", "signal", sig) case <-shutdownCh: + log.Warn("received shutdown") } log.Warn("Shutting down...") From ccd0a67b01c31da932c2a3b052269ba4d703df17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 18 Sep 2020 20:34:23 +0200 Subject: [PATCH 221/885] wip fixing wdpost tests --- storage/wdpost_run_test.go | 61 ++++++++++++++++++++++++-------------- 1 file changed, 38 insertions(+), 23 deletions(-) diff --git a/storage/wdpost_run_test.go b/storage/wdpost_run_test.go index af529a75e..1a930b41a 100644 --- a/storage/wdpost_run_test.go +++ b/storage/wdpost_run_test.go @@ -7,8 +7,6 @@ import ( "github.com/filecoin-project/go-state-types/dline" - "golang.org/x/xerrors" - "github.com/stretchr/testify/require" "github.com/filecoin-project/go-address" @@ -17,17 +15,20 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/ipfs/go-cid" - "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" - "github.com/filecoin-project/specs-actors/actors/runtime/proof" - "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/crypto" + "github.com/filecoin-project/go-state-types/network" + "github.com/filecoin-project/specs-actors/actors/builtin" + v0builtin "github.com/filecoin-project/specs-actors/actors/builtin" + v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" + "github.com/filecoin-project/specs-actors/actors/runtime/proof" tutils "github.com/filecoin-project/specs-actors/support/testing" + + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" ) type mockStorageMinerAPI struct { - partitions []*miner.Partition + partitions []api.Partition pushedMessages chan *types.Message } @@ -37,6 +38,14 @@ func newMockStorageMinerAPI() *mockStorageMinerAPI { } } +func (m *mockStorageMinerAPI) StateMinerInfo(ctx context.Context, a address.Address, key types.TipSetKey) (miner.MinerInfo, error) { + panic("implement me") +} + +func (m *mockStorageMinerAPI) StateNetworkVersion(ctx context.Context, key types.TipSetKey) (network.Version, error) { + panic("implement me") +} + func (m *mockStorageMinerAPI) ChainGetRandomnessFromTickets(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error) { return abi.Randomness("ticket rand"), nil } @@ -45,18 +54,18 @@ func (m *mockStorageMinerAPI) ChainGetRandomnessFromBeacon(ctx context.Context, return abi.Randomness("beacon rand"), nil } -func (m *mockStorageMinerAPI) setPartitions(ps []*miner.Partition) { +func (m *mockStorageMinerAPI) setPartitions(ps []api.Partition) { m.partitions = append(m.partitions, ps...) } -func (m *mockStorageMinerAPI) StateMinerPartitions(ctx context.Context, address address.Address, u uint64, key types.TipSetKey) ([]*miner.Partition, error) { +func (m *mockStorageMinerAPI) StateMinerPartitions(ctx context.Context, a address.Address, dlIdx uint64, tsk types.TipSetKey) ([]api.Partition, error) { return m.partitions, nil } -func (m *mockStorageMinerAPI) StateMinerSectors(ctx context.Context, address address.Address, field *bitfield.BitField, b bool, key types.TipSetKey) ([]*api.ChainSectorInfo, error) { - var sis []*api.ChainSectorInfo +func (m *mockStorageMinerAPI) StateMinerSectors(ctx context.Context, address address.Address, field *bitfield.BitField, b bool, key types.TipSetKey) ([]*miner.ChainSectorInfo, error) { + var sis []*miner.ChainSectorInfo _ = field.ForEach(func(i uint64) error { - sis = append(sis, &api.ChainSectorInfo{ + sis = append(sis, &miner.ChainSectorInfo{ Info: miner.SectorOnChainInfo{ SectorNumber: abi.SectorNumber(i), }, @@ -67,10 +76,6 @@ func (m *mockStorageMinerAPI) StateMinerSectors(ctx context.Context, address add return sis, nil } -func (m *mockStorageMinerAPI) StateMinerInfo(ctx context.Context, address address.Address, key types.TipSetKey) (api.MinerInfo, error) { - return api.MinerInfo{}, xerrors.Errorf("err") -} - func (m *mockStorageMinerAPI) MpoolPushMessage(ctx context.Context, message *types.Message, spec *api.MessageSendSpec) (*types.SignedMessage, error) { m.pushedMessages <- message return &types.SignedMessage{ @@ -127,21 +132,27 @@ func TestWDPostDoPost(t *testing.T) { require.NoError(t, err) // Work out the number of partitions that can be included in a message // without exceeding the message sector limit - partitionsPerMsg := int(miner.AddressedSectorsMax / sectorsPerPartition) + + require.NoError(t, err) + partitionsPerMsg := int(v0miner.AddressedSectorsMax / sectorsPerPartition) // Enough partitions to fill expectedMsgCount-1 messages partitionCount := (expectedMsgCount - 1) * partitionsPerMsg // Add an extra partition that should be included in the last message partitionCount++ - var partitions []*miner.Partition + var partitions []api.Partition for p := 0; p < partitionCount; p++ { sectors := bitfield.New() for s := uint64(0); s < sectorsPerPartition; s++ { sectors.Set(s) } - partitions = append(partitions, &miner.Partition{ - Sectors: sectors, + partitions = append(partitions, api.Partition{ + AllSectors: sectors, + FaultySectors: bitfield.New(), + RecoveringSectors: bitfield.New(), + LiveSectors: sectors, + ActiveSectors: sectors, }) } mockStgMinerAPI.setPartitions(partitions) @@ -204,7 +215,7 @@ func (m *mockStorageMinerAPI) StateCall(ctx context.Context, message *types.Mess panic("implement me") } -func (m *mockStorageMinerAPI) StateMinerDeadlines(ctx context.Context, maddr address.Address, tok types.TipSetKey) ([]*miner.Deadline, error) { +func (m *mockStorageMinerAPI) StateMinerDeadlines(ctx context.Context, maddr address.Address, tok types.TipSetKey) ([]api.Deadline, error) { panic("implement me") } @@ -216,7 +227,7 @@ func (m *mockStorageMinerAPI) StateSectorGetInfo(ctx context.Context, address ad panic("implement me") } -func (m *mockStorageMinerAPI) StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*api.SectorLocation, error) { +func (m *mockStorageMinerAPI) StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*miner.SectorLocation, error) { panic("implement me") } @@ -237,7 +248,9 @@ func (m *mockStorageMinerAPI) StateSearchMsg(ctx context.Context, cid cid.Cid) ( } func (m *mockStorageMinerAPI) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) { - panic("implement me") + return &types.Actor{ + Code: v0builtin.StorageMinerActorCodeID, + }, nil } func (m *mockStorageMinerAPI) StateGetReceipt(ctx context.Context, cid cid.Cid, key types.TipSetKey) (*types.MessageReceipt, error) { @@ -303,3 +316,5 @@ func (m *mockStorageMinerAPI) WalletBalance(ctx context.Context, address address func (m *mockStorageMinerAPI) WalletHas(ctx context.Context, address address.Address) (bool, error) { panic("implement me") } + +var _ storageMinerApi = &mockStorageMinerAPI{} From c3046f487cf01a0a4d06c42510f04c33ddd3965a Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Fri, 18 Sep 2020 14:41:03 -0400 Subject: [PATCH 222/885] Don't use -1 as default sector number --- cmd/lotus-pcr/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/lotus-pcr/main.go b/cmd/lotus-pcr/main.go index 3dedb24fc..53c8223ed 100644 --- a/cmd/lotus-pcr/main.go +++ b/cmd/lotus-pcr/main.go @@ -412,7 +412,7 @@ func (r *refunder) ProcessTipset(ctx context.Context, tipset *types.TipSet, refu sn = proveCommitSector.SectorNumber } else { // TODO: ActorUpgrade - sn = -1 + sn = 0 } // We use the parent tipset key because precommit information is removed when ProveCommitSector is executed From fb2b25c2975b9f996646d71e9fd0b3410d7572b0 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 18 Sep 2020 13:43:14 -0700 Subject: [PATCH 223/885] finish upgrading chainwatch --- chain/actors/builtin/init/init.go | 3 + chain/actors/builtin/init/v0.go | 19 ++++ chain/actors/builtin/power/power.go | 4 + chain/actors/builtin/power/v0.go | 12 +++ chain/actors/builtin/reward/reward.go | 11 ++- chain/actors/builtin/reward/v0.go | 22 ++++- .../processor/common_actors.go | 29 ++---- cmd/lotus-chainwatch/processor/miner.go | 29 ++---- cmd/lotus-chainwatch/processor/power.go | 76 +++++++------- cmd/lotus-chainwatch/processor/reward.go | 99 ++++++++++++------- node/impl/full/state.go | 4 +- 11 files changed, 185 insertions(+), 123 deletions(-) diff --git a/chain/actors/builtin/init/init.go b/chain/actors/builtin/init/init.go index 3b1a49564..7485c599f 100644 --- a/chain/actors/builtin/init/init.go +++ b/chain/actors/builtin/init/init.go @@ -4,6 +4,7 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/cbor" v0builtin "github.com/filecoin-project/specs-actors/actors/builtin" @@ -33,4 +34,6 @@ type State interface { ResolveAddress(address address.Address) (address.Address, bool, error) MapAddressToNewID(address address.Address) (address.Address, error) NetworkName() (dtypes.NetworkName, error) + + ForEachActor(func(id abi.ActorID, address address.Address) error) error } diff --git a/chain/actors/builtin/init/v0.go b/chain/actors/builtin/init/v0.go index 711b702e2..e286e1ef2 100644 --- a/chain/actors/builtin/init/v0.go +++ b/chain/actors/builtin/init/v0.go @@ -2,11 +2,15 @@ package init import ( "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + cbg "github.com/whyrusleeping/cbor-gen" init_ "github.com/filecoin-project/specs-actors/actors/builtin/init" "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/node/modules/dtypes" + + v0adt "github.com/filecoin-project/specs-actors/actors/util/adt" ) type v0State struct { @@ -22,6 +26,21 @@ func (s *v0State) MapAddressToNewID(address address.Address) (address.Address, e return s.State.MapAddressToNewID(s.store, address) } +func (s *v0State) ForEachActor(cb func(id abi.ActorID, address address.Address) error) error { + addrs, err := v0adt.AsMap(s.store, s.State.AddressMap) + if err != nil { + return err + } + var actorID cbg.CborInt + return addrs.ForEach(&actorID, func(key string) error { + addr, err := address.NewFromBytes([]byte(key)) + if err != nil { + return err + } + return cb(abi.ActorID(actorID), addr) + }) +} + func (s *v0State) NetworkName() (dtypes.NetworkName, error) { return dtypes.NetworkName(s.State.NetworkName), nil } diff --git a/chain/actors/builtin/power/power.go b/chain/actors/builtin/power/power.go index 65148f0a5..7170526bf 100644 --- a/chain/actors/builtin/power/power.go +++ b/chain/actors/builtin/power/power.go @@ -33,8 +33,12 @@ type State interface { TotalLocked() (abi.TokenAmount, error) TotalPower() (Claim, error) + TotalCommitted() (Claim, error) TotalPowerSmoothed() (builtin.FilterEstimate, error) + // MinerCounts returns the number of miners. Participating is the number + // with power above the minimum miner threshold. + MinerCounts() (participating, total uint64, err error) MinerPower(address.Address) (Claim, bool, error) MinerNominalPowerMeetsConsensusMinimum(address.Address) (bool, error) ListAllMiners() ([]address.Address, error) diff --git a/chain/actors/builtin/power/v0.go b/chain/actors/builtin/power/v0.go index 5cf6920c8..9730be893 100644 --- a/chain/actors/builtin/power/v0.go +++ b/chain/actors/builtin/power/v0.go @@ -24,6 +24,14 @@ func (s *v0State) TotalPower() (Claim, error) { }, nil } +// Committed power to the network. Includes miners below the minimum threshold. +func (s *v0State) TotalCommitted() (Claim, error) { + return Claim{ + RawBytePower: s.TotalBytesCommitted, + QualityAdjPower: s.TotalQABytesCommitted, + }, nil +} + func (s *v0State) MinerPower(addr address.Address) (Claim, bool, error) { claims, err := adt.AsMap(s.store, s.Claims) if err != nil { @@ -48,6 +56,10 @@ func (s *v0State) TotalPowerSmoothed() (builtin.FilterEstimate, error) { return *s.State.ThisEpochQAPowerSmoothed, nil } +func (s *v0State) MinerCounts() (uint64, uint64, error) { + return uint64(s.State.MinerAboveMinPowerCount), uint64(s.State.MinerCount), nil +} + func (s *v0State) ListAllMiners() ([]address.Address, error) { claims, err := adt.AsMap(s.store, s.Claims) if err != nil { diff --git a/chain/actors/builtin/reward/reward.go b/chain/actors/builtin/reward/reward.go index 52a26ab15..66df887fc 100644 --- a/chain/actors/builtin/reward/reward.go +++ b/chain/actors/builtin/reward/reward.go @@ -30,8 +30,15 @@ func Load(store adt.Store, act *types.Actor) (st State, err error) { type State interface { cbor.Er - RewardSmoothed() (builtin.FilterEstimate, error) - EffectiveBaselinePower() (abi.StoragePower, error) ThisEpochBaselinePower() (abi.StoragePower, error) + ThisEpochReward() (abi.StoragePower, error) + ThisEpochRewardSmoothed() (builtin.FilterEstimate, error) + + EffectiveBaselinePower() (abi.StoragePower, error) + EffectiveNetworkTime() (abi.ChainEpoch, error) + TotalStoragePowerReward() (abi.TokenAmount, error) + + CumsumBaseline() (abi.StoragePower, error) + CumsumRealized() (abi.StoragePower, error) } diff --git a/chain/actors/builtin/reward/v0.go b/chain/actors/builtin/reward/v0.go index 16ac2b071..d12eccf59 100644 --- a/chain/actors/builtin/reward/v0.go +++ b/chain/actors/builtin/reward/v0.go @@ -12,10 +12,18 @@ type v0State struct { store adt.Store } -func (s *v0State) RewardSmoothed() (builtin.FilterEstimate, error) { +func (s *v0State) ThisEpochReward() (abi.StoragePower, error) { + return s.State.ThisEpochReward, nil +} + +func (s *v0State) ThisEpochRewardSmoothed() (builtin.FilterEstimate, error) { return *s.State.ThisEpochRewardSmoothed, nil } +func (s *v0State) ThisEpochBaselinePower() (abi.StoragePower, error) { + return s.State.ThisEpochBaselinePower, nil +} + func (s *v0State) TotalStoragePowerReward() (abi.TokenAmount, error) { return s.State.TotalMined, nil } @@ -24,6 +32,14 @@ func (s *v0State) EffectiveBaselinePower() (abi.StoragePower, error) { return s.State.EffectiveBaselinePower, nil } -func (s *v0State) ThisEpochBaselinePower() (abi.StoragePower, error) { - return s.State.ThisEpochBaselinePower, nil +func (s *v0State) EffectiveNetworkTime() (abi.ChainEpoch, error) { + return s.State.EffectiveNetworkTime, nil +} + +func (s *v0State) CumsumBaseline() (abi.StoragePower, error) { + return s.State.CumsumBaseline, nil +} + +func (s *v0State) CumsumRealized() (abi.StoragePower, error) { + return s.State.CumsumBaseline, nil } diff --git a/cmd/lotus-chainwatch/processor/common_actors.go b/cmd/lotus-chainwatch/processor/common_actors.go index d6aec7f90..56520880c 100644 --- a/cmd/lotus-chainwatch/processor/common_actors.go +++ b/cmd/lotus-chainwatch/processor/common_actors.go @@ -1,7 +1,6 @@ package processor import ( - "bytes" "context" "time" @@ -9,14 +8,13 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + _init "github.com/filecoin-project/lotus/chain/actors/builtin/init" "github.com/filecoin-project/lotus/chain/events/state" "github.com/filecoin-project/lotus/chain/types" cw_util "github.com/filecoin-project/lotus/cmd/lotus-chainwatch/util" "github.com/filecoin-project/specs-actors/actors/builtin" - _init "github.com/filecoin-project/specs-actors/actors/builtin/init" - "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/ipfs/go-cid" - typegen "github.com/whyrusleeping/cbor-gen" ) func (p *Processor) setupCommonActors() error { @@ -150,32 +148,17 @@ func (p Processor) storeActorAddresses(ctx context.Context, actors map[cid.Cid]A return err } - initActorRaw, err := p.node.ChainReadObj(ctx, initActor.Head) - if err != nil { - return err - } - - var initActorState _init.State - if err := initActorState.UnmarshalCBOR(bytes.NewReader(initActorRaw)); err != nil { - return err - } - ctxStore := cw_util.NewAPIIpldStore(ctx, p.node) - addrMap, err := adt.AsMap(ctxStore, initActorState.AddressMap) + initActorState, err := _init.Load(cw_util.NewAPIIpldStore(ctx, p.node), initActor) if err != nil { return err } // gross.. - var actorID typegen.CborInt - if err := addrMap.ForEach(&actorID, func(key string) error { - longAddr, err := address.NewFromBytes([]byte(key)) + if err := initActorState.ForEachActor(func(id abi.ActorID, addr address.Address) error { + idAddr, err := address.NewIDAddress(uint64(id)) if err != nil { return err } - shortAddr, err := address.NewIDAddress(uint64(actorID)) - if err != nil { - return err - } - addressToID[longAddr] = shortAddr + addressToID[addr] = idAddr return nil }); err != nil { return err diff --git a/cmd/lotus-chainwatch/processor/miner.go b/cmd/lotus-chainwatch/processor/miner.go index 1fd2a119f..17eef3afa 100644 --- a/cmd/lotus-chainwatch/processor/miner.go +++ b/cmd/lotus-chainwatch/processor/miner.go @@ -1,9 +1,7 @@ package processor import ( - "bytes" "context" - "fmt" "strings" "time" @@ -15,13 +13,11 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" - "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/builtin/power" - "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/apibstore" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + "github.com/filecoin-project/lotus/chain/actors/builtin/power" "github.com/filecoin-project/lotus/chain/events/state" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" @@ -212,7 +208,7 @@ func (p *Processor) processMiners(ctx context.Context, minerTips map[types.TipSe // TODO add parallel calls if this becomes slow for tipset, miners := range minerTips { // get the power actors claims map - minersClaims, err := getPowerActorClaimsMap(ctx, p.node, tipset) + powerState, err := getPowerActorState(ctx, p.node, tipset) if err != nil { return nil, err } @@ -222,10 +218,9 @@ func (p *Processor) processMiners(ctx context.Context, minerTips map[types.TipSe var mi minerActorInfo mi.common = act - var claim power.Claim // get miner claim from power actors claim map and store if found, else the miner had no claim at // this tipset - found, err := minersClaims.Get(abi.AddrKey(act.addr), &claim) + claim, found, err := powerState.MinerPower(act.addr) if err != nil { return nil, err } @@ -1027,22 +1022,10 @@ func (p *Processor) storeMinersPower(miners []minerActorInfo) error { } // load the power actor state clam as an adt.Map at the tipset `ts`. -func getPowerActorClaimsMap(ctx context.Context, api api.FullNode, ts types.TipSetKey) (*adt.Map, error) { - powerActor, err := api.StateGetActor(ctx, builtin.StoragePowerActorAddr, ts) +func getPowerActorState(ctx context.Context, api api.FullNode, ts types.TipSetKey) (power.State, error) { + powerActor, err := api.StateGetActor(ctx, power.Address, ts) if err != nil { return nil, err } - - powerRaw, err := api.ChainReadObj(ctx, powerActor.Head) - if err != nil { - return nil, err - } - - var powerActorState power.State - if err := powerActorState.UnmarshalCBOR(bytes.NewReader(powerRaw)); err != nil { - return nil, fmt.Errorf("failed to unmarshal power actor state: %w", err) - } - - s := cw_util.NewAPIIpldStore(ctx, api) - return adt.AsMap(s, powerActorState.Claims) + return power.Load(cw_util.NewAPIIpldStore(ctx, api), powerActor) } diff --git a/cmd/lotus-chainwatch/processor/power.go b/cmd/lotus-chainwatch/processor/power.go index dfd7eddd7..403580e3b 100644 --- a/cmd/lotus-chainwatch/processor/power.go +++ b/cmd/lotus-chainwatch/processor/power.go @@ -1,16 +1,14 @@ package processor import ( - "bytes" "context" "time" "golang.org/x/xerrors" "github.com/filecoin-project/go-state-types/big" - "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/builtin/power" - "github.com/filecoin-project/specs-actors/actors/util/smoothing" + + "github.com/filecoin-project/lotus/chain/actors/builtin" ) type powerActorInfo struct { @@ -22,10 +20,7 @@ type powerActorInfo struct { totalQualityAdjustedBytesCommitted big.Int totalPledgeCollateral big.Int - newRawBytes big.Int - newQualityAdjustedBytes big.Int - newPledgeCollateral big.Int - newQAPowerSmoothed *smoothing.FilterEstimate + qaPowerSmoothed builtin.FilterEstimate minerCount int64 minerCountAboveMinimumPower int64 @@ -44,10 +39,6 @@ create table if not exists chain_power constraint power_smoothing_estimates_pk primary key, - new_raw_bytes_power text not null, - new_qa_bytes_power text not null, - new_pledge_collateral text not null, - total_raw_bytes_power text not null, total_raw_bytes_committed text not null, total_qa_bytes_power text not null, @@ -92,35 +83,49 @@ func (p *Processor) processPowerActors(ctx context.Context, powerTips ActorTips) var pw powerActorInfo pw.common = act - powerActor, err := p.node.StateGetActor(ctx, builtin.StoragePowerActorAddr, tipset) + powerActorState, err := getPowerActorState(ctx, p.node, tipset) if err != nil { return nil, xerrors.Errorf("get power state (@ %s): %w", pw.common.stateroot.String(), err) } - powerStateRaw, err := p.node.ChainReadObj(ctx, powerActor.Head) - if err != nil { - return nil, xerrors.Errorf("read state obj (@ %s): %w", pw.common.stateroot.String(), err) + if totalPower, err := powerActorState.TotalPower(); err != nil { + return nil, xerrors.Errorf("failed to compute total power: %w", err) + } else { + pw.totalRawBytes = totalPower.RawBytePower + pw.totalQualityAdjustedBytes = totalPower.QualityAdjPower } - var powerActorState power.State - if err := powerActorState.UnmarshalCBOR(bytes.NewReader(powerStateRaw)); err != nil { - return nil, xerrors.Errorf("unmarshal state (@ %s): %w", pw.common.stateroot.String(), err) + if totalCommitted, err := powerActorState.TotalCommitted(); err != nil { + return nil, xerrors.Errorf("failed to compute total committed: %w", err) + } else { + pw.totalRawBytesCommitted = totalCommitted.RawBytePower + pw.totalQualityAdjustedBytesCommitted = totalCommitted.QualityAdjPower } - pw.totalRawBytes = powerActorState.TotalRawBytePower - pw.totalRawBytesCommitted = powerActorState.TotalBytesCommitted - pw.totalQualityAdjustedBytes = powerActorState.TotalQualityAdjPower - pw.totalQualityAdjustedBytesCommitted = powerActorState.TotalQABytesCommitted - pw.totalPledgeCollateral = powerActorState.TotalPledgeCollateral + if totalLocked, err := powerActorState.TotalLocked(); err != nil { + return nil, xerrors.Errorf("failed to compute total locked: %w", err) + } else { + pw.totalPledgeCollateral = totalLocked + } - pw.newRawBytes = powerActorState.ThisEpochRawBytePower - pw.newQualityAdjustedBytes = powerActorState.ThisEpochQualityAdjPower - pw.newPledgeCollateral = powerActorState.ThisEpochPledgeCollateral - pw.newQAPowerSmoothed = powerActorState.ThisEpochQAPowerSmoothed + if powerSmoothed, err := powerActorState.TotalPowerSmoothed(); err != nil { + return nil, xerrors.Errorf("failed to determine smoothed power: %w", err) + } else { + pw.qaPowerSmoothed = powerSmoothed + } - pw.minerCount = powerActorState.MinerCount - pw.minerCountAboveMinimumPower = powerActorState.MinerAboveMinPowerCount - out = append(out, pw) + // NOTE: this doesn't set new* fields. Previously, we + // filled these using ThisEpoch* fields from the actor + // state, but these fields are effectively internal + // state and don't represent "new" power, as was + // assumed. + + if participating, total, err := powerActorState.MinerCounts(); err != nil { + return nil, xerrors.Errorf("failed to count miners: %w", err) + } else { + pw.minerCountAboveMinimumPower = int64(participating) + pw.minerCount = int64(total) + } } } @@ -142,7 +147,7 @@ func (p *Processor) storePowerSmoothingEstimates(powerStates []powerActorInfo) e return xerrors.Errorf("prep chain_power: %w", err) } - stmt, err := tx.Prepare(`copy cp (state_root, new_raw_bytes_power, new_qa_bytes_power, new_pledge_collateral, total_raw_bytes_power, total_raw_bytes_committed, total_qa_bytes_power, total_qa_bytes_committed, total_pledge_collateral, qa_smoothed_position_estimate, qa_smoothed_velocity_estimate, miner_count, minimum_consensus_miner_count) from stdin;`) + stmt, err := tx.Prepare(`copy cp (state_root, total_raw_bytes_power, total_raw_bytes_committed, total_qa_bytes_power, total_qa_bytes_committed, total_pledge_collateral, qa_smoothed_position_estimate, qa_smoothed_velocity_estimate, miner_count, minimum_consensus_miner_count) from stdin;`) if err != nil { return xerrors.Errorf("prepare tmp chain_power: %w", err) } @@ -150,9 +155,6 @@ func (p *Processor) storePowerSmoothingEstimates(powerStates []powerActorInfo) e for _, ps := range powerStates { if _, err := stmt.Exec( ps.common.stateroot.String(), - ps.newRawBytes.String(), - ps.newQualityAdjustedBytes.String(), - ps.newPledgeCollateral.String(), ps.totalRawBytes.String(), ps.totalRawBytesCommitted.String(), @@ -160,8 +162,8 @@ func (p *Processor) storePowerSmoothingEstimates(powerStates []powerActorInfo) e ps.totalQualityAdjustedBytesCommitted.String(), ps.totalPledgeCollateral.String(), - ps.newQAPowerSmoothed.PositionEstimate.String(), - ps.newQAPowerSmoothed.VelocityEstimate.String(), + ps.qaPowerSmoothed.PositionEstimate.String(), + ps.qaPowerSmoothed.VelocityEstimate.String(), ps.minerCount, ps.minerCountAboveMinimumPower, diff --git a/cmd/lotus-chainwatch/processor/reward.go b/cmd/lotus-chainwatch/processor/reward.go index 5bdb478df..35d4bbf01 100644 --- a/cmd/lotus-chainwatch/processor/reward.go +++ b/cmd/lotus-chainwatch/processor/reward.go @@ -1,18 +1,18 @@ package processor import ( - "bytes" "context" "time" "golang.org/x/xerrors" "github.com/filecoin-project/go-state-types/big" - "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/builtin/reward" - "github.com/filecoin-project/specs-actors/actors/util/smoothing" + "github.com/filecoin-project/lotus/chain/actors/builtin" + "github.com/filecoin-project/lotus/chain/actors/builtin/reward" "github.com/filecoin-project/lotus/chain/types" + + cw_util "github.com/filecoin-project/lotus/cmd/lotus-chainwatch/util" ) type rewardActorInfo struct { @@ -24,13 +24,66 @@ type rewardActorInfo struct { effectiveNetworkTime int64 effectiveBaselinePower big.Int + // NOTE: These variables are wrong. Talk to @ZX about fixing. These _do + // not_ represent "new" anything. newBaselinePower big.Int newBaseReward big.Int - newSmoothingEstimate *smoothing.FilterEstimate + newSmoothingEstimate builtin.FilterEstimate totalMinedReward big.Int } +func (rw *rewardActorInfo) set(s reward.State) error { + if p, err := s.CumsumBaseline(); err != nil { + return xerrors.Errorf("getting cumsum baseline power (@ %s): %w", rw.common.stateroot.String(), err) + } else { + rw.cumSumBaselinePower = p + } + + if p, err := s.CumsumRealized(); err != nil { + return xerrors.Errorf("getting cumsum realized power (@ %s): %w", rw.common.stateroot.String(), err) + } else { + rw.cumSumRealizedPower = p + } + + if t, err := s.EffectiveNetworkTime(); err != nil { + return xerrors.Errorf("getting effective network time (@ %s): %w", rw.common.stateroot.String(), err) + } else { + rw.effectiveNetworkTime = int64(t) + } + + if p, err := s.EffectiveBaselinePower(); err != nil { + return xerrors.Errorf("getting effective baseline power (@ %s): %w", rw.common.stateroot.String(), err) + } else { + rw.effectiveBaselinePower = p + } + + if t, err := s.TotalStoragePowerReward(); err != nil { + return xerrors.Errorf("getting total mined (@ %s): %w", rw.common.stateroot.String(), err) + } else { + rw.totalMinedReward = t + } + + if p, err := s.ThisEpochBaselinePower(); err != nil { + return xerrors.Errorf("getting this epoch baseline power (@ %s): %w", rw.common.stateroot.String(), err) + } else { + rw.newBaselinePower = p + } + + if t, err := s.ThisEpochReward(); err != nil { + return xerrors.Errorf("getting this epoch baseline power (@ %s): %w", rw.common.stateroot.String(), err) + } else { + rw.newBaseReward = t + } + + if e, err := s.ThisEpochRewardSmoothed(); err != nil { + return xerrors.Errorf("getting this epoch baseline power (@ %s): %w", rw.common.stateroot.String(), err) + } else { + rw.newSmoothingEstimate = e + } + return nil +} + func (p *Processor) setupRewards() error { tx, err := p.db.Begin() if err != nil { @@ -89,29 +142,19 @@ func (p *Processor) processRewardActors(ctx context.Context, rewardTips ActorTip rw.common = act // get reward actor states at each tipset once for all updates - rewardActor, err := p.node.StateGetActor(ctx, builtin.RewardActorAddr, tipset) + rewardActor, err := p.node.StateGetActor(ctx, reward.Address, tipset) if err != nil { return nil, xerrors.Errorf("get reward state (@ %s): %w", rw.common.stateroot.String(), err) } - rewardStateRaw, err := p.node.ChainReadObj(ctx, rewardActor.Head) + rewardActorState, err := reward.Load(cw_util.NewAPIIpldStore(ctx, p.node), rewardActor) if err != nil { return nil, xerrors.Errorf("read state obj (@ %s): %w", rw.common.stateroot.String(), err) } - - var rewardActorState reward.State - if err := rewardActorState.UnmarshalCBOR(bytes.NewReader(rewardStateRaw)); err != nil { - return nil, xerrors.Errorf("unmarshal state (@ %s): %w", rw.common.stateroot.String(), err) + if err := rw.set(rewardActorState); err != nil { + return nil, err } - rw.cumSumBaselinePower = rewardActorState.CumsumBaseline - rw.cumSumRealizedPower = rewardActorState.CumsumRealized - rw.effectiveNetworkTime = int64(rewardActorState.EffectiveNetworkTime) - rw.effectiveBaselinePower = rewardActorState.EffectiveBaselinePower - rw.newBaselinePower = rewardActorState.ThisEpochBaselinePower - rw.newBaseReward = rewardActorState.ThisEpochReward - rw.newSmoothingEstimate = rewardActorState.ThisEpochRewardSmoothed - rw.totalMinedReward = rewardActorState.TotalMined out = append(out, rw) } } @@ -126,29 +169,19 @@ func (p *Processor) processRewardActors(ctx context.Context, rewardTips ActorTip rw.common.stateroot = tipset.ParentState() rw.common.parentTsKey = tipset.Parents() // get reward actor states at each tipset once for all updates - rewardActor, err := p.node.StateGetActor(ctx, builtin.RewardActorAddr, tsKey) + rewardActor, err := p.node.StateGetActor(ctx, reward.Address, tsKey) if err != nil { return nil, err } - rewardStateRaw, err := p.node.ChainReadObj(ctx, rewardActor.Head) + rewardActorState, err := reward.Load(cw_util.NewAPIIpldStore(ctx, p.node), rewardActor) if err != nil { - return nil, err + return nil, xerrors.Errorf("read state obj (@ %s): %w", rw.common.stateroot.String(), err) } - var rewardActorState reward.State - if err := rewardActorState.UnmarshalCBOR(bytes.NewReader(rewardStateRaw)); err != nil { + if err := rw.set(rewardActorState); err != nil { return nil, err } - - rw.cumSumBaselinePower = rewardActorState.CumsumBaseline - rw.cumSumRealizedPower = rewardActorState.CumsumRealized - rw.effectiveNetworkTime = int64(rewardActorState.EffectiveNetworkTime) - rw.effectiveBaselinePower = rewardActorState.EffectiveBaselinePower - rw.newBaselinePower = rewardActorState.ThisEpochBaselinePower - rw.newBaseReward = rewardActorState.ThisEpochReward - rw.newSmoothingEstimate = rewardActorState.ThisEpochRewardSmoothed - rw.totalMinedReward = rewardActorState.TotalMined out = append(out, rw) } diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 05fcc5171..3bed0dfbb 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -940,7 +940,7 @@ func (a *StateAPI) StateMinerPreCommitDepositForPower(ctx context.Context, maddr return types.EmptyInt, xerrors.Errorf("loading miner actor: %w", err) } else if s, err := reward.Load(store, act); err != nil { return types.EmptyInt, xerrors.Errorf("loading reward actor state: %w", err) - } else if r, err := s.RewardSmoothed(); err != nil { + } else if r, err := s.ThisEpochRewardSmoothed(); err != nil { return types.EmptyInt, xerrors.Errorf("failed to determine total reward: %w", err) } else { rewardSmoothed = r @@ -1011,7 +1011,7 @@ func (a *StateAPI) StateMinerInitialPledgeCollateral(ctx context.Context, maddr return types.EmptyInt, xerrors.Errorf("loading miner actor: %w", err) } else if s, err := reward.Load(store, act); err != nil { return types.EmptyInt, xerrors.Errorf("loading reward actor state: %w", err) - } else if r, err := s.RewardSmoothed(); err != nil { + } else if r, err := s.ThisEpochRewardSmoothed(); err != nil { return types.EmptyInt, xerrors.Errorf("failed to determine total reward: %w", err) } else if p, err := s.ThisEpochBaselinePower(); err != nil { return types.EmptyInt, xerrors.Errorf("failed to determine baseline power: %w", err) From 298efa221cf8908e7aefb2476b997420e18c554a Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 18 Sep 2020 13:52:07 -0700 Subject: [PATCH 224/885] fix lint errors --- cmd/lotus-chainwatch/processor/power.go | 37 ++++++++++--------- cmd/lotus-chainwatch/processor/reward.go | 47 ++++++++++-------------- 2 files changed, 39 insertions(+), 45 deletions(-) diff --git a/cmd/lotus-chainwatch/processor/power.go b/cmd/lotus-chainwatch/processor/power.go index 403580e3b..726a46706 100644 --- a/cmd/lotus-chainwatch/processor/power.go +++ b/cmd/lotus-chainwatch/processor/power.go @@ -88,30 +88,24 @@ func (p *Processor) processPowerActors(ctx context.Context, powerTips ActorTips) return nil, xerrors.Errorf("get power state (@ %s): %w", pw.common.stateroot.String(), err) } - if totalPower, err := powerActorState.TotalPower(); err != nil { + totalPower, err := powerActorState.TotalPower() + if err != nil { return nil, xerrors.Errorf("failed to compute total power: %w", err) - } else { - pw.totalRawBytes = totalPower.RawBytePower - pw.totalQualityAdjustedBytes = totalPower.QualityAdjPower } - if totalCommitted, err := powerActorState.TotalCommitted(); err != nil { + totalCommitted, err := powerActorState.TotalCommitted() + if err != nil { return nil, xerrors.Errorf("failed to compute total committed: %w", err) - } else { - pw.totalRawBytesCommitted = totalCommitted.RawBytePower - pw.totalQualityAdjustedBytesCommitted = totalCommitted.QualityAdjPower } - if totalLocked, err := powerActorState.TotalLocked(); err != nil { + totalLocked, err := powerActorState.TotalLocked() + if err != nil { return nil, xerrors.Errorf("failed to compute total locked: %w", err) - } else { - pw.totalPledgeCollateral = totalLocked } - if powerSmoothed, err := powerActorState.TotalPowerSmoothed(); err != nil { + powerSmoothed, err := powerActorState.TotalPowerSmoothed() + if err != nil { return nil, xerrors.Errorf("failed to determine smoothed power: %w", err) - } else { - pw.qaPowerSmoothed = powerSmoothed } // NOTE: this doesn't set new* fields. Previously, we @@ -120,12 +114,19 @@ func (p *Processor) processPowerActors(ctx context.Context, powerTips ActorTips) // state and don't represent "new" power, as was // assumed. - if participating, total, err := powerActorState.MinerCounts(); err != nil { + participatingMiners, totalMiners, err := powerActorState.MinerCounts() + if err != nil { return nil, xerrors.Errorf("failed to count miners: %w", err) - } else { - pw.minerCountAboveMinimumPower = int64(participating) - pw.minerCount = int64(total) } + + pw.totalRawBytes = totalPower.RawBytePower + pw.totalQualityAdjustedBytes = totalPower.QualityAdjPower + pw.totalRawBytesCommitted = totalCommitted.RawBytePower + pw.totalQualityAdjustedBytesCommitted = totalCommitted.QualityAdjPower + pw.totalPledgeCollateral = totalLocked + pw.qaPowerSmoothed = powerSmoothed + pw.minerCountAboveMinimumPower = int64(participatingMiners) + pw.minerCount = int64(totalMiners) } } diff --git a/cmd/lotus-chainwatch/processor/reward.go b/cmd/lotus-chainwatch/processor/reward.go index 35d4bbf01..72a329c87 100644 --- a/cmd/lotus-chainwatch/processor/reward.go +++ b/cmd/lotus-chainwatch/processor/reward.go @@ -6,6 +6,7 @@ import ( "golang.org/x/xerrors" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/lotus/chain/actors/builtin" @@ -21,7 +22,7 @@ type rewardActorInfo struct { cumSumBaselinePower big.Int cumSumRealizedPower big.Int - effectiveNetworkTime int64 + effectiveNetworkTime abi.ChainEpoch effectiveBaselinePower big.Int // NOTE: These variables are wrong. Talk to @ZX about fixing. These _do @@ -33,53 +34,45 @@ type rewardActorInfo struct { totalMinedReward big.Int } -func (rw *rewardActorInfo) set(s reward.State) error { - if p, err := s.CumsumBaseline(); err != nil { +func (rw *rewardActorInfo) set(s reward.State) (err error) { + rw.cumSumBaselinePower, err = s.CumsumBaseline() + if err != nil { return xerrors.Errorf("getting cumsum baseline power (@ %s): %w", rw.common.stateroot.String(), err) - } else { - rw.cumSumBaselinePower = p } - if p, err := s.CumsumRealized(); err != nil { + rw.cumSumRealizedPower, err = s.CumsumRealized() + if err != nil { return xerrors.Errorf("getting cumsum realized power (@ %s): %w", rw.common.stateroot.String(), err) - } else { - rw.cumSumRealizedPower = p } - if t, err := s.EffectiveNetworkTime(); err != nil { + rw.effectiveNetworkTime, err = s.EffectiveNetworkTime() + if err != nil { return xerrors.Errorf("getting effective network time (@ %s): %w", rw.common.stateroot.String(), err) - } else { - rw.effectiveNetworkTime = int64(t) } - if p, err := s.EffectiveBaselinePower(); err != nil { + rw.effectiveBaselinePower, err = s.EffectiveBaselinePower() + if err != nil { return xerrors.Errorf("getting effective baseline power (@ %s): %w", rw.common.stateroot.String(), err) - } else { - rw.effectiveBaselinePower = p } - if t, err := s.TotalStoragePowerReward(); err != nil { + rw.totalMinedReward, err = s.TotalStoragePowerReward() + if err != nil { return xerrors.Errorf("getting total mined (@ %s): %w", rw.common.stateroot.String(), err) - } else { - rw.totalMinedReward = t } - if p, err := s.ThisEpochBaselinePower(); err != nil { + rw.newBaselinePower, err = s.ThisEpochBaselinePower() + if err != nil { return xerrors.Errorf("getting this epoch baseline power (@ %s): %w", rw.common.stateroot.String(), err) - } else { - rw.newBaselinePower = p } - if t, err := s.ThisEpochReward(); err != nil { + rw.newBaseReward, err = s.ThisEpochReward() + if err != nil { return xerrors.Errorf("getting this epoch baseline power (@ %s): %w", rw.common.stateroot.String(), err) - } else { - rw.newBaseReward = t } - if e, err := s.ThisEpochRewardSmoothed(); err != nil { + rw.newSmoothingEstimate, err = s.ThisEpochRewardSmoothed() + if err != nil { return xerrors.Errorf("getting this epoch baseline power (@ %s): %w", rw.common.stateroot.String(), err) - } else { - rw.newSmoothingEstimate = e } return nil } @@ -213,7 +206,7 @@ func (p *Processor) persistRewardActors(ctx context.Context, rewards []rewardAct rewardState.common.stateroot.String(), rewardState.cumSumBaselinePower.String(), rewardState.cumSumRealizedPower.String(), - rewardState.effectiveNetworkTime, + uint64(rewardState.effectiveNetworkTime), rewardState.effectiveBaselinePower.String(), rewardState.newBaselinePower.String(), rewardState.newBaseReward.String(), From 8747c6083e6ecc1c60620c8dcf67bb07a304d8d4 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 18 Sep 2020 14:20:53 -0700 Subject: [PATCH 225/885] abstract over account actor --- chain/actors/builtin/account/account.go | 31 +++++++++++++++++++++++++ chain/actors/builtin/account/v0.go | 16 +++++++++++++ chain/vm/vm.go | 13 ++++------- 3 files changed, 52 insertions(+), 8 deletions(-) create mode 100644 chain/actors/builtin/account/account.go create mode 100644 chain/actors/builtin/account/v0.go diff --git a/chain/actors/builtin/account/account.go b/chain/actors/builtin/account/account.go new file mode 100644 index 000000000..d4cbe4177 --- /dev/null +++ b/chain/actors/builtin/account/account.go @@ -0,0 +1,31 @@ +package account + +import ( + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/cbor" + v0builtin "github.com/filecoin-project/specs-actors/actors/builtin" + + "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/chain/types" +) + +func Load(store adt.Store, act *types.Actor) (State, error) { + switch act.Code { + case v0builtin.AccountActorCodeID: + out := v0State{store: store} + err := store.Get(store.Context(), act.Head, &out) + if err != nil { + return nil, err + } + return &out, nil + } + return nil, xerrors.Errorf("unknown actor code %s", act.Code) +} + +type State interface { + cbor.Marshaler + + PubkeyAddress() (address.Address, error) +} diff --git a/chain/actors/builtin/account/v0.go b/chain/actors/builtin/account/v0.go new file mode 100644 index 000000000..6001dd7bf --- /dev/null +++ b/chain/actors/builtin/account/v0.go @@ -0,0 +1,16 @@ +package account + +import ( + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/specs-actors/actors/builtin/account" +) + +type v0State struct { + account.State + store adt.Store +} + +func (s *v0State) PubkeyAddress() (address.Address, error) { + return s.Address, nil +} diff --git a/chain/vm/vm.go b/chain/vm/vm.go index 542dfee69..25c937ca3 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -23,10 +23,11 @@ import ( "github.com/filecoin-project/go-state-types/exitcode" "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/builtin/account" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/actors/aerrors" + "github.com/filecoin-project/lotus/chain/actors/builtin/account" "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/blockstore" @@ -49,16 +50,12 @@ func ResolveToKeyAddr(state types.StateTree, cst cbor.IpldStore, addr address.Ad return address.Undef, xerrors.Errorf("failed to find actor: %s", addr) } - if act.Code != builtin.AccountActorCodeID { - return address.Undef, xerrors.Errorf("address %s was not for an account actor", addr) - } - - var aast account.State - if err := cst.Get(context.TODO(), act.Head, &aast); err != nil { + aast, err := account.Load(adt.WrapStore(context.TODO(), cst), act) + if err != nil { return address.Undef, xerrors.Errorf("failed to get account actor state for %s: %w", addr, err) } - return aast.Address, nil + return aast.PubkeyAddress() } var _ cbor.IpldBlockstore = (*gasChargingBlocks)(nil) From 24df873498f0af9ec38672a7c6c9024ec6546b5f Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 18 Sep 2020 14:21:05 -0700 Subject: [PATCH 226/885] rename imports --- chain/vm/invoker.go | 54 ++++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/chain/vm/invoker.go b/chain/vm/invoker.go index b31b45767..501b933e2 100644 --- a/chain/vm/invoker.go +++ b/chain/vm/invoker.go @@ -6,28 +6,28 @@ import ( "fmt" "reflect" - "github.com/filecoin-project/go-state-types/exitcode" - "github.com/filecoin-project/specs-actors/actors/builtin/account" - v0verifreg "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" - "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" - "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/chain/actors/aerrors" - "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/builtin/cron" - init_ "github.com/filecoin-project/specs-actors/actors/builtin/init" - "github.com/filecoin-project/specs-actors/actors/builtin/market" + v0builtin "github.com/filecoin-project/specs-actors/actors/builtin" + v0account "github.com/filecoin-project/specs-actors/actors/builtin/account" + v0cron "github.com/filecoin-project/specs-actors/actors/builtin/cron" + v0init "github.com/filecoin-project/specs-actors/actors/builtin/init" + v0market "github.com/filecoin-project/specs-actors/actors/builtin/market" v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" v0msig "github.com/filecoin-project/specs-actors/actors/builtin/multisig" - "github.com/filecoin-project/specs-actors/actors/builtin/paych" + v0paych "github.com/filecoin-project/specs-actors/actors/builtin/paych" v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" v0reward "github.com/filecoin-project/specs-actors/actors/builtin/reward" - "github.com/filecoin-project/specs-actors/actors/builtin/system" - "github.com/filecoin-project/specs-actors/actors/runtime" + v0system "github.com/filecoin-project/specs-actors/actors/builtin/system" + v0verifreg "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" vmr "github.com/filecoin-project/specs-actors/actors/runtime" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/exitcode" + + "github.com/filecoin-project/lotus/chain/actors/aerrors" ) type Invoker struct { @@ -35,7 +35,7 @@ type Invoker struct { builtInState map[cid.Cid]reflect.Type } -type invokeFunc func(rt runtime.Runtime, params []byte) ([]byte, aerrors.ActorError) +type invokeFunc func(rt vmr.Runtime, params []byte) ([]byte, aerrors.ActorError) type nativeCode []invokeFunc func NewInvoker() *Invoker { @@ -46,22 +46,22 @@ func NewInvoker() *Invoker { // add builtInCode using: register(cid, singleton) // NETUPGRADE: register code IDs for v2, etc. - inv.Register(builtin.SystemActorCodeID, system.Actor{}, abi.EmptyValue{}) - inv.Register(builtin.InitActorCodeID, init_.Actor{}, init_.State{}) - inv.Register(builtin.RewardActorCodeID, v0reward.Actor{}, v0reward.State{}) - inv.Register(builtin.CronActorCodeID, cron.Actor{}, cron.State{}) - inv.Register(builtin.StoragePowerActorCodeID, v0power.Actor{}, v0power.State{}) - inv.Register(builtin.StorageMarketActorCodeID, market.Actor{}, market.State{}) - inv.Register(builtin.StorageMinerActorCodeID, v0miner.Actor{}, v0miner.State{}) - inv.Register(builtin.MultisigActorCodeID, v0msig.Actor{}, v0msig.State{}) - inv.Register(builtin.PaymentChannelActorCodeID, paych.Actor{}, paych.State{}) - inv.Register(builtin.VerifiedRegistryActorCodeID, v0verifreg.Actor{}, v0verifreg.State{}) - inv.Register(builtin.AccountActorCodeID, account.Actor{}, account.State{}) + inv.Register(v0builtin.SystemActorCodeID, v0system.Actor{}, abi.EmptyValue{}) + inv.Register(v0builtin.InitActorCodeID, v0init.Actor{}, v0init.State{}) + inv.Register(v0builtin.RewardActorCodeID, v0reward.Actor{}, v0reward.State{}) + inv.Register(v0builtin.CronActorCodeID, v0cron.Actor{}, v0cron.State{}) + inv.Register(v0builtin.StoragePowerActorCodeID, v0power.Actor{}, v0power.State{}) + inv.Register(v0builtin.StorageMarketActorCodeID, v0market.Actor{}, v0market.State{}) + inv.Register(v0builtin.StorageMinerActorCodeID, v0miner.Actor{}, v0miner.State{}) + inv.Register(v0builtin.MultisigActorCodeID, v0msig.Actor{}, v0msig.State{}) + inv.Register(v0builtin.PaymentChannelActorCodeID, v0paych.Actor{}, v0paych.State{}) + inv.Register(v0builtin.VerifiedRegistryActorCodeID, v0verifreg.Actor{}, v0verifreg.State{}) + inv.Register(v0builtin.AccountActorCodeID, v0account.Actor{}, v0account.State{}) return inv } -func (inv *Invoker) Invoke(codeCid cid.Cid, rt runtime.Runtime, method abi.MethodNum, params []byte) ([]byte, aerrors.ActorError) { +func (inv *Invoker) Invoke(codeCid cid.Cid, rt vmr.Runtime, method abi.MethodNum, params []byte) ([]byte, aerrors.ActorError) { code, ok := inv.builtInCode[codeCid] if !ok { @@ -177,7 +177,7 @@ func DecodeParams(b []byte, out interface{}) error { } func DumpActorState(code cid.Cid, b []byte) (interface{}, error) { - if code == builtin.AccountActorCodeID { // Account code special case + if code == v0builtin.AccountActorCodeID { // Account code special case return nil, nil } From e60027c00af38a2dee6b3de14c4f28a582b21069 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 18 Sep 2020 14:22:38 -0700 Subject: [PATCH 227/885] remove todo --- chain/stmgr/stmgr.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index fee76369a..d183b7ce2 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -1183,9 +1183,7 @@ func (sm *StateManager) GetMarketState(ctx context.Context, ts *types.TipSet) (m return nil, err } - // TODO maybe there needs to be code here to differentiate address based on ts height? - addr := builtin.StorageMarketActorAddr - act, err := st.GetActor(addr) + act, err := st.GetActor(market.Address) if err != nil { return nil, err } From 8285eda8e507358f5a64080e3894573f6b5b131c Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 18 Sep 2020 14:46:42 -0700 Subject: [PATCH 228/885] migrate storage miner info --- chain/actors/builtin/miner/miner.go | 10 +++++++ chain/actors/builtin/miner/v0.go | 16 +++++++++++ cmd/lotus-storage-miner/info.go | 44 +++++++++++++---------------- 3 files changed, 45 insertions(+), 25 deletions(-) diff --git a/chain/actors/builtin/miner/miner.go b/chain/actors/builtin/miner/miner.go index 82dc002ca..c67e773bf 100644 --- a/chain/actors/builtin/miner/miner.go +++ b/chain/actors/builtin/miner/miner.go @@ -35,8 +35,12 @@ func Load(store adt.Store, act *types.Actor) (st State, err error) { type State interface { cbor.Marshaler + // Total available balance to spend. AvailableBalance(abi.TokenAmount) (abi.TokenAmount, error) + // Funds that will vest by the given epoch. VestedFunds(abi.ChainEpoch) (abi.TokenAmount, error) + // Funds locked for various reasons. + LockedFunds() (LockedFunds, error) GetSector(abi.SectorNumber) (*SectorOnChainInfo, error) FindSector(abi.SectorNumber) (*SectorLocation, error) @@ -138,3 +142,9 @@ type PreCommitChanges struct { Added []SectorPreCommitOnChainInfo Removed []SectorPreCommitOnChainInfo } + +type LockedFunds struct { + VestingFunds abi.TokenAmount + InitialPledgeRequirement abi.TokenAmount + PreCommitDeposits abi.TokenAmount +} diff --git a/chain/actors/builtin/miner/v0.go b/chain/actors/builtin/miner/v0.go index d333dff76..0f27d7d7f 100644 --- a/chain/actors/builtin/miner/v0.go +++ b/chain/actors/builtin/miner/v0.go @@ -40,6 +40,22 @@ func (s *v0State) VestedFunds(epoch abi.ChainEpoch) (abi.TokenAmount, error) { return s.CheckVestedFunds(s.store, epoch) } +func (s *v0State) LockedFunds() (LockedFunds, error) { + return LockedFunds{ + VestingFunds: s.State.LockedFunds, + InitialPledgeRequirement: s.State.InitialPledgeRequirement, + PreCommitDeposits: s.State.PreCommitDeposits, + }, nil +} + +func (s *v0State) InitialPledge() (abi.TokenAmount, error) { + return s.State.InitialPledgeRequirement, nil +} + +func (s *v0State) PreCommitDeposits() (abi.TokenAmount, error) { + return s.State.PreCommitDeposits, nil +} + func (s *v0State) GetSector(num abi.SectorNumber) (*SectorOnChainInfo, error) { info, ok, err := s.State.GetSector(s.store, num) if !ok || err != nil { diff --git a/cmd/lotus-storage-miner/info.go b/cmd/lotus-storage-miner/info.go index 3a7720c52..3ccfd67da 100644 --- a/cmd/lotus-storage-miner/info.go +++ b/cmd/lotus-storage-miner/info.go @@ -1,7 +1,6 @@ package main import ( - "bytes" "context" "fmt" "sort" @@ -16,12 +15,12 @@ import ( "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-state-types/abi" sealing "github.com/filecoin-project/lotus/extern/storage-sealing" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" - "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/apibstore" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/lib/blockstore" @@ -54,11 +53,6 @@ func infoCmdAct(cctx *cli.Context) error { ctx := lcli.ReqContext(cctx) - head, err := api.ChainHead(ctx) - if err != nil { - return xerrors.Errorf("getting chain head: %w", err) - } - maddr, err := getActorAddress(ctx, nodeApi, cctx.String("actor")) if err != nil { return err @@ -68,15 +62,11 @@ func infoCmdAct(cctx *cli.Context) error { if err != nil { return err } - var mas miner.State - { - rmas, err := api.ChainReadObj(ctx, mact.Head) - if err != nil { - return err - } - if err := mas.UnmarshalCBOR(bytes.NewReader(rmas)); err != nil { - return err - } + + tbs := bufbstore.NewTieredBstore(apibstore.NewAPIBlockstore(api), blockstore.NewTemporary()) + mas, err := miner.Load(adt.WrapStore(ctx, cbor.NewCborStore(tbs)), mact) + if err != nil { + return err } fmt.Printf("Miner: %s\n", color.BlueString("%s", maddr)) @@ -172,17 +162,21 @@ func infoCmdAct(cctx *cli.Context) error { fmt.Printf("\tActive: %d, %s (Verified: %d, %s)\n", nactiveDeals, types.SizeStr(types.NewInt(uint64(activeDealBytes))), nVerifDeals, types.SizeStr(types.NewInt(uint64(activeVerifDealBytes)))) fmt.Println() - tbs := bufbstore.NewTieredBstore(apibstore.NewAPIBlockstore(api), blockstore.NewTemporary()) - _, err = mas.UnlockVestedFunds(adt.WrapStore(ctx, cbor.NewCborStore(tbs)), head.Height()) + // NOTE: there's no need to unlock anything here. Funds only + // vest on deadline boundaries, and they're unlocked by cron. + lockedFunds, err := mas.LockedFunds() if err != nil { - return xerrors.Errorf("calculating vested funds: %w", err) + return xerrors.Errorf("getting locked funds: %w", err) + } + availBalance, err := mas.AvailableBalance(mact.Balance) + if err != nil { + return xerrors.Errorf("getting available balance: %w", err) } - fmt.Printf("Miner Balance: %s\n", color.YellowString("%s", types.FIL(mact.Balance))) - fmt.Printf("\tPreCommit: %s\n", types.FIL(mas.PreCommitDeposits)) - fmt.Printf("\tPledge: %s\n", types.FIL(mas.InitialPledgeRequirement)) - fmt.Printf("\tLocked: %s\n", types.FIL(mas.LockedFunds)) - color.Green("\tAvailable: %s", types.FIL(mas.GetAvailableBalance(mact.Balance))) + fmt.Printf("\tPreCommit: %s\n", types.FIL(lockedFunds.PreCommitDeposits)) + fmt.Printf("\tPledge: %s\n", types.FIL(lockedFunds.InitialPledgeRequirement)) + fmt.Printf("\tVesting: %s\n", types.FIL(lockedFunds.VestingFunds)) + color.Green("\tAvailable: %s", types.FIL(availBalance)) wb, err := api.WalletBalance(ctx, mi.Worker) if err != nil { return xerrors.Errorf("getting worker balance: %w", err) From 4a7055c32874a08f13ec850e63480dd30a30e66c Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 18 Sep 2020 14:46:55 -0700 Subject: [PATCH 229/885] fix state loading in vm --- chain/vm/syscalls.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/chain/vm/syscalls.go b/chain/vm/syscalls.go index aab1812d9..a7f5dab0c 100644 --- a/chain/vm/syscalls.go +++ b/chain/vm/syscalls.go @@ -7,8 +7,6 @@ import ( goruntime "runtime" "sync" - "github.com/filecoin-project/lotus/chain/actors/builtin/miner" - "github.com/filecoin-project/specs-actors/actors/runtime/proof" "github.com/filecoin-project/go-address" @@ -20,6 +18,8 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/crypto" + "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/sigs" @@ -192,8 +192,8 @@ func (ss *syscallShim) VerifyBlockSig(blk *types.BlockHeader) error { } // use that to get the miner state - var mas miner.State - if err = ss.cst.Get(ss.ctx, act.Head, &mas); err != nil { + mas, err := miner.Load(adt.WrapStore(ss.ctx, ss.cst), act) + if err != nil { return err } From f741ce6e3090fefe18d8d480a507b435b54d9fb5 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 18 Sep 2020 14:51:18 -0700 Subject: [PATCH 230/885] fixup some more imports --- cmd/lotus-storage-miner/actor.go | 11 ++++++----- cmd/lotus-storage-miner/init.go | 17 +++++++++-------- cmd/lotus-storage-miner/sectors.go | 3 +-- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/cmd/lotus-storage-miner/actor.go b/cmd/lotus-storage-miner/actor.go index c84e006d7..2493c5a1e 100644 --- a/cmd/lotus-storage-miner/actor.go +++ b/cmd/lotus-storage-miner/actor.go @@ -14,8 +14,9 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" + v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/types" @@ -87,7 +88,7 @@ var actorSetAddrsCmd = &cli.Command{ return err } - params, err := actors.SerializeParams(&miner.ChangeMultiaddrsParams{NewMultiaddrs: addrs}) + params, err := actors.SerializeParams(&v0miner.ChangeMultiaddrsParams{NewMultiaddrs: addrs}) if err != nil { return err } @@ -152,7 +153,7 @@ var actorSetPeeridCmd = &cli.Command{ return err } - params, err := actors.SerializeParams(&miner.ChangePeerIDParams{NewID: abi.PeerID(pid)}) + params, err := actors.SerializeParams(&v0miner.ChangePeerIDParams{NewID: abi.PeerID(pid)}) if err != nil { return err } @@ -225,7 +226,7 @@ var actorWithdrawCmd = &cli.Command{ } } - params, err := actors.SerializeParams(&miner.WithdrawBalanceParams{ + params, err := actors.SerializeParams(&v0miner.WithdrawBalanceParams{ AmountRequested: amount, // Default to attempting to withdraw all the extra funds in the miner actor }) if err != nil { @@ -450,7 +451,7 @@ var actorControlSet = &cli.Command{ return nil } - cwp := &miner.ChangeWorkerAddressParams{ + cwp := &v0miner.ChangeWorkerAddressParams{ NewWorker: mi.Worker, NewControlAddrs: toSet, } diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index 8325aae8b..715daead8 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -31,9 +31,10 @@ import ( sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/extern/sector-storage/stores" - "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/builtin/market" - miner2 "github.com/filecoin-project/specs-actors/actors/builtin/miner" + + v0builtin "github.com/filecoin-project/specs-actors/actors/builtin" + v0market "github.com/filecoin-project/specs-actors/actors/builtin/market" + v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" lapi "github.com/filecoin-project/lotus/api" @@ -372,7 +373,7 @@ func migratePreSealMeta(ctx context.Context, api lapi.FullNode, metadata string, return mds.Put(datastore.NewKey(modules.StorageCounterDSPrefix), buf[:size]) } -func findMarketDealID(ctx context.Context, api lapi.FullNode, deal market.DealProposal) (abi.DealID, error) { +func findMarketDealID(ctx context.Context, api lapi.FullNode, deal v0market.DealProposal) (abi.DealID, error) { // TODO: find a better way // (this is only used by genesis miners) @@ -566,7 +567,7 @@ func configureStorageMiner(ctx context.Context, api lapi.FullNode, addr address. return xerrors.Errorf("getWorkerAddr returned bad address: %w", err) } - enc, err := actors.SerializeParams(&miner2.ChangePeerIDParams{NewID: abi.PeerID(peerid)}) + enc, err := actors.SerializeParams(&v0miner.ChangePeerIDParams{NewID: abi.PeerID(peerid)}) if err != nil { return err } @@ -574,7 +575,7 @@ func configureStorageMiner(ctx context.Context, api lapi.FullNode, addr address. msg := &types.Message{ To: addr, From: mi.Worker, - Method: builtin.MethodsMiner.ChangePeerID, + Method: v0builtin.MethodsMiner.ChangePeerID, Params: enc, Value: types.NewInt(0), GasPremium: gasPrice, @@ -653,11 +654,11 @@ func createStorageMiner(ctx context.Context, api lapi.FullNode, peerid peer.ID, } createStorageMinerMsg := &types.Message{ - To: builtin.StoragePowerActorAddr, + To: v0builtin.StoragePowerActorAddr, From: sender, Value: big.Zero(), - Method: builtin.MethodsPower.CreateMiner, + Method: v0builtin.MethodsPower.CreateMiner, Params: params, GasLimit: 0, diff --git a/cmd/lotus-storage-miner/sectors.go b/cmd/lotus-storage-miner/sectors.go index 06f09fe20..032a68ac2 100644 --- a/cmd/lotus-storage-miner/sectors.go +++ b/cmd/lotus-storage-miner/sectors.go @@ -8,14 +8,13 @@ import ( "text/tabwriter" "time" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" - "github.com/urfave/cli/v2" "golang.org/x/xerrors" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" ) From c130806c374905ec8afa1c25d0201bd1c2c9fe48 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 18 Sep 2020 14:59:14 -0700 Subject: [PATCH 231/885] compile fix --- cmd/lotus-storage-miner/sectors.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/lotus-storage-miner/sectors.go b/cmd/lotus-storage-miner/sectors.go index 032a68ac2..5659bed79 100644 --- a/cmd/lotus-storage-miner/sectors.go +++ b/cmd/lotus-storage-miner/sectors.go @@ -13,6 +13,8 @@ import ( "github.com/filecoin-project/go-state-types/abi" + v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" @@ -378,7 +380,7 @@ var sectorsCapacityCollateralCmd = &cli.Command{ Expiration: abi.ChainEpoch(cctx.Uint64("expiration")), } if pci.Expiration == 0 { - pci.Expiration = miner.MaxSectorExpirationExtension + pci.Expiration = v0miner.MaxSectorExpirationExtension } pc, err := nApi.StateMinerInitialPledgeCollateral(ctx, maddr, pci, types.EmptyTSK) if err != nil { From 1bf3b4989dae6ddde14ed8ae5d81b15057901c0f Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 18 Sep 2020 14:59:27 -0700 Subject: [PATCH 232/885] rename imports to match actors code `sed -i 's/\bv0\(\w\)\(\w*\)/\L\1\E\20/g' **/*.go` --- build/params_2k.go | 12 +-- build/params_shared_funcs.go | 6 +- build/params_shared_vals.go | 4 +- build/params_testground.go | 4 +- build/params_testnet.go | 8 +- chain/actors/adt/adt.go | 8 +- chain/actors/adt/diff_adt_test.go | 10 +- chain/actors/builtin/account/account.go | 6 +- chain/actors/builtin/account/v0.go | 4 +- chain/actors/builtin/builtin.go | 4 +- chain/actors/builtin/init/init.go | 8 +- chain/actors/builtin/init/v0.go | 14 +-- chain/actors/builtin/market/market.go | 8 +- chain/actors/builtin/market/v0.go | 106 +++++++++--------- chain/actors/builtin/miner/miner.go | 30 +++--- chain/actors/builtin/miner/v0.go | 120 ++++++++++----------- chain/actors/builtin/multisig/multisig.go | 6 +- chain/actors/builtin/multisig/v0.go | 10 +- chain/actors/builtin/paych/paych.go | 6 +- chain/actors/builtin/paych/v0.go | 30 +++--- chain/actors/builtin/power/power.go | 8 +- chain/actors/builtin/power/v0.go | 24 ++--- chain/actors/builtin/reward/reward.go | 8 +- chain/actors/builtin/reward/v0.go | 18 ++-- chain/actors/builtin/verifreg/v0.go | 12 +-- chain/actors/builtin/verifreg/verifreg.go | 8 +- chain/events/state/predicates.go | 4 +- chain/events/state/predicates_test.go | 78 +++++++------- chain/gen/gen.go | 4 +- chain/gen/gen_test.go | 12 +-- chain/gen/genesis/genesis.go | 60 +++++------ chain/gen/genesis/miners.go | 42 ++++---- chain/gen/genesis/t02_reward.go | 4 +- chain/gen/genesis/t04_power.go | 4 +- chain/gen/genesis/t06_vreg.go | 4 +- chain/stmgr/forks.go | 14 +-- chain/stmgr/forks_test.go | 12 +-- chain/stmgr/stmgr.go | 16 +-- chain/stmgr/utils.go | 30 +++--- chain/store/store_test.go | 12 +-- chain/sync_test.go | 12 +-- chain/vectors/gen/main.go | 8 +- chain/vm/invoker.go | 48 ++++----- cli/multisig.go | 14 +-- cli/paych_test.go | 12 +-- cmd/lotus-pcr/main.go | 4 +- cmd/lotus-storage-miner/actor.go | 10 +- cmd/lotus-storage-miner/init.go | 22 ++-- cmd/lotus-storage-miner/sectors.go | 4 +- extern/storage-sealing/checks.go | 8 +- extern/storage-sealing/constants.go | 4 +- extern/storage-sealing/precommit_policy.go | 4 +- extern/storage-sealing/sealing.go | 4 +- extern/storage-sealing/states_sealing.go | 8 +- node/impl/client/client.go | 4 +- node/impl/full/state.go | 14 +-- node/node_test.go | 14 +-- node/test/builder.go | 4 +- paychmgr/manager.go | 14 +-- paychmgr/paych.go | 22 ++-- paychmgr/paych_test.go | 36 +++---- paychmgr/simple.go | 4 +- storage/adapter_storage_miner.go | 4 +- storage/miner.go | 4 +- storage/wdpost_run_test.go | 8 +- 65 files changed, 534 insertions(+), 534 deletions(-) diff --git a/build/params_2k.go b/build/params_2k.go index 61c5fd71d..4a49da22b 100644 --- a/build/params_2k.go +++ b/build/params_2k.go @@ -5,9 +5,9 @@ package build import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" - v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" - v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" - v0verifreg "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" + power0 "github.com/filecoin-project/specs-actors/actors/builtin/power" + verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" ) const UpgradeBreezeHeight = -1 @@ -20,11 +20,11 @@ var DrandSchedule = map[abi.ChainEpoch]DrandEnum{ } func init() { - v0power.ConsensusMinerMinPower = big.NewInt(2048) - v0miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ + power0.ConsensusMinerMinPower = big.NewInt(2048) + miner0.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } - v0verifreg.MinVerifiedDealSize = big.NewInt(256) + verifreg0.MinVerifiedDealSize = big.NewInt(256) BuildType |= Build2k } diff --git a/build/params_shared_funcs.go b/build/params_shared_funcs.go index 08f16cefd..0e9739914 100644 --- a/build/params_shared_funcs.go +++ b/build/params_shared_funcs.go @@ -6,14 +6,14 @@ import ( "github.com/libp2p/go-libp2p-core/protocol" "github.com/filecoin-project/go-state-types/abi" - v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/lotus/node/modules/dtypes" ) func DefaultSectorSize() abi.SectorSize { - szs := make([]abi.SectorSize, 0, len(v0miner.SupportedProofTypes)) - for spt := range v0miner.SupportedProofTypes { + szs := make([]abi.SectorSize, 0, len(miner0.SupportedProofTypes)) + for spt := range miner0.SupportedProofTypes { ss, err := spt.SectorSize() if err != nil { panic(err) diff --git a/build/params_shared_vals.go b/build/params_shared_vals.go index ac7796ae7..3ee9f52ec 100644 --- a/build/params_shared_vals.go +++ b/build/params_shared_vals.go @@ -9,7 +9,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/specs-actors/actors/builtin" - v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" ) // ///// @@ -32,7 +32,7 @@ const ForkLengthThreshold = Finality var BlocksPerEpoch = uint64(builtin.ExpectedLeadersPerEpoch) // Epochs -const Finality = v0miner.ChainFinality +const Finality = miner0.ChainFinality const MessageConfidence = uint64(5) // constants for Weight calculation diff --git a/build/params_testground.go b/build/params_testground.go index 1b30ae2e9..954b5ccfd 100644 --- a/build/params_testground.go +++ b/build/params_testground.go @@ -13,7 +13,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/specs-actors/actors/builtin" - v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" ) var ( @@ -32,7 +32,7 @@ var ( AllowableClockDriftSecs = uint64(1) - Finality = v0miner.ChainFinality + Finality = miner0.ChainFinality ForkLengthThreshold = Finality SlashablePowerDelay = 20 diff --git a/build/params_testnet.go b/build/params_testnet.go index a879d3ba7..108aba20c 100644 --- a/build/params_testnet.go +++ b/build/params_testnet.go @@ -8,8 +8,8 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/specs-actors/actors/builtin" - v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" - v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" + power0 "github.com/filecoin-project/specs-actors/actors/builtin/power" ) var DrandSchedule = map[abi.ChainEpoch]DrandEnum{ @@ -23,8 +23,8 @@ const BreezeGasTampingDuration = 120 const UpgradeSmokeHeight = 51000 func init() { - v0power.ConsensusMinerMinPower = big.NewInt(10 << 40) - v0miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ + power0.ConsensusMinerMinPower = big.NewInt(10 << 40) + miner0.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ abi.RegisteredSealProof_StackedDrg32GiBV1: {}, abi.RegisteredSealProof_StackedDrg64GiBV1: {}, } diff --git a/chain/actors/adt/adt.go b/chain/actors/adt/adt.go index ebf32c3c4..fd5ee3f87 100644 --- a/chain/actors/adt/adt.go +++ b/chain/actors/adt/adt.go @@ -8,7 +8,7 @@ import ( "github.com/filecoin-project/go-state-types/cbor" "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/chain/actors/builtin" - v0adt "github.com/filecoin-project/specs-actors/actors/util/adt" + adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" ) type Map interface { @@ -24,7 +24,7 @@ type Map interface { func AsMap(store Store, root cid.Cid, version builtin.Version) (Map, error) { switch version { case builtin.Version0: - return v0adt.AsMap(store, root) + return adt0.AsMap(store, root) } return nil, xerrors.Errorf("unknown network version: %d", version) } @@ -32,7 +32,7 @@ func AsMap(store Store, root cid.Cid, version builtin.Version) (Map, error) { func NewMap(store Store, version builtin.Version) (Map, error) { switch version { case builtin.Version0: - return v0adt.MakeEmptyMap(store), nil + return adt0.MakeEmptyMap(store), nil } return nil, xerrors.Errorf("unknown network version: %d", version) } @@ -51,7 +51,7 @@ type Array interface { func AsArray(store Store, root cid.Cid, version network.Version) (Array, error) { switch builtin.VersionForNetwork(version) { case builtin.Version0: - return v0adt.AsArray(store, root) + return adt0.AsArray(store, root) } return nil, xerrors.Errorf("unknown network version: %d", version) } diff --git a/chain/actors/adt/diff_adt_test.go b/chain/actors/adt/diff_adt_test.go index 436e28bbf..1c0726003 100644 --- a/chain/actors/adt/diff_adt_test.go +++ b/chain/actors/adt/diff_adt_test.go @@ -13,7 +13,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/specs-actors/actors/runtime" - v0adt "github.com/filecoin-project/specs-actors/actors/util/adt" + adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" bstore "github.com/filecoin-project/lotus/lib/blockstore" ) @@ -22,8 +22,8 @@ func TestDiffAdtArray(t *testing.T) { ctxstoreA := newContextStore() ctxstoreB := newContextStore() - arrA := v0adt.MakeEmptyArray(ctxstoreA) - arrB := v0adt.MakeEmptyArray(ctxstoreB) + arrA := adt0.MakeEmptyArray(ctxstoreA) + arrB := adt0.MakeEmptyArray(ctxstoreB) require.NoError(t, arrA.Set(0, runtime.CBORBytes([]byte{0}))) // delete @@ -76,8 +76,8 @@ func TestDiffAdtMap(t *testing.T) { ctxstoreA := newContextStore() ctxstoreB := newContextStore() - mapA := v0adt.MakeEmptyMap(ctxstoreA) - mapB := v0adt.MakeEmptyMap(ctxstoreB) + mapA := adt0.MakeEmptyMap(ctxstoreA) + mapB := adt0.MakeEmptyMap(ctxstoreB) require.NoError(t, mapA.Put(abi.UIntKey(0), runtime.CBORBytes([]byte{0}))) // delete diff --git a/chain/actors/builtin/account/account.go b/chain/actors/builtin/account/account.go index d4cbe4177..5b90580ec 100644 --- a/chain/actors/builtin/account/account.go +++ b/chain/actors/builtin/account/account.go @@ -5,7 +5,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/cbor" - v0builtin "github.com/filecoin-project/specs-actors/actors/builtin" + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/types" @@ -13,8 +13,8 @@ import ( func Load(store adt.Store, act *types.Actor) (State, error) { switch act.Code { - case v0builtin.AccountActorCodeID: - out := v0State{store: store} + case builtin0.AccountActorCodeID: + out := state0{store: store} err := store.Get(store.Context(), act.Head, &out) if err != nil { return nil, err diff --git a/chain/actors/builtin/account/v0.go b/chain/actors/builtin/account/v0.go index 6001dd7bf..535255d0e 100644 --- a/chain/actors/builtin/account/v0.go +++ b/chain/actors/builtin/account/v0.go @@ -6,11 +6,11 @@ import ( "github.com/filecoin-project/specs-actors/actors/builtin/account" ) -type v0State struct { +type state0 struct { account.State store adt.Store } -func (s *v0State) PubkeyAddress() (address.Address, error) { +func (s *state0) PubkeyAddress() (address.Address, error) { return s.Address, nil } diff --git a/chain/actors/builtin/builtin.go b/chain/actors/builtin/builtin.go index bee8e59d6..4ce77804c 100644 --- a/chain/actors/builtin/builtin.go +++ b/chain/actors/builtin/builtin.go @@ -5,7 +5,7 @@ import ( "github.com/filecoin-project/go-state-types/network" - v0smoothing "github.com/filecoin-project/specs-actors/actors/util/smoothing" + smoothing0 "github.com/filecoin-project/specs-actors/actors/util/smoothing" ) type Version int @@ -25,4 +25,4 @@ func VersionForNetwork(version network.Version) Version { } // TODO: find some way to abstract over this. -type FilterEstimate = v0smoothing.FilterEstimate +type FilterEstimate = smoothing0.FilterEstimate diff --git a/chain/actors/builtin/init/init.go b/chain/actors/builtin/init/init.go index 7485c599f..1164891f8 100644 --- a/chain/actors/builtin/init/init.go +++ b/chain/actors/builtin/init/init.go @@ -6,19 +6,19 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/cbor" - v0builtin "github.com/filecoin-project/specs-actors/actors/builtin" + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/modules/dtypes" ) -var Address = v0builtin.InitActorAddr +var Address = builtin0.InitActorAddr func Load(store adt.Store, act *types.Actor) (State, error) { switch act.Code { - case v0builtin.InitActorCodeID: - out := v0State{store: store} + case builtin0.InitActorCodeID: + out := state0{store: store} err := store.Get(store.Context(), act.Head, &out) if err != nil { return nil, err diff --git a/chain/actors/builtin/init/v0.go b/chain/actors/builtin/init/v0.go index e286e1ef2..0e8395a08 100644 --- a/chain/actors/builtin/init/v0.go +++ b/chain/actors/builtin/init/v0.go @@ -10,24 +10,24 @@ import ( "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/node/modules/dtypes" - v0adt "github.com/filecoin-project/specs-actors/actors/util/adt" + adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" ) -type v0State struct { +type state0 struct { init_.State store adt.Store } -func (s *v0State) ResolveAddress(address address.Address) (address.Address, bool, error) { +func (s *state0) ResolveAddress(address address.Address) (address.Address, bool, error) { return s.State.ResolveAddress(s.store, address) } -func (s *v0State) MapAddressToNewID(address address.Address) (address.Address, error) { +func (s *state0) MapAddressToNewID(address address.Address) (address.Address, error) { return s.State.MapAddressToNewID(s.store, address) } -func (s *v0State) ForEachActor(cb func(id abi.ActorID, address address.Address) error) error { - addrs, err := v0adt.AsMap(s.store, s.State.AddressMap) +func (s *state0) ForEachActor(cb func(id abi.ActorID, address address.Address) error) error { + addrs, err := adt0.AsMap(s.store, s.State.AddressMap) if err != nil { return err } @@ -41,6 +41,6 @@ func (s *v0State) ForEachActor(cb func(id abi.ActorID, address address.Address) }) } -func (s *v0State) NetworkName() (dtypes.NetworkName, error) { +func (s *state0) NetworkName() (dtypes.NetworkName, error) { return dtypes.NetworkName(s.State.NetworkName), nil } diff --git a/chain/actors/builtin/market/market.go b/chain/actors/builtin/market/market.go index c65fa093d..fa5f027b3 100644 --- a/chain/actors/builtin/market/market.go +++ b/chain/actors/builtin/market/market.go @@ -6,7 +6,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/cbor" - v0builtin "github.com/filecoin-project/specs-actors/actors/builtin" + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" @@ -14,12 +14,12 @@ import ( "github.com/filecoin-project/lotus/chain/types" ) -var Address = v0builtin.StorageMarketActorAddr +var Address = builtin0.StorageMarketActorAddr func Load(store adt.Store, act *types.Actor) (st State, err error) { switch act.Code { - case v0builtin.StorageMarketActorCodeID: - out := v0State{store: store} + case builtin0.StorageMarketActorCodeID: + out := state0{store: store} err := store.Get(store.Context(), act.Head, &out) if err != nil { return nil, err diff --git a/chain/actors/builtin/market/v0.go b/chain/actors/builtin/market/v0.go index 671907ad4..27eee4c50 100644 --- a/chain/actors/builtin/market/v0.go +++ b/chain/actors/builtin/market/v0.go @@ -8,95 +8,95 @@ import ( "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/specs-actors/actors/builtin/market" - v0adt "github.com/filecoin-project/specs-actors/actors/util/adt" + adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" cbg "github.com/whyrusleeping/cbor-gen" ) -type v0State struct { +type state0 struct { market.State store adt.Store } -func (s *v0State) TotalLocked() (abi.TokenAmount, error) { +func (s *state0) TotalLocked() (abi.TokenAmount, error) { fml := types.BigAdd(s.TotalClientLockedCollateral, s.TotalProviderLockedCollateral) fml = types.BigAdd(fml, s.TotalClientStorageFee) return fml, nil } -func (s *v0State) BalancesChanged(otherState State) bool { - v0otherState, ok := otherState.(*v0State) +func (s *state0) BalancesChanged(otherState State) bool { + otherState0, ok := otherState.(*state0) if !ok { // there's no way to compare differnt versions of the state, so let's // just say that means the state of balances has changed return true } - return !s.State.EscrowTable.Equals(v0otherState.State.EscrowTable) || !s.State.LockedTable.Equals(v0otherState.State.LockedTable) + return !s.State.EscrowTable.Equals(otherState0.State.EscrowTable) || !s.State.LockedTable.Equals(otherState0.State.LockedTable) } -func (s *v0State) StatesChanged(otherState State) bool { - v0otherState, ok := otherState.(*v0State) +func (s *state0) StatesChanged(otherState State) bool { + otherState0, ok := otherState.(*state0) if !ok { // there's no way to compare differnt versions of the state, so let's // just say that means the state of balances has changed return true } - return !s.State.States.Equals(v0otherState.State.States) + return !s.State.States.Equals(otherState0.State.States) } -func (s *v0State) States() (DealStates, error) { - stateArray, err := v0adt.AsArray(s.store, s.State.States) +func (s *state0) States() (DealStates, error) { + stateArray, err := adt0.AsArray(s.store, s.State.States) if err != nil { return nil, err } - return &v0DealStates{stateArray}, nil + return &dealStates0{stateArray}, nil } -func (s *v0State) ProposalsChanged(otherState State) bool { - v0otherState, ok := otherState.(*v0State) +func (s *state0) ProposalsChanged(otherState State) bool { + otherState0, ok := otherState.(*state0) if !ok { // there's no way to compare differnt versions of the state, so let's // just say that means the state of balances has changed return true } - return !s.State.Proposals.Equals(v0otherState.State.Proposals) + return !s.State.Proposals.Equals(otherState0.State.Proposals) } -func (s *v0State) Proposals() (DealProposals, error) { - proposalArray, err := v0adt.AsArray(s.store, s.State.Proposals) +func (s *state0) Proposals() (DealProposals, error) { + proposalArray, err := adt0.AsArray(s.store, s.State.Proposals) if err != nil { return nil, err } - return &v0DealProposals{proposalArray}, nil + return &dealProposals0{proposalArray}, nil } -func (s *v0State) EscrowTable() (BalanceTable, error) { - bt, err := v0adt.AsBalanceTable(s.store, s.State.EscrowTable) +func (s *state0) EscrowTable() (BalanceTable, error) { + bt, err := adt0.AsBalanceTable(s.store, s.State.EscrowTable) if err != nil { return nil, err } - return &v0BalanceTable{bt}, nil + return &balanceTable0{bt}, nil } -func (s *v0State) LockedTable() (BalanceTable, error) { - bt, err := v0adt.AsBalanceTable(s.store, s.State.LockedTable) +func (s *state0) LockedTable() (BalanceTable, error) { + bt, err := adt0.AsBalanceTable(s.store, s.State.LockedTable) if err != nil { return nil, err } - return &v0BalanceTable{bt}, nil + return &balanceTable0{bt}, nil } -func (s *v0State) VerifyDealsForActivation( +func (s *state0) VerifyDealsForActivation( minerAddr address.Address, deals []abi.DealID, currEpoch, sectorExpiry abi.ChainEpoch, ) (weight, verifiedWeight abi.DealWeight, err error) { return market.ValidateDealsForActivation(&s.State, s.store, deals, minerAddr, sectorExpiry, currEpoch) } -type v0BalanceTable struct { - *v0adt.BalanceTable +type balanceTable0 struct { + *adt0.BalanceTable } -func (bt *v0BalanceTable) ForEach(cb func(address.Address, abi.TokenAmount) error) error { - asMap := (*v0adt.Map)(bt.BalanceTable) +func (bt *balanceTable0) ForEach(cb func(address.Address, abi.TokenAmount) error) error { + asMap := (*adt0.Map)(bt.BalanceTable) var ta abi.TokenAmount return asMap.ForEach(&ta, func(key string) error { a, err := address.NewFromBytes([]byte(key)) @@ -107,33 +107,33 @@ func (bt *v0BalanceTable) ForEach(cb func(address.Address, abi.TokenAmount) erro }) } -type v0DealStates struct { +type dealStates0 struct { adt.Array } -func (s *v0DealStates) Get(dealID abi.DealID) (*DealState, bool, error) { - var v0deal market.DealState - found, err := s.Array.Get(uint64(dealID), &v0deal) +func (s *dealStates0) Get(dealID abi.DealID) (*DealState, bool, error) { + var deal0 market.DealState + found, err := s.Array.Get(uint64(dealID), &deal0) if err != nil { return nil, false, err } if !found { return nil, false, nil } - deal := fromV0DealState(v0deal) + deal := fromV0DealState(deal0) return &deal, true, nil } -func (s *v0DealStates) decode(val *cbg.Deferred) (*DealState, error) { - var v0ds market.DealState - if err := v0ds.UnmarshalCBOR(bytes.NewReader(val.Raw)); err != nil { +func (s *dealStates0) decode(val *cbg.Deferred) (*DealState, error) { + var ds0 market.DealState + if err := ds0.UnmarshalCBOR(bytes.NewReader(val.Raw)); err != nil { return nil, err } - ds := fromV0DealState(v0ds) + ds := fromV0DealState(ds0) return &ds, nil } -func (s *v0DealStates) array() adt.Array { +func (s *dealStates0) array() adt.Array { return s.Array } @@ -141,40 +141,40 @@ func fromV0DealState(v0 market.DealState) DealState { return (DealState)(v0) } -type v0DealProposals struct { +type dealProposals0 struct { adt.Array } -func (s *v0DealProposals) Get(dealID abi.DealID) (*DealProposal, bool, error) { - var v0proposal market.DealProposal - found, err := s.Array.Get(uint64(dealID), &v0proposal) +func (s *dealProposals0) Get(dealID abi.DealID) (*DealProposal, bool, error) { + var proposal0 market.DealProposal + found, err := s.Array.Get(uint64(dealID), &proposal0) if err != nil { return nil, false, err } if !found { return nil, false, nil } - proposal := fromV0DealProposal(v0proposal) + proposal := fromV0DealProposal(proposal0) return &proposal, true, nil } -func (s *v0DealProposals) ForEach(cb func(dealID abi.DealID, dp DealProposal) error) error { - var v0dp market.DealProposal - return s.Array.ForEach(&v0dp, func(idx int64) error { - return cb(abi.DealID(idx), fromV0DealProposal(v0dp)) +func (s *dealProposals0) ForEach(cb func(dealID abi.DealID, dp DealProposal) error) error { + var dp0 market.DealProposal + return s.Array.ForEach(&dp0, func(idx int64) error { + return cb(abi.DealID(idx), fromV0DealProposal(dp0)) }) } -func (s *v0DealProposals) decode(val *cbg.Deferred) (*DealProposal, error) { - var v0dp market.DealProposal - if err := v0dp.UnmarshalCBOR(bytes.NewReader(val.Raw)); err != nil { +func (s *dealProposals0) decode(val *cbg.Deferred) (*DealProposal, error) { + var dp0 market.DealProposal + if err := dp0.UnmarshalCBOR(bytes.NewReader(val.Raw)); err != nil { return nil, err } - dp := fromV0DealProposal(v0dp) + dp := fromV0DealProposal(dp0) return &dp, nil } -func (s *v0DealProposals) array() adt.Array { +func (s *dealProposals0) array() adt.Array { return s.Array } diff --git a/chain/actors/builtin/miner/miner.go b/chain/actors/builtin/miner/miner.go index c67e773bf..b9cff99da 100644 --- a/chain/actors/builtin/miner/miner.go +++ b/chain/actors/builtin/miner/miner.go @@ -10,19 +10,19 @@ import ( "github.com/filecoin-project/go-bitfield" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/cbor" - v0builtin "github.com/filecoin-project/specs-actors/actors/builtin" - v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/types" ) -var Address = v0builtin.InitActorAddr +var Address = builtin0.InitActorAddr func Load(store adt.Store, act *types.Actor) (st State, err error) { switch act.Code { - case v0builtin.StorageMinerActorCodeID: - out := v0State{store: store} + case builtin0.StorageMinerActorCodeID: + out := state0{store: store} err := store.Get(store.Context(), act.Head, &out) if err != nil { return nil, err @@ -83,18 +83,18 @@ type Partition interface { ActiveSectors() (bitfield.BitField, error) } -type SectorOnChainInfo = v0miner.SectorOnChainInfo -type SectorPreCommitInfo = v0miner.SectorPreCommitInfo -type SectorPreCommitOnChainInfo = v0miner.SectorPreCommitOnChainInfo -type PoStPartition = v0miner.PoStPartition -type RecoveryDeclaration = v0miner.RecoveryDeclaration -type FaultDeclaration = v0miner.FaultDeclaration +type SectorOnChainInfo = miner0.SectorOnChainInfo +type SectorPreCommitInfo = miner0.SectorPreCommitInfo +type SectorPreCommitOnChainInfo = miner0.SectorPreCommitOnChainInfo +type PoStPartition = miner0.PoStPartition +type RecoveryDeclaration = miner0.RecoveryDeclaration +type FaultDeclaration = miner0.FaultDeclaration // Params -type DeclareFaultsParams = v0miner.DeclareFaultsParams -type DeclareFaultsRecoveredParams = v0miner.DeclareFaultsRecoveredParams -type SubmitWindowedPoStParams = v0miner.SubmitWindowedPoStParams -type ProveCommitSectorParams = v0miner.ProveCommitSectorParams +type DeclareFaultsParams = miner0.DeclareFaultsParams +type DeclareFaultsRecoveredParams = miner0.DeclareFaultsRecoveredParams +type SubmitWindowedPoStParams = miner0.SubmitWindowedPoStParams +type ProveCommitSectorParams = miner0.ProveCommitSectorParams type MinerInfo struct { Owner address.Address // Must be an ID-address. diff --git a/chain/actors/builtin/miner/v0.go b/chain/actors/builtin/miner/v0.go index 0f27d7d7f..26a91edd1 100644 --- a/chain/actors/builtin/miner/v0.go +++ b/chain/actors/builtin/miner/v0.go @@ -9,38 +9,38 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/dline" "github.com/filecoin-project/lotus/chain/actors/adt" - v0adt "github.com/filecoin-project/specs-actors/actors/util/adt" + adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/libp2p/go-libp2p-core/peer" cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" - v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" ) -type v0State struct { - v0miner.State +type state0 struct { + miner0.State store adt.Store } -type v0Deadline struct { - v0miner.Deadline +type deadline0 struct { + miner0.Deadline store adt.Store } -type v0Partition struct { - v0miner.Partition +type partition0 struct { + miner0.Partition store adt.Store } -func (s *v0State) AvailableBalance(bal abi.TokenAmount) (abi.TokenAmount, error) { +func (s *state0) AvailableBalance(bal abi.TokenAmount) (abi.TokenAmount, error) { return s.GetAvailableBalance(bal), nil } -func (s *v0State) VestedFunds(epoch abi.ChainEpoch) (abi.TokenAmount, error) { +func (s *state0) VestedFunds(epoch abi.ChainEpoch) (abi.TokenAmount, error) { return s.CheckVestedFunds(s.store, epoch) } -func (s *v0State) LockedFunds() (LockedFunds, error) { +func (s *state0) LockedFunds() (LockedFunds, error) { return LockedFunds{ VestingFunds: s.State.LockedFunds, InitialPledgeRequirement: s.State.InitialPledgeRequirement, @@ -48,15 +48,15 @@ func (s *v0State) LockedFunds() (LockedFunds, error) { }, nil } -func (s *v0State) InitialPledge() (abi.TokenAmount, error) { +func (s *state0) InitialPledge() (abi.TokenAmount, error) { return s.State.InitialPledgeRequirement, nil } -func (s *v0State) PreCommitDeposits() (abi.TokenAmount, error) { +func (s *state0) PreCommitDeposits() (abi.TokenAmount, error) { return s.State.PreCommitDeposits, nil } -func (s *v0State) GetSector(num abi.SectorNumber) (*SectorOnChainInfo, error) { +func (s *state0) GetSector(num abi.SectorNumber) (*SectorOnChainInfo, error) { info, ok, err := s.State.GetSector(s.store, num) if !ok || err != nil { return nil, err @@ -65,7 +65,7 @@ func (s *v0State) GetSector(num abi.SectorNumber) (*SectorOnChainInfo, error) { return info, nil } -func (s *v0State) FindSector(num abi.SectorNumber) (*SectorLocation, error) { +func (s *state0) FindSector(num abi.SectorNumber) (*SectorLocation, error) { dlIdx, partIdx, err := s.State.FindSector(s.store, num) if err != nil { return nil, err @@ -81,7 +81,7 @@ func (s *v0State) FindSector(num abi.SectorNumber) (*SectorLocation, error) { // If the sector isn't found or has already been terminated, this method returns // nil and no error. If the sector does not expire early, the Early expiration // field is 0. -func (s *v0State) GetSectorExpiration(num abi.SectorNumber) (*SectorExpiration, error) { +func (s *state0) GetSectorExpiration(num abi.SectorNumber) (*SectorExpiration, error) { dls, err := s.State.LoadDeadlines(s.store) if err != nil { return nil, err @@ -94,13 +94,13 @@ func (s *v0State) GetSectorExpiration(num abi.SectorNumber) (*SectorExpiration, // of the expiration queue. stopErr := errors.New("stop") out := SectorExpiration{} - err = dls.ForEach(s.store, func(dlIdx uint64, dl *v0miner.Deadline) error { + err = dls.ForEach(s.store, func(dlIdx uint64, dl *miner0.Deadline) error { partitions, err := dl.PartitionsArray(s.store) if err != nil { return err } quant := s.State.QuantSpecForDeadline(dlIdx) - var part v0miner.Partition + var part miner0.Partition return partitions.ForEach(&part, func(partIdx int64) error { if found, err := part.Sectors.IsSet(uint64(num)); err != nil { return err @@ -114,11 +114,11 @@ func (s *v0State) GetSectorExpiration(num abi.SectorNumber) (*SectorExpiration, return stopErr } - q, err := v0miner.LoadExpirationQueue(s.store, part.ExpirationsEpochs, quant) + q, err := miner0.LoadExpirationQueue(s.store, part.ExpirationsEpochs, quant) if err != nil { return err } - var exp v0miner.ExpirationSet + var exp miner0.ExpirationSet return q.ForEach(&exp, func(epoch int64) error { if early, err := exp.EarlySectors.IsSet(uint64(num)); err != nil { return err @@ -148,7 +148,7 @@ func (s *v0State) GetSectorExpiration(num abi.SectorNumber) (*SectorExpiration, return &out, nil } -func (s *v0State) GetPrecommittedSector(num abi.SectorNumber) (*SectorPreCommitOnChainInfo, error) { +func (s *state0) GetPrecommittedSector(num abi.SectorNumber) (*SectorPreCommitOnChainInfo, error) { info, ok, err := s.State.GetPrecommittedSector(s.store, num) if !ok || err != nil { return nil, err @@ -157,8 +157,8 @@ func (s *v0State) GetPrecommittedSector(num abi.SectorNumber) (*SectorPreCommitO return info, nil } -func (s *v0State) LoadSectorsFromSet(filter *bitfield.BitField, filterOut bool) (adt.Array, error) { - a, err := v0adt.AsArray(s.store, s.State.Sectors) +func (s *state0) LoadSectorsFromSet(filter *bitfield.BitField, filterOut bool) (adt.Array, error) { + a, err := adt0.AsArray(s.store, s.State.Sectors) if err != nil { return nil, err } @@ -185,11 +185,11 @@ func (s *v0State) LoadSectorsFromSet(filter *bitfield.BitField, filterOut bool) return a, nil } -func (s *v0State) LoadPreCommittedSectors() (adt.Map, error) { - return v0adt.AsMap(s.store, s.State.PreCommittedSectors) +func (s *state0) LoadPreCommittedSectors() (adt.Map, error) { + return adt0.AsMap(s.store, s.State.PreCommittedSectors) } -func (s *v0State) IsAllocated(num abi.SectorNumber) (bool, error) { +func (s *state0) IsAllocated(num abi.SectorNumber) (bool, error) { var allocatedSectors bitfield.BitField if err := s.store.Get(s.store.Context(), s.State.AllocatedSectors, &allocatedSectors); err != nil { return false, err @@ -198,7 +198,7 @@ func (s *v0State) IsAllocated(num abi.SectorNumber) (bool, error) { return allocatedSectors.IsSet(uint64(num)) } -func (s *v0State) LoadDeadline(idx uint64) (Deadline, error) { +func (s *state0) LoadDeadline(idx uint64) (Deadline, error) { dls, err := s.State.LoadDeadlines(s.store) if err != nil { return nil, err @@ -207,39 +207,39 @@ func (s *v0State) LoadDeadline(idx uint64) (Deadline, error) { if err != nil { return nil, err } - return &v0Deadline{*dl, s.store}, nil + return &deadline0{*dl, s.store}, nil } -func (s *v0State) ForEachDeadline(cb func(uint64, Deadline) error) error { +func (s *state0) ForEachDeadline(cb func(uint64, Deadline) error) error { dls, err := s.State.LoadDeadlines(s.store) if err != nil { return err } - return dls.ForEach(s.store, func(i uint64, dl *v0miner.Deadline) error { - return cb(i, &v0Deadline{*dl, s.store}) + return dls.ForEach(s.store, func(i uint64, dl *miner0.Deadline) error { + return cb(i, &deadline0{*dl, s.store}) }) } -func (s *v0State) NumDeadlines() (uint64, error) { - return v0miner.WPoStPeriodDeadlines, nil +func (s *state0) NumDeadlines() (uint64, error) { + return miner0.WPoStPeriodDeadlines, nil } // Max sectors per PoSt -func (s *v0State) MaxAddressedSectors() (uint64, error) { - return v0miner.AddressedSectorsMax, nil +func (s *state0) MaxAddressedSectors() (uint64, error) { + return miner0.AddressedSectorsMax, nil } -func (s *v0State) DeadlinesChanged(other State) bool { - v0other, ok := other.(*v0State) +func (s *state0) DeadlinesChanged(other State) bool { + other0, ok := other.(*state0) if !ok { // treat an upgrade as a change, always return true } - return s.State.Deadlines.Equals(v0other.Deadlines) + return s.State.Deadlines.Equals(other0.Deadlines) } -func (s *v0State) Info() (MinerInfo, error) { +func (s *state0) Info() (MinerInfo, error) { info, err := s.State.GetInfo(s.store) if err != nil { return MinerInfo{}, err @@ -273,71 +273,71 @@ func (s *v0State) Info() (MinerInfo, error) { return mi, nil } -func (s *v0State) DeadlineInfo(epoch abi.ChainEpoch) *dline.Info { +func (s *state0) DeadlineInfo(epoch abi.ChainEpoch) *dline.Info { return s.State.DeadlineInfo(epoch) } -func (s *v0State) sectors() (adt.Array, error) { - return v0adt.AsArray(s.store, s.Sectors) +func (s *state0) sectors() (adt.Array, error) { + return adt0.AsArray(s.store, s.Sectors) } -func (s *v0State) decodeSectorOnChainInfo(val *cbg.Deferred) (SectorOnChainInfo, error) { - var si v0miner.SectorOnChainInfo +func (s *state0) decodeSectorOnChainInfo(val *cbg.Deferred) (SectorOnChainInfo, error) { + var si miner0.SectorOnChainInfo err := si.UnmarshalCBOR(bytes.NewReader(val.Raw)) return si, err } -func (s *v0State) precommits() (adt.Map, error) { - return v0adt.AsMap(s.store, s.PreCommittedSectors) +func (s *state0) precommits() (adt.Map, error) { + return adt0.AsMap(s.store, s.PreCommittedSectors) } -func (s *v0State) decodeSectorPreCommitOnChainInfo(val *cbg.Deferred) (SectorPreCommitOnChainInfo, error) { - var sp v0miner.SectorPreCommitOnChainInfo +func (s *state0) decodeSectorPreCommitOnChainInfo(val *cbg.Deferred) (SectorPreCommitOnChainInfo, error) { + var sp miner0.SectorPreCommitOnChainInfo err := sp.UnmarshalCBOR(bytes.NewReader(val.Raw)) return sp, err } -func (d *v0Deadline) LoadPartition(idx uint64) (Partition, error) { +func (d *deadline0) LoadPartition(idx uint64) (Partition, error) { p, err := d.Deadline.LoadPartition(d.store, idx) if err != nil { return nil, err } - return &v0Partition{*p, d.store}, nil + return &partition0{*p, d.store}, nil } -func (d *v0Deadline) ForEachPartition(cb func(uint64, Partition) error) error { +func (d *deadline0) ForEachPartition(cb func(uint64, Partition) error) error { ps, err := d.Deadline.PartitionsArray(d.store) if err != nil { return err } - var part v0miner.Partition + var part miner0.Partition return ps.ForEach(&part, func(i int64) error { - return cb(uint64(i), &v0Partition{part, d.store}) + return cb(uint64(i), &partition0{part, d.store}) }) } -func (d *v0Deadline) PartitionsChanged(other Deadline) bool { - v0other, ok := other.(*v0Deadline) +func (d *deadline0) PartitionsChanged(other Deadline) bool { + other0, ok := other.(*deadline0) if !ok { // treat an upgrade as a change, always return true } - return d.Deadline.Partitions.Equals(v0other.Deadline.Partitions) + return d.Deadline.Partitions.Equals(other0.Deadline.Partitions) } -func (d *v0Deadline) PostSubmissions() (bitfield.BitField, error) { +func (d *deadline0) PostSubmissions() (bitfield.BitField, error) { return d.Deadline.PostSubmissions, nil } -func (p *v0Partition) AllSectors() (bitfield.BitField, error) { +func (p *partition0) AllSectors() (bitfield.BitField, error) { return p.Partition.Sectors, nil } -func (p *v0Partition) FaultySectors() (bitfield.BitField, error) { +func (p *partition0) FaultySectors() (bitfield.BitField, error) { return p.Partition.Faults, nil } -func (p *v0Partition) RecoveringSectors() (bitfield.BitField, error) { +func (p *partition0) RecoveringSectors() (bitfield.BitField, error) { return p.Partition.Recoveries, nil } diff --git a/chain/actors/builtin/multisig/multisig.go b/chain/actors/builtin/multisig/multisig.go index fc58599a9..86a68c178 100644 --- a/chain/actors/builtin/multisig/multisig.go +++ b/chain/actors/builtin/multisig/multisig.go @@ -5,7 +5,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/cbor" - v0builtin "github.com/filecoin-project/specs-actors/actors/builtin" + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/types" @@ -13,8 +13,8 @@ import ( func Load(store adt.Store, act *types.Actor) (State, error) { switch act.Code { - case v0builtin.MultisigActorCodeID: - out := v0State{store: store} + case builtin0.MultisigActorCodeID: + out := state0{store: store} err := store.Get(store.Context(), act.Head, &out) if err != nil { return nil, err diff --git a/chain/actors/builtin/multisig/v0.go b/chain/actors/builtin/multisig/v0.go index dc464d9af..ded834d5f 100644 --- a/chain/actors/builtin/multisig/v0.go +++ b/chain/actors/builtin/multisig/v0.go @@ -6,23 +6,23 @@ import ( "github.com/filecoin-project/specs-actors/actors/builtin/multisig" ) -type v0State struct { +type state0 struct { multisig.State store adt.Store } -func (s *v0State) LockedBalance(currEpoch abi.ChainEpoch) (abi.TokenAmount, error) { +func (s *state0) LockedBalance(currEpoch abi.ChainEpoch) (abi.TokenAmount, error) { return s.State.AmountLocked(currEpoch - s.State.StartEpoch), nil } -func (s *v0State) StartEpoch() abi.ChainEpoch { +func (s *state0) StartEpoch() abi.ChainEpoch { return s.State.StartEpoch } -func (s *v0State) UnlockDuration() abi.ChainEpoch { +func (s *state0) UnlockDuration() abi.ChainEpoch { return s.State.UnlockDuration } -func (s *v0State) InitialBalance() abi.TokenAmount { +func (s *state0) InitialBalance() abi.TokenAmount { return s.State.InitialBalance } diff --git a/chain/actors/builtin/paych/paych.go b/chain/actors/builtin/paych/paych.go index 974d64fde..5eec5f08b 100644 --- a/chain/actors/builtin/paych/paych.go +++ b/chain/actors/builtin/paych/paych.go @@ -7,7 +7,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" big "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/cbor" - v0builtin "github.com/filecoin-project/specs-actors/actors/builtin" + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/types" @@ -16,8 +16,8 @@ import ( // Load returns an abstract copy of payment channel state, irregardless of actor version func Load(store adt.Store, act *types.Actor) (State, error) { switch act.Code { - case v0builtin.PaymentChannelActorCodeID: - out := v0State{store: store} + case builtin0.PaymentChannelActorCodeID: + out := state0{store: store} err := store.Get(store.Context(), act.Head, &out) if err != nil { return nil, err diff --git a/chain/actors/builtin/paych/v0.go b/chain/actors/builtin/paych/v0.go index 7d63b8913..0c4b2f218 100644 --- a/chain/actors/builtin/paych/v0.go +++ b/chain/actors/builtin/paych/v0.go @@ -6,42 +6,42 @@ import ( big "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/specs-actors/actors/builtin/paych" - v0adt "github.com/filecoin-project/specs-actors/actors/util/adt" + adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" ) -type v0State struct { +type state0 struct { paych.State store adt.Store - lsAmt *v0adt.Array + lsAmt *adt0.Array } // Channel owner, who has funded the actor -func (s *v0State) From() address.Address { +func (s *state0) From() address.Address { return s.State.From } // Recipient of payouts from channel -func (s *v0State) To() address.Address { +func (s *state0) To() address.Address { return s.State.To } // Height at which the channel can be `Collected` -func (s *v0State) SettlingAt() abi.ChainEpoch { +func (s *state0) SettlingAt() abi.ChainEpoch { return s.State.SettlingAt } // Amount successfully redeemed through the payment channel, paid out on `Collect()` -func (s *v0State) ToSend() abi.TokenAmount { +func (s *state0) ToSend() abi.TokenAmount { return s.State.ToSend } -func (s *v0State) getOrLoadLsAmt() (*v0adt.Array, error) { +func (s *state0) getOrLoadLsAmt() (*adt0.Array, error) { if s.lsAmt != nil { return s.lsAmt, nil } // Get the lane state from the chain - lsamt, err := v0adt.AsArray(s.store, s.State.LaneStates) + lsamt, err := adt0.AsArray(s.store, s.State.LaneStates) if err != nil { return nil, err } @@ -51,7 +51,7 @@ func (s *v0State) getOrLoadLsAmt() (*v0adt.Array, error) { } // Get total number of lanes -func (s *v0State) LaneCount() (uint64, error) { +func (s *state0) LaneCount() (uint64, error) { lsamt, err := s.getOrLoadLsAmt() if err != nil { return 0, err @@ -60,7 +60,7 @@ func (s *v0State) LaneCount() (uint64, error) { } // Iterate lane states -func (s *v0State) ForEachLaneState(cb func(idx uint64, dl LaneState) error) error { +func (s *state0) ForEachLaneState(cb func(idx uint64, dl LaneState) error) error { // Get the lane state from the chain lsamt, err := s.getOrLoadLsAmt() if err != nil { @@ -72,18 +72,18 @@ func (s *v0State) ForEachLaneState(cb func(idx uint64, dl LaneState) error) erro // very large index. var ls paych.LaneState return lsamt.ForEach(&ls, func(i int64) error { - return cb(uint64(i), &v0LaneState{ls}) + return cb(uint64(i), &laneState0{ls}) }) } -type v0LaneState struct { +type laneState0 struct { paych.LaneState } -func (ls *v0LaneState) Redeemed() big.Int { +func (ls *laneState0) Redeemed() big.Int { return ls.LaneState.Redeemed } -func (ls *v0LaneState) Nonce() uint64 { +func (ls *laneState0) Nonce() uint64 { return ls.LaneState.Nonce } diff --git a/chain/actors/builtin/power/power.go b/chain/actors/builtin/power/power.go index 7170526bf..e4bb52d44 100644 --- a/chain/actors/builtin/power/power.go +++ b/chain/actors/builtin/power/power.go @@ -6,19 +6,19 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/cbor" - v0builtin "github.com/filecoin-project/specs-actors/actors/builtin" + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/types" ) -var Address = v0builtin.StoragePowerActorAddr +var Address = builtin0.StoragePowerActorAddr func Load(store adt.Store, act *types.Actor) (st State, err error) { switch act.Code { - case v0builtin.StoragePowerActorCodeID: - out := v0State{store: store} + case builtin0.StoragePowerActorCodeID: + out := state0{store: store} err := store.Get(store.Context(), act.Head, &out) if err != nil { return nil, err diff --git a/chain/actors/builtin/power/v0.go b/chain/actors/builtin/power/v0.go index 9730be893..1d5a93d2b 100644 --- a/chain/actors/builtin/power/v0.go +++ b/chain/actors/builtin/power/v0.go @@ -4,20 +4,20 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/actors/builtin" - v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" + power0 "github.com/filecoin-project/specs-actors/actors/builtin/power" "github.com/filecoin-project/specs-actors/actors/util/adt" ) -type v0State struct { - v0power.State +type state0 struct { + power0.State store adt.Store } -func (s *v0State) TotalLocked() (abi.TokenAmount, error) { +func (s *state0) TotalLocked() (abi.TokenAmount, error) { return s.TotalPledgeCollateral, nil } -func (s *v0State) TotalPower() (Claim, error) { +func (s *state0) TotalPower() (Claim, error) { return Claim{ RawBytePower: s.TotalRawBytePower, QualityAdjPower: s.TotalQualityAdjPower, @@ -25,19 +25,19 @@ func (s *v0State) TotalPower() (Claim, error) { } // Committed power to the network. Includes miners below the minimum threshold. -func (s *v0State) TotalCommitted() (Claim, error) { +func (s *state0) TotalCommitted() (Claim, error) { return Claim{ RawBytePower: s.TotalBytesCommitted, QualityAdjPower: s.TotalQABytesCommitted, }, nil } -func (s *v0State) MinerPower(addr address.Address) (Claim, bool, error) { +func (s *state0) MinerPower(addr address.Address) (Claim, bool, error) { claims, err := adt.AsMap(s.store, s.Claims) if err != nil { return Claim{}, false, err } - var claim v0power.Claim + var claim power0.Claim ok, err := claims.Get(abi.AddrKey(addr), &claim) if err != nil { return Claim{}, false, err @@ -48,19 +48,19 @@ func (s *v0State) MinerPower(addr address.Address) (Claim, bool, error) { }, ok, nil } -func (s *v0State) MinerNominalPowerMeetsConsensusMinimum(a address.Address) (bool, error) { +func (s *state0) MinerNominalPowerMeetsConsensusMinimum(a address.Address) (bool, error) { return s.State.MinerNominalPowerMeetsConsensusMinimum(s.store, a) } -func (s *v0State) TotalPowerSmoothed() (builtin.FilterEstimate, error) { +func (s *state0) TotalPowerSmoothed() (builtin.FilterEstimate, error) { return *s.State.ThisEpochQAPowerSmoothed, nil } -func (s *v0State) MinerCounts() (uint64, uint64, error) { +func (s *state0) MinerCounts() (uint64, uint64, error) { return uint64(s.State.MinerAboveMinPowerCount), uint64(s.State.MinerCount), nil } -func (s *v0State) ListAllMiners() ([]address.Address, error) { +func (s *state0) ListAllMiners() ([]address.Address, error) { claims, err := adt.AsMap(s.store, s.Claims) if err != nil { return nil, err diff --git a/chain/actors/builtin/reward/reward.go b/chain/actors/builtin/reward/reward.go index 66df887fc..b56292fff 100644 --- a/chain/actors/builtin/reward/reward.go +++ b/chain/actors/builtin/reward/reward.go @@ -5,19 +5,19 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-state-types/cbor" - v0builtin "github.com/filecoin-project/specs-actors/actors/builtin" + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/types" ) -var Address = v0builtin.RewardActorAddr +var Address = builtin0.RewardActorAddr func Load(store adt.Store, act *types.Actor) (st State, err error) { switch act.Code { - case v0builtin.RewardActorCodeID: - out := v0State{store: store} + case builtin0.RewardActorCodeID: + out := state0{store: store} err := store.Get(store.Context(), act.Head, &out) if err != nil { return nil, err diff --git a/chain/actors/builtin/reward/v0.go b/chain/actors/builtin/reward/v0.go index d12eccf59..50ad49971 100644 --- a/chain/actors/builtin/reward/v0.go +++ b/chain/actors/builtin/reward/v0.go @@ -7,39 +7,39 @@ import ( "github.com/filecoin-project/specs-actors/actors/util/adt" ) -type v0State struct { +type state0 struct { reward.State store adt.Store } -func (s *v0State) ThisEpochReward() (abi.StoragePower, error) { +func (s *state0) ThisEpochReward() (abi.StoragePower, error) { return s.State.ThisEpochReward, nil } -func (s *v0State) ThisEpochRewardSmoothed() (builtin.FilterEstimate, error) { +func (s *state0) ThisEpochRewardSmoothed() (builtin.FilterEstimate, error) { return *s.State.ThisEpochRewardSmoothed, nil } -func (s *v0State) ThisEpochBaselinePower() (abi.StoragePower, error) { +func (s *state0) ThisEpochBaselinePower() (abi.StoragePower, error) { return s.State.ThisEpochBaselinePower, nil } -func (s *v0State) TotalStoragePowerReward() (abi.TokenAmount, error) { +func (s *state0) TotalStoragePowerReward() (abi.TokenAmount, error) { return s.State.TotalMined, nil } -func (s *v0State) EffectiveBaselinePower() (abi.StoragePower, error) { +func (s *state0) EffectiveBaselinePower() (abi.StoragePower, error) { return s.State.EffectiveBaselinePower, nil } -func (s *v0State) EffectiveNetworkTime() (abi.ChainEpoch, error) { +func (s *state0) EffectiveNetworkTime() (abi.ChainEpoch, error) { return s.State.EffectiveNetworkTime, nil } -func (s *v0State) CumsumBaseline() (abi.StoragePower, error) { +func (s *state0) CumsumBaseline() (abi.StoragePower, error) { return s.State.CumsumBaseline, nil } -func (s *v0State) CumsumRealized() (abi.StoragePower, error) { +func (s *state0) CumsumRealized() (abi.StoragePower, error) { return s.State.CumsumBaseline, nil } diff --git a/chain/actors/builtin/verifreg/v0.go b/chain/actors/builtin/verifreg/v0.go index cbaf4d236..c397d6679 100644 --- a/chain/actors/builtin/verifreg/v0.go +++ b/chain/actors/builtin/verifreg/v0.go @@ -4,24 +4,24 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" - v0verifreg "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" - v0adt "github.com/filecoin-project/specs-actors/actors/util/adt" + verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" + adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/chain/actors/adt" ) -type v0State struct { - v0verifreg.State +type state0 struct { + verifreg0.State store adt.Store } -func (s *v0State) VerifiedClientDataCap(addr address.Address) (bool, abi.StoragePower, error) { +func (s *state0) VerifiedClientDataCap(addr address.Address) (bool, abi.StoragePower, error) { if addr.Protocol() != address.ID { return false, big.Zero(), xerrors.Errorf("can only look up ID addresses") } - vh, err := v0adt.AsMap(s.store, s.VerifiedClients) + vh, err := adt0.AsMap(s.store, s.VerifiedClients) if err != nil { return false, big.Zero(), xerrors.Errorf("loading verified clients: %w", err) } diff --git a/chain/actors/builtin/verifreg/verifreg.go b/chain/actors/builtin/verifreg/verifreg.go index 4cb5bb55b..000a94349 100644 --- a/chain/actors/builtin/verifreg/verifreg.go +++ b/chain/actors/builtin/verifreg/verifreg.go @@ -6,18 +6,18 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-state-types/cbor" - v0builtin "github.com/filecoin-project/specs-actors/actors/builtin" + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/types" ) -var Address = v0builtin.VerifiedRegistryActorAddr +var Address = builtin0.VerifiedRegistryActorAddr func Load(store adt.Store, act *types.Actor) (State, error) { switch act.Code { - case v0builtin.VerifiedRegistryActorCodeID: - out := v0State{store: store} + case builtin0.VerifiedRegistryActorCodeID: + out := state0{store: store} err := store.Get(store.Context(), act.Head, &out) if err != nil { return nil, err diff --git a/chain/events/state/predicates.go b/chain/events/state/predicates.go index 21e1720e6..56dfb981d 100644 --- a/chain/events/state/predicates.go +++ b/chain/events/state/predicates.go @@ -509,12 +509,12 @@ func (sp *StatePredicates) OnAddressMapChange() DiffInitActorStateFunc { return false, nil, nil } - oldAddrs, err := v0adt.AsMap(ctxStore, oldState.AddressMap) + oldAddrs, err := adt0.AsMap(ctxStore, oldState.AddressMap) if err != nil { return false, nil, err } - newAddrs, err := v0adt.AsMap(ctxStore, newState.AddressMap) + newAddrs, err := adt0.AsMap(ctxStore, newState.AddressMap) if err != nil { return false, nil, err } diff --git a/chain/events/state/predicates_test.go b/chain/events/state/predicates_test.go index 604eec75d..832f6a0a7 100644 --- a/chain/events/state/predicates_test.go +++ b/chain/events/state/predicates_test.go @@ -19,10 +19,10 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/lotus/chain/actors/builtin/market" - v0builtin "github.com/filecoin-project/specs-actors/actors/builtin" - v0market "github.com/filecoin-project/specs-actors/actors/builtin/market" + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" + market0 "github.com/filecoin-project/specs-actors/actors/builtin/market" - v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/specs-actors/actors/util/adt" tutils "github.com/filecoin-project/specs-actors/support/testing" @@ -74,22 +74,22 @@ func TestMarketPredicates(t *testing.T) { bs := bstore.NewTemporarySync() store := adt.WrapStore(ctx, cbornode.NewCborStore(bs)) - oldDeal1 := &v0market.DealState{ + oldDeal1 := &market0.DealState{ SectorStartEpoch: 1, LastUpdatedEpoch: 2, SlashEpoch: 0, } - oldDeal2 := &v0market.DealState{ + oldDeal2 := &market0.DealState{ SectorStartEpoch: 4, LastUpdatedEpoch: 5, SlashEpoch: 0, } - oldDeals := map[abi.DealID]*v0market.DealState{ + oldDeals := map[abi.DealID]*market0.DealState{ abi.DealID(1): oldDeal1, abi.DealID(2): oldDeal2, } - oldProp1 := &v0market.DealProposal{ + oldProp1 := &market0.DealProposal{ PieceCID: dummyCid, PieceSize: 0, VerifiedDeal: false, @@ -101,7 +101,7 @@ func TestMarketPredicates(t *testing.T) { ProviderCollateral: big.Zero(), ClientCollateral: big.Zero(), } - oldProp2 := &v0market.DealProposal{ + oldProp2 := &market0.DealProposal{ PieceCID: dummyCid, PieceSize: 0, VerifiedDeal: false, @@ -113,7 +113,7 @@ func TestMarketPredicates(t *testing.T) { ProviderCollateral: big.Zero(), ClientCollateral: big.Zero(), } - oldProps := map[abi.DealID]*v0market.DealProposal{ + oldProps := map[abi.DealID]*market0.DealProposal{ abi.DealID(1): oldProp1, abi.DealID(2): oldProp2, } @@ -127,7 +127,7 @@ func TestMarketPredicates(t *testing.T) { oldStateC := createMarketState(ctx, t, store, oldDeals, oldProps, oldBalances) - newDeal1 := &v0market.DealState{ + newDeal1 := &market0.DealState{ SectorStartEpoch: 1, LastUpdatedEpoch: 3, SlashEpoch: 0, @@ -136,19 +136,19 @@ func TestMarketPredicates(t *testing.T) { // deal 2 removed // added - newDeal3 := &v0market.DealState{ + newDeal3 := &market0.DealState{ SectorStartEpoch: 1, LastUpdatedEpoch: 2, SlashEpoch: 3, } - newDeals := map[abi.DealID]*v0market.DealState{ + newDeals := map[abi.DealID]*market0.DealState{ abi.DealID(1): newDeal1, // deal 2 was removed abi.DealID(3): newDeal3, } // added - newProp3 := &v0market.DealProposal{ + newProp3 := &market0.DealProposal{ PieceCID: dummyCid, PieceSize: 0, VerifiedDeal: false, @@ -160,7 +160,7 @@ func TestMarketPredicates(t *testing.T) { ProviderCollateral: big.Zero(), ClientCollateral: big.Zero(), } - newProps := map[abi.DealID]*v0market.DealProposal{ + newProps := map[abi.DealID]*market0.DealProposal{ abi.DealID(1): oldProp1, // 1 was persisted // prop 2 was removed abi.DealID(3): newProp3, // new @@ -183,8 +183,8 @@ func TestMarketPredicates(t *testing.T) { require.NoError(t, err) api := newMockAPI(bs) - api.setActor(oldState.Key(), &types.Actor{Code: v0builtin.StorageMarketActorCodeID, Head: oldStateC}) - api.setActor(newState.Key(), &types.Actor{Code: v0builtin.StorageMarketActorCodeID, Head: newStateC}) + api.setActor(oldState.Key(), &types.Actor{Code: builtin0.StorageMarketActorCodeID, Head: oldStateC}) + api.setActor(newState.Key(), &types.Actor{Code: builtin0.StorageMarketActorCodeID, Head: newStateC}) t.Run("deal ID predicate", func(t *testing.T) { preds := NewStatePredicates(api) @@ -239,11 +239,11 @@ func TestMarketPredicates(t *testing.T) { t.Fatal("No state change so this should not be called") return false, nil, nil }) - v0marketState := createEmptyMarketState(t, store) - marketCid, err := store.Put(ctx, v0marketState) + marketState0 := createEmptyMarketState(t, store) + marketCid, err := store.Put(ctx, marketState0) require.NoError(t, err) marketState, err := market.Load(store, &types.Actor{ - Code: v0builtin.StorageMarketActorCodeID, + Code: builtin0.StorageMarketActorCodeID, Head: marketCid, }) require.NoError(t, err) @@ -352,11 +352,11 @@ func TestMarketPredicates(t *testing.T) { t.Fatal("No state change so this should not be called") return false, nil, nil }) - v0marketState := createEmptyMarketState(t, store) - marketCid, err := store.Put(ctx, v0marketState) + marketState0 := createEmptyMarketState(t, store) + marketCid, err := store.Put(ctx, marketState0) require.NoError(t, err) marketState, err := market.Load(store, &types.Actor{ - Code: v0builtin.StorageMarketActorCodeID, + Code: builtin0.StorageMarketActorCodeID, Head: marketCid, }) require.NoError(t, err) @@ -379,12 +379,12 @@ func TestMinerSectorChange(t *testing.T) { } owner, worker := nextIDAddrF(), nextIDAddrF() - si0 := newSectorOnChainInfo(0, tutils.MakeCID("0", &v0miner.SealedCIDPrefix), big.NewInt(0), abi.ChainEpoch(0), abi.ChainEpoch(10)) - si1 := newSectorOnChainInfo(1, tutils.MakeCID("1", &v0miner.SealedCIDPrefix), big.NewInt(1), abi.ChainEpoch(1), abi.ChainEpoch(11)) - si2 := newSectorOnChainInfo(2, tutils.MakeCID("2", &v0miner.SealedCIDPrefix), big.NewInt(2), abi.ChainEpoch(2), abi.ChainEpoch(11)) + si0 := newSectorOnChainInfo(0, tutils.MakeCID("0", &miner0.SealedCIDPrefix), big.NewInt(0), abi.ChainEpoch(0), abi.ChainEpoch(10)) + si1 := newSectorOnChainInfo(1, tutils.MakeCID("1", &miner0.SealedCIDPrefix), big.NewInt(1), abi.ChainEpoch(1), abi.ChainEpoch(11)) + si2 := newSectorOnChainInfo(2, tutils.MakeCID("2", &miner0.SealedCIDPrefix), big.NewInt(2), abi.ChainEpoch(2), abi.ChainEpoch(11)) oldMinerC := createMinerState(ctx, t, store, owner, worker, []miner.SectorOnChainInfo{si0, si1, si2}) - si3 := newSectorOnChainInfo(3, tutils.MakeCID("3", &v0miner.SealedCIDPrefix), big.NewInt(3), abi.ChainEpoch(3), abi.ChainEpoch(12)) + si3 := newSectorOnChainInfo(3, tutils.MakeCID("3", &miner0.SealedCIDPrefix), big.NewInt(3), abi.ChainEpoch(3), abi.ChainEpoch(12)) // 0 delete // 1 extend // 2 same @@ -400,8 +400,8 @@ func TestMinerSectorChange(t *testing.T) { require.NoError(t, err) api := newMockAPI(bs) - api.setActor(oldState.Key(), &types.Actor{Head: oldMinerC, Code: v0builtin.StorageMinerActorCodeID}) - api.setActor(newState.Key(), &types.Actor{Head: newMinerC, Code: v0builtin.StorageMinerActorCodeID}) + api.setActor(oldState.Key(), &types.Actor{Head: oldMinerC, Code: builtin0.StorageMinerActorCodeID}) + api.setActor(newState.Key(), &types.Actor{Head: newMinerC, Code: builtin0.StorageMinerActorCodeID}) preds := NewStatePredicates(api) @@ -467,7 +467,7 @@ type balance struct { locked abi.TokenAmount } -func createMarketState(ctx context.Context, t *testing.T, store adt.Store, deals map[abi.DealID]*v0market.DealState, props map[abi.DealID]*v0market.DealProposal, balances map[address.Address]balance) cid.Cid { +func createMarketState(ctx context.Context, t *testing.T, store adt.Store, deals map[abi.DealID]*market0.DealState, props map[abi.DealID]*market0.DealProposal, balances map[address.Address]balance) cid.Cid { dealRootCid := createDealAMT(ctx, t, store, deals) propRootCid := createProposalAMT(ctx, t, store, props) balancesCids := createBalanceTable(ctx, t, store, balances) @@ -482,15 +482,15 @@ func createMarketState(ctx context.Context, t *testing.T, store adt.Store, deals return stateC } -func createEmptyMarketState(t *testing.T, store adt.Store) *v0market.State { +func createEmptyMarketState(t *testing.T, store adt.Store) *market0.State { emptyArrayCid, err := adt.MakeEmptyArray(store).Root() require.NoError(t, err) emptyMap, err := adt.MakeEmptyMap(store).Root() require.NoError(t, err) - return v0market.ConstructState(emptyArrayCid, emptyMap, emptyMap) + return market0.ConstructState(emptyArrayCid, emptyMap, emptyMap) } -func createDealAMT(ctx context.Context, t *testing.T, store adt.Store, deals map[abi.DealID]*v0market.DealState) cid.Cid { +func createDealAMT(ctx context.Context, t *testing.T, store adt.Store, deals map[abi.DealID]*market0.DealState) cid.Cid { root := adt.MakeEmptyArray(store) for dealID, dealState := range deals { err := root.Set(uint64(dealID), dealState) @@ -501,7 +501,7 @@ func createDealAMT(ctx context.Context, t *testing.T, store adt.Store, deals map return rootCid } -func createProposalAMT(ctx context.Context, t *testing.T, store adt.Store, props map[abi.DealID]*v0market.DealProposal) cid.Cid { +func createProposalAMT(ctx context.Context, t *testing.T, store adt.Store, props map[abi.DealID]*market0.DealProposal) cid.Cid { root := adt.MakeEmptyArray(store) for dealID, prop := range props { err := root.Set(uint64(dealID), prop) @@ -549,20 +549,20 @@ func createMinerState(ctx context.Context, t *testing.T, store adt.Store, owner, return stateC } -func createEmptyMinerState(ctx context.Context, t *testing.T, store adt.Store, owner, worker address.Address) *v0miner.State { +func createEmptyMinerState(ctx context.Context, t *testing.T, store adt.Store, owner, worker address.Address) *miner0.State { emptyArrayCid, err := adt.MakeEmptyArray(store).Root() require.NoError(t, err) emptyMap, err := adt.MakeEmptyMap(store).Root() require.NoError(t, err) - emptyDeadline, err := store.Put(store.Context(), v0miner.ConstructDeadline(emptyArrayCid)) + emptyDeadline, err := store.Put(store.Context(), miner0.ConstructDeadline(emptyArrayCid)) require.NoError(t, err) - emptyVestingFunds := v0miner.ConstructVestingFunds() + emptyVestingFunds := miner0.ConstructVestingFunds() emptyVestingFundsCid, err := store.Put(store.Context(), emptyVestingFunds) require.NoError(t, err) - emptyDeadlines := v0miner.ConstructDeadlines(emptyDeadline) + emptyDeadlines := miner0.ConstructDeadlines(emptyDeadline) emptyDeadlinesCid, err := store.Put(store.Context(), emptyDeadlines) require.NoError(t, err) @@ -572,7 +572,7 @@ func createEmptyMinerState(ctx context.Context, t *testing.T, store adt.Store, o emptyBitfieldCid, err := store.Put(store.Context(), emptyBitfield) require.NoError(t, err) - state, err := v0miner.ConstructState(minerInfo, 123, emptyBitfieldCid, emptyArrayCid, emptyMap, emptyDeadlinesCid, emptyVestingFundsCid) + state, err := miner0.ConstructState(minerInfo, 123, emptyBitfieldCid, emptyArrayCid, emptyMap, emptyDeadlinesCid, emptyVestingFundsCid) require.NoError(t, err) return state @@ -625,7 +625,7 @@ func newSectorPreCommitInfo(sectorNo abi.SectorNumber, sealed cid.Cid, expiratio } } -func dealEquality(expected v0market.DealState, actual market.DealState) bool { +func dealEquality(expected market0.DealState, actual market.DealState) bool { return expected.LastUpdatedEpoch == actual.LastUpdatedEpoch && expected.SectorStartEpoch == actual.SectorStartEpoch && expected.SlashEpoch == actual.SlashEpoch diff --git a/chain/gen/gen.go b/chain/gen/gen.go index 4163f0b2d..c4ecf1d41 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -14,7 +14,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" - v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" block "github.com/ipfs/go-block-format" "github.com/ipfs/go-blockservice" "github.com/ipfs/go-cid" @@ -121,7 +121,7 @@ var DefaultRemainderAccountActor = genesis.Actor{ } func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) { - v0miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ + miner0.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } diff --git a/chain/gen/gen_test.go b/chain/gen/gen_test.go index 9d1262f77..be913f5f2 100644 --- a/chain/gen/gen_test.go +++ b/chain/gen/gen_test.go @@ -5,20 +5,20 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" - v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" - v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" - v0verifreg "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" + power0 "github.com/filecoin-project/specs-actors/actors/builtin/power" + verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" _ "github.com/filecoin-project/lotus/lib/sigs/bls" _ "github.com/filecoin-project/lotus/lib/sigs/secp" ) func init() { - v0miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ + miner0.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } - v0power.ConsensusMinerMinPower = big.NewInt(2048) - v0verifreg.MinVerifiedDealSize = big.NewInt(256) + power0.ConsensusMinerMinPower = big.NewInt(2048) + verifreg0.MinVerifiedDealSize = big.NewInt(256) } func testGeneration(t testing.TB, n int, msgs int, sectors int) { diff --git a/chain/gen/genesis/genesis.go b/chain/gen/genesis/genesis.go index be2ed70aa..039e284cd 100644 --- a/chain/gen/genesis/genesis.go +++ b/chain/gen/genesis/genesis.go @@ -17,11 +17,11 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" - v0builtin "github.com/filecoin-project/specs-actors/actors/builtin" - v0account "github.com/filecoin-project/specs-actors/actors/builtin/account" - v0multisig "github.com/filecoin-project/specs-actors/actors/builtin/multisig" - v0verifreg "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" - v0adt "github.com/filecoin-project/specs-actors/actors/util/adt" + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" + account0 "github.com/filecoin-project/specs-actors/actors/builtin/account" + multisig0 "github.com/filecoin-project/specs-actors/actors/builtin/multisig" + verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" + adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors/builtin" @@ -126,7 +126,7 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge if err != nil { return nil, nil, xerrors.Errorf("setup init actor: %w", err) } - if err := state.SetActor(v0builtin.SystemActorAddr, sysact); err != nil { + if err := state.SetActor(builtin0.SystemActorAddr, sysact); err != nil { return nil, nil, xerrors.Errorf("set init actor: %w", err) } @@ -136,7 +136,7 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge if err != nil { return nil, nil, xerrors.Errorf("setup init actor: %w", err) } - if err := state.SetActor(v0builtin.InitActorAddr, initact); err != nil { + if err := state.SetActor(builtin0.InitActorAddr, initact); err != nil { return nil, nil, xerrors.Errorf("set init actor: %w", err) } @@ -147,7 +147,7 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge return nil, nil, xerrors.Errorf("setup init actor: %w", err) } - err = state.SetActor(v0builtin.RewardActorAddr, rewact) + err = state.SetActor(builtin0.RewardActorAddr, rewact) if err != nil { return nil, nil, xerrors.Errorf("set network account actor: %w", err) } @@ -157,7 +157,7 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge if err != nil { return nil, nil, xerrors.Errorf("setup cron actor: %w", err) } - if err := state.SetActor(v0builtin.CronActorAddr, cronact); err != nil { + if err := state.SetActor(builtin0.CronActorAddr, cronact); err != nil { return nil, nil, xerrors.Errorf("set cron actor: %w", err) } @@ -166,7 +166,7 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge if err != nil { return nil, nil, xerrors.Errorf("setup storage market actor: %w", err) } - if err := state.SetActor(v0builtin.StoragePowerActorAddr, spact); err != nil { + if err := state.SetActor(builtin0.StoragePowerActorAddr, spact); err != nil { return nil, nil, xerrors.Errorf("set storage market actor: %w", err) } @@ -175,7 +175,7 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge if err != nil { return nil, nil, xerrors.Errorf("setup storage market actor: %w", err) } - if err := state.SetActor(v0builtin.StorageMarketActorAddr, marketact); err != nil { + if err := state.SetActor(builtin0.StorageMarketActorAddr, marketact); err != nil { return nil, nil, xerrors.Errorf("set market actor: %w", err) } @@ -184,20 +184,20 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge if err != nil { return nil, nil, xerrors.Errorf("setup storage market actor: %w", err) } - if err := state.SetActor(v0builtin.VerifiedRegistryActorAddr, verifact); err != nil { + if err := state.SetActor(builtin0.VerifiedRegistryActorAddr, verifact); err != nil { return nil, nil, xerrors.Errorf("set market actor: %w", err) } - burntRoot, err := cst.Put(ctx, &v0account.State{ - Address: v0builtin.BurntFundsActorAddr, + burntRoot, err := cst.Put(ctx, &account0.State{ + Address: builtin0.BurntFundsActorAddr, }) if err != nil { return nil, nil, xerrors.Errorf("failed to setup burnt funds actor state: %w", err) } // Setup burnt-funds - err = state.SetActor(v0builtin.BurntFundsActorAddr, &types.Actor{ - Code: v0builtin.AccountActorCodeID, + err = state.SetActor(builtin0.BurntFundsActorAddr, &types.Actor{ + Code: builtin0.AccountActorCodeID, Balance: types.NewInt(0), Head: burntRoot, }) @@ -262,13 +262,13 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge return nil, nil, err } - verifierState, err := cst.Put(ctx, &v0account.State{Address: verifierAd}) + verifierState, err := cst.Put(ctx, &account0.State{Address: verifierAd}) if err != nil { return nil, nil, err } err = state.SetActor(verifierId, &types.Actor{ - Code: v0builtin.AccountActorCodeID, + Code: builtin0.AccountActorCodeID, Balance: types.NewInt(0), Head: verifierState, }) @@ -315,7 +315,7 @@ func createAccountActor(ctx context.Context, cst cbor.IpldStore, state *state.St if err := json.Unmarshal(info.Meta, &ainfo); err != nil { return xerrors.Errorf("unmarshaling account meta: %w", err) } - st, err := cst.Put(ctx, &v0account.State{Address: ainfo.Owner}) + st, err := cst.Put(ctx, &account0.State{Address: ainfo.Owner}) if err != nil { return err } @@ -326,7 +326,7 @@ func createAccountActor(ctx context.Context, cst cbor.IpldStore, state *state.St } err = state.SetActor(ida, &types.Actor{ - Code: v0builtin.AccountActorCodeID, + Code: builtin0.AccountActorCodeID, Balance: info.Balance, Head: st, }) @@ -344,7 +344,7 @@ func createMultisigAccount(ctx context.Context, bs bstore.Blockstore, cst cbor.I if err := json.Unmarshal(info.Meta, &ainfo); err != nil { return xerrors.Errorf("unmarshaling account meta: %w", err) } - pending, err := v0adt.MakeEmptyMap(v0adt.WrapStore(ctx, cst)).Root() + pending, err := adt0.MakeEmptyMap(adt0.WrapStore(ctx, cst)).Root() if err != nil { return xerrors.Errorf("failed to create empty map: %v", err) } @@ -364,12 +364,12 @@ func createMultisigAccount(ctx context.Context, bs bstore.Blockstore, cst cbor.I continue } - st, err := cst.Put(ctx, &v0account.State{Address: e}) + st, err := cst.Put(ctx, &account0.State{Address: e}) if err != nil { return err } err = state.SetActor(idAddress, &types.Actor{ - Code: v0builtin.AccountActorCodeID, + Code: builtin0.AccountActorCodeID, Balance: types.NewInt(0), Head: st, }) @@ -379,7 +379,7 @@ func createMultisigAccount(ctx context.Context, bs bstore.Blockstore, cst cbor.I signers = append(signers, idAddress) } - st, err := cst.Put(ctx, &v0multisig.State{ + st, err := cst.Put(ctx, &multisig0.State{ Signers: signers, NumApprovalsThreshold: uint64(ainfo.Threshold), StartEpoch: abi.ChainEpoch(ainfo.VestingStart), @@ -391,7 +391,7 @@ func createMultisigAccount(ctx context.Context, bs bstore.Blockstore, cst cbor.I return err } err = state.SetActor(ida, &types.Actor{ - Code: v0builtin.MultisigActorCodeID, + Code: builtin0.MultisigActorCodeID, Balance: info.Balance, Head: st, }) @@ -442,7 +442,7 @@ func VerifyPreSealedData(ctx context.Context, cs *store.ChainStore, stateroot ci return cid.Undef, err } - _, err = doExecValue(ctx, vm, v0builtin.VerifiedRegistryActorAddr, verifregRoot, types.NewInt(0), v0builtin.MethodsVerifiedRegistry.AddVerifier, mustEnc(&v0verifreg.AddVerifierParams{ + _, err = doExecValue(ctx, vm, builtin0.VerifiedRegistryActorAddr, verifregRoot, types.NewInt(0), builtin0.MethodsVerifiedRegistry.AddVerifier, mustEnc(&verifreg0.AddVerifierParams{ Address: verifier, Allowance: abi.NewStoragePower(int64(sum)), // eh, close enough @@ -453,7 +453,7 @@ func VerifyPreSealedData(ctx context.Context, cs *store.ChainStore, stateroot ci } for c, amt := range verifNeeds { - _, err := doExecValue(ctx, vm, v0builtin.VerifiedRegistryActorAddr, verifier, types.NewInt(0), v0builtin.MethodsVerifiedRegistry.AddVerifiedClient, mustEnc(&v0verifreg.AddVerifiedClientParams{ + _, err := doExecValue(ctx, vm, builtin0.VerifiedRegistryActorAddr, verifier, types.NewInt(0), builtin0.MethodsVerifiedRegistry.AddVerifiedClient, mustEnc(&verifreg0.AddVerifiedClientParams{ Address: c, Allowance: abi.NewStoragePower(int64(amt)), })) @@ -495,8 +495,8 @@ func MakeGenesisBlock(ctx context.Context, bs bstore.Blockstore, sys vm.SyscallB return nil, xerrors.Errorf("setup miners failed: %w", err) } - store := v0adt.WrapStore(ctx, cbor.NewCborStore(bs)) - emptyroot, err := v0adt.MakeEmptyArray(store).Root() + store := adt0.WrapStore(ctx, cbor.NewCborStore(bs)) + emptyroot, err := adt0.MakeEmptyArray(store).Root() if err != nil { return nil, xerrors.Errorf("amt build failed: %w", err) } @@ -544,7 +544,7 @@ func MakeGenesisBlock(ctx context.Context, bs bstore.Blockstore, sys vm.SyscallB } b := &types.BlockHeader{ - Miner: v0builtin.SystemActorAddr, + Miner: builtin0.SystemActorAddr, Ticket: genesisticket, Parents: []cid.Cid{filecoinGenesisCid}, Height: 0, diff --git a/chain/gen/genesis/miners.go b/chain/gen/genesis/miners.go index aff226882..853c9c4a0 100644 --- a/chain/gen/genesis/miners.go +++ b/chain/gen/genesis/miners.go @@ -20,9 +20,9 @@ import ( "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/builtin/market" - v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" - v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" - v0reward "github.com/filecoin-project/specs-actors/actors/builtin/reward" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" + power0 "github.com/filecoin-project/specs-actors/actors/builtin/power" + reward0 "github.com/filecoin-project/specs-actors/actors/builtin/reward" "github.com/filecoin-project/specs-actors/actors/runtime" "github.com/filecoin-project/lotus/chain/state" @@ -101,7 +101,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid } { - constructorParams := &v0power.CreateMinerParams{ + constructorParams := &power0.CreateMinerParams{ Owner: m.Worker, Worker: m.Worker, Peer: []byte(m.PeerId), @@ -114,7 +114,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid return cid.Undef, xerrors.Errorf("failed to create genesis miner: %w", err) } - var ma v0power.CreateMinerReturn + var ma power0.CreateMinerReturn if err := ma.UnmarshalCBOR(bytes.NewReader(rval)); err != nil { return cid.Undef, xerrors.Errorf("unmarshaling CreateMinerReturn: %w", err) } @@ -126,9 +126,9 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid minerInfos[i].maddr = ma.IDAddress // TODO: ActorUpgrade - err = vm.MutateState(ctx, minerInfos[i].maddr, func(cst cbor.IpldStore, st *v0miner.State) error { - maxPeriods := v0miner.MaxSectorExpirationExtension / v0miner.WPoStProvingPeriod - minerInfos[i].presealExp = (maxPeriods-1)*v0miner.WPoStProvingPeriod + st.ProvingPeriodStart - 1 + err = vm.MutateState(ctx, minerInfos[i].maddr, func(cst cbor.IpldStore, st *miner0.State) error { + maxPeriods := miner0.MaxSectorExpirationExtension / miner0.WPoStProvingPeriod + minerInfos[i].presealExp = (maxPeriods-1)*miner0.WPoStProvingPeriod + st.ProvingPeriodStart - 1 return nil }) @@ -204,13 +204,13 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid return cid.Undef, xerrors.Errorf("getting deal weight: %w", err) } - sectorWeight := v0miner.QAPowerForWeight(m.SectorSize, minerInfos[i].presealExp, dweight.DealWeight, dweight.VerifiedDealWeight) + sectorWeight := miner0.QAPowerForWeight(m.SectorSize, minerInfos[i].presealExp, dweight.DealWeight, dweight.VerifiedDealWeight) qaPow = types.BigAdd(qaPow, sectorWeight) } } - err = vm.MutateState(ctx, builtin.StoragePowerActorAddr, func(cst cbor.IpldStore, st *v0power.State) error { + err = vm.MutateState(ctx, builtin.StoragePowerActorAddr, func(cst cbor.IpldStore, st *power0.State) error { st.TotalQualityAdjPower = qaPow st.TotalRawBytePower = rawPow @@ -222,8 +222,8 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid return cid.Undef, xerrors.Errorf("mutating state: %w", err) } - err = vm.MutateState(ctx, builtin.RewardActorAddr, func(sct cbor.IpldStore, st *v0reward.State) error { - *st = *v0reward.ConstructState(qaPow) + err = vm.MutateState(ctx, builtin.RewardActorAddr, func(sct cbor.IpldStore, st *reward0.State) error { + *st = *reward0.ConstructState(qaPow) return nil }) if err != nil { @@ -249,10 +249,10 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid return cid.Undef, xerrors.Errorf("getting deal weight: %w", err) } - sectorWeight := v0miner.QAPowerForWeight(m.SectorSize, minerInfos[i].presealExp, dweight.DealWeight, dweight.VerifiedDealWeight) + sectorWeight := miner0.QAPowerForWeight(m.SectorSize, minerInfos[i].presealExp, dweight.DealWeight, dweight.VerifiedDealWeight) // we've added fake power for this sector above, remove it now - err = vm.MutateState(ctx, builtin.StoragePowerActorAddr, func(cst cbor.IpldStore, st *v0power.State) error { + err = vm.MutateState(ctx, builtin.StoragePowerActorAddr, func(cst cbor.IpldStore, st *power0.State) error { st.TotalQualityAdjPower = types.BigSub(st.TotalQualityAdjPower, sectorWeight) //nolint:scopelint st.TotalRawBytePower = types.BigSub(st.TotalRawBytePower, types.NewInt(uint64(m.SectorSize))) return nil @@ -271,9 +271,9 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid return cid.Undef, xerrors.Errorf("getting current total power: %w", err) } - pcd := v0miner.PreCommitDepositForPower(epochReward.ThisEpochRewardSmoothed, tpow.QualityAdjPowerSmoothed, sectorWeight) + pcd := miner0.PreCommitDepositForPower(epochReward.ThisEpochRewardSmoothed, tpow.QualityAdjPowerSmoothed, sectorWeight) - pledge := v0miner.InitialPledgeForPower( + pledge := miner0.InitialPledgeForPower( sectorWeight, epochReward.ThisEpochBaselinePower, tpow.PledgeCollateral, @@ -304,7 +304,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid } // Sanity-check total network power - err = vm.MutateState(ctx, builtin.StoragePowerActorAddr, func(cst cbor.IpldStore, st *v0power.State) error { + err = vm.MutateState(ctx, builtin.StoragePowerActorAddr, func(cst cbor.IpldStore, st *power0.State) error { if !st.TotalRawBytePower.Equals(rawPow) { return xerrors.Errorf("st.TotalRawBytePower doesn't match previously calculated rawPow") } @@ -343,12 +343,12 @@ func (fr *fakeRand) GetBeaconRandomness(ctx context.Context, personalization cry return out, nil } -func currentTotalPower(ctx context.Context, vm *vm.VM, maddr address.Address) (*v0power.CurrentTotalPowerReturn, error) { +func currentTotalPower(ctx context.Context, vm *vm.VM, maddr address.Address) (*power0.CurrentTotalPowerReturn, error) { pwret, err := doExecValue(ctx, vm, builtin.StoragePowerActorAddr, maddr, big.Zero(), builtin.MethodsPower.CurrentTotalPower, nil) if err != nil { return nil, err } - var pwr v0power.CurrentTotalPowerReturn + var pwr power0.CurrentTotalPowerReturn if err := pwr.UnmarshalCBOR(bytes.NewReader(pwret)); err != nil { return nil, err } @@ -381,13 +381,13 @@ func dealWeight(ctx context.Context, vm *vm.VM, maddr address.Address, dealIDs [ return dealWeights, nil } -func currentEpochBlockReward(ctx context.Context, vm *vm.VM, maddr address.Address) (*v0reward.ThisEpochRewardReturn, error) { +func currentEpochBlockReward(ctx context.Context, vm *vm.VM, maddr address.Address) (*reward0.ThisEpochRewardReturn, error) { rwret, err := doExecValue(ctx, vm, builtin.RewardActorAddr, maddr, big.Zero(), builtin.MethodsReward.ThisEpochReward, nil) if err != nil { return nil, err } - var epochReward v0reward.ThisEpochRewardReturn + var epochReward reward0.ThisEpochRewardReturn if err := epochReward.UnmarshalCBOR(bytes.NewReader(rwret)); err != nil { return nil, err } diff --git a/chain/gen/genesis/t02_reward.go b/chain/gen/genesis/t02_reward.go index e29e390f9..92531051b 100644 --- a/chain/gen/genesis/t02_reward.go +++ b/chain/gen/genesis/t02_reward.go @@ -6,7 +6,7 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/specs-actors/actors/builtin" - v0reward "github.com/filecoin-project/specs-actors/actors/builtin/reward" + reward0 "github.com/filecoin-project/specs-actors/actors/builtin/reward" cbor "github.com/ipfs/go-ipld-cbor" "github.com/filecoin-project/lotus/build" @@ -17,7 +17,7 @@ import ( func SetupRewardActor(bs bstore.Blockstore, qaPower big.Int) (*types.Actor, error) { cst := cbor.NewCborStore(bs) - st := v0reward.ConstructState(qaPower) + st := reward0.ConstructState(qaPower) hcid, err := cst.Put(context.TODO(), st) if err != nil { diff --git a/chain/gen/genesis/t04_power.go b/chain/gen/genesis/t04_power.go index 40ea68079..2f1303ba4 100644 --- a/chain/gen/genesis/t04_power.go +++ b/chain/gen/genesis/t04_power.go @@ -6,7 +6,7 @@ import ( "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/util/adt" - v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" + power0 "github.com/filecoin-project/specs-actors/actors/builtin/power" cbor "github.com/ipfs/go-ipld-cbor" "github.com/filecoin-project/lotus/chain/types" @@ -30,7 +30,7 @@ func SetupStoragePowerActor(bs bstore.Blockstore) (*types.Actor, error) { return nil, err } - sms := v0power.ConstructState(emptyMap, emptyMultiMap) + sms := power0.ConstructState(emptyMap, emptyMultiMap) stcid, err := store.Put(store.Context(), sms) if err != nil { diff --git a/chain/gen/genesis/t06_vreg.go b/chain/gen/genesis/t06_vreg.go index d91cdf7b1..1709b205f 100644 --- a/chain/gen/genesis/t06_vreg.go +++ b/chain/gen/genesis/t06_vreg.go @@ -7,7 +7,7 @@ import ( cbor "github.com/ipfs/go-ipld-cbor" "github.com/filecoin-project/specs-actors/actors/builtin" - v0verifreg "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" + verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/lotus/chain/types" @@ -34,7 +34,7 @@ func SetupVerifiedRegistryActor(bs bstore.Blockstore) (*types.Actor, error) { return nil, err } - sms := v0verifreg.ConstructState(h, RootVerifierID) + sms := verifreg0.ConstructState(h, RootVerifierID) stcid, err := store.Put(store.Context(), sms) if err != nil { diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index 650536718..bb496849b 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -9,8 +9,8 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/specs-actors/actors/builtin" - v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" - v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" + power0 "github.com/filecoin-project/specs-actors/actors/builtin/power" "github.com/filecoin-project/specs-actors/actors/util/adt" cbor "github.com/ipfs/go-ipld-cbor" "golang.org/x/xerrors" @@ -135,7 +135,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, tree types }) } case builtin.StorageMinerActorCodeID: - var st v0miner.State + var st miner0.State if err := sm.ChainStore().Store(ctx).Get(ctx, act.Head, &st); err != nil { return xerrors.Errorf("failed to load miner state: %w", err) } @@ -172,7 +172,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, tree types } // pull up power table to give miners back some funds proportional to their power - var ps v0power.State + var ps power0.State powAct, err := tree.GetActor(builtin.StoragePowerActorAddr) if err != nil { return xerrors.Errorf("failed to load power actor: %w", err) @@ -211,12 +211,12 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, tree types }) } case builtin.StorageMinerActorCodeID: - var st v0miner.State + var st miner0.State if err := sm.ChainStore().Store(ctx).Get(ctx, act.Head, &st); err != nil { return xerrors.Errorf("failed to load miner state: %w", err) } - var minfo v0miner.MinerInfo + var minfo miner0.MinerInfo if err := cst.Get(ctx, st.Info, &minfo); err != nil { return xerrors.Errorf("failed to get miner info: %w", err) } @@ -240,7 +240,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, tree types // Now make sure to give each miner who had power at the lookback some FIL lbact, err := lbtree.GetActor(addr) if err == nil { - var lbst v0miner.State + var lbst miner0.State if err := sm.ChainStore().Store(ctx).Get(ctx, lbact.Head, &lbst); err != nil { return xerrors.Errorf("failed to load miner state: %w", err) } diff --git a/chain/stmgr/forks_test.go b/chain/stmgr/forks_test.go index c4fb1b3be..87d328ce1 100644 --- a/chain/stmgr/forks_test.go +++ b/chain/stmgr/forks_test.go @@ -11,9 +11,9 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/specs-actors/actors/builtin" init_ "github.com/filecoin-project/specs-actors/actors/builtin/init" - v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" - v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" - v0verifreg "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" + power0 "github.com/filecoin-project/specs-actors/actors/builtin/power" + verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" "github.com/filecoin-project/specs-actors/actors/runtime" "golang.org/x/xerrors" @@ -33,11 +33,11 @@ import ( ) func init() { - v0miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ + miner0.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } - v0power.ConsensusMinerMinPower = big.NewInt(2048) - v0verifreg.MinVerifiedDealSize = big.NewInt(256) + power0.ConsensusMinerMinPower = big.NewInt(2048) + verifreg0.MinVerifiedDealSize = big.NewInt(256) } const testForkHeight = 40 diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index d183b7ce2..e6e557930 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -5,7 +5,7 @@ import ( "fmt" "sync" - v0msig "github.com/filecoin-project/specs-actors/actors/builtin/multisig" + msig0 "github.com/filecoin-project/specs-actors/actors/builtin/multisig" "github.com/filecoin-project/lotus/chain/actors/builtin/multisig" @@ -23,7 +23,7 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/specs-actors/actors/builtin" - v0reward "github.com/filecoin-project/specs-actors/actors/builtin/reward" + reward0 "github.com/filecoin-project/specs-actors/actors/builtin/reward" "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/lotus/api" @@ -251,7 +251,7 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp nv := sm.GetNtwkVersion(ctx, epoch) if nv < build.ActorUpgradeNetworkVersion { - params, err = actors.SerializeParams(&v0reward.AwardBlockRewardParams{ + params, err = actors.SerializeParams(&reward0.AwardBlockRewardParams{ Miner: b.Miner, Penalty: penalty, GasReward: gasReward, @@ -797,7 +797,7 @@ func (sm *StateManager) SetVMConstructor(nvm func(context.Context, *vm.VMOpts) ( } type genesisInfo struct { - genesisMsigs []v0msig.State + genesisMsigs []msig0.State // info about the Accounts in the genesis state genesisActors []genesisActor genesisPledge abi.TokenAmount @@ -898,9 +898,9 @@ func (sm *StateManager) setupGenesisActors(ctx context.Context) error { return xerrors.Errorf("error setting up genesis infos: %w", err) } - gi.genesisMsigs = make([]v0msig.State, 0, len(totalsByEpoch)) + gi.genesisMsigs = make([]msig0.State, 0, len(totalsByEpoch)) for k, v := range totalsByEpoch { - ns := v0msig.State{ + ns := msig0.State{ InitialBalance: v, UnlockDuration: k, PendingTxns: cid.Undef, @@ -975,9 +975,9 @@ func (sm *StateManager) setupGenesisActorsTestnet(ctx context.Context) error { totalsByEpoch[sixYears] = big.NewInt(100_000_000) totalsByEpoch[sixYears] = big.Add(totalsByEpoch[sixYears], big.NewInt(300_000_000)) - gi.genesisMsigs = make([]v0msig.State, 0, len(totalsByEpoch)) + gi.genesisMsigs = make([]msig0.State, 0, len(totalsByEpoch)) for k, v := range totalsByEpoch { - ns := v0msig.State{ + ns := msig0.State{ InitialBalance: v, UnlockDuration: k, PendingTxns: cid.Undef, diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index 8c55fc1f2..3684b9e77 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -9,11 +9,11 @@ import ( "runtime" "strings" - v0init "github.com/filecoin-project/specs-actors/actors/builtin/init" - v0market "github.com/filecoin-project/specs-actors/actors/builtin/market" - v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" + init0 "github.com/filecoin-project/specs-actors/actors/builtin/init" + market0 "github.com/filecoin-project/specs-actors/actors/builtin/market" + power0 "github.com/filecoin-project/specs-actors/actors/builtin/power" - v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" saruntime "github.com/filecoin-project/specs-actors/actors/runtime" "github.com/filecoin-project/specs-actors/actors/runtime/proof" @@ -31,10 +31,10 @@ import ( "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/builtin/account" "github.com/filecoin-project/specs-actors/actors/builtin/cron" - v0msig "github.com/filecoin-project/specs-actors/actors/builtin/multisig" + msig0 "github.com/filecoin-project/specs-actors/actors/builtin/multisig" "github.com/filecoin-project/specs-actors/actors/builtin/paych" - v0reward "github.com/filecoin-project/specs-actors/actors/builtin/reward" - v0verifreg "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" + reward0 "github.com/filecoin-project/specs-actors/actors/builtin/reward" + verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" @@ -169,7 +169,7 @@ func GetMinerSectorSet(ctx context.Context, sm *StateManager, ts *types.TipSet, var sset []*miner.ChainSectorInfo var v cbg.Deferred if err := sectors.ForEach(&v, func(i int64) error { - var oci v0miner.SectorOnChainInfo + var oci miner0.SectorOnChainInfo if err := oci.UnmarshalCBOR(bytes.NewReader(v.Raw)); err != nil { return err } @@ -555,16 +555,16 @@ var MethodsMap = map[cid.Cid]map[abi.MethodNum]MethodMeta{} func init() { cidToMethods := map[cid.Cid][2]interface{}{ // builtin.SystemActorCodeID: {builtin.MethodsSystem, system.Actor{} }- apparently it doesn't have methods - builtin.InitActorCodeID: {builtin.MethodsInit, v0init.Actor{}}, + builtin.InitActorCodeID: {builtin.MethodsInit, init0.Actor{}}, builtin.CronActorCodeID: {builtin.MethodsCron, cron.Actor{}}, builtin.AccountActorCodeID: {builtin.MethodsAccount, account.Actor{}}, - builtin.StoragePowerActorCodeID: {builtin.MethodsPower, v0power.Actor{}}, - builtin.StorageMinerActorCodeID: {builtin.MethodsMiner, v0miner.Actor{}}, - builtin.StorageMarketActorCodeID: {builtin.MethodsMarket, v0market.Actor{}}, + builtin.StoragePowerActorCodeID: {builtin.MethodsPower, power0.Actor{}}, + builtin.StorageMinerActorCodeID: {builtin.MethodsMiner, miner0.Actor{}}, + builtin.StorageMarketActorCodeID: {builtin.MethodsMarket, market0.Actor{}}, builtin.PaymentChannelActorCodeID: {builtin.MethodsPaych, paych.Actor{}}, - builtin.MultisigActorCodeID: {builtin.MethodsMultisig, v0msig.Actor{}}, - builtin.RewardActorCodeID: {builtin.MethodsReward, v0reward.Actor{}}, - builtin.VerifiedRegistryActorCodeID: {builtin.MethodsVerifiedRegistry, v0verifreg.Actor{}}, + builtin.MultisigActorCodeID: {builtin.MethodsMultisig, msig0.Actor{}}, + builtin.RewardActorCodeID: {builtin.MethodsReward, reward0.Actor{}}, + builtin.VerifiedRegistryActorCodeID: {builtin.MethodsVerifiedRegistry, verifreg0.Actor{}}, } for c, m := range cidToMethods { diff --git a/chain/store/store_test.go b/chain/store/store_test.go index 83238a67d..d5e092559 100644 --- a/chain/store/store_test.go +++ b/chain/store/store_test.go @@ -10,9 +10,9 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" - v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" - v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" - v0verifreg "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" + power0 "github.com/filecoin-project/specs-actors/actors/builtin/power" + verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/store" @@ -22,11 +22,11 @@ import ( ) func init() { - v0miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ + miner0.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } - v0power.ConsensusMinerMinPower = big.NewInt(2048) - v0verifreg.MinVerifiedDealSize = big.NewInt(256) + power0.ConsensusMinerMinPower = big.NewInt(2048) + verifreg0.MinVerifiedDealSize = big.NewInt(256) } func BenchmarkGetRandomness(b *testing.B) { diff --git a/chain/sync_test.go b/chain/sync_test.go index 2baa5d8a4..53001caf8 100644 --- a/chain/sync_test.go +++ b/chain/sync_test.go @@ -20,9 +20,9 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" - v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" - v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" - v0verifreg "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" + power0 "github.com/filecoin-project/specs-actors/actors/builtin/power" + verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" @@ -43,11 +43,11 @@ func init() { if err != nil { panic(err) } - v0miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ + miner0.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } - v0power.ConsensusMinerMinPower = big.NewInt(2048) - v0verifreg.MinVerifiedDealSize = big.NewInt(256) + power0.ConsensusMinerMinPower = big.NewInt(2048) + verifreg0.MinVerifiedDealSize = big.NewInt(256) } const source = 0 diff --git a/chain/vectors/gen/main.go b/chain/vectors/gen/main.go index 36b770f03..2b1c6f340 100644 --- a/chain/vectors/gen/main.go +++ b/chain/vectors/gen/main.go @@ -6,7 +6,7 @@ import ( "math/rand" "os" - v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" + power0 "github.com/filecoin-project/specs-actors/actors/builtin/power" "github.com/filecoin-project/go-address" "golang.org/x/xerrors" @@ -19,15 +19,15 @@ import ( "github.com/filecoin-project/lotus/chain/types/mock" "github.com/filecoin-project/lotus/chain/vectors" "github.com/filecoin-project/lotus/chain/wallet" - v0verifreg "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" + verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" _ "github.com/filecoin-project/lotus/lib/sigs/bls" _ "github.com/filecoin-project/lotus/lib/sigs/secp" ) func init() { - v0verifreg.MinVerifiedDealSize = big.NewInt(2048) - v0power.ConsensusMinerMinPower = big.NewInt(2048) + verifreg0.MinVerifiedDealSize = big.NewInt(2048) + power0.ConsensusMinerMinPower = big.NewInt(2048) } func MakeHeaderVectors() []vectors.HeaderVector { diff --git a/chain/vm/invoker.go b/chain/vm/invoker.go index 501b933e2..0a83e273d 100644 --- a/chain/vm/invoker.go +++ b/chain/vm/invoker.go @@ -10,18 +10,18 @@ import ( cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" - v0builtin "github.com/filecoin-project/specs-actors/actors/builtin" - v0account "github.com/filecoin-project/specs-actors/actors/builtin/account" - v0cron "github.com/filecoin-project/specs-actors/actors/builtin/cron" - v0init "github.com/filecoin-project/specs-actors/actors/builtin/init" - v0market "github.com/filecoin-project/specs-actors/actors/builtin/market" - v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" - v0msig "github.com/filecoin-project/specs-actors/actors/builtin/multisig" - v0paych "github.com/filecoin-project/specs-actors/actors/builtin/paych" - v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" - v0reward "github.com/filecoin-project/specs-actors/actors/builtin/reward" - v0system "github.com/filecoin-project/specs-actors/actors/builtin/system" - v0verifreg "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" + account0 "github.com/filecoin-project/specs-actors/actors/builtin/account" + cron0 "github.com/filecoin-project/specs-actors/actors/builtin/cron" + init0 "github.com/filecoin-project/specs-actors/actors/builtin/init" + market0 "github.com/filecoin-project/specs-actors/actors/builtin/market" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" + msig0 "github.com/filecoin-project/specs-actors/actors/builtin/multisig" + paych0 "github.com/filecoin-project/specs-actors/actors/builtin/paych" + power0 "github.com/filecoin-project/specs-actors/actors/builtin/power" + reward0 "github.com/filecoin-project/specs-actors/actors/builtin/reward" + system0 "github.com/filecoin-project/specs-actors/actors/builtin/system" + verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" vmr "github.com/filecoin-project/specs-actors/actors/runtime" "github.com/filecoin-project/go-state-types/abi" @@ -46,17 +46,17 @@ func NewInvoker() *Invoker { // add builtInCode using: register(cid, singleton) // NETUPGRADE: register code IDs for v2, etc. - inv.Register(v0builtin.SystemActorCodeID, v0system.Actor{}, abi.EmptyValue{}) - inv.Register(v0builtin.InitActorCodeID, v0init.Actor{}, v0init.State{}) - inv.Register(v0builtin.RewardActorCodeID, v0reward.Actor{}, v0reward.State{}) - inv.Register(v0builtin.CronActorCodeID, v0cron.Actor{}, v0cron.State{}) - inv.Register(v0builtin.StoragePowerActorCodeID, v0power.Actor{}, v0power.State{}) - inv.Register(v0builtin.StorageMarketActorCodeID, v0market.Actor{}, v0market.State{}) - inv.Register(v0builtin.StorageMinerActorCodeID, v0miner.Actor{}, v0miner.State{}) - inv.Register(v0builtin.MultisigActorCodeID, v0msig.Actor{}, v0msig.State{}) - inv.Register(v0builtin.PaymentChannelActorCodeID, v0paych.Actor{}, v0paych.State{}) - inv.Register(v0builtin.VerifiedRegistryActorCodeID, v0verifreg.Actor{}, v0verifreg.State{}) - inv.Register(v0builtin.AccountActorCodeID, v0account.Actor{}, v0account.State{}) + inv.Register(builtin0.SystemActorCodeID, system0.Actor{}, abi.EmptyValue{}) + inv.Register(builtin0.InitActorCodeID, init0.Actor{}, init0.State{}) + inv.Register(builtin0.RewardActorCodeID, reward0.Actor{}, reward0.State{}) + inv.Register(builtin0.CronActorCodeID, cron0.Actor{}, cron0.State{}) + inv.Register(builtin0.StoragePowerActorCodeID, power0.Actor{}, power0.State{}) + inv.Register(builtin0.StorageMarketActorCodeID, market0.Actor{}, market0.State{}) + inv.Register(builtin0.StorageMinerActorCodeID, miner0.Actor{}, miner0.State{}) + inv.Register(builtin0.MultisigActorCodeID, msig0.Actor{}, msig0.State{}) + inv.Register(builtin0.PaymentChannelActorCodeID, paych0.Actor{}, paych0.State{}) + inv.Register(builtin0.VerifiedRegistryActorCodeID, verifreg0.Actor{}, verifreg0.State{}) + inv.Register(builtin0.AccountActorCodeID, account0.Actor{}, account0.State{}) return inv } @@ -177,7 +177,7 @@ func DecodeParams(b []byte, out interface{}) error { } func DumpActorState(code cid.Cid, b []byte) (interface{}, error) { - if code == v0builtin.AccountActorCodeID { // Account code special case + if code == builtin0.AccountActorCodeID { // Account code special case return nil, nil } diff --git a/cli/multisig.go b/cli/multisig.go index ffbbd733b..23095bf06 100644 --- a/cli/multisig.go +++ b/cli/multisig.go @@ -17,7 +17,7 @@ import ( "github.com/filecoin-project/go-address" init_ "github.com/filecoin-project/specs-actors/actors/builtin/init" - v0msig "github.com/filecoin-project/specs-actors/actors/builtin/multisig" + msig0 "github.com/filecoin-project/specs-actors/actors/builtin/multisig" cid "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" "github.com/urfave/cli/v2" @@ -199,7 +199,7 @@ var msigInspectCmd = &cli.Command{ return err } - var mstate v0msig.State + var mstate msig0.State if err := mstate.UnmarshalCBOR(bytes.NewReader(obj)); err != nil { return err } @@ -251,7 +251,7 @@ var msigInspectCmd = &cli.Command{ }, } -func GetMultisigPending(ctx context.Context, lapi api.FullNode, hroot cid.Cid) (map[int64]*v0msig.Transaction, error) { +func GetMultisigPending(ctx context.Context, lapi api.FullNode, hroot cid.Cid) (map[int64]*msig0.Transaction, error) { bs := apibstore.NewAPIBlockstore(lapi) store := adt.WrapStore(ctx, cbor.NewCborStore(bs)) @@ -260,8 +260,8 @@ func GetMultisigPending(ctx context.Context, lapi api.FullNode, hroot cid.Cid) ( return nil, err } - txs := make(map[int64]*v0msig.Transaction) - var tx v0msig.Transaction + txs := make(map[int64]*msig0.Transaction) + var tx msig0.Transaction err = nd.ForEach(&tx, func(k string) error { txid, _ := binary.Varint([]byte(k)) @@ -276,7 +276,7 @@ func GetMultisigPending(ctx context.Context, lapi api.FullNode, hroot cid.Cid) ( return txs, nil } -func state(tx *v0msig.Transaction) string { +func state(tx *msig0.Transaction) string { /* // TODO(why): I strongly disagree with not having these... but i need to move forward if tx.Complete { return "done" @@ -385,7 +385,7 @@ var msigProposeCmd = &cli.Command{ return fmt.Errorf("proposal returned exit %d", wait.Receipt.ExitCode) } - var retval v0msig.ProposeReturn + var retval msig0.ProposeReturn if err := retval.UnmarshalCBOR(bytes.NewReader(wait.Receipt.Return)); err != nil { return fmt.Errorf("failed to unmarshal propose return value: %w", err) } diff --git a/cli/paych_test.go b/cli/paych_test.go index 598c178eb..49631f1fc 100644 --- a/cli/paych_test.go +++ b/cli/paych_test.go @@ -15,9 +15,9 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/go-state-types/big" - v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" - v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" - v0verifreg "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" + power0 "github.com/filecoin-project/specs-actors/actors/builtin/power" + verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" "github.com/multiformats/go-multiaddr" @@ -40,11 +40,11 @@ import ( ) func init() { - v0power.ConsensusMinerMinPower = big.NewInt(2048) - v0miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ + power0.ConsensusMinerMinPower = big.NewInt(2048) + miner0.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } - v0verifreg.MinVerifiedDealSize = big.NewInt(256) + verifreg0.MinVerifiedDealSize = big.NewInt(256) } // TestPaymentChannels does a basic test to exercise the payment channel CLI diff --git a/cmd/lotus-pcr/main.go b/cmd/lotus-pcr/main.go index 53c8223ed..c8179d9b6 100644 --- a/cmd/lotus-pcr/main.go +++ b/cmd/lotus-pcr/main.go @@ -12,7 +12,7 @@ import ( "strconv" "time" - v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" @@ -403,7 +403,7 @@ func (r *refunder) ProcessTipset(ctx context.Context, tipset *types.TipSet, refu } if nv < build.ActorUpgradeNetworkVersion { - var proveCommitSector v0miner.ProveCommitSectorParams + var proveCommitSector miner0.ProveCommitSectorParams if err := proveCommitSector.UnmarshalCBOR(bytes.NewBuffer(m.Params)); err != nil { log.Warnw("failed to decode provecommit params", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To) continue diff --git a/cmd/lotus-storage-miner/actor.go b/cmd/lotus-storage-miner/actor.go index 2493c5a1e..ea21aa90d 100644 --- a/cmd/lotus-storage-miner/actor.go +++ b/cmd/lotus-storage-miner/actor.go @@ -16,7 +16,7 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/specs-actors/actors/builtin" - v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/types" @@ -88,7 +88,7 @@ var actorSetAddrsCmd = &cli.Command{ return err } - params, err := actors.SerializeParams(&v0miner.ChangeMultiaddrsParams{NewMultiaddrs: addrs}) + params, err := actors.SerializeParams(&miner0.ChangeMultiaddrsParams{NewMultiaddrs: addrs}) if err != nil { return err } @@ -153,7 +153,7 @@ var actorSetPeeridCmd = &cli.Command{ return err } - params, err := actors.SerializeParams(&v0miner.ChangePeerIDParams{NewID: abi.PeerID(pid)}) + params, err := actors.SerializeParams(&miner0.ChangePeerIDParams{NewID: abi.PeerID(pid)}) if err != nil { return err } @@ -226,7 +226,7 @@ var actorWithdrawCmd = &cli.Command{ } } - params, err := actors.SerializeParams(&v0miner.WithdrawBalanceParams{ + params, err := actors.SerializeParams(&miner0.WithdrawBalanceParams{ AmountRequested: amount, // Default to attempting to withdraw all the extra funds in the miner actor }) if err != nil { @@ -451,7 +451,7 @@ var actorControlSet = &cli.Command{ return nil } - cwp := &v0miner.ChangeWorkerAddressParams{ + cwp := &miner0.ChangeWorkerAddressParams{ NewWorker: mi.Worker, NewControlAddrs: toSet, } diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index 715daead8..4faf6a25b 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -32,10 +32,10 @@ import ( "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/extern/sector-storage/stores" - v0builtin "github.com/filecoin-project/specs-actors/actors/builtin" - v0market "github.com/filecoin-project/specs-actors/actors/builtin/market" - v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" - v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" + market0 "github.com/filecoin-project/specs-actors/actors/builtin/market" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" + power0 "github.com/filecoin-project/specs-actors/actors/builtin/power" lapi "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" @@ -373,7 +373,7 @@ func migratePreSealMeta(ctx context.Context, api lapi.FullNode, metadata string, return mds.Put(datastore.NewKey(modules.StorageCounterDSPrefix), buf[:size]) } -func findMarketDealID(ctx context.Context, api lapi.FullNode, deal v0market.DealProposal) (abi.DealID, error) { +func findMarketDealID(ctx context.Context, api lapi.FullNode, deal market0.DealProposal) (abi.DealID, error) { // TODO: find a better way // (this is only used by genesis miners) @@ -567,7 +567,7 @@ func configureStorageMiner(ctx context.Context, api lapi.FullNode, addr address. return xerrors.Errorf("getWorkerAddr returned bad address: %w", err) } - enc, err := actors.SerializeParams(&v0miner.ChangePeerIDParams{NewID: abi.PeerID(peerid)}) + enc, err := actors.SerializeParams(&miner0.ChangePeerIDParams{NewID: abi.PeerID(peerid)}) if err != nil { return err } @@ -575,7 +575,7 @@ func configureStorageMiner(ctx context.Context, api lapi.FullNode, addr address. msg := &types.Message{ To: addr, From: mi.Worker, - Method: v0builtin.MethodsMiner.ChangePeerID, + Method: builtin0.MethodsMiner.ChangePeerID, Params: enc, Value: types.NewInt(0), GasPremium: gasPrice, @@ -634,7 +634,7 @@ func createStorageMiner(ctx context.Context, api lapi.FullNode, peerid peer.ID, return address.Undef, err } - params, err := actors.SerializeParams(&v0power.CreateMinerParams{ + params, err := actors.SerializeParams(&power0.CreateMinerParams{ Owner: owner, Worker: worker, SealProofType: spt, @@ -654,11 +654,11 @@ func createStorageMiner(ctx context.Context, api lapi.FullNode, peerid peer.ID, } createStorageMinerMsg := &types.Message{ - To: v0builtin.StoragePowerActorAddr, + To: builtin0.StoragePowerActorAddr, From: sender, Value: big.Zero(), - Method: v0builtin.MethodsPower.CreateMiner, + Method: builtin0.MethodsPower.CreateMiner, Params: params, GasLimit: 0, @@ -682,7 +682,7 @@ func createStorageMiner(ctx context.Context, api lapi.FullNode, peerid peer.ID, return address.Undef, xerrors.Errorf("create miner failed: exit code %d", mw.Receipt.ExitCode) } - var retval v0power.CreateMinerReturn + var retval power0.CreateMinerReturn if err := retval.UnmarshalCBOR(bytes.NewReader(mw.Receipt.Return)); err != nil { return address.Undef, err } diff --git a/cmd/lotus-storage-miner/sectors.go b/cmd/lotus-storage-miner/sectors.go index 5659bed79..83550dbda 100644 --- a/cmd/lotus-storage-miner/sectors.go +++ b/cmd/lotus-storage-miner/sectors.go @@ -13,7 +13,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" - v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" @@ -380,7 +380,7 @@ var sectorsCapacityCollateralCmd = &cli.Command{ Expiration: abi.ChainEpoch(cctx.Uint64("expiration")), } if pci.Expiration == 0 { - pci.Expiration = v0miner.MaxSectorExpirationExtension + pci.Expiration = miner0.MaxSectorExpirationExtension } pc, err := nApi.StateMinerInitialPledgeCollateral(ctx, maddr, pci, types.EmptyTSK) if err != nil { diff --git a/extern/storage-sealing/checks.go b/extern/storage-sealing/checks.go index 677cef1e9..27b62f49a 100644 --- a/extern/storage-sealing/checks.go +++ b/extern/storage-sealing/checks.go @@ -5,9 +5,9 @@ import ( "context" "github.com/filecoin-project/lotus/build" - v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" - v0proof "github.com/filecoin-project/specs-actors/actors/runtime/proof" + proof0 "github.com/filecoin-project/specs-actors/actors/runtime/proof" "golang.org/x/xerrors" @@ -102,7 +102,7 @@ func checkPrecommit(ctx context.Context, maddr address.Address, si SectorInfo, t var msd abi.ChainEpoch if nv < build.ActorUpgradeNetworkVersion { - msd = v0miner.MaxSealDuration[si.SectorType] + msd = miner0.MaxSealDuration[si.SectorType] } else { // TODO: ActorUpgrade msd = 0 @@ -190,7 +190,7 @@ func (m *Sealing) checkCommit(ctx context.Context, si SectorInfo, proof []byte, log.Warn("on-chain sealed CID doesn't match!") } - ok, err := m.verif.VerifySeal(v0proof.SealVerifyInfo{ + ok, err := m.verif.VerifySeal(proof0.SealVerifyInfo{ SectorID: m.minerSector(si.SectorNumber), SealedCID: pci.Info.SealedCID, SealProof: spt, diff --git a/extern/storage-sealing/constants.go b/extern/storage-sealing/constants.go index 06c48fa4c..8c7fa5abc 100644 --- a/extern/storage-sealing/constants.go +++ b/extern/storage-sealing/constants.go @@ -1,11 +1,11 @@ package sealing import ( - v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" ) // Epochs -const SealRandomnessLookback = v0miner.ChainFinality +const SealRandomnessLookback = miner0.ChainFinality // Epochs const InteractivePoRepConfidence = 6 diff --git a/extern/storage-sealing/precommit_policy.go b/extern/storage-sealing/precommit_policy.go index 76d867144..2ee6c1afc 100644 --- a/extern/storage-sealing/precommit_policy.go +++ b/extern/storage-sealing/precommit_policy.go @@ -7,7 +7,7 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/go-state-types/abi" - v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" ) type PreCommitPolicy interface { @@ -87,7 +87,7 @@ func (p *BasicPreCommitPolicy) Expiration(ctx context.Context, ps ...Piece) (abi var wpp abi.ChainEpoch if nv < build.ActorUpgradeNetworkVersion { - wpp = v0miner.WPoStProvingPeriod + wpp = miner0.WPoStProvingPeriod } else { // TODO: ActorUpgrade wpp = 0 diff --git a/extern/storage-sealing/sealing.go b/extern/storage-sealing/sealing.go index 01551e6d7..d5772e1d6 100644 --- a/extern/storage-sealing/sealing.go +++ b/extern/storage-sealing/sealing.go @@ -9,7 +9,7 @@ import ( "time" "github.com/filecoin-project/lotus/build" - v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/go-state-types/network" "github.com/ipfs/go-cid" @@ -430,7 +430,7 @@ func (m *Sealing) getPreCommitChallengeDelay(ctx context.Context, tok TipSetToke } if nv < build.ActorUpgradeNetworkVersion { - return v0miner.PreCommitChallengeDelay, nil + return miner0.PreCommitChallengeDelay, nil } // TODO: ActorUpgrade diff --git a/extern/storage-sealing/states_sealing.go b/extern/storage-sealing/states_sealing.go index 6ae42a91f..0109326fb 100644 --- a/extern/storage-sealing/states_sealing.go +++ b/extern/storage-sealing/states_sealing.go @@ -6,7 +6,7 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" - v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" "golang.org/x/xerrors" @@ -191,8 +191,8 @@ func (m *Sealing) handlePreCommitting(ctx statemachine.Context, sector SectorInf var msd abi.ChainEpoch var mse abi.ChainEpoch if nv < build.ActorUpgradeNetworkVersion { - msd = v0miner.MaxSealDuration[sector.SectorType] - mse = v0miner.MinSectorExpiration + msd = miner0.MaxSealDuration[sector.SectorType] + mse = miner0.MinSectorExpiration } else { // TODO: ActorUpgrade msd = 0 @@ -387,7 +387,7 @@ func (m *Sealing) handleSubmitCommit(ctx statemachine.Context, sector SectorInfo enc := new(bytes.Buffer) if nv < build.ActorUpgradeNetworkVersion { - params := &v0miner.ProveCommitSectorParams{ + params := &miner0.ProveCommitSectorParams{ SectorNumber: sector.SectorNumber, Proof: sector.Proof, } diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 6c60269ab..8b47144af 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -41,7 +41,7 @@ import ( "github.com/filecoin-project/go-multistore" "github.com/filecoin-project/go-padreader" "github.com/filecoin-project/go-state-types/abi" - v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" marketevents "github.com/filecoin-project/lotus/markets/loggers" @@ -87,7 +87,7 @@ func calcDealExpiration(minDuration uint64, md *dline.Info, startEpoch abi.Chain minExp := startEpoch + abi.ChainEpoch(minDuration) // Align on miners ProvingPeriodBoundary - return minExp + v0miner.WPoStProvingPeriod - (minExp % v0miner.WPoStProvingPeriod) + (md.PeriodStart % v0miner.WPoStProvingPeriod) - 1 + return minExp + miner0.WPoStProvingPeriod - (minExp % miner0.WPoStProvingPeriod) + (md.PeriodStart % miner0.WPoStProvingPeriod) - 1 } func (a *API) imgr() *importmgr.Mgr { diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 3bed0dfbb..76bd3926b 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -5,7 +5,7 @@ import ( "context" "strconv" - v0market "github.com/filecoin-project/specs-actors/actors/builtin/market" + market0 "github.com/filecoin-project/specs-actors/actors/builtin/market" "github.com/filecoin-project/lotus/chain/actors/builtin/verifreg" @@ -24,7 +24,7 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/specs-actors/actors/builtin" - v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/specs-actors/actors/util/smoothing" @@ -921,7 +921,7 @@ func (a *StateAPI) StateMinerPreCommitDepositForPower(ctx context.Context, maddr duration := pci.Expiration - ts.Height() // TODO: ActorUpgrade - sectorWeight = v0miner.QAPowerForWeight(ssize, duration, w, vw) + sectorWeight = miner0.QAPowerForWeight(ssize, duration, w, vw) } var powerSmoothed smoothing.FilterEstimate @@ -947,7 +947,7 @@ func (a *StateAPI) StateMinerPreCommitDepositForPower(ctx context.Context, maddr } // TODO: ActorUpgrade - deposit := v0miner.PreCommitDepositForPower(&rewardSmoothed, &powerSmoothed, sectorWeight) + deposit := miner0.PreCommitDepositForPower(&rewardSmoothed, &powerSmoothed, sectorWeight) return types.BigDiv(types.BigMul(deposit, initialPledgeNum), initialPledgeDen), nil } @@ -983,7 +983,7 @@ func (a *StateAPI) StateMinerInitialPledgeCollateral(ctx context.Context, maddr duration := pci.Expiration - ts.Height() // TODO: handle changes to this function across actor upgrades. - sectorWeight = v0miner.QAPowerForWeight(ssize, duration, w, vw) + sectorWeight = miner0.QAPowerForWeight(ssize, duration, w, vw) } var ( @@ -1027,7 +1027,7 @@ func (a *StateAPI) StateMinerInitialPledgeCollateral(ctx context.Context, maddr // TODO: ActorUpgrade - initialPledge := v0miner.InitialPledgeForPower( + initialPledge := miner0.InitialPledgeForPower( sectorWeight, baselinePower, pledgeCollateral, @@ -1145,7 +1145,7 @@ func (a *StateAPI) StateDealProviderCollateralBounds(ctx context.Context, size a return api.DealCollateralBounds{}, xerrors.Errorf("getting reward baseline power: %w", err) } - min, max := v0market.DealProviderCollateralBounds(size, + min, max := market0.DealProviderCollateralBounds(size, verified, powClaim.RawBytePower, powClaim.QualityAdjPower, diff --git a/node/node_test.go b/node/node_test.go index 68ee178fc..0e404dc0b 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -10,9 +10,9 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/lotus/lib/lotuslog" - v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" - v0power "github.com/filecoin-project/specs-actors/actors/builtin/power" - v0verifreg "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" + power0 "github.com/filecoin-project/specs-actors/actors/builtin/power" + verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" logging "github.com/ipfs/go-log/v2" "github.com/filecoin-project/lotus/api/test" @@ -21,11 +21,11 @@ import ( func init() { _ = logging.SetLogLevel("*", "INFO") - v0power.ConsensusMinerMinPower = big.NewInt(2048) - v0miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ + power0.ConsensusMinerMinPower = big.NewInt(2048) + miner0.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ abi.RegisteredSealProof_StackedDrg2KiBV1: {}, } - v0verifreg.MinVerifiedDealSize = big.NewInt(256) + verifreg0.MinVerifiedDealSize = big.NewInt(256) } func TestAPI(t *testing.T) { @@ -68,7 +68,7 @@ func TestAPIDealFlowReal(t *testing.T) { logging.SetLogLevel("sub", "ERROR") logging.SetLogLevel("storageminer", "ERROR") - v0miner.PreCommitChallengeDelay = 5 + miner0.PreCommitChallengeDelay = 5 t.Run("basic", func(t *testing.T) { test.TestDealFlow(t, builder.Builder, time.Second, false, false) diff --git a/node/test/builder.go b/node/test/builder.go index 2b26c13fc..c496b1e4c 100644 --- a/node/test/builder.go +++ b/node/test/builder.go @@ -37,7 +37,7 @@ import ( "github.com/filecoin-project/lotus/node/repo" "github.com/filecoin-project/lotus/storage/mockstorage" "github.com/filecoin-project/specs-actors/actors/builtin" - v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/ipfs/go-datastore" "github.com/libp2p/go-libp2p-core/crypto" "github.com/libp2p/go-libp2p-core/peer" @@ -83,7 +83,7 @@ func CreateTestStorageNode(ctx context.Context, t *testing.T, waddr address.Addr peerid, err := peer.IDFromPrivateKey(pk) require.NoError(t, err) - enc, err := actors.SerializeParams(&v0miner.ChangePeerIDParams{NewID: abi.PeerID(peerid)}) + enc, err := actors.SerializeParams(&miner0.ChangePeerIDParams{NewID: abi.PeerID(peerid)}) require.NoError(t, err) msg := &types.Message{ diff --git a/paychmgr/manager.go b/paychmgr/manager.go index 4556c37be..d9c568cbe 100644 --- a/paychmgr/manager.go +++ b/paychmgr/manager.go @@ -12,7 +12,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/crypto" - v0paych "github.com/filecoin-project/specs-actors/actors/builtin/paych" + paych0 "github.com/filecoin-project/specs-actors/actors/builtin/paych" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors/adt" @@ -221,7 +221,7 @@ func (pm *Manager) GetChannelInfo(addr address.Address) (*ChannelInfo, error) { return ca.getChannelInfo(addr) } -func (pm *Manager) CreateVoucher(ctx context.Context, ch address.Address, voucher v0paych.SignedVoucher) (*api.VoucherCreateResult, error) { +func (pm *Manager) CreateVoucher(ctx context.Context, ch address.Address, voucher paych0.SignedVoucher) (*api.VoucherCreateResult, error) { ca, err := pm.accessorByAddress(ch) if err != nil { return nil, err @@ -233,7 +233,7 @@ func (pm *Manager) CreateVoucher(ctx context.Context, ch address.Address, vouche // CheckVoucherValid checks if the given voucher is valid (is or could become spendable at some point). // If the channel is not in the store, fetches the channel from state (and checks that // the channel To address is owned by the wallet). -func (pm *Manager) CheckVoucherValid(ctx context.Context, ch address.Address, sv *v0paych.SignedVoucher) error { +func (pm *Manager) CheckVoucherValid(ctx context.Context, ch address.Address, sv *paych0.SignedVoucher) error { // Get an accessor for the channel, creating it from state if necessary ca, err := pm.inboundChannelAccessor(ctx, ch) if err != nil { @@ -245,7 +245,7 @@ func (pm *Manager) CheckVoucherValid(ctx context.Context, ch address.Address, sv } // CheckVoucherSpendable checks if the given voucher is currently spendable -func (pm *Manager) CheckVoucherSpendable(ctx context.Context, ch address.Address, sv *v0paych.SignedVoucher, secret []byte, proof []byte) (bool, error) { +func (pm *Manager) CheckVoucherSpendable(ctx context.Context, ch address.Address, sv *paych0.SignedVoucher, secret []byte, proof []byte) (bool, error) { ca, err := pm.accessorByAddress(ch) if err != nil { return false, err @@ -256,7 +256,7 @@ func (pm *Manager) CheckVoucherSpendable(ctx context.Context, ch address.Address // AddVoucherOutbound adds a voucher for an outbound channel. // Returns an error if the channel is not already in the store. -func (pm *Manager) AddVoucherOutbound(ctx context.Context, ch address.Address, sv *v0paych.SignedVoucher, proof []byte, minDelta types.BigInt) (types.BigInt, error) { +func (pm *Manager) AddVoucherOutbound(ctx context.Context, ch address.Address, sv *paych0.SignedVoucher, proof []byte, minDelta types.BigInt) (types.BigInt, error) { ca, err := pm.accessorByAddress(ch) if err != nil { return types.NewInt(0), err @@ -267,7 +267,7 @@ func (pm *Manager) AddVoucherOutbound(ctx context.Context, ch address.Address, s // AddVoucherInbound adds a voucher for an inbound channel. // If the channel is not in the store, fetches the channel from state (and checks that // the channel To address is owned by the wallet). -func (pm *Manager) AddVoucherInbound(ctx context.Context, ch address.Address, sv *v0paych.SignedVoucher, proof []byte, minDelta types.BigInt) (types.BigInt, error) { +func (pm *Manager) AddVoucherInbound(ctx context.Context, ch address.Address, sv *paych0.SignedVoucher, proof []byte, minDelta types.BigInt) (types.BigInt, error) { // Get an accessor for the channel, creating it from state if necessary ca, err := pm.inboundChannelAccessor(ctx, ch) if err != nil { @@ -336,7 +336,7 @@ func (pm *Manager) trackInboundChannel(ctx context.Context, ch address.Address) return pm.store.TrackChannel(stateCi) } -func (pm *Manager) SubmitVoucher(ctx context.Context, ch address.Address, sv *v0paych.SignedVoucher, secret []byte, proof []byte) (cid.Cid, error) { +func (pm *Manager) SubmitVoucher(ctx context.Context, ch address.Address, sv *paych0.SignedVoucher, secret []byte, proof []byte) (cid.Cid, error) { ca, err := pm.accessorByAddress(ch) if err != nil { return cid.Undef, err diff --git a/paychmgr/paych.go b/paychmgr/paych.go index 53f16b4fc..11dfd2a1e 100644 --- a/paychmgr/paych.go +++ b/paychmgr/paych.go @@ -12,7 +12,7 @@ import ( cborutil "github.com/filecoin-project/go-cbor-util" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/specs-actors/actors/builtin" - v0paych "github.com/filecoin-project/specs-actors/actors/builtin/paych" + paych0 "github.com/filecoin-project/specs-actors/actors/builtin/paych" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" @@ -103,7 +103,7 @@ func (ca *channelAccessor) outboundActiveByFromTo(from, to address.Address) (*Ch // nonce, signing the voucher and storing it in the local datastore. // If there are not enough funds in the channel to create the voucher, returns // the shortfall in funds. -func (ca *channelAccessor) createVoucher(ctx context.Context, ch address.Address, voucher v0paych.SignedVoucher) (*api.VoucherCreateResult, error) { +func (ca *channelAccessor) createVoucher(ctx context.Context, ch address.Address, voucher paych0.SignedVoucher) (*api.VoucherCreateResult, error) { ca.lk.Lock() defer ca.lk.Unlock() @@ -162,14 +162,14 @@ func (ca *channelAccessor) nextNonceForLane(ci *ChannelInfo, lane uint64) uint64 return maxnonce + 1 } -func (ca *channelAccessor) checkVoucherValid(ctx context.Context, ch address.Address, sv *v0paych.SignedVoucher) (map[uint64]paych.LaneState, error) { +func (ca *channelAccessor) checkVoucherValid(ctx context.Context, ch address.Address, sv *paych0.SignedVoucher) (map[uint64]paych.LaneState, error) { ca.lk.Lock() defer ca.lk.Unlock() return ca.checkVoucherValidUnlocked(ctx, ch, sv) } -func (ca *channelAccessor) checkVoucherValidUnlocked(ctx context.Context, ch address.Address, sv *v0paych.SignedVoucher) (map[uint64]paych.LaneState, error) { +func (ca *channelAccessor) checkVoucherValidUnlocked(ctx context.Context, ch address.Address, sv *paych0.SignedVoucher) (map[uint64]paych.LaneState, error) { if sv.ChannelAddr != ch { return nil, xerrors.Errorf("voucher ChannelAddr doesn't match channel address, got %s, expected %s", sv.ChannelAddr, ch) } @@ -251,7 +251,7 @@ func (ca *channelAccessor) checkVoucherValidUnlocked(ctx context.Context, ch add return laneStates, nil } -func (ca *channelAccessor) checkVoucherSpendable(ctx context.Context, ch address.Address, sv *v0paych.SignedVoucher, secret []byte, proof []byte) (bool, error) { +func (ca *channelAccessor) checkVoucherSpendable(ctx context.Context, ch address.Address, sv *paych0.SignedVoucher, secret []byte, proof []byte) (bool, error) { ca.lk.Lock() defer ca.lk.Unlock() @@ -290,7 +290,7 @@ func (ca *channelAccessor) checkVoucherSpendable(ctx context.Context, ch address } } - enc, err := actors.SerializeParams(&v0paych.UpdateChannelStateParams{ + enc, err := actors.SerializeParams(&paych0.UpdateChannelStateParams{ Sv: *sv, Secret: secret, Proof: proof, @@ -325,14 +325,14 @@ func (ca *channelAccessor) getPaychRecipient(ctx context.Context, ch address.Add return state.To(), nil } -func (ca *channelAccessor) addVoucher(ctx context.Context, ch address.Address, sv *v0paych.SignedVoucher, proof []byte, minDelta types.BigInt) (types.BigInt, error) { +func (ca *channelAccessor) addVoucher(ctx context.Context, ch address.Address, sv *paych0.SignedVoucher, proof []byte, minDelta types.BigInt) (types.BigInt, error) { ca.lk.Lock() defer ca.lk.Unlock() return ca.addVoucherUnlocked(ctx, ch, sv, proof, minDelta) } -func (ca *channelAccessor) addVoucherUnlocked(ctx context.Context, ch address.Address, sv *v0paych.SignedVoucher, proof []byte, minDelta types.BigInt) (types.BigInt, error) { +func (ca *channelAccessor) addVoucherUnlocked(ctx context.Context, ch address.Address, sv *paych0.SignedVoucher, proof []byte, minDelta types.BigInt) (types.BigInt, error) { ci, err := ca.store.ByAddress(ch) if err != nil { return types.BigInt{}, err @@ -396,7 +396,7 @@ func (ca *channelAccessor) addVoucherUnlocked(ctx context.Context, ch address.Ad return delta, ca.store.putChannelInfo(ci) } -func (ca *channelAccessor) submitVoucher(ctx context.Context, ch address.Address, sv *v0paych.SignedVoucher, secret []byte, proof []byte) (cid.Cid, error) { +func (ca *channelAccessor) submitVoucher(ctx context.Context, ch address.Address, sv *paych0.SignedVoucher, secret []byte, proof []byte) (cid.Cid, error) { ca.lk.Lock() defer ca.lk.Unlock() @@ -437,7 +437,7 @@ func (ca *channelAccessor) submitVoucher(ctx context.Context, ch address.Address } } - enc, err := actors.SerializeParams(&v0paych.UpdateChannelStateParams{ + enc, err := actors.SerializeParams(&paych0.UpdateChannelStateParams{ Sv: *sv, Secret: secret, Proof: proof, @@ -543,7 +543,7 @@ func (ca *channelAccessor) laneState(ctx context.Context, state paych.State, ch } // Get the total redeemed amount across all lanes, after applying the voucher -func (ca *channelAccessor) totalRedeemedWithVoucher(laneStates map[uint64]paych.LaneState, sv *v0paych.SignedVoucher) (big.Int, error) { +func (ca *channelAccessor) totalRedeemedWithVoucher(laneStates map[uint64]paych.LaneState, sv *paych0.SignedVoucher) (big.Int, error) { // TODO: merges if len(sv.Merges) != 0 { return big.Int{}, xerrors.Errorf("dont currently support paych lane merges") diff --git a/paychmgr/paych_test.go b/paychmgr/paych_test.go index 434c83e9c..b27b1e540 100644 --- a/paychmgr/paych_test.go +++ b/paychmgr/paych_test.go @@ -15,7 +15,7 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/specs-actors/actors/builtin" - v0paych "github.com/filecoin-project/specs-actors/actors/builtin/paych" + paych0 "github.com/filecoin-project/specs-actors/actors/builtin/paych" tutils "github.com/filecoin-project/specs-actors/support/testing" "github.com/filecoin-project/lotus/api" @@ -339,7 +339,7 @@ func TestCreateVoucher(t *testing.T) { // Create a voucher in lane 1 voucherLane1Amt := big.NewInt(5) - voucher := v0paych.SignedVoucher{ + voucher := paych0.SignedVoucher{ Lane: 1, Amount: voucherLane1Amt, } @@ -354,7 +354,7 @@ func TestCreateVoucher(t *testing.T) { // Create a voucher in lane 1 again, with a higher amount voucherLane1Amt = big.NewInt(8) - voucher = v0paych.SignedVoucher{ + voucher = paych0.SignedVoucher{ Lane: 1, Amount: voucherLane1Amt, } @@ -369,7 +369,7 @@ func TestCreateVoucher(t *testing.T) { // Create a voucher in lane 2 that covers all the remaining funds // in the channel voucherLane2Amt := big.Sub(s.amt, voucherLane1Amt) - voucher = v0paych.SignedVoucher{ + voucher = paych0.SignedVoucher{ Lane: 2, Amount: voucherLane2Amt, } @@ -383,7 +383,7 @@ func TestCreateVoucher(t *testing.T) { // Create a voucher in lane 2 that exceeds the remaining funds in the // channel voucherLane2Amt = big.Add(voucherLane2Amt, big.NewInt(1)) - voucher = v0paych.SignedVoucher{ + voucher = paych0.SignedVoucher{ Lane: 2, Amount: voucherLane2Amt, } @@ -772,7 +772,7 @@ func TestCheckSpendable(t *testing.T) { // Check that the secret and proof were passed through correctly lastCall := s.mock.getLastCall() - var p v0paych.UpdateChannelStateParams + var p paych0.UpdateChannelStateParams err = p.UnmarshalCBOR(bytes.NewReader(lastCall.Params)) require.NoError(t, err) require.Equal(t, otherProof, p.Proof) @@ -786,7 +786,7 @@ func TestCheckSpendable(t *testing.T) { require.True(t, spendable) lastCall = s.mock.getLastCall() - var p2 v0paych.UpdateChannelStateParams + var p2 paych0.UpdateChannelStateParams err = p2.UnmarshalCBOR(bytes.NewReader(lastCall.Params)) require.NoError(t, err) require.Equal(t, proof, p2.Proof) @@ -843,7 +843,7 @@ func TestSubmitVoucher(t *testing.T) { // Check that the secret and proof were passed through correctly msg := s.mock.pushedMessages(submitCid) - var p v0paych.UpdateChannelStateParams + var p paych0.UpdateChannelStateParams err = p.UnmarshalCBOR(bytes.NewReader(msg.Message.Params)) require.NoError(t, err) require.Equal(t, submitProof, p.Proof) @@ -863,7 +863,7 @@ func TestSubmitVoucher(t *testing.T) { require.NoError(t, err) msg = s.mock.pushedMessages(submitCid) - var p2 v0paych.UpdateChannelStateParams + var p2 paych0.UpdateChannelStateParams err = p2.UnmarshalCBOR(bytes.NewReader(msg.Message.Params)) require.NoError(t, err) require.Equal(t, addVoucherProof2, p2.Proof) @@ -879,7 +879,7 @@ func TestSubmitVoucher(t *testing.T) { require.NoError(t, err) msg = s.mock.pushedMessages(submitCid) - var p3 v0paych.UpdateChannelStateParams + var p3 paych0.UpdateChannelStateParams err = p3.UnmarshalCBOR(bytes.NewReader(msg.Message.Params)) require.NoError(t, err) require.Equal(t, proof3, p3.Proof) @@ -966,8 +966,8 @@ func testGenerateKeyPair(t *testing.T) ([]byte, []byte) { return priv, pub } -func createTestVoucher(t *testing.T, ch address.Address, voucherLane uint64, nonce uint64, voucherAmount big.Int, key []byte) *v0paych.SignedVoucher { - sv := &v0paych.SignedVoucher{ +func createTestVoucher(t *testing.T, ch address.Address, voucherLane uint64, nonce uint64, voucherAmount big.Int, key []byte) *paych0.SignedVoucher { + sv := &paych0.SignedVoucher{ ChannelAddr: ch, Lane: voucherLane, Nonce: nonce, @@ -982,13 +982,13 @@ func createTestVoucher(t *testing.T, ch address.Address, voucherLane uint64, non return sv } -func createTestVoucherWithExtra(t *testing.T, ch address.Address, voucherLane uint64, nonce uint64, voucherAmount big.Int, key []byte) *v0paych.SignedVoucher { - sv := &v0paych.SignedVoucher{ +func createTestVoucherWithExtra(t *testing.T, ch address.Address, voucherLane uint64, nonce uint64, voucherAmount big.Int, key []byte) *paych0.SignedVoucher { + sv := &paych0.SignedVoucher{ ChannelAddr: ch, Lane: voucherLane, Nonce: nonce, Amount: voucherAmount, - Extra: &v0paych.ModVerifyParams{ + Extra: &paych0.ModVerifyParams{ Actor: tutils.NewActorAddr(t, "act"), }, } @@ -1006,13 +1006,13 @@ type mockBestSpendableAPI struct { mgr *Manager } -func (m *mockBestSpendableAPI) PaychVoucherList(ctx context.Context, ch address.Address) ([]*v0paych.SignedVoucher, error) { +func (m *mockBestSpendableAPI) PaychVoucherList(ctx context.Context, ch address.Address) ([]*paych0.SignedVoucher, error) { vi, err := m.mgr.ListVouchers(ctx, ch) if err != nil { return nil, err } - out := make([]*v0paych.SignedVoucher, len(vi)) + out := make([]*paych0.SignedVoucher, len(vi)) for k, v := range vi { out[k] = v.Voucher } @@ -1020,7 +1020,7 @@ func (m *mockBestSpendableAPI) PaychVoucherList(ctx context.Context, ch address. return out, nil } -func (m *mockBestSpendableAPI) PaychVoucherCheckSpendable(ctx context.Context, ch address.Address, voucher *v0paych.SignedVoucher, secret []byte, proof []byte) (bool, error) { +func (m *mockBestSpendableAPI) PaychVoucherCheckSpendable(ctx context.Context, ch address.Address, voucher *paych0.SignedVoucher, secret []byte, proof []byte) (bool, error) { return m.mgr.CheckVoucherSpendable(ctx, ch, voucher, secret, proof) } diff --git a/paychmgr/simple.go b/paychmgr/simple.go index 2f6dc0593..46bbea62e 100644 --- a/paychmgr/simple.go +++ b/paychmgr/simple.go @@ -14,7 +14,7 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/specs-actors/actors/builtin" init_ "github.com/filecoin-project/specs-actors/actors/builtin/init" - v0paych "github.com/filecoin-project/specs-actors/actors/builtin/paych" + paych0 "github.com/filecoin-project/specs-actors/actors/builtin/paych" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" @@ -382,7 +382,7 @@ func (ca *channelAccessor) processTask(ctx context.Context, amt types.BigInt) *p // createPaych sends a message to create the channel and returns the message cid func (ca *channelAccessor) createPaych(ctx context.Context, amt types.BigInt) (cid.Cid, error) { - params, aerr := actors.SerializeParams(&v0paych.ConstructorParams{From: ca.from, To: ca.to}) + params, aerr := actors.SerializeParams(&paych0.ConstructorParams{From: ca.from, To: ca.to}) if aerr != nil { return cid.Undef, aerr } diff --git a/storage/adapter_storage_miner.go b/storage/adapter_storage_miner.go index efbd95817..db3ae63d9 100644 --- a/storage/adapter_storage_miner.go +++ b/storage/adapter_storage_miner.go @@ -15,7 +15,7 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/specs-actors/actors/builtin" - v0market "github.com/filecoin-project/specs-actors/actors/builtin/market" + market0 "github.com/filecoin-project/specs-actors/actors/builtin/market" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/apibstore" @@ -137,7 +137,7 @@ func (s SealingAPIAdapter) StateComputeDataCommitment(ctx context.Context, maddr return cid.Undef, xerrors.Errorf("failed to unmarshal TipSetToken to TipSetKey: %w", err) } - ccparams, err := actors.SerializeParams(&v0market.ComputeDataCommitmentParams{ + ccparams, err := actors.SerializeParams(&market0.ComputeDataCommitmentParams{ DealIDs: deals, SectorType: sectorType, }) diff --git a/storage/miner.go b/storage/miner.go index d7780898d..6a5d4183b 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -5,7 +5,7 @@ import ( "errors" "time" - v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/go-state-types/network" @@ -146,7 +146,7 @@ func (m *Miner) Run(ctx context.Context) error { evts := events.NewEvents(ctx, m.api) adaptedAPI := NewSealingAPIAdapter(m.api) // TODO: Maybe we update this policy after actor upgrades? - pcp := sealing.NewBasicPreCommitPolicy(adaptedAPI, v0miner.MaxSectorExpirationExtension-(v0miner.WPoStProvingPeriod*2), md.PeriodStart%v0miner.WPoStProvingPeriod) + pcp := sealing.NewBasicPreCommitPolicy(adaptedAPI, miner0.MaxSectorExpirationExtension-(miner0.WPoStProvingPeriod*2), md.PeriodStart%miner0.WPoStProvingPeriod) m.sealing = sealing.New(adaptedAPI, fc, NewEventsAdapter(evts), m.maddr, m.ds, m.sealer, m.sc, m.verif, &pcp, sealing.GetSealingConfigFunc(m.getSealConfig), m.handleSealingNotifications) go m.sealing.Run(ctx) //nolint:errcheck // logged intside the function diff --git a/storage/wdpost_run_test.go b/storage/wdpost_run_test.go index 1a930b41a..58014d632 100644 --- a/storage/wdpost_run_test.go +++ b/storage/wdpost_run_test.go @@ -19,8 +19,8 @@ import ( "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/specs-actors/actors/builtin" - v0builtin "github.com/filecoin-project/specs-actors/actors/builtin" - v0miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/specs-actors/actors/runtime/proof" tutils "github.com/filecoin-project/specs-actors/support/testing" @@ -134,7 +134,7 @@ func TestWDPostDoPost(t *testing.T) { // without exceeding the message sector limit require.NoError(t, err) - partitionsPerMsg := int(v0miner.AddressedSectorsMax / sectorsPerPartition) + partitionsPerMsg := int(miner0.AddressedSectorsMax / sectorsPerPartition) // Enough partitions to fill expectedMsgCount-1 messages partitionCount := (expectedMsgCount - 1) * partitionsPerMsg @@ -249,7 +249,7 @@ func (m *mockStorageMinerAPI) StateSearchMsg(ctx context.Context, cid cid.Cid) ( func (m *mockStorageMinerAPI) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) { return &types.Actor{ - Code: v0builtin.StorageMinerActorCodeID, + Code: builtin0.StorageMinerActorCodeID, }, nil } From 35bce5a5c691c015abc6b5fc51e85e63bb4915e4 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 18 Sep 2020 15:40:49 -0700 Subject: [PATCH 233/885] revert post changes 1. Calling the specific partition/deadline APIs is faster. 2. It's _much_ easier to test this way. --- chain/actors/builtin/miner/miner.go | 2 - chain/actors/builtin/miner/v0.go | 5 -- storage/miner.go | 1 + storage/wdpost_run.go | 101 ++++++---------------------- 4 files changed, 22 insertions(+), 87 deletions(-) diff --git a/chain/actors/builtin/miner/miner.go b/chain/actors/builtin/miner/miner.go index b9cff99da..d25652d30 100644 --- a/chain/actors/builtin/miner/miner.go +++ b/chain/actors/builtin/miner/miner.go @@ -58,8 +58,6 @@ type State interface { DeadlineInfo(epoch abi.ChainEpoch) *dline.Info - MaxAddressedSectors() (uint64, error) - // Diff helpers. Used by Diff* functions internally. sectors() (adt.Array, error) decodeSectorOnChainInfo(*cbg.Deferred) (SectorOnChainInfo, error) diff --git a/chain/actors/builtin/miner/v0.go b/chain/actors/builtin/miner/v0.go index 26a91edd1..37df8c217 100644 --- a/chain/actors/builtin/miner/v0.go +++ b/chain/actors/builtin/miner/v0.go @@ -224,11 +224,6 @@ func (s *state0) NumDeadlines() (uint64, error) { return miner0.WPoStPeriodDeadlines, nil } -// Max sectors per PoSt -func (s *state0) MaxAddressedSectors() (uint64, error) { - return miner0.AddressedSectorsMax, nil -} - func (s *state0) DeadlinesChanged(other State) bool { other0, ok := other.(*state0) if !ok { diff --git a/storage/miner.go b/storage/miner.go index 6a5d4183b..61c4000f2 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -77,6 +77,7 @@ type storageMinerApi interface { StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*miner.SectorLocation, error) StateMinerInfo(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error) StateMinerDeadlines(context.Context, address.Address, types.TipSetKey) ([]api.Deadline, error) + StateMinerPartitions(context.Context, address.Address, uint64, types.TipSetKey) ([]api.Partition, error) StateMinerProvingDeadline(context.Context, address.Address, types.TipSetKey) (*dline.Info, error) StateMinerPreCommitDepositForPower(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error) StateMinerInitialPledgeCollateral(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error) diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 2f5d97b76..06740b108 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -5,6 +5,8 @@ import ( "context" "time" + "github.com/filecoin-project/specs-actors/actors/runtime/proof" + "github.com/filecoin-project/go-bitfield" "github.com/filecoin-project/go-address" @@ -18,16 +20,14 @@ import ( "go.opencensus.io/trace" "golang.org/x/xerrors" - "github.com/filecoin-project/specs-actors/actors/runtime/proof" - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/api/apibstore" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" - "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/journal" + + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" ) func (s *WindowPoStScheduler) failPost(err error, deadline *dline.Info) { @@ -154,7 +154,7 @@ func (s *WindowPoStScheduler) checkSectors(ctx context.Context, check bitfield.B return sbf, nil } -func (s *WindowPoStScheduler) checkNextRecoveries(ctx context.Context, dlIdx uint64, partitions []miner.Partition) ([]miner.RecoveryDeclaration, *types.SignedMessage, error) { +func (s *WindowPoStScheduler) checkNextRecoveries(ctx context.Context, dlIdx uint64, partitions []api.Partition) ([]miner.RecoveryDeclaration, *types.SignedMessage, error) { ctx, span := trace.StartSpan(ctx, "storage.checkNextRecoveries") defer span.End() @@ -164,15 +164,7 @@ func (s *WindowPoStScheduler) checkNextRecoveries(ctx context.Context, dlIdx uin } for partIdx, partition := range partitions { - faults, err := partition.FaultySectors() - if err != nil { - return nil, nil, xerrors.Errorf("getting faults: %w", err) - } - recovering, err := partition.RecoveringSectors() - if err != nil { - return nil, nil, xerrors.Errorf("getting recovering: %w", err) - } - unrecovered, err := bitfield.SubtractBitField(faults, recovering) + unrecovered, err := bitfield.SubtractBitField(partition.FaultySectors, partition.RecoveringSectors) if err != nil { return nil, nil, xerrors.Errorf("subtracting recovered set from fault set: %w", err) } @@ -253,7 +245,7 @@ func (s *WindowPoStScheduler) checkNextRecoveries(ctx context.Context, dlIdx uin return recoveries, sm, nil } -func (s *WindowPoStScheduler) checkNextFaults(ctx context.Context, dlIdx uint64, partitions []miner.Partition) ([]miner.FaultDeclaration, *types.SignedMessage, error) { +func (s *WindowPoStScheduler) checkNextFaults(ctx context.Context, dlIdx uint64, partitions []api.Partition) ([]miner.FaultDeclaration, *types.SignedMessage, error) { ctx, span := trace.StartSpan(ctx, "storage.checkNextFaults") defer span.End() @@ -263,17 +255,12 @@ func (s *WindowPoStScheduler) checkNextFaults(ctx context.Context, dlIdx uint64, } for partIdx, partition := range partitions { - toCheck, err := partition.ActiveSectors() - if err != nil { - return nil, nil, xerrors.Errorf("getting active sectors: %w", err) - } - - good, err := s.checkSectors(ctx, toCheck) + good, err := s.checkSectors(ctx, partition.ActiveSectors) if err != nil { return nil, nil, xerrors.Errorf("checking sectors: %w", err) } - faulty, err := bitfield.SubtractBitField(toCheck, good) + faulty, err := bitfield.SubtractBitField(partition.ActiveSectors, good) if err != nil { return nil, nil, xerrors.Errorf("calculating faulty sector set: %w", err) } @@ -341,17 +328,6 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty ctx, span := trace.StartSpan(ctx, "storage.runPost") defer span.End() - stor := store.ActorStore(ctx, apibstore.NewAPIBlockstore(s.api)) - act, err := s.api.StateGetActor(context.TODO(), s.actor, ts.Key()) - if err != nil { - return nil, xerrors.Errorf("resolving actor: %w", err) - } - - mas, err := miner.Load(stor, act) - if err != nil { - return nil, xerrors.Errorf("getting miner state: %w", err) - } - go func() { // TODO: extract from runPost, run on fault cutoff boundaries @@ -359,18 +335,9 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty // late to declare them for this deadline declDeadline := (di.Index + 2) % di.WPoStPeriodDeadlines - dl, err := mas.LoadDeadline(declDeadline) + partitions, err := s.api.StateMinerPartitions(context.TODO(), s.actor, declDeadline, ts.Key()) if err != nil { - log.Errorf("loading deadline: %v", err) - return - } - var partitions []miner.Partition - err = dl.ForEachPartition(func(_ uint64, part miner.Partition) error { - partitions = append(partitions, part) - return nil - }) - if err != nil { - log.Errorf("loading partitions: %v", err) + log.Errorf("getting partitions: %v", err) return } @@ -429,24 +396,15 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty return nil, xerrors.Errorf("failed to get chain randomness for window post (ts=%d; deadline=%d): %w", ts.Height(), di, err) } - dl, err := mas.LoadDeadline(di.Index) - if err != nil { - return nil, xerrors.Errorf("loading deadline: %w", err) - } - // Get the partitions for the given deadline - var partitions []miner.Partition - err = dl.ForEachPartition(func(_ uint64, part miner.Partition) error { - partitions = append(partitions, part) - return nil - }) + partitions, err := s.api.StateMinerPartitions(ctx, s.actor, di.Index, ts.Key()) if err != nil { - return nil, xerrors.Errorf("loading partitions: %w", err) + return nil, xerrors.Errorf("getting partitions: %w", err) } // Split partitions into batches, so as not to exceed the number of sectors // allowed in a single message - partitionBatches, err := s.batchPartitions(partitions, mas) + partitionBatches, err := s.batchPartitions(partitions) if err != nil { return nil, err } @@ -475,16 +433,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty var sinfos []proof.SectorInfo for partIdx, partition := range batch { // TODO: Can do this in parallel - toProve, err := partition.ActiveSectors() - if err != nil { - return nil, xerrors.Errorf("getting active sectors: %w", err) - } - - recs, err := partition.RecoveringSectors() - if err != nil { - return nil, xerrors.Errorf("getting recovering sectors: %w", err) - } - toProve, err = bitfield.MergeBitFields(toProve, recs) + toProve, err := bitfield.MergeBitFields(partition.ActiveSectors, partition.RecoveringSectors) if err != nil { return nil, xerrors.Errorf("adding recoveries to set of sectors to prove: %w", err) } @@ -511,12 +460,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty skipCount += sc - partitionSectors, err := partition.AllSectors() - if err != nil { - return nil, xerrors.Errorf("getting partition sectors: %w", err) - } - - ssi, err := s.sectorsForProof(ctx, good, partitionSectors, ts) + ssi, err := s.sectorsForProof(ctx, good, partition.AllSectors, ts) if err != nil { return nil, xerrors.Errorf("getting sorted sector info: %w", err) } @@ -609,18 +553,13 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty return posts, nil } -func (s *WindowPoStScheduler) batchPartitions(partitions []miner.Partition, mas miner.State) ([][]miner.Partition, error) { +func (s *WindowPoStScheduler) batchPartitions(partitions []api.Partition) ([][]api.Partition, error) { // Get the number of sectors allowed in a partition, for this proof size sectorsPerPartition, err := builtin.PoStProofWindowPoStPartitionSectors(s.proofType) if err != nil { return nil, xerrors.Errorf("getting sectors per partition: %w", err) } - maxSectors, err := mas.MaxAddressedSectors() - if err != nil { - return nil, err - } - // We don't want to exceed the number of sectors allowed in a message. // So given the number of sectors in a partition, work out the number of // partitions that can be in a message without exceeding sectors per @@ -631,7 +570,9 @@ func (s *WindowPoStScheduler) batchPartitions(partitions []miner.Partition, mas // sectors per partition 3: ooo // partitions per message 2: oooOOO // <1><2> (3rd doesn't fit) - partitionsPerMsg := int(maxSectors / sectorsPerPartition) + // TODO(NETUPGRADE): we're going to need some form of policy abstraction + // where we can get policy from the future. Unfortunately, we can't just get this from the state. + partitionsPerMsg := int(miner0.AddressedSectorsMax / sectorsPerPartition) // The number of messages will be: // ceiling(number of partitions / partitions per message) @@ -641,7 +582,7 @@ func (s *WindowPoStScheduler) batchPartitions(partitions []miner.Partition, mas } // Split the partitions into batches - batches := make([][]miner.Partition, 0, batchCount) + batches := make([][]api.Partition, 0, batchCount) for i := 0; i < len(partitions); i += partitionsPerMsg { end := i + partitionsPerMsg if end > len(partitions) { From 26de0db75756b85f5135017fa43282b21b3255eb Mon Sep 17 00:00:00 2001 From: Travis Person Date: Wed, 9 Sep 2020 18:45:09 +0000 Subject: [PATCH 234/885] lotus-pcr: add recover-miners command --- cmd/lotus-pcr/main.go | 167 ++++++++++++++++++++++++++++++++++++++--- tools/stats/metrics.go | 22 ++++-- 2 files changed, 172 insertions(+), 17 deletions(-) diff --git a/cmd/lotus-pcr/main.go b/cmd/lotus-pcr/main.go index dc12693ca..473fc7ff1 100644 --- a/cmd/lotus-pcr/main.go +++ b/cmd/lotus-pcr/main.go @@ -38,6 +38,7 @@ var log = logging.Logger("main") func main() { local := []*cli.Command{ runCmd, + recoverMinersCmd, versionCmd, } @@ -101,6 +102,80 @@ var versionCmd = &cli.Command{ }, } +var recoverMinersCmd = &cli.Command{ + Name: "recover-miners", + Usage: "Ensure all miners with a negative available balance have a FIL surplus across accounts", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "from", + EnvVars: []string{"LOTUS_PCR_FROM"}, + Usage: "wallet address to send refund from", + }, + &cli.BoolFlag{ + Name: "no-sync", + EnvVars: []string{"LOTUS_PCR_NO_SYNC"}, + Usage: "do not wait for chain sync to complete", + }, + &cli.BoolFlag{ + Name: "dry-run", + EnvVars: []string{"LOTUS_PCR_DRY_RUN"}, + Usage: "do not send any messages", + Value: false, + }, + &cli.IntFlag{ + Name: "miner-total-funds-threashold", + EnvVars: []string{"LOTUS_PCR_MINER_TOTAL_FUNDS_THREASHOLD"}, + Usage: "total filecoin across all accounts that should be met, if the miner balancer drops below zero", + Value: 0, + }, + }, + Action: func(cctx *cli.Context) error { + ctx := context.Background() + api, closer, err := stats.GetFullNodeAPI(cctx.Context, cctx.String("lotus-path")) + if err != nil { + log.Fatal(err) + } + defer closer() + + from, err := address.NewFromString(cctx.String("from")) + if err != nil { + return xerrors.Errorf("parsing source address (provide correct --from flag!): %w", err) + } + + if !cctx.Bool("no-sync") { + if err := stats.WaitForSyncComplete(ctx, api); err != nil { + log.Fatal(err) + } + } + + dryRun := cctx.Bool("dry-run") + minerTotalFundsThreashold := uint64(cctx.Int("miner-total-funds-threashold")) + + rf := &refunder{ + api: api, + wallet: from, + dryRun: dryRun, + minerTotalFundsThreashold: types.FromFil(minerTotalFundsThreashold), + } + + refundTipset, err := api.ChainHead(ctx) + if err != nil { + return err + } + + negativeBalancerRefund, err := rf.EnsureMinerMinimums(ctx, refundTipset, NewMinersRefund()) + if err != nil { + return err + } + + if err := rf.Refund(ctx, "refund negative balancer miner", refundTipset, negativeBalancerRefund, 0); err != nil { + return err + } + + return nil + }, +} + var runCmd = &cli.Command{ Name: "run", Usage: "Start message reimpursement", @@ -230,7 +305,7 @@ var runCmd = &cli.Command{ return err } - if err := rf.Refund(ctx, refundTipset, refunds, rounds); err != nil { + if err := rf.Refund(ctx, "refund stats", refundTipset, refunds, rounds); err != nil { return err } @@ -289,7 +364,6 @@ func (m *MinersRefund) Track(addr address.Address, value types.BigInt) { m.count = m.count + 1 m.totalRefunds = types.BigAdd(m.totalRefunds, value) - m.refunds[addr] = types.BigAdd(m.refunds[addr], value) } @@ -318,8 +392,12 @@ type refunderNodeApi interface { ChainGetParentMessages(ctx context.Context, blockCid cid.Cid) ([]api.Message, error) ChainGetParentReceipts(ctx context.Context, blockCid cid.Cid) ([]*types.MessageReceipt, error) ChainGetTipSetByHeight(ctx context.Context, epoch abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) + ChainReadObj(context.Context, cid.Cid) ([]byte, error) StateMinerInitialPledgeCollateral(ctx context.Context, addr address.Address, precommitInfo miner.SectorPreCommitInfo, tsk types.TipSetKey) (types.BigInt, error) + StateMinerInfo(context.Context, address.Address, types.TipSetKey) (api.MinerInfo, error) StateSectorPreCommitInfo(ctx context.Context, addr address.Address, sector abi.SectorNumber, tsk types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) + StateMinerAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) + StateListMiners(context.Context, types.TipSetKey) ([]address.Address, error) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) MpoolPushMessage(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec) (*types.SignedMessage, error) GasEstimateGasPremium(ctx context.Context, nblocksincl uint64, sender address.Address, gaslimit int64, tsk types.TipSetKey) (types.BigInt, error) @@ -327,12 +405,81 @@ type refunderNodeApi interface { } type refunder struct { - api refunderNodeApi - wallet address.Address - percentExtra int - dryRun bool - preCommitEnabled bool - proveCommitEnabled bool + api refunderNodeApi + wallet address.Address + percentExtra int + dryRun bool + preCommitEnabled bool + proveCommitEnabled bool + minerTotalFundsThreashold big.Int +} + +func (r *refunder) EnsureMinerMinimums(ctx context.Context, tipset *types.TipSet, refunds *MinersRefund) (*MinersRefund, error) { + miners, err := r.api.StateListMiners(ctx, tipset.Key()) + if err != nil { + return nil, err + } + + for _, maddr := range miners { + mact, err := r.api.StateGetActor(ctx, maddr, types.EmptyTSK) + if err != nil { + log.Errorw("failed", "err", err, "height", tipset.Height(), "key", tipset.Key(), "miner", maddr) + continue + } + + if !mact.Balance.GreaterThan(big.Zero()) { + continue + } + + minerAvailableBalance, err := r.api.StateMinerAvailableBalance(ctx, maddr, tipset.Key()) + if err != nil { + log.Errorw("failed", "err", err, "height", tipset.Height(), "key", tipset.Key(), "miner", maddr) + continue + } + + if minerAvailableBalance.GreaterThanEqual(big.Zero()) { + log.Debugw("skipping over miner with positive balance", "height", tipset.Height(), "key", tipset.Key(), "miner", maddr) + continue + } + + // Look up and find all addresses associated with the miner + minerInfo, err := r.api.StateMinerInfo(ctx, maddr, tipset.Key()) + + allAddresses := []address.Address{minerInfo.Worker, minerInfo.Owner} + allAddresses = append(allAddresses, minerInfo.ControlAddresses...) + + // Sum the balancer of all the addresses + addrSum := big.Zero() + addrCheck := make(map[address.Address]struct{}, len(allAddresses)) + for _, addr := range allAddresses { + if _, found := addrCheck[addr]; !found { + balance, err := r.api.WalletBalance(ctx, addr) + if err != nil { + log.Errorw("failed", "err", err, "height", tipset.Height(), "key", tipset.Key(), "miner", maddr) + continue + } + + addrSum = big.Add(addrSum, balance) + addrCheck[addr] = struct{}{} + } + } + + totalAvailableBalance := big.Add(addrSum, minerAvailableBalance) + + // If the miner has available balance they should use it + if totalAvailableBalance.GreaterThanEqual(r.minerTotalFundsThreashold) { + log.Debugw("skipping over miner with enough funds cross all accounts", "height", tipset.Height(), "key", tipset.Key(), "miner", maddr, "miner_available_balance", minerAvailableBalance, "wallet_total_balances", addrSum) + continue + } + + // Calculate the required FIL to bring the miner up to the minimum across all of their accounts + refundValue := big.Add(totalAvailableBalance.Abs(), r.minerTotalFundsThreashold) + refunds.Track(maddr, refundValue) + + log.Debugw("processing negative balance miner", "miner", maddr, "refund", refundValue) + } + + return refunds, nil } func (r *refunder) ProcessTipset(ctx context.Context, tipset *types.TipSet, refunds *MinersRefund) (*MinersRefund, error) { @@ -464,7 +611,7 @@ func (r *refunder) ProcessTipset(ctx context.Context, tipset *types.TipSet, refu return refunds, nil } -func (r *refunder) Refund(ctx context.Context, tipset *types.TipSet, refunds *MinersRefund, rounds int) error { +func (r *refunder) Refund(ctx context.Context, name string, tipset *types.TipSet, refunds *MinersRefund, rounds int) error { if refunds.Count() == 0 { log.Debugw("no messages to refund in tipset", "height", tipset.Height(), "key", tipset.Key()) return nil @@ -522,7 +669,7 @@ func (r *refunder) Refund(ctx context.Context, tipset *types.TipSet, refunds *Mi refundSum = types.BigAdd(refundSum, msg.Value) } - log.Infow("refund stats", "tipsets_processed", rounds, "height", tipset.Height(), "key", tipset.Key(), "messages_sent", len(messages)-failures, "refund_sum", refundSum, "messages_failures", failures, "messages_processed", refunds.Count()) + log.Infow(name, "tipsets_processed", rounds, "height", tipset.Height(), "key", tipset.Key(), "messages_sent", len(messages)-failures, "refund_sum", refundSum, "messages_failures", failures, "messages_processed", refunds.Count()) return nil } diff --git a/tools/stats/metrics.go b/tools/stats/metrics.go index e50ac953f..eeef72e8e 100644 --- a/tools/stats/metrics.go +++ b/tools/stats/metrics.go @@ -201,16 +201,24 @@ func RecordTipsetPoints(ctx context.Context, api api.FullNode, pl *PointList, ti return nil } -type apiIpldStore struct { +type ApiIpldStore struct { ctx context.Context - api api.FullNode + api apiIpldStoreApi } -func (ht *apiIpldStore) Context() context.Context { +type apiIpldStoreApi interface { + ChainReadObj(context.Context, cid.Cid) ([]byte, error) +} + +func NewApiIpldStore(ctx context.Context, api apiIpldStoreApi) *ApiIpldStore { + return &ApiIpldStore{ctx, api} +} + +func (ht *ApiIpldStore) Context() context.Context { return ht.ctx } -func (ht *apiIpldStore) Get(ctx context.Context, c cid.Cid, out interface{}) error { +func (ht *ApiIpldStore) Get(ctx context.Context, c cid.Cid, out interface{}) error { raw, err := ht.api.ChainReadObj(ctx, c) if err != nil { return err @@ -227,8 +235,8 @@ func (ht *apiIpldStore) Get(ctx context.Context, c cid.Cid, out interface{}) err return fmt.Errorf("Object does not implement CBORUnmarshaler") } -func (ht *apiIpldStore) Put(ctx context.Context, v interface{}) (cid.Cid, error) { - return cid.Undef, fmt.Errorf("Put is not implemented on apiIpldStore") +func (ht *ApiIpldStore) Put(ctx context.Context, v interface{}) (cid.Cid, error) { + return cid.Undef, fmt.Errorf("Put is not implemented on ApiIpldStore") } func RecordTipsetStatePoints(ctx context.Context, api api.FullNode, pl *PointList, tipset *types.TipSet) error { @@ -279,7 +287,7 @@ func RecordTipsetStatePoints(ctx context.Context, api api.FullNode, pl *PointLis return fmt.Errorf("failed to unmarshal power actor state: %w", err) } - s := &apiIpldStore{ctx, api} + s := NewApiIpldStore(ctx, api) mp, err := adt.AsMap(s, powerActorState.Claims) if err != nil { return err From e39036dd499ccc80697a2c38444953d77a1d7777 Mon Sep 17 00:00:00 2001 From: Travis Person Date: Thu, 10 Sep 2020 07:36:24 +0000 Subject: [PATCH 235/885] lotus-pcr: find miners command --- cmd/lotus-pcr/main.go | 258 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 228 insertions(+), 30 deletions(-) diff --git a/cmd/lotus-pcr/main.go b/cmd/lotus-pcr/main.go index 473fc7ff1..66279c29a 100644 --- a/cmd/lotus-pcr/main.go +++ b/cmd/lotus-pcr/main.go @@ -1,10 +1,13 @@ package main import ( + "bufio" "bytes" "context" + "encoding/csv" "fmt" "io/ioutil" + "math" "net/http" _ "net/http/pprof" "os" @@ -20,13 +23,14 @@ import ( "golang.org/x/xerrors" + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-bitfield" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/exitcode" "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/builtin/miner" - "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" @@ -39,6 +43,7 @@ func main() { local := []*cli.Command{ runCmd, recoverMinersCmd, + findMinersCmd, versionCmd, } @@ -102,6 +107,92 @@ var versionCmd = &cli.Command{ }, } +var findMinersCmd = &cli.Command{ + Name: "find-miners", + Usage: "find miners with a desired minimum balance", + Description: `Find miners returns a list of miners and their balances that are below a + threhold value. By default only the miner actor available balance is considered but other + account balances can be included by enabling them through the flags. + + Examples + Find all miners with an available balance below 100 FIL + + lotus-pcr find-miners --threshold 100 + + Find all miners with a balance below zero, which includes the owner and worker balances + + lotus-pcr find-miners --threshold 0 --owner --worker +`, + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "no-sync", + EnvVars: []string{"LOTUS_PCR_NO_SYNC"}, + Usage: "do not wait for chain sync to complete", + }, + &cli.IntFlag{ + Name: "threshold", + EnvVars: []string{"LOTUS_PCR_THRESHOLD"}, + Usage: "balance below this limit will be printed", + Value: 0, + }, + &cli.BoolFlag{ + Name: "owner", + Usage: "include owner balance", + Value: false, + }, + &cli.BoolFlag{ + Name: "worker", + Usage: "include worker balance", + Value: false, + }, + &cli.BoolFlag{ + Name: "control", + Usage: "include control balance", + Value: false, + }, + }, + Action: func(cctx *cli.Context) error { + ctx := context.Background() + api, closer, err := stats.GetFullNodeAPI(cctx.Context, cctx.String("lotus-path")) + if err != nil { + log.Fatal(err) + } + defer closer() + + if !cctx.Bool("no-sync") { + if err := stats.WaitForSyncComplete(ctx, api); err != nil { + log.Fatal(err) + } + } + + owner := cctx.Bool("owner") + worker := cctx.Bool("worker") + control := cctx.Bool("control") + threshold := uint64(cctx.Int("threshold")) + + rf := &refunder{ + api: api, + threshold: types.FromFil(threshold), + } + + refundTipset, err := api.ChainHead(ctx) + if err != nil { + return err + } + + balanceRefund, err := rf.FindMiners(ctx, refundTipset, NewMinersRefund(), owner, worker, control) + if err != nil { + return err + } + + for _, maddr := range balanceRefund.Miners() { + fmt.Printf("%s\t%s\n", maddr, types.FIL(balanceRefund.GetRefund(maddr))) + } + + return nil + }, +} + var recoverMinersCmd = &cli.Command{ Name: "recover-miners", Usage: "Ensure all miners with a negative available balance have a FIL surplus across accounts", @@ -123,11 +214,15 @@ var recoverMinersCmd = &cli.Command{ Value: false, }, &cli.IntFlag{ - Name: "miner-total-funds-threashold", - EnvVars: []string{"LOTUS_PCR_MINER_TOTAL_FUNDS_THREASHOLD"}, - Usage: "total filecoin across all accounts that should be met, if the miner balancer drops below zero", + Name: "threshold", + EnvVars: []string{"LOTUS_PCR_THRESHOLD"}, + Usage: "total filecoin across all accounts that should be met, if the miner balance drops below zero", Value: 0, }, + &cli.StringFlag{ + Name: "output", + Usage: "dump data as a csv format to this file", + }, }, Action: func(cctx *cli.Context) error { ctx := context.Background() @@ -149,13 +244,13 @@ var recoverMinersCmd = &cli.Command{ } dryRun := cctx.Bool("dry-run") - minerTotalFundsThreashold := uint64(cctx.Int("miner-total-funds-threashold")) + threshold := uint64(cctx.Int("threshold")) rf := &refunder{ - api: api, - wallet: from, - dryRun: dryRun, - minerTotalFundsThreashold: types.FromFil(minerTotalFundsThreashold), + api: api, + wallet: from, + dryRun: dryRun, + threshold: types.FromFil(threshold), } refundTipset, err := api.ChainHead(ctx) @@ -163,12 +258,12 @@ var recoverMinersCmd = &cli.Command{ return err } - negativeBalancerRefund, err := rf.EnsureMinerMinimums(ctx, refundTipset, NewMinersRefund()) + balanceRefund, err := rf.EnsureMinerMinimums(ctx, refundTipset, NewMinersRefund(), cctx.String("output")) if err != nil { return err } - if err := rf.Refund(ctx, "refund negative balancer miner", refundTipset, negativeBalancerRefund, 0); err != nil { + if err := rf.Refund(ctx, "refund to recover miner", refundTipset, balanceRefund, 0); err != nil { return err } @@ -397,6 +492,8 @@ type refunderNodeApi interface { StateMinerInfo(context.Context, address.Address, types.TipSetKey) (api.MinerInfo, error) StateSectorPreCommitInfo(ctx context.Context, addr address.Address, sector abi.SectorNumber, tsk types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) StateMinerAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) + StateMinerSectors(ctx context.Context, addr address.Address, filter *bitfield.BitField, filterOut bool, tsk types.TipSetKey) ([]*api.ChainSectorInfo, error) + StateMinerFaults(ctx context.Context, addr address.Address, tsk types.TipSetKey) (bitfield.BitField, error) StateListMiners(context.Context, types.TipSetKey) ([]address.Address, error) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) MpoolPushMessage(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec) (*types.SignedMessage, error) @@ -405,16 +502,16 @@ type refunderNodeApi interface { } type refunder struct { - api refunderNodeApi - wallet address.Address - percentExtra int - dryRun bool - preCommitEnabled bool - proveCommitEnabled bool - minerTotalFundsThreashold big.Int + api refunderNodeApi + wallet address.Address + percentExtra int + dryRun bool + preCommitEnabled bool + proveCommitEnabled bool + threshold big.Int } -func (r *refunder) EnsureMinerMinimums(ctx context.Context, tipset *types.TipSet, refunds *MinersRefund) (*MinersRefund, error) { +func (r *refunder) FindMiners(ctx context.Context, tipset *types.TipSet, refunds *MinersRefund, owner, worker, control bool) (*MinersRefund, error) { miners, err := r.api.StateListMiners(ctx, tipset.Key()) if err != nil { return nil, err @@ -437,8 +534,91 @@ func (r *refunder) EnsureMinerMinimums(ctx context.Context, tipset *types.TipSet continue } - if minerAvailableBalance.GreaterThanEqual(big.Zero()) { - log.Debugw("skipping over miner with positive balance", "height", tipset.Height(), "key", tipset.Key(), "miner", maddr) + // Look up and find all addresses associated with the miner + minerInfo, err := r.api.StateMinerInfo(ctx, maddr, tipset.Key()) + + allAddresses := []address.Address{} + + if worker { + allAddresses = append(allAddresses, minerInfo.Worker) + } + + if owner { + allAddresses = append(allAddresses, minerInfo.Owner) + } + + if control { + allAddresses = append(allAddresses, minerInfo.ControlAddresses...) + } + + // Sum the balancer of all the addresses + addrSum := big.Zero() + addrCheck := make(map[address.Address]struct{}, len(allAddresses)) + for _, addr := range allAddresses { + if _, found := addrCheck[addr]; !found { + balance, err := r.api.WalletBalance(ctx, addr) + if err != nil { + log.Errorw("failed", "err", err, "height", tipset.Height(), "key", tipset.Key(), "miner", maddr) + continue + } + + addrSum = big.Add(addrSum, balance) + addrCheck[addr] = struct{}{} + } + } + + totalAvailableBalance := big.Add(addrSum, minerAvailableBalance) + + if totalAvailableBalance.GreaterThanEqual(r.threshold) { + continue + } + + refunds.Track(maddr, totalAvailableBalance) + + log.Debugw("processing miner", "miner", maddr, "sectors", "available_balance", totalAvailableBalance) + } + + return refunds, nil +} + +func (r *refunder) EnsureMinerMinimums(ctx context.Context, tipset *types.TipSet, refunds *MinersRefund, output string) (*MinersRefund, error) { + miners, err := r.api.StateListMiners(ctx, tipset.Key()) + if err != nil { + return nil, err + } + + w := ioutil.Discard + if len(output) != 0 { + f, err := os.Create(output) + if err != nil { + return nil, err + } + + defer f.Close() + + w = bufio.NewWriter(f) + } + + csvOut := csv.NewWriter(w) + defer csvOut.Flush() + if err := csvOut.Write([]string{"MinerID", "Sectors", "CombinedBalance", "ProposedSend"}); err != nil { + return nil, err + } + + for _, maddr := range miners { + mact, err := r.api.StateGetActor(ctx, maddr, types.EmptyTSK) + if err != nil { + log.Errorw("failed", "err", err, "height", tipset.Height(), "key", tipset.Key(), "miner", maddr) + continue + } + + if !mact.Balance.GreaterThan(big.Zero()) { + continue + } + + minerAvailableBalance, err := r.api.StateMinerAvailableBalance(ctx, maddr, tipset.Key()) + if err != nil { + log.Errorw("failed", "err", err, "height", tipset.Height(), "key", tipset.Key(), "miner", maddr) continue } @@ -464,19 +644,37 @@ func (r *refunder) EnsureMinerMinimums(ctx context.Context, tipset *types.TipSet } } - totalAvailableBalance := big.Add(addrSum, minerAvailableBalance) - - // If the miner has available balance they should use it - if totalAvailableBalance.GreaterThanEqual(r.minerTotalFundsThreashold) { - log.Debugw("skipping over miner with enough funds cross all accounts", "height", tipset.Height(), "key", tipset.Key(), "miner", maddr, "miner_available_balance", minerAvailableBalance, "wallet_total_balances", addrSum) + sectorInfo, err := r.api.StateMinerSectors(ctx, maddr, nil, false, tipset.Key()) + if err != nil { + log.Errorw("failed to look up miner sectors", "err", err, "height", tipset.Height(), "key", tipset.Key(), "miner", maddr) continue } - // Calculate the required FIL to bring the miner up to the minimum across all of their accounts - refundValue := big.Add(totalAvailableBalance.Abs(), r.minerTotalFundsThreashold) - refunds.Track(maddr, refundValue) + if len(sectorInfo) == 0 { + log.Debugw("skipping miner with zero sectors", "height", tipset.Height(), "key", tipset.Key(), "miner", maddr) + continue + } - log.Debugw("processing negative balance miner", "miner", maddr, "refund", refundValue) + totalAvailableBalance := big.Add(addrSum, minerAvailableBalance) + + numSectorInfo := float64(len(sectorInfo)) + filAmount := uint64(math.Ceil(math.Max(math.Log(numSectorInfo)*math.Sqrt(numSectorInfo)/4, 20))) + attoFilAmount := big.Mul(big.NewIntUnsigned(filAmount), big.NewIntUnsigned(build.FilecoinPrecision)) + + if totalAvailableBalance.GreaterThanEqual(attoFilAmount) { + log.Debugw("skipping over miner with total available balance larger than refund", "height", tipset.Height(), "key", tipset.Key(), "miner", maddr, "available_balance", totalAvailableBalance, "possible_refund", attoFilAmount) + continue + } + + refundValue := big.Sub(attoFilAmount, totalAvailableBalance) + + refunds.Track(maddr, refundValue) + record := []string{maddr.String(), fmt.Sprintf("%d", len(sectorInfo)), totalAvailableBalance.String(), refundValue.String()} + if err := csvOut.Write(record); err != nil { + return nil, err + } + + log.Debugw("processing miner", "miner", maddr, "sectors", len(sectorInfo), "available_balance", totalAvailableBalance, "refund", refundValue) } return refunds, nil From e6bbc03ca8e4207907d8fb44df3818b074815c48 Mon Sep 17 00:00:00 2001 From: Travis Person Date: Sat, 19 Sep 2020 00:51:49 +0000 Subject: [PATCH 236/885] lotus-pcr: update miner recovery --- cmd/lotus-pcr/main.go | 343 +++++++++++++++++++++++++++++++++--------- 1 file changed, 269 insertions(+), 74 deletions(-) diff --git a/cmd/lotus-pcr/main.go b/cmd/lotus-pcr/main.go index 66279c29a..c80bbad48 100644 --- a/cmd/lotus-pcr/main.go +++ b/cmd/lotus-pcr/main.go @@ -7,7 +7,6 @@ import ( "encoding/csv" "fmt" "io/ioutil" - "math" "net/http" _ "net/http/pprof" "os" @@ -213,16 +212,28 @@ var recoverMinersCmd = &cli.Command{ Usage: "do not send any messages", Value: false, }, - &cli.IntFlag{ - Name: "threshold", - EnvVars: []string{"LOTUS_PCR_THRESHOLD"}, - Usage: "total filecoin across all accounts that should be met, if the miner balance drops below zero", - Value: 0, - }, &cli.StringFlag{ Name: "output", Usage: "dump data as a csv format to this file", }, + &cli.IntFlag{ + Name: "miner-recovery-cutoff", + EnvVars: []string{"LOTUS_PCR_MINER_RECOVERY_CUTOFF"}, + Usage: "maximum amount of FIL that can be sent to any one miner before refund percent is applied", + Value: 3000, + }, + &cli.IntFlag{ + Name: "miner-recovery-bonus", + EnvVars: []string{"LOTUS_PCR_MINER_RECOVERY_BONUS"}, + Usage: "additional FIL to send to each miner", + Value: 5, + }, + &cli.IntFlag{ + Name: "miner-recovery-refund-percent", + EnvVars: []string{"LOTUS_PCR_MINER_RECOVERY_REFUND_PERCENT"}, + Usage: "percent of refund to issue", + Value: 110, + }, }, Action: func(cctx *cli.Context) error { ctx := context.Background() @@ -244,13 +255,17 @@ var recoverMinersCmd = &cli.Command{ } dryRun := cctx.Bool("dry-run") - threshold := uint64(cctx.Int("threshold")) + minerRecoveryRefundPercent := cctx.Int("miner-recovery-refund-percent") + minerRecoveryCutoff := uint64(cctx.Int("miner-recovery-cutoff")) + minerRecoveryBonus := uint64(cctx.Int("miner-recovery-bonus")) rf := &refunder{ - api: api, - wallet: from, - dryRun: dryRun, - threshold: types.FromFil(threshold), + api: api, + wallet: from, + dryRun: dryRun, + minerRecoveryRefundPercent: minerRecoveryRefundPercent, + minerRecoveryCutoff: types.FromFil(minerRecoveryCutoff), + minerRecoveryBonus: types.FromFil(minerRecoveryBonus), } refundTipset, err := api.ChainHead(ctx) @@ -286,10 +301,10 @@ var runCmd = &cli.Command{ Usage: "do not wait for chain sync to complete", }, &cli.IntFlag{ - Name: "percent-extra", - EnvVars: []string{"LOTUS_PCR_PERCENT_EXTRA"}, - Usage: "extra funds to send above the refund", - Value: 3, + Name: "refund-percent", + EnvVars: []string{"LOTUS_PCR_REFUND_PERCENT"}, + Usage: "percent of refund to issue", + Value: 103, }, &cli.IntFlag{ Name: "max-message-queue", @@ -327,6 +342,36 @@ var runCmd = &cli.Command{ Usage: "the number of tipsets to delay message processing to smooth chain reorgs", Value: int(build.MessageConfidence), }, + &cli.BoolFlag{ + Name: "miner-recovery", + EnvVars: []string{"LOTUS_PCR_MINER_RECOVERY"}, + Usage: "run the miner recovery job", + Value: false, + }, + &cli.IntFlag{ + Name: "miner-recovery-period", + EnvVars: []string{"LOTUS_PCR_MINER_RECOVERY_PERIOD"}, + Usage: "interval between running miner recovery", + Value: 2880, + }, + &cli.IntFlag{ + Name: "miner-recovery-cutoff", + EnvVars: []string{"LOTUS_PCR_MINER_RECOVERY_CUTOFF"}, + Usage: "maximum amount of FIL that can be sent to any one miner before refund percent is applied", + Value: 3000, + }, + &cli.IntFlag{ + Name: "miner-recovery-bonus", + EnvVars: []string{"LOTUS_PCR_MINER_RECOVERY_BONUS"}, + Usage: "additional FIL to send to each miner", + Value: 5, + }, + &cli.IntFlag{ + Name: "miner-recovery-refund-percent", + EnvVars: []string{"LOTUS_PCR_MINER_RECOVERY_REFUND_PERCENT"}, + Usage: "percent of refund to issue", + Value: 110, + }, }, Action: func(cctx *cli.Context) error { go func() { @@ -365,24 +410,33 @@ var runCmd = &cli.Command{ log.Fatal(err) } - percentExtra := cctx.Int("percent-extra") + refundPercent := cctx.Int("refund-percent") maxMessageQueue := cctx.Int("max-message-queue") dryRun := cctx.Bool("dry-run") preCommitEnabled := cctx.Bool("pre-commit") proveCommitEnabled := cctx.Bool("prove-commit") aggregateTipsets := cctx.Int("aggregate-tipsets") + minerRecoveryEnabled := cctx.Bool("miner-recovery") + minerRecoveryPeriod := abi.ChainEpoch(int64(cctx.Int("miner-recovery-period"))) + minerRecoveryRefundPercent := cctx.Int("miner-recovery-refund-percent") + minerRecoveryCutoff := uint64(cctx.Int("miner-recovery-cutoff")) + minerRecoveryBonus := uint64(cctx.Int("miner-recovery-bonus")) rf := &refunder{ - api: api, - wallet: from, - percentExtra: percentExtra, - dryRun: dryRun, - preCommitEnabled: preCommitEnabled, - proveCommitEnabled: proveCommitEnabled, + api: api, + wallet: from, + refundPercent: refundPercent, + minerRecoveryRefundPercent: minerRecoveryRefundPercent, + minerRecoveryCutoff: types.FromFil(minerRecoveryCutoff), + minerRecoveryBonus: types.FromFil(minerRecoveryBonus), + dryRun: dryRun, + preCommitEnabled: preCommitEnabled, + proveCommitEnabled: proveCommitEnabled, } var refunds *MinersRefund = NewMinersRefund() var rounds int = 0 + nextMinerRecovery := r.MinerRecoveryHeight() + minerRecoveryPeriod for tipset := range tipsetsCh { refunds, err = rf.ProcessTipset(ctx, tipset, refunds) @@ -390,16 +444,33 @@ var runCmd = &cli.Command{ return err } - rounds = rounds + 1 - if rounds < aggregateTipsets { - continue - } - refundTipset, err := api.ChainHead(ctx) if err != nil { return err } + if minerRecoveryEnabled && refundTipset.Height() >= nextMinerRecovery { + recoveryRefund, err := rf.EnsureMinerMinimums(ctx, refundTipset, NewMinersRefund(), "") + if err != nil { + return err + } + + if err := rf.Refund(ctx, "refund to recover miners", refundTipset, recoveryRefund, 0); err != nil { + return err + } + + if err := r.SetMinerRecoveryHeight(tipset.Height()); err != nil { + return err + } + + nextMinerRecovery = r.MinerRecoveryHeight() + minerRecoveryPeriod + } + + rounds = rounds + 1 + if rounds < aggregateTipsets { + continue + } + if err := rf.Refund(ctx, "refund stats", refundTipset, refunds, rounds); err != nil { return err } @@ -502,13 +573,16 @@ type refunderNodeApi interface { } type refunder struct { - api refunderNodeApi - wallet address.Address - percentExtra int - dryRun bool - preCommitEnabled bool - proveCommitEnabled bool - threshold big.Int + api refunderNodeApi + wallet address.Address + refundPercent int + minerRecoveryRefundPercent int + minerRecoveryCutoff big.Int + minerRecoveryBonus big.Int + dryRun bool + preCommitEnabled bool + proveCommitEnabled bool + threshold big.Int } func (r *refunder) FindMiners(ctx context.Context, tipset *types.TipSet, refunds *MinersRefund, owner, worker, control bool) (*MinersRefund, error) { @@ -601,7 +675,7 @@ func (r *refunder) EnsureMinerMinimums(ctx context.Context, tipset *types.TipSet csvOut := csv.NewWriter(w) defer csvOut.Flush() - if err := csvOut.Write([]string{"MinerID", "Sectors", "CombinedBalance", "ProposedSend"}); err != nil { + if err := csvOut.Write([]string{"MinerID", "FaultedSectors", "AvailableBalance", "ProposedRefund"}); err != nil { return nil, err } @@ -644,37 +718,85 @@ func (r *refunder) EnsureMinerMinimums(ctx context.Context, tipset *types.TipSet } } - sectorInfo, err := r.api.StateMinerSectors(ctx, maddr, nil, false, tipset.Key()) + faults, err := r.api.StateMinerFaults(ctx, maddr, tipset.Key()) if err != nil { - log.Errorw("failed to look up miner sectors", "err", err, "height", tipset.Height(), "key", tipset.Key(), "miner", maddr) + log.Errorw("failed to look up miner faults", "err", err, "height", tipset.Height(), "key", tipset.Key(), "miner", maddr) continue } - if len(sectorInfo) == 0 { - log.Debugw("skipping miner with zero sectors", "height", tipset.Height(), "key", tipset.Key(), "miner", maddr) + faultsCount, err := faults.Count() + if err != nil { + log.Errorw("failed to get count of faults", "err", err, "height", tipset.Height(), "key", tipset.Key(), "miner", maddr) + continue + } + + if faultsCount == 0 { + log.Debugw("skipping miner with zero faults", "height", tipset.Height(), "key", tipset.Key(), "miner", maddr) continue } totalAvailableBalance := big.Add(addrSum, minerAvailableBalance) + balanceCutoff := big.Mul(big.Div(big.NewIntUnsigned(faultsCount), big.NewInt(10)), big.NewIntUnsigned(build.FilecoinPrecision)) - numSectorInfo := float64(len(sectorInfo)) - filAmount := uint64(math.Ceil(math.Max(math.Log(numSectorInfo)*math.Sqrt(numSectorInfo)/4, 20))) - attoFilAmount := big.Mul(big.NewIntUnsigned(filAmount), big.NewIntUnsigned(build.FilecoinPrecision)) - - if totalAvailableBalance.GreaterThanEqual(attoFilAmount) { - log.Debugw("skipping over miner with total available balance larger than refund", "height", tipset.Height(), "key", tipset.Key(), "miner", maddr, "available_balance", totalAvailableBalance, "possible_refund", attoFilAmount) + if totalAvailableBalance.GreaterThan(balanceCutoff) { + log.Debugw( + "skipping over miner with total available balance larger than refund", + "height", tipset.Height(), + "key", tipset.Key(), + "miner", maddr, + "available_balance", totalAvailableBalance, + "balance_cutoff", balanceCutoff, + "faults_count", faultsCount, + "available_balance_fil", big.Div(totalAvailableBalance, big.NewIntUnsigned(build.FilecoinPrecision)).Int64(), + "balance_cutoff_fil", big.Div(balanceCutoff, big.NewIntUnsigned(build.FilecoinPrecision)).Int64(), + ) continue } - refundValue := big.Sub(attoFilAmount, totalAvailableBalance) + refundValue := big.Sub(balanceCutoff, totalAvailableBalance) + if r.minerRecoveryRefundPercent > 0 { + refundValue = types.BigMul(types.BigDiv(refundValue, types.NewInt(100)), types.NewInt(uint64(r.minerRecoveryRefundPercent))) + } + + refundValue = big.Add(refundValue, r.minerRecoveryBonus) + + if refundValue.GreaterThan(r.minerRecoveryCutoff) { + log.Infow( + "skipping over miner with refund greater than refund cutoff", + "height", tipset.Height(), + "key", tipset.Key(), + "miner", maddr, + "available_balance", totalAvailableBalance, + "balance_cutoff", balanceCutoff, + "faults_count", faultsCount, + "refund", refundValue, + "available_balance_fil", big.Div(totalAvailableBalance, big.NewIntUnsigned(build.FilecoinPrecision)).Int64(), + "balance_cutoff_fil", big.Div(balanceCutoff, big.NewIntUnsigned(build.FilecoinPrecision)).Int64(), + "refund_fil", big.Div(refundValue, big.NewIntUnsigned(build.FilecoinPrecision)).Int64(), + ) + continue + } refunds.Track(maddr, refundValue) - record := []string{maddr.String(), fmt.Sprintf("%d", len(sectorInfo)), totalAvailableBalance.String(), refundValue.String()} + record := []string{ + maddr.String(), + fmt.Sprintf("%d", faultsCount), + big.Div(totalAvailableBalance, big.NewIntUnsigned(build.FilecoinPrecision)).String(), + big.Div(refundValue, big.NewIntUnsigned(build.FilecoinPrecision)).String(), + } if err := csvOut.Write(record); err != nil { return nil, err } - log.Debugw("processing miner", "miner", maddr, "sectors", len(sectorInfo), "available_balance", totalAvailableBalance, "refund", refundValue) + log.Debugw( + "processing miner", + "miner", maddr, + "faults_count", faultsCount, + "available_balance", totalAvailableBalance, + "refund", refundValue, + "available_balance_fil", big.Div(totalAvailableBalance, big.NewIntUnsigned(build.FilecoinPrecision)).Int64(), + "refund_fil", big.Div(refundValue, big.NewIntUnsigned(build.FilecoinPrecision)).Int64(), + ) } return refunds, nil @@ -794,17 +916,36 @@ func (r *refunder) ProcessTipset(ctx context.Context, tipset *types.TipSet, refu continue } - if r.percentExtra > 0 { - refundValue = types.BigAdd(refundValue, types.BigMul(types.BigDiv(refundValue, types.NewInt(100)), types.NewInt(uint64(r.percentExtra)))) + if r.refundPercent > 0 { + refundValue = types.BigMul(types.BigDiv(refundValue, types.NewInt(100)), types.NewInt(uint64(r.refundPercent))) } - log.Debugw("processing message", "method", messageMethod, "cid", msg.Cid, "from", m.From, "to", m.To, "value", m.Value, "gas_fee_cap", m.GasFeeCap, "gas_premium", m.GasPremium, "gas_used", recps[i].GasUsed, "refund", refundValue) + log.Debugw( + "processing message", + "method", messageMethod, + "cid", msg.Cid, + "from", m.From, + "to", m.To, + "value", m.Value, + "gas_fee_cap", m.GasFeeCap, + "gas_premium", m.GasPremium, + "gas_used", recps[i].GasUsed, + "refund", refundValue, + "refund_fil", big.Div(refundValue, big.NewIntUnsigned(build.FilecoinPrecision)).Int64(), + ) refunds.Track(m.From, refundValue) tipsetRefunds.Track(m.From, refundValue) } - log.Infow("tipset stats", "height", tipset.Height(), "key", tipset.Key(), "total_refunds", tipsetRefunds.TotalRefunds(), "messages_processed", tipsetRefunds.Count()) + log.Infow( + "tipset stats", + "height", tipset.Height(), + "key", tipset.Key(), + "total_refunds", tipsetRefunds.TotalRefunds(), + "total_refunds_fil", big.Div(tipsetRefunds.TotalRefunds(), big.NewIntUnsigned(build.FilecoinPrecision)).Int64(), + "messages_processed", tipsetRefunds.Count(), + ) return refunds, nil } @@ -867,13 +1008,24 @@ func (r *refunder) Refund(ctx context.Context, name string, tipset *types.TipSet refundSum = types.BigAdd(refundSum, msg.Value) } - log.Infow(name, "tipsets_processed", rounds, "height", tipset.Height(), "key", tipset.Key(), "messages_sent", len(messages)-failures, "refund_sum", refundSum, "messages_failures", failures, "messages_processed", refunds.Count()) + log.Infow( + name, + "tipsets_processed", rounds, + "height", tipset.Height(), + "key", tipset.Key(), + "messages_sent", len(messages)-failures, + "refund_sum", refundSum, + "refund_sum_fil", big.Div(refundSum, big.NewIntUnsigned(build.FilecoinPrecision)).Int64(), + "messages_failures", failures, + "messages_processed", refunds.Count(), + ) return nil } type Repo struct { - last abi.ChainEpoch - path string + lastHeight abi.ChainEpoch + lastMinerRecoveryHeight abi.ChainEpoch + path string } func NewRepo(path string) (*Repo, error) { @@ -883,8 +1035,9 @@ func NewRepo(path string) (*Repo, error) { } return &Repo{ - last: 0, - path: path, + lastHeight: 0, + lastMinerRecoveryHeight: 0, + path: path, }, nil } @@ -915,43 +1068,66 @@ func (r *Repo) init() error { return nil } -func (r *Repo) Open() (err error) { - if err = r.init(); err != nil { - return +func (r *Repo) Open() error { + if err := r.init(); err != nil { + return err } - var f *os.File + if err := r.loadHeight(); err != nil { + return err + } - f, err = os.OpenFile(filepath.Join(r.path, "height"), os.O_RDWR|os.O_CREATE, 0644) + if err := r.loadMinerRecoveryHeight(); err != nil { + return err + } + + return nil +} + +func loadChainEpoch(fn string) (abi.ChainEpoch, error) { + f, err := os.OpenFile(fn, os.O_RDWR|os.O_CREATE, 0644) if err != nil { - return + return 0, err } defer func() { err = f.Close() }() - var raw []byte - - raw, err = ioutil.ReadAll(f) + raw, err := ioutil.ReadAll(f) if err != nil { - return + return 0, err } height, err := strconv.Atoi(string(bytes.TrimSpace(raw))) if err != nil { - return + return 0, err } - r.last = abi.ChainEpoch(height) - return + return abi.ChainEpoch(height), nil +} + +func (r *Repo) loadHeight() error { + var err error + r.lastHeight, err = loadChainEpoch(filepath.Join(r.path, "height")) + return err +} + +func (r *Repo) loadMinerRecoveryHeight() error { + var err error + r.lastMinerRecoveryHeight, err = loadChainEpoch(filepath.Join(r.path, "miner_recovery_height")) + return err } func (r *Repo) Height() abi.ChainEpoch { - return r.last + return r.lastHeight +} + +func (r *Repo) MinerRecoveryHeight() abi.ChainEpoch { + return r.lastMinerRecoveryHeight } func (r *Repo) SetHeight(last abi.ChainEpoch) (err error) { - r.last = last + r.lastHeight = last var f *os.File f, err = os.OpenFile(filepath.Join(r.path, "height"), os.O_RDWR, 0644) if err != nil { @@ -962,7 +1138,26 @@ func (r *Repo) SetHeight(last abi.ChainEpoch) (err error) { err = f.Close() }() - if _, err = fmt.Fprintf(f, "%d", r.last); err != nil { + if _, err = fmt.Fprintf(f, "%d", r.lastHeight); err != nil { + return + } + + return +} + +func (r *Repo) SetMinerRecoveryHeight(last abi.ChainEpoch) (err error) { + r.lastMinerRecoveryHeight = last + var f *os.File + f, err = os.OpenFile(filepath.Join(r.path, "miner_recovery_height"), os.O_RDWR, 0644) + if err != nil { + return + } + + defer func() { + err = f.Close() + }() + + if _, err = fmt.Fprintf(f, "%d", r.lastMinerRecoveryHeight); err != nil { return } From 7c3f638f6853bd155b0b4becc59f395d735ea2ce Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Sat, 19 Sep 2020 00:30:24 -0400 Subject: [PATCH 237/885] Abstract FilterEstimate, PreCommitDepositForPower, and InitialPledgeForPower --- chain/actors/builtin/builtin.go | 11 +++-- chain/actors/builtin/power/v0.go | 5 ++- chain/actors/builtin/reward/reward.go | 3 ++ chain/actors/builtin/reward/v0.go | 29 +++++++++++++- node/impl/full/state.go | 58 ++++++++++++--------------- 5 files changed, 68 insertions(+), 38 deletions(-) diff --git a/chain/actors/builtin/builtin.go b/chain/actors/builtin/builtin.go index 4ce77804c..4906f0fa4 100644 --- a/chain/actors/builtin/builtin.go +++ b/chain/actors/builtin/builtin.go @@ -3,9 +3,9 @@ package builtin import ( "fmt" - "github.com/filecoin-project/go-state-types/network" + "github.com/filecoin-project/go-state-types/big" - smoothing0 "github.com/filecoin-project/specs-actors/actors/util/smoothing" + "github.com/filecoin-project/go-state-types/network" ) type Version int @@ -24,5 +24,8 @@ func VersionForNetwork(version network.Version) Version { } } -// TODO: find some way to abstract over this. -type FilterEstimate = smoothing0.FilterEstimate +// TODO: Why does actors have 2 different versions of this? +type FilterEstimate struct { + PositionEstimate big.Int + VelocityEstimate big.Int +} diff --git a/chain/actors/builtin/power/v0.go b/chain/actors/builtin/power/v0.go index 1d5a93d2b..94c8f749e 100644 --- a/chain/actors/builtin/power/v0.go +++ b/chain/actors/builtin/power/v0.go @@ -53,7 +53,10 @@ func (s *state0) MinerNominalPowerMeetsConsensusMinimum(a address.Address) (bool } func (s *state0) TotalPowerSmoothed() (builtin.FilterEstimate, error) { - return *s.State.ThisEpochQAPowerSmoothed, nil + return builtin.FilterEstimate{ + PositionEstimate: s.State.ThisEpochQAPowerSmoothed.PositionEstimate, + VelocityEstimate: s.State.ThisEpochQAPowerSmoothed.VelocityEstimate, + }, nil } func (s *state0) MinerCounts() (uint64, uint64, error) { diff --git a/chain/actors/builtin/reward/reward.go b/chain/actors/builtin/reward/reward.go index b56292fff..3f887914d 100644 --- a/chain/actors/builtin/reward/reward.go +++ b/chain/actors/builtin/reward/reward.go @@ -41,4 +41,7 @@ type State interface { CumsumBaseline() (abi.StoragePower, error) CumsumRealized() (abi.StoragePower, error) + + InitialPledgeForPower(abi.StoragePower, abi.TokenAmount, *builtin.FilterEstimate, abi.TokenAmount) (abi.TokenAmount, error) + PreCommitDepositForPower(builtin.FilterEstimate, abi.StoragePower) (abi.TokenAmount, error) } diff --git a/chain/actors/builtin/reward/v0.go b/chain/actors/builtin/reward/v0.go index 50ad49971..3efcaf93a 100644 --- a/chain/actors/builtin/reward/v0.go +++ b/chain/actors/builtin/reward/v0.go @@ -3,8 +3,10 @@ package reward import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/actors/builtin" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/specs-actors/actors/builtin/reward" "github.com/filecoin-project/specs-actors/actors/util/adt" + "github.com/filecoin-project/specs-actors/actors/util/smoothing" ) type state0 struct { @@ -17,7 +19,10 @@ func (s *state0) ThisEpochReward() (abi.StoragePower, error) { } func (s *state0) ThisEpochRewardSmoothed() (builtin.FilterEstimate, error) { - return *s.State.ThisEpochRewardSmoothed, nil + return builtin.FilterEstimate{ + PositionEstimate: s.State.ThisEpochRewardSmoothed.PositionEstimate, + VelocityEstimate: s.State.ThisEpochRewardSmoothed.VelocityEstimate, + }, nil } func (s *state0) ThisEpochBaselinePower() (abi.StoragePower, error) { @@ -43,3 +48,25 @@ func (s *state0) CumsumBaseline() (abi.StoragePower, error) { func (s *state0) CumsumRealized() (abi.StoragePower, error) { return s.State.CumsumBaseline, nil } + +func (s *state0) InitialPledgeForPower(sectorWeight abi.StoragePower, networkTotalPledge abi.TokenAmount, networkQAPower *builtin.FilterEstimate, circSupply abi.TokenAmount) (abi.TokenAmount, error) { + return miner0.InitialPledgeForPower( + sectorWeight, + s.State.ThisEpochBaselinePower, + networkTotalPledge, + s.State.ThisEpochRewardSmoothed, + &smoothing.FilterEstimate{ + PositionEstimate: networkQAPower.PositionEstimate, + VelocityEstimate: networkQAPower.VelocityEstimate, + }, + circSupply), nil +} + +func (s *state0) PreCommitDepositForPower(networkQAPower builtin.FilterEstimate, sectorWeight abi.StoragePower) (abi.TokenAmount, error) { + return miner0.PreCommitDepositForPower(s.State.ThisEpochRewardSmoothed, + &smoothing.FilterEstimate{ + PositionEstimate: networkQAPower.PositionEstimate, + VelocityEstimate: networkQAPower.VelocityEstimate, + }, + sectorWeight), nil +} diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 76bd3926b..e4400b073 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -5,6 +5,8 @@ import ( "context" "strconv" + builtin2 "github.com/filecoin-project/lotus/chain/actors/builtin" + market0 "github.com/filecoin-project/specs-actors/actors/builtin/market" "github.com/filecoin-project/lotus/chain/actors/builtin/verifreg" @@ -26,7 +28,6 @@ import ( "github.com/filecoin-project/specs-actors/actors/builtin" miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/specs-actors/actors/util/adt" - "github.com/filecoin-project/specs-actors/actors/util/smoothing" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors/builtin/market" @@ -924,7 +925,7 @@ func (a *StateAPI) StateMinerPreCommitDepositForPower(ctx context.Context, maddr sectorWeight = miner0.QAPowerForWeight(ssize, duration, w, vw) } - var powerSmoothed smoothing.FilterEstimate + var powerSmoothed builtin2.FilterEstimate if act, err := state.GetActor(power.Address); err != nil { return types.EmptyInt, xerrors.Errorf("loading miner actor: %w", err) } else if s, err := power.Load(store, act); err != nil { @@ -935,19 +936,20 @@ func (a *StateAPI) StateMinerPreCommitDepositForPower(ctx context.Context, maddr powerSmoothed = p } - var rewardSmoothed smoothing.FilterEstimate - if act, err := state.GetActor(reward.Address); err != nil { + rewardActor, err := state.GetActor(reward.Address) + if err != nil { return types.EmptyInt, xerrors.Errorf("loading miner actor: %w", err) - } else if s, err := reward.Load(store, act); err != nil { - return types.EmptyInt, xerrors.Errorf("loading reward actor state: %w", err) - } else if r, err := s.ThisEpochRewardSmoothed(); err != nil { - return types.EmptyInt, xerrors.Errorf("failed to determine total reward: %w", err) - } else { - rewardSmoothed = r } - // TODO: ActorUpgrade - deposit := miner0.PreCommitDepositForPower(&rewardSmoothed, &powerSmoothed, sectorWeight) + rewardState, err := reward.Load(store, rewardActor) + if err != nil { + return types.EmptyInt, xerrors.Errorf("loading reward actor state: %w", err) + } + + deposit, err := rewardState.PreCommitDepositForPower(powerSmoothed, sectorWeight) + if err != nil { + return big.Zero(), xerrors.Errorf("calculating precommit deposit: %w", err) + } return types.BigDiv(types.BigMul(deposit, initialPledgeNum), initialPledgeDen), nil } @@ -982,12 +984,12 @@ func (a *StateAPI) StateMinerInitialPledgeCollateral(ctx context.Context, maddr // NB: not exactly accurate, but should always lead us to *over* estimate, not under duration := pci.Expiration - ts.Height() - // TODO: handle changes to this function across actor upgrades. + // TODO: ActorUpgrade sectorWeight = miner0.QAPowerForWeight(ssize, duration, w, vw) } var ( - powerSmoothed smoothing.FilterEstimate + powerSmoothed builtin2.FilterEstimate pledgeCollateral abi.TokenAmount ) if act, err := state.GetActor(power.Address); err != nil { @@ -1003,21 +1005,14 @@ func (a *StateAPI) StateMinerInitialPledgeCollateral(ctx context.Context, maddr pledgeCollateral = c } - var ( - rewardSmoothed smoothing.FilterEstimate - baselinePower abi.StoragePower - ) - if act, err := state.GetActor(reward.Address); err != nil { + rewardActor, err := state.GetActor(reward.Address) + if err != nil { return types.EmptyInt, xerrors.Errorf("loading miner actor: %w", err) - } else if s, err := reward.Load(store, act); err != nil { + } + + rewardState, err := reward.Load(store, rewardActor) + if err != nil { return types.EmptyInt, xerrors.Errorf("loading reward actor state: %w", err) - } else if r, err := s.ThisEpochRewardSmoothed(); err != nil { - return types.EmptyInt, xerrors.Errorf("failed to determine total reward: %w", err) - } else if p, err := s.ThisEpochBaselinePower(); err != nil { - return types.EmptyInt, xerrors.Errorf("failed to determine baseline power: %w", err) - } else { - rewardSmoothed = r - baselinePower = p } circSupply, err := a.StateCirculatingSupply(ctx, ts.Key()) @@ -1025,16 +1020,15 @@ func (a *StateAPI) StateMinerInitialPledgeCollateral(ctx context.Context, maddr return big.Zero(), xerrors.Errorf("getting circulating supply: %w", err) } - // TODO: ActorUpgrade - - initialPledge := miner0.InitialPledgeForPower( + initialPledge, err := rewardState.InitialPledgeForPower( sectorWeight, - baselinePower, pledgeCollateral, - &rewardSmoothed, &powerSmoothed, circSupply.FilCirculating, ) + if err != nil { + return big.Zero(), xerrors.Errorf("calculating initial pledge: %w", err) + } return types.BigDiv(types.BigMul(initialPledge, initialPledgeNum), initialPledgeDen), nil } From 6b40e1e30dabc87ddecfa84861b77aaf0046100e Mon Sep 17 00:00:00 2001 From: shepf Date: Sat, 19 Sep 2020 14:04:48 +0800 Subject: [PATCH 238/885] sync wait add heightDiff maybe useful! --- cli/sync.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/cli/sync.go b/cli/sync.go index bff34960e..3b4e2e9fb 100644 --- a/cli/sync.go +++ b/cli/sync.go @@ -249,14 +249,24 @@ func SyncWait(ctx context.Context, napi api.FullNode) error { ss := state.ActiveSyncs[working] + var baseHeight abi.ChainEpoch var target []cid.Cid var theight abi.ChainEpoch + var heightDiff int64 + + if ss.Base != nil { + baseHeight = ss.Base.Height() + heightDiff = int64(ss.Base.Height()) + } if ss.Target != nil { target = ss.Target.Cids() theight = ss.Target.Height() + heightDiff = int64(ss.Target.Height()) - heightDiff + } else { + heightDiff = 0 } - fmt.Printf("\r\x1b[2KWorker %d: Target Height: %d\tTarget: %s\tState: %s\tHeight: %d", working, theight, target, ss.Stage, ss.Height) + fmt.Printf("\r\x1b[2KWorker %d: Base Height: %d\tTarget Height: %d\t Height diff: %d\tTarget: %s\tState: %s\tHeight: %d", working, baseHeight, theight, heightDiff, target, ss.Stage, ss.Height) if time.Now().Unix()-int64(head.MinTimestamp()) < int64(build.BlockDelaySecs) { fmt.Println("\nDone!") From c401d2ec4ec58dd7ddf5e12598c04dcdcd8bf309 Mon Sep 17 00:00:00 2001 From: Jerry <1032246642@qq.com> Date: Sat, 19 Sep 2020 15:48:02 +0800 Subject: [PATCH 239/885] fix storage find error --- cmd/lotus-storage-miner/storage.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/lotus-storage-miner/storage.go b/cmd/lotus-storage-miner/storage.go index 71d88de6d..8a3687877 100644 --- a/cmd/lotus-storage-miner/storage.go +++ b/cmd/lotus-storage-miner/storage.go @@ -370,7 +370,7 @@ var storageFindCmd = &cli.Command{ } fmt.Printf("In %s (%s)\n", info.id, types[:len(types)-2]) - fmt.Printf("\tSealing: %t; Storage: %t\n", info.store.CanSeal, info.store.CanSeal) + fmt.Printf("\tSealing: %t; Storage: %t\n", info.store.CanSeal, info.store.CanStore) if localPath, ok := local[info.id]; ok { fmt.Printf("\tLocal (%s)\n", localPath) } else { From b3d0a5fb4a240500b8088dcbc068737fa27c3323 Mon Sep 17 00:00:00 2001 From: Travis Person Date: Sun, 20 Sep 2020 20:16:20 +0000 Subject: [PATCH 240/885] lotus-shed: add consensus check command --- cmd/lotus-shed/consensus.go | 286 ++++++++++++++++++++++++++++++++++++ cmd/lotus-shed/main.go | 8 + 2 files changed, 294 insertions(+) create mode 100644 cmd/lotus-shed/consensus.go diff --git a/cmd/lotus-shed/consensus.go b/cmd/lotus-shed/consensus.go new file mode 100644 index 000000000..59d9555df --- /dev/null +++ b/cmd/lotus-shed/consensus.go @@ -0,0 +1,286 @@ +package main + +import ( + "bufio" + "fmt" + "io" + "os" + "strconv" + "strings" + "time" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/api/client" + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/types" + lcli "github.com/filecoin-project/lotus/cli" + "github.com/libp2p/go-libp2p-core/peer" + "github.com/multiformats/go-multiaddr" + "github.com/urfave/cli/v2" +) + +var consensusCmd = &cli.Command{ + Name: "consensus", + Usage: "tools for gathering information about consensus between nodes", + Flags: []cli.Flag{}, + Subcommands: []*cli.Command{ + consensusCheckCmd, + }, +} + +type consensusItem struct { + multiaddr multiaddr.Multiaddr + genesisTipset *types.TipSet + targetTipset *types.TipSet + headTipset *types.TipSet + peerID peer.ID + version api.Version + api api.FullNode +} + +var consensusCheckCmd = &cli.Command{ + Name: "check", + Usage: "verify if all nodes agree upon a common tipset for a given tipset height", + Description: `Consensus check verifies that all nodes share a common tipset for a given + height. + + The height flag specifies a chain height to start a comparison from. There are two special + arguments for this flag. All other expected values should be chain tipset heights. + + @common - Use the maximum common chain height between all nodes + @expected - Use the current time and the genesis timestamp to determine a height + + Examples + + Find the highest common tipset and look back 10 tipsets + lotus-shed consensus check --height @common --lookback 10 + + Calculate the expected tipset height and look back 10 tipsets + lotus-shed consensus check --height @expected --lookback 10 + + Check if nodes all share a common genesis + lotus-shed consensus check --height 0 + + Check that all nodes agree upon the tipset for 1day post genesis + lotus-shed consensus check --height 2880 --lookback 0 + `, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "height", + Value: "@common", + Usage: "height of tipset to start check from", + }, + &cli.IntFlag{ + Name: "lookback", + Value: int(build.MessageConfidence * 2), + Usage: "number of tipsets behind to look back when comparing nodes", + }, + }, + Action: func(cctx *cli.Context) error { + filePath := cctx.Args().First() + + var input *bufio.Reader + if cctx.Args().Len() == 0 { + input = bufio.NewReader(os.Stdin) + } else { + var err error + inputFile, err := os.Open(filePath) + if err != nil { + return err + } + defer inputFile.Close() //nolint:errcheck + input = bufio.NewReader(inputFile) + } + + var nodes []*consensusItem + ctx := lcli.ReqContext(cctx) + + for { + strma, errR := input.ReadString('\n') + strma = strings.TrimSpace(strma) + + if len(strma) == 0 { + if errR == io.EOF { + break + } + continue + } + + apima, err := multiaddr.NewMultiaddr(strma) + if err != nil { + return err + } + ainfo := lcli.APIInfo{Addr: apima} + addr, err := ainfo.DialArgs() + if err != nil { + return err + } + + api, closer, err := client.NewFullNodeRPC(cctx.Context, addr, nil) + if err != nil { + return err + } + defer closer() + + peerID, err := api.ID(ctx) + if err != nil { + return err + } + + version, err := api.Version(ctx) + if err != nil { + return err + } + + genesisTipset, err := api.ChainGetGenesis(ctx) + if err != nil { + return err + } + + headTipset, err := api.ChainHead(ctx) + if err != nil { + return err + } + + nodes = append(nodes, &consensusItem{ + genesisTipset: genesisTipset, + headTipset: headTipset, + multiaddr: apima, + api: api, + peerID: peerID, + version: version, + }) + + if errR != nil && errR != io.EOF { + return err + } + + if errR == io.EOF { + break + } + } + + if len(nodes) == 0 { + return fmt.Errorf("no nodes") + } + + genesisBuckets := make(map[types.TipSetKey][]*consensusItem) + for _, node := range nodes { + genesisBuckets[node.genesisTipset.Key()] = append(genesisBuckets[node.genesisTipset.Key()], node) + + } + + if len(genesisBuckets) != 1 { + for _, nodes := range genesisBuckets { + for _, node := range nodes { + log.Errorw( + "genesis do not match", + "genesis_tipset", node.genesisTipset.Key(), + "peer_id", node.peerID, + "version", node.version, + ) + } + } + + return fmt.Errorf("genesis does not match between all nodes") + } + + target := abi.ChainEpoch(0) + + switch cctx.String("height") { + case "@common": + minTipset := nodes[0].headTipset + for _, node := range nodes { + if node.headTipset.Height() < minTipset.Height() { + minTipset = node.headTipset + } + } + + target = minTipset.Height() + case "@expected": + tnow := uint64(time.Now().Unix()) + tgen := nodes[0].genesisTipset.MinTimestamp() + + target = abi.ChainEpoch((tnow - tgen) / build.BlockDelaySecs) + default: + h, err := strconv.Atoi(strings.TrimSpace(cctx.String("height"))) + if err != nil { + return fmt.Errorf("failed to parse string: %s", cctx.String("height")) + } + + target = abi.ChainEpoch(h) + } + + lookback := abi.ChainEpoch(cctx.Int("lookback")) + if lookback > target { + target = abi.ChainEpoch(0) + } else { + target = target - lookback + } + + for _, node := range nodes { + targetTipset, err := node.api.ChainGetTipSetByHeight(ctx, target, types.EmptyTSK) + if err != nil { + log.Errorw("error checking target", "err", err) + node.targetTipset = nil + } else { + node.targetTipset = targetTipset + } + + } + for _, node := range nodes { + log.Debugw( + "node info", + "peer_id", node.peerID, + "version", node.version, + "genesis_tipset", node.genesisTipset.Key(), + "head_tipset", node.headTipset.Key(), + "target_tipset", node.targetTipset.Key(), + ) + } + + targetBuckets := make(map[types.TipSetKey][]*consensusItem) + for _, node := range nodes { + if node.targetTipset == nil { + targetBuckets[types.EmptyTSK] = append(targetBuckets[types.EmptyTSK], node) + continue + } + + targetBuckets[node.targetTipset.Key()] = append(targetBuckets[node.targetTipset.Key()], node) + } + + if nodes, ok := targetBuckets[types.EmptyTSK]; ok { + for _, node := range nodes { + log.Errorw( + "targeted tipset not found", + "peer_id", node.peerID, + "version", node.version, + "genesis_tipset", node.genesisTipset.Key(), + "head_tipset", node.headTipset.Key(), + "target_tipset", node.targetTipset.Key(), + ) + } + + return fmt.Errorf("targeted tipset not found") + } + + if len(targetBuckets) != 1 { + for _, nodes := range targetBuckets { + for _, node := range nodes { + log.Errorw( + "targeted tipset not found", + "peer_id", node.peerID, + "version", node.version, + "genesis_tipset", node.genesisTipset.Key(), + "head_tipset", node.headTipset.Key(), + "target_tipset", node.targetTipset.Key(), + ) + } + } + return fmt.Errorf("nodes not in consensus at tipset height %d", target) + } + + return nil + }, +} diff --git a/cmd/lotus-shed/main.go b/cmd/lotus-shed/main.go index cff3059b6..1a56756d1 100644 --- a/cmd/lotus-shed/main.go +++ b/cmd/lotus-shed/main.go @@ -35,6 +35,7 @@ func main() { mathCmd, mpoolStatsCmd, exportChainCmd, + consensusCmd, } app := &cli.App{ @@ -49,6 +50,13 @@ func main() { Hidden: true, Value: "~/.lotus", // TODO: Consider XDG_DATA_HOME }, + &cli.StringFlag{ + Name: "log-level", + Value: "info", + }, + }, + Before: func(cctx *cli.Context) error { + return logging.SetLogLevel("lotus-shed", cctx.String("log-level")) }, } From b25dd2a00d9c4d346087438d33a8d789a50aa8f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sun, 20 Sep 2020 23:04:45 +0200 Subject: [PATCH 241/885] More correct / fasterer GetSectorsForWinningPoSt --- chain/actors/builtin/miner/v0.go | 12 ++++++------ chain/stmgr/utils.go | 20 +++++++++++++++----- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/chain/actors/builtin/miner/v0.go b/chain/actors/builtin/miner/v0.go index 37df8c217..7985896f3 100644 --- a/chain/actors/builtin/miner/v0.go +++ b/chain/actors/builtin/miner/v0.go @@ -163,9 +163,9 @@ func (s *state0) LoadSectorsFromSet(filter *bitfield.BitField, filterOut bool) ( return nil, err } - var v cbg.Deferred - if err := a.ForEach(&v, func(i int64) error { - if filter != nil { + if filter != nil { + var v cbg.Deferred + if err := a.ForEach(&v, func(i int64) error { set, err := filter.IsSet(uint64(i)) if err != nil { return xerrors.Errorf("filter check error: %w", err) @@ -176,10 +176,10 @@ func (s *state0) LoadSectorsFromSet(filter *bitfield.BitField, filterOut bool) ( return xerrors.Errorf("filtering error: %w", err) } } + return nil + }); err != nil { + return nil, err } - return nil - }); err != nil { - return nil, err } return a, nil diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index 3684b9e77..203d49490 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -197,12 +197,21 @@ func GetSectorsForWinningPoSt(ctx context.Context, pv ffiwrapper.Verifier, sm *S return nil, xerrors.Errorf("failed to load miner actor state: %w", err) } - // TODO (!!): This was partition.Sectors-partition.Faults originally, which was likely very wrong, and will need to be an upgrade - // ^ THE CURRENT THING HERE WON'T SYNC v + // TODO (!!): Actor Update: Make this active sectors - provingSectors, err := miner.AllPartSectors(mas, miner.Partition.ActiveSectors) + allSectors, err := miner.AllPartSectors(mas, miner.Partition.AllSectors) if err != nil { - return nil, xerrors.Errorf("merge partition proving sets: %w", err) + return nil, xerrors.Errorf("get all sectors: %w", err) + } + + faultySectors, err := miner.AllPartSectors(mas, miner.Partition.FaultySectors) + if err != nil { + return nil, xerrors.Errorf("get faulty sectors: %w", err) + } + + provingSectors, err := bitfield.SubtractBitField(allSectors, faultySectors) // TODO: This is wrong, as it can contain faaults, change to just ActiveSectors in an upgrade + if err != nil { + return nil, xerrors.Errorf("calc proving sectors: %w", err) } numProvSect, err := provingSectors.Count() @@ -240,7 +249,8 @@ func GetSectorsForWinningPoSt(ctx context.Context, pv ffiwrapper.Verifier, sm *S return nil, xerrors.Errorf("generating winning post challenges: %w", err) } - sectors, err := mas.LoadSectorsFromSet(&provingSectors, false) + // we don't need to filter here (and it's **very** slow) + sectors, err := mas.LoadSectorsFromSet(nil, false) if err != nil { return nil, xerrors.Errorf("loading proving sectors: %w", err) } From ed285f1114ab1a9fd2a2d5b4f149077a8c3b851c Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Sun, 20 Sep 2020 17:53:46 -0400 Subject: [PATCH 242/885] Abstract SectorOnChainInfo and SectorPreCommitOnChainInfo --- chain/actors/builtin/builtin.go | 9 +++++---- chain/actors/builtin/market/v0.go | 6 +++--- chain/actors/builtin/miner/miner.go | 26 ++++++++++++++++++++++++-- chain/actors/builtin/miner/v0.go | 26 ++++++++++++++++++++++---- chain/actors/builtin/power/v0.go | 5 +---- chain/actors/builtin/reward/v0.go | 5 +---- 6 files changed, 56 insertions(+), 21 deletions(-) diff --git a/chain/actors/builtin/builtin.go b/chain/actors/builtin/builtin.go index 4906f0fa4..9ad976564 100644 --- a/chain/actors/builtin/builtin.go +++ b/chain/actors/builtin/builtin.go @@ -3,7 +3,7 @@ package builtin import ( "fmt" - "github.com/filecoin-project/go-state-types/big" + smoothing0 "github.com/filecoin-project/specs-actors/actors/util/smoothing" "github.com/filecoin-project/go-state-types/network" ) @@ -25,7 +25,8 @@ func VersionForNetwork(version network.Version) Version { } // TODO: Why does actors have 2 different versions of this? -type FilterEstimate struct { - PositionEstimate big.Int - VelocityEstimate big.Int +type FilterEstimate = smoothing0.FilterEstimate + +func FromV0FilterEstimate(v0 smoothing0.FilterEstimate) FilterEstimate { + return (FilterEstimate)(v0) } diff --git a/chain/actors/builtin/market/v0.go b/chain/actors/builtin/market/v0.go index 27eee4c50..e184b668d 100644 --- a/chain/actors/builtin/market/v0.go +++ b/chain/actors/builtin/market/v0.go @@ -26,7 +26,7 @@ func (s *state0) TotalLocked() (abi.TokenAmount, error) { func (s *state0) BalancesChanged(otherState State) bool { otherState0, ok := otherState.(*state0) if !ok { - // there's no way to compare differnt versions of the state, so let's + // there's no way to compare different versions of the state, so let's // just say that means the state of balances has changed return true } @@ -36,7 +36,7 @@ func (s *state0) BalancesChanged(otherState State) bool { func (s *state0) StatesChanged(otherState State) bool { otherState0, ok := otherState.(*state0) if !ok { - // there's no way to compare differnt versions of the state, so let's + // there's no way to compare different versions of the state, so let's // just say that means the state of balances has changed return true } @@ -54,7 +54,7 @@ func (s *state0) States() (DealStates, error) { func (s *state0) ProposalsChanged(otherState State) bool { otherState0, ok := otherState.(*state0) if !ok { - // there's no way to compare differnt versions of the state, so let's + // there's no way to compare different versions of the state, so let's // just say that means the state of balances has changed return true } diff --git a/chain/actors/builtin/miner/miner.go b/chain/actors/builtin/miner/miner.go index d25652d30..704ea1291 100644 --- a/chain/actors/builtin/miner/miner.go +++ b/chain/actors/builtin/miner/miner.go @@ -2,6 +2,7 @@ package miner import ( "github.com/filecoin-project/go-state-types/dline" + "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p-core/peer" cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" @@ -81,9 +82,30 @@ type Partition interface { ActiveSectors() (bitfield.BitField, error) } -type SectorOnChainInfo = miner0.SectorOnChainInfo +type SectorOnChainInfo struct { + SectorNumber abi.SectorNumber + SealProof abi.RegisteredSealProof + SealedCID cid.Cid + DealIDs []abi.DealID + Activation abi.ChainEpoch + Expiration abi.ChainEpoch + DealWeight abi.DealWeight + VerifiedDealWeight abi.DealWeight + InitialPledge abi.TokenAmount + ExpectedDayReward abi.TokenAmount + ExpectedStoragePledge abi.TokenAmount +} + type SectorPreCommitInfo = miner0.SectorPreCommitInfo -type SectorPreCommitOnChainInfo = miner0.SectorPreCommitOnChainInfo + +type SectorPreCommitOnChainInfo struct { + Info SectorPreCommitInfo + PreCommitDeposit abi.TokenAmount + PreCommitEpoch abi.ChainEpoch + DealWeight abi.DealWeight + VerifiedDealWeight abi.DealWeight +} + type PoStPartition = miner0.PoStPartition type RecoveryDeclaration = miner0.RecoveryDeclaration type FaultDeclaration = miner0.FaultDeclaration diff --git a/chain/actors/builtin/miner/v0.go b/chain/actors/builtin/miner/v0.go index 7985896f3..71306fafd 100644 --- a/chain/actors/builtin/miner/v0.go +++ b/chain/actors/builtin/miner/v0.go @@ -62,7 +62,8 @@ func (s *state0) GetSector(num abi.SectorNumber) (*SectorOnChainInfo, error) { return nil, err } - return info, nil + ret := fromV0SectorOnChainInfo(*info) + return &ret, nil } func (s *state0) FindSector(num abi.SectorNumber) (*SectorLocation, error) { @@ -154,7 +155,8 @@ func (s *state0) GetPrecommittedSector(num abi.SectorNumber) (*SectorPreCommitOn return nil, err } - return info, nil + ret := fromV0SectorPreCommitOnChainInfo(*info) + return &ret, nil } func (s *state0) LoadSectorsFromSet(filter *bitfield.BitField, filterOut bool) (adt.Array, error) { @@ -279,7 +281,11 @@ func (s *state0) sectors() (adt.Array, error) { func (s *state0) decodeSectorOnChainInfo(val *cbg.Deferred) (SectorOnChainInfo, error) { var si miner0.SectorOnChainInfo err := si.UnmarshalCBOR(bytes.NewReader(val.Raw)) - return si, err + if err != nil { + return SectorOnChainInfo{}, err + } + + return fromV0SectorOnChainInfo(si), nil } func (s *state0) precommits() (adt.Map, error) { @@ -289,7 +295,11 @@ func (s *state0) precommits() (adt.Map, error) { func (s *state0) decodeSectorPreCommitOnChainInfo(val *cbg.Deferred) (SectorPreCommitOnChainInfo, error) { var sp miner0.SectorPreCommitOnChainInfo err := sp.UnmarshalCBOR(bytes.NewReader(val.Raw)) - return sp, err + if err != nil { + return SectorPreCommitOnChainInfo{}, err + } + + return fromV0SectorPreCommitOnChainInfo(sp), nil } func (d *deadline0) LoadPartition(idx uint64) (Partition, error) { @@ -336,3 +346,11 @@ func (p *partition0) FaultySectors() (bitfield.BitField, error) { func (p *partition0) RecoveringSectors() (bitfield.BitField, error) { return p.Partition.Recoveries, nil } + +func fromV0SectorOnChainInfo(v0 miner0.SectorOnChainInfo) SectorOnChainInfo { + return (SectorOnChainInfo)(v0) +} + +func fromV0SectorPreCommitOnChainInfo(v0 miner0.SectorPreCommitOnChainInfo) SectorPreCommitOnChainInfo { + return (SectorPreCommitOnChainInfo)(v0) +} diff --git a/chain/actors/builtin/power/v0.go b/chain/actors/builtin/power/v0.go index 94c8f749e..e19984f72 100644 --- a/chain/actors/builtin/power/v0.go +++ b/chain/actors/builtin/power/v0.go @@ -53,10 +53,7 @@ func (s *state0) MinerNominalPowerMeetsConsensusMinimum(a address.Address) (bool } func (s *state0) TotalPowerSmoothed() (builtin.FilterEstimate, error) { - return builtin.FilterEstimate{ - PositionEstimate: s.State.ThisEpochQAPowerSmoothed.PositionEstimate, - VelocityEstimate: s.State.ThisEpochQAPowerSmoothed.VelocityEstimate, - }, nil + return builtin.FromV0FilterEstimate(*s.State.ThisEpochQAPowerSmoothed), nil } func (s *state0) MinerCounts() (uint64, uint64, error) { diff --git a/chain/actors/builtin/reward/v0.go b/chain/actors/builtin/reward/v0.go index 3efcaf93a..981905ba5 100644 --- a/chain/actors/builtin/reward/v0.go +++ b/chain/actors/builtin/reward/v0.go @@ -19,10 +19,7 @@ func (s *state0) ThisEpochReward() (abi.StoragePower, error) { } func (s *state0) ThisEpochRewardSmoothed() (builtin.FilterEstimate, error) { - return builtin.FilterEstimate{ - PositionEstimate: s.State.ThisEpochRewardSmoothed.PositionEstimate, - VelocityEstimate: s.State.ThisEpochRewardSmoothed.VelocityEstimate, - }, nil + return builtin.FromV0FilterEstimate(*s.State.ThisEpochRewardSmoothed), nil } func (s *state0) ThisEpochBaselinePower() (abi.StoragePower, error) { From a95e34f742355deea28c2c1376d5ce7e4d1846b8 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Mon, 21 Sep 2020 01:58:20 -0400 Subject: [PATCH 243/885] Fix build --- chain/actors/builtin/miner/cbor_gen.go | 313 +++++++++++++++++++++++++ chain/actors/builtin/miner/v0.go | 46 +++- chain/stmgr/utils.go | 2 +- gen/main.go | 9 + 4 files changed, 358 insertions(+), 12 deletions(-) create mode 100644 chain/actors/builtin/miner/cbor_gen.go diff --git a/chain/actors/builtin/miner/cbor_gen.go b/chain/actors/builtin/miner/cbor_gen.go new file mode 100644 index 000000000..16819d5c4 --- /dev/null +++ b/chain/actors/builtin/miner/cbor_gen.go @@ -0,0 +1,313 @@ +// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. + +package miner + +import ( + "fmt" + "io" + + abi "github.com/filecoin-project/go-state-types/abi" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +var _ = xerrors.Errorf + +var lengthBufSectorOnChainInfo = []byte{139} + +func (t *SectorOnChainInfo) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write(lengthBufSectorOnChainInfo); err != nil { + return err + } + + scratch := make([]byte, 9) + + // t.SectorNumber (abi.SectorNumber) (uint64) + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.SectorNumber)); err != nil { + return err + } + + // t.SealProof (abi.RegisteredSealProof) (int64) + if t.SealProof >= 0 { + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.SealProof)); err != nil { + return err + } + } else { + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.SealProof-1)); err != nil { + return err + } + } + + // t.SealedCID (cid.Cid) (struct) + + if err := cbg.WriteCidBuf(scratch, w, t.SealedCID); err != nil { + return xerrors.Errorf("failed to write cid field t.SealedCID: %w", err) + } + + // t.DealIDs ([]abi.DealID) (slice) + if len(t.DealIDs) > cbg.MaxLength { + return xerrors.Errorf("Slice value in field t.DealIDs was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.DealIDs))); err != nil { + return err + } + for _, v := range t.DealIDs { + if err := cbg.CborWriteHeader(w, cbg.MajUnsignedInt, uint64(v)); err != nil { + return err + } + } + + // t.Activation (abi.ChainEpoch) (int64) + if t.Activation >= 0 { + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Activation)); err != nil { + return err + } + } else { + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.Activation-1)); err != nil { + return err + } + } + + // t.Expiration (abi.ChainEpoch) (int64) + if t.Expiration >= 0 { + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Expiration)); err != nil { + return err + } + } else { + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.Expiration-1)); err != nil { + return err + } + } + + // t.DealWeight (big.Int) (struct) + if err := t.DealWeight.MarshalCBOR(w); err != nil { + return err + } + + // t.VerifiedDealWeight (big.Int) (struct) + if err := t.VerifiedDealWeight.MarshalCBOR(w); err != nil { + return err + } + + // t.InitialPledge (big.Int) (struct) + if err := t.InitialPledge.MarshalCBOR(w); err != nil { + return err + } + + // t.ExpectedDayReward (big.Int) (struct) + if err := t.ExpectedDayReward.MarshalCBOR(w); err != nil { + return err + } + + // t.ExpectedStoragePledge (big.Int) (struct) + if err := t.ExpectedStoragePledge.MarshalCBOR(w); err != nil { + return err + } + return nil +} + +func (t *SectorOnChainInfo) UnmarshalCBOR(r io.Reader) error { + *t = SectorOnChainInfo{} + + br := cbg.GetPeeker(r) + scratch := make([]byte, 8) + + maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 11 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.SectorNumber (abi.SectorNumber) (uint64) + + { + + maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.SectorNumber = abi.SectorNumber(extra) + + } + // t.SealProof (abi.RegisteredSealProof) (int64) + { + maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.SealProof = abi.RegisteredSealProof(extraI) + } + // t.SealedCID (cid.Cid) (struct) + + { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.SealedCID: %w", err) + } + + t.SealedCID = c + + } + // t.DealIDs ([]abi.DealID) (slice) + + maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + + if extra > cbg.MaxLength { + return fmt.Errorf("t.DealIDs: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.DealIDs = make([]abi.DealID, extra) + } + + for i := 0; i < int(extra); i++ { + + maj, val, err := cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return xerrors.Errorf("failed to read uint64 for t.DealIDs slice: %w", err) + } + + if maj != cbg.MajUnsignedInt { + return xerrors.Errorf("value read for array t.DealIDs was not a uint, instead got %d", maj) + } + + t.DealIDs[i] = abi.DealID(val) + } + + // t.Activation (abi.ChainEpoch) (int64) + { + maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.Activation = abi.ChainEpoch(extraI) + } + // t.Expiration (abi.ChainEpoch) (int64) + { + maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.Expiration = abi.ChainEpoch(extraI) + } + // t.DealWeight (big.Int) (struct) + + { + + if err := t.DealWeight.UnmarshalCBOR(br); err != nil { + return xerrors.Errorf("unmarshaling t.DealWeight: %w", err) + } + + } + // t.VerifiedDealWeight (big.Int) (struct) + + { + + if err := t.VerifiedDealWeight.UnmarshalCBOR(br); err != nil { + return xerrors.Errorf("unmarshaling t.VerifiedDealWeight: %w", err) + } + + } + // t.InitialPledge (big.Int) (struct) + + { + + if err := t.InitialPledge.UnmarshalCBOR(br); err != nil { + return xerrors.Errorf("unmarshaling t.InitialPledge: %w", err) + } + + } + // t.ExpectedDayReward (big.Int) (struct) + + { + + if err := t.ExpectedDayReward.UnmarshalCBOR(br); err != nil { + return xerrors.Errorf("unmarshaling t.ExpectedDayReward: %w", err) + } + + } + // t.ExpectedStoragePledge (big.Int) (struct) + + { + + if err := t.ExpectedStoragePledge.UnmarshalCBOR(br); err != nil { + return xerrors.Errorf("unmarshaling t.ExpectedStoragePledge: %w", err) + } + + } + return nil +} diff --git a/chain/actors/builtin/miner/v0.go b/chain/actors/builtin/miner/v0.go index 71306fafd..6d4e7c589 100644 --- a/chain/actors/builtin/miner/v0.go +++ b/chain/actors/builtin/miner/v0.go @@ -165,26 +165,50 @@ func (s *state0) LoadSectorsFromSet(filter *bitfield.BitField, filterOut bool) ( return nil, err } - if filter != nil { - var v cbg.Deferred - if err := a.ForEach(&v, func(i int64) error { + ret := adt0.MakeEmptyArray(s.store) + var v cbg.Deferred + if err := a.ForEach(&v, func(i int64) error { + include := true + if filter != nil { set, err := filter.IsSet(uint64(i)) if err != nil { return xerrors.Errorf("filter check error: %w", err) } if set == filterOut { - err = a.Delete(uint64(i)) - if err != nil { - return xerrors.Errorf("filtering error: %w", err) - } + include = false } - return nil - }); err != nil { - return nil, err } + + if include { + var oci miner0.SectorOnChainInfo + if err := oci.UnmarshalCBOR(bytes.NewReader(v.Raw)); err != nil { + return err + } + + noci := SectorOnChainInfo{ + SectorNumber: oci.SectorNumber, + SealProof: oci.SealProof, + SealedCID: oci.SealedCID, + DealIDs: oci.DealIDs, + Activation: oci.Activation, + Expiration: oci.Expiration, + DealWeight: oci.DealWeight, + VerifiedDealWeight: oci.VerifiedDealWeight, + InitialPledge: oci.InitialPledge, + ExpectedDayReward: oci.ExpectedDayReward, + ExpectedStoragePledge: oci.ExpectedStoragePledge, + } + + if err := ret.Set(uint64(i), &noci); err != nil { + return err + } + } + return nil + }); err != nil { + return nil, err } - return a, nil + return ret, nil } func (s *state0) LoadPreCommittedSectors() (adt.Map, error) { diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index 203d49490..20d7bd54f 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -169,7 +169,7 @@ func GetMinerSectorSet(ctx context.Context, sm *StateManager, ts *types.TipSet, var sset []*miner.ChainSectorInfo var v cbg.Deferred if err := sectors.ForEach(&v, func(i int64) error { - var oci miner0.SectorOnChainInfo + var oci miner.SectorOnChainInfo if err := oci.UnmarshalCBOR(bytes.NewReader(v.Raw)); err != nil { return err } diff --git a/gen/main.go b/gen/main.go index be227663c..90b24e3a7 100644 --- a/gen/main.go +++ b/gen/main.go @@ -4,6 +4,8 @@ import ( "fmt" "os" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + gen "github.com/whyrusleeping/cbor-gen" "github.com/filecoin-project/lotus/api" @@ -76,4 +78,11 @@ func main() { os.Exit(1) } + err = gen.WriteTupleEncodersToFile("./chain/actors/builtin/miner/cbor_gen.go", "miner", + miner.SectorOnChainInfo{}, + ) + if err != nil { + fmt.Println(err) + os.Exit(1) + } } From b355eb75ebd99f4c4ea84b0c1288afdcdc91c0af Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Mon, 21 Sep 2020 02:19:32 -0400 Subject: [PATCH 244/885] Add compile-time checks that actors interfaces are correctly implemented --- chain/actors/builtin/account/v0.go | 2 ++ chain/actors/builtin/init/v0.go | 2 ++ chain/actors/builtin/market/v0.go | 2 ++ chain/actors/builtin/miner/v0.go | 2 ++ chain/actors/builtin/multisig/v0.go | 2 ++ chain/actors/builtin/paych/v0.go | 2 ++ chain/actors/builtin/power/v0.go | 2 ++ chain/actors/builtin/reward/v0.go | 2 ++ chain/actors/builtin/verifreg/v0.go | 2 ++ 9 files changed, 18 insertions(+) diff --git a/chain/actors/builtin/account/v0.go b/chain/actors/builtin/account/v0.go index 535255d0e..30bafbfd3 100644 --- a/chain/actors/builtin/account/v0.go +++ b/chain/actors/builtin/account/v0.go @@ -6,6 +6,8 @@ import ( "github.com/filecoin-project/specs-actors/actors/builtin/account" ) +var _ State = (*state0)(nil) + type state0 struct { account.State store adt.Store diff --git a/chain/actors/builtin/init/v0.go b/chain/actors/builtin/init/v0.go index 0e8395a08..717ed9669 100644 --- a/chain/actors/builtin/init/v0.go +++ b/chain/actors/builtin/init/v0.go @@ -13,6 +13,8 @@ import ( adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" ) +var _ State = (*state0)(nil) + type state0 struct { init_.State store adt.Store diff --git a/chain/actors/builtin/market/v0.go b/chain/actors/builtin/market/v0.go index e184b668d..433445dab 100644 --- a/chain/actors/builtin/market/v0.go +++ b/chain/actors/builtin/market/v0.go @@ -12,6 +12,8 @@ import ( cbg "github.com/whyrusleeping/cbor-gen" ) +var _ State = (*state0)(nil) + type state0 struct { market.State store adt.Store diff --git a/chain/actors/builtin/miner/v0.go b/chain/actors/builtin/miner/v0.go index 6d4e7c589..010ae6699 100644 --- a/chain/actors/builtin/miner/v0.go +++ b/chain/actors/builtin/miner/v0.go @@ -17,6 +17,8 @@ import ( miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" ) +var _ State = (*state0)(nil) + type state0 struct { miner0.State store adt.Store diff --git a/chain/actors/builtin/multisig/v0.go b/chain/actors/builtin/multisig/v0.go index ded834d5f..5de5ca6ad 100644 --- a/chain/actors/builtin/multisig/v0.go +++ b/chain/actors/builtin/multisig/v0.go @@ -6,6 +6,8 @@ import ( "github.com/filecoin-project/specs-actors/actors/builtin/multisig" ) +var _ State = (*state0)(nil) + type state0 struct { multisig.State store adt.Store diff --git a/chain/actors/builtin/paych/v0.go b/chain/actors/builtin/paych/v0.go index 0c4b2f218..b69cf46ce 100644 --- a/chain/actors/builtin/paych/v0.go +++ b/chain/actors/builtin/paych/v0.go @@ -9,6 +9,8 @@ import ( adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" ) +var _ State = (*state0)(nil) + type state0 struct { paych.State store adt.Store diff --git a/chain/actors/builtin/power/v0.go b/chain/actors/builtin/power/v0.go index e19984f72..f2fe96dad 100644 --- a/chain/actors/builtin/power/v0.go +++ b/chain/actors/builtin/power/v0.go @@ -8,6 +8,8 @@ import ( "github.com/filecoin-project/specs-actors/actors/util/adt" ) +var _ State = (*state0)(nil) + type state0 struct { power0.State store adt.Store diff --git a/chain/actors/builtin/reward/v0.go b/chain/actors/builtin/reward/v0.go index 981905ba5..df7117b67 100644 --- a/chain/actors/builtin/reward/v0.go +++ b/chain/actors/builtin/reward/v0.go @@ -9,6 +9,8 @@ import ( "github.com/filecoin-project/specs-actors/actors/util/smoothing" ) +var _ State = (*state0)(nil) + type state0 struct { reward.State store adt.Store diff --git a/chain/actors/builtin/verifreg/v0.go b/chain/actors/builtin/verifreg/v0.go index c397d6679..f64c27310 100644 --- a/chain/actors/builtin/verifreg/v0.go +++ b/chain/actors/builtin/verifreg/v0.go @@ -11,6 +11,8 @@ import ( "github.com/filecoin-project/lotus/chain/actors/adt" ) +var _ State = (*state0)(nil) + type state0 struct { verifreg0.State store adt.Store From f135ec84682f072626e90b9b43f65ad535ddaa33 Mon Sep 17 00:00:00 2001 From: vyzo Date: Mon, 21 Sep 2020 09:21:25 +0300 Subject: [PATCH 245/885] fix handling of end of sync --- chain/sync.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/chain/sync.go b/chain/sync.go index 03ae1cd4f..74dc5aa1a 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -1486,11 +1486,7 @@ func (syncer *Syncer) iterFullTipsets(ctx context.Context, headers []*types.TipS batchSize := concurrentSyncRequests * syncRequestBatchSize if i < batchSize { - if i == 0 { - batchSize = 1 - } else { - batchSize = i - } + batchSize = i + 1 } ss.SetStage(api.StageFetchingMessages) From 286fe042715cce1e11d2bcf67de8cdd28558190d Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Mon, 21 Sep 2020 02:52:45 -0400 Subject: [PATCH 246/885] Add some TODOs --- chain/stmgr/utils.go | 58 +++++++++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index 20d7bd54f..1056818a6 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -9,11 +9,7 @@ import ( "runtime" "strings" - init0 "github.com/filecoin-project/specs-actors/actors/builtin/init" - market0 "github.com/filecoin-project/specs-actors/actors/builtin/market" - power0 "github.com/filecoin-project/specs-actors/actors/builtin/power" - - miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" + "github.com/filecoin-project/specs-actors/actors/builtin/cron" saruntime "github.com/filecoin-project/specs-actors/actors/runtime" "github.com/filecoin-project/specs-actors/actors/runtime/proof" @@ -28,11 +24,14 @@ import ( "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/lotus/chain/actors/builtin/power" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" - "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/builtin/account" - "github.com/filecoin-project/specs-actors/actors/builtin/cron" + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" + account0 "github.com/filecoin-project/specs-actors/actors/builtin/account" + init0 "github.com/filecoin-project/specs-actors/actors/builtin/init" + market0 "github.com/filecoin-project/specs-actors/actors/builtin/market" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" msig0 "github.com/filecoin-project/specs-actors/actors/builtin/multisig" - "github.com/filecoin-project/specs-actors/actors/builtin/paych" + paych0 "github.com/filecoin-project/specs-actors/actors/builtin/paych" + power0 "github.com/filecoin-project/specs-actors/actors/builtin/power" reward0 "github.com/filecoin-project/specs-actors/actors/builtin/reward" verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" @@ -89,7 +88,8 @@ func GetPower(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr add } func GetPowerRaw(ctx context.Context, sm *StateManager, st cid.Cid, maddr address.Address) (power.Claim, power.Claim, bool, error) { - act, err := sm.LoadActorRaw(ctx, builtin.StoragePowerActorAddr, st) + // TODO: ActorUpgrade + act, err := sm.LoadActorRaw(ctx, builtin0.StoragePowerActorAddr, st) if err != nil { return power.Claim{}, power.Claim{}, false, xerrors.Errorf("(get sset) failed to load power actor state: %w", err) } @@ -308,7 +308,8 @@ func StateMinerInfo(ctx context.Context, sm *StateManager, ts *types.TipSet, mad } func GetMinerSlashed(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) (bool, error) { - act, err := sm.LoadActor(ctx, builtin.StoragePowerActorAddr, ts) + // TODO: ActorUpgrade + act, err := sm.LoadActor(ctx, builtin0.StoragePowerActorAddr, ts) if err != nil { return false, xerrors.Errorf("failed to load power actor: %w", err) } @@ -331,7 +332,8 @@ func GetMinerSlashed(ctx context.Context, sm *StateManager, ts *types.TipSet, ma } func GetStorageDeal(ctx context.Context, sm *StateManager, dealID abi.DealID, ts *types.TipSet) (*api.MarketDeal, error) { - act, err := sm.LoadActor(ctx, builtin.StorageMarketActorAddr, ts) + // TODO: ActorUpgrade + act, err := sm.LoadActor(ctx, builtin0.StorageMarketActorAddr, ts) if err != nil { return nil, xerrors.Errorf("failed to load market actor: %w", err) } @@ -375,7 +377,8 @@ func GetStorageDeal(ctx context.Context, sm *StateManager, dealID abi.DealID, ts } func ListMinerActors(ctx context.Context, sm *StateManager, ts *types.TipSet) ([]address.Address, error) { - act, err := sm.LoadActor(ctx, builtin.StoragePowerActorAddr, ts) + // TODO: ActorUpgrade + act, err := sm.LoadActor(ctx, builtin0.StoragePowerActorAddr, ts) if err != nil { return nil, xerrors.Errorf("failed to load power actor: %w", err) } @@ -565,16 +568,16 @@ var MethodsMap = map[cid.Cid]map[abi.MethodNum]MethodMeta{} func init() { cidToMethods := map[cid.Cid][2]interface{}{ // builtin.SystemActorCodeID: {builtin.MethodsSystem, system.Actor{} }- apparently it doesn't have methods - builtin.InitActorCodeID: {builtin.MethodsInit, init0.Actor{}}, - builtin.CronActorCodeID: {builtin.MethodsCron, cron.Actor{}}, - builtin.AccountActorCodeID: {builtin.MethodsAccount, account.Actor{}}, - builtin.StoragePowerActorCodeID: {builtin.MethodsPower, power0.Actor{}}, - builtin.StorageMinerActorCodeID: {builtin.MethodsMiner, miner0.Actor{}}, - builtin.StorageMarketActorCodeID: {builtin.MethodsMarket, market0.Actor{}}, - builtin.PaymentChannelActorCodeID: {builtin.MethodsPaych, paych.Actor{}}, - builtin.MultisigActorCodeID: {builtin.MethodsMultisig, msig0.Actor{}}, - builtin.RewardActorCodeID: {builtin.MethodsReward, reward0.Actor{}}, - builtin.VerifiedRegistryActorCodeID: {builtin.MethodsVerifiedRegistry, verifreg0.Actor{}}, + builtin0.InitActorCodeID: {builtin0.MethodsInit, init0.Actor{}}, + builtin0.CronActorCodeID: {builtin0.MethodsCron, cron.Actor{}}, + builtin0.AccountActorCodeID: {builtin0.MethodsAccount, account0.Actor{}}, + builtin0.StoragePowerActorCodeID: {builtin0.MethodsPower, power0.Actor{}}, + builtin0.StorageMinerActorCodeID: {builtin0.MethodsMiner, miner0.Actor{}}, + builtin0.StorageMarketActorCodeID: {builtin0.MethodsMarket, market0.Actor{}}, + builtin0.PaymentChannelActorCodeID: {builtin0.MethodsPaych, paych0.Actor{}}, + builtin0.MultisigActorCodeID: {builtin0.MethodsMultisig, msig0.Actor{}}, + builtin0.RewardActorCodeID: {builtin0.MethodsReward, reward0.Actor{}}, + builtin0.VerifiedRegistryActorCodeID: {builtin0.MethodsVerifiedRegistry, verifreg0.Actor{}}, } for c, m := range cidToMethods { @@ -582,7 +585,7 @@ func init() { methods := make(map[abi.MethodNum]MethodMeta, len(exports)) // Explicitly add send, it's special. - methods[builtin.MethodSend] = MethodMeta{ + methods[builtin0.MethodSend] = MethodMeta{ Name: "Send", Params: reflect.TypeOf(new(abi.EmptyValue)), Ret: reflect.TypeOf(new(abi.EmptyValue)), @@ -622,9 +625,9 @@ func init() { } switch abi.MethodNum(number) { - case builtin.MethodSend: + case builtin0.MethodSend: panic("method 0 is reserved for Send") - case builtin.MethodConstructor: + case builtin0.MethodConstructor: if fnName != "Constructor" { panic("method 1 is reserved for Constructor") } @@ -654,7 +657,8 @@ func GetReturnType(ctx context.Context, sm *StateManager, to address.Address, me } func MinerHasMinPower(ctx context.Context, sm *StateManager, addr address.Address, ts *types.TipSet) (bool, error) { - pact, err := sm.LoadActor(ctx, builtin.StoragePowerActorAddr, ts) + // TODO: ActorUpgrade + pact, err := sm.LoadActor(ctx, builtin0.StoragePowerActorAddr, ts) if err != nil { return false, xerrors.Errorf("loading power actor state: %w", err) } From a307e4593a005888395db34e5af4691ee50d765d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 21 Sep 2020 09:52:57 +0200 Subject: [PATCH 247/885] wdpost: Fix TestWDPostDoPost --- storage/wdpost_run_test.go | 54 ++++++++++++++++++++++++++++---------- 1 file changed, 40 insertions(+), 14 deletions(-) diff --git a/storage/wdpost_run_test.go b/storage/wdpost_run_test.go index 58014d632..bd8e35d4f 100644 --- a/storage/wdpost_run_test.go +++ b/storage/wdpost_run_test.go @@ -5,18 +5,16 @@ import ( "context" "testing" - "github.com/filecoin-project/go-state-types/dline" - "github.com/stretchr/testify/require" + "github.com/ipfs/go-cid" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-bitfield" - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/types" - "github.com/ipfs/go-cid" - "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" + "github.com/filecoin-project/go-state-types/dline" "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/specs-actors/actors/builtin" builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" @@ -24,7 +22,9 @@ import ( "github.com/filecoin-project/specs-actors/actors/runtime/proof" tutils "github.com/filecoin-project/specs-actors/support/testing" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + "github.com/filecoin-project/lotus/chain/types" ) type mockStorageMinerAPI struct { @@ -39,7 +39,10 @@ func newMockStorageMinerAPI() *mockStorageMinerAPI { } func (m *mockStorageMinerAPI) StateMinerInfo(ctx context.Context, a address.Address, key types.TipSetKey) (miner.MinerInfo, error) { - panic("implement me") + return miner.MinerInfo{ + Worker: tutils.NewIDAddr(nil, 101), + Owner: tutils.NewIDAddr(nil, 101), + }, nil } func (m *mockStorageMinerAPI) StateNetworkVersion(ctx context.Context, key types.TipSetKey) (network.Version, error) { @@ -167,7 +170,13 @@ func TestWDPostDoPost(t *testing.T) { worker: workerAct, } - di := &dline.Info{} + di := &dline.Info{ + WPoStPeriodDeadlines: miner0.WPoStPeriodDeadlines, + WPoStProvingPeriod: miner0.WPoStProvingPeriod, + WPoStChallengeWindow: miner0.WPoStChallengeWindow, + WPoStChallengeLookback: miner0.WPoStChallengeLookback, + FaultDeclarationCutoff: miner0.FaultDeclarationCutoff, + } ts := mockTipSet(t) scheduler.doPost(ctx, di, ts) @@ -232,7 +241,20 @@ func (m *mockStorageMinerAPI) StateSectorPartition(ctx context.Context, maddr ad } func (m *mockStorageMinerAPI) StateMinerProvingDeadline(ctx context.Context, address address.Address, key types.TipSetKey) (*dline.Info, error) { - panic("implement me") + return &dline.Info{ + CurrentEpoch: 0, + PeriodStart: 0, + Index: 0, + Open: 0, + Close: 0, + Challenge: 0, + FaultCutoff: 0, + WPoStPeriodDeadlines: miner0.WPoStPeriodDeadlines, + WPoStProvingPeriod: miner0.WPoStProvingPeriod, + WPoStChallengeWindow: miner0.WPoStChallengeWindow, + WPoStChallengeLookback: miner0.WPoStChallengeLookback, + FaultDeclarationCutoff: miner0.FaultDeclarationCutoff, + }, nil } func (m *mockStorageMinerAPI) StateMinerPreCommitDepositForPower(ctx context.Context, address address.Address, info miner.SectorPreCommitInfo, key types.TipSetKey) (types.BigInt, error) { @@ -270,11 +292,15 @@ func (m *mockStorageMinerAPI) StateMinerRecoveries(ctx context.Context, address } func (m *mockStorageMinerAPI) StateAccountKey(ctx context.Context, address address.Address, key types.TipSetKey) (address.Address, error) { - panic("implement me") + return address, nil } func (m *mockStorageMinerAPI) GasEstimateMessageGas(ctx context.Context, message *types.Message, spec *api.MessageSendSpec, key types.TipSetKey) (*types.Message, error) { - panic("implement me") + msg := *message + msg.GasFeeCap = big.NewInt(1) + msg.GasPremium = big.NewInt(1) + msg.GasLimit = 2 + return &msg, nil } func (m *mockStorageMinerAPI) ChainHead(ctx context.Context) (*types.TipSet, error) { @@ -306,15 +332,15 @@ func (m *mockStorageMinerAPI) ChainGetTipSet(ctx context.Context, key types.TipS } func (m *mockStorageMinerAPI) WalletSign(ctx context.Context, address address.Address, bytes []byte) (*crypto.Signature, error) { - panic("implement me") + return nil, nil } func (m *mockStorageMinerAPI) WalletBalance(ctx context.Context, address address.Address) (types.BigInt, error) { - panic("implement me") + return big.NewInt(333), nil } func (m *mockStorageMinerAPI) WalletHas(ctx context.Context, address address.Address) (bool, error) { - panic("implement me") + return true, nil } var _ storageMinerApi = &mockStorageMinerAPI{} From cfa041ca8ad7d54c1faf7e942ce70ef3e1313068 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Mon, 14 Sep 2020 12:41:00 +0100 Subject: [PATCH 248/885] refactor: chaos caller validation --- conformance/chaos/actor.go | 34 +++++--- conformance/chaos/cbor_gen.go | 159 ++++++++++++++++++++++++++++++++++ conformance/chaos/gen/gen.go | 1 + 3 files changed, 181 insertions(+), 13 deletions(-) diff --git a/conformance/chaos/actor.go b/conformance/chaos/actor.go index 005a06f0d..bc68b086f 100644 --- a/conformance/chaos/actor.go +++ b/conformance/chaos/actor.go @@ -7,8 +7,6 @@ import ( "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/runtime" "github.com/ipfs/go-cid" - - typegen "github.com/whyrusleeping/cbor-gen" ) //go:generate go run ./gen @@ -31,10 +29,14 @@ type Actor struct{} type CallerValidationBranch int64 const ( + // CallerValidationBranchNone causes no caller validation to take place. CallerValidationBranchNone CallerValidationBranch = iota + // CallerValidationBranchTwice causes Runtime.ValidateImmediateCallerAcceptAny to be called twice. CallerValidationBranchTwice - CallerValidationBranchAddrNilSet - CallerValidationBranchTypeNilSet + // CallerValidationBranchIs causes caller validation against CallerValidationArgs.Addrs. + CallerValidationBranchIs + // CallerValidationBranchType causes caller validation against CallerValidationArgs.Types. + CallerValidationBranchType ) // MutateStateBranch is an enum used to select the type of state mutation to attempt. @@ -123,23 +125,29 @@ func (a Actor) Constructor(_ runtime.Runtime, _ *abi.EmptyValue) *abi.EmptyValue panic("constructor should not be called; the Chaos actor is a singleton actor") } +// CallerValidationArgs are the arguments to Actor.CallerValidation. +type CallerValidationArgs struct { + Branch CallerValidationBranch + Addrs []address.Address + Types []cid.Cid +} + // CallerValidation violates VM call validation constraints. // // CallerValidationBranchNone performs no validation. // CallerValidationBranchTwice validates twice. -// CallerValidationBranchAddrNilSet validates against an empty caller -// address set. -// CallerValidationBranchTypeNilSet validates against an empty caller type set. -func (a Actor) CallerValidation(rt runtime.Runtime, branch *typegen.CborInt) *abi.EmptyValue { - switch CallerValidationBranch(*branch) { +// CallerValidationBranchIs validates caller against CallerValidationArgs.Addrs. +// CallerValidationBranchType validates caller against CallerValidationArgs.Types. +func (a Actor) CallerValidation(rt runtime.Runtime, args *CallerValidationArgs) *abi.EmptyValue { + switch args.Branch { case CallerValidationBranchNone: case CallerValidationBranchTwice: rt.ValidateImmediateCallerAcceptAny() rt.ValidateImmediateCallerAcceptAny() - case CallerValidationBranchAddrNilSet: - rt.ValidateImmediateCallerIs() - case CallerValidationBranchTypeNilSet: - rt.ValidateImmediateCallerType() + case CallerValidationBranchIs: + rt.ValidateImmediateCallerIs(args.Addrs...) + case CallerValidationBranchType: + rt.ValidateImmediateCallerType(args.Types...) default: panic("invalid branch passed to CallerValidation") } diff --git a/conformance/chaos/cbor_gen.go b/conformance/chaos/cbor_gen.go index 61e36e661..882af7026 100644 --- a/conformance/chaos/cbor_gen.go +++ b/conformance/chaos/cbor_gen.go @@ -6,8 +6,10 @@ import ( "fmt" "io" + address "github.com/filecoin-project/go-address" abi "github.com/filecoin-project/go-state-types/abi" exitcode "github.com/filecoin-project/go-state-types/exitcode" + cid "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" xerrors "golang.org/x/xerrors" ) @@ -115,6 +117,163 @@ func (t *State) UnmarshalCBOR(r io.Reader) error { return nil } +var lengthBufCallerValidationArgs = []byte{131} + +func (t *CallerValidationArgs) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write(lengthBufCallerValidationArgs); err != nil { + return err + } + + scratch := make([]byte, 9) + + // t.Branch (chaos.CallerValidationBranch) (int64) + if t.Branch >= 0 { + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Branch)); err != nil { + return err + } + } else { + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.Branch-1)); err != nil { + return err + } + } + + // t.Addrs ([]address.Address) (slice) + if len(t.Addrs) > cbg.MaxLength { + return xerrors.Errorf("Slice value in field t.Addrs was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Addrs))); err != nil { + return err + } + for _, v := range t.Addrs { + if err := v.MarshalCBOR(w); err != nil { + return err + } + } + + // t.Types ([]cid.Cid) (slice) + if len(t.Types) > cbg.MaxLength { + return xerrors.Errorf("Slice value in field t.Types was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Types))); err != nil { + return err + } + for _, v := range t.Types { + if err := cbg.WriteCidBuf(scratch, w, v); err != nil { + return xerrors.Errorf("failed writing cid field t.Types: %w", err) + } + } + return nil +} + +func (t *CallerValidationArgs) UnmarshalCBOR(r io.Reader) error { + *t = CallerValidationArgs{} + + br := cbg.GetPeeker(r) + scratch := make([]byte, 8) + + maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 3 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.Branch (chaos.CallerValidationBranch) (int64) + { + maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.Branch = CallerValidationBranch(extraI) + } + // t.Addrs ([]address.Address) (slice) + + maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + + if extra > cbg.MaxLength { + return fmt.Errorf("t.Addrs: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.Addrs = make([]address.Address, extra) + } + + for i := 0; i < int(extra); i++ { + + var v address.Address + if err := v.UnmarshalCBOR(br); err != nil { + return err + } + + t.Addrs[i] = v + } + + // t.Types ([]cid.Cid) (slice) + + maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + + if extra > cbg.MaxLength { + return fmt.Errorf("t.Types: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.Types = make([]cid.Cid, extra) + } + + for i := 0; i < int(extra); i++ { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("reading cid field t.Types failed: %w", err) + } + t.Types[i] = c + } + + return nil +} + var lengthBufCreateActorArgs = []byte{132} func (t *CreateActorArgs) MarshalCBOR(w io.Writer) error { diff --git a/conformance/chaos/gen/gen.go b/conformance/chaos/gen/gen.go index ea04aee21..ac97da99e 100644 --- a/conformance/chaos/gen/gen.go +++ b/conformance/chaos/gen/gen.go @@ -9,6 +9,7 @@ import ( func main() { if err := gen.WriteTupleEncodersToFile("./cbor_gen.go", "chaos", chaos.State{}, + chaos.CallerValidationArgs{}, chaos.CreateActorArgs{}, chaos.ResolveAddressResponse{}, chaos.SendArgs{}, From ab53abf04eaee92973daade2d0f55a44b2ca972a Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 15 Sep 2020 14:00:33 +0100 Subject: [PATCH 249/885] feat: add CallerValidationBranchIsReceiver and tests --- conformance/chaos/actor.go | 4 ++ conformance/chaos/actor_test.go | 119 ++++++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+) diff --git a/conformance/chaos/actor.go b/conformance/chaos/actor.go index bc68b086f..c56abcb8b 100644 --- a/conformance/chaos/actor.go +++ b/conformance/chaos/actor.go @@ -35,6 +35,8 @@ const ( CallerValidationBranchTwice // CallerValidationBranchIs causes caller validation against CallerValidationArgs.Addrs. CallerValidationBranchIs + // CallerValidationBranchIsReceiver causes validation that the caller was also the receiver. + CallerValidationBranchIsReceiver // CallerValidationBranchType causes caller validation against CallerValidationArgs.Types. CallerValidationBranchType ) @@ -146,6 +148,8 @@ func (a Actor) CallerValidation(rt runtime.Runtime, args *CallerValidationArgs) rt.ValidateImmediateCallerAcceptAny() case CallerValidationBranchIs: rt.ValidateImmediateCallerIs(args.Addrs...) + case CallerValidationBranchIsReceiver: + rt.ValidateImmediateCallerIs(rt.Message().Receiver()) case CallerValidationBranchType: rt.ValidateImmediateCallerType(args.Types...) default: diff --git a/conformance/chaos/actor_test.go b/conformance/chaos/actor_test.go index 2e899980d..c9d804309 100644 --- a/conformance/chaos/actor_test.go +++ b/conformance/chaos/actor_test.go @@ -2,6 +2,9 @@ package chaos import ( "context" + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/specs-actors/actors/builtin" + "github.com/ipfs/go-cid" "testing" "github.com/filecoin-project/go-state-types/abi" @@ -25,6 +28,108 @@ func TestSingleton(t *testing.T) { rt.Verify() } +func TestCallerValidationNone(t *testing.T) { + receiver := atesting.NewIDAddr(t, 100) + builder := mock.NewBuilder(context.Background(), receiver) + + rt := builder.Build(t) + var a Actor + + rt.Call(a.CallerValidation, &CallerValidationArgs{Branch: CallerValidationBranchNone}) + rt.Verify() +} + +func TestCallerValidationIs(t *testing.T) { + caller := atesting.NewIDAddr(t, 100) + receiver := atesting.NewIDAddr(t, 101) + builder := mock.NewBuilder(context.Background(), receiver) + + rt := builder.Build(t) + rt.SetCaller(caller, builtin.AccountActorCodeID) + var a Actor + + caddrs := []address.Address{atesting.NewIDAddr(t, 101)} + + rt.ExpectValidateCallerAddr(caddrs...) + // FIXME: https://github.com/filecoin-project/specs-actors/pull/1155 + rt.ExpectAbort(exitcode.ErrForbidden, func() { + rt.Call(a.CallerValidation, &CallerValidationArgs{ + Branch: CallerValidationBranchIs, + Addrs: caddrs, + }) + }) + rt.Verify() + + rt.ExpectValidateCallerAddr(caller) + rt.Call(a.CallerValidation, &CallerValidationArgs{ + Branch: CallerValidationBranchIs, + Addrs: []address.Address{caller}, + }) + rt.Verify() +} + +func TestCallerValidationIsReceiver(t *testing.T) { + caller := atesting.NewIDAddr(t, 100) + receiver := atesting.NewIDAddr(t, 101) + builder := mock.NewBuilder(context.Background(), receiver) + + rt := builder.Build(t) + var a Actor + + rt.SetCaller(caller, builtin.AccountActorCodeID) + rt.ExpectValidateCallerAddr(receiver) + // FIXME: https://github.com/filecoin-project/specs-actors/pull/1155 + rt.ExpectAbort(exitcode.ErrForbidden, func() { + rt.Call(a.CallerValidation, &CallerValidationArgs{Branch: CallerValidationBranchIsReceiver}) + }) + rt.Verify() + + rt.SetCaller(receiver, builtin.AccountActorCodeID) + rt.ExpectValidateCallerAddr(receiver) + rt.Call(a.CallerValidation, &CallerValidationArgs{Branch: CallerValidationBranchIsReceiver}) + rt.Verify() +} + +func TestCallerValidationType(t *testing.T) { + caller := atesting.NewIDAddr(t, 100) + receiver := atesting.NewIDAddr(t, 101) + builder := mock.NewBuilder(context.Background(), receiver) + + rt := builder.Build(t) + rt.SetCaller(caller, builtin.AccountActorCodeID) + var a Actor + + rt.ExpectValidateCallerType(builtin.CronActorCodeID) + // FIXME: https://github.com/filecoin-project/specs-actors/pull/1155 + rt.ExpectAbort(exitcode.ErrForbidden, func() { + rt.Call(a.CallerValidation, &CallerValidationArgs{ + Branch: CallerValidationBranchType, + Types: []cid.Cid{builtin.CronActorCodeID}, + }) + }) + rt.Verify() + + rt.ExpectValidateCallerType(builtin.AccountActorCodeID) + rt.Call(a.CallerValidation, &CallerValidationArgs{ + Branch: CallerValidationBranchType, + Types: []cid.Cid{builtin.AccountActorCodeID}, + }) + rt.Verify() +} + +func TestCallerValidationInvalidBranch(t *testing.T) { + receiver := atesting.NewIDAddr(t, 100) + builder := mock.NewBuilder(context.Background(), receiver) + + rt := builder.Build(t) + var a Actor + + rt.ExpectAssertionFailure("invalid branch passed to CallerValidation", func() { + rt.Call(a.CallerValidation, &CallerValidationArgs{Branch: -1}) + }) + rt.Verify() +} + func TestDeleteActor(t *testing.T) { receiver := atesting.NewIDAddr(t, 100) beneficiary := atesting.NewIDAddr(t, 101) @@ -118,6 +223,20 @@ func TestMutateStateReadonly(t *testing.T) { rt.Verify() } +func TestMutateStateInvalidBranch(t *testing.T) { + receiver := atesting.NewIDAddr(t, 100) + builder := mock.NewBuilder(context.Background(), receiver) + + rt := builder.Build(t) + var a Actor + + rt.ExpectValidateCallerAny() + rt.ExpectAssertionFailure("unknown mutation type", func() { + rt.Call(a.MutateState, &MutateStateArgs{Branch: -1}) + }) + rt.Verify() +} + func TestAbortWith(t *testing.T) { receiver := atesting.NewIDAddr(t, 100) builder := mock.NewBuilder(context.Background(), receiver) From 49a94b7cbab9b2acef1fa1765035f4c8a5ce9002 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 15 Sep 2020 14:10:20 +0100 Subject: [PATCH 250/885] chore: appease linter --- conformance/chaos/actor_test.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/conformance/chaos/actor_test.go b/conformance/chaos/actor_test.go index c9d804309..675c617ab 100644 --- a/conformance/chaos/actor_test.go +++ b/conformance/chaos/actor_test.go @@ -2,16 +2,15 @@ package chaos import ( "context" - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/ipfs/go-cid" "testing" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/exitcode" "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/support/mock" atesting "github.com/filecoin-project/specs-actors/support/testing" + "github.com/ipfs/go-cid" ) func TestSingleton(t *testing.T) { From d607ca6118a9d859b99da43e1ba74251faf0d498 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 15 Sep 2020 14:56:58 +0100 Subject: [PATCH 251/885] refactor: remove IsReceiver branch --- conformance/chaos/actor.go | 4 ---- conformance/chaos/actor_test.go | 22 ---------------------- 2 files changed, 26 deletions(-) diff --git a/conformance/chaos/actor.go b/conformance/chaos/actor.go index c56abcb8b..bc68b086f 100644 --- a/conformance/chaos/actor.go +++ b/conformance/chaos/actor.go @@ -35,8 +35,6 @@ const ( CallerValidationBranchTwice // CallerValidationBranchIs causes caller validation against CallerValidationArgs.Addrs. CallerValidationBranchIs - // CallerValidationBranchIsReceiver causes validation that the caller was also the receiver. - CallerValidationBranchIsReceiver // CallerValidationBranchType causes caller validation against CallerValidationArgs.Types. CallerValidationBranchType ) @@ -148,8 +146,6 @@ func (a Actor) CallerValidation(rt runtime.Runtime, args *CallerValidationArgs) rt.ValidateImmediateCallerAcceptAny() case CallerValidationBranchIs: rt.ValidateImmediateCallerIs(args.Addrs...) - case CallerValidationBranchIsReceiver: - rt.ValidateImmediateCallerIs(rt.Message().Receiver()) case CallerValidationBranchType: rt.ValidateImmediateCallerType(args.Types...) default: diff --git a/conformance/chaos/actor_test.go b/conformance/chaos/actor_test.go index 675c617ab..a6345da53 100644 --- a/conformance/chaos/actor_test.go +++ b/conformance/chaos/actor_test.go @@ -67,28 +67,6 @@ func TestCallerValidationIs(t *testing.T) { rt.Verify() } -func TestCallerValidationIsReceiver(t *testing.T) { - caller := atesting.NewIDAddr(t, 100) - receiver := atesting.NewIDAddr(t, 101) - builder := mock.NewBuilder(context.Background(), receiver) - - rt := builder.Build(t) - var a Actor - - rt.SetCaller(caller, builtin.AccountActorCodeID) - rt.ExpectValidateCallerAddr(receiver) - // FIXME: https://github.com/filecoin-project/specs-actors/pull/1155 - rt.ExpectAbort(exitcode.ErrForbidden, func() { - rt.Call(a.CallerValidation, &CallerValidationArgs{Branch: CallerValidationBranchIsReceiver}) - }) - rt.Verify() - - rt.SetCaller(receiver, builtin.AccountActorCodeID) - rt.ExpectValidateCallerAddr(receiver) - rt.Call(a.CallerValidation, &CallerValidationArgs{Branch: CallerValidationBranchIsReceiver}) - rt.Verify() -} - func TestCallerValidationType(t *testing.T) { caller := atesting.NewIDAddr(t, 100) receiver := atesting.NewIDAddr(t, 101) From dfaad08a8b57b08b3bbde9ef633dec97bb497ce7 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 15 Sep 2020 15:03:57 +0100 Subject: [PATCH 252/885] fix: update cbor gen file --- conformance/chaos/cbor_gen.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/conformance/chaos/cbor_gen.go b/conformance/chaos/cbor_gen.go index 882af7026..0d34264ee 100644 --- a/conformance/chaos/cbor_gen.go +++ b/conformance/chaos/cbor_gen.go @@ -6,12 +6,12 @@ import ( "fmt" "io" - address "github.com/filecoin-project/go-address" - abi "github.com/filecoin-project/go-state-types/abi" - exitcode "github.com/filecoin-project/go-state-types/exitcode" - cid "github.com/ipfs/go-cid" + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/exitcode" + "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" - xerrors "golang.org/x/xerrors" + "golang.org/x/xerrors" ) var _ = xerrors.Errorf From c1a9cf7edd9a8d089e97fccc70e97b73dffc93de Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 15 Sep 2020 15:47:11 +0100 Subject: [PATCH 253/885] refactor: renames --- conformance/chaos/actor.go | 16 ++++++++-------- conformance/chaos/actor_test.go | 8 ++++---- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/conformance/chaos/actor.go b/conformance/chaos/actor.go index bc68b086f..477204374 100644 --- a/conformance/chaos/actor.go +++ b/conformance/chaos/actor.go @@ -33,10 +33,10 @@ const ( CallerValidationBranchNone CallerValidationBranch = iota // CallerValidationBranchTwice causes Runtime.ValidateImmediateCallerAcceptAny to be called twice. CallerValidationBranchTwice - // CallerValidationBranchIs causes caller validation against CallerValidationArgs.Addrs. - CallerValidationBranchIs - // CallerValidationBranchType causes caller validation against CallerValidationArgs.Types. - CallerValidationBranchType + // CallerValidationBranchIsAddress causes caller validation against CallerValidationArgs.Addrs. + CallerValidationBranchIsAddress + // CallerValidationBranchIsType causes caller validation against CallerValidationArgs.Types. + CallerValidationBranchIsType ) // MutateStateBranch is an enum used to select the type of state mutation to attempt. @@ -136,17 +136,17 @@ type CallerValidationArgs struct { // // CallerValidationBranchNone performs no validation. // CallerValidationBranchTwice validates twice. -// CallerValidationBranchIs validates caller against CallerValidationArgs.Addrs. -// CallerValidationBranchType validates caller against CallerValidationArgs.Types. +// CallerValidationBranchIsAddress validates caller against CallerValidationArgs.Addrs. +// CallerValidationBranchIsType validates caller against CallerValidationArgs.Types. func (a Actor) CallerValidation(rt runtime.Runtime, args *CallerValidationArgs) *abi.EmptyValue { switch args.Branch { case CallerValidationBranchNone: case CallerValidationBranchTwice: rt.ValidateImmediateCallerAcceptAny() rt.ValidateImmediateCallerAcceptAny() - case CallerValidationBranchIs: + case CallerValidationBranchIsAddress: rt.ValidateImmediateCallerIs(args.Addrs...) - case CallerValidationBranchType: + case CallerValidationBranchIsType: rt.ValidateImmediateCallerType(args.Types...) default: panic("invalid branch passed to CallerValidation") diff --git a/conformance/chaos/actor_test.go b/conformance/chaos/actor_test.go index a6345da53..2061efb82 100644 --- a/conformance/chaos/actor_test.go +++ b/conformance/chaos/actor_test.go @@ -53,7 +53,7 @@ func TestCallerValidationIs(t *testing.T) { // FIXME: https://github.com/filecoin-project/specs-actors/pull/1155 rt.ExpectAbort(exitcode.ErrForbidden, func() { rt.Call(a.CallerValidation, &CallerValidationArgs{ - Branch: CallerValidationBranchIs, + Branch: CallerValidationBranchIsAddress, Addrs: caddrs, }) }) @@ -61,7 +61,7 @@ func TestCallerValidationIs(t *testing.T) { rt.ExpectValidateCallerAddr(caller) rt.Call(a.CallerValidation, &CallerValidationArgs{ - Branch: CallerValidationBranchIs, + Branch: CallerValidationBranchIsAddress, Addrs: []address.Address{caller}, }) rt.Verify() @@ -80,7 +80,7 @@ func TestCallerValidationType(t *testing.T) { // FIXME: https://github.com/filecoin-project/specs-actors/pull/1155 rt.ExpectAbort(exitcode.ErrForbidden, func() { rt.Call(a.CallerValidation, &CallerValidationArgs{ - Branch: CallerValidationBranchType, + Branch: CallerValidationBranchIsType, Types: []cid.Cid{builtin.CronActorCodeID}, }) }) @@ -88,7 +88,7 @@ func TestCallerValidationType(t *testing.T) { rt.ExpectValidateCallerType(builtin.AccountActorCodeID) rt.Call(a.CallerValidation, &CallerValidationArgs{ - Branch: CallerValidationBranchType, + Branch: CallerValidationBranchIsType, Types: []cid.Cid{builtin.AccountActorCodeID}, }) rt.Verify() From d9951cdff402b400015b5213097c8db67049f8b2 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Mon, 21 Sep 2020 10:10:31 +0100 Subject: [PATCH 254/885] fix: regen --- conformance/chaos/cbor_gen.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/conformance/chaos/cbor_gen.go b/conformance/chaos/cbor_gen.go index 0d34264ee..882af7026 100644 --- a/conformance/chaos/cbor_gen.go +++ b/conformance/chaos/cbor_gen.go @@ -6,12 +6,12 @@ import ( "fmt" "io" - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/exitcode" - "github.com/ipfs/go-cid" + address "github.com/filecoin-project/go-address" + abi "github.com/filecoin-project/go-state-types/abi" + exitcode "github.com/filecoin-project/go-state-types/exitcode" + cid "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" - "golang.org/x/xerrors" + xerrors "golang.org/x/xerrors" ) var _ = xerrors.Errorf From a79291070e216309e81aa929b0409bece4bf1820 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Mon, 21 Sep 2020 11:53:38 +0100 Subject: [PATCH 255/885] chore: update test-vectors --- extern/test-vectors | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/test-vectors b/extern/test-vectors index 7d3becbeb..9db1dd8c9 160000 --- a/extern/test-vectors +++ b/extern/test-vectors @@ -1 +1 @@ -Subproject commit 7d3becbeb5b932baed419c43390595b5e5cece12 +Subproject commit 9db1dd8c9f85bd80eb624e77130a02a3aaf3a1f9 From d9f2b909ac5127589f7e385df5897d7bcd476da6 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Mon, 21 Sep 2020 11:58:45 +0100 Subject: [PATCH 256/885] chore: update test-vectors --- extern/test-vectors | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/test-vectors b/extern/test-vectors index 9db1dd8c9..6bea015ed 160000 --- a/extern/test-vectors +++ b/extern/test-vectors @@ -1 +1 @@ -Subproject commit 9db1dd8c9f85bd80eb624e77130a02a3aaf3a1f9 +Subproject commit 6bea015edddde116001a4251dce3c4a9966c25d9 From 58a85f378cc00301014bf67b6445a8df5c54e295 Mon Sep 17 00:00:00 2001 From: vyzo Date: Mon, 21 Sep 2020 18:33:31 +0300 Subject: [PATCH 257/885] refactor response compressed index validation into its own function --- chain/exchange/client.go | 64 ++++++++++++++++++++++++---------------- 1 file changed, 39 insertions(+), 25 deletions(-) diff --git a/chain/exchange/client.go b/chain/exchange/client.go index 57563d5b2..20a130c1d 100644 --- a/chain/exchange/client.go +++ b/chain/exchange/client.go @@ -210,31 +210,9 @@ func (c *client) processResponse(req *Request, res *Response) (*validatedRespons // If the headers were also returned check that the compression // indexes are valid before `toFullTipSets()` is called by the // consumer. - for tipsetIdx := 0; tipsetIdx < resLength; tipsetIdx++ { - msgs := res.Chain[tipsetIdx].Messages - blocksNum := len(res.Chain[tipsetIdx].Blocks) - if len(msgs.BlsIncludes) != blocksNum { - return nil, xerrors.Errorf("BlsIncludes (%d) does not match number of blocks (%d)", - len(msgs.BlsIncludes), blocksNum) - } - if len(msgs.SecpkIncludes) != blocksNum { - return nil, xerrors.Errorf("SecpkIncludes (%d) does not match number of blocks (%d)", - len(msgs.SecpkIncludes), blocksNum) - } - for blockIdx := 0; blockIdx < blocksNum; blockIdx++ { - for _, mi := range msgs.BlsIncludes[blockIdx] { - if int(mi) >= len(msgs.Bls) { - return nil, xerrors.Errorf("index in BlsIncludes (%d) exceeds number of messages (%d)", - mi, len(msgs.Bls)) - } - } - for _, mi := range msgs.SecpkIncludes[blockIdx] { - if int(mi) >= len(msgs.Secpk) { - return nil, xerrors.Errorf("index in SecpkIncludes (%d) exceeds number of messages (%d)", - mi, len(msgs.Secpk)) - } - } - } + err := c.validateCompressedIndices(res.Chain) + if err != nil { + return nil, err } } } @@ -242,6 +220,42 @@ func (c *client) processResponse(req *Request, res *Response) (*validatedRespons return validRes, nil } +func (c *client) validateCompressedIndices(chain []*BSTipSet) error { + resLength := len(chain) + for tipsetIdx := 0; tipsetIdx < resLength; tipsetIdx++ { + msgs := chain[tipsetIdx].Messages + blocksNum := len(chain[tipsetIdx].Blocks) + + if len(msgs.BlsIncludes) != blocksNum { + return xerrors.Errorf("BlsIncludes (%d) does not match number of blocks (%d)", + len(msgs.BlsIncludes), blocksNum) + } + + if len(msgs.SecpkIncludes) != blocksNum { + return xerrors.Errorf("SecpkIncludes (%d) does not match number of blocks (%d)", + len(msgs.SecpkIncludes), blocksNum) + } + + for blockIdx := 0; blockIdx < blocksNum; blockIdx++ { + for _, mi := range msgs.BlsIncludes[blockIdx] { + if int(mi) >= len(msgs.Bls) { + return xerrors.Errorf("index in BlsIncludes (%d) exceeds number of messages (%d)", + mi, len(msgs.Bls)) + } + } + + for _, mi := range msgs.SecpkIncludes[blockIdx] { + if int(mi) >= len(msgs.Secpk) { + return xerrors.Errorf("index in SecpkIncludes (%d) exceeds number of messages (%d)", + mi, len(msgs.Secpk)) + } + } + } + } + + return nil +} + // GetBlocks implements Client.GetBlocks(). Refer to the godocs there. func (c *client) GetBlocks(ctx context.Context, tsk types.TipSetKey, count int) ([]*types.TipSet, error) { ctx, span := trace.StartSpan(ctx, "bsync.GetBlocks") From 44b52941f7a2af1c3cd69fd1324ce7454f48eb32 Mon Sep 17 00:00:00 2001 From: vyzo Date: Mon, 21 Sep 2020 18:50:41 +0300 Subject: [PATCH 258/885] add option to validate messages against expected tipset --- chain/exchange/client.go | 16 ++++++++++++++++ chain/exchange/protocol.go | 13 +++++++++---- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/chain/exchange/client.go b/chain/exchange/client.go index 20a130c1d..fcdcd544a 100644 --- a/chain/exchange/client.go +++ b/chain/exchange/client.go @@ -215,6 +215,22 @@ func (c *client) processResponse(req *Request, res *Response) (*validatedRespons return nil, err } } + + if options.ValidateMessages { + chain := make([]*BSTipSet, 0, resLength) + for i, resChain := range res.Chain { + next := &BSTipSet{ + Blocks: req.TipSets[i].Blocks(), + Messages: resChain.Messages, + } + chain = append(chain, next) + } + + err := c.validateCompressedIndices(chain) + if err != nil { + return nil, err + } + } } return validRes, nil diff --git a/chain/exchange/protocol.go b/chain/exchange/protocol.go index ac02cf60f..b1fe69bd2 100644 --- a/chain/exchange/protocol.go +++ b/chain/exchange/protocol.go @@ -57,6 +57,8 @@ type Request struct { // Request options, see `Options` type for more details. Compressed // in a single `uint64` to save space. Options uint64 + // Request tipsets for validation + TipSets []*types.TipSet } // `Request` processed and validated to query the tipsets needed. @@ -71,13 +73,15 @@ type validatedRequest struct { const ( Headers = 1 << iota Messages + Validate ) // Decompressed options into separate struct members for easy access // during internal processing.. type parsedOptions struct { - IncludeHeaders bool - IncludeMessages bool + IncludeHeaders bool + IncludeMessages bool + ValidateMessages bool } func (options *parsedOptions) noOptionsSet() bool { @@ -87,8 +91,9 @@ func (options *parsedOptions) noOptionsSet() bool { func parseOptions(optfield uint64) *parsedOptions { return &parsedOptions{ - IncludeHeaders: optfield&(uint64(Headers)) != 0, - IncludeMessages: optfield&(uint64(Messages)) != 0, + IncludeHeaders: optfield&(uint64(Headers)) != 0, + IncludeMessages: optfield&(uint64(Messages)) != 0, + ValidateMessages: optfield&(uint64(Validate)) != 0, } } From 5663b2d697c4bc809373f759e9a978b74b6846ab Mon Sep 17 00:00:00 2001 From: vyzo Date: Mon, 21 Sep 2020 18:58:52 +0300 Subject: [PATCH 259/885] change GetChainMessages api to include tipsets for validation --- chain/exchange/client.go | 9 +++++++-- chain/exchange/interfaces.go | 7 +++---- chain/sync.go | 4 ++-- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/chain/exchange/client.go b/chain/exchange/client.go index fcdcd544a..a99362745 100644 --- a/chain/exchange/client.go +++ b/chain/exchange/client.go @@ -217,6 +217,7 @@ func (c *client) processResponse(req *Request, res *Response) (*validatedRespons } if options.ValidateMessages { + // if the request includes target tipsets, validate against them chain := make([]*BSTipSet, 0, resLength) for i, resChain := range res.Chain { next := &BSTipSet{ @@ -318,7 +319,10 @@ func (c *client) GetFullTipSet(ctx context.Context, peer peer.ID, tsk types.TipS } // GetChainMessages implements Client.GetChainMessages(). Refer to the godocs there. -func (c *client) GetChainMessages(ctx context.Context, head *types.TipSet, length uint64) ([]*CompactedMessages, error) { +func (c *client) GetChainMessages(ctx context.Context, tipsets []*types.TipSet) ([]*CompactedMessages, error) { + head := tipsets[0] + length := uint64(len(tipsets)) + ctx, span := trace.StartSpan(ctx, "GetChainMessages") if span.IsRecordingEvents() { span.AddAttributes( @@ -331,7 +335,8 @@ func (c *client) GetChainMessages(ctx context.Context, head *types.TipSet, lengt req := &Request{ Head: head.Cids(), Length: length, - Options: Messages, + Options: Messages | Validate, + TipSets: tipsets, } validRes, err := c.doRequest(ctx, req, nil) diff --git a/chain/exchange/interfaces.go b/chain/exchange/interfaces.go index 79d8fd4b1..acc0854da 100644 --- a/chain/exchange/interfaces.go +++ b/chain/exchange/interfaces.go @@ -32,10 +32,9 @@ type Client interface { // or less. GetBlocks(ctx context.Context, tsk types.TipSetKey, count int) ([]*types.TipSet, error) - // GetChainMessages fetches messages from the network, from the provided - // tipset *backwards*, returning the messages from as many tipsets as the - // count parameter, or less. - GetChainMessages(ctx context.Context, head *types.TipSet, length uint64) ([]*CompactedMessages, error) + // GetChainMessages fetches messages from the network, starting from the first provided tipset + // and returning messages from as many tipsets as requested or less. + GetChainMessages(ctx context.Context, tipsets []*types.TipSet) ([]*CompactedMessages, error) // GetFullTipSet fetches a full tipset from a given peer. If successful, // the fetched object contains block headers and all messages in full form. diff --git a/chain/sync.go b/chain/sync.go index 74dc5aa1a..b47365b4b 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -1555,7 +1555,7 @@ func (syncer *Syncer) fetchMessages(ctx context.Context, headers []*types.TipSet failed := false for offset := 0; !failed && offset < nreq; { nextI := j + offset - nextHeader := headers[nextI] + lastI := j + nreq var requestErr error var requestResult []*exchange.CompactedMessages @@ -1566,7 +1566,7 @@ func (syncer *Syncer) fetchMessages(ctx context.Context, headers []*types.TipSet log.Infof("fetching messages at %d", startOffset+nextI) } - result, err := syncer.Exchange.GetChainMessages(ctx, nextHeader, uint64(nreq-offset)) + result, err := syncer.Exchange.GetChainMessages(ctx, headers[nextI:lastI]) if err != nil { requestErr = multierror.Append(requestErr, err) } else { From 0b5e4a961245f09634d9db750f928249b60ff3a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 21 Sep 2020 18:28:32 +0200 Subject: [PATCH 260/885] Make GetSectorsForWinningPoSt fast again --- chain/actors/adt/adt.go | 17 ++++++ chain/actors/builtin/miner/miner.go | 4 +- chain/actors/builtin/miner/v0.go | 83 +++++++++++++++-------------- 3 files changed, 61 insertions(+), 43 deletions(-) diff --git a/chain/actors/adt/adt.go b/chain/actors/adt/adt.go index fd5ee3f87..81c2e3a86 100644 --- a/chain/actors/adt/adt.go +++ b/chain/actors/adt/adt.go @@ -55,3 +55,20 @@ func AsArray(store Store, root cid.Cid, version network.Version) (Array, error) } return nil, xerrors.Errorf("unknown network version: %d", version) } + +type ROnlyArray interface { + Get(idx uint64, v cbor.Unmarshaler) (bool, error) + ForEach(v cbor.Unmarshaler, fn func(idx int64) error) error +} + +type ProxyArray struct { + GetFunc func(idx uint64, v cbor.Unmarshaler) (bool, error) + ForEachFunc func(v cbor.Unmarshaler, fn func(idx int64) error) error +} + +func (a *ProxyArray) Get(idx uint64, v cbor.Unmarshaler) (bool, error) { + return a.GetFunc(idx, v) +} +func (a *ProxyArray) ForEach(v cbor.Unmarshaler, fn func(idx int64) error) error { + return a.ForEachFunc(v, fn) +} diff --git a/chain/actors/builtin/miner/miner.go b/chain/actors/builtin/miner/miner.go index 704ea1291..b20539b8f 100644 --- a/chain/actors/builtin/miner/miner.go +++ b/chain/actors/builtin/miner/miner.go @@ -1,7 +1,6 @@ package miner import ( - "github.com/filecoin-project/go-state-types/dline" "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p-core/peer" cbg "github.com/whyrusleeping/cbor-gen" @@ -11,6 +10,7 @@ import ( "github.com/filecoin-project/go-bitfield" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/cbor" + "github.com/filecoin-project/go-state-types/dline" builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" @@ -47,7 +47,7 @@ type State interface { FindSector(abi.SectorNumber) (*SectorLocation, error) GetSectorExpiration(abi.SectorNumber) (*SectorExpiration, error) GetPrecommittedSector(abi.SectorNumber) (*SectorPreCommitOnChainInfo, error) - LoadSectorsFromSet(filter *bitfield.BitField, filterOut bool) (adt.Array, error) + LoadSectorsFromSet(filter *bitfield.BitField, filterOut bool) (adt.ROnlyArray, error) IsAllocated(abi.SectorNumber) (bool, error) LoadDeadline(idx uint64) (Deadline, error) diff --git a/chain/actors/builtin/miner/v0.go b/chain/actors/builtin/miner/v0.go index 010ae6699..ddc0b71a9 100644 --- a/chain/actors/builtin/miner/v0.go +++ b/chain/actors/builtin/miner/v0.go @@ -4,17 +4,19 @@ import ( "bytes" "errors" - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-bitfield" - "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/dline" - "github.com/filecoin-project/lotus/chain/actors/adt" - adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/libp2p/go-libp2p-core/peer" cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-bitfield" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/cbor" + "github.com/filecoin-project/go-state-types/dline" miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" + adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" + + "github.com/filecoin-project/lotus/chain/actors/adt" ) var _ State = (*state0)(nil) @@ -161,56 +163,55 @@ func (s *state0) GetPrecommittedSector(num abi.SectorNumber) (*SectorPreCommitOn return &ret, nil } -func (s *state0) LoadSectorsFromSet(filter *bitfield.BitField, filterOut bool) (adt.Array, error) { +func (s *state0) LoadSectorsFromSet(filter *bitfield.BitField, filterOut bool) (adt.ROnlyArray, error) { a, err := adt0.AsArray(s.store, s.State.Sectors) if err != nil { return nil, err } - ret := adt0.MakeEmptyArray(s.store) - var v cbg.Deferred - if err := a.ForEach(&v, func(i int64) error { + incl := func(i uint64) (bool, error) { include := true if filter != nil { - set, err := filter.IsSet(uint64(i)) + set, err := filter.IsSet(i) if err != nil { - return xerrors.Errorf("filter check error: %w", err) + return false, xerrors.Errorf("filter check error: %w", err) } if set == filterOut { include = false } } - - if include { - var oci miner0.SectorOnChainInfo - if err := oci.UnmarshalCBOR(bytes.NewReader(v.Raw)); err != nil { - return err - } - - noci := SectorOnChainInfo{ - SectorNumber: oci.SectorNumber, - SealProof: oci.SealProof, - SealedCID: oci.SealedCID, - DealIDs: oci.DealIDs, - Activation: oci.Activation, - Expiration: oci.Expiration, - DealWeight: oci.DealWeight, - VerifiedDealWeight: oci.VerifiedDealWeight, - InitialPledge: oci.InitialPledge, - ExpectedDayReward: oci.ExpectedDayReward, - ExpectedStoragePledge: oci.ExpectedStoragePledge, - } - - if err := ret.Set(uint64(i), &noci); err != nil { - return err - } - } - return nil - }); err != nil { - return nil, err + return include, nil } - return ret, nil + return &adt.ProxyArray{ + GetFunc: func(idx uint64, v cbor.Unmarshaler) (bool, error) { + i, err := incl(idx) + if err != nil { + return false, err + } + if !i { + return false, nil + } + + // TODO: ActorUpgrade potentially convert + + return a.Get(idx, v) + }, + ForEachFunc: func(v cbor.Unmarshaler, fn func(int64) error) error { + // TODO: ActorUpgrade potentially convert the output + return a.ForEach(v, func(i int64) error { + include, err := incl(uint64(i)) + if err != nil { + return err + } + if !include { + return nil + } + + return fn(i) + }) + }, + }, nil } func (s *state0) LoadPreCommittedSectors() (adt.Map, error) { From a5e7fd8f5c0e8ed2c900bd23860e16d2336411f1 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Mon, 21 Sep 2020 18:43:57 +0200 Subject: [PATCH 261/885] Re-add docs-check step to circle --- .circleci/config.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index d8f149889..acd447f69 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -346,6 +346,15 @@ jobs: - run: git --no-pager diff - run: git --no-pager diff --quiet + docs-check: + executor: golang + steps: + - install-deps + - prepare + - run: make docsgen + - run: git --no-pager diff + - run: git --no-pager diff --quiet + lint: &lint description: | Run golangci-lint. @@ -415,6 +424,7 @@ workflows: - mod-tidy-check - gofmt - cbor-gen-check + - docs-check - test: codecov-upload: true test-suite-name: full From 1096f8acc6900dccc7db160cafe9284362f17db2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 21 Sep 2020 18:59:05 +0200 Subject: [PATCH 262/885] miner: Invert show-removed option in sectors list --- cmd/lotus-storage-miner/sectors.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/lotus-storage-miner/sectors.go b/cmd/lotus-storage-miner/sectors.go index 0e6fe8435..f77c6b61c 100644 --- a/cmd/lotus-storage-miner/sectors.go +++ b/cmd/lotus-storage-miner/sectors.go @@ -2,7 +2,6 @@ package main import ( "fmt" - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" "os" "sort" "strconv" @@ -19,6 +18,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" + sealing "github.com/filecoin-project/lotus/extern/storage-sealing" ) var sectorsCmd = &cli.Command{ @@ -139,8 +139,8 @@ var sectorsListCmd = &cli.Command{ Usage: "List sectors", Flags: []cli.Flag{ &cli.BoolFlag{ - Name: "hide-removed", - Usage: "to hide Removed sectors", + Name: "show-removed", + Usage: "show removed sectors", }, }, Action: func(cctx *cli.Context) error { @@ -199,11 +199,11 @@ var sectorsListCmd = &cli.Command{ continue } - if !cctx.Bool("hide-removed") || st.State != api.SectorState(sealing.Removed) { + if cctx.Bool("show-removed") || st.State != api.SectorState(sealing.Removed) { _, inSSet := commitedIDs[s] _, inASet := activeIDs[s] - fmt.Fprintf(w, "%d: %s\tsSet: %s\tactive: %s\ttktH: %d\tseedH: %d\tdeals: %v\t toUpgrade:%t\n", + _, _ = fmt.Fprintf(w, "%d: %s\tsSet: %s\tactive: %s\ttktH: %d\tseedH: %d\tdeals: %v\t toUpgrade:%t\n", s, st.State, yesno(inSSet), From ad3db0e83c5f6723573eec1464bc3bc48d29c001 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Mon, 21 Sep 2020 19:05:39 +0200 Subject: [PATCH 263/885] Move unclassified docs back to the docs root --- .../en/{unclassified => }/WIP-arch-complementary-notes.md | 0 documentation/en/{unclassified => }/block-validation.md | 0 documentation/en/{unclassified => }/create-miner.md | 0 documentation/en/{unclassified => }/dev-tools-pond-ui.md | 0 documentation/en/{unclassified => }/sealing-procs.md | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename documentation/en/{unclassified => }/WIP-arch-complementary-notes.md (100%) rename documentation/en/{unclassified => }/block-validation.md (100%) rename documentation/en/{unclassified => }/create-miner.md (100%) rename documentation/en/{unclassified => }/dev-tools-pond-ui.md (100%) rename documentation/en/{unclassified => }/sealing-procs.md (100%) diff --git a/documentation/en/unclassified/WIP-arch-complementary-notes.md b/documentation/en/WIP-arch-complementary-notes.md similarity index 100% rename from documentation/en/unclassified/WIP-arch-complementary-notes.md rename to documentation/en/WIP-arch-complementary-notes.md diff --git a/documentation/en/unclassified/block-validation.md b/documentation/en/block-validation.md similarity index 100% rename from documentation/en/unclassified/block-validation.md rename to documentation/en/block-validation.md diff --git a/documentation/en/unclassified/create-miner.md b/documentation/en/create-miner.md similarity index 100% rename from documentation/en/unclassified/create-miner.md rename to documentation/en/create-miner.md diff --git a/documentation/en/unclassified/dev-tools-pond-ui.md b/documentation/en/dev-tools-pond-ui.md similarity index 100% rename from documentation/en/unclassified/dev-tools-pond-ui.md rename to documentation/en/dev-tools-pond-ui.md diff --git a/documentation/en/unclassified/sealing-procs.md b/documentation/en/sealing-procs.md similarity index 100% rename from documentation/en/unclassified/sealing-procs.md rename to documentation/en/sealing-procs.md From 025663118f65dc50d0dd13364fd57b7cddab70ed Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 21 Sep 2020 10:30:09 -0700 Subject: [PATCH 264/885] non-destructive diff --- chain/actors/adt/diff_adt.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/chain/actors/adt/diff_adt.go b/chain/actors/adt/diff_adt.go index 0784b77a2..160e12e19 100644 --- a/chain/actors/adt/diff_adt.go +++ b/chain/actors/adt/diff_adt.go @@ -27,6 +27,7 @@ type AdtArrayDiff interface { // - All values that exist in preArr and in curArr are passed to AdtArrayDiff.Modify() // - It is the responsibility of AdtArrayDiff.Modify() to determine if the values it was passed have been modified. func DiffAdtArray(preArr, curArr Array, out AdtArrayDiff) error { + notNew := make(map[int64]struct{}, curArr.Length()) prevVal := new(typegen.Deferred) if err := preArr.ForEach(prevVal, func(i int64) error { curVal := new(typegen.Deferred) @@ -47,14 +48,17 @@ func DiffAdtArray(preArr, curArr Array, out AdtArrayDiff) error { return err } } - - return curArr.Delete(uint64(i)) + notNew[i] = struct{}{} + return nil }); err != nil { return err } curVal := new(typegen.Deferred) return curArr.ForEach(curVal, func(i int64) error { + if _, ok := notNew[i]; ok { + return nil + } return out.Add(uint64(i), curVal) }) } @@ -76,6 +80,7 @@ type AdtMapDiff interface { } func DiffAdtMap(preMap, curMap Map, out AdtMapDiff) error { + notNew := make(map[string]struct{}) prevVal := new(typegen.Deferred) if err := preMap.ForEach(prevVal, func(key string) error { curVal := new(typegen.Deferred) @@ -101,14 +106,17 @@ func DiffAdtMap(preMap, curMap Map, out AdtMapDiff) error { return err } } - - return curMap.Delete(k) + notNew[key] = struct{}{} + return nil }); err != nil { return err } curVal := new(typegen.Deferred) return curMap.ForEach(curVal, func(key string) error { + if _, ok := notNew[key]; ok { + return nil + } return out.Add(key, curVal) }) } From 4cf0c105eb12685b384fad6fb3e249898687511f Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 21 Sep 2020 12:05:01 -0700 Subject: [PATCH 265/885] optimize sector loading And avoid exposing "arrays" via the miner abstraction. We may change these structures later. --- api/api_full.go | 6 +-- api/apistruct/struct.go | 10 ++-- chain/actors/adt/adt.go | 17 ------- chain/actors/builtin/miner/miner.go | 7 +-- chain/actors/builtin/miner/v0.go | 72 ++++++++++------------------- chain/stmgr/utils.go | 71 +++++++++------------------- cli/state.go | 6 +-- cmd/lotus-storage-miner/sectors.go | 6 +-- documentation/en/api-methods.md | 3 -- node/impl/full/state.go | 8 ++-- storage/miner.go | 2 +- storage/wdpost_run.go | 16 +++---- storage/wdpost_run_test.go | 16 +++---- 13 files changed, 81 insertions(+), 159 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 61bf1fd8d..f39d0e9bc 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -313,11 +313,9 @@ type FullNode interface { // StateNetworkName returns the name of the network the node is synced to StateNetworkName(context.Context) (dtypes.NetworkName, error) // StateMinerSectors returns info about the given miner's sectors. If the filter bitfield is nil, all sectors are included. - // If the filterOut boolean is set to true, any sectors in the filter are excluded. - // If false, only those sectors in the filter are included. - StateMinerSectors(context.Context, address.Address, *bitfield.BitField, bool, types.TipSetKey) ([]*miner.ChainSectorInfo, error) + StateMinerSectors(context.Context, address.Address, *bitfield.BitField, types.TipSetKey) ([]*miner.SectorOnChainInfo, error) // StateMinerActiveSectors returns info about sectors that a given miner is actively proving. - StateMinerActiveSectors(context.Context, address.Address, types.TipSetKey) ([]*miner.ChainSectorInfo, error) + StateMinerActiveSectors(context.Context, address.Address, types.TipSetKey) ([]*miner.SectorOnChainInfo, error) // StateMinerProvingDeadline calculates the deadline at some epoch for a proving period // and returns the deadline-related calculations. StateMinerProvingDeadline(context.Context, address.Address, types.TipSetKey) (*dline.Info, error) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index cbe78c555..91a545479 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -162,8 +162,8 @@ type FullNodeStruct struct { ClientRetrieveTryRestartInsufficientFunds func(ctx context.Context, paymentChannel address.Address) error `perm:"write"` StateNetworkName func(context.Context) (dtypes.NetworkName, error) `perm:"read"` - StateMinerSectors func(context.Context, address.Address, *bitfield.BitField, bool, types.TipSetKey) ([]*miner.ChainSectorInfo, error) `perm:"read"` - StateMinerActiveSectors func(context.Context, address.Address, types.TipSetKey) ([]*miner.ChainSectorInfo, error) `perm:"read"` + StateMinerSectors func(context.Context, address.Address, *bitfield.BitField, types.TipSetKey) ([]*miner.SectorOnChainInfo, error) `perm:"read"` + StateMinerActiveSectors func(context.Context, address.Address, types.TipSetKey) ([]*miner.SectorOnChainInfo, error) `perm:"read"` StateMinerProvingDeadline func(context.Context, address.Address, types.TipSetKey) (*dline.Info, error) `perm:"read"` StateMinerPower func(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error) `perm:"read"` StateMinerInfo func(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error) `perm:"read"` @@ -734,11 +734,11 @@ func (c *FullNodeStruct) StateNetworkName(ctx context.Context) (dtypes.NetworkNa return c.Internal.StateNetworkName(ctx) } -func (c *FullNodeStruct) StateMinerSectors(ctx context.Context, addr address.Address, filter *bitfield.BitField, filterOut bool, tsk types.TipSetKey) ([]*miner.ChainSectorInfo, error) { - return c.Internal.StateMinerSectors(ctx, addr, filter, filterOut, tsk) +func (c *FullNodeStruct) StateMinerSectors(ctx context.Context, addr address.Address, sectorNos *bitfield.BitField, tsk types.TipSetKey) ([]*miner.SectorOnChainInfo, error) { + return c.Internal.StateMinerSectors(ctx, addr, sectorNos, tsk) } -func (c *FullNodeStruct) StateMinerActiveSectors(ctx context.Context, addr address.Address, tsk types.TipSetKey) ([]*miner.ChainSectorInfo, error) { +func (c *FullNodeStruct) StateMinerActiveSectors(ctx context.Context, addr address.Address, tsk types.TipSetKey) ([]*miner.SectorOnChainInfo, error) { return c.Internal.StateMinerActiveSectors(ctx, addr, tsk) } diff --git a/chain/actors/adt/adt.go b/chain/actors/adt/adt.go index 81c2e3a86..fd5ee3f87 100644 --- a/chain/actors/adt/adt.go +++ b/chain/actors/adt/adt.go @@ -55,20 +55,3 @@ func AsArray(store Store, root cid.Cid, version network.Version) (Array, error) } return nil, xerrors.Errorf("unknown network version: %d", version) } - -type ROnlyArray interface { - Get(idx uint64, v cbor.Unmarshaler) (bool, error) - ForEach(v cbor.Unmarshaler, fn func(idx int64) error) error -} - -type ProxyArray struct { - GetFunc func(idx uint64, v cbor.Unmarshaler) (bool, error) - ForEachFunc func(v cbor.Unmarshaler, fn func(idx int64) error) error -} - -func (a *ProxyArray) Get(idx uint64, v cbor.Unmarshaler) (bool, error) { - return a.GetFunc(idx, v) -} -func (a *ProxyArray) ForEach(v cbor.Unmarshaler, fn func(idx int64) error) error { - return a.ForEachFunc(v, fn) -} diff --git a/chain/actors/builtin/miner/miner.go b/chain/actors/builtin/miner/miner.go index b20539b8f..cf5eea742 100644 --- a/chain/actors/builtin/miner/miner.go +++ b/chain/actors/builtin/miner/miner.go @@ -47,7 +47,7 @@ type State interface { FindSector(abi.SectorNumber) (*SectorLocation, error) GetSectorExpiration(abi.SectorNumber) (*SectorExpiration, error) GetPrecommittedSector(abi.SectorNumber) (*SectorPreCommitOnChainInfo, error) - LoadSectorsFromSet(filter *bitfield.BitField, filterOut bool) (adt.ROnlyArray, error) + LoadSectors(sectorNos *bitfield.BitField) ([]*SectorOnChainInfo, error) IsAllocated(abi.SectorNumber) (bool, error) LoadDeadline(idx uint64) (Deadline, error) @@ -129,11 +129,6 @@ type MinerInfo struct { WindowPoStPartitionSectors uint64 } -type ChainSectorInfo struct { - Info SectorOnChainInfo - ID abi.SectorNumber -} - type SectorExpiration struct { OnTime abi.ChainEpoch diff --git a/chain/actors/builtin/miner/v0.go b/chain/actors/builtin/miner/v0.go index ddc0b71a9..30ce7e73e 100644 --- a/chain/actors/builtin/miner/v0.go +++ b/chain/actors/builtin/miner/v0.go @@ -6,12 +6,10 @@ import ( "github.com/libp2p/go-libp2p-core/peer" cbg "github.com/whyrusleeping/cbor-gen" - "golang.org/x/xerrors" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-bitfield" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/cbor" "github.com/filecoin-project/go-state-types/dline" miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" @@ -163,59 +161,37 @@ func (s *state0) GetPrecommittedSector(num abi.SectorNumber) (*SectorPreCommitOn return &ret, nil } -func (s *state0) LoadSectorsFromSet(filter *bitfield.BitField, filterOut bool) (adt.ROnlyArray, error) { - a, err := adt0.AsArray(s.store, s.State.Sectors) +func (s *state0) LoadSectors(snos *bitfield.BitField) ([]*SectorOnChainInfo, error) { + sectors, err := miner0.LoadSectors(s.store, s.State.Sectors) if err != nil { return nil, err } - incl := func(i uint64) (bool, error) { - include := true - if filter != nil { - set, err := filter.IsSet(i) - if err != nil { - return false, xerrors.Errorf("filter check error: %w", err) - } - if set == filterOut { - include = false - } + // If no sector numbers are specified, load all. + if snos == nil { + infos := make([]*SectorOnChainInfo, 0, sectors.Length()) + var info0 miner0.SectorOnChainInfo + if err := sectors.ForEach(&info0, func(i int64) error { + info := fromV0SectorOnChainInfo(info0) + infos[i] = &info + return nil + }); err != nil { + return nil, err } - return include, nil + return infos, nil } - return &adt.ProxyArray{ - GetFunc: func(idx uint64, v cbor.Unmarshaler) (bool, error) { - i, err := incl(idx) - if err != nil { - return false, err - } - if !i { - return false, nil - } - - // TODO: ActorUpgrade potentially convert - - return a.Get(idx, v) - }, - ForEachFunc: func(v cbor.Unmarshaler, fn func(int64) error) error { - // TODO: ActorUpgrade potentially convert the output - return a.ForEach(v, func(i int64) error { - include, err := incl(uint64(i)) - if err != nil { - return err - } - if !include { - return nil - } - - return fn(i) - }) - }, - }, nil -} - -func (s *state0) LoadPreCommittedSectors() (adt.Map, error) { - return adt0.AsMap(s.store, s.State.PreCommittedSectors) + // Otherwise, load selected. + infos0, err := sectors.Load(*snos) + if err != nil { + return nil, err + } + infos := make([]*SectorOnChainInfo, len(infos0)) + for i, info0 := range infos0 { + info := fromV0SectorOnChainInfo(*info0) + infos[i] = &info + } + return infos, nil } func (s *state0) IsAllocated(num abi.SectorNumber) (bool, error) { diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index 1056818a6..1e670dace 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -150,7 +150,7 @@ func MinerSectorInfo(ctx context.Context, sm *StateManager, maddr address.Addres return mas.GetSector(sid) } -func GetMinerSectorSet(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address, filter *bitfield.BitField, filterOut bool) ([]*miner.ChainSectorInfo, error) { +func GetMinerSectorSet(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address, snos *bitfield.BitField) ([]*miner.SectorOnChainInfo, error) { act, err := sm.LoadActor(ctx, maddr, ts) if err != nil { return nil, xerrors.Errorf("(get sset) failed to load miner actor: %w", err) @@ -161,29 +161,7 @@ func GetMinerSectorSet(ctx context.Context, sm *StateManager, ts *types.TipSet, return nil, xerrors.Errorf("(get sset) failed to load miner actor state: %w", err) } - sectors, err := mas.LoadSectorsFromSet(filter, filterOut) - if err != nil { - return nil, xerrors.Errorf("(get sset) failed to load sectors: %w", err) - } - - var sset []*miner.ChainSectorInfo - var v cbg.Deferred - if err := sectors.ForEach(&v, func(i int64) error { - var oci miner.SectorOnChainInfo - if err := oci.UnmarshalCBOR(bytes.NewReader(v.Raw)); err != nil { - return err - } - sset = append(sset, &miner.ChainSectorInfo{ - Info: oci, - ID: abi.SectorNumber(i), - }) - - return nil - }); err != nil { - return nil, err - } - - return sset, nil + return mas.LoadSectors(snos) } func GetSectorsForWinningPoSt(ctx context.Context, pv ffiwrapper.Verifier, sm *StateManager, st cid.Cid, maddr address.Address, rand abi.PoStRandomness) ([]proof.SectorInfo, error) { @@ -249,35 +227,30 @@ func GetSectorsForWinningPoSt(ctx context.Context, pv ffiwrapper.Verifier, sm *S return nil, xerrors.Errorf("generating winning post challenges: %w", err) } - // we don't need to filter here (and it's **very** slow) - sectors, err := mas.LoadSectorsFromSet(nil, false) + iter, err := provingSectors.BitIterator() + if err != nil { + return nil, xerrors.Errorf("iterating over proving sectors: %w", err) + } + + // Select winning sectors by _index_ in the all-sectors bitfield. + selectedSectors := bitfield.New() + prev := uint64(0) + for _, n := range ids { + sno, err := iter.Nth(n - prev) + if err != nil { + return nil, xerrors.Errorf("iterating over proving sectors: %w", err) + } + selectedSectors.Set(sno) + prev = n + } + + sectors, err := mas.LoadSectors(&selectedSectors) if err != nil { return nil, xerrors.Errorf("loading proving sectors: %w", err) } - out := make([]proof.SectorInfo, len(ids)) - for i, n := range ids { - sb, err := provingSectors.Slice(n, 1) - if err != nil { - return nil, err - } - - sid, err := sb.First() - if err != nil { - return nil, err - } - - var sinfo miner.SectorOnChainInfo - found, err := sectors.Get(sid, &sinfo) - - if err != nil { - return nil, xerrors.Errorf("loading sector info: %w", err) - } - - if !found { - return nil, xerrors.Errorf("didn't find sector info for sector %d", n) - } - + out := make([]proof.SectorInfo, len(sectors)) + for i, sinfo := range sectors { out[i] = proof.SectorInfo{ SealProof: spt, SectorNumber: sinfo.SectorNumber, diff --git a/cli/state.go b/cli/state.go index 2476d7c59..d96c93c54 100644 --- a/cli/state.go +++ b/cli/state.go @@ -259,13 +259,13 @@ var stateSectorsCmd = &cli.Command{ return err } - sectors, err := api.StateMinerSectors(ctx, maddr, nil, true, ts.Key()) + sectors, err := api.StateMinerSectors(ctx, maddr, nil, ts.Key()) if err != nil { return err } for _, s := range sectors { - fmt.Printf("%d: %x\n", s.Info.SectorNumber, s.Info.SealedCID) + fmt.Printf("%d: %x\n", s.SectorNumber, s.SealedCID) } return nil @@ -305,7 +305,7 @@ var stateActiveSectorsCmd = &cli.Command{ } for _, s := range sectors { - fmt.Printf("%d: %x\n", s.Info.SectorNumber, s.Info.SealedCID) + fmt.Printf("%d: %x\n", s.SectorNumber, s.SealedCID) } return nil diff --git a/cmd/lotus-storage-miner/sectors.go b/cmd/lotus-storage-miner/sectors.go index e3512cfe4..27a5c31be 100644 --- a/cmd/lotus-storage-miner/sectors.go +++ b/cmd/lotus-storage-miner/sectors.go @@ -176,16 +176,16 @@ var sectorsListCmd = &cli.Command{ } activeIDs := make(map[abi.SectorNumber]struct{}, len(activeSet)) for _, info := range activeSet { - activeIDs[info.ID] = struct{}{} + activeIDs[info.SectorNumber] = struct{}{} } - sset, err := fullApi.StateMinerSectors(ctx, maddr, nil, true, types.EmptyTSK) + sset, err := fullApi.StateMinerSectors(ctx, maddr, nil, types.EmptyTSK) if err != nil { return err } commitedIDs := make(map[abi.SectorNumber]struct{}, len(activeSet)) for _, info := range sset { - commitedIDs[info.ID] = struct{}{} + commitedIDs[info.SectorNumber] = struct{}{} } sort.Slice(list, func(i, j int) bool { diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index 5616bbd26..fd201f18a 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -3714,8 +3714,6 @@ Response: ### StateMinerSectors StateMinerSectors returns info about the given miner's sectors. If the filter bitfield is nil, all sectors are included. -If the filterOut boolean is set to true, any sectors in the filter are excluded. -If false, only those sectors in the filter are included. Perms: read @@ -3727,7 +3725,6 @@ Inputs: [ 0 ], - true, [ { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" diff --git a/node/impl/full/state.go b/node/impl/full/state.go index e4400b073..8d40da0ec 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -64,15 +64,15 @@ func (a *StateAPI) StateNetworkName(ctx context.Context) (dtypes.NetworkName, er return stmgr.GetNetworkName(ctx, a.StateManager, a.Chain.GetHeaviestTipSet().ParentState()) } -func (a *StateAPI) StateMinerSectors(ctx context.Context, addr address.Address, filter *bitfield.BitField, filterOut bool, tsk types.TipSetKey) ([]*miner.ChainSectorInfo, error) { +func (a *StateAPI) StateMinerSectors(ctx context.Context, addr address.Address, sectorNos *bitfield.BitField, tsk types.TipSetKey) ([]*miner.SectorOnChainInfo, error) { ts, err := a.Chain.GetTipSetFromKey(tsk) if err != nil { return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - return stmgr.GetMinerSectorSet(ctx, a.StateManager, ts, addr, filter, filterOut) + return stmgr.GetMinerSectorSet(ctx, a.StateManager, ts, addr, sectorNos) } -func (a *StateAPI) StateMinerActiveSectors(ctx context.Context, maddr address.Address, tsk types.TipSetKey) ([]*miner.ChainSectorInfo, error) { // TODO: only used in cli +func (a *StateAPI) StateMinerActiveSectors(ctx context.Context, maddr address.Address, tsk types.TipSetKey) ([]*miner.SectorOnChainInfo, error) { // TODO: only used in cli ts, err := a.Chain.GetTipSetFromKey(tsk) if err != nil { return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) @@ -93,7 +93,7 @@ func (a *StateAPI) StateMinerActiveSectors(ctx context.Context, maddr address.Ad return nil, xerrors.Errorf("merge partition active sets: %w", err) } - return stmgr.GetMinerSectorSet(ctx, a.StateManager, ts, maddr, &activeSectors, false) + return stmgr.GetMinerSectorSet(ctx, a.StateManager, ts, maddr, &activeSectors) } func (a *StateAPI) StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (miner.MinerInfo, error) { diff --git a/storage/miner.go b/storage/miner.go index 61c4000f2..a64ee977e 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -71,7 +71,7 @@ type SealingStateEvt struct { type storageMinerApi interface { // Call a read only method on actors (no interaction with the chain required) StateCall(context.Context, *types.Message, types.TipSetKey) (*api.InvocResult, error) - StateMinerSectors(context.Context, address.Address, *bitfield.BitField, bool, types.TipSetKey) ([]*miner.ChainSectorInfo, error) + StateMinerSectors(context.Context, address.Address, *bitfield.BitField, types.TipSetKey) ([]*miner.SectorOnChainInfo, error) StateSectorPreCommitInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) StateSectorGetInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorOnChainInfo, error) StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*miner.SectorLocation, error) diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 06740b108..254a2f0d3 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -594,7 +594,7 @@ func (s *WindowPoStScheduler) batchPartitions(partitions []api.Partition) ([][]a } func (s *WindowPoStScheduler) sectorsForProof(ctx context.Context, goodSectors, allSectors bitfield.BitField, ts *types.TipSet) ([]proof.SectorInfo, error) { - sset, err := s.api.StateMinerSectors(ctx, s.actor, &goodSectors, false, ts.Key()) + sset, err := s.api.StateMinerSectors(ctx, s.actor, &goodSectors, ts.Key()) if err != nil { return nil, err } @@ -604,17 +604,17 @@ func (s *WindowPoStScheduler) sectorsForProof(ctx context.Context, goodSectors, } substitute := proof.SectorInfo{ - SectorNumber: sset[0].ID, - SealedCID: sset[0].Info.SealedCID, - SealProof: sset[0].Info.SealProof, + SectorNumber: sset[0].SectorNumber, + SealedCID: sset[0].SealedCID, + SealProof: sset[0].SealProof, } sectorByID := make(map[uint64]proof.SectorInfo, len(sset)) for _, sector := range sset { - sectorByID[uint64(sector.ID)] = proof.SectorInfo{ - SectorNumber: sector.ID, - SealedCID: sector.Info.SealedCID, - SealProof: sector.Info.SealProof, + sectorByID[uint64(sector.SectorNumber)] = proof.SectorInfo{ + SectorNumber: sector.SectorNumber, + SealedCID: sector.SealedCID, + SealProof: sector.SealProof, } } diff --git a/storage/wdpost_run_test.go b/storage/wdpost_run_test.go index bd8e35d4f..1797cf35c 100644 --- a/storage/wdpost_run_test.go +++ b/storage/wdpost_run_test.go @@ -65,14 +65,14 @@ func (m *mockStorageMinerAPI) StateMinerPartitions(ctx context.Context, a addres return m.partitions, nil } -func (m *mockStorageMinerAPI) StateMinerSectors(ctx context.Context, address address.Address, field *bitfield.BitField, b bool, key types.TipSetKey) ([]*miner.ChainSectorInfo, error) { - var sis []*miner.ChainSectorInfo - _ = field.ForEach(func(i uint64) error { - sis = append(sis, &miner.ChainSectorInfo{ - Info: miner.SectorOnChainInfo{ - SectorNumber: abi.SectorNumber(i), - }, - ID: abi.SectorNumber(i), +func (m *mockStorageMinerAPI) StateMinerSectors(ctx context.Context, address address.Address, snos *bitfield.BitField, key types.TipSetKey) ([]*miner.SectorOnChainInfo, error) { + var sis []*miner.SectorOnChainInfo + if snos == nil { + panic("unsupported") + } + _ = snos.ForEach(func(i uint64) error { + sis = append(sis, &miner.SectorOnChainInfo{ + SectorNumber: abi.SectorNumber(i), }) return nil }) From 24ae9205fef921e1e15b59c291859c42ef5851ad Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 21 Sep 2020 12:50:12 -0700 Subject: [PATCH 266/885] cross-version state tree diff --- chain/state/statetree.go | 43 ++++++++++++++++++++++++++++++++ node/impl/full/state.go | 53 +++++----------------------------------- 2 files changed, 49 insertions(+), 47 deletions(-) diff --git a/chain/state/statetree.go b/chain/state/statetree.go index b3a3b7cee..fb6a14a9b 100644 --- a/chain/state/statetree.go +++ b/chain/state/statetree.go @@ -1,6 +1,7 @@ package state import ( + "bytes" "context" "fmt" @@ -14,6 +15,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/actors/builtin" init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init" + cbg "github.com/whyrusleeping/cbor-gen" "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/types" @@ -391,3 +393,44 @@ func (st *StateTree) ForEach(f func(address.Address, *types.Actor) error) error return f(addr, &act) }) } + +func Diff(oldTree, newTree *StateTree) (map[string]types.Actor, error) { + out := map[string]types.Actor{} + + var ( + ncval, ocval cbg.Deferred + buf = bytes.NewReader(nil) + ) + if err := newTree.root.ForEach(&ncval, func(k string) error { + var act types.Actor + + addr, err := address.NewFromBytes([]byte(k)) + if err != nil { + return xerrors.Errorf("address in state tree was not valid: %w", err) + } + + found, err := oldTree.root.Get(abi.AddrKey(addr), &ocval) + if err != nil { + return err + } + + if found && bytes.Equal(ocval.Raw, ncval.Raw) { + return nil // not changed + } + + buf.Reset(ncval.Raw) + err = act.UnmarshalCBOR(buf) + buf.Reset(nil) + + if err != nil { + return err + } + + out[addr.String()] = act + + return nil + }); err != nil { + return nil, err + } + return out, nil +} diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 8d40da0ec..b64e60260 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -16,7 +16,6 @@ import ( cid "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" - cbg "github.com/whyrusleeping/cbor-gen" "go.uber.org/fx" "golang.org/x/xerrors" @@ -27,7 +26,6 @@ import ( "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/specs-actors/actors/builtin" miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" - "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors/builtin/market" @@ -618,58 +616,19 @@ func (a *StateAPI) StateMarketStorageDeal(ctx context.Context, dealId abi.DealID } func (a *StateAPI) StateChangedActors(ctx context.Context, old cid.Cid, new cid.Cid) (map[string]types.Actor, error) { - store := adt.WrapStore(ctx, cbor.NewCborStore(a.Chain.Blockstore())) + store := a.Chain.Store(ctx) - nh, err := adt.AsMap(store, new) + oldTree, err := state.LoadStateTree(store, old) if err != nil { - return nil, err + return nil, xerrors.Errorf("failed to load old state tree: %w", err) } - oh, err := adt.AsMap(store, old) + newTree, err := state.LoadStateTree(store, new) if err != nil { - return nil, err + return nil, xerrors.Errorf("failed to load new state tree: %w", err) } - out := map[string]types.Actor{} - - var ( - ncval, ocval cbg.Deferred - buf = bytes.NewReader(nil) - ) - err = nh.ForEach(&ncval, func(k string) error { - var act types.Actor - - addr, err := address.NewFromBytes([]byte(k)) - if err != nil { - return xerrors.Errorf("address in state tree was not valid: %w", err) - } - - found, err := oh.Get(abi.AddrKey(addr), &ocval) - if err != nil { - return err - } - - if found && bytes.Equal(ocval.Raw, ncval.Raw) { - return nil // not changed - } - - buf.Reset(ncval.Raw) - err = act.UnmarshalCBOR(buf) - buf.Reset(nil) - - if err != nil { - return err - } - - out[addr.String()] = act - - return nil - }) - if err != nil { - return nil, err - } - - return out, nil + state.Diff(oldTree, newTree) } func (a *StateAPI) StateMinerSectorCount(ctx context.Context, addr address.Address, tsk types.TipSetKey) (api.MinerSectors, error) { From f9a896f6f2f33e30f4cbd627613255488cbc2aec Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 21 Sep 2020 13:10:41 -0700 Subject: [PATCH 267/885] fix compile --- node/impl/full/state.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/impl/full/state.go b/node/impl/full/state.go index b64e60260..df747ba3f 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -628,7 +628,7 @@ func (a *StateAPI) StateChangedActors(ctx context.Context, old cid.Cid, new cid. return nil, xerrors.Errorf("failed to load new state tree: %w", err) } - state.Diff(oldTree, newTree) + return state.Diff(oldTree, newTree) } func (a *StateAPI) StateMinerSectorCount(ctx context.Context, addr address.Address, tsk types.TipSetKey) (api.MinerSectors, error) { From 0ab2459fcec5e85ea6496174b350b201e0b1518a Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 21 Sep 2020 13:13:38 -0700 Subject: [PATCH 268/885] convert multisig to new actor abstractions --- chain/actors/builtin/multisig/multisig.go | 9 ++ chain/actors/builtin/multisig/v0.go | 33 ++++++- cli/multisig.go | 103 +++++++--------------- 3 files changed, 74 insertions(+), 71 deletions(-) diff --git a/chain/actors/builtin/multisig/multisig.go b/chain/actors/builtin/multisig/multisig.go index 86a68c178..226eed75b 100644 --- a/chain/actors/builtin/multisig/multisig.go +++ b/chain/actors/builtin/multisig/multisig.go @@ -3,9 +3,12 @@ package multisig import ( "golang.org/x/xerrors" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/cbor" + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" + msig0 "github.com/filecoin-project/specs-actors/actors/builtin/multisig" "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/types" @@ -31,4 +34,10 @@ type State interface { StartEpoch() abi.ChainEpoch UnlockDuration() abi.ChainEpoch InitialBalance() abi.TokenAmount + Threshold() uint64 + Signers() []address.Address + + ForEachPendingTxn(func(id int64, txn Transaction) error) error } + +type Transaction = msig0.Transaction diff --git a/chain/actors/builtin/multisig/v0.go b/chain/actors/builtin/multisig/v0.go index 5de5ca6ad..e81a367cf 100644 --- a/chain/actors/builtin/multisig/v0.go +++ b/chain/actors/builtin/multisig/v0.go @@ -1,15 +1,21 @@ package multisig import ( + "encoding/binary" + + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/actors/adt" - "github.com/filecoin-project/specs-actors/actors/builtin/multisig" + "golang.org/x/xerrors" + + msig0 "github.com/filecoin-project/specs-actors/actors/builtin/multisig" + adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" ) var _ State = (*state0)(nil) type state0 struct { - multisig.State + msig0.State store adt.Store } @@ -28,3 +34,26 @@ func (s *state0) UnlockDuration() abi.ChainEpoch { func (s *state0) InitialBalance() abi.TokenAmount { return s.State.InitialBalance } + +func (s *state0) Threshold() uint64 { + return s.State.NumApprovalsThreshold +} + +func (s *state0) Signers() []address.Address { + return s.State.Signers +} + +func (s *state0) ForEachPendingTxn(cb func(id int64, txn Transaction) error) error { + arr, err := adt0.AsMap(s.store, s.State.PendingTxns) + if err != nil { + return err + } + var out msig0.Transaction + return arr.ForEach(&out, func(key string) error { + txid, n := binary.Varint([]byte(key)) + if n <= 0 { + return xerrors.Errorf("invalid pending transaction key: %v", key) + } + return cb(txid, (Transaction)(out)) + }) +} diff --git a/cli/multisig.go b/cli/multisig.go index 23095bf06..2aabf5d7e 100644 --- a/cli/multisig.go +++ b/cli/multisig.go @@ -2,8 +2,6 @@ package cli import ( "bytes" - "context" - "encoding/binary" "encoding/hex" "fmt" "os" @@ -12,21 +10,21 @@ import ( "text/tabwriter" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/go-address" - init_ "github.com/filecoin-project/specs-actors/actors/builtin/init" - msig0 "github.com/filecoin-project/specs-actors/actors/builtin/multisig" - cid "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" "github.com/urfave/cli/v2" "golang.org/x/xerrors" - "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/specs-actors/actors/builtin" + init0 "github.com/filecoin-project/specs-actors/actors/builtin/init" + msig0 "github.com/filecoin-project/specs-actors/actors/builtin/multisig" + "github.com/filecoin-project/lotus/api/apibstore" "github.com/filecoin-project/lotus/build" - types "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/chain/actors/builtin/multisig" + "github.com/filecoin-project/lotus/chain/types" ) var multisigCmd = &cli.Command{ @@ -146,7 +144,7 @@ var msigCreateCmd = &cli.Command{ // get address of newly created miner - var execreturn init_.ExecReturn + var execreturn init0.ExecReturn if err := execreturn.UnmarshalCBOR(bytes.NewReader(wait.Receipt.Return)); err != nil { return err } @@ -179,50 +177,54 @@ var msigInspectCmd = &cli.Command{ defer closer() ctx := ReqContext(cctx) + store := adt.WrapStore(ctx, cbor.NewCborStore(apibstore.NewAPIBlockstore(api))) + maddr, err := address.NewFromString(cctx.Args().First()) if err != nil { return err } - act, err := api.StateGetActor(ctx, maddr, types.EmptyTSK) - if err != nil { - return err - } - - obj, err := api.ChainReadObj(ctx, act.Head) - if err != nil { - return err - } - head, err := api.ChainHead(ctx) if err != nil { return err } - var mstate msig0.State - if err := mstate.UnmarshalCBOR(bytes.NewReader(obj)); err != nil { + act, err := api.StateGetActor(ctx, maddr, head.Key()) + if err != nil { + return err + } + + mstate, err := multisig.Load(store, act) + if err != nil { + return err + } + locked, err := mstate.LockedBalance(head.Height()) + if err != nil { return err } - locked := mstate.AmountLocked(head.Height() - mstate.StartEpoch) fmt.Printf("Balance: %s\n", types.FIL(act.Balance)) fmt.Printf("Spendable: %s\n", types.FIL(types.BigSub(act.Balance, locked))) if cctx.Bool("vesting") { - fmt.Printf("InitialBalance: %s\n", types.FIL(mstate.InitialBalance)) - fmt.Printf("StartEpoch: %d\n", mstate.StartEpoch) - fmt.Printf("UnlockDuration: %d\n", mstate.UnlockDuration) + fmt.Printf("InitialBalance: %s\n", types.FIL(mstate.InitialBalance())) + fmt.Printf("StartEpoch: %d\n", mstate.StartEpoch()) + fmt.Printf("UnlockDuration: %d\n", mstate.UnlockDuration()) } - fmt.Printf("Threshold: %d / %d\n", mstate.NumApprovalsThreshold, len(mstate.Signers)) + signers := mstate.Signers() + fmt.Printf("Threshold: %d / %d\n", mstate.Threshold(), len(signers)) fmt.Println("Signers:") - for _, s := range mstate.Signers { + for _, s := range signers { fmt.Printf("\t%s\n", s) } - pending, err := GetMultisigPending(ctx, api, mstate.PendingTxns) - if err != nil { - return fmt.Errorf("reading pending transactions: %w", err) + pending := make(map[int64]multisig.Transaction) + if err := mstate.ForEachPendingTxn(func(id int64, txn multisig.Transaction) error { + pending[id] = txn + return nil + }); err != nil { + return xerrors.Errorf("reading pending transactions: %w", err) } fmt.Println("Transactions: ", len(pending)) @@ -239,7 +241,7 @@ var msigInspectCmd = &cli.Command{ fmt.Fprintf(w, "ID\tState\tApprovals\tTo\tValue\tMethod\tParams\n") for _, txid := range txids { tx := pending[txid] - fmt.Fprintf(w, "%d\t%s\t%d\t%s\t%s\t%d\t%x\n", txid, state(tx), len(tx.Approved), tx.To, types.FIL(tx.Value), tx.Method, tx.Params) + fmt.Fprintf(w, "%d\t%s\t%d\t%s\t%s\t%d\t%x\n", txid, "pending", len(tx.Approved), tx.To, types.FIL(tx.Value), tx.Method, tx.Params) } if err := w.Flush(); err != nil { return xerrors.Errorf("flushing output: %+v", err) @@ -251,43 +253,6 @@ var msigInspectCmd = &cli.Command{ }, } -func GetMultisigPending(ctx context.Context, lapi api.FullNode, hroot cid.Cid) (map[int64]*msig0.Transaction, error) { - bs := apibstore.NewAPIBlockstore(lapi) - store := adt.WrapStore(ctx, cbor.NewCborStore(bs)) - - nd, err := adt.AsMap(store, hroot) - if err != nil { - return nil, err - } - - txs := make(map[int64]*msig0.Transaction) - var tx msig0.Transaction - err = nd.ForEach(&tx, func(k string) error { - txid, _ := binary.Varint([]byte(k)) - - cpy := tx // copy so we don't clobber on future iterations. - txs[txid] = &cpy - return nil - }) - if err != nil { - return nil, xerrors.Errorf("failed to iterate transactions hamt: %w", err) - } - - return txs, nil -} - -func state(tx *msig0.Transaction) string { - /* // TODO(why): I strongly disagree with not having these... but i need to move forward - if tx.Complete { - return "done" - } - if tx.Canceled { - return "canceled" - } - */ - return "pending" -} - var msigProposeCmd = &cli.Command{ Name: "propose", Usage: "Propose a multisig transaction", From a41bf74badf8586ebb20bcdba01d14cf006e5ff4 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 21 Sep 2020 13:25:03 -0700 Subject: [PATCH 269/885] fix: remove incorrect variable --- chain/actors/builtin/miner/miner.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/chain/actors/builtin/miner/miner.go b/chain/actors/builtin/miner/miner.go index cf5eea742..ab2043954 100644 --- a/chain/actors/builtin/miner/miner.go +++ b/chain/actors/builtin/miner/miner.go @@ -18,8 +18,6 @@ import ( "github.com/filecoin-project/lotus/chain/types" ) -var Address = builtin0.InitActorAddr - func Load(store adt.Store, act *types.Actor) (st State, err error) { switch act.Code { case builtin0.StorageMinerActorCodeID: From 916421b247b8a075ca808c139df7dd52aa396f81 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 21 Sep 2020 13:43:47 -0700 Subject: [PATCH 270/885] convert lotus-shed balances --- chain/actors/builtin/miner/miner.go | 10 ++++++++ chain/actors/builtin/miner/v0.go | 15 ++++++++++++ cmd/lotus-shed/balances.go | 37 ++++++++++++++++------------- 3 files changed, 46 insertions(+), 16 deletions(-) diff --git a/chain/actors/builtin/miner/miner.go b/chain/actors/builtin/miner/miner.go index ab2043954..5f1c26013 100644 --- a/chain/actors/builtin/miner/miner.go +++ b/chain/actors/builtin/miner/miner.go @@ -18,6 +18,15 @@ import ( "github.com/filecoin-project/lotus/chain/types" ) +// Returns true if the specified actor code ID is a miner actor. +func Is(code cid.Cid) bool { + switch code { + case builtin0.StorageMinerActorCodeID: + return true + } + return false +} + func Load(store adt.Store, act *types.Actor) (st State, err error) { switch act.Code { case builtin0.StorageMinerActorCodeID: @@ -46,6 +55,7 @@ type State interface { GetSectorExpiration(abi.SectorNumber) (*SectorExpiration, error) GetPrecommittedSector(abi.SectorNumber) (*SectorPreCommitOnChainInfo, error) LoadSectors(sectorNos *bitfield.BitField) ([]*SectorOnChainInfo, error) + NumLiveSectors() (uint64, error) IsAllocated(abi.SectorNumber) (bool, error) LoadDeadline(idx uint64) (Deadline, error) diff --git a/chain/actors/builtin/miner/v0.go b/chain/actors/builtin/miner/v0.go index 30ce7e73e..a56778156 100644 --- a/chain/actors/builtin/miner/v0.go +++ b/chain/actors/builtin/miner/v0.go @@ -79,6 +79,21 @@ func (s *state0) FindSector(num abi.SectorNumber) (*SectorLocation, error) { }, nil } +func (s *state0) NumLiveSectors() (uint64, error) { + dls, err := s.State.LoadDeadlines(s.store) + if err != nil { + return 0, err + } + var total uint64 + if err := dls.ForEach(s.store, func(dlIdx uint64, dl *miner0.Deadline) error { + total += dl.LiveSectors + return nil + }); err != nil { + return 0, err + } + return total, nil +} + // GetSectorExpiration returns the effective expiration of the given sector. // // If the sector isn't found or has already been terminated, this method returns diff --git a/cmd/lotus-shed/balances.go b/cmd/lotus-shed/balances.go index 248f3b26e..413f86375 100644 --- a/cmd/lotus-shed/balances.go +++ b/cmd/lotus-shed/balances.go @@ -12,6 +12,8 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/store" @@ -21,9 +23,6 @@ import ( "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/lib/blockstore" "github.com/filecoin-project/lotus/node/repo" - "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" - "github.com/filecoin-project/specs-actors/actors/util/adt" ) type accountInfo struct { @@ -90,7 +89,7 @@ var chainBalanceCmd = &cli.Command{ Type: string(act.Code.Hash()[2:]), } - if act.Code == builtin.StorageMinerActorCodeID { + if miner.Is(act.Code) { pow, err := api.StateMinerPower(ctx, addr, tsk) if err != nil { return xerrors.Errorf("failed to get power: %w", err) @@ -167,6 +166,7 @@ var chainBalanceStateCmd = &cli.Command{ cs := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier)) cst := cbor.NewCborStore(bs) + store := adt.WrapStore(ctx, cst) sm := stmgr.NewStateManager(cs) @@ -191,7 +191,7 @@ var chainBalanceStateCmd = &cli.Command{ PreCommits: types.FIL(big.NewInt(0)), } - if act.Code == builtin.StorageMinerActorCodeID && minerInfo { + if minerInfo && miner.Is(act.Code) { pow, _, _, err := stmgr.GetPowerRaw(ctx, sm, sroot, addr) if err != nil { return xerrors.Errorf("failed to get power: %w", err) @@ -199,24 +199,29 @@ var chainBalanceStateCmd = &cli.Command{ ai.Power = pow.RawBytePower - var st miner.State - if err := cst.Get(ctx, act.Head, &st); err != nil { + st, err := miner.Load(store, act) + if err != nil { return xerrors.Errorf("failed to read miner state: %w", err) } - sectors, err := adt.AsArray(cs.Store(ctx), st.Sectors) + liveSectorCount, err := st.NumLiveSectors() if err != nil { - return xerrors.Errorf("failed to load sector set: %w", err) + return xerrors.Errorf("failed to compute live sector count: %w", err) } - ai.InitialPledge = types.FIL(st.InitialPledgeRequirement) - ai.LockedFunds = types.FIL(st.LockedFunds) - ai.PreCommits = types.FIL(st.PreCommitDeposits) - ai.Sectors = sectors.Length() + lockedFunds, err := st.LockedFunds() + if err != nil { + return xerrors.Errorf("failed to compute locked funds: %w", err) + } - var minfo miner.MinerInfo - if err := cst.Get(ctx, st.Info, &minfo); err != nil { - return xerrors.Errorf("failed to read miner info: %w", err) + ai.InitialPledge = types.FIL(lockedFunds.InitialPledgeRequirement) + ai.LockedFunds = types.FIL(lockedFunds.VestingFunds) + ai.PreCommits = types.FIL(lockedFunds.PreCommitDeposits) + ai.Sectors = liveSectorCount + + minfo, err := st.Info() + if err != nil { + return xerrors.Errorf("failed to get miner info: %w", err) } ai.Worker = minfo.Worker From 4bab784e401bcaec79a30ac0a36b75cc98e73fdd Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 21 Sep 2020 15:04:39 -0700 Subject: [PATCH 271/885] migrate lotus-shed verifreg to specs-actors abstractions --- chain/actors/builtin/verifreg/v0.go | 40 ++++++++- chain/actors/builtin/verifreg/verifreg.go | 3 + cmd/lotus-shed/verifreg.go | 100 ++++++++-------------- 3 files changed, 76 insertions(+), 67 deletions(-) diff --git a/chain/actors/builtin/verifreg/v0.go b/chain/actors/builtin/verifreg/v0.go index f64c27310..c59a58811 100644 --- a/chain/actors/builtin/verifreg/v0.go +++ b/chain/actors/builtin/verifreg/v0.go @@ -6,6 +6,7 @@ import ( "github.com/filecoin-project/go-state-types/big" verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" + "github.com/ipfs/go-cid" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/chain/actors/adt" @@ -18,22 +19,53 @@ type state0 struct { store adt.Store } -func (s *state0) VerifiedClientDataCap(addr address.Address) (bool, abi.StoragePower, error) { +func getDataCap(store adt.Store, root cid.Cid, addr address.Address) (bool, abi.StoragePower, error) { if addr.Protocol() != address.ID { return false, big.Zero(), xerrors.Errorf("can only look up ID addresses") } - vh, err := adt0.AsMap(s.store, s.VerifiedClients) + vh, err := adt0.AsMap(store, root) if err != nil { - return false, big.Zero(), xerrors.Errorf("loading verified clients: %w", err) + return false, big.Zero(), xerrors.Errorf("loading verifreg: %w", err) } var dcap abi.StoragePower if found, err := vh.Get(abi.AddrKey(addr), &dcap); err != nil { - return false, big.Zero(), xerrors.Errorf("looking up verified clients: %w", err) + return false, big.Zero(), xerrors.Errorf("looking up addr: %w", err) } else if !found { return false, big.Zero(), nil } return true, dcap, nil } + +func (s *state0) VerifiedClientDataCap(addr address.Address) (bool, abi.StoragePower, error) { + return getDataCap(s.store, s.State.VerifiedClients, addr) +} + +func (s *state0) VerifierDataCap(addr address.Address) (bool, abi.StoragePower, error) { + return getDataCap(s.store, s.State.Verifiers, addr) +} + +func forEachCap(store adt.Store, root cid.Cid, cb func(addr address.Address, dcap abi.StoragePower) error) error { + vh, err := adt0.AsMap(store, root) + if err != nil { + return xerrors.Errorf("loading verified clients: %w", err) + } + var dcap abi.StoragePower + return vh.ForEach(&dcap, func(key string) error { + a, err := address.NewFromBytes([]byte(key)) + if err != nil { + return err + } + return cb(a, dcap) + }) +} + +func (s *state0) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error { + return forEachCap(s.store, s.State.Verifiers, cb) +} + +func (s *state0) ForEachClient(cb func(addr address.Address, dcap abi.StoragePower) error) error { + return forEachCap(s.store, s.State.VerifiedClients, cb) +} diff --git a/chain/actors/builtin/verifreg/verifreg.go b/chain/actors/builtin/verifreg/verifreg.go index 000a94349..c861f862f 100644 --- a/chain/actors/builtin/verifreg/verifreg.go +++ b/chain/actors/builtin/verifreg/verifreg.go @@ -31,4 +31,7 @@ type State interface { cbor.Marshaler VerifiedClientDataCap(address.Address) (bool, abi.StoragePower, error) + VerifierDataCap(address.Address) (bool, abi.StoragePower, error) + ForEachVerifier(func(addr address.Address, dcap abi.StoragePower) error) error + ForEachClient(func(addr address.Address, dcap abi.StoragePower) error) error } diff --git a/cmd/lotus-shed/verifreg.go b/cmd/lotus-shed/verifreg.go index 9f475261d..3e2f34f4b 100644 --- a/cmd/lotus-shed/verifreg.go +++ b/cmd/lotus-shed/verifreg.go @@ -8,13 +8,15 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" - "github.com/filecoin-project/specs-actors/actors/util/adt" + + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" + verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" "github.com/filecoin-project/lotus/api/apibstore" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/chain/actors/builtin/verifreg" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" cbor "github.com/ipfs/go-ipld-cbor" @@ -57,7 +59,7 @@ var verifRegAddVerifierCmd = &cli.Command{ return err } - params, err := actors.SerializeParams(&verifreg.AddVerifierParams{Address: target, Allowance: allowance}) + params, err := actors.SerializeParams(&verifreg0.AddVerifierParams{Address: target, Allowance: allowance}) if err != nil { return err } @@ -70,9 +72,9 @@ var verifRegAddVerifierCmd = &cli.Command{ ctx := lcli.ReqContext(cctx) msg := &types.Message{ - To: builtin.VerifiedRegistryActorAddr, + To: verifreg.Address, From: fromk, - Method: builtin.MethodsVerifiedRegistry.AddVerifier, + Method: builtin0.MethodsVerifiedRegistry.AddVerifier, Params: params, } @@ -131,7 +133,7 @@ var verifRegVerifyClientCmd = &cli.Command{ return err } - params, err := actors.SerializeParams(&verifreg.AddVerifiedClientParams{Address: target, Allowance: allowance}) + params, err := actors.SerializeParams(&verifreg0.AddVerifiedClientParams{Address: target, Allowance: allowance}) if err != nil { return err } @@ -144,9 +146,9 @@ var verifRegVerifyClientCmd = &cli.Command{ ctx := lcli.ReqContext(cctx) msg := &types.Message{ - To: builtin.VerifiedRegistryActorAddr, + To: verifreg.Address, From: fromk, - Method: builtin.MethodsVerifiedRegistry.AddVerifiedClient, + Method: builtin0.MethodsVerifiedRegistry.AddVerifiedClient, Params: params, } @@ -181,7 +183,7 @@ var verifRegListVerifiersCmd = &cli.Command{ defer closer() ctx := lcli.ReqContext(cctx) - act, err := api.StateGetActor(ctx, builtin.VerifiedRegistryActorAddr, types.EmptyTSK) + act, err := api.StateGetActor(ctx, verifreg.Address, types.EmptyTSK) if err != nil { return err } @@ -189,31 +191,14 @@ var verifRegListVerifiersCmd = &cli.Command{ apibs := apibstore.NewAPIBlockstore(api) store := adt.WrapStore(ctx, cbor.NewCborStore(apibs)) - var st verifreg.State - if err := store.Get(ctx, act.Head, &st); err != nil { - return err - } - - vh, err := adt.AsMap(store, st.Verifiers) + st, err := verifreg.Load(store, act) if err != nil { return err } - - var dcap verifreg.DataCap - if err := vh.ForEach(&dcap, func(k string) error { - addr, err := address.NewFromBytes([]byte(k)) - if err != nil { - return err - } - - fmt.Printf("%s: %s\n", addr, dcap) - - return nil - }); err != nil { + return st.ForEachVerifier(func(addr address.Address, dcap abi.StoragePower) error { + _, err := fmt.Printf("%s: %s\n", addr, dcap) return err - } - - return nil + }) }, } @@ -228,7 +213,7 @@ var verifRegListClientsCmd = &cli.Command{ defer closer() ctx := lcli.ReqContext(cctx) - act, err := api.StateGetActor(ctx, builtin.VerifiedRegistryActorAddr, types.EmptyTSK) + act, err := api.StateGetActor(ctx, verifreg.Address, types.EmptyTSK) if err != nil { return err } @@ -236,31 +221,14 @@ var verifRegListClientsCmd = &cli.Command{ apibs := apibstore.NewAPIBlockstore(api) store := adt.WrapStore(ctx, cbor.NewCborStore(apibs)) - var st verifreg.State - if err := store.Get(ctx, act.Head, &st); err != nil { - return err - } - - vh, err := adt.AsMap(store, st.VerifiedClients) + st, err := verifreg.Load(store, act) if err != nil { return err } - - var dcap verifreg.DataCap - if err := vh.ForEach(&dcap, func(k string) error { - addr, err := address.NewFromBytes([]byte(k)) - if err != nil { - return err - } - - fmt.Printf("%s: %s\n", addr, dcap) - - return nil - }); err != nil { + return st.ForEachClient(func(addr address.Address, dcap abi.StoragePower) error { + _, err := fmt.Printf("%s: %s\n", addr, dcap) return err - } - - return nil + }) }, } @@ -318,7 +286,17 @@ var verifRegCheckVerifierCmd = &cli.Command{ defer closer() ctx := lcli.ReqContext(cctx) - act, err := api.StateGetActor(ctx, builtin.VerifiedRegistryActorAddr, types.EmptyTSK) + head, err := api.ChainHead(ctx) + if err != nil { + return err + } + + vid, err := api.StateLookupID(ctx, vaddr, head.Key()) + if err != nil { + return err + } + + act, err := api.StateGetActor(ctx, verifreg.Address, head.Key()) if err != nil { return err } @@ -326,20 +304,16 @@ var verifRegCheckVerifierCmd = &cli.Command{ apibs := apibstore.NewAPIBlockstore(api) store := adt.WrapStore(ctx, cbor.NewCborStore(apibs)) - var st verifreg.State - if err := store.Get(ctx, act.Head, &st); err != nil { - return err - } - - vh, err := adt.AsMap(store, st.Verifiers) + st, err := verifreg.Load(store, act) if err != nil { return err } - var dcap verifreg.DataCap - if found, err := vh.Get(abi.AddrKey(vaddr), &dcap); err != nil { + found, dcap, err := st.VerifierDataCap(vid) + if err != nil { return err - } else if !found { + } + if !found { return fmt.Errorf("not found") } From 3f0106cfe5ba375b3e2ddad07dba8d5bef976a9b Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 21 Sep 2020 15:18:30 -0700 Subject: [PATCH 272/885] migrate lotus-shed/genesis-verify to actor abstraction --- chain/actors/builtin/miner/miner.go | 9 ------ chain/types/actor.go | 12 ++++++-- cmd/lotus-shed/balances.go | 4 +-- cmd/lotus-shed/genesis-verify.go | 48 ++++++++++++++++------------- cmd/lotus-shed/mempool-stats.go | 7 +++-- 5 files changed, 43 insertions(+), 37 deletions(-) diff --git a/chain/actors/builtin/miner/miner.go b/chain/actors/builtin/miner/miner.go index 5f1c26013..c2b2e3a0d 100644 --- a/chain/actors/builtin/miner/miner.go +++ b/chain/actors/builtin/miner/miner.go @@ -18,15 +18,6 @@ import ( "github.com/filecoin-project/lotus/chain/types" ) -// Returns true if the specified actor code ID is a miner actor. -func Is(code cid.Cid) bool { - switch code { - case builtin0.StorageMinerActorCodeID: - return true - } - return false -} - func Load(store adt.Store, act *types.Actor) (st State, err error) { switch act.Code { case builtin0.StorageMinerActorCodeID: diff --git a/chain/types/actor.go b/chain/types/actor.go index bb5635995..aa3ba2146 100644 --- a/chain/types/actor.go +++ b/chain/types/actor.go @@ -5,7 +5,7 @@ import ( "github.com/ipfs/go-cid" - "github.com/filecoin-project/specs-actors/actors/builtin" + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" ) var ErrActorNotFound = errors.New("actor not found") @@ -19,5 +19,13 @@ type Actor struct { } func (a *Actor) IsAccountActor() bool { - return a.Code == builtin.AccountActorCodeID + return a.Code == builtin0.AccountActorCodeID +} + +func (a *Actor) IsStorageMinerActor() bool { + return a.Code == builtin0.StorageMinerActorCodeID +} + +func (a *Actor) IsMultisigActor() bool { + return a.Code == builtin0.MultisigActorCodeID } diff --git a/cmd/lotus-shed/balances.go b/cmd/lotus-shed/balances.go index 413f86375..de92aa8b6 100644 --- a/cmd/lotus-shed/balances.go +++ b/cmd/lotus-shed/balances.go @@ -89,7 +89,7 @@ var chainBalanceCmd = &cli.Command{ Type: string(act.Code.Hash()[2:]), } - if miner.Is(act.Code) { + if act.IsStorageMinerActor() { pow, err := api.StateMinerPower(ctx, addr, tsk) if err != nil { return xerrors.Errorf("failed to get power: %w", err) @@ -191,7 +191,7 @@ var chainBalanceStateCmd = &cli.Command{ PreCommits: types.FIL(big.NewInt(0)), } - if minerInfo && miner.Is(act.Code) { + if minerInfo && act.IsStorageMinerActor() { pow, _, _, err := stmgr.GetPowerRaw(ctx, sm, sroot, addr) if err != nil { return xerrors.Errorf("failed to get power: %w", err) diff --git a/cmd/lotus-shed/genesis-verify.go b/cmd/lotus-shed/genesis-verify.go index 043cb72bb..1225d817a 100644 --- a/cmd/lotus-shed/genesis-verify.go +++ b/cmd/lotus-shed/genesis-verify.go @@ -14,16 +14,17 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/chain/actors/builtin/account" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + "github.com/filecoin-project/lotus/chain/actors/builtin/multisig" "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/blockstore" - "github.com/filecoin-project/specs-actors/actors/builtin" - saacc "github.com/filecoin-project/specs-actors/actors/builtin/account" - saminer "github.com/filecoin-project/specs-actors/actors/builtin/miner" - samsig "github.com/filecoin-project/specs-actors/actors/builtin/multisig" ) type addrInfo struct { @@ -90,36 +91,41 @@ var genesisVerifyCmd = &cli.Command{ kminers := make(map[address.Address]minerInfo) ctx := context.TODO() + store := adt.WrapStore(ctx, cst) if err := stree.ForEach(func(addr address.Address, act *types.Actor) error { - switch act.Code { - case builtin.StorageMinerActorCodeID: - var st saminer.State - if err := cst.Get(ctx, act.Head, &st); err != nil { - return err + switch { + case act.IsStorageMinerActor(): + _, err := miner.Load(store, act) + if err != nil { + return xerrors.Errorf("miner actor: %w", err) } - + // TODO: actually verify something here? kminers[addr] = minerInfo{} - case builtin.MultisigActorCodeID: - var st samsig.State - if err := cst.Get(ctx, act.Head, &st); err != nil { + case act.IsMultisigActor(): + st, err := multisig.Load(store, act) + if err != nil { return xerrors.Errorf("multisig actor: %w", err) } kmultisigs[addr] = msigInfo{ Balance: types.FIL(act.Balance), - Signers: st.Signers, - Threshold: st.NumApprovalsThreshold, + Signers: st.Signers(), + Threshold: st.Threshold(), } msigAddrs = append(msigAddrs, addr) - case builtin.AccountActorCodeID: - var st saacc.State - if err := cst.Get(ctx, act.Head, &st); err != nil { - log.Warn(xerrors.Errorf("account actor %s: %w", addr, err)) + case act.IsAccountActor(): + st, err := account.Load(store, act) + if err != nil { + // TODO: magik6k: this _used_ to log instead of failing, why? + return xerrors.Errorf("account actor %s: %w", addr, err) + } + pkaddr, err := st.PubkeyAddress() + if err != nil { + return xerrors.Errorf("failed to get actor pk address %s: %w", addr, err) } - kaccounts[addr] = addrInfo{ - Key: st.Address, + Key: pkaddr, Balance: types.FIL(act.Balance.Copy()), } accAddrs = append(accAddrs, addr) diff --git a/cmd/lotus-shed/mempool-stats.go b/cmd/lotus-shed/mempool-stats.go index b81cf2704..165c01432 100644 --- a/cmd/lotus-shed/mempool-stats.go +++ b/cmd/lotus-shed/mempool-stats.go @@ -14,11 +14,12 @@ import ( "go.opencensus.io/stats/view" "go.opencensus.io/tag" + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" + "github.com/filecoin-project/go-address" lapi "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" - "github.com/filecoin-project/specs-actors/actors/builtin" ) var ( @@ -121,7 +122,7 @@ var mpoolStatsCmd = &cli.Command{ return false, err } - ism := act.Code == builtin.StorageMinerActorCodeID + ism := act.IsStorageMinerActor() mcache[addr] = ism return ism, nil } @@ -143,7 +144,7 @@ var mpoolStatsCmd = &cli.Command{ seen: time.Now(), } - if u.Message.Message.Method == builtin.MethodsMiner.SubmitWindowedPoSt { + if u.Message.Message.Method == builtin0.MethodsMiner.SubmitWindowedPoSt { miner, err := isMiner(u.Message.Message.To) if err != nil { From 63f026f7c3fb52f894ed80d6fef7bdd1f7de8650 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 21 Sep 2020 15:24:45 -0700 Subject: [PATCH 273/885] migrate some more imports --- paychmgr/store.go | 15 ++++++++------- paychmgr/util.go | 11 ++++++----- storage/adapter_storage_miner.go | 7 ++++--- storage/miner.go | 6 +++--- storage/mockstorage/preseal.go | 5 +++-- storage/wdpost_run.go | 17 ++++++++--------- storage/wdpost_run_test.go | 13 ++++++------- 7 files changed, 38 insertions(+), 36 deletions(-) diff --git a/paychmgr/store.go b/paychmgr/store.go index 46249fa36..23916669e 100644 --- a/paychmgr/store.go +++ b/paychmgr/store.go @@ -12,12 +12,13 @@ import ( "github.com/filecoin-project/lotus/chain/types" cborutil "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/specs-actors/actors/builtin/paych" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" dsq "github.com/ipfs/go-datastore/query" + paych0 "github.com/filecoin-project/specs-actors/actors/builtin/paych" + "github.com/filecoin-project/go-address" cborrpc "github.com/filecoin-project/go-cbor-util" @@ -48,7 +49,7 @@ const ( ) type VoucherInfo struct { - Voucher *paych.SignedVoucher + Voucher *paych0.SignedVoucher Proof []byte Submitted bool } @@ -102,7 +103,7 @@ func (ci *ChannelInfo) to() address.Address { // infoForVoucher gets the VoucherInfo for the given voucher. // returns nil if the channel doesn't have the voucher. -func (ci *ChannelInfo) infoForVoucher(sv *paych.SignedVoucher) (*VoucherInfo, error) { +func (ci *ChannelInfo) infoForVoucher(sv *paych0.SignedVoucher) (*VoucherInfo, error) { for _, v := range ci.Vouchers { eq, err := cborutil.Equals(sv, v.Voucher) if err != nil { @@ -115,7 +116,7 @@ func (ci *ChannelInfo) infoForVoucher(sv *paych.SignedVoucher) (*VoucherInfo, er return nil, nil } -func (ci *ChannelInfo) hasVoucher(sv *paych.SignedVoucher) (bool, error) { +func (ci *ChannelInfo) hasVoucher(sv *paych0.SignedVoucher) (bool, error) { vi, err := ci.infoForVoucher(sv) return vi != nil, err } @@ -123,7 +124,7 @@ func (ci *ChannelInfo) hasVoucher(sv *paych.SignedVoucher) (bool, error) { // markVoucherSubmitted marks the voucher, and any vouchers of lower nonce // in the same lane, as being submitted. // Note: This method doesn't write anything to the store. -func (ci *ChannelInfo) markVoucherSubmitted(sv *paych.SignedVoucher) error { +func (ci *ChannelInfo) markVoucherSubmitted(sv *paych0.SignedVoucher) error { vi, err := ci.infoForVoucher(sv) if err != nil { return err @@ -147,7 +148,7 @@ func (ci *ChannelInfo) markVoucherSubmitted(sv *paych.SignedVoucher) error { } // wasVoucherSubmitted returns true if the voucher has been submitted -func (ci *ChannelInfo) wasVoucherSubmitted(sv *paych.SignedVoucher) (bool, error) { +func (ci *ChannelInfo) wasVoucherSubmitted(sv *paych0.SignedVoucher) (bool, error) { vi, err := ci.infoForVoucher(sv) if err != nil { return false, err @@ -276,7 +277,7 @@ func (ps *Store) VouchersForPaych(ch address.Address) ([]*VoucherInfo, error) { return ci.Vouchers, nil } -func (ps *Store) MarkVoucherSubmitted(ci *ChannelInfo, sv *paych.SignedVoucher) error { +func (ps *Store) MarkVoucherSubmitted(ci *ChannelInfo, sv *paych0.SignedVoucher) error { err := ci.markVoucherSubmitted(sv) if err != nil { return err diff --git a/paychmgr/util.go b/paychmgr/util.go index 0509f8a24..2a8181c15 100644 --- a/paychmgr/util.go +++ b/paychmgr/util.go @@ -4,21 +4,22 @@ import ( "context" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/specs-actors/actors/builtin/paych" + + paych0 "github.com/filecoin-project/specs-actors/actors/builtin/paych" ) type BestSpendableAPI interface { - PaychVoucherList(context.Context, address.Address) ([]*paych.SignedVoucher, error) - PaychVoucherCheckSpendable(context.Context, address.Address, *paych.SignedVoucher, []byte, []byte) (bool, error) + PaychVoucherList(context.Context, address.Address) ([]*paych0.SignedVoucher, error) + PaychVoucherCheckSpendable(context.Context, address.Address, *paych0.SignedVoucher, []byte, []byte) (bool, error) } -func BestSpendableByLane(ctx context.Context, api BestSpendableAPI, ch address.Address) (map[uint64]*paych.SignedVoucher, error) { +func BestSpendableByLane(ctx context.Context, api BestSpendableAPI, ch address.Address) (map[uint64]*paych0.SignedVoucher, error) { vouchers, err := api.PaychVoucherList(ctx, ch) if err != nil { return nil, err } - bestByLane := make(map[uint64]*paych.SignedVoucher) + bestByLane := make(map[uint64]*paych0.SignedVoucher) for _, voucher := range vouchers { spendable, err := api.PaychVoucherCheckSpendable(ctx, ch, voucher, nil, nil) if err != nil { diff --git a/storage/adapter_storage_miner.go b/storage/adapter_storage_miner.go index db3ae63d9..380fb4471 100644 --- a/storage/adapter_storage_miner.go +++ b/storage/adapter_storage_miner.go @@ -14,7 +14,8 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" - "github.com/filecoin-project/specs-actors/actors/builtin" + + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" market0 "github.com/filecoin-project/specs-actors/actors/builtin/market" "github.com/filecoin-project/lotus/api" @@ -146,10 +147,10 @@ func (s SealingAPIAdapter) StateComputeDataCommitment(ctx context.Context, maddr } ccmt := &types.Message{ - To: builtin.StorageMarketActorAddr, + To: market.Address, From: maddr, Value: types.NewInt(0), - Method: builtin.MethodsMarket.ComputeDataCommitment, + Method: builtin0.MethodsMarket.ComputeDataCommitment, Params: ccparams, } r, err := s.delegate.StateCall(ctx, ccmt, tsk) diff --git a/storage/miner.go b/storage/miner.go index a64ee977e..c1b50fe89 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -6,13 +6,13 @@ import ( "time" miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" + proof0 "github.com/filecoin-project/specs-actors/actors/runtime/proof" "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/go-state-types/dline" "github.com/filecoin-project/go-bitfield" - "github.com/filecoin-project/specs-actors/actors/runtime/proof" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" @@ -235,9 +235,9 @@ func (wpp *StorageWpp) GenerateCandidates(ctx context.Context, randomness abi.Po return cds, nil } -func (wpp *StorageWpp) ComputeProof(ctx context.Context, ssi []proof.SectorInfo, rand abi.PoStRandomness) ([]proof.PoStProof, error) { +func (wpp *StorageWpp) ComputeProof(ctx context.Context, ssi []proof0.SectorInfo, rand abi.PoStRandomness) ([]proof0.PoStProof, error) { if build.InsecurePoStValidation { - return []proof.PoStProof{{ProofBytes: []byte("valid proof")}}, nil + return []proof0.PoStProof{{ProofBytes: []byte("valid proof")}}, nil } log.Infof("Computing WinningPoSt ;%+v; %v", ssi, rand) diff --git a/storage/mockstorage/preseal.go b/storage/mockstorage/preseal.go index da063020d..8ca789ba6 100644 --- a/storage/mockstorage/preseal.go +++ b/storage/mockstorage/preseal.go @@ -9,7 +9,8 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/lotus/extern/sector-storage/mock" - "github.com/filecoin-project/specs-actors/actors/builtin/market" + + market0 "github.com/filecoin-project/specs-actors/actors/builtin/market" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/wallet" @@ -48,7 +49,7 @@ func PreSeal(ssize abi.SectorSize, maddr address.Address, sectors int) (*genesis r := mock.CommDR(d) preseal.CommR, _ = commcid.ReplicaCommitmentV1ToCID(r[:]) preseal.SectorID = abi.SectorNumber(i + 1) - preseal.Deal = market.DealProposal{ + preseal.Deal = market0.DealProposal{ PieceCID: preseal.CommD, PieceSize: abi.PaddedPieceSize(ssize), Client: k.Address, diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 254a2f0d3..9a497f879 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -5,8 +5,6 @@ import ( "context" "time" - "github.com/filecoin-project/specs-actors/actors/runtime/proof" - "github.com/filecoin-project/go-bitfield" "github.com/filecoin-project/go-address" @@ -14,20 +12,21 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/go-state-types/dline" - "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/ipfs/go-cid" "go.opencensus.io/trace" "golang.org/x/xerrors" + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" + "github.com/filecoin-project/specs-actors/actors/runtime/proof" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/journal" - - miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" ) func (s *WindowPoStScheduler) failPost(err error, deadline *dline.Info) { @@ -219,7 +218,7 @@ func (s *WindowPoStScheduler) checkNextRecoveries(ctx context.Context, dlIdx uin msg := &types.Message{ To: s.actor, From: s.worker, - Method: builtin.MethodsMiner.DeclareFaultsRecovered, + Method: builtin0.MethodsMiner.DeclareFaultsRecovered, Params: enc, Value: types.NewInt(0), } @@ -298,7 +297,7 @@ func (s *WindowPoStScheduler) checkNextFaults(ctx context.Context, dlIdx uint64, msg := &types.Message{ To: s.actor, From: s.worker, - Method: builtin.MethodsMiner.DeclareFaults, + Method: builtin0.MethodsMiner.DeclareFaults, Params: enc, Value: types.NewInt(0), // TODO: Is there a fee? } @@ -555,7 +554,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty func (s *WindowPoStScheduler) batchPartitions(partitions []api.Partition) ([][]api.Partition, error) { // Get the number of sectors allowed in a partition, for this proof size - sectorsPerPartition, err := builtin.PoStProofWindowPoStPartitionSectors(s.proofType) + sectorsPerPartition, err := builtin0.PoStProofWindowPoStPartitionSectors(s.proofType) if err != nil { return nil, xerrors.Errorf("getting sectors per partition: %w", err) } @@ -647,7 +646,7 @@ func (s *WindowPoStScheduler) submitPost(ctx context.Context, proof *miner.Submi msg := &types.Message{ To: s.actor, From: s.worker, - Method: builtin.MethodsMiner.SubmitWindowedPoSt, + Method: builtin0.MethodsMiner.SubmitWindowedPoSt, Params: enc, Value: types.NewInt(0), } diff --git a/storage/wdpost_run_test.go b/storage/wdpost_run_test.go index 1797cf35c..10be2fbcd 100644 --- a/storage/wdpost_run_test.go +++ b/storage/wdpost_run_test.go @@ -16,10 +16,9 @@ import ( "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/go-state-types/dline" "github.com/filecoin-project/go-state-types/network" - "github.com/filecoin-project/specs-actors/actors/builtin" builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" - "github.com/filecoin-project/specs-actors/actors/runtime/proof" + proof0 "github.com/filecoin-project/specs-actors/actors/runtime/proof" tutils "github.com/filecoin-project/specs-actors/support/testing" "github.com/filecoin-project/lotus/api" @@ -97,12 +96,12 @@ func (m *mockStorageMinerAPI) StateWaitMsg(ctx context.Context, cid cid.Cid, con type mockProver struct { } -func (m *mockProver) GenerateWinningPoSt(context.Context, abi.ActorID, []proof.SectorInfo, abi.PoStRandomness) ([]proof.PoStProof, error) { +func (m *mockProver) GenerateWinningPoSt(context.Context, abi.ActorID, []proof0.SectorInfo, abi.PoStRandomness) ([]proof0.PoStProof, error) { panic("implement me") } -func (m *mockProver) GenerateWindowPoSt(ctx context.Context, aid abi.ActorID, sis []proof.SectorInfo, pr abi.PoStRandomness) ([]proof.PoStProof, []abi.SectorID, error) { - return []proof.PoStProof{ +func (m *mockProver) GenerateWindowPoSt(ctx context.Context, aid abi.ActorID, sis []proof0.SectorInfo, pr abi.PoStRandomness) ([]proof0.PoStProof, []abi.SectorID, error) { + return []proof0.PoStProof{ { PoStProof: abi.RegisteredPoStProof_StackedDrgWindow2KiBV1, ProofBytes: []byte("post-proof"), @@ -131,7 +130,7 @@ func TestWDPostDoPost(t *testing.T) { mockStgMinerAPI := newMockStorageMinerAPI() // Get the number of sectors allowed in a partition for this proof type - sectorsPerPartition, err := builtin.PoStProofWindowPoStPartitionSectors(proofType) + sectorsPerPartition, err := builtin0.PoStProofWindowPoStPartitionSectors(proofType) require.NoError(t, err) // Work out the number of partitions that can be included in a message // without exceeding the message sector limit @@ -183,7 +182,7 @@ func TestWDPostDoPost(t *testing.T) { // Read the window PoST messages for i := 0; i < expectedMsgCount; i++ { msg := <-mockStgMinerAPI.pushedMessages - require.Equal(t, builtin.MethodsMiner.SubmitWindowedPoSt, msg.Method) + require.Equal(t, builtin0.MethodsMiner.SubmitWindowedPoSt, msg.Method) var params miner.SubmitWindowedPoStParams err := params.UnmarshalCBOR(bytes.NewReader(msg.Params)) require.NoError(t, err) From d33dd4f7bc9bc79b0c6e7957ab115c430d97bc79 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 21 Sep 2020 16:01:29 -0700 Subject: [PATCH 274/885] more renames --- chain/types/actor.go | 4 +++ markets/storageadapter/provider.go | 26 ++++++++-------- miner/miner.go | 4 +-- node/impl/full/gas.go | 7 +++-- node/impl/full/multisig.go | 49 ++++++++++++++++-------------- node/impl/full/state.go | 10 +++--- node/impl/paych/paych.go | 21 +++++++------ paychmgr/settler/settler.go | 15 ++++----- paychmgr/simple.go | 6 ++-- 9 files changed, 77 insertions(+), 65 deletions(-) diff --git a/chain/types/actor.go b/chain/types/actor.go index aa3ba2146..eb8e05c49 100644 --- a/chain/types/actor.go +++ b/chain/types/actor.go @@ -29,3 +29,7 @@ func (a *Actor) IsStorageMinerActor() bool { func (a *Actor) IsMultisigActor() bool { return a.Code == builtin0.MultisigActorCodeID } + +func (a *Actor) IsPaymentChannelActor() bool { + return a.Code == builtin0.PaymentChannelActorCodeID +} diff --git a/markets/storageadapter/provider.go b/markets/storageadapter/provider.go index 04c1055df..9f610d76a 100644 --- a/markets/storageadapter/provider.go +++ b/markets/storageadapter/provider.go @@ -8,25 +8,27 @@ import ( "io" "time" - "github.com/filecoin-project/lotus/chain/actors/builtin/miner" - "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" "golang.org/x/xerrors" + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" + market0 "github.com/filecoin-project/specs-actors/actors/builtin/market" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-fil-markets/shared" "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/go-state-types/exitcode" - "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/builtin/market" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/events" + + "github.com/filecoin-project/lotus/chain/actors/builtin/market" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/events/state" "github.com/filecoin-project/lotus/chain/types" sealing "github.com/filecoin-project/lotus/extern/storage-sealing" @@ -67,8 +69,8 @@ func (n *ProviderNodeAdapter) PublishDeals(ctx context.Context, deal storagemark return cid.Undef, err } - params, err := actors.SerializeParams(&market.PublishStorageDealsParams{ - Deals: []market.ClientDealProposal{deal.ClientDealProposal}, + params, err := actors.SerializeParams(&market0.PublishStorageDealsParams{ + Deals: []market0.ClientDealProposal{deal.ClientDealProposal}, }) if err != nil { @@ -77,10 +79,10 @@ func (n *ProviderNodeAdapter) PublishDeals(ctx context.Context, deal storagemark // TODO: We may want this to happen after fetching data smsg, err := n.MpoolPushMessage(ctx, &types.Message{ - To: builtin.StorageMarketActorAddr, + To: market.Address, From: mi.Worker, Value: types.NewInt(0), - Method: builtin.MethodsMarket.PublishStorageDeals, + Method: builtin0.MethodsMarket.PublishStorageDeals, Params: params, }, nil) if err != nil { @@ -177,10 +179,10 @@ func (n *ProviderNodeAdapter) EnsureFunds(ctx context.Context, addr, wallet addr func (n *ProviderNodeAdapter) AddFunds(ctx context.Context, addr address.Address, amount abi.TokenAmount) (cid.Cid, error) { // (Provider Node API) smsg, err := n.MpoolPushMessage(ctx, &types.Message{ - To: builtin.StorageMarketActorAddr, + To: market.Address, From: addr, Value: amount, - Method: builtin.MethodsMarket.AddBalance, + Method: builtin0.MethodsMarket.AddBalance, }, nil) if err != nil { return cid.Undef, err @@ -302,7 +304,7 @@ func (n *ProviderNodeAdapter) OnDealSectorCommitted(ctx context.Context, provide } switch msg.Method { - case builtin.MethodsMiner.PreCommitSector: + case builtin0.MethodsMiner.PreCommitSector: var params miner.SectorPreCommitInfo if err := params.UnmarshalCBOR(bytes.NewReader(msg.Params)); err != nil { return true, false, xerrors.Errorf("unmarshal pre commit: %w", err) @@ -317,7 +319,7 @@ func (n *ProviderNodeAdapter) OnDealSectorCommitted(ctx context.Context, provide } return true, false, nil - case builtin.MethodsMiner.ProveCommitSector: + case builtin0.MethodsMiner.ProveCommitSector: var params miner.ProveCommitSectorParams if err := params.UnmarshalCBOR(bytes.NewReader(msg.Params)); err != nil { return true, false, xerrors.Errorf("failed to unmarshal prove commit sector params: %w", err) diff --git a/miner/miner.go b/miner/miner.go index 2d8591992..5e8d8cf37 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -9,7 +9,7 @@ import ( "sync" "time" - "github.com/filecoin-project/specs-actors/actors/runtime/proof" + proof0 "github.com/filecoin-project/specs-actors/actors/runtime/proof" "github.com/filecoin-project/lotus/chain/gen/slashfilter" @@ -489,7 +489,7 @@ func (m *Miner) computeTicket(ctx context.Context, brand *types.BeaconEntry, bas } func (m *Miner) createBlock(base *MiningBase, addr address.Address, ticket *types.Ticket, - eproof *types.ElectionProof, bvals []types.BeaconEntry, wpostProof []proof.PoStProof, msgs []*types.SignedMessage) (*types.BlockMsg, error) { + eproof *types.ElectionProof, bvals []types.BeaconEntry, wpostProof []proof0.PoStProof, msgs []*types.SignedMessage) (*types.BlockMsg, error) { uts := base.TipSet.MinTimestamp() + build.BlockDelaySecs*(uint64(base.NullRounds)+1) nheight := base.TipSet.Height() + base.NullRounds + 1 diff --git a/node/impl/full/gas.go b/node/impl/full/gas.go index b207a1f6d..f03807c80 100644 --- a/node/impl/full/gas.go +++ b/node/impl/full/gas.go @@ -13,7 +13,8 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/exitcode" - "github.com/filecoin-project/specs-actors/actors/builtin" + + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" @@ -181,10 +182,10 @@ func (a *GasAPI) GasEstimateGasLimit(ctx context.Context, msgIn *types.Message, return res.MsgRct.GasUsed, nil } - if !act.Code.Equals(builtin.PaymentChannelActorCodeID) { + if !act.IsPaymentChannelActor() { return res.MsgRct.GasUsed, nil } - if msgIn.Method != builtin.MethodsPaych.Collect { + if msgIn.Method != builtin0.MethodsPaych.Collect { return res.MsgRct.GasUsed, nil } diff --git a/node/impl/full/multisig.go b/node/impl/full/multisig.go index 2322d5a1e..8c15a27be 100644 --- a/node/impl/full/multisig.go +++ b/node/impl/full/multisig.go @@ -9,10 +9,12 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" + init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/specs-actors/actors/builtin" - init_ "github.com/filecoin-project/specs-actors/actors/builtin/init" - samsig "github.com/filecoin-project/specs-actors/actors/builtin/multisig" + + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" + init0 "github.com/filecoin-project/specs-actors/actors/builtin/init" + multisig0 "github.com/filecoin-project/specs-actors/actors/builtin/multisig" "github.com/ipfs/go-cid" "github.com/minio/blake2b-simd" @@ -46,7 +48,7 @@ func (a *MsigAPI) MsigCreate(ctx context.Context, req uint64, addrs []address.Ad } // Set up constructor parameters for multisig - msigParams := &samsig.ConstructorParams{ + msigParams := &multisig0.ConstructorParams{ Signers: addrs, NumApprovalsThreshold: req, UnlockDuration: duration, @@ -58,8 +60,9 @@ func (a *MsigAPI) MsigCreate(ctx context.Context, req uint64, addrs []address.Ad } // new actors are created by invoking 'exec' on the init actor with the constructor params - execParams := &init_.ExecParams{ - CodeCID: builtin.MultisigActorCodeID, + // TODO: network upgrade? + execParams := &init0.ExecParams{ + CodeCID: builtin0.MultisigActorCodeID, ConstructorParams: enc, } @@ -70,9 +73,9 @@ func (a *MsigAPI) MsigCreate(ctx context.Context, req uint64, addrs []address.Ad // now we create the message to send this with msg := types.Message{ - To: builtin.InitActorAddr, + To: init_.Address, From: src, - Method: builtin.MethodsInit.Exec, + Method: builtin0.MethodsInit.Exec, Params: enc, Value: val, } @@ -104,7 +107,7 @@ func (a *MsigAPI) MsigPropose(ctx context.Context, msig address.Address, to addr return cid.Undef, xerrors.Errorf("must provide source address") } - enc, actErr := actors.SerializeParams(&samsig.ProposeParams{ + enc, actErr := actors.SerializeParams(&multisig0.ProposeParams{ To: to, Value: amt, Method: abi.MethodNum(method), @@ -118,7 +121,7 @@ func (a *MsigAPI) MsigPropose(ctx context.Context, msig address.Address, to addr To: msig, From: src, Value: types.NewInt(0), - Method: builtin.MethodsMultisig.Propose, + Method: builtin0.MethodsMultisig.Propose, Params: enc, } @@ -136,7 +139,7 @@ func (a *MsigAPI) MsigAddPropose(ctx context.Context, msig address.Address, src return cid.Undef, actErr } - return a.MsigPropose(ctx, msig, msig, big.Zero(), src, uint64(builtin.MethodsMultisig.AddSigner), enc) + return a.MsigPropose(ctx, msig, msig, big.Zero(), src, uint64(builtin0.MethodsMultisig.AddSigner), enc) } func (a *MsigAPI) MsigAddApprove(ctx context.Context, msig address.Address, src address.Address, txID uint64, proposer address.Address, newAdd address.Address, inc bool) (cid.Cid, error) { @@ -145,7 +148,7 @@ func (a *MsigAPI) MsigAddApprove(ctx context.Context, msig address.Address, src return cid.Undef, actErr } - return a.MsigApprove(ctx, msig, txID, proposer, msig, big.Zero(), src, uint64(builtin.MethodsMultisig.AddSigner), enc) + return a.MsigApprove(ctx, msig, txID, proposer, msig, big.Zero(), src, uint64(builtin0.MethodsMultisig.AddSigner), enc) } func (a *MsigAPI) MsigAddCancel(ctx context.Context, msig address.Address, src address.Address, txID uint64, newAdd address.Address, inc bool) (cid.Cid, error) { @@ -154,7 +157,7 @@ func (a *MsigAPI) MsigAddCancel(ctx context.Context, msig address.Address, src a return cid.Undef, actErr } - return a.MsigCancel(ctx, msig, txID, msig, big.Zero(), src, uint64(builtin.MethodsMultisig.AddSigner), enc) + return a.MsigCancel(ctx, msig, txID, msig, big.Zero(), src, uint64(builtin0.MethodsMultisig.AddSigner), enc) } func (a *MsigAPI) MsigSwapPropose(ctx context.Context, msig address.Address, src address.Address, oldAdd address.Address, newAdd address.Address) (cid.Cid, error) { @@ -163,7 +166,7 @@ func (a *MsigAPI) MsigSwapPropose(ctx context.Context, msig address.Address, src return cid.Undef, actErr } - return a.MsigPropose(ctx, msig, msig, big.Zero(), src, uint64(builtin.MethodsMultisig.SwapSigner), enc) + return a.MsigPropose(ctx, msig, msig, big.Zero(), src, uint64(builtin0.MethodsMultisig.SwapSigner), enc) } func (a *MsigAPI) MsigSwapApprove(ctx context.Context, msig address.Address, src address.Address, txID uint64, proposer address.Address, oldAdd address.Address, newAdd address.Address) (cid.Cid, error) { @@ -172,7 +175,7 @@ func (a *MsigAPI) MsigSwapApprove(ctx context.Context, msig address.Address, src return cid.Undef, actErr } - return a.MsigApprove(ctx, msig, txID, proposer, msig, big.Zero(), src, uint64(builtin.MethodsMultisig.SwapSigner), enc) + return a.MsigApprove(ctx, msig, txID, proposer, msig, big.Zero(), src, uint64(builtin0.MethodsMultisig.SwapSigner), enc) } func (a *MsigAPI) MsigSwapCancel(ctx context.Context, msig address.Address, src address.Address, txID uint64, oldAdd address.Address, newAdd address.Address) (cid.Cid, error) { @@ -181,7 +184,7 @@ func (a *MsigAPI) MsigSwapCancel(ctx context.Context, msig address.Address, src return cid.Undef, actErr } - return a.MsigCancel(ctx, msig, txID, msig, big.Zero(), src, uint64(builtin.MethodsMultisig.SwapSigner), enc) + return a.MsigCancel(ctx, msig, txID, msig, big.Zero(), src, uint64(builtin0.MethodsMultisig.SwapSigner), enc) } func (a *MsigAPI) MsigApprove(ctx context.Context, msig address.Address, txID uint64, proposer address.Address, to address.Address, amt types.BigInt, src address.Address, method uint64, params []byte) (cid.Cid, error) { @@ -217,7 +220,7 @@ func (a *MsigAPI) msigApproveOrCancel(ctx context.Context, operation api.MsigPro proposer = proposerID } - p := samsig.ProposalHashData{ + p := multisig0.ProposalHashData{ Requester: proposer, To: to, Value: amt, @@ -231,8 +234,8 @@ func (a *MsigAPI) msigApproveOrCancel(ctx context.Context, operation api.MsigPro } phash := blake2b.Sum256(pser) - enc, err := actors.SerializeParams(&samsig.TxnIDParams{ - ID: samsig.TxnID(txID), + enc, err := actors.SerializeParams(&multisig0.TxnIDParams{ + ID: multisig0.TxnID(txID), ProposalHash: phash[:], }) @@ -248,9 +251,9 @@ func (a *MsigAPI) msigApproveOrCancel(ctx context.Context, operation api.MsigPro */ switch operation { case api.MsigApprove: - msigResponseMethod = builtin.MethodsMultisig.Approve + msigResponseMethod = builtin0.MethodsMultisig.Approve case api.MsigCancel: - msigResponseMethod = builtin.MethodsMultisig.Cancel + msigResponseMethod = builtin0.MethodsMultisig.Cancel default: return cid.Undef, xerrors.Errorf("Invalid operation for msigApproveOrCancel") } @@ -272,7 +275,7 @@ func (a *MsigAPI) msigApproveOrCancel(ctx context.Context, operation api.MsigPro } func serializeAddParams(new address.Address, inc bool) ([]byte, error) { - enc, actErr := actors.SerializeParams(&samsig.AddSignerParams{ + enc, actErr := actors.SerializeParams(&multisig0.AddSignerParams{ Signer: new, Increase: inc, }) @@ -284,7 +287,7 @@ func serializeAddParams(new address.Address, inc bool) ([]byte, error) { } func serializeSwapParams(old address.Address, new address.Address) ([]byte, error) { - enc, actErr := actors.SerializeParams(&samsig.SwapSignerParams{ + enc, actErr := actors.SerializeParams(&multisig0.SwapSignerParams{ From: old, To: new, }) diff --git a/node/impl/full/state.go b/node/impl/full/state.go index df747ba3f..0b2d86718 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -7,7 +7,9 @@ import ( builtin2 "github.com/filecoin-project/lotus/chain/actors/builtin" + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" market0 "github.com/filecoin-project/specs-actors/actors/builtin/market" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/actors/builtin/verifreg" @@ -24,8 +26,6 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" - "github.com/filecoin-project/specs-actors/actors/builtin" - miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors/builtin/market" @@ -1025,7 +1025,7 @@ func (a *StateAPI) StateMinerAvailableBalance(ctx context.Context, maddr address // Returns zero if there is no entry in the data cap table for the // address. func (a *StateAPI) StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) { - act, err := a.StateGetActor(ctx, builtin.VerifiedRegistryActorAddr, tsk) + act, err := a.StateGetActor(ctx, builtin0.VerifiedRegistryActorAddr, tsk) if err != nil { return nil, err } @@ -1063,12 +1063,12 @@ func (a *StateAPI) StateDealProviderCollateralBounds(ctx context.Context, size a return api.DealCollateralBounds{}, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - pact, err := a.StateGetActor(ctx, builtin.StoragePowerActorAddr, tsk) + pact, err := a.StateGetActor(ctx, builtin0.StoragePowerActorAddr, tsk) if err != nil { return api.DealCollateralBounds{}, xerrors.Errorf("failed to load power actor: %w", err) } - ract, err := a.StateGetActor(ctx, builtin.RewardActorAddr, tsk) + ract, err := a.StateGetActor(ctx, builtin0.RewardActorAddr, tsk) if err != nil { return api.DealCollateralBounds{}, xerrors.Errorf("failed to load reward actor: %w", err) } diff --git a/node/impl/paych/paych.go b/node/impl/paych/paych.go index 94fcc320d..b2e6be0ca 100644 --- a/node/impl/paych/paych.go +++ b/node/impl/paych/paych.go @@ -9,7 +9,8 @@ import ( "go.uber.org/fx" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/specs-actors/actors/builtin/paych" + + paych0 "github.com/filecoin-project/specs-actors/actors/builtin/paych" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" @@ -70,10 +71,10 @@ func (a *PaychAPI) PaychNewPayment(ctx context.Context, from, to address.Address return nil, err } - svs := make([]*paych.SignedVoucher, len(vouchers)) + svs := make([]*paych0.SignedVoucher, len(vouchers)) for i, v := range vouchers { - sv, err := a.PaychMgr.CreateVoucher(ctx, ch.Channel, paych.SignedVoucher{ + sv, err := a.PaychMgr.CreateVoucher(ctx, ch.Channel, paych0.SignedVoucher{ Amount: v.Amount, Lane: lane, @@ -122,15 +123,15 @@ func (a *PaychAPI) PaychCollect(ctx context.Context, addr address.Address) (cid. return a.PaychMgr.Collect(ctx, addr) } -func (a *PaychAPI) PaychVoucherCheckValid(ctx context.Context, ch address.Address, sv *paych.SignedVoucher) error { +func (a *PaychAPI) PaychVoucherCheckValid(ctx context.Context, ch address.Address, sv *paych0.SignedVoucher) error { return a.PaychMgr.CheckVoucherValid(ctx, ch, sv) } -func (a *PaychAPI) PaychVoucherCheckSpendable(ctx context.Context, ch address.Address, sv *paych.SignedVoucher, secret []byte, proof []byte) (bool, error) { +func (a *PaychAPI) PaychVoucherCheckSpendable(ctx context.Context, ch address.Address, sv *paych0.SignedVoucher, secret []byte, proof []byte) (bool, error) { return a.PaychMgr.CheckVoucherSpendable(ctx, ch, sv, secret, proof) } -func (a *PaychAPI) PaychVoucherAdd(ctx context.Context, ch address.Address, sv *paych.SignedVoucher, proof []byte, minDelta types.BigInt) (types.BigInt, error) { +func (a *PaychAPI) PaychVoucherAdd(ctx context.Context, ch address.Address, sv *paych0.SignedVoucher, proof []byte, minDelta types.BigInt) (types.BigInt, error) { return a.PaychMgr.AddVoucherInbound(ctx, ch, sv, proof, minDelta) } @@ -142,16 +143,16 @@ func (a *PaychAPI) PaychVoucherAdd(ctx context.Context, ch address.Address, sv * // If there are insufficient funds in the channel to create the voucher, // returns a nil voucher and the shortfall. func (a *PaychAPI) PaychVoucherCreate(ctx context.Context, pch address.Address, amt types.BigInt, lane uint64) (*api.VoucherCreateResult, error) { - return a.PaychMgr.CreateVoucher(ctx, pch, paych.SignedVoucher{Amount: amt, Lane: lane}) + return a.PaychMgr.CreateVoucher(ctx, pch, paych0.SignedVoucher{Amount: amt, Lane: lane}) } -func (a *PaychAPI) PaychVoucherList(ctx context.Context, pch address.Address) ([]*paych.SignedVoucher, error) { +func (a *PaychAPI) PaychVoucherList(ctx context.Context, pch address.Address) ([]*paych0.SignedVoucher, error) { vi, err := a.PaychMgr.ListVouchers(ctx, pch) if err != nil { return nil, err } - out := make([]*paych.SignedVoucher, len(vi)) + out := make([]*paych0.SignedVoucher, len(vi)) for k, v := range vi { out[k] = v.Voucher } @@ -159,6 +160,6 @@ func (a *PaychAPI) PaychVoucherList(ctx context.Context, pch address.Address) ([ return out, nil } -func (a *PaychAPI) PaychVoucherSubmit(ctx context.Context, ch address.Address, sv *paych.SignedVoucher, secret []byte, proof []byte) (cid.Cid, error) { +func (a *PaychAPI) PaychVoucherSubmit(ctx context.Context, ch address.Address, sv *paych0.SignedVoucher, secret []byte, proof []byte) (cid.Cid, error) { return a.PaychMgr.SubmitVoucher(ctx, ch, sv, secret, proof) } diff --git a/paychmgr/settler/settler.go b/paychmgr/settler/settler.go index 45f24cdd9..654ed66cc 100644 --- a/paychmgr/settler/settler.go +++ b/paychmgr/settler/settler.go @@ -13,8 +13,9 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/builtin/paych" + + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" + paych0 "github.com/filecoin-project/specs-actors/actors/builtin/paych" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" @@ -38,9 +39,9 @@ type API struct { type settlerAPI interface { PaychList(context.Context) ([]address.Address, error) PaychStatus(context.Context, address.Address) (*api.PaychStatus, error) - PaychVoucherCheckSpendable(context.Context, address.Address, *paych.SignedVoucher, []byte, []byte) (bool, error) - PaychVoucherList(context.Context, address.Address) ([]*paych.SignedVoucher, error) - PaychVoucherSubmit(context.Context, address.Address, *paych.SignedVoucher, []byte, []byte) (cid.Cid, error) + PaychVoucherCheckSpendable(context.Context, address.Address, *paych0.SignedVoucher, []byte, []byte) (bool, error) + PaychVoucherList(context.Context, address.Address) ([]*paych0.SignedVoucher, error) + PaychVoucherSubmit(context.Context, address.Address, *paych0.SignedVoucher, []byte, []byte) (cid.Cid, error) StateWaitMsg(ctx context.Context, cid cid.Cid, confidence uint64) (*api.MsgLookup, error) } @@ -85,7 +86,7 @@ func (pcs *paymentChannelSettler) messageHandler(msg *types.Message, rec *types. if err != nil { return true, err } - go func(voucher *paych.SignedVoucher, submitMessageCID cid.Cid) { + go func(voucher *paych0.SignedVoucher, submitMessageCID cid.Cid) { defer wg.Done() msgLookup, err := pcs.api.StateWaitMsg(pcs.ctx, submitMessageCID, build.MessageConfidence) if err != nil { @@ -106,7 +107,7 @@ func (pcs *paymentChannelSettler) revertHandler(ctx context.Context, ts *types.T func (pcs *paymentChannelSettler) matcher(msg *types.Message) (matchOnce bool, matched bool, err error) { // Check if this is a settle payment channel message - if msg.Method != builtin.MethodsPaych.Settle { + if msg.Method != builtin0.MethodsPaych.Settle { return false, false, nil } // Check if this payment channel is of concern to this node (i.e. tracked in payment channel store), diff --git a/paychmgr/simple.go b/paychmgr/simple.go index 46bbea62e..815b8acab 100644 --- a/paychmgr/simple.go +++ b/paychmgr/simple.go @@ -13,7 +13,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/specs-actors/actors/builtin" - init_ "github.com/filecoin-project/specs-actors/actors/builtin/init" + init0 "github.com/filecoin-project/specs-actors/actors/builtin/init" paych0 "github.com/filecoin-project/specs-actors/actors/builtin/paych" "github.com/filecoin-project/lotus/api" @@ -387,7 +387,7 @@ func (ca *channelAccessor) createPaych(ctx context.Context, amt types.BigInt) (c return cid.Undef, aerr } - enc, aerr := actors.SerializeParams(&init_.ExecParams{ + enc, aerr := actors.SerializeParams(&init0.ExecParams{ CodeCID: builtin.PaymentChannelActorCodeID, ConstructorParams: params, }) @@ -452,7 +452,7 @@ func (ca *channelAccessor) waitPaychCreateMsg(channelID string, mcid cid.Cid) er return err } - var decodedReturn init_.ExecReturn + var decodedReturn init0.ExecReturn err = decodedReturn.UnmarshalCBOR(bytes.NewReader(mwait.Receipt.Return)) if err != nil { log.Error(err) From d56da1b014b24c6d163f60546c51462ab968f383 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Tue, 22 Sep 2020 00:12:07 -0400 Subject: [PATCH 275/885] Refinements to stmgr and utils --- chain/actors/builtin/builtin.go | 8 +++ chain/actors/builtin/reward/reward.go | 3 ++ chain/stmgr/stmgr.go | 75 ++++++++++++--------------- chain/stmgr/utils.go | 15 ++---- node/impl/full/state.go | 19 +++---- 5 files changed, 56 insertions(+), 64 deletions(-) diff --git a/chain/actors/builtin/builtin.go b/chain/actors/builtin/builtin.go index 9ad976564..3230705dd 100644 --- a/chain/actors/builtin/builtin.go +++ b/chain/actors/builtin/builtin.go @@ -3,6 +3,9 @@ package builtin import ( "fmt" + "github.com/filecoin-project/go-state-types/abi" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" + smoothing0 "github.com/filecoin-project/specs-actors/actors/util/smoothing" "github.com/filecoin-project/go-state-types/network" @@ -30,3 +33,8 @@ type FilterEstimate = smoothing0.FilterEstimate func FromV0FilterEstimate(v0 smoothing0.FilterEstimate) FilterEstimate { return (FilterEstimate)(v0) } + +// Doesn't change between actors v0 and v1 +func QAPowerForWeight(size abi.SectorSize, duration abi.ChainEpoch, dealWeight, verifiedWeight abi.DealWeight) abi.StoragePower { + return miner0.QAPowerForWeight(size, duration, dealWeight, verifiedWeight) +} diff --git a/chain/actors/builtin/reward/reward.go b/chain/actors/builtin/reward/reward.go index 3f887914d..a4f936d9b 100644 --- a/chain/actors/builtin/reward/reward.go +++ b/chain/actors/builtin/reward/reward.go @@ -2,6 +2,7 @@ package reward import ( "github.com/filecoin-project/go-state-types/abi" + reward0 "github.com/filecoin-project/specs-actors/actors/builtin/reward" "golang.org/x/xerrors" "github.com/filecoin-project/go-state-types/cbor" @@ -45,3 +46,5 @@ type State interface { InitialPledgeForPower(abi.StoragePower, abi.TokenAmount, *builtin.FilterEstimate, abi.TokenAmount) (abi.TokenAmount, error) PreCommitDepositForPower(builtin.FilterEstimate, abi.StoragePower) (abi.TokenAmount, error) } + +type AwardBlockRewardParams = reward0.AwardBlockRewardParams diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index e6e557930..2f8d460aa 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -22,8 +22,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/network" - "github.com/filecoin-project/specs-actors/actors/builtin" - reward0 "github.com/filecoin-project/specs-actors/actors/builtin/reward" + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/lotus/api" @@ -169,20 +168,20 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp runCron := func() error { // TODO: this nonce-getting is a tiny bit ugly - ca, err := vmi.StateTree().GetActor(builtin.SystemActorAddr) + ca, err := vmi.StateTree().GetActor(builtin0.SystemActorAddr) if err != nil { return err } cronMsg := &types.Message{ - To: builtin.CronActorAddr, - From: builtin.SystemActorAddr, + To: builtin0.CronActorAddr, + From: builtin0.SystemActorAddr, Nonce: ca.Nonce, Value: types.NewInt(0), GasFeeCap: types.NewInt(0), GasPremium: types.NewInt(0), GasLimit: build.BlockGasLimit * 10000, // Make super sure this is never too little - Method: builtin.MethodsCron.EpochTick, + Method: builtin0.MethodsCron.EpochTick, Params: nil, } ret, err := vmi.ApplyImplicitMessage(ctx, cronMsg) @@ -247,42 +246,34 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp processedMsgs[m.Cid()] = true } - var params []byte - - nv := sm.GetNtwkVersion(ctx, epoch) - if nv < build.ActorUpgradeNetworkVersion { - params, err = actors.SerializeParams(&reward0.AwardBlockRewardParams{ - Miner: b.Miner, - Penalty: penalty, - GasReward: gasReward, - WinCount: b.WinCount, - }) - if err != nil { - return cid.Undef, cid.Undef, xerrors.Errorf("failed to serialize award params: %w", err) - } - } else { - // TODO: ActorUpgrade - params = nil + params, err := actors.SerializeParams(&reward.AwardBlockRewardParams{ + Miner: b.Miner, + Penalty: penalty, + GasReward: gasReward, + WinCount: b.WinCount, + }) + if err != nil { + return cid.Undef, cid.Undef, xerrors.Errorf("failed to serialize award params: %w", err) } - sysAct, err := vmi.StateTree().GetActor(builtin.SystemActorAddr) - if err != nil { + sysAct, actErr := vmi.StateTree().GetActor(builtin0.SystemActorAddr) + if actErr != nil { return cid.Undef, cid.Undef, xerrors.Errorf("failed to get system actor: %w", err) } rwMsg := &types.Message{ - From: builtin.SystemActorAddr, - To: builtin.RewardActorAddr, + From: builtin0.SystemActorAddr, + To: reward.Address, Nonce: sysAct.Nonce, Value: types.NewInt(0), GasFeeCap: types.NewInt(0), GasPremium: types.NewInt(0), GasLimit: 1 << 30, - Method: builtin.MethodsReward.AwardBlockReward, + Method: builtin0.MethodsReward.AwardBlockReward, Params: params, } - ret, err := vmi.ApplyImplicitMessage(ctx, rwMsg) - if err != nil { + ret, actErr := vmi.ApplyImplicitMessage(ctx, rwMsg) + if actErr != nil { return cid.Undef, cid.Undef, xerrors.Errorf("failed to apply reward message for miner %s: %w", b.Miner, err) } if cb != nil { @@ -725,7 +716,7 @@ func (sm *StateManager) MarketBalance(ctx context.Context, addr address.Address, return api.MarketBalance{}, err } - act, err := st.GetActor(builtin.StorageMarketActorAddr) + act, err := st.GetActor(market.Address) if err != nil { return api.MarketBalance{}, err } @@ -853,7 +844,7 @@ func (sm *StateManager) setupGenesisActors(ctx context.Context) error { totalsByEpoch := make(map[abi.ChainEpoch]abi.TokenAmount) var act types.Actor err = r.ForEach(&act, func(k string) error { - if act.Code == builtin.MultisigActorCodeID { + if act.Code == builtin0.MultisigActorCodeID { var s multisig.State err := sm.cs.Store(ctx).Get(ctx, act.Head, &s) if err != nil { @@ -871,7 +862,7 @@ func (sm *StateManager) setupGenesisActors(ctx context.Context) error { totalsByEpoch[s.UnlockDuration()] = s.InitialBalance() } - } else if act.Code == builtin.AccountActorCodeID { + } else if act.Code == builtin0.AccountActorCodeID { // should exclude burnt funds actor and "remainder account actor" // should only ever be "faucet" accounts in testnets kaddr, err := address.NewFromBytes([]byte(k)) @@ -879,7 +870,7 @@ func (sm *StateManager) setupGenesisActors(ctx context.Context) error { return xerrors.Errorf("decoding address: %w", err) } - if kaddr != builtin.BurntFundsActorAddr { + if kaddr != builtin0.BurntFundsActorAddr { kid, err := sTree.LookupID(kaddr) if err != nil { return xerrors.Errorf("resolving address: %w", err) @@ -954,24 +945,24 @@ func (sm *StateManager) setupGenesisActorsTestnet(ctx context.Context) error { totalsByEpoch := make(map[abi.ChainEpoch]abi.TokenAmount) // 6 months - sixMonths := abi.ChainEpoch(183 * builtin.EpochsInDay) + sixMonths := abi.ChainEpoch(183 * builtin0.EpochsInDay) totalsByEpoch[sixMonths] = big.NewInt(49_929_341) totalsByEpoch[sixMonths] = big.Add(totalsByEpoch[sixMonths], big.NewInt(32_787_700)) // 1 year - oneYear := abi.ChainEpoch(365 * builtin.EpochsInDay) + oneYear := abi.ChainEpoch(365 * builtin0.EpochsInDay) totalsByEpoch[oneYear] = big.NewInt(22_421_712) // 2 years - twoYears := abi.ChainEpoch(2 * 365 * builtin.EpochsInDay) + twoYears := abi.ChainEpoch(2 * 365 * builtin0.EpochsInDay) totalsByEpoch[twoYears] = big.NewInt(7_223_364) // 3 years - threeYears := abi.ChainEpoch(3 * 365 * builtin.EpochsInDay) + threeYears := abi.ChainEpoch(3 * 365 * builtin0.EpochsInDay) totalsByEpoch[threeYears] = big.NewInt(87_637_883) // 6 years - sixYears := abi.ChainEpoch(6 * 365 * builtin.EpochsInDay) + sixYears := abi.ChainEpoch(6 * 365 * builtin0.EpochsInDay) totalsByEpoch[sixYears] = big.NewInt(100_000_000) totalsByEpoch[sixYears] = big.Add(totalsByEpoch[sixYears], big.NewInt(300_000_000)) @@ -1020,7 +1011,7 @@ func (sm *StateManager) GetFilVested(ctx context.Context, height abi.ChainEpoch, } func GetFilMined(ctx context.Context, st *state.StateTree) (abi.TokenAmount, error) { - ractor, err := st.GetActor(builtin.RewardActorAddr) + ractor, err := st.GetActor(reward.Address) if err != nil { return big.Zero(), xerrors.Errorf("failed to load reward actor state: %w", err) } @@ -1034,7 +1025,7 @@ func GetFilMined(ctx context.Context, st *state.StateTree) (abi.TokenAmount, err } func getFilMarketLocked(ctx context.Context, st *state.StateTree) (abi.TokenAmount, error) { - act, err := st.GetActor(builtin.StorageMarketActorAddr) + act, err := st.GetActor(market.Address) if err != nil { return big.Zero(), xerrors.Errorf("failed to load market actor: %w", err) } @@ -1048,7 +1039,7 @@ func getFilMarketLocked(ctx context.Context, st *state.StateTree) (abi.TokenAmou } func getFilPowerLocked(ctx context.Context, st *state.StateTree) (abi.TokenAmount, error) { - pactor, err := st.GetActor(builtin.StoragePowerActorAddr) + pactor, err := st.GetActor(power.Address) if err != nil { return big.Zero(), xerrors.Errorf("failed to load power actor: %w", err) } @@ -1077,7 +1068,7 @@ func (sm *StateManager) GetFilLocked(ctx context.Context, st *state.StateTree) ( } func GetFilBurnt(ctx context.Context, st *state.StateTree) (abi.TokenAmount, error) { - burnt, err := st.GetActor(builtin.BurntFundsActorAddr) + burnt, err := st.GetActor(builtin0.BurntFundsActorAddr) if err != nil { return big.Zero(), xerrors.Errorf("failed to load burnt actor: %w", err) } diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index 1e670dace..c9e8d32da 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -88,8 +88,7 @@ func GetPower(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr add } func GetPowerRaw(ctx context.Context, sm *StateManager, st cid.Cid, maddr address.Address) (power.Claim, power.Claim, bool, error) { - // TODO: ActorUpgrade - act, err := sm.LoadActorRaw(ctx, builtin0.StoragePowerActorAddr, st) + act, err := sm.LoadActorRaw(ctx, power.Address, st) if err != nil { return power.Claim{}, power.Claim{}, false, xerrors.Errorf("(get sset) failed to load power actor state: %w", err) } @@ -281,8 +280,7 @@ func StateMinerInfo(ctx context.Context, sm *StateManager, ts *types.TipSet, mad } func GetMinerSlashed(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) (bool, error) { - // TODO: ActorUpgrade - act, err := sm.LoadActor(ctx, builtin0.StoragePowerActorAddr, ts) + act, err := sm.LoadActor(ctx, power.Address, ts) if err != nil { return false, xerrors.Errorf("failed to load power actor: %w", err) } @@ -305,8 +303,7 @@ func GetMinerSlashed(ctx context.Context, sm *StateManager, ts *types.TipSet, ma } func GetStorageDeal(ctx context.Context, sm *StateManager, dealID abi.DealID, ts *types.TipSet) (*api.MarketDeal, error) { - // TODO: ActorUpgrade - act, err := sm.LoadActor(ctx, builtin0.StorageMarketActorAddr, ts) + act, err := sm.LoadActor(ctx, market.Address, ts) if err != nil { return nil, xerrors.Errorf("failed to load market actor: %w", err) } @@ -350,8 +347,7 @@ func GetStorageDeal(ctx context.Context, sm *StateManager, dealID abi.DealID, ts } func ListMinerActors(ctx context.Context, sm *StateManager, ts *types.TipSet) ([]address.Address, error) { - // TODO: ActorUpgrade - act, err := sm.LoadActor(ctx, builtin0.StoragePowerActorAddr, ts) + act, err := sm.LoadActor(ctx, power.Address, ts) if err != nil { return nil, xerrors.Errorf("failed to load power actor: %w", err) } @@ -630,8 +626,7 @@ func GetReturnType(ctx context.Context, sm *StateManager, to address.Address, me } func MinerHasMinPower(ctx context.Context, sm *StateManager, addr address.Address, ts *types.TipSet) (bool, error) { - // TODO: ActorUpgrade - pact, err := sm.LoadActor(ctx, builtin0.StoragePowerActorAddr, ts) + pact, err := sm.LoadActor(ctx, power.Address, ts) if err != nil { return false, xerrors.Errorf("loading power actor state: %w", err) } diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 0b2d86718..109a265ea 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -5,11 +5,10 @@ import ( "context" "strconv" - builtin2 "github.com/filecoin-project/lotus/chain/actors/builtin" + lotusbuiltin "github.com/filecoin-project/lotus/chain/actors/builtin" builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" market0 "github.com/filecoin-project/specs-actors/actors/builtin/market" - miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/actors/builtin/verifreg" @@ -871,7 +870,7 @@ func (a *StateAPI) StateMinerPreCommitDepositForPower(ctx context.Context, maddr var sectorWeight abi.StoragePower if act, err := state.GetActor(market.Address); err != nil { - return types.EmptyInt, xerrors.Errorf("loading miner actor %s: %w", maddr, err) + return types.EmptyInt, xerrors.Errorf("loading market actor %s: %w", maddr, err) } else if s, err := market.Load(store, act); err != nil { return types.EmptyInt, xerrors.Errorf("loading market actor state %s: %w", maddr, err) } else if w, vw, err := s.VerifyDealsForActivation(maddr, pci.DealIDs, ts.Height(), pci.Expiration); err != nil { @@ -879,14 +878,12 @@ func (a *StateAPI) StateMinerPreCommitDepositForPower(ctx context.Context, maddr } else { // NB: not exactly accurate, but should always lead us to *over* estimate, not under duration := pci.Expiration - ts.Height() - - // TODO: ActorUpgrade - sectorWeight = miner0.QAPowerForWeight(ssize, duration, w, vw) + sectorWeight = lotusbuiltin.QAPowerForWeight(ssize, duration, w, vw) } - var powerSmoothed builtin2.FilterEstimate + var powerSmoothed lotusbuiltin.FilterEstimate if act, err := state.GetActor(power.Address); err != nil { - return types.EmptyInt, xerrors.Errorf("loading miner actor: %w", err) + return types.EmptyInt, xerrors.Errorf("loading power actor: %w", err) } else if s, err := power.Load(store, act); err != nil { return types.EmptyInt, xerrors.Errorf("loading power actor state: %w", err) } else if p, err := s.TotalPowerSmoothed(); err != nil { @@ -942,13 +939,11 @@ func (a *StateAPI) StateMinerInitialPledgeCollateral(ctx context.Context, maddr } else { // NB: not exactly accurate, but should always lead us to *over* estimate, not under duration := pci.Expiration - ts.Height() - - // TODO: ActorUpgrade - sectorWeight = miner0.QAPowerForWeight(ssize, duration, w, vw) + sectorWeight = lotusbuiltin.QAPowerForWeight(ssize, duration, w, vw) } var ( - powerSmoothed builtin2.FilterEstimate + powerSmoothed lotusbuiltin.FilterEstimate pledgeCollateral abi.TokenAmount ) if act, err := state.GetActor(power.Address); err != nil { From 1dc69e397e66a0c622d5e2b0bc865c5e25241206 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Tue, 22 Sep 2020 00:35:15 -0400 Subject: [PATCH 276/885] Resolve some unnecessary actor upgrade TODOs --- chain/actors/builtin/miner/miner.go | 6 ++++ cmd/lotus-pcr/main.go | 19 +++--------- extern/storage-sealing/checks.go | 13 +++----- extern/storage-sealing/precommit_policy.go | 21 +++---------- extern/storage-sealing/sealing.go | 17 ---------- extern/storage-sealing/states_sealing.go | 36 ++++++---------------- 6 files changed, 28 insertions(+), 84 deletions(-) diff --git a/chain/actors/builtin/miner/miner.go b/chain/actors/builtin/miner/miner.go index c2b2e3a0d..dbb4ddd73 100644 --- a/chain/actors/builtin/miner/miner.go +++ b/chain/actors/builtin/miner/miner.go @@ -18,6 +18,12 @@ import ( "github.com/filecoin-project/lotus/chain/types" ) +// Unchanged between v0 and v1 actors +var PreCommitChallengeDelay = miner0.PreCommitChallengeDelay +var WPoStProvingPeriod = miner0.WPoStProvingPeriod + +const MinSectorExpiration = miner0.MinSectorExpiration + func Load(store adt.Store, act *types.Actor) (st State, err error) { switch act.Code { case builtin0.StorageMinerActorCodeID: diff --git a/cmd/lotus-pcr/main.go b/cmd/lotus-pcr/main.go index c8179d9b6..fed42427c 100644 --- a/cmd/lotus-pcr/main.go +++ b/cmd/lotus-pcr/main.go @@ -396,24 +396,13 @@ func (r *refunder) ProcessTipset(ctx context.Context, tipset *types.TipSet, refu var sn abi.SectorNumber - nv, err := r.api.StateNetworkVersion(ctx, tipset.Key()) - if err != nil { - log.Warnw("failed to get network version") + var proveCommitSector miner0.ProveCommitSectorParams + if err := proveCommitSector.UnmarshalCBOR(bytes.NewBuffer(m.Params)); err != nil { + log.Warnw("failed to decode provecommit params", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To) continue } - if nv < build.ActorUpgradeNetworkVersion { - var proveCommitSector miner0.ProveCommitSectorParams - if err := proveCommitSector.UnmarshalCBOR(bytes.NewBuffer(m.Params)); err != nil { - log.Warnw("failed to decode provecommit params", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To) - continue - } - - sn = proveCommitSector.SectorNumber - } else { - // TODO: ActorUpgrade - sn = 0 - } + sn = proveCommitSector.SectorNumber // We use the parent tipset key because precommit information is removed when ProveCommitSector is executed precommitChainInfo, err := r.api.StateSectorPreCommitInfo(ctx, m.To, sn, tipset.Parents()) diff --git a/extern/storage-sealing/checks.go b/extern/storage-sealing/checks.go index 27b62f49a..3d9aedeb4 100644 --- a/extern/storage-sealing/checks.go +++ b/extern/storage-sealing/checks.go @@ -4,6 +4,8 @@ import ( "bytes" "context" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + "github.com/filecoin-project/lotus/build" miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" @@ -104,7 +106,7 @@ func checkPrecommit(ctx context.Context, maddr address.Address, si SectorInfo, t if nv < build.ActorUpgradeNetworkVersion { msd = miner0.MaxSealDuration[si.SectorType] } else { - // TODO: ActorUpgrade + // TODO: ActorUpgrade(use MaxProveCommitDuration) msd = 0 } @@ -154,13 +156,8 @@ func (m *Sealing) checkCommit(ctx context.Context, si SectorInfo, proof []byte, return &ErrNoPrecommit{xerrors.Errorf("precommit info not found on-chain")} } - pccd, err := m.getPreCommitChallengeDelay(ctx, tok) - if err != nil { - return xerrors.Errorf("failed to get precommit challenge delay: %w", err) - } - - if pci.PreCommitEpoch+pccd != si.SeedEpoch { - return &ErrBadSeed{xerrors.Errorf("seed epoch doesn't match on chain info: %d != %d", pci.PreCommitEpoch+pccd, si.SeedEpoch)} + if pci.PreCommitEpoch+miner.PreCommitChallengeDelay != si.SeedEpoch { + return &ErrBadSeed{xerrors.Errorf("seed epoch doesn't match on chain info: %d != %d", pci.PreCommitEpoch+miner.PreCommitChallengeDelay, si.SeedEpoch)} } buf := new(bytes.Buffer) diff --git a/extern/storage-sealing/precommit_policy.go b/extern/storage-sealing/precommit_policy.go index 2ee6c1afc..0b774b56f 100644 --- a/extern/storage-sealing/precommit_policy.go +++ b/extern/storage-sealing/precommit_policy.go @@ -3,11 +3,11 @@ package sealing import ( "context" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + "github.com/filecoin-project/go-state-types/network" - "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/go-state-types/abi" - miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" ) type PreCommitPolicy interface { @@ -52,7 +52,7 @@ func NewBasicPreCommitPolicy(api Chain, duration abi.ChainEpoch, provingBoundary // Expiration produces the pre-commit sector expiration epoch for an encoded // replica containing the provided enumeration of pieces and deals. func (p *BasicPreCommitPolicy) Expiration(ctx context.Context, ps ...Piece) (abi.ChainEpoch, error) { - tok, epoch, err := p.api.ChainHead(ctx) + _, epoch, err := p.api.ChainHead(ctx) if err != nil { return 0, err } @@ -80,20 +80,7 @@ func (p *BasicPreCommitPolicy) Expiration(ctx context.Context, ps ...Piece) (abi end = &tmp } - nv, err := p.api.StateNetworkVersion(ctx, tok) - if err != nil { - return 0, err - } - - var wpp abi.ChainEpoch - if nv < build.ActorUpgradeNetworkVersion { - wpp = miner0.WPoStProvingPeriod - } else { - // TODO: ActorUpgrade - wpp = 0 - } - - *end += wpp - (*end % wpp) + p.provingBoundary - 1 + *end += miner.WPoStProvingPeriod - (*end % miner.WPoStProvingPeriod) + p.provingBoundary - 1 return *end, nil } diff --git a/extern/storage-sealing/sealing.go b/extern/storage-sealing/sealing.go index d5772e1d6..1ba53661a 100644 --- a/extern/storage-sealing/sealing.go +++ b/extern/storage-sealing/sealing.go @@ -8,9 +8,6 @@ import ( "sync" "time" - "github.com/filecoin-project/lotus/build" - miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" - "github.com/filecoin-project/go-state-types/network" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" @@ -422,17 +419,3 @@ func getDealPerSectorLimit(size abi.SectorSize) uint64 { } return 512 } - -func (m *Sealing) getPreCommitChallengeDelay(ctx context.Context, tok TipSetToken) (abi.ChainEpoch, error) { - nv, err := m.api.StateNetworkVersion(ctx, tok) - if err != nil { - return -1, xerrors.Errorf("failed to get network version: %w", err) - } - - if nv < build.ActorUpgradeNetworkVersion { - return miner0.PreCommitChallengeDelay, nil - } - - // TODO: ActorUpgrade - return -1, nil -} diff --git a/extern/storage-sealing/states_sealing.go b/extern/storage-sealing/states_sealing.go index 0109326fb..443365160 100644 --- a/extern/storage-sealing/states_sealing.go +++ b/extern/storage-sealing/states_sealing.go @@ -189,17 +189,14 @@ func (m *Sealing) handlePreCommitting(ctx statemachine.Context, sector SectorInf } var msd abi.ChainEpoch - var mse abi.ChainEpoch if nv < build.ActorUpgradeNetworkVersion { msd = miner0.MaxSealDuration[sector.SectorType] - mse = miner0.MinSectorExpiration } else { - // TODO: ActorUpgrade + // TODO: ActorUpgrade(use MaxProveCommitDuration) msd = 0 - mse = 0 } - if minExpiration := height + msd + mse + 10; expiration < minExpiration { + if minExpiration := height + msd + miner.MinSectorExpiration + 10; expiration < minExpiration { expiration = minExpiration } // TODO: enforce a reasonable _maximum_ sector lifetime? @@ -284,12 +281,7 @@ func (m *Sealing) handleWaitSeed(ctx statemachine.Context, sector SectorInfo) er return ctx.Send(SectorChainPreCommitFailed{error: xerrors.Errorf("precommit info not found on chain")}) } - pccd, err := m.getPreCommitChallengeDelay(ctx.Context(), tok) - if err != nil { - return ctx.Send(SectorChainPreCommitFailed{xerrors.Errorf("failed to get precommit challenge delay: %w", err)}) - } - - randHeight := pci.PreCommitEpoch + pccd + randHeight := pci.PreCommitEpoch + miner.PreCommitChallengeDelay err = m.events.ChainAt(func(ectx context.Context, _ TipSetToken, curH abi.ChainEpoch) error { // in case of null blocks the randomness can land after the tipset we @@ -380,24 +372,14 @@ func (m *Sealing) handleSubmitCommit(ctx statemachine.Context, sector SectorInfo return ctx.Send(SectorCommitFailed{xerrors.Errorf("commit check error: %w", err)}) } - nv, err := m.api.StateNetworkVersion(ctx.Context(), tok) - if err != nil { - return ctx.Send(SectorCommitFailed{xerrors.Errorf("failed to get network version: %w", err)}) + enc := new(bytes.Buffer) + params := &miner.ProveCommitSectorParams{ + SectorNumber: sector.SectorNumber, + Proof: sector.Proof, } - enc := new(bytes.Buffer) - if nv < build.ActorUpgradeNetworkVersion { - params := &miner0.ProveCommitSectorParams{ - SectorNumber: sector.SectorNumber, - Proof: sector.Proof, - } - - if err := params.MarshalCBOR(enc); err != nil { - return ctx.Send(SectorCommitFailed{xerrors.Errorf("could not serialize commit sector parameters: %w", err)}) - } - } else { - // TODO: ActorUpgrade - enc = nil + if err := params.MarshalCBOR(enc); err != nil { + return ctx.Send(SectorCommitFailed{xerrors.Errorf("could not serialize commit sector parameters: %w", err)}) } waddr, err := m.api.StateMinerWorkerAddress(ctx.Context(), m.maddr, tok) From 001ba17d37c4217e7e7a4737123132062f517772 Mon Sep 17 00:00:00 2001 From: zgfzgf <1901989065@qq.com> Date: Tue, 22 Sep 2020 15:21:35 +0800 Subject: [PATCH 277/885] break error --- miner/miner.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/miner/miner.go b/miner/miner.go index 2d8591992..144b2f413 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -151,6 +151,7 @@ func (m *Miner) mine(ctx context.Context) { var lastBase MiningBase for { + minerStop: select { case <-m.stop: stopping := m.stopping @@ -171,7 +172,7 @@ func (m *Miner) mine(ctx context.Context) { if err != nil { log.Errorf("failed to get best mining candidate: %s", err) if !m.niceSleep(time.Second * 5) { - break + goto minerStop } continue } @@ -203,7 +204,7 @@ func (m *Miner) mine(ctx context.Context) { if err != nil { log.Errorf("failed getting beacon entry: %s", err) if !m.niceSleep(time.Second) { - break + goto minerStop } continue } @@ -214,7 +215,7 @@ func (m *Miner) mine(ctx context.Context) { if base.TipSet.Equals(lastBase.TipSet) && lastBase.NullRounds == base.NullRounds { log.Warnf("BestMiningCandidate from the previous round: %s (nulls:%d)", lastBase.TipSet.Cids(), lastBase.NullRounds) if !m.niceSleep(time.Duration(build.BlockDelaySecs) * time.Second) { - break + goto minerStop } continue } @@ -225,7 +226,7 @@ func (m *Miner) mine(ctx context.Context) { if err != nil { log.Errorf("mining block failed: %+v", err) if !m.niceSleep(time.Second) { - break + goto minerStop } onDone(false, 0, err) continue From 9b91628d858a35f49de2f0f9ecc691a1cba3fbaf Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 22 Sep 2020 11:05:12 -0500 Subject: [PATCH 278/885] begin adding simple api server for deal stats --- cmd/lotus-shed/dealtracker.go | 78 +++++++++++++++++++++++++++++++++++ cmd/lotus-shed/main.go | 1 + 2 files changed, 79 insertions(+) create mode 100644 cmd/lotus-shed/dealtracker.go diff --git a/cmd/lotus-shed/dealtracker.go b/cmd/lotus-shed/dealtracker.go new file mode 100644 index 000000000..9f97337ee --- /dev/null +++ b/cmd/lotus-shed/dealtracker.go @@ -0,0 +1,78 @@ +package main + +import ( + "context" + "encoding/json" + "net/http" + + "github.com/filecoin-project/lotus/api" + lcli "github.com/filecoin-project/lotus/cli" + "github.com/urfave/cli/v2" +) + +type dealStatsServer struct { + api api.FullNode +} + +type dealCountResp struct { + Total int64 `json:"total"` + Epoch int64 `json:"epoch"` +} + +func (dss *dealStatsServer) handleStorageDealCount(w http.ResponseWriter, r *http.Request) { + ctx := context.Background() + + head, err := dss.api.ChainHead(ctx) + if err != nil { + log.Warnf("failed to get chain head: %s", err) + w.WriteHeader(500) + return + } + + deals, err := dss.api.StateMarketDeals(ctx, head.Key()) + if err != nil { + log.Warnf("failed to get market deals: %s", err) + w.WriteHeader(500) + return + } + + if err := json.NewEncoder(w).Encode(&dealCountResp{ + Total: int64(len(deals)), + Epoch: int64(head.Height()), + }); err != nil { + log.Warnf("failed to write back deal count response: %s", err) + return + } +} + +func (dss *dealStatsServer) handleStorageDealAverageSize(w http.ResponseWriter, r *http.Request) { + +} + +func (dss *dealStatsServer) handleStorageDealTotalReal(w http.ResponseWriter, r *http.Request) { + +} + +var serveDealStatsCmd = &cli.Command{ + Name: "serve-deal-stats", + Flags: []cli.Flag{}, + Action: func(cctx *cli.Context) error { + api, closer, err := lcli.GetFullNodeAPI(cctx) + if err != nil { + return err + } + + defer closer() + ctx := lcli.ReqContext(cctx) + + _ = ctx + + dss := &dealStatsServer{api} + + http.HandleFunc("/api/storagedeal/count", dss.handleStorageDealCount) + http.HandleFunc("/api/storagedeal/averagesize", dss.handleStorageDealAverageSize) + http.HandleFunc("/api/storagedeal/totalreal", dss.handleStorageDealTotalReal) + + panic(http.ListenAndServe(":7272", nil)) + }, +} diff --git a/cmd/lotus-shed/main.go b/cmd/lotus-shed/main.go index 1a56756d1..118b4ea72 100644 --- a/cmd/lotus-shed/main.go +++ b/cmd/lotus-shed/main.go @@ -36,6 +36,7 @@ func main() { mpoolStatsCmd, exportChainCmd, consensusCmd, + serveDealStatsCmd, } app := &cli.App{ From fc15888697effe357f177d00c35f07fc7666330f Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 22 Sep 2020 10:52:23 -0700 Subject: [PATCH 279/885] fixup some imports --- markets/storageadapter/client.go | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/markets/storageadapter/client.go b/markets/storageadapter/client.go index 781715903..fabc5b197 100644 --- a/markets/storageadapter/client.go +++ b/markets/storageadapter/client.go @@ -6,9 +6,9 @@ import ( "bytes" "context" - "github.com/filecoin-project/lotus/chain/actors/builtin/miner" - "github.com/filecoin-project/go-state-types/big" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin" + market0 "github.com/filecoin-project/specs-actors/actors/builtin/market" "golang.org/x/xerrors" @@ -20,6 +20,7 @@ import ( "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/go-state-types/exitcode" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/events" "github.com/filecoin-project/lotus/chain/events/state" "github.com/filecoin-project/lotus/chain/market" @@ -29,8 +30,6 @@ import ( "github.com/filecoin-project/lotus/lib/sigs" "github.com/filecoin-project/lotus/markets/utils" "github.com/filecoin-project/lotus/node/impl/full" - "github.com/filecoin-project/specs-actors/actors/builtin" - samarket "github.com/filecoin-project/specs-actors/actors/builtin/market" "github.com/ipfs/go-cid" ) @@ -102,10 +101,10 @@ func (c *ClientNodeAdapter) VerifySignature(ctx context.Context, sig crypto.Sign func (c *ClientNodeAdapter) AddFunds(ctx context.Context, addr address.Address, amount abi.TokenAmount) (cid.Cid, error) { // (Provider Node API) smsg, err := c.MpoolPushMessage(ctx, &types.Message{ - To: builtin.StorageMarketActorAddr, + To: miner0.StorageMarketActorAddr, From: addr, Value: amount, - Method: builtin.MethodsMarket.AddBalance, + Method: miner0.MethodsMarket.AddBalance, }, nil) if err != nil { return cid.Undef, err @@ -156,15 +155,15 @@ func (c *ClientNodeAdapter) ValidatePublishedDeal(ctx context.Context, deal stor return 0, xerrors.Errorf("deal wasn't published by storage provider: from=%s, provider=%s", pubmsg.From, deal.Proposal.Provider) } - if pubmsg.To != builtin.StorageMarketActorAddr { + if pubmsg.To != miner0.StorageMarketActorAddr { return 0, xerrors.Errorf("deal publish message wasn't set to StorageMarket actor (to=%s)", pubmsg.To) } - if pubmsg.Method != builtin.MethodsMarket.PublishStorageDeals { + if pubmsg.Method != miner0.MethodsMarket.PublishStorageDeals { return 0, xerrors.Errorf("deal publish message called incorrect method (method=%s)", pubmsg.Method) } - var params samarket.PublishStorageDealsParams + var params market0.PublishStorageDealsParams if err := params.UnmarshalCBOR(bytes.NewReader(pubmsg.Params)); err != nil { return 0, err } @@ -196,7 +195,7 @@ func (c *ClientNodeAdapter) ValidatePublishedDeal(ctx context.Context, deal stor return 0, xerrors.Errorf("deal publish failed: exit=%d", ret.ExitCode) } - var res samarket.PublishStorageDealsReturn + var res market0.PublishStorageDealsReturn if err := res.UnmarshalCBOR(bytes.NewReader(ret.Return)); err != nil { return 0, err } @@ -274,7 +273,7 @@ func (c *ClientNodeAdapter) OnDealSectorCommitted(ctx context.Context, provider } switch msg.Method { - case builtin.MethodsMiner.PreCommitSector: + case miner0.MethodsMiner.PreCommitSector: var params miner.SectorPreCommitInfo if err := params.UnmarshalCBOR(bytes.NewReader(msg.Params)); err != nil { return true, false, xerrors.Errorf("unmarshal pre commit: %w", err) @@ -289,7 +288,7 @@ func (c *ClientNodeAdapter) OnDealSectorCommitted(ctx context.Context, provider } return true, false, nil - case builtin.MethodsMiner.ProveCommitSector: + case miner0.MethodsMiner.ProveCommitSector: var params miner.ProveCommitSectorParams if err := params.UnmarshalCBOR(bytes.NewReader(msg.Params)); err != nil { return true, false, xerrors.Errorf("failed to unmarshal prove commit sector params: %w", err) @@ -411,7 +410,7 @@ func (c *ClientNodeAdapter) OnDealExpiredOrSlashed(ctx context.Context, dealID a return nil } -func (c *ClientNodeAdapter) SignProposal(ctx context.Context, signer address.Address, proposal samarket.DealProposal) (*samarket.ClientDealProposal, error) { +func (c *ClientNodeAdapter) SignProposal(ctx context.Context, signer address.Address, proposal market0.DealProposal) (*market0.ClientDealProposal, error) { // TODO: output spec signed proposal buf, err := cborutil.Dump(&proposal) if err != nil { @@ -428,7 +427,7 @@ func (c *ClientNodeAdapter) SignProposal(ctx context.Context, signer address.Add return nil, err } - return &samarket.ClientDealProposal{ + return &market0.ClientDealProposal{ Proposal: proposal, ClientSignature: *sig, }, nil From f96698b54df95fcb44bbcf9cd62e9f710832196f Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 22 Sep 2020 12:54:39 -0500 Subject: [PATCH 280/885] finish up other endpoints --- cmd/lotus-shed/dealtracker.go | 136 +++++++++++++++++++++++++++++++++- 1 file changed, 134 insertions(+), 2 deletions(-) diff --git a/cmd/lotus-shed/dealtracker.go b/cmd/lotus-shed/dealtracker.go index 9f97337ee..1d3dad058 100644 --- a/cmd/lotus-shed/dealtracker.go +++ b/cmd/lotus-shed/dealtracker.go @@ -5,8 +5,10 @@ import ( "encoding/json" "net/http" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/api" lcli "github.com/filecoin-project/lotus/cli" + "github.com/ipfs/go-cid" "github.com/urfave/cli/v2" ) @@ -14,11 +16,34 @@ type dealStatsServer struct { api api.FullNode } +var filteredClients map[address.Address]bool + +func init() { + filteredClients = make(map[address.Address]bool) + for _, a := range []string{"t0112", "t0113", "t0114", "t010089"} { + addr, err := address.NewFromString(a) + if err != nil { + panic(err) + } + filteredClients[addr] = true + } +} + type dealCountResp struct { Total int64 `json:"total"` Epoch int64 `json:"epoch"` } +func filterDeals(deals map[string]api.MarketDeal) []*api.MarketDeal { + out := make([]*api.MarketDeal, 0, len(deals)) + for _, d := range deals { + if !filteredClients[d.Proposal.Client] { + out = append(out, &d) + } + } + return out +} + func (dss *dealStatsServer) handleStorageDealCount(w http.ResponseWriter, r *http.Request) { ctx := context.Background() @@ -36,8 +61,15 @@ func (dss *dealStatsServer) handleStorageDealCount(w http.ResponseWriter, r *htt return } + var count int64 + for _, d := range deals { + if !filteredClients[d.Proposal.Client] { + count++ + } + } + if err := json.NewEncoder(w).Encode(&dealCountResp{ - Total: int64(len(deals)), + Total: count, Epoch: int64(head.Height()), }); err != nil { log.Warnf("failed to write back deal count response: %s", err) @@ -45,14 +77,113 @@ func (dss *dealStatsServer) handleStorageDealCount(w http.ResponseWriter, r *htt } } -func (dss *dealStatsServer) handleStorageDealAverageSize(w http.ResponseWriter, r *http.Request) { +type dealAverageResp struct { + AverageSize int64 `json:"averageSize"` + Epoch int64 `json:"epoch"` +} +func (dss *dealStatsServer) handleStorageDealAverageSize(w http.ResponseWriter, r *http.Request) { + ctx := context.Background() + + head, err := dss.api.ChainHead(ctx) + if err != nil { + log.Warnf("failed to get chain head: %s", err) + w.WriteHeader(500) + return + } + + deals, err := dss.api.StateMarketDeals(ctx, head.Key()) + if err != nil { + log.Warnf("failed to get market deals: %s", err) + w.WriteHeader(500) + return + } + + var count int64 + var totalBytes int64 + for _, d := range deals { + if !filteredClients[d.Proposal.Client] { + count++ + totalBytes += int64(d.Proposal.PieceSize.Unpadded()) + } + } + + if err := json.NewEncoder(w).Encode(&dealAverageResp{ + AverageSize: totalBytes / count, + Epoch: int64(head.Height()), + }); err != nil { + log.Warnf("failed to write back deal average response: %s", err) + return + } } func (dss *dealStatsServer) handleStorageDealTotalReal(w http.ResponseWriter, r *http.Request) { } +type clientStatsOutput struct { + Client address.Address `json:"client"` + DataSize int64 `json:"data_size"` + NumCids int `json:"num_cids"` + NumDeals int `json:"num_deals"` + NumMiners int `json:"num_miners"` + + cids map[cid.Cid]bool + providers map[address.Address]bool +} + +func (dss *dealStatsServer) handleStorageClientStats(w http.ResponseWriter, r *http.Request) { + ctx := context.Background() + + head, err := dss.api.ChainHead(ctx) + if err != nil { + log.Warnf("failed to get chain head: %s", err) + w.WriteHeader(500) + return + } + + deals, err := dss.api.StateMarketDeals(ctx, head.Key()) + if err != nil { + log.Warnf("failed to get market deals: %s", err) + w.WriteHeader(500) + return + } + + stats := make(map[address.Address]*clientStatsOutput) + + for _, d := range deals { + if filteredClients[d.Proposal.Client] { + continue + } + + st, ok := stats[d.Proposal.Client] + if !ok { + st = &clientStatsOutput{ + Client: d.Proposal.Client, + cids: make(map[cid.Cid]bool), + providers: make(map[address.Address]bool), + } + stats[d.Proposal.Client] = st + } + + st.DataSize += int64(d.Proposal.PieceSize.Unpadded()) + st.cids[d.Proposal.PieceCID] = true + st.providers[d.Proposal.Provider] = true + st.NumDeals++ + } + + out := make([]*clientStatsOutput, 0, len(stats)) + for _, cso := range stats { + cso.NumCids = len(cso.cids) + cso.NumMiners = len(cso.providers) + } + + if err := json.NewEncoder(w).Encode(out); err != nil { + log.Warnf("failed to write back client stats response: %s", err) + return + } +} + var serveDealStatsCmd = &cli.Command{ Name: "serve-deal-stats", Flags: []cli.Flag{}, @@ -72,6 +203,7 @@ var serveDealStatsCmd = &cli.Command{ http.HandleFunc("/api/storagedeal/count", dss.handleStorageDealCount) http.HandleFunc("/api/storagedeal/averagesize", dss.handleStorageDealAverageSize) http.HandleFunc("/api/storagedeal/totalreal", dss.handleStorageDealTotalReal) + http.HandleFunc("/api/storagedeal/clientstats", dss.handleStorageClientStats) panic(http.ListenAndServe(":7272", nil)) }, From 5314ba8c6d1008ae2ea10aa4017c9a88a4ab4144 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 22 Sep 2020 10:55:29 -0700 Subject: [PATCH 281/885] remove ptr indirection --- chain/events/state/predicates.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/chain/events/state/predicates.go b/chain/events/state/predicates.go index 56dfb981d..909533d15 100644 --- a/chain/events/state/predicates.go +++ b/chain/events/state/predicates.go @@ -303,7 +303,7 @@ func (sp *StatePredicates) OnInitActorChange(diffInitActorState DiffInitActorSta if err != nil { return false, nil, err } - return diffInitActorState(ctx, &oldState, &newState) + return diffInitActorState(ctx, oldState, newState) }) } @@ -405,7 +405,7 @@ type AddressChange struct { To AddressPair } -type DiffInitActorStateFunc func(ctx context.Context, oldState *init_.State, newState *init_.State) (changed bool, user UserData, err error) +type DiffInitActorStateFunc func(ctx context.Context, oldState init_.State, newState init_.State) (changed bool, user UserData, err error) func (i *InitActorAddressChanges) AsKey(key string) (abi.Keyer, error) { addr, err := address.NewFromBytes([]byte(key)) @@ -493,7 +493,7 @@ func (i *InitActorAddressChanges) Remove(key string, val *typegen.Deferred) erro } func (sp *StatePredicates) OnAddressMapChange() DiffInitActorStateFunc { - return func(ctx context.Context, oldState, newState *init_.State) (changed bool, user UserData, err error) { + return func(ctx context.Context, oldState, newState init_.State) (changed bool, user UserData, err error) { /*ctxStore := &contextStore{ ctx: ctx, cst: sp.cst, From ab070f2ebee287e6cddf9b43bc2abac77e9eeb9b Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 22 Sep 2020 11:09:41 -0700 Subject: [PATCH 282/885] copy actor object when iterating over actors This is a pretty big footgun. --- chain/state/statetree.go | 1 + 1 file changed, 1 insertion(+) diff --git a/chain/state/statetree.go b/chain/state/statetree.go index fb6a14a9b..a654d224b 100644 --- a/chain/state/statetree.go +++ b/chain/state/statetree.go @@ -385,6 +385,7 @@ func (st *StateTree) MutateActor(addr address.Address, f func(*types.Actor) erro func (st *StateTree) ForEach(f func(address.Address, *types.Actor) error) error { var act types.Actor return st.root.ForEach(&act, func(k string) error { + act := act // copy addr, err := address.NewFromBytes([]byte(k)) if err != nil { return xerrors.Errorf("invalid address (%x) found in state tree key: %w", []byte(k), err) From bc24fdbd14bde171f59f61d6f0061cbc1955b9d0 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 22 Sep 2020 11:09:56 -0700 Subject: [PATCH 283/885] finish migrating statemanager --- chain/actors/adt/adt.go | 8 ++++++ chain/stmgr/stmgr.go | 64 +++++++++++++++++------------------------ 2 files changed, 35 insertions(+), 37 deletions(-) diff --git a/chain/actors/adt/adt.go b/chain/actors/adt/adt.go index fd5ee3f87..39dd5cebc 100644 --- a/chain/actors/adt/adt.go +++ b/chain/actors/adt/adt.go @@ -55,3 +55,11 @@ func AsArray(store Store, root cid.Cid, version network.Version) (Array, error) } return nil, xerrors.Errorf("unknown network version: %d", version) } + +func NewArray(store Store, version builtin.Version) (Array, error) { + switch version { + case builtin.Version0: + return adt0.MakeEmptyArray(store), nil + } + return nil, xerrors.Errorf("unknown network version: %d", version) +} diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index 2f8d460aa..af729143c 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -5,12 +5,9 @@ import ( "fmt" "sync" + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" msig0 "github.com/filecoin-project/specs-actors/actors/builtin/multisig" - "github.com/filecoin-project/lotus/chain/actors/builtin/multisig" - - "github.com/filecoin-project/lotus/chain/actors/builtin/reward" - "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" logging "github.com/ipfs/go-log/v2" @@ -22,15 +19,17 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/network" - builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/builtin/market" + "github.com/filecoin-project/lotus/chain/actors/builtin/multisig" "github.com/filecoin-project/lotus/chain/actors/builtin/paych" "github.com/filecoin-project/lotus/chain/actors/builtin/power" + "github.com/filecoin-project/lotus/chain/actors/builtin/reward" "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" @@ -291,7 +290,11 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp return cid.Cid{}, cid.Cid{}, err } - rectarr := adt.MakeEmptyArray(sm.cs.Store(ctx)) + // XXX: Is the height correct? Or should it be epoch-1? + rectarr, err := adt.NewArray(sm.cs.Store(ctx), builtin.VersionForNetwork(sm.GetNtwkVersion(ctx, epoch))) + if err != nil { + return cid.Undef, cid.Undef, xerrors.Errorf("failed to create receipts amt: %w", err) + } for i, receipt := range receipts { if err := rectarr.Set(uint64(i), receipt); err != nil { return cid.Undef, cid.Undef, xerrors.Errorf("failed to build receipts amt: %w", err) @@ -689,17 +692,13 @@ func (sm *StateManager) ListAllActors(ctx context.Context, ts *types.TipSet) ([] return nil, err } - r, err := adt.AsMap(sm.cs.Store(ctx), st) + stateTree, err := sm.StateTree(st) if err != nil { return nil, err } var out []address.Address - err = r.ForEach(nil, func(k string) error { - addr, err := address.NewFromBytes([]byte(k)) - if err != nil { - return xerrors.Errorf("address in state tree was not valid: %w", err) - } + err = stateTree.ForEach(func(addr address.Address, act *types.Actor) error { out = append(out, addr) return nil }) @@ -836,17 +835,10 @@ func (sm *StateManager) setupGenesisActors(ctx context.Context) error { return xerrors.Errorf("setting up genesis pledge: %w", err) } - r, err := adt.AsMap(sm.cs.Store(ctx), st) - if err != nil { - return xerrors.Errorf("getting genesis actors: %w", err) - } - totalsByEpoch := make(map[abi.ChainEpoch]abi.TokenAmount) - var act types.Actor - err = r.ForEach(&act, func(k string) error { - if act.Code == builtin0.MultisigActorCodeID { - var s multisig.State - err := sm.cs.Store(ctx).Get(ctx, act.Head, &s) + err = sTree.ForEach(func(kaddr address.Address, act *types.Actor) error { + if act.IsMultisigActor() { + s, err := multisig.Load(sm.cs.Store(ctx), act) if err != nil { return err } @@ -862,25 +854,22 @@ func (sm *StateManager) setupGenesisActors(ctx context.Context) error { totalsByEpoch[s.UnlockDuration()] = s.InitialBalance() } - } else if act.Code == builtin0.AccountActorCodeID { + } else if act.IsAccountActor() { // should exclude burnt funds actor and "remainder account actor" // should only ever be "faucet" accounts in testnets - kaddr, err := address.NewFromBytes([]byte(k)) + if kaddr == builtin0.BurntFundsActorAddr { + return nil + } + + kid, err := sTree.LookupID(kaddr) if err != nil { - return xerrors.Errorf("decoding address: %w", err) + return xerrors.Errorf("resolving address: %w", err) } - if kaddr != builtin0.BurntFundsActorAddr { - kid, err := sTree.LookupID(kaddr) - if err != nil { - return xerrors.Errorf("resolving address: %w", err) - } - - gi.genesisActors = append(gi.genesisActors, genesisActor{ - addr: kid, - initBal: act.Balance, - }) - } + gi.genesisActors = append(gi.genesisActors, genesisActor{ + addr: kid, + initBal: act.Balance, + }) } return nil }) @@ -889,6 +878,7 @@ func (sm *StateManager) setupGenesisActors(ctx context.Context) error { return xerrors.Errorf("error setting up genesis infos: %w", err) } + // TODO: use network upgrade abstractions or always start at actors v0? gi.genesisMsigs = make([]msig0.State, 0, len(totalsByEpoch)) for k, v := range totalsByEpoch { ns := msig0.State{ From 441d7ff790fde269fa950d9106f591178b55648d Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 22 Sep 2020 11:14:55 -0700 Subject: [PATCH 284/885] cleanup imports some more --- chain/stmgr/forks.go | 12 +++++++----- chain/stmgr/utils.go | 22 ++++++++++------------ 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index bb496849b..32e598065 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -6,12 +6,14 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" + + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" + power0 "github.com/filecoin-project/specs-actors/actors/builtin/power" + adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/specs-actors/actors/builtin" - miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" - power0 "github.com/filecoin-project/specs-actors/actors/builtin/power" - "github.com/filecoin-project/specs-actors/actors/util/adt" cbor "github.com/ipfs/go-ipld-cbor" "golang.org/x/xerrors" ) @@ -221,7 +223,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, tree types return xerrors.Errorf("failed to get miner info: %w", err) } - sectorsArr, err := adt.AsArray(sm.ChainStore().Store(ctx), st.Sectors) + sectorsArr, err := adt0.AsArray(sm.ChainStore().Store(ctx), st.Sectors) if err != nil { return xerrors.Errorf("failed to load sectors array: %w", err) } @@ -245,7 +247,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, tree types return xerrors.Errorf("failed to load miner state: %w", err) } - lbsectors, err := adt.AsArray(sm.ChainStore().Store(ctx), lbst.Sectors) + lbsectors, err := adt0.AsArray(sm.ChainStore().Store(ctx), lbst.Sectors) if err != nil { return xerrors.Errorf("failed to load lb sectors array: %w", err) } diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index c9e8d32da..3493afca3 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -9,11 +9,6 @@ import ( "runtime" "strings" - "github.com/filecoin-project/specs-actors/actors/builtin/cron" - - saruntime "github.com/filecoin-project/specs-actors/actors/runtime" - "github.com/filecoin-project/specs-actors/actors/runtime/proof" - cid "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" @@ -22,10 +17,10 @@ import ( "github.com/filecoin-project/go-bitfield" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/crypto" - "github.com/filecoin-project/lotus/chain/actors/builtin/power" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" account0 "github.com/filecoin-project/specs-actors/actors/builtin/account" + cron0 "github.com/filecoin-project/specs-actors/actors/builtin/cron" init0 "github.com/filecoin-project/specs-actors/actors/builtin/init" market0 "github.com/filecoin-project/specs-actors/actors/builtin/market" miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" @@ -34,17 +29,20 @@ import ( power0 "github.com/filecoin-project/specs-actors/actors/builtin/power" reward0 "github.com/filecoin-project/specs-actors/actors/builtin/reward" verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" + proof0 "github.com/filecoin-project/specs-actors/actors/runtime/proof" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init" "github.com/filecoin-project/lotus/chain/actors/builtin/market" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + "github.com/filecoin-project/lotus/chain/actors/builtin/power" "github.com/filecoin-project/lotus/chain/beacon" "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" + "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/node/modules/dtypes" ) @@ -163,7 +161,7 @@ func GetMinerSectorSet(ctx context.Context, sm *StateManager, ts *types.TipSet, return mas.LoadSectors(snos) } -func GetSectorsForWinningPoSt(ctx context.Context, pv ffiwrapper.Verifier, sm *StateManager, st cid.Cid, maddr address.Address, rand abi.PoStRandomness) ([]proof.SectorInfo, error) { +func GetSectorsForWinningPoSt(ctx context.Context, pv ffiwrapper.Verifier, sm *StateManager, st cid.Cid, maddr address.Address, rand abi.PoStRandomness) ([]proof0.SectorInfo, error) { act, err := sm.LoadActorRaw(ctx, maddr, st) if err != nil { return nil, xerrors.Errorf("failed to load miner actor: %w", err) @@ -248,9 +246,9 @@ func GetSectorsForWinningPoSt(ctx context.Context, pv ffiwrapper.Verifier, sm *S return nil, xerrors.Errorf("loading proving sectors: %w", err) } - out := make([]proof.SectorInfo, len(sectors)) + out := make([]proof0.SectorInfo, len(sectors)) for i, sinfo := range sectors { - out[i] = proof.SectorInfo{ + out[i] = proof0.SectorInfo{ SealProof: spt, SectorNumber: sinfo.SectorNumber, SealedCID: sinfo.SealedCID, @@ -538,7 +536,7 @@ func init() { cidToMethods := map[cid.Cid][2]interface{}{ // builtin.SystemActorCodeID: {builtin.MethodsSystem, system.Actor{} }- apparently it doesn't have methods builtin0.InitActorCodeID: {builtin0.MethodsInit, init0.Actor{}}, - builtin0.CronActorCodeID: {builtin0.MethodsCron, cron.Actor{}}, + builtin0.CronActorCodeID: {builtin0.MethodsCron, cron0.Actor{}}, builtin0.AccountActorCodeID: {builtin0.MethodsAccount, account0.Actor{}}, builtin0.StoragePowerActorCodeID: {builtin0.MethodsPower, power0.Actor{}}, builtin0.StorageMinerActorCodeID: {builtin0.MethodsMiner, miner0.Actor{}}, @@ -550,7 +548,7 @@ func init() { } for c, m := range cidToMethods { - exports := m[1].(saruntime.Invokee).Exports() + exports := m[1].(vm.Invokee).Exports() methods := make(map[abi.MethodNum]MethodMeta, len(exports)) // Explicitly add send, it's special. From c66f087f4c4f67396fa24da377f41fac777c781c Mon Sep 17 00:00:00 2001 From: Travis Person Date: Tue, 22 Sep 2020 18:15:42 +0000 Subject: [PATCH 285/885] lotus-miner: add more help text to storage / attach --- cmd/lotus-storage-miner/storage.go | 19 +++++++++++++++++++ extern/sector-storage/stores/local.go | 9 +++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/cmd/lotus-storage-miner/storage.go b/cmd/lotus-storage-miner/storage.go index 8a3687877..0c366cb01 100644 --- a/cmd/lotus-storage-miner/storage.go +++ b/cmd/lotus-storage-miner/storage.go @@ -31,6 +31,9 @@ const metaFile = "sectorstore.json" var storageCmd = &cli.Command{ Name: "storage", Usage: "manage sector storage", + Description: `Sectors can be stored across many filesystem paths. These commands provide ways to + manage the storage the miner will used to store sectors long term for proving (refernces as 'store') + as well as how sectors will be stored while moving through the sealing pipeline (references as 'seal').`, Subcommands: []*cli.Command{ storageAttachCmd, storageListCmd, @@ -41,6 +44,22 @@ var storageCmd = &cli.Command{ var storageAttachCmd = &cli.Command{ Name: "attach", Usage: "attach local storage path", + Description: `Storage can be attach to the miner using this command. The storage volume list is stored local + to the miner in $LOTUS_MINER_PATH/storage.json. We do not recommend modifying this value without further + understanding of the storage system. + + Each storage volume contains a configuration file which descripbes the capabilities of the volume. When the + '--init' flag is provided, this file will be created using the additional flags. + + Weight + A high weight value means data will be more likely to be stored + + Seal + Intermittment data for the sealing process will be stored here + + Store + Finalized sectors that will be proved over will be stored here + `, Flags: []cli.Flag{ &cli.BoolFlag{ Name: "init", diff --git a/extern/sector-storage/stores/local.go b/extern/sector-storage/stores/local.go index 75387acc8..70b37f10e 100644 --- a/extern/sector-storage/stores/local.go +++ b/extern/sector-storage/stores/local.go @@ -30,10 +30,15 @@ type StoragePath struct { // LocalStorageMeta [path]/sectorstore.json type LocalStorageMeta struct { - ID ID + ID ID + + // A height wait means data is more likely to be stored here Weight uint64 // 0 = readonly - CanSeal bool + // Intermittment data for the sealing process will be stored here + CanSeal bool + + // Finalized sectors that will be proved over will be stored here CanStore bool } From c91774be3bb8d8fcd98cdb5de96f1654de622ffe Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 22 Sep 2020 11:19:28 -0700 Subject: [PATCH 286/885] remove old comment --- cmd/lotus-shed/balances.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/lotus-shed/balances.go b/cmd/lotus-shed/balances.go index de92aa8b6..c156de931 100644 --- a/cmd/lotus-shed/balances.go +++ b/cmd/lotus-shed/balances.go @@ -170,7 +170,6 @@ var chainBalanceStateCmd = &cli.Command{ sm := stmgr.NewStateManager(cs) - // Options: (a) encode the version in the chain or (b) pass a flag. tree, err := state.LoadStateTree(cst, sroot) if err != nil { return err From 3153ab9ae3672a5a7b1aec1384364fdc50169e13 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 22 Sep 2020 13:24:49 -0500 Subject: [PATCH 287/885] allow graceful shutdown --- cmd/lotus-shed/dealtracker.go | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/cmd/lotus-shed/dealtracker.go b/cmd/lotus-shed/dealtracker.go index 1d3dad058..cbe375803 100644 --- a/cmd/lotus-shed/dealtracker.go +++ b/cmd/lotus-shed/dealtracker.go @@ -3,6 +3,7 @@ package main import ( "context" "encoding/json" + "net" "net/http" "github.com/filecoin-project/go-address" @@ -200,11 +201,28 @@ var serveDealStatsCmd = &cli.Command{ dss := &dealStatsServer{api} - http.HandleFunc("/api/storagedeal/count", dss.handleStorageDealCount) - http.HandleFunc("/api/storagedeal/averagesize", dss.handleStorageDealAverageSize) - http.HandleFunc("/api/storagedeal/totalreal", dss.handleStorageDealTotalReal) - http.HandleFunc("/api/storagedeal/clientstats", dss.handleStorageClientStats) + mux := &http.ServeMux{} + mux.HandleFunc("/api/storagedeal/count", dss.handleStorageDealCount) + mux.HandleFunc("/api/storagedeal/averagesize", dss.handleStorageDealAverageSize) + mux.HandleFunc("/api/storagedeal/totalreal", dss.handleStorageDealTotalReal) + mux.HandleFunc("/api/storagedeal/clientstats", dss.handleStorageClientStats) - panic(http.ListenAndServe(":7272", nil)) + s := &http.Server{ + Addr: ":7272", + Handler: mux, + } + + go func() { + <-ctx.Done() + s.Shutdown(context.TODO()) + }() + + list, err := net.Listen("tcp", ":7272") + if err != nil { + panic(err) + } + + s.Serve(list) + return nil }, } From 3cf2fd595f7f74561a5856befa69afe6ee8d570f Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 22 Sep 2020 13:27:58 -0500 Subject: [PATCH 288/885] fix appending to array --- cmd/lotus-shed/dealtracker.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/lotus-shed/dealtracker.go b/cmd/lotus-shed/dealtracker.go index cbe375803..57f42bc83 100644 --- a/cmd/lotus-shed/dealtracker.go +++ b/cmd/lotus-shed/dealtracker.go @@ -177,6 +177,8 @@ func (dss *dealStatsServer) handleStorageClientStats(w http.ResponseWriter, r *h for _, cso := range stats { cso.NumCids = len(cso.cids) cso.NumMiners = len(cso.providers) + + out = append(out, cso) } if err := json.NewEncoder(w).Encode(out); err != nil { From 88ada66280807b2000f78a5fca5f186bbf233df5 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 22 Sep 2020 13:31:01 -0500 Subject: [PATCH 289/885] finish up the total bytes endpoint --- cmd/lotus-shed/dealtracker.go | 37 ++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/cmd/lotus-shed/dealtracker.go b/cmd/lotus-shed/dealtracker.go index 57f42bc83..18dc959f7 100644 --- a/cmd/lotus-shed/dealtracker.go +++ b/cmd/lotus-shed/dealtracker.go @@ -79,7 +79,7 @@ func (dss *dealStatsServer) handleStorageDealCount(w http.ResponseWriter, r *htt } type dealAverageResp struct { - AverageSize int64 `json:"averageSize"` + AverageSize int64 `json:"average_size"` Epoch int64 `json:"epoch"` } @@ -118,7 +118,42 @@ func (dss *dealStatsServer) handleStorageDealAverageSize(w http.ResponseWriter, } } +type dealTotalResp struct { + TotalBytes int64 `json:"total_size"` + Epoch int64 `json:"epoch"` +} + func (dss *dealStatsServer) handleStorageDealTotalReal(w http.ResponseWriter, r *http.Request) { + ctx := context.Background() + + head, err := dss.api.ChainHead(ctx) + if err != nil { + log.Warnf("failed to get chain head: %s", err) + w.WriteHeader(500) + return + } + + deals, err := dss.api.StateMarketDeals(ctx, head.Key()) + if err != nil { + log.Warnf("failed to get market deals: %s", err) + w.WriteHeader(500) + return + } + + var totalBytes int64 + for _, d := range deals { + if !filteredClients[d.Proposal.Client] { + totalBytes += int64(d.Proposal.PieceSize.Unpadded()) + } + } + + if err := json.NewEncoder(w).Encode(&dealTotalResp{ + TotalBytes: totalBytes, + Epoch: int64(head.Height()), + }); err != nil { + log.Warnf("failed to write back deal average response: %s", err) + return + } } From 2967c4ec109c2287b05b2732b9f04c1c101320b9 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 22 Sep 2020 12:02:29 -0700 Subject: [PATCH 290/885] use abstract actor type methods --- chain/stmgr/forks.go | 16 ++++++++-------- chain/vm/vm.go | 2 +- cli/multisig.go | 3 +-- cmd/lotus-pcr/main.go | 6 +++--- 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index 32e598065..f9418c4d8 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -7,13 +7,13 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" power0 "github.com/filecoin-project/specs-actors/actors/builtin/power" adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/specs-actors/actors/builtin" cbor "github.com/ipfs/go-ipld-cbor" "golang.org/x/xerrors" ) @@ -123,7 +123,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, tree types // Take all excess funds away, put them into the reserve account err = fetree.ForEach(func(addr address.Address, act *types.Actor) error { switch act.Code { - case builtin.AccountActorCodeID, builtin.MultisigActorCodeID, builtin.PaymentChannelActorCodeID: + case builtin0.AccountActorCodeID, builtin0.MultisigActorCodeID, builtin0.PaymentChannelActorCodeID: sysAcc, err := isSystemAccount(addr) if err != nil { return xerrors.Errorf("checking system account: %w", err) @@ -136,7 +136,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, tree types Amt: act.Balance, }) } - case builtin.StorageMinerActorCodeID: + case builtin0.StorageMinerActorCodeID: var st miner0.State if err := sm.ChainStore().Store(ctx).Get(ctx, act.Head, &st); err != nil { return xerrors.Errorf("failed to load miner state: %w", err) @@ -175,7 +175,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, tree types // pull up power table to give miners back some funds proportional to their power var ps power0.State - powAct, err := tree.GetActor(builtin.StoragePowerActorAddr) + powAct, err := tree.GetActor(builtin0.StoragePowerActorAddr) if err != nil { return xerrors.Errorf("failed to load power actor: %w", err) } @@ -203,7 +203,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, tree types } switch act.Code { - case builtin.AccountActorCodeID, builtin.MultisigActorCodeID, builtin.PaymentChannelActorCodeID: + case builtin0.AccountActorCodeID, builtin0.MultisigActorCodeID, builtin0.PaymentChannelActorCodeID: nbalance := big.Min(prevBalance, AccountCap) if nbalance.Sign() != 0 { transfersBack = append(transfersBack, transfer{ @@ -212,7 +212,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, tree types Amt: nbalance, }) } - case builtin.StorageMinerActorCodeID: + case builtin0.StorageMinerActorCodeID: var st miner0.State if err := sm.ChainStore().Store(ctx).Get(ctx, act.Head, &st); err != nil { return xerrors.Errorf("failed to load miner state: %w", err) @@ -277,11 +277,11 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, tree types } // transfer all burnt funds back to the reserve account - burntAct, err := tree.GetActor(builtin.BurntFundsActorAddr) + burntAct, err := tree.GetActor(builtin0.BurntFundsActorAddr) if err != nil { return xerrors.Errorf("failed to load burnt funds actor: %w", err) } - if err := doTransfer(tree, builtin.BurntFundsActorAddr, ReserveAddress, burntAct.Balance); err != nil { + if err := doTransfer(tree, builtin0.BurntFundsActorAddr, ReserveAddress, burntAct.Balance); err != nil { return xerrors.Errorf("failed to unburn funds: %w", err) } diff --git a/chain/vm/vm.go b/chain/vm/vm.go index 25c937ca3..049918fba 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -384,7 +384,7 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, } // this should never happen, but is currently still exercised by some tests - if !fromActor.Code.Equals(builtin.AccountActorCodeID) { + if !fromActor.IsAccountActor() { gasOutputs := ZeroGasOutputs() gasOutputs.MinerPenalty = minerPenaltyAmount return &ApplyRet{ diff --git a/cli/multisig.go b/cli/multisig.go index 2aabf5d7e..81c6502d5 100644 --- a/cli/multisig.go +++ b/cli/multisig.go @@ -16,7 +16,6 @@ import ( "github.com/urfave/cli/v2" "golang.org/x/xerrors" - "github.com/filecoin-project/specs-actors/actors/builtin" init0 "github.com/filecoin-project/specs-actors/actors/builtin/init" msig0 "github.com/filecoin-project/specs-actors/actors/builtin/multisig" @@ -330,7 +329,7 @@ var msigProposeCmd = &cli.Command{ return fmt.Errorf("failed to look up multisig %s: %w", msig, err) } - if act.Code != builtin.MultisigActorCodeID { + if !act.IsMultisigActor() { return fmt.Errorf("actor %s is not a multisig actor", msig) } diff --git a/cmd/lotus-pcr/main.go b/cmd/lotus-pcr/main.go index fed42427c..36961a663 100644 --- a/cmd/lotus-pcr/main.go +++ b/cmd/lotus-pcr/main.go @@ -12,10 +12,10 @@ import ( "strconv" "time" + "github.com/filecoin-project/specs-actors/actors/builtin" miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/go-state-types/network" - "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" @@ -28,11 +28,11 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/exitcode" - "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/tools/stats" ) @@ -375,7 +375,7 @@ func (r *refunder) ProcessTipset(ctx context.Context, tipset *types.TipSet, refu continue } - if a.Code != builtin.StorageMinerActorCodeID { + if !a.IsStorageMinerActor() { continue } From 46fb0e74cd3d23c5fd0c49c2022106839dbb6051 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 22 Sep 2020 14:09:33 -0700 Subject: [PATCH 291/885] add deal state iterator --- chain/actors/builtin/market/market.go | 1 + chain/actors/builtin/market/v0.go | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/chain/actors/builtin/market/market.go b/chain/actors/builtin/market/market.go index fa5f027b3..9cb0ffc13 100644 --- a/chain/actors/builtin/market/market.go +++ b/chain/actors/builtin/market/market.go @@ -50,6 +50,7 @@ type BalanceTable interface { } type DealStates interface { + ForEach(cb func(id abi.DealID, ds DealState) error) error Get(id abi.DealID) (*DealState, bool, error) array() adt.Array diff --git a/chain/actors/builtin/market/v0.go b/chain/actors/builtin/market/v0.go index 433445dab..14b8b7d93 100644 --- a/chain/actors/builtin/market/v0.go +++ b/chain/actors/builtin/market/v0.go @@ -126,6 +126,13 @@ func (s *dealStates0) Get(dealID abi.DealID) (*DealState, bool, error) { return &deal, true, nil } +func (s *dealStates0) ForEach(cb func(dealID abi.DealID, ds DealState) error) error { + var ds0 market.DealState + return s.Array.ForEach(&ds0, func(idx int64) error { + return cb(abi.DealID(idx), fromV0DealState(ds0)) + }) +} + func (s *dealStates0) decode(val *cbg.Deferred) (*DealState, error) { var ds0 market.DealState if err := ds0.UnmarshalCBOR(bytes.NewReader(val.Raw)); err != nil { From 77f81fc49be3d0fd67438795e41416bb5af18476 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Tue, 22 Sep 2020 19:07:32 -0400 Subject: [PATCH 292/885] Test fix --- node/node_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/node/node_test.go b/node/node_test.go index 0e404dc0b..54498eec3 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -5,6 +5,8 @@ import ( "testing" "time" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + builder "github.com/filecoin-project/lotus/node/test" "github.com/filecoin-project/go-state-types/abi" @@ -68,6 +70,8 @@ func TestAPIDealFlowReal(t *testing.T) { logging.SetLogLevel("sub", "ERROR") logging.SetLogLevel("storageminer", "ERROR") + // TODO: Do this better. + miner.PreCommitChallengeDelay = 5 miner0.PreCommitChallengeDelay = 5 t.Run("basic", func(t *testing.T) { From dcf2735836ba5993f9a876139ff8476fc9c18a8e Mon Sep 17 00:00:00 2001 From: Lucas Molas Date: Tue, 22 Sep 2020 20:19:31 -0300 Subject: [PATCH 293/885] move validation from protocol to API --- chain/exchange/client.go | 45 +++++++++++++++++++++++++++----------- chain/exchange/protocol.go | 15 +++++-------- 2 files changed, 38 insertions(+), 22 deletions(-) diff --git a/chain/exchange/client.go b/chain/exchange/client.go index a99362745..371c50aed 100644 --- a/chain/exchange/client.go +++ b/chain/exchange/client.go @@ -65,7 +65,15 @@ func NewClient(lc fx.Lifecycle, host host.Host, pmgr peermgr.MaybePeerMgr) Clien // request options without disrupting external calls. In the future the // consumers should be forced to use a more standardized service and // adhere to a single API derived from this function. -func (c *client) doRequest(ctx context.Context, req *Request, singlePeer *peer.ID) (*validatedResponse, error) { +func (c *client) doRequest( + ctx context.Context, + req *Request, + singlePeer *peer.ID, + // In the `GetChainMessages` case, we won't request the headers but we still + // need them to check the integrity of the `CompactedMessages` in the response + // so the tipset blocks need to be provided by the caller. + tipsets []*types.TipSet, +) (*validatedResponse, error) { // Validate request. if req.Length == 0 { return nil, xerrors.Errorf("invalid request of length 0") @@ -116,7 +124,7 @@ func (c *client) doRequest(ctx context.Context, req *Request, singlePeer *peer.I } // Process and validate response. - validRes, err := c.processResponse(req, res) + validRes, err := c.processResponse(req, res, tipsets) if err != nil { log.Warnf("processing peer %s response failed: %s", peer.String(), err) @@ -144,7 +152,7 @@ func (c *client) doRequest(ctx context.Context, req *Request, singlePeer *peer.I // errors. Peer penalization should happen here then, before returning, so // we can apply the correct penalties depending on the cause of the error. // FIXME: Add the `peer` as argument once we implement penalties. -func (c *client) processResponse(req *Request, res *Response) (*validatedResponse, error) { +func (c *client) processResponse(req *Request, res *Response, tipsets []*types.TipSet) (*validatedResponse, error) { err := res.statusToError() if err != nil { return nil, xerrors.Errorf("status error: %s", err) @@ -176,6 +184,16 @@ func (c *client) processResponse(req *Request, res *Response) (*validatedRespons // Check for valid block sets and extract them into `TipSet`s. validRes.tipsets = make([]*types.TipSet, resLength) for i := 0; i < resLength; i++ { + if res.Chain[i] == nil { + return nil, xerrors.Errorf("response with nil tipset in pos %d", i) + } + for blockIdx, block := range res.Chain[i].Blocks { + if block == nil { + return nil, xerrors.Errorf("tipset with nil block in pos %d", blockIdx) + // FIXME: Maybe we should move this check to `NewTipSet`. + } + } + validRes.tipsets[i], err = types.NewTipSet(res.Chain[i].Blocks) if err != nil { return nil, xerrors.Errorf("invalid tipset blocks at height (head - %d): %w", i, err) @@ -214,14 +232,16 @@ func (c *client) processResponse(req *Request, res *Response) (*validatedRespons if err != nil { return nil, err } - } - - if options.ValidateMessages { - // if the request includes target tipsets, validate against them + } else { + // If we didn't request the headers they should have been provided + // by the caller. + if len(tipsets) < len(res.Chain) { + return nil, xerrors.Errorf("not enought tipsets provided for message response validation, needed %d, have %d", len(res.Chain), len(tipsets)) + } chain := make([]*BSTipSet, 0, resLength) for i, resChain := range res.Chain { next := &BSTipSet{ - Blocks: req.TipSets[i].Blocks(), + Blocks: tipsets[i].Blocks(), Messages: resChain.Messages, } chain = append(chain, next) @@ -290,7 +310,7 @@ func (c *client) GetBlocks(ctx context.Context, tsk types.TipSetKey, count int) Options: Headers, } - validRes, err := c.doRequest(ctx, req, nil) + validRes, err := c.doRequest(ctx, req, nil, nil) if err != nil { return nil, err } @@ -308,7 +328,7 @@ func (c *client) GetFullTipSet(ctx context.Context, peer peer.ID, tsk types.TipS Options: Headers | Messages, } - validRes, err := c.doRequest(ctx, req, &peer) + validRes, err := c.doRequest(ctx, req, &peer, nil) if err != nil { return nil, err } @@ -335,11 +355,10 @@ func (c *client) GetChainMessages(ctx context.Context, tipsets []*types.TipSet) req := &Request{ Head: head.Cids(), Length: length, - Options: Messages | Validate, - TipSets: tipsets, + Options: Messages, } - validRes, err := c.doRequest(ctx, req, nil) + validRes, err := c.doRequest(ctx, req, nil, tipsets) if err != nil { return nil, err } diff --git a/chain/exchange/protocol.go b/chain/exchange/protocol.go index b1fe69bd2..211479335 100644 --- a/chain/exchange/protocol.go +++ b/chain/exchange/protocol.go @@ -57,8 +57,6 @@ type Request struct { // Request options, see `Options` type for more details. Compressed // in a single `uint64` to save space. Options uint64 - // Request tipsets for validation - TipSets []*types.TipSet } // `Request` processed and validated to query the tipsets needed. @@ -73,15 +71,13 @@ type validatedRequest struct { const ( Headers = 1 << iota Messages - Validate ) // Decompressed options into separate struct members for easy access // during internal processing.. type parsedOptions struct { - IncludeHeaders bool - IncludeMessages bool - ValidateMessages bool + IncludeHeaders bool + IncludeMessages bool } func (options *parsedOptions) noOptionsSet() bool { @@ -91,9 +87,8 @@ func (options *parsedOptions) noOptionsSet() bool { func parseOptions(optfield uint64) *parsedOptions { return &parsedOptions{ - IncludeHeaders: optfield&(uint64(Headers)) != 0, - IncludeMessages: optfield&(uint64(Messages)) != 0, - ValidateMessages: optfield&(uint64(Validate)) != 0, + IncludeHeaders: optfield&(uint64(Headers)) != 0, + IncludeMessages: optfield&(uint64(Messages)) != 0, } } @@ -144,6 +139,8 @@ func (res *Response) statusToError() error { // FIXME: Rename. type BSTipSet struct { + // List of blocks belonging to a single tipset to which the + // `CompactedMessages` are linked. Blocks []*types.BlockHeader Messages *CompactedMessages } From 773714792f18a503ca1ad4d586734a7be7bc1223 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 22 Sep 2020 18:40:03 -0700 Subject: [PATCH 294/885] update oni --- extern/oni | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/oni b/extern/oni index 8b7e7d438..9a0d5cd73 160000 --- a/extern/oni +++ b/extern/oni @@ -1 +1 @@ -Subproject commit 8b7e7d438c4cc38a0d2d671876d4590ad20655b3 +Subproject commit 9a0d5cd739de77b357589ac1fc8b756ed27299be From ed4bf9b8fe26ce6fc98d0f3bdb1147523116b8ce Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Wed, 23 Sep 2020 00:48:35 -0400 Subject: [PATCH 295/885] API shouldn't depend on actors directly --- api/api_full.go | 8 ++++---- api/apistruct/struct.go | 2 +- chain/actors/builtin/builtin.go | 3 +++ chain/actors/builtin/paych/paych.go | 4 ++++ node/impl/paych/paych.go | 21 ++++++++++----------- 5 files changed, 22 insertions(+), 16 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index f39d0e9bc..11060a9ea 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -21,9 +21,9 @@ import ( "github.com/filecoin-project/go-state-types/dline" "github.com/filecoin-project/lotus/chain/actors/builtin/market" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" - "github.com/filecoin-project/specs-actors/actors/builtin/paych" - "github.com/filecoin-project/specs-actors/actors/runtime/proof" + "github.com/filecoin-project/lotus/chain/actors/builtin/paych" + "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/builtin/power" "github.com/filecoin-project/lotus/chain/types" marketevents "github.com/filecoin-project/lotus/markets/loggers" @@ -788,7 +788,7 @@ type CirculatingSupply struct { type MiningBaseInfo struct { MinerPower types.BigInt NetworkPower types.BigInt - Sectors []proof.SectorInfo + Sectors []builtin.SectorInfo WorkerKey address.Address SectorSize abi.SectorSize PrevBeaconEntry types.BeaconEntry @@ -805,7 +805,7 @@ type BlockTemplate struct { Messages []*types.SignedMessage Epoch abi.ChainEpoch Timestamp uint64 - WinningPoStProof []proof.PoStProof + WinningPoStProof []builtin.PoStProof } type DataSize struct { diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 91a545479..aea7aae8c 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -29,12 +29,12 @@ import ( "github.com/filecoin-project/lotus/extern/sector-storage/stores" "github.com/filecoin-project/lotus/extern/sector-storage/storiface" marketevents "github.com/filecoin-project/lotus/markets/loggers" - "github.com/filecoin-project/specs-actors/actors/builtin/paych" "github.com/filecoin-project/specs-storage/storage" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + "github.com/filecoin-project/lotus/chain/actors/builtin/paych" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/modules/dtypes" ) diff --git a/chain/actors/builtin/builtin.go b/chain/actors/builtin/builtin.go index 3230705dd..4079e694a 100644 --- a/chain/actors/builtin/builtin.go +++ b/chain/actors/builtin/builtin.go @@ -5,6 +5,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" + proof0 "github.com/filecoin-project/specs-actors/actors/runtime/proof" smoothing0 "github.com/filecoin-project/specs-actors/actors/util/smoothing" @@ -28,6 +29,8 @@ func VersionForNetwork(version network.Version) Version { } // TODO: Why does actors have 2 different versions of this? +type SectorInfo = proof0.SectorInfo +type PoStProof = proof0.PoStProof type FilterEstimate = smoothing0.FilterEstimate func FromV0FilterEstimate(v0 smoothing0.FilterEstimate) FilterEstimate { diff --git a/chain/actors/builtin/paych/paych.go b/chain/actors/builtin/paych/paych.go index 5eec5f08b..0413dcdd0 100644 --- a/chain/actors/builtin/paych/paych.go +++ b/chain/actors/builtin/paych/paych.go @@ -8,6 +8,7 @@ import ( big "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/cbor" builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" + paych0 "github.com/filecoin-project/specs-actors/actors/builtin/paych" "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/types" @@ -54,3 +55,6 @@ type LaneState interface { Redeemed() big.Int Nonce() uint64 } + +type SignedVoucher = paych0.SignedVoucher +type ModVerifyParams = paych0.ModVerifyParams diff --git a/node/impl/paych/paych.go b/node/impl/paych/paych.go index b2e6be0ca..af0a1db15 100644 --- a/node/impl/paych/paych.go +++ b/node/impl/paych/paych.go @@ -10,9 +10,8 @@ import ( "github.com/filecoin-project/go-address" - paych0 "github.com/filecoin-project/specs-actors/actors/builtin/paych" - "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/actors/builtin/paych" "github.com/filecoin-project/lotus/chain/types" full "github.com/filecoin-project/lotus/node/impl/full" "github.com/filecoin-project/lotus/paychmgr" @@ -71,10 +70,10 @@ func (a *PaychAPI) PaychNewPayment(ctx context.Context, from, to address.Address return nil, err } - svs := make([]*paych0.SignedVoucher, len(vouchers)) + svs := make([]*paych.SignedVoucher, len(vouchers)) for i, v := range vouchers { - sv, err := a.PaychMgr.CreateVoucher(ctx, ch.Channel, paych0.SignedVoucher{ + sv, err := a.PaychMgr.CreateVoucher(ctx, ch.Channel, paych.SignedVoucher{ Amount: v.Amount, Lane: lane, @@ -123,15 +122,15 @@ func (a *PaychAPI) PaychCollect(ctx context.Context, addr address.Address) (cid. return a.PaychMgr.Collect(ctx, addr) } -func (a *PaychAPI) PaychVoucherCheckValid(ctx context.Context, ch address.Address, sv *paych0.SignedVoucher) error { +func (a *PaychAPI) PaychVoucherCheckValid(ctx context.Context, ch address.Address, sv *paych.SignedVoucher) error { return a.PaychMgr.CheckVoucherValid(ctx, ch, sv) } -func (a *PaychAPI) PaychVoucherCheckSpendable(ctx context.Context, ch address.Address, sv *paych0.SignedVoucher, secret []byte, proof []byte) (bool, error) { +func (a *PaychAPI) PaychVoucherCheckSpendable(ctx context.Context, ch address.Address, sv *paych.SignedVoucher, secret []byte, proof []byte) (bool, error) { return a.PaychMgr.CheckVoucherSpendable(ctx, ch, sv, secret, proof) } -func (a *PaychAPI) PaychVoucherAdd(ctx context.Context, ch address.Address, sv *paych0.SignedVoucher, proof []byte, minDelta types.BigInt) (types.BigInt, error) { +func (a *PaychAPI) PaychVoucherAdd(ctx context.Context, ch address.Address, sv *paych.SignedVoucher, proof []byte, minDelta types.BigInt) (types.BigInt, error) { return a.PaychMgr.AddVoucherInbound(ctx, ch, sv, proof, minDelta) } @@ -143,16 +142,16 @@ func (a *PaychAPI) PaychVoucherAdd(ctx context.Context, ch address.Address, sv * // If there are insufficient funds in the channel to create the voucher, // returns a nil voucher and the shortfall. func (a *PaychAPI) PaychVoucherCreate(ctx context.Context, pch address.Address, amt types.BigInt, lane uint64) (*api.VoucherCreateResult, error) { - return a.PaychMgr.CreateVoucher(ctx, pch, paych0.SignedVoucher{Amount: amt, Lane: lane}) + return a.PaychMgr.CreateVoucher(ctx, pch, paych.SignedVoucher{Amount: amt, Lane: lane}) } -func (a *PaychAPI) PaychVoucherList(ctx context.Context, pch address.Address) ([]*paych0.SignedVoucher, error) { +func (a *PaychAPI) PaychVoucherList(ctx context.Context, pch address.Address) ([]*paych.SignedVoucher, error) { vi, err := a.PaychMgr.ListVouchers(ctx, pch) if err != nil { return nil, err } - out := make([]*paych0.SignedVoucher, len(vi)) + out := make([]*paych.SignedVoucher, len(vi)) for k, v := range vi { out[k] = v.Voucher } @@ -160,6 +159,6 @@ func (a *PaychAPI) PaychVoucherList(ctx context.Context, pch address.Address) ([ return out, nil } -func (a *PaychAPI) PaychVoucherSubmit(ctx context.Context, ch address.Address, sv *paych0.SignedVoucher, secret []byte, proof []byte) (cid.Cid, error) { +func (a *PaychAPI) PaychVoucherSubmit(ctx context.Context, ch address.Address, sv *paych.SignedVoucher, secret []byte, proof []byte) (cid.Cid, error) { return a.PaychMgr.SubmitVoucher(ctx, ch, sv, secret, proof) } From e27fc03f5541288ef4a6c4080af9587ac706144a Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Wed, 23 Sep 2020 01:36:55 -0400 Subject: [PATCH 296/885] Reward state interface only needs cbor.Marshaler --- chain/actors/builtin/reward/reward.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chain/actors/builtin/reward/reward.go b/chain/actors/builtin/reward/reward.go index a4f936d9b..cfa82c774 100644 --- a/chain/actors/builtin/reward/reward.go +++ b/chain/actors/builtin/reward/reward.go @@ -29,7 +29,7 @@ func Load(store adt.Store, act *types.Actor) (st State, err error) { } type State interface { - cbor.Er + cbor.Marshaler ThisEpochBaselinePower() (abi.StoragePower, error) ThisEpochReward() (abi.StoragePower, error) From 476e7992e8e58de64c848081b03136a5e51c1644 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Wed, 23 Sep 2020 01:19:43 -0400 Subject: [PATCH 297/885] Add an error return to all actor state interface methods --- chain/actors/builtin/market/market.go | 6 +- chain/actors/builtin/market/v0.go | 18 ++--- chain/actors/builtin/miner/miner.go | 6 +- chain/actors/builtin/miner/v0.go | 16 ++--- chain/actors/builtin/multisig/multisig.go | 10 +-- chain/actors/builtin/multisig/v0.go | 20 +++--- chain/actors/builtin/paych/mock/mock.go | 24 +++---- chain/actors/builtin/paych/paych.go | 12 ++-- chain/actors/builtin/paych/v0.go | 24 +++---- chain/events/state/predicates.go | 37 ++++++++-- chain/stmgr/stmgr.go | 23 +++++-- cli/multisig.go | 29 ++++++-- cli/paych_test.go | 4 +- cmd/lotus-chainwatch/processor/miner.go | 6 +- cmd/lotus-shed/genesis-verify.go | 13 +++- node/impl/full/state.go | 7 +- paychmgr/paych.go | 83 +++++++++++++++++------ paychmgr/simple.go | 6 +- paychmgr/state.go | 12 +++- 19 files changed, 245 insertions(+), 111 deletions(-) diff --git a/chain/actors/builtin/market/market.go b/chain/actors/builtin/market/market.go index 9cb0ffc13..84872b04b 100644 --- a/chain/actors/builtin/market/market.go +++ b/chain/actors/builtin/market/market.go @@ -31,13 +31,13 @@ func Load(store adt.Store, act *types.Actor) (st State, err error) { type State interface { cbor.Marshaler - BalancesChanged(State) bool + BalancesChanged(State) (bool, error) EscrowTable() (BalanceTable, error) LockedTable() (BalanceTable, error) TotalLocked() (abi.TokenAmount, error) - StatesChanged(State) bool + StatesChanged(State) (bool, error) States() (DealStates, error) - ProposalsChanged(State) bool + ProposalsChanged(State) (bool, error) Proposals() (DealProposals, error) VerifyDealsForActivation( minerAddr address.Address, deals []abi.DealID, currEpoch, sectorExpiry abi.ChainEpoch, diff --git a/chain/actors/builtin/market/v0.go b/chain/actors/builtin/market/v0.go index 14b8b7d93..2727f513d 100644 --- a/chain/actors/builtin/market/v0.go +++ b/chain/actors/builtin/market/v0.go @@ -25,24 +25,24 @@ func (s *state0) TotalLocked() (abi.TokenAmount, error) { return fml, nil } -func (s *state0) BalancesChanged(otherState State) bool { +func (s *state0) BalancesChanged(otherState State) (bool, error) { otherState0, ok := otherState.(*state0) if !ok { // there's no way to compare different versions of the state, so let's // just say that means the state of balances has changed - return true + return true, nil } - return !s.State.EscrowTable.Equals(otherState0.State.EscrowTable) || !s.State.LockedTable.Equals(otherState0.State.LockedTable) + return !s.State.EscrowTable.Equals(otherState0.State.EscrowTable) || !s.State.LockedTable.Equals(otherState0.State.LockedTable), nil } -func (s *state0) StatesChanged(otherState State) bool { +func (s *state0) StatesChanged(otherState State) (bool, error) { otherState0, ok := otherState.(*state0) if !ok { // there's no way to compare different versions of the state, so let's // just say that means the state of balances has changed - return true + return true, nil } - return !s.State.States.Equals(otherState0.State.States) + return !s.State.States.Equals(otherState0.State.States), nil } func (s *state0) States() (DealStates, error) { @@ -53,14 +53,14 @@ func (s *state0) States() (DealStates, error) { return &dealStates0{stateArray}, nil } -func (s *state0) ProposalsChanged(otherState State) bool { +func (s *state0) ProposalsChanged(otherState State) (bool, error) { otherState0, ok := otherState.(*state0) if !ok { // there's no way to compare different versions of the state, so let's // just say that means the state of balances has changed - return true + return true, nil } - return !s.State.Proposals.Equals(otherState0.State.Proposals) + return !s.State.Proposals.Equals(otherState0.State.Proposals), nil } func (s *state0) Proposals() (DealProposals, error) { diff --git a/chain/actors/builtin/miner/miner.go b/chain/actors/builtin/miner/miner.go index dbb4ddd73..50a0fc5ca 100644 --- a/chain/actors/builtin/miner/miner.go +++ b/chain/actors/builtin/miner/miner.go @@ -58,11 +58,11 @@ type State interface { LoadDeadline(idx uint64) (Deadline, error) ForEachDeadline(cb func(idx uint64, dl Deadline) error) error NumDeadlines() (uint64, error) - DeadlinesChanged(State) bool + DeadlinesChanged(State) (bool, error) Info() (MinerInfo, error) - DeadlineInfo(epoch abi.ChainEpoch) *dline.Info + DeadlineInfo(epoch abi.ChainEpoch) (*dline.Info, error) // Diff helpers. Used by Diff* functions internally. sectors() (adt.Array, error) @@ -76,7 +76,7 @@ type Deadline interface { ForEachPartition(cb func(idx uint64, part Partition) error) error PostSubmissions() (bitfield.BitField, error) - PartitionsChanged(Deadline) bool + PartitionsChanged(Deadline) (bool, error) } type Partition interface { diff --git a/chain/actors/builtin/miner/v0.go b/chain/actors/builtin/miner/v0.go index a56778156..e515b9ed6 100644 --- a/chain/actors/builtin/miner/v0.go +++ b/chain/actors/builtin/miner/v0.go @@ -244,14 +244,14 @@ func (s *state0) NumDeadlines() (uint64, error) { return miner0.WPoStPeriodDeadlines, nil } -func (s *state0) DeadlinesChanged(other State) bool { +func (s *state0) DeadlinesChanged(other State) (bool, error) { other0, ok := other.(*state0) if !ok { // treat an upgrade as a change, always - return true + return true, nil } - return s.State.Deadlines.Equals(other0.Deadlines) + return s.State.Deadlines.Equals(other0.Deadlines), nil } func (s *state0) Info() (MinerInfo, error) { @@ -288,8 +288,8 @@ func (s *state0) Info() (MinerInfo, error) { return mi, nil } -func (s *state0) DeadlineInfo(epoch abi.ChainEpoch) *dline.Info { - return s.State.DeadlineInfo(epoch) +func (s *state0) DeadlineInfo(epoch abi.ChainEpoch) (*dline.Info, error) { + return s.State.DeadlineInfo(epoch), nil } func (s *state0) sectors() (adt.Array, error) { @@ -339,14 +339,14 @@ func (d *deadline0) ForEachPartition(cb func(uint64, Partition) error) error { }) } -func (d *deadline0) PartitionsChanged(other Deadline) bool { +func (d *deadline0) PartitionsChanged(other Deadline) (bool, error) { other0, ok := other.(*deadline0) if !ok { // treat an upgrade as a change, always - return true + return true, nil } - return d.Deadline.Partitions.Equals(other0.Deadline.Partitions) + return d.Deadline.Partitions.Equals(other0.Deadline.Partitions), nil } func (d *deadline0) PostSubmissions() (bitfield.BitField, error) { diff --git a/chain/actors/builtin/multisig/multisig.go b/chain/actors/builtin/multisig/multisig.go index 226eed75b..884b6f493 100644 --- a/chain/actors/builtin/multisig/multisig.go +++ b/chain/actors/builtin/multisig/multisig.go @@ -31,11 +31,11 @@ type State interface { cbor.Marshaler LockedBalance(epoch abi.ChainEpoch) (abi.TokenAmount, error) - StartEpoch() abi.ChainEpoch - UnlockDuration() abi.ChainEpoch - InitialBalance() abi.TokenAmount - Threshold() uint64 - Signers() []address.Address + StartEpoch() (abi.ChainEpoch, error) + UnlockDuration() (abi.ChainEpoch, error) + InitialBalance() (abi.TokenAmount, error) + Threshold() (uint64, error) + Signers() ([]address.Address, error) ForEachPendingTxn(func(id int64, txn Transaction) error) error } diff --git a/chain/actors/builtin/multisig/v0.go b/chain/actors/builtin/multisig/v0.go index e81a367cf..ae0a7ac0e 100644 --- a/chain/actors/builtin/multisig/v0.go +++ b/chain/actors/builtin/multisig/v0.go @@ -23,24 +23,24 @@ func (s *state0) LockedBalance(currEpoch abi.ChainEpoch) (abi.TokenAmount, error return s.State.AmountLocked(currEpoch - s.State.StartEpoch), nil } -func (s *state0) StartEpoch() abi.ChainEpoch { - return s.State.StartEpoch +func (s *state0) StartEpoch() (abi.ChainEpoch, error) { + return s.State.StartEpoch, nil } -func (s *state0) UnlockDuration() abi.ChainEpoch { - return s.State.UnlockDuration +func (s *state0) UnlockDuration() (abi.ChainEpoch, error) { + return s.State.UnlockDuration, nil } -func (s *state0) InitialBalance() abi.TokenAmount { - return s.State.InitialBalance +func (s *state0) InitialBalance() (abi.TokenAmount, error) { + return s.State.InitialBalance, nil } -func (s *state0) Threshold() uint64 { - return s.State.NumApprovalsThreshold +func (s *state0) Threshold() (uint64, error) { + return s.State.NumApprovalsThreshold, nil } -func (s *state0) Signers() []address.Address { - return s.State.Signers +func (s *state0) Signers() ([]address.Address, error) { + return s.State.Signers, nil } func (s *state0) ForEachPendingTxn(cb func(id int64, txn Transaction) error) error { diff --git a/chain/actors/builtin/paych/mock/mock.go b/chain/actors/builtin/paych/mock/mock.go index 31f7fba93..c4903f3ac 100644 --- a/chain/actors/builtin/paych/mock/mock.go +++ b/chain/actors/builtin/paych/mock/mock.go @@ -45,23 +45,23 @@ func (ms *mockState) MarshalCBOR(io.Writer) error { } // Channel owner, who has funded the actor -func (ms *mockState) From() address.Address { - return ms.from +func (ms *mockState) From() (address.Address, error) { + return ms.from, nil } // Recipient of payouts from channel -func (ms *mockState) To() address.Address { - return ms.to +func (ms *mockState) To() (address.Address, error) { + return ms.to, nil } // Height at which the channel can be `Collected` -func (ms *mockState) SettlingAt() abi.ChainEpoch { - return ms.settlingAt +func (ms *mockState) SettlingAt() (abi.ChainEpoch, error) { + return ms.settlingAt, nil } // Amount successfully redeemed through the payment channel, paid out on `Collect()` -func (ms *mockState) ToSend() abi.TokenAmount { - return ms.toSend +func (ms *mockState) ToSend() (abi.TokenAmount, error) { + return ms.toSend, nil } // Get total number of lanes @@ -80,10 +80,10 @@ func (ms *mockState) ForEachLaneState(cb func(idx uint64, dl paych.LaneState) er return lastErr } -func (mls *mockLaneState) Redeemed() big.Int { - return mls.redeemed +func (mls *mockLaneState) Redeemed() (big.Int, error) { + return mls.redeemed, nil } -func (mls *mockLaneState) Nonce() uint64 { - return mls.nonce +func (mls *mockLaneState) Nonce() (uint64, error) { + return mls.nonce, nil } diff --git a/chain/actors/builtin/paych/paych.go b/chain/actors/builtin/paych/paych.go index 0413dcdd0..dad54163f 100644 --- a/chain/actors/builtin/paych/paych.go +++ b/chain/actors/builtin/paych/paych.go @@ -33,15 +33,15 @@ func Load(store adt.Store, act *types.Actor) (State, error) { type State interface { cbor.Marshaler // Channel owner, who has funded the actor - From() address.Address + From() (address.Address, error) // Recipient of payouts from channel - To() address.Address + To() (address.Address, error) // Height at which the channel can be `Collected` - SettlingAt() abi.ChainEpoch + SettlingAt() (abi.ChainEpoch, error) // Amount successfully redeemed through the payment channel, paid out on `Collect()` - ToSend() abi.TokenAmount + ToSend() (abi.TokenAmount, error) // Get total number of lanes LaneCount() (uint64, error) @@ -52,8 +52,8 @@ type State interface { // LaneState is an abstract copy of the state of a single lane type LaneState interface { - Redeemed() big.Int - Nonce() uint64 + Redeemed() (big.Int, error) + Nonce() (uint64, error) } type SignedVoucher = paych0.SignedVoucher diff --git a/chain/actors/builtin/paych/v0.go b/chain/actors/builtin/paych/v0.go index b69cf46ce..c0eea1000 100644 --- a/chain/actors/builtin/paych/v0.go +++ b/chain/actors/builtin/paych/v0.go @@ -18,23 +18,23 @@ type state0 struct { } // Channel owner, who has funded the actor -func (s *state0) From() address.Address { - return s.State.From +func (s *state0) From() (address.Address, error) { + return s.State.From, nil } // Recipient of payouts from channel -func (s *state0) To() address.Address { - return s.State.To +func (s *state0) To() (address.Address, error) { + return s.State.To, nil } // Height at which the channel can be `Collected` -func (s *state0) SettlingAt() abi.ChainEpoch { - return s.State.SettlingAt +func (s *state0) SettlingAt() (abi.ChainEpoch, error) { + return s.State.SettlingAt, nil } // Amount successfully redeemed through the payment channel, paid out on `Collect()` -func (s *state0) ToSend() abi.TokenAmount { - return s.State.ToSend +func (s *state0) ToSend() (abi.TokenAmount, error) { + return s.State.ToSend, nil } func (s *state0) getOrLoadLsAmt() (*adt0.Array, error) { @@ -82,10 +82,10 @@ type laneState0 struct { paych.LaneState } -func (ls *laneState0) Redeemed() big.Int { - return ls.LaneState.Redeemed +func (ls *laneState0) Redeemed() (big.Int, error) { + return ls.LaneState.Redeemed, nil } -func (ls *laneState0) Nonce() uint64 { - return ls.LaneState.Nonce +func (ls *laneState0) Nonce() (uint64, error) { + return ls.LaneState.Nonce, nil } diff --git a/chain/events/state/predicates.go b/chain/events/state/predicates.go index 909533d15..e6c2fa084 100644 --- a/chain/events/state/predicates.go +++ b/chain/events/state/predicates.go @@ -97,7 +97,12 @@ type DiffBalanceTablesFunc func(ctx context.Context, oldBalanceTable, newBalance // OnBalanceChanged runs when the escrow table for available balances changes func (sp *StatePredicates) OnBalanceChanged(diffBalances DiffBalanceTablesFunc) DiffStorageMarketStateFunc { return func(ctx context.Context, oldState market.State, newState market.State) (changed bool, user UserData, err error) { - if !oldState.BalancesChanged(newState) { + bc, err := oldState.BalancesChanged(newState) + if err != nil { + return false, nil, err + } + + if !bc { return false, nil, nil } @@ -132,7 +137,12 @@ type DiffAdtArraysFunc func(ctx context.Context, oldDealStateRoot, newDealStateR // OnDealStateChanged calls diffDealStates when the market deal state changes func (sp *StatePredicates) OnDealStateChanged(diffDealStates DiffDealStatesFunc) DiffStorageMarketStateFunc { return func(ctx context.Context, oldState market.State, newState market.State) (changed bool, user UserData, err error) { - if !oldState.StatesChanged(newState) { + sc, err := oldState.StatesChanged(newState) + if err != nil { + return false, nil, err + } + + if !sc { return false, nil, nil } @@ -152,7 +162,12 @@ func (sp *StatePredicates) OnDealStateChanged(diffDealStates DiffDealStatesFunc) // OnDealProposalChanged calls diffDealProps when the market proposal state changes func (sp *StatePredicates) OnDealProposalChanged(diffDealProps DiffDealProposalsFunc) DiffStorageMarketStateFunc { return func(ctx context.Context, oldState market.State, newState market.State) (changed bool, user UserData, err error) { - if !oldState.ProposalsChanged(newState) { + pc, err := oldState.ProposalsChanged(newState) + if err != nil { + return false, nil, err + } + + if !pc { return false, nil, nil } @@ -379,12 +394,22 @@ type PayChToSendChange struct { // OnToSendAmountChanges monitors changes on the total amount to send from one party to the other on a payment channel func (sp *StatePredicates) OnToSendAmountChanges() DiffPaymentChannelStateFunc { return func(ctx context.Context, oldState paych.State, newState paych.State) (changed bool, user UserData, err error) { - if oldState.ToSend().Equals(newState.ToSend()) { + ots, err := oldState.ToSend() + if err != nil { + return false, nil, err + } + + nts, err := newState.ToSend() + if err != nil { + return false, nil, err + } + + if ots.Equals(nts) { return false, nil, nil } return true, &PayChToSendChange{ - OldToSend: oldState.ToSend(), - NewToSend: newState.ToSend(), + OldToSend: ots, + NewToSend: nts, }, nil } } diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index af729143c..eaf9215db 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -843,15 +843,30 @@ func (sm *StateManager) setupGenesisActors(ctx context.Context) error { return err } - if s.StartEpoch() != 0 { + se, err := s.StartEpoch() + if err != nil { + return err + } + + if se != 0 { return xerrors.New("genesis multisig doesn't start vesting at epoch 0!") } - ot, f := totalsByEpoch[s.UnlockDuration()] + ud, err := s.UnlockDuration() + if err != nil { + return err + } + + ib, err := s.InitialBalance() + if err != nil { + return err + } + + ot, f := totalsByEpoch[ud] if f { - totalsByEpoch[s.UnlockDuration()] = big.Add(ot, s.InitialBalance()) + totalsByEpoch[ud] = big.Add(ot, ib) } else { - totalsByEpoch[s.UnlockDuration()] = s.InitialBalance() + totalsByEpoch[ud] = ib } } else if act.IsAccountActor() { diff --git a/cli/multisig.go b/cli/multisig.go index 81c6502d5..0cb4028a9 100644 --- a/cli/multisig.go +++ b/cli/multisig.go @@ -206,13 +206,32 @@ var msigInspectCmd = &cli.Command{ fmt.Printf("Spendable: %s\n", types.FIL(types.BigSub(act.Balance, locked))) if cctx.Bool("vesting") { - fmt.Printf("InitialBalance: %s\n", types.FIL(mstate.InitialBalance())) - fmt.Printf("StartEpoch: %d\n", mstate.StartEpoch()) - fmt.Printf("UnlockDuration: %d\n", mstate.UnlockDuration()) + ib, err := mstate.InitialBalance() + if err != nil { + return err + } + fmt.Printf("InitialBalance: %s\n", types.FIL(ib)) + se, err := mstate.StartEpoch() + if err != nil { + return err + } + fmt.Printf("StartEpoch: %d\n", se) + ud, err := mstate.UnlockDuration() + if err != nil { + return err + } + fmt.Printf("UnlockDuration: %d\n", ud) } - signers := mstate.Signers() - fmt.Printf("Threshold: %d / %d\n", mstate.Threshold(), len(signers)) + signers, err := mstate.Signers() + if err != nil { + return err + } + threshold, err := mstate.Threshold() + if err != nil { + return err + } + fmt.Printf("Threshold: %d / %d\n", threshold, len(signers)) fmt.Println("Signers:") for _, s := range signers { fmt.Printf("\t%s\n", s) diff --git a/cli/paych_test.go b/cli/paych_test.go index 49631f1fc..47a84fe5e 100644 --- a/cli/paych_test.go +++ b/cli/paych_test.go @@ -89,7 +89,9 @@ func TestPaymentChannels(t *testing.T) { // Wait for the chain to reach the settle height chState := getPaychState(ctx, t, paymentReceiver, chAddr) - waitForHeight(ctx, t, paymentReceiver, chState.SettlingAt()) + sa, err := chState.SettlingAt() + require.NoError(t, err) + waitForHeight(ctx, t, paymentReceiver, sa) // receiver: paych collect cmd = []string{chAddr.String()} diff --git a/cmd/lotus-chainwatch/processor/miner.go b/cmd/lotus-chainwatch/processor/miner.go index 17eef3afa..3a37a82f8 100644 --- a/cmd/lotus-chainwatch/processor/miner.go +++ b/cmd/lotus-chainwatch/processor/miner.go @@ -684,7 +684,11 @@ func (p *Processor) diffMinerPartitions(ctx context.Context, m minerActorInfo, e return err } curMiner := m.state - if !prevMiner.DeadlinesChanged(curMiner) { + dc, err := prevMiner.DeadlinesChanged(curMiner) + if err != nil { + return err + } + if !dc { return nil } panic("TODO") diff --git a/cmd/lotus-shed/genesis-verify.go b/cmd/lotus-shed/genesis-verify.go index 1225d817a..da2c82359 100644 --- a/cmd/lotus-shed/genesis-verify.go +++ b/cmd/lotus-shed/genesis-verify.go @@ -108,10 +108,19 @@ var genesisVerifyCmd = &cli.Command{ return xerrors.Errorf("multisig actor: %w", err) } + signers, err := st.Signers() + if err != nil { + return xerrors.Errorf("multisig actor: %w", err) + } + threshold, err := st.Threshold() + if err != nil { + return xerrors.Errorf("multisig actor: %w", err) + } + kmultisigs[addr] = msigInfo{ Balance: types.FIL(act.Balance), - Signers: st.Signers(), - Threshold: st.Threshold(), + Signers: signers, + Threshold: threshold, } msigAddrs = append(msigAddrs, addr) case act.IsAccountActor(): diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 109a265ea..f8bf92a92 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -212,7 +212,12 @@ func (a *StateAPI) StateMinerProvingDeadline(ctx context.Context, addr address.A return nil, xerrors.Errorf("failed to load miner actor state: %w", err) } - return mas.DeadlineInfo(ts.Height()).NextNotElapsed(), nil + di, err := mas.DeadlineInfo(ts.Height()) + if err != nil { + return nil, xerrors.Errorf("failed to get deadline info: %w", err) + } + + return di.NextNotElapsed(), nil } func (a *StateAPI) StateMinerFaults(ctx context.Context, addr address.Address, tsk types.TipSetKey) (bitfield.BitField, error) { diff --git a/paychmgr/paych.go b/paychmgr/paych.go index 11dfd2a1e..f856b9890 100644 --- a/paychmgr/paych.go +++ b/paychmgr/paych.go @@ -48,12 +48,12 @@ type laneState struct { nonce uint64 } -func (ls laneState) Redeemed() big.Int { - return ls.redeemed +func (ls laneState) Redeemed() (big.Int, error) { + return ls.redeemed, nil } -func (ls laneState) Nonce() uint64 { - return ls.nonce +func (ls laneState) Nonce() (uint64, error) { + return ls.nonce, nil } // channelAccessor is used to simplify locking when accessing a channel @@ -181,7 +181,12 @@ func (ca *channelAccessor) checkVoucherValidUnlocked(ctx context.Context, ch add } // Load channel "From" account actor state - from, err := ca.api.ResolveToKeyAddress(ctx, pchState.From(), nil) + f, err := pchState.From() + if err != nil { + return nil, err + } + + from, err := ca.api.ResolveToKeyAddress(ctx, f, nil) if err != nil { return nil, err } @@ -208,13 +213,24 @@ func (ca *channelAccessor) checkVoucherValidUnlocked(ctx context.Context, ch add // If the new voucher nonce value is less than the highest known // nonce for the lane ls, lsExists := laneStates[sv.Lane] - if lsExists && sv.Nonce <= ls.Nonce() { - return nil, fmt.Errorf("nonce too low") - } + if lsExists { + n, err := ls.Nonce() + if err != nil { + return nil, err + } - // If the voucher amount is less than the highest known voucher amount - if lsExists && sv.Amount.LessThanEqual(ls.Redeemed()) { - return nil, fmt.Errorf("voucher amount is lower than amount for voucher with lower nonce") + if sv.Nonce <= n { + return nil, fmt.Errorf("nonce too low") + } + + // If the voucher amount is less than the highest known voucher amount + r, err := ls.Redeemed() + if err != nil { + return nil, err + } + if sv.Amount.LessThanEqual(r) { + return nil, fmt.Errorf("voucher amount is lower than amount for voucher with lower nonce") + } } // Total redeemed is the total redeemed amount for all lanes, including @@ -239,7 +255,12 @@ func (ca *channelAccessor) checkVoucherValidUnlocked(ctx context.Context, ch add // Total required balance = total redeemed + toSend // Must not exceed actor balance - newTotal := types.BigAdd(totalRedeemed, pchState.ToSend()) + ts, err := pchState.ToSend() + if err != nil { + return nil, err + } + + newTotal := types.BigAdd(totalRedeemed, ts) if act.Balance.LessThan(newTotal) { return nil, newErrInsufficientFunds(types.BigSub(newTotal, act.Balance)) } @@ -322,7 +343,7 @@ func (ca *channelAccessor) getPaychRecipient(ctx context.Context, ch address.Add return address.Address{}, err } - return state.To(), nil + return state.To() } func (ca *channelAccessor) addVoucher(ctx context.Context, ch address.Address, sv *paych0.SignedVoucher, proof []byte, minDelta types.BigInt) (types.BigInt, error) { @@ -376,7 +397,10 @@ func (ca *channelAccessor) addVoucherUnlocked(ctx context.Context, ch address.Ad laneState, exists := laneStates[sv.Lane] redeemed := big.NewInt(0) if exists { - redeemed = laneState.Redeemed() + redeemed, err = laneState.Redeemed() + if err != nil { + return types.NewInt(0), err + } } delta := types.BigSub(sv.Amount, redeemed) @@ -531,9 +555,14 @@ func (ca *channelAccessor) laneState(ctx context.Context, state paych.State, ch // If there's a voucher for a lane that isn't in chain state just // create it ls, ok := laneStates[v.Voucher.Lane] - - if ok && v.Voucher.Nonce < ls.Nonce() { - continue + if ok { + n, err := ls.Nonce() + if err != nil { + return nil, err + } + if v.Voucher.Nonce < n { + continue + } } laneStates[v.Voucher.Lane] = laneState{v.Voucher.Amount, v.Voucher.Nonce} @@ -551,17 +580,31 @@ func (ca *channelAccessor) totalRedeemedWithVoucher(laneStates map[uint64]paych. total := big.NewInt(0) for _, ls := range laneStates { - total = big.Add(total, ls.Redeemed()) + r, err := ls.Redeemed() + if err != nil { + return big.Int{}, err + } + total = big.Add(total, r) } lane, ok := laneStates[sv.Lane] if ok { // If the voucher is for an existing lane, and the voucher nonce // is higher than the lane nonce - if sv.Nonce > lane.Nonce() { + n, err := lane.Nonce() + if err != nil { + return big.Int{}, err + } + + if sv.Nonce > n { // Add the delta between the redeemed amount and the voucher // amount to the total - delta := big.Sub(sv.Amount, lane.Redeemed()) + r, err := lane.Redeemed() + if err != nil { + return big.Int{}, err + } + + delta := big.Sub(sv.Amount, r) total = big.Add(total, delta) } } else { diff --git a/paychmgr/simple.go b/paychmgr/simple.go index 815b8acab..d2a42a673 100644 --- a/paychmgr/simple.go +++ b/paychmgr/simple.go @@ -317,7 +317,11 @@ func (ca *channelAccessor) currentAvailableFunds(channelID string, queuedAmt typ } for _, ls := range laneStates { - totalRedeemed = types.BigAdd(totalRedeemed, ls.Redeemed()) + r, err := ls.Redeemed() + if err != nil { + return nil, err + } + totalRedeemed = types.BigAdd(totalRedeemed, r) } } diff --git a/paychmgr/state.go b/paychmgr/state.go index 2571ef73e..65963d2a0 100644 --- a/paychmgr/state.go +++ b/paychmgr/state.go @@ -24,11 +24,19 @@ func (ca *stateAccessor) loadStateChannelInfo(ctx context.Context, ch address.Ad } // Load channel "From" account actor state - from, err := ca.sm.ResolveToKeyAddress(ctx, st.From(), nil) + f, err := st.From() if err != nil { return nil, err } - to, err := ca.sm.ResolveToKeyAddress(ctx, st.To(), nil) + from, err := ca.sm.ResolveToKeyAddress(ctx, f, nil) + if err != nil { + return nil, err + } + t, err := st.To() + if err != nil { + return nil, err + } + to, err := ca.sm.ResolveToKeyAddress(ctx, t, nil) if err != nil { return nil, err } From 819180f739c7d6c15eb7d4c00ab5218c7d30096f Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Wed, 23 Sep 2020 02:14:28 -0400 Subject: [PATCH 298/885] Implement inefficient OnAddressMapChange predicate --- chain/events/state/predicates.go | 72 ++++++++++++++++++++++++-------- 1 file changed, 55 insertions(+), 17 deletions(-) diff --git a/chain/events/state/predicates.go b/chain/events/state/predicates.go index e6c2fa084..99b8480dc 100644 --- a/chain/events/state/predicates.go +++ b/chain/events/state/predicates.go @@ -519,32 +519,72 @@ func (i *InitActorAddressChanges) Remove(key string, val *typegen.Deferred) erro func (sp *StatePredicates) OnAddressMapChange() DiffInitActorStateFunc { return func(ctx context.Context, oldState, newState init_.State) (changed bool, user UserData, err error) { - /*ctxStore := &contextStore{ - ctx: ctx, - cst: sp.cst, - } - addressChanges := &InitActorAddressChanges{ Added: []AddressPair{}, Modified: []AddressChange{}, Removed: []AddressPair{}, } - if oldState.AddressMap.Equals(newState.AddressMap) { - return false, nil, nil - } + err = oldState.ForEachActor(func(oldId abi.ActorID, oldAddress address.Address) error { + oldIdAddress, err := address.NewIDAddress(uint64(oldId)) + if err != nil { + return err + } + + newIdAddress, found, err := newState.ResolveAddress(oldAddress) + if err != nil { + return err + } + + if !found { + addressChanges.Removed = append(addressChanges.Removed, AddressPair{ + ID: oldIdAddress, + PK: oldAddress, + }) + } + + if oldIdAddress != newIdAddress { + addressChanges.Modified = append(addressChanges.Modified, AddressChange{ + From: AddressPair{ + ID: oldIdAddress, + PK: oldAddress, + }, + To: AddressPair{ + ID: newIdAddress, + PK: oldAddress, + }, + }) + } + + return nil + }) - oldAddrs, err := adt0.AsMap(ctxStore, oldState.AddressMap) if err != nil { return false, nil, err } - newAddrs, err := adt0.AsMap(ctxStore, newState.AddressMap) - if err != nil { - return false, nil, err - } + err = newState.ForEachActor(func(newId abi.ActorID, newAddress address.Address) error { + newIdAddress, err := address.NewIDAddress(uint64(newId)) + if err != nil { + return err + } - if err := adt.DiffAdtMap(oldAddrs, newAddrs, addressChanges); err != nil { + _, found, err := newState.ResolveAddress(newAddress) + if err != nil { + return err + } + + if !found { + addressChanges.Added = append(addressChanges.Added, AddressPair{ + ID: newIdAddress, + PK: newAddress, + }) + } + + return nil + }) + + if err != nil { return false, nil, err } @@ -552,8 +592,6 @@ func (sp *StatePredicates) OnAddressMapChange() DiffInitActorStateFunc { return false, nil, nil } - return true, addressChanges, nil*/ - - panic("TODO") + return true, addressChanges, nil } } From a876a0ba443afee1477567b06679edf2f3ce21f2 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Wed, 23 Sep 2020 02:18:52 -0400 Subject: [PATCH 299/885] Use actor state addresses --- chain/actors/builtin/market/market.go | 7 +++++ chain/gen/genesis/miners.go | 37 +++++++++++++++------------ chain/market/fundmgr.go | 3 ++- chain/market/fundmgr_test.go | 3 ++- chain/stmgr/forks_test.go | 3 ++- chain/store/weight.go | 3 +-- chain/sync.go | 3 +-- chain/vm/vm.go | 3 ++- paychmgr/paychget_test.go | 9 ++++--- paychmgr/simple.go | 3 ++- 10 files changed, 45 insertions(+), 29 deletions(-) diff --git a/chain/actors/builtin/market/market.go b/chain/actors/builtin/market/market.go index 84872b04b..fef0c03f9 100644 --- a/chain/actors/builtin/market/market.go +++ b/chain/actors/builtin/market/market.go @@ -7,6 +7,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/cbor" builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" + market0 "github.com/filecoin-project/specs-actors/actors/builtin/market" "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" @@ -65,6 +66,12 @@ type DealProposals interface { decode(*cbg.Deferred) (*DealProposal, error) } +type PublishStorageDealsParams = market0.PublishStorageDealsParams +type PublishStorageDealsReturn = market0.PublishStorageDealsReturn +type VerifyDealsForActivationParams = market0.VerifyDealsForActivationParams + +type ClientDealProposal = market0.ClientDealProposal + type DealState struct { SectorStartEpoch abi.ChainEpoch // -1 if not yet included in proven sector LastUpdatedEpoch abi.ChainEpoch // -1 if deal state never updated diff --git a/chain/gen/genesis/miners.go b/chain/gen/genesis/miners.go index 853c9c4a0..1023e5efa 100644 --- a/chain/gen/genesis/miners.go +++ b/chain/gen/genesis/miners.go @@ -6,6 +6,12 @@ import ( "fmt" "math/rand" + market0 "github.com/filecoin-project/specs-actors/actors/builtin/market" + + "github.com/filecoin-project/lotus/chain/actors/builtin/power" + "github.com/filecoin-project/lotus/chain/actors/builtin/reward" + + "github.com/filecoin-project/lotus/chain/actors/builtin/market" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/ipfs/go-cid" @@ -19,7 +25,6 @@ import ( "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/builtin/market" miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" power0 "github.com/filecoin-project/specs-actors/actors/builtin/power" reward0 "github.com/filecoin-project/specs-actors/actors/builtin/reward" @@ -109,7 +114,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid } params := mustEnc(constructorParams) - rval, err := doExecValue(ctx, vm, builtin.StoragePowerActorAddr, m.Owner, m.PowerBalance, builtin.MethodsPower.CreateMiner, params) + rval, err := doExecValue(ctx, vm, power.Address, m.Owner, m.PowerBalance, builtin.MethodsPower.CreateMiner, params) if err != nil { return cid.Undef, xerrors.Errorf("failed to create genesis miner: %w", err) } @@ -141,7 +146,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid if m.MarketBalance.GreaterThan(big.Zero()) { params := mustEnc(&minerInfos[i].maddr) - _, err := doExecValue(ctx, vm, builtin.StorageMarketActorAddr, m.Worker, m.MarketBalance, builtin.MethodsMarket.AddBalance, params) + _, err := doExecValue(ctx, vm, market.Address, m.Worker, m.MarketBalance, builtin.MethodsMarket.AddBalance, params) if err != nil { return cid.Undef, xerrors.Errorf("failed to create genesis miner (add balance): %w", err) } @@ -153,7 +158,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid publish := func(params *market.PublishStorageDealsParams) error { fmt.Printf("publishing %d storage deals on miner %s with worker %s\n", len(params.Deals), params.Deals[0].Proposal.Provider, m.Worker) - ret, err := doExecValue(ctx, vm, builtin.StorageMarketActorAddr, m.Worker, big.Zero(), builtin.MethodsMarket.PublishStorageDeals, mustEnc(params)) + ret, err := doExecValue(ctx, vm, market.Address, m.Worker, big.Zero(), builtin.MethodsMarket.PublishStorageDeals, mustEnc(params)) if err != nil { return xerrors.Errorf("failed to create genesis miner (publish deals): %w", err) } @@ -210,7 +215,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid } } - err = vm.MutateState(ctx, builtin.StoragePowerActorAddr, func(cst cbor.IpldStore, st *power0.State) error { + err = vm.MutateState(ctx, power.Address, func(cst cbor.IpldStore, st *power0.State) error { st.TotalQualityAdjPower = qaPow st.TotalRawBytePower = rawPow @@ -222,7 +227,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid return cid.Undef, xerrors.Errorf("mutating state: %w", err) } - err = vm.MutateState(ctx, builtin.RewardActorAddr, func(sct cbor.IpldStore, st *reward0.State) error { + err = vm.MutateState(ctx, reward.Address, func(sct cbor.IpldStore, st *reward0.State) error { *st = *reward0.ConstructState(qaPow) return nil }) @@ -252,7 +257,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid sectorWeight := miner0.QAPowerForWeight(m.SectorSize, minerInfos[i].presealExp, dweight.DealWeight, dweight.VerifiedDealWeight) // we've added fake power for this sector above, remove it now - err = vm.MutateState(ctx, builtin.StoragePowerActorAddr, func(cst cbor.IpldStore, st *power0.State) error { + err = vm.MutateState(ctx, power.Address, func(cst cbor.IpldStore, st *power0.State) error { st.TotalQualityAdjPower = types.BigSub(st.TotalQualityAdjPower, sectorWeight) //nolint:scopelint st.TotalRawBytePower = types.BigSub(st.TotalRawBytePower, types.NewInt(uint64(m.SectorSize))) return nil @@ -295,7 +300,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid Sectors: []abi.SectorNumber{preseal.SectorID}, } - _, err = doExecValue(ctx, vm, minerInfos[i].maddr, builtin.StoragePowerActorAddr, big.Zero(), builtin.MethodsMiner.ConfirmSectorProofsValid, mustEnc(confirmParams)) + _, err = doExecValue(ctx, vm, minerInfos[i].maddr, power.Address, big.Zero(), builtin.MethodsMiner.ConfirmSectorProofsValid, mustEnc(confirmParams)) if err != nil { return cid.Undef, xerrors.Errorf("failed to confirm presealed sectors: %w", err) } @@ -304,7 +309,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid } // Sanity-check total network power - err = vm.MutateState(ctx, builtin.StoragePowerActorAddr, func(cst cbor.IpldStore, st *power0.State) error { + err = vm.MutateState(ctx, power.Address, func(cst cbor.IpldStore, st *power0.State) error { if !st.TotalRawBytePower.Equals(rawPow) { return xerrors.Errorf("st.TotalRawBytePower doesn't match previously calculated rawPow") } @@ -344,7 +349,7 @@ func (fr *fakeRand) GetBeaconRandomness(ctx context.Context, personalization cry } func currentTotalPower(ctx context.Context, vm *vm.VM, maddr address.Address) (*power0.CurrentTotalPowerReturn, error) { - pwret, err := doExecValue(ctx, vm, builtin.StoragePowerActorAddr, maddr, big.Zero(), builtin.MethodsPower.CurrentTotalPower, nil) + pwret, err := doExecValue(ctx, vm, power.Address, maddr, big.Zero(), builtin.MethodsPower.CurrentTotalPower, nil) if err != nil { return nil, err } @@ -356,33 +361,33 @@ func currentTotalPower(ctx context.Context, vm *vm.VM, maddr address.Address) (* return &pwr, nil } -func dealWeight(ctx context.Context, vm *vm.VM, maddr address.Address, dealIDs []abi.DealID, sectorStart, sectorExpiry abi.ChainEpoch) (market.VerifyDealsForActivationReturn, error) { +func dealWeight(ctx context.Context, vm *vm.VM, maddr address.Address, dealIDs []abi.DealID, sectorStart, sectorExpiry abi.ChainEpoch) (market0.VerifyDealsForActivationReturn, error) { params := &market.VerifyDealsForActivationParams{ DealIDs: dealIDs, SectorStart: sectorStart, SectorExpiry: sectorExpiry, } - var dealWeights market.VerifyDealsForActivationReturn + var dealWeights market0.VerifyDealsForActivationReturn ret, err := doExecValue(ctx, vm, - builtin.StorageMarketActorAddr, + market.Address, maddr, abi.NewTokenAmount(0), builtin.MethodsMarket.VerifyDealsForActivation, mustEnc(params), ) if err != nil { - return market.VerifyDealsForActivationReturn{}, err + return market0.VerifyDealsForActivationReturn{}, err } if err := dealWeights.UnmarshalCBOR(bytes.NewReader(ret)); err != nil { - return market.VerifyDealsForActivationReturn{}, err + return market0.VerifyDealsForActivationReturn{}, err } return dealWeights, nil } func currentEpochBlockReward(ctx context.Context, vm *vm.VM, maddr address.Address) (*reward0.ThisEpochRewardReturn, error) { - rwret, err := doExecValue(ctx, vm, builtin.RewardActorAddr, maddr, big.Zero(), builtin.MethodsReward.ThisEpochReward, nil) + rwret, err := doExecValue(ctx, vm, reward.Address, maddr, big.Zero(), builtin.MethodsReward.ThisEpochReward, nil) if err != nil { return nil, err } diff --git a/chain/market/fundmgr.go b/chain/market/fundmgr.go index edf73d9bd..aef3b98eb 100644 --- a/chain/market/fundmgr.go +++ b/chain/market/fundmgr.go @@ -15,6 +15,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/actors/builtin/market" "github.com/filecoin-project/lotus/chain/events" "github.com/filecoin-project/lotus/chain/events/state" "github.com/filecoin-project/lotus/chain/types" @@ -151,7 +152,7 @@ func (fm *FundMgr) EnsureAvailable(ctx context.Context, addr, wallet address.Add } smsg, err := fm.api.MpoolPushMessage(ctx, &types.Message{ - To: builtin.StorageMarketActorAddr, + To: market.Address, From: wallet, Value: toAdd, Method: builtin.MethodsMarket.AddBalance, diff --git a/chain/market/fundmgr_test.go b/chain/market/fundmgr_test.go index c0e69c51c..b05db55d8 100644 --- a/chain/market/fundmgr_test.go +++ b/chain/market/fundmgr_test.go @@ -17,6 +17,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/actors/builtin/market" "github.com/filecoin-project/lotus/chain/types" ) @@ -47,7 +48,7 @@ func (fapi *fakeAPI) MpoolPushMessage(ctx context.Context, msg *types.Message, s func addFundsMsg(toAdd abi.TokenAmount, addr address.Address, wallet address.Address) *types.Message { params, _ := actors.SerializeParams(&addr) return &types.Message{ - To: builtin.StorageMarketActorAddr, + To: market.Address, From: wallet, Value: toAdd, Method: builtin.MethodsMarket.AddBalance, diff --git a/chain/stmgr/forks_test.go b/chain/stmgr/forks_test.go index 87d328ce1..8c6d2ce40 100644 --- a/chain/stmgr/forks_test.go +++ b/chain/stmgr/forks_test.go @@ -19,6 +19,7 @@ import ( "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/aerrors" + lotusinit "github.com/filecoin-project/lotus/chain/actors/builtin/init" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/stmgr" . "github.com/filecoin-project/lotus/chain/stmgr" @@ -168,7 +169,7 @@ func TestForkHeightTriggers(t *testing.T) { m := &types.Message{ From: cg.Banker(), - To: builtin.InitActorAddr, + To: lotusinit.Address, Method: builtin.MethodsInit.Exec, Params: enc, GasLimit: types.TestGasLimit, diff --git a/chain/store/weight.go b/chain/store/weight.go index 2d83738c5..9100df315 100644 --- a/chain/store/weight.go +++ b/chain/store/weight.go @@ -10,7 +10,6 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/specs-actors/actors/builtin" cbor "github.com/ipfs/go-ipld-cbor" "golang.org/x/xerrors" ) @@ -35,7 +34,7 @@ func (cs *ChainStore) Weight(ctx context.Context, ts *types.TipSet) (types.BigIn return types.NewInt(0), xerrors.Errorf("load state tree: %w", err) } - act, err := state.GetActor(builtin.StoragePowerActorAddr) + act, err := state.GetActor(power.Address) if err != nil { return types.NewInt(0), xerrors.Errorf("get power actor: %w", err) } diff --git a/chain/sync.go b/chain/sync.go index 972321787..f641f078b 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -33,7 +33,6 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" - "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/util/adt" blst "github.com/supranational/blst/bindings/go" @@ -640,7 +639,7 @@ func (syncer *Syncer) ValidateTipSet(ctx context.Context, fts *store.FullTipSet) } func (syncer *Syncer) minerIsValid(ctx context.Context, maddr address.Address, baseTs *types.TipSet) error { - act, err := syncer.sm.LoadActor(ctx, builtin.StoragePowerActorAddr, baseTs) + act, err := syncer.sm.LoadActor(ctx, power.Address, baseTs) if err != nil { return xerrors.Errorf("failed to load power actor: %w", err) } diff --git a/chain/vm/vm.go b/chain/vm/vm.go index 049918fba..35ece7c57 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -28,6 +28,7 @@ import ( "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/actors/aerrors" "github.com/filecoin-project/lotus/chain/actors/builtin/account" + "github.com/filecoin-project/lotus/chain/actors/builtin/reward" "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/blockstore" @@ -493,7 +494,7 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, return nil, xerrors.Errorf("failed to burn base fee: %w", err) } - if err := vm.transferFromGasHolder(builtin.RewardActorAddr, gasHolder, gasOutputs.MinerTip); err != nil { + if err := vm.transferFromGasHolder(reward.Address, gasHolder, gasOutputs.MinerTip); err != nil { return nil, xerrors.Errorf("failed to give miner gas reward: %w", err) } diff --git a/paychmgr/paychget_test.go b/paychmgr/paychget_test.go index 93233c54f..430e66c67 100644 --- a/paychmgr/paychget_test.go +++ b/paychmgr/paychget_test.go @@ -19,6 +19,7 @@ import ( init_ "github.com/filecoin-project/specs-actors/actors/builtin/init" tutils "github.com/filecoin-project/specs-actors/support/testing" + lotusinit "github.com/filecoin-project/lotus/chain/actors/builtin/init" "github.com/filecoin-project/lotus/chain/actors/builtin/paych" paychmock "github.com/filecoin-project/lotus/chain/actors/builtin/paych/mock" "github.com/filecoin-project/lotus/chain/types" @@ -60,7 +61,7 @@ func TestPaychGetCreateChannelMsg(t *testing.T) { pushedMsg := mock.pushedMessages(mcid) require.Equal(t, from, pushedMsg.Message.From) - require.Equal(t, builtin.InitActorAddr, pushedMsg.Message.To) + require.Equal(t, lotusinit.Address, pushedMsg.Message.To) require.Equal(t, amt, pushedMsg.Message.Value) } @@ -712,7 +713,7 @@ func TestPaychGetMergeAddFunds(t *testing.T) { // Check create message amount is correct createMsg := mock.pushedMessages(createMsgCid) require.Equal(t, from, createMsg.Message.From) - require.Equal(t, builtin.InitActorAddr, createMsg.Message.To) + require.Equal(t, lotusinit.Address, createMsg.Message.To) require.Equal(t, createAmt, createMsg.Message.Value) // Check merged add funds amount is the sum of the individual @@ -808,7 +809,7 @@ func TestPaychGetMergeAddFundsCtxCancelOne(t *testing.T) { // Check create message amount is correct createMsg := mock.pushedMessages(createMsgCid) require.Equal(t, from, createMsg.Message.From) - require.Equal(t, builtin.InitActorAddr, createMsg.Message.To) + require.Equal(t, lotusinit.Address, createMsg.Message.To) require.Equal(t, createAmt, createMsg.Message.Value) // Check merged add funds amount only includes the second add funds amount @@ -890,7 +891,7 @@ func TestPaychGetMergeAddFundsCtxCancelAll(t *testing.T) { // Check create message amount is correct createMsg := mock.pushedMessages(createMsgCid) require.Equal(t, from, createMsg.Message.From) - require.Equal(t, builtin.InitActorAddr, createMsg.Message.To) + require.Equal(t, lotusinit.Address, createMsg.Message.To) require.Equal(t, createAmt, createMsg.Message.Value) } diff --git a/paychmgr/simple.go b/paychmgr/simple.go index d2a42a673..d49ccafe6 100644 --- a/paychmgr/simple.go +++ b/paychmgr/simple.go @@ -19,6 +19,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" + lotusinit "github.com/filecoin-project/lotus/chain/actors/builtin/init" "github.com/filecoin-project/lotus/chain/types" ) @@ -400,7 +401,7 @@ func (ca *channelAccessor) createPaych(ctx context.Context, amt types.BigInt) (c } msg := &types.Message{ - To: builtin.InitActorAddr, + To: lotusinit.Address, From: ca.from, Value: amt, Method: builtin.MethodsInit.Exec, From a4ade158f4e065b8729fa140fb1cc553391d7d03 Mon Sep 17 00:00:00 2001 From: zgfzgf <1901989065@qq.com> Date: Wed, 23 Sep 2020 20:24:19 +0800 Subject: [PATCH 300/885] change minerStop to minerLoop --- miner/miner.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/miner/miner.go b/miner/miner.go index 144b2f413..3b1e65a72 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -149,9 +149,8 @@ func (m *Miner) mine(ctx context.Context) { defer span.End() var lastBase MiningBase - +minerLoop: for { - minerStop: select { case <-m.stop: stopping := m.stopping @@ -172,7 +171,7 @@ func (m *Miner) mine(ctx context.Context) { if err != nil { log.Errorf("failed to get best mining candidate: %s", err) if !m.niceSleep(time.Second * 5) { - goto minerStop + goto minerLoop } continue } @@ -204,7 +203,7 @@ func (m *Miner) mine(ctx context.Context) { if err != nil { log.Errorf("failed getting beacon entry: %s", err) if !m.niceSleep(time.Second) { - goto minerStop + goto minerLoop } continue } @@ -215,7 +214,7 @@ func (m *Miner) mine(ctx context.Context) { if base.TipSet.Equals(lastBase.TipSet) && lastBase.NullRounds == base.NullRounds { log.Warnf("BestMiningCandidate from the previous round: %s (nulls:%d)", lastBase.TipSet.Cids(), lastBase.NullRounds) if !m.niceSleep(time.Duration(build.BlockDelaySecs) * time.Second) { - goto minerStop + goto minerLoop } continue } @@ -226,7 +225,7 @@ func (m *Miner) mine(ctx context.Context) { if err != nil { log.Errorf("mining block failed: %+v", err) if !m.niceSleep(time.Second) { - goto minerStop + goto minerLoop } onDone(false, 0, err) continue From 91a43c477c5ec8c8911ada49cbe5485b49273472 Mon Sep 17 00:00:00 2001 From: jennijuju Date: Mon, 21 Sep 2020 21:41:17 -0400 Subject: [PATCH 301/885] When doing `sectors update-state`, show a list of existing states if user inputs an invalid one. --- cmd/lotus-storage-miner/sectors.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/cmd/lotus-storage-miner/sectors.go b/cmd/lotus-storage-miner/sectors.go index 27a5c31be..370962bdc 100644 --- a/cmd/lotus-storage-miner/sectors.go +++ b/cmd/lotus-storage-miner/sectors.go @@ -404,8 +404,9 @@ var sectorsCapacityCollateralCmd = &cli.Command{ } var sectorsUpdateCmd = &cli.Command{ - Name: "update-state", - Usage: "ADVANCED: manually update the state of a sector, this may aid in error recovery", + Name: "update-state", + Usage: "ADVANCED: manually update the state of a sector, this may aid in error recovery", + ArgsUsage: " ", Flags: []cli.Flag{ &cli.BoolFlag{ Name: "really-do-it", @@ -431,8 +432,13 @@ var sectorsUpdateCmd = &cli.Command{ return xerrors.Errorf("could not parse sector number: %w", err) } - if _, ok := sealing.ExistSectorStateList[sealing.SectorState(cctx.Args().Get(1))]; !ok { - return xerrors.Errorf("Not existing sector state") + newState := cctx.Args().Get(1) + if _, ok := sealing.ExistSectorStateList[sealing.SectorState(newState)]; !ok { + fmt.Printf(" \"%s\" is not a valid state. Possible states for sectors are: \n", newState) + for state := range sealing.ExistSectorStateList { + fmt.Printf("%s\n", string(state)) + } + return nil } return nodeApi.SectorsUpdate(ctx, abi.SectorNumber(id), api.SectorState(cctx.Args().Get(1))) From f1ab1af6170e822803073f9ae74ad20d00fdda43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Wed, 23 Sep 2020 17:42:01 +0100 Subject: [PATCH 302/885] add init.State#Remove() for testing. --- chain/actors/builtin/init/init.go | 5 +++++ chain/actors/builtin/init/v0.go | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/chain/actors/builtin/init/init.go b/chain/actors/builtin/init/init.go index 1164891f8..f235450c2 100644 --- a/chain/actors/builtin/init/init.go +++ b/chain/actors/builtin/init/init.go @@ -36,4 +36,9 @@ type State interface { NetworkName() (dtypes.NetworkName, error) ForEachActor(func(id abi.ActorID, address address.Address) error) error + + // Remove exists to support tooling that manipulates state for testing. + // It should not be used in production code, as init actor entries are + // immutable. + Remove(addrs ...address.Address) error } diff --git a/chain/actors/builtin/init/v0.go b/chain/actors/builtin/init/v0.go index 717ed9669..425ba654c 100644 --- a/chain/actors/builtin/init/v0.go +++ b/chain/actors/builtin/init/v0.go @@ -4,6 +4,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" cbg "github.com/whyrusleeping/cbor-gen" + "golang.org/x/xerrors" init_ "github.com/filecoin-project/specs-actors/actors/builtin/init" @@ -46,3 +47,21 @@ func (s *state0) ForEachActor(cb func(id abi.ActorID, address address.Address) e func (s *state0) NetworkName() (dtypes.NetworkName, error) { return dtypes.NetworkName(s.State.NetworkName), nil } + +func (s *state0) Remove(addrs ...address.Address) (err error) { + m, err := adt0.AsMap(s.store, s.State.AddressMap) + if err != nil { + return err + } + for _, addr := range addrs { + if err = m.Delete(abi.AddrKey(addr)); err != nil { + return xerrors.Errorf("failed to delete entry for address: %s; err: %w", addr, err) + } + } + amr, err := m.Root() + if err != nil { + return xerrors.Errorf("failed to get address map root: %w", err) + } + s.State.AddressMap = amr + return nil +} From 6c5ed3f07f287e375c171818f001f13e5f0dd7ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 23 Sep 2020 19:31:36 +0200 Subject: [PATCH 303/885] Some safeguards on chain delete-obj --- cli/chain.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/cli/chain.go b/cli/chain.go index 0f1e36518..bd72c2030 100644 --- a/cli/chain.go +++ b/cli/chain.go @@ -195,9 +195,15 @@ var chainReadObjCmd = &cli.Command{ } var chainDeleteObjCmd = &cli.Command{ - Name: "delete-obj", - Usage: "Delete an object", - ArgsUsage: "[objectCid]", + Name: "delete-obj", + Usage: "Delete an object from the chain blockstore", + Description: "WARNING: Removing wrong objects from the chain blockstore may lead to sync issues", + ArgsUsage: "[objectCid]", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "really-do-it", + }, + }, Action: func(cctx *cli.Context) error { api, closer, err := GetFullNodeAPI(cctx) if err != nil { @@ -211,6 +217,10 @@ var chainDeleteObjCmd = &cli.Command{ return fmt.Errorf("failed to parse cid input: %s", err) } + if !cctx.Bool("really-do-it") { + return xerrors.Errorf("pass the --really-do-it flag to proceed") + } + err = api.ChainDeleteObj(ctx, c) if err != nil { return err From 3fc791b0e8c1c12b2d729123e6cf3bb9386a1eca Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Wed, 23 Sep 2020 12:16:26 -0700 Subject: [PATCH 304/885] feat(markets): update markets v0.6.2 --- go.mod | 10 +++++----- go.sum | 31 ++++++++++++++----------------- node/impl/client/client.go | 2 +- 3 files changed, 20 insertions(+), 23 deletions(-) diff --git a/go.mod b/go.mod index 6706a3eda..cfdf4cb1d 100644 --- a/go.mod +++ b/go.mod @@ -25,9 +25,9 @@ require ( github.com/filecoin-project/go-bitfield v0.2.0 github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 - github.com/filecoin-project/go-data-transfer v0.6.4 + github.com/filecoin-project/go-data-transfer v0.6.5 github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f - github.com/filecoin-project/go-fil-markets v0.6.1-0.20200917052354-ee0af754c6e9 + github.com/filecoin-project/go-fil-markets v0.6.2 github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200822201400-474f4fdccc52 github.com/filecoin-project/go-multistore v0.0.3 github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 @@ -60,7 +60,7 @@ require ( github.com/ipfs/go-ds-measure v0.1.0 github.com/ipfs/go-filestore v1.0.0 github.com/ipfs/go-fs-lock v0.0.6 - github.com/ipfs/go-graphsync v0.1.2 + github.com/ipfs/go-graphsync v0.2.0 github.com/ipfs/go-ipfs-blockstore v1.0.1 github.com/ipfs/go-ipfs-chunker v0.0.5 github.com/ipfs/go-ipfs-ds-help v1.0.0 @@ -77,8 +77,8 @@ require ( github.com/ipfs/go-path v0.0.7 github.com/ipfs/go-unixfs v0.2.4 github.com/ipfs/interface-go-ipfs-core v0.2.3 - github.com/ipld/go-car v0.1.1-0.20200526133713-1c7508d55aae - github.com/ipld/go-ipld-prime v0.0.4-0.20200828224805-5ff8c8b0b6ef + github.com/ipld/go-car v0.1.1-0.20200923150018-8cdef32e2da4 + github.com/ipld/go-ipld-prime v0.5.1-0.20200828233916-988837377a7f github.com/kelseyhightower/envconfig v1.4.0 github.com/lib/pq v1.7.0 github.com/libp2p/go-eventbus v0.2.1 diff --git a/go.sum b/go.sum index e62cb92c9..3b31e6c3b 100644 --- a/go.sum +++ b/go.sum @@ -222,13 +222,12 @@ github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:a github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= -github.com/filecoin-project/go-data-transfer v0.6.3/go.mod h1:PmBKVXkhh67/tnEdJXQwDHl5mT+7Tbcwe1NPninqhnM= -github.com/filecoin-project/go-data-transfer v0.6.4 h1:Q08ABa+cOTOLoAyHeA94fPLcwu53p6eeAaxMxQb0m0A= -github.com/filecoin-project/go-data-transfer v0.6.4/go.mod h1:PmBKVXkhh67/tnEdJXQwDHl5mT+7Tbcwe1NPninqhnM= +github.com/filecoin-project/go-data-transfer v0.6.5 h1:oP20la8Z0CLrw0uqvt6xVgw6rOevZeGJ9GNQeC0OCSU= +github.com/filecoin-project/go-data-transfer v0.6.5/go.mod h1:I9Ylb/UiZyqnI41wUoCXq/le0nDLhlwpFQCtNPxEPOA= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f h1:GxJzR3oRIMTPtpZ0b7QF8FKPK6/iPAc7trhlL5k/g+s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= -github.com/filecoin-project/go-fil-markets v0.6.1-0.20200917052354-ee0af754c6e9 h1:SnCUC9wHDId9TtV8PsQp8q1OOsi+NOLOwitIDnAgUa4= -github.com/filecoin-project/go-fil-markets v0.6.1-0.20200917052354-ee0af754c6e9/go.mod h1:PLr9svZxsnHkae1Ky7+66g7fP9AlneVxIVu+oSMq56A= +github.com/filecoin-project/go-fil-markets v0.6.2 h1:9Z57KeaQSa1liCmT1pH6SIjrn9mGTDFJXmR2WQVuaiY= +github.com/filecoin-project/go-fil-markets v0.6.2/go.mod h1:wtN4Hc/1hoVCpWhSWYxwUxH3PQtjSkWWuC1nQjiIWog= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24= @@ -396,9 +395,8 @@ github.com/gxed/go-shellwords v1.0.3/go.mod h1:N7paucT91ByIjmVJHhvoarjoQnmsi3Jd3 github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= github.com/gxed/pubsub v0.0.0-20180201040156-26ebdf44f824/go.mod h1:OiEWyHgK+CWrmOlVquHaIK1vhpUJydC9m0Je6mhaiNE= -github.com/hannahhoward/cbor-gen-for v0.0.0-20191218204337-9ab7b1bcc099/go.mod h1:WVPCl0HO/0RAL5+vBH2GMxBomlxBF70MAS78+Lu1//k= -github.com/hannahhoward/cbor-gen-for v0.0.0-20200723175505-5892b522820a h1:wfqh5oiHXvn3Rk54xy8Cwqh+HnYihGnjMNzdNb3/ld0= -github.com/hannahhoward/cbor-gen-for v0.0.0-20200723175505-5892b522820a/go.mod h1:jvfsLIxk0fY/2BKSQ1xf2406AKA5dwMmKKv0ADcOfN8= +github.com/hannahhoward/cbor-gen-for v0.0.0-20200817222906-ea96cece81f1 h1:F9k+7wv5OIk1zcq23QpdiL0hfDuXPjuOmMNaC6fgQ0Q= +github.com/hannahhoward/cbor-gen-for v0.0.0-20200817222906-ea96cece81f1/go.mod h1:jvfsLIxk0fY/2BKSQ1xf2406AKA5dwMmKKv0ADcOfN8= github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e h1:3YKHER4nmd7b5qy5t0GWDTwSn4OyRgfAXSmo6VnryBY= github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e/go.mod h1:I8h3MITA53gN9OnWGCgaMa0JWVRdXthWw4M3CPM54OY= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= @@ -504,8 +502,8 @@ github.com/ipfs/go-filestore v1.0.0/go.mod h1:/XOCuNtIe2f1YPbiXdYvD0BKLA0JR1MgPi github.com/ipfs/go-fs-lock v0.0.6 h1:sn3TWwNVQqSeNjlWy6zQ1uUGAZrV3hPOyEA6y1/N2a0= github.com/ipfs/go-fs-lock v0.0.6/go.mod h1:OTR+Rj9sHiRubJh3dRhD15Juhd/+w6VPOY28L7zESmM= github.com/ipfs/go-graphsync v0.1.0/go.mod h1:jMXfqIEDFukLPZHqDPp8tJMbHO9Rmeb9CEGevngQbmE= -github.com/ipfs/go-graphsync v0.1.2 h1:25Ll9kIXCE+DY0dicvfS3KMw+U5sd01b/FJbA7KAbhg= -github.com/ipfs/go-graphsync v0.1.2/go.mod h1:sLXVXm1OxtE2XYPw62MuXCdAuNwkAdsbnfrmos5odbA= +github.com/ipfs/go-graphsync v0.2.0 h1:x94MvHLNuRwBlZzVal7tR1RYK7T7H6bqQLPopxDbIF0= +github.com/ipfs/go-graphsync v0.2.0/go.mod h1:gEBvJUNelzMkaRPJTpg/jaKN4AQW/7wDWu0K92D8o10= github.com/ipfs/go-hamt-ipld v0.1.1/go.mod h1:1EZCr2v0jlCnhpa+aZ0JZYp8Tt2w16+JJOAVz17YcDk= github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw= @@ -609,14 +607,14 @@ github.com/ipfs/iptb v1.4.0 h1:YFYTrCkLMRwk/35IMyC6+yjoQSHTEcNcefBStLJzgvo= github.com/ipfs/iptb v1.4.0/go.mod h1:1rzHpCYtNp87/+hTxG5TfCVn/yMY3dKnLn8tBiMfdmg= github.com/ipfs/iptb-plugins v0.2.1 h1:au4HWn9/pRPbkxA08pDx2oRAs4cnbgQWgV0teYXuuGA= github.com/ipfs/iptb-plugins v0.2.1/go.mod h1:QXMbtIWZ+jRsW8a4h13qAKU7jcM7qaittO8wOsTP0Rs= -github.com/ipld/go-car v0.1.1-0.20200526133713-1c7508d55aae h1:OV9dxl8iPMCOD8Vi/hvFwRh3JWPXqmkYSVxWr9JnEzM= -github.com/ipld/go-car v0.1.1-0.20200526133713-1c7508d55aae/go.mod h1:2mvxpu4dKRnuH3mj5u6KW/tmRSCcXvy/KYiJ4nC6h4c= +github.com/ipld/go-car v0.1.1-0.20200923150018-8cdef32e2da4 h1:6phjU3kXvCEWOZpu+Ob0w6DzgPFZmDLgLPxJhD8RxEY= +github.com/ipld/go-car v0.1.1-0.20200923150018-8cdef32e2da4/go.mod h1:xrMEcuSq+D1vEwl+YAXsg/JfA98XGpXDwnkIL4Aimqw= github.com/ipld/go-ipld-prime v0.0.2-0.20200428162820-8b59dc292b8e/go.mod h1:uVIwe/u0H4VdKv3kaN1ck7uCb6yD9cFLS9/ELyXbsw8= -github.com/ipld/go-ipld-prime v0.0.4-0.20200828224805-5ff8c8b0b6ef h1:/yPelt/0CuzZsmRkYzBBnJ499JnAOGaIaAXHujx96ic= -github.com/ipld/go-ipld-prime v0.0.4-0.20200828224805-5ff8c8b0b6ef/go.mod h1:uVIwe/u0H4VdKv3kaN1ck7uCb6yD9cFLS9/ELyXbsw8= +github.com/ipld/go-ipld-prime v0.5.1-0.20200828233916-988837377a7f h1:XpOuNQ5GbXxUcSukbQcW9jkE7REpaFGJU2/T00fo9kA= +github.com/ipld/go-ipld-prime v0.5.1-0.20200828233916-988837377a7f/go.mod h1:0xEgdD6MKbZ1vF0GC+YcR/C4SQCAlRuOjIJ2i0HxqzM= github.com/ipld/go-ipld-prime-proto v0.0.0-20200428191222-c1ffdadc01e1/go.mod h1:OAV6xBmuTLsPZ+epzKkPB1e25FHk/vCtyatkdHcArLs= -github.com/ipld/go-ipld-prime-proto v0.0.0-20200828231332-ae0aea07222b h1:ZtlW6pubN17TDaStlxgrwEXXwwUfJaXu9RobwczXato= -github.com/ipld/go-ipld-prime-proto v0.0.0-20200828231332-ae0aea07222b/go.mod h1:OAV6xBmuTLsPZ+epzKkPB1e25FHk/vCtyatkdHcArLs= +github.com/ipld/go-ipld-prime-proto v0.0.0-20200922192210-9a2bfd4440a6 h1:6Mq+tZGSEMEoJJ1NbJRhddeelkXZcU8yfH/ZRYUo/Es= +github.com/ipld/go-ipld-prime-proto v0.0.0-20200922192210-9a2bfd4440a6/go.mod h1:3pHYooM9Ea65jewRwrb2u5uHZCNkNTe9ABsVB+SrkH0= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52/go.mod h1:fdg+/X9Gg4AsAIzWpEHwnqd+QY3b7lajxyjE1m4hkq4= github.com/jackpal/gateway v1.0.4/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= @@ -1362,7 +1360,6 @@ github.com/weaveworks/promrus v1.2.0/go.mod h1:SaE82+OJ91yqjrE1rsvBWVzNZKcHYFtMU github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc/go.mod h1:r45hJU7yEoA81k6MWNhpMj/kms0n14dkzkxYHoB96UM= github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba h1:X4n8JG2e2biEZZXdBKt9HX7DN3bYGFUqljqqy0DqgnY= github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba/go.mod h1:CHQnYnQUEPydYCwuy8lmTHfGmdw9TKrhWV0xLx8l0oM= -github.com/whyrusleeping/cbor-gen v0.0.0-20191212224538-d370462a7e8a/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/cbor-gen v0.0.0-20191216205031-b047b6acb3c0/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= github.com/whyrusleeping/cbor-gen v0.0.0-20200402171437-3d27c146c105/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 8b47144af..81978af16 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -711,7 +711,7 @@ func (a *API) ClientGenCar(ctx context.Context, ref api.FileRef, outputPath stri // TODO: does that defer mean to remove the whole blockstore? defer bufferedDS.Remove(ctx, c) //nolint:errcheck - ssb := builder.NewSelectorSpecBuilder(basicnode.Style.Any) + ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) // entire DAG selector allSelector := ssb.ExploreRecursive(selector.RecursionLimitNone(), From 1c1d23d14273dbdca95b6a03537635799ad33c0e Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 23 Sep 2020 12:36:15 -0700 Subject: [PATCH 305/885] fix out-of-bounds when loading all sector infos fixes #3972 --- chain/actors/builtin/miner/v0.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chain/actors/builtin/miner/v0.go b/chain/actors/builtin/miner/v0.go index e515b9ed6..9cdfc25bc 100644 --- a/chain/actors/builtin/miner/v0.go +++ b/chain/actors/builtin/miner/v0.go @@ -186,9 +186,9 @@ func (s *state0) LoadSectors(snos *bitfield.BitField) ([]*SectorOnChainInfo, err if snos == nil { infos := make([]*SectorOnChainInfo, 0, sectors.Length()) var info0 miner0.SectorOnChainInfo - if err := sectors.ForEach(&info0, func(i int64) error { + if err := sectors.ForEach(&info0, func(_ int64) error { info := fromV0SectorOnChainInfo(info0) - infos[i] = &info + infos = append(infos, &info) return nil }); err != nil { return nil, err From a1281273bc20b7911ee7052936a8fda8f2eda776 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 23 Sep 2020 22:10:02 +0200 Subject: [PATCH 306/885] shed dealtracker: fix lint, env var filter --- cmd/lotus-shed/dealtracker.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/cmd/lotus-shed/dealtracker.go b/cmd/lotus-shed/dealtracker.go index 18dc959f7..d39f51bd1 100644 --- a/cmd/lotus-shed/dealtracker.go +++ b/cmd/lotus-shed/dealtracker.go @@ -5,6 +5,8 @@ import ( "encoding/json" "net" "net/http" + "os" + "strings" "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/api" @@ -20,8 +22,15 @@ type dealStatsServer struct { var filteredClients map[address.Address]bool func init() { + fc := []string{"t0112", "t0113", "t0114", "t010089"} + + filtered, set := os.LookupEnv("FILTERED_CLIENTS") + if set { + fc = strings.Split(filtered, ":") + } + filteredClients = make(map[address.Address]bool) - for _, a := range []string{"t0112", "t0113", "t0114", "t010089"} { + for _, a := range fc { addr, err := address.NewFromString(a) if err != nil { panic(err) @@ -35,16 +44,6 @@ type dealCountResp struct { Epoch int64 `json:"epoch"` } -func filterDeals(deals map[string]api.MarketDeal) []*api.MarketDeal { - out := make([]*api.MarketDeal, 0, len(deals)) - for _, d := range deals { - if !filteredClients[d.Proposal.Client] { - out = append(out, &d) - } - } - return out -} - func (dss *dealStatsServer) handleStorageDealCount(w http.ResponseWriter, r *http.Request) { ctx := context.Background() @@ -251,15 +250,16 @@ var serveDealStatsCmd = &cli.Command{ go func() { <-ctx.Done() - s.Shutdown(context.TODO()) + if err := s.Shutdown(context.TODO()); err != nil { + log.Error(err) + } }() - list, err := net.Listen("tcp", ":7272") + list, err := net.Listen("tcp", ":7272") // nolint if err != nil { panic(err) } - s.Serve(list) - return nil + return s.Serve(list) }, } From 46f5b62a761b9e300b455a7da6776dbf0f2cf497 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 23 Sep 2020 12:47:41 -0700 Subject: [PATCH 307/885] Remove a misleading miner actor abstraction We shouldn't implement CBOR functions on the "abstract" miner info. Otherwise, we could end up trying to actually _use_ this "abstract" info when decoding state (which won't work across version). Also remove the use of these CBOR functions, and instead explicitly use miner0 types. We'll have to abstract over versions eventually, but we'll probably need some form of abstract miner builder (or maybe even adding some "add sector", etc. functions to the current miner abstraction? --- chain/actors/builtin/miner/cbor_gen.go | 313 ------------------------- chain/events/state/predicates_test.go | 6 +- gen/main.go | 10 - 3 files changed, 3 insertions(+), 326 deletions(-) delete mode 100644 chain/actors/builtin/miner/cbor_gen.go diff --git a/chain/actors/builtin/miner/cbor_gen.go b/chain/actors/builtin/miner/cbor_gen.go deleted file mode 100644 index 16819d5c4..000000000 --- a/chain/actors/builtin/miner/cbor_gen.go +++ /dev/null @@ -1,313 +0,0 @@ -// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. - -package miner - -import ( - "fmt" - "io" - - abi "github.com/filecoin-project/go-state-types/abi" - cbg "github.com/whyrusleeping/cbor-gen" - xerrors "golang.org/x/xerrors" -) - -var _ = xerrors.Errorf - -var lengthBufSectorOnChainInfo = []byte{139} - -func (t *SectorOnChainInfo) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write(lengthBufSectorOnChainInfo); err != nil { - return err - } - - scratch := make([]byte, 9) - - // t.SectorNumber (abi.SectorNumber) (uint64) - - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.SectorNumber)); err != nil { - return err - } - - // t.SealProof (abi.RegisteredSealProof) (int64) - if t.SealProof >= 0 { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.SealProof)); err != nil { - return err - } - } else { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.SealProof-1)); err != nil { - return err - } - } - - // t.SealedCID (cid.Cid) (struct) - - if err := cbg.WriteCidBuf(scratch, w, t.SealedCID); err != nil { - return xerrors.Errorf("failed to write cid field t.SealedCID: %w", err) - } - - // t.DealIDs ([]abi.DealID) (slice) - if len(t.DealIDs) > cbg.MaxLength { - return xerrors.Errorf("Slice value in field t.DealIDs was too long") - } - - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.DealIDs))); err != nil { - return err - } - for _, v := range t.DealIDs { - if err := cbg.CborWriteHeader(w, cbg.MajUnsignedInt, uint64(v)); err != nil { - return err - } - } - - // t.Activation (abi.ChainEpoch) (int64) - if t.Activation >= 0 { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Activation)); err != nil { - return err - } - } else { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.Activation-1)); err != nil { - return err - } - } - - // t.Expiration (abi.ChainEpoch) (int64) - if t.Expiration >= 0 { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Expiration)); err != nil { - return err - } - } else { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.Expiration-1)); err != nil { - return err - } - } - - // t.DealWeight (big.Int) (struct) - if err := t.DealWeight.MarshalCBOR(w); err != nil { - return err - } - - // t.VerifiedDealWeight (big.Int) (struct) - if err := t.VerifiedDealWeight.MarshalCBOR(w); err != nil { - return err - } - - // t.InitialPledge (big.Int) (struct) - if err := t.InitialPledge.MarshalCBOR(w); err != nil { - return err - } - - // t.ExpectedDayReward (big.Int) (struct) - if err := t.ExpectedDayReward.MarshalCBOR(w); err != nil { - return err - } - - // t.ExpectedStoragePledge (big.Int) (struct) - if err := t.ExpectedStoragePledge.MarshalCBOR(w); err != nil { - return err - } - return nil -} - -func (t *SectorOnChainInfo) UnmarshalCBOR(r io.Reader) error { - *t = SectorOnChainInfo{} - - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) - - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 11 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.SectorNumber (abi.SectorNumber) (uint64) - - { - - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.SectorNumber = abi.SectorNumber(extra) - - } - // t.SealProof (abi.RegisteredSealProof) (int64) - { - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) - var extraI int64 - if err != nil { - return err - } - switch maj { - case cbg.MajUnsignedInt: - extraI = int64(extra) - if extraI < 0 { - return fmt.Errorf("int64 positive overflow") - } - case cbg.MajNegativeInt: - extraI = int64(extra) - if extraI < 0 { - return fmt.Errorf("int64 negative oveflow") - } - extraI = -1 - extraI - default: - return fmt.Errorf("wrong type for int64 field: %d", maj) - } - - t.SealProof = abi.RegisteredSealProof(extraI) - } - // t.SealedCID (cid.Cid) (struct) - - { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.SealedCID: %w", err) - } - - t.SealedCID = c - - } - // t.DealIDs ([]abi.DealID) (slice) - - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) - if err != nil { - return err - } - - if extra > cbg.MaxLength { - return fmt.Errorf("t.DealIDs: array too large (%d)", extra) - } - - if maj != cbg.MajArray { - return fmt.Errorf("expected cbor array") - } - - if extra > 0 { - t.DealIDs = make([]abi.DealID, extra) - } - - for i := 0; i < int(extra); i++ { - - maj, val, err := cbg.CborReadHeaderBuf(br, scratch) - if err != nil { - return xerrors.Errorf("failed to read uint64 for t.DealIDs slice: %w", err) - } - - if maj != cbg.MajUnsignedInt { - return xerrors.Errorf("value read for array t.DealIDs was not a uint, instead got %d", maj) - } - - t.DealIDs[i] = abi.DealID(val) - } - - // t.Activation (abi.ChainEpoch) (int64) - { - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) - var extraI int64 - if err != nil { - return err - } - switch maj { - case cbg.MajUnsignedInt: - extraI = int64(extra) - if extraI < 0 { - return fmt.Errorf("int64 positive overflow") - } - case cbg.MajNegativeInt: - extraI = int64(extra) - if extraI < 0 { - return fmt.Errorf("int64 negative oveflow") - } - extraI = -1 - extraI - default: - return fmt.Errorf("wrong type for int64 field: %d", maj) - } - - t.Activation = abi.ChainEpoch(extraI) - } - // t.Expiration (abi.ChainEpoch) (int64) - { - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) - var extraI int64 - if err != nil { - return err - } - switch maj { - case cbg.MajUnsignedInt: - extraI = int64(extra) - if extraI < 0 { - return fmt.Errorf("int64 positive overflow") - } - case cbg.MajNegativeInt: - extraI = int64(extra) - if extraI < 0 { - return fmt.Errorf("int64 negative oveflow") - } - extraI = -1 - extraI - default: - return fmt.Errorf("wrong type for int64 field: %d", maj) - } - - t.Expiration = abi.ChainEpoch(extraI) - } - // t.DealWeight (big.Int) (struct) - - { - - if err := t.DealWeight.UnmarshalCBOR(br); err != nil { - return xerrors.Errorf("unmarshaling t.DealWeight: %w", err) - } - - } - // t.VerifiedDealWeight (big.Int) (struct) - - { - - if err := t.VerifiedDealWeight.UnmarshalCBOR(br); err != nil { - return xerrors.Errorf("unmarshaling t.VerifiedDealWeight: %w", err) - } - - } - // t.InitialPledge (big.Int) (struct) - - { - - if err := t.InitialPledge.UnmarshalCBOR(br); err != nil { - return xerrors.Errorf("unmarshaling t.InitialPledge: %w", err) - } - - } - // t.ExpectedDayReward (big.Int) (struct) - - { - - if err := t.ExpectedDayReward.UnmarshalCBOR(br); err != nil { - return xerrors.Errorf("unmarshaling t.ExpectedDayReward: %w", err) - } - - } - // t.ExpectedStoragePledge (big.Int) (struct) - - { - - if err := t.ExpectedStoragePledge.UnmarshalCBOR(br); err != nil { - return xerrors.Errorf("unmarshaling t.ExpectedStoragePledge: %w", err) - } - - } - return nil -} diff --git a/chain/events/state/predicates_test.go b/chain/events/state/predicates_test.go index 832f6a0a7..461ac4997 100644 --- a/chain/events/state/predicates_test.go +++ b/chain/events/state/predicates_test.go @@ -581,7 +581,7 @@ func createEmptyMinerState(ctx context.Context, t *testing.T, store adt.Store, o func createSectorsAMT(ctx context.Context, t *testing.T, store adt.Store, sectors []miner.SectorOnChainInfo) cid.Cid { root := adt.MakeEmptyArray(store) for _, sector := range sectors { - sector := sector + sector := (miner0.SectorOnChainInfo)(sector) err := root.Set(uint64(sector.SectorNumber), §or) require.NoError(t, err) } @@ -614,8 +614,8 @@ const ( ) // returns a unique SectorPreCommitInfo with each invocation with SectorNumber set to `sectorNo`. -func newSectorPreCommitInfo(sectorNo abi.SectorNumber, sealed cid.Cid, expiration abi.ChainEpoch) *miner.SectorPreCommitInfo { - return &miner.SectorPreCommitInfo{ +func newSectorPreCommitInfo(sectorNo abi.SectorNumber, sealed cid.Cid, expiration abi.ChainEpoch) *miner0.SectorPreCommitInfo { + return &miner0.SectorPreCommitInfo{ SealProof: abi.RegisteredSealProof_StackedDrg32GiBV1, SectorNumber: sectorNo, SealedCID: sealed, diff --git a/gen/main.go b/gen/main.go index 90b24e3a7..bcb43a8f0 100644 --- a/gen/main.go +++ b/gen/main.go @@ -4,8 +4,6 @@ import ( "fmt" "os" - "github.com/filecoin-project/lotus/chain/actors/builtin/miner" - gen "github.com/whyrusleeping/cbor-gen" "github.com/filecoin-project/lotus/api" @@ -77,12 +75,4 @@ func main() { fmt.Println(err) os.Exit(1) } - - err = gen.WriteTupleEncodersToFile("./chain/actors/builtin/miner/cbor_gen.go", "miner", - miner.SectorOnChainInfo{}, - ) - if err != nil { - fmt.Println(err) - os.Exit(1) - } } From 32a699d6a35a50eb9fe38af98fa60617ec6d3e27 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 23 Sep 2020 12:24:51 -0700 Subject: [PATCH 308/885] Add some actors policy setters for testing Addresses: * https://github.com/filecoin-project/lotus/pull/3781/files/a307e4593a005888395db34e5af4691ee50d765d#r491966115 * https://github.com/filecoin-project/lotus/pull/3781/files/a307e4593a005888395db34e5af4691ee50d765d#r491966634 Note: This puts everything into a policy package to avoid a dependency cycle between the build package, the miner package, and the types package. This is also why I introduced a GetPreCommitChallengeDelay function and removed the variable. --- build/params_2k.go | 14 +++--- build/params_testnet.go | 19 ++++---- chain/actors/builtin/miner/miner.go | 1 - chain/actors/policy/policy.go | 56 ++++++++++++++++++++++++ chain/gen/gen.go | 7 ++- chain/gen/gen_test.go | 13 ++---- chain/stmgr/forks_test.go | 13 ++---- chain/store/store_test.go | 13 ++---- chain/sync_test.go | 13 ++---- cli/paych_test.go | 48 +++++++++----------- cmd/lotus-bench/main.go | 4 +- extern/storage-sealing/checks.go | 6 +-- extern/storage-sealing/states_sealing.go | 3 +- lotuspond/spawn.go | 6 +-- node/node_test.go | 24 +++++----- 15 files changed, 128 insertions(+), 112 deletions(-) create mode 100644 chain/actors/policy/policy.go diff --git a/build/params_2k.go b/build/params_2k.go index 4a49da22b..3edd0fb82 100644 --- a/build/params_2k.go +++ b/build/params_2k.go @@ -4,10 +4,8 @@ package build import ( "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/big" - miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" - power0 "github.com/filecoin-project/specs-actors/actors/builtin/power" - verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" + + "github.com/filecoin-project/lotus/chain/actors/policy" ) const UpgradeBreezeHeight = -1 @@ -20,11 +18,9 @@ var DrandSchedule = map[abi.ChainEpoch]DrandEnum{ } func init() { - power0.ConsensusMinerMinPower = big.NewInt(2048) - miner0.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ - abi.RegisteredSealProof_StackedDrg2KiBV1: {}, - } - verifreg0.MinVerifiedDealSize = big.NewInt(256) + policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg2KiBV1) + policy.SetConsensusMinerMinPower(abi.NewStoragePower(2048)) + policy.SetMinVerifiedDealSize(abi.NewStoragePower(256)) BuildType |= Build2k } diff --git a/build/params_testnet.go b/build/params_testnet.go index 108aba20c..13d2ff62e 100644 --- a/build/params_testnet.go +++ b/build/params_testnet.go @@ -6,10 +6,9 @@ package build import ( "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/big" - "github.com/filecoin-project/specs-actors/actors/builtin" - miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" - power0 "github.com/filecoin-project/specs-actors/actors/builtin/power" + "github.com/filecoin-project/lotus/chain/actors/policy" + + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" ) var DrandSchedule = map[abi.ChainEpoch]DrandEnum{ @@ -23,13 +22,13 @@ const BreezeGasTampingDuration = 120 const UpgradeSmokeHeight = 51000 func init() { - power0.ConsensusMinerMinPower = big.NewInt(10 << 40) - miner0.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ - abi.RegisteredSealProof_StackedDrg32GiBV1: {}, - abi.RegisteredSealProof_StackedDrg64GiBV1: {}, - } + policy.SetConsensusMinerMinPower(abi.NewStoragePower(10 << 40)) + policy.SetSupportedProofTypes( + abi.RegisteredSealProof_StackedDrg32GiBV1, + abi.RegisteredSealProof_StackedDrg64GiBV1, + ) } -const BlockDelaySecs = uint64(builtin.EpochDurationSeconds) +const BlockDelaySecs = uint64(builtin0.EpochDurationSeconds) const PropagationDelaySecs = uint64(6) diff --git a/chain/actors/builtin/miner/miner.go b/chain/actors/builtin/miner/miner.go index 50a0fc5ca..1a4c466b9 100644 --- a/chain/actors/builtin/miner/miner.go +++ b/chain/actors/builtin/miner/miner.go @@ -19,7 +19,6 @@ import ( ) // Unchanged between v0 and v1 actors -var PreCommitChallengeDelay = miner0.PreCommitChallengeDelay var WPoStProvingPeriod = miner0.WPoStProvingPeriod const MinSectorExpiration = miner0.MinSectorExpiration diff --git a/chain/actors/policy/policy.go b/chain/actors/policy/policy.go new file mode 100644 index 000000000..eec52e855 --- /dev/null +++ b/chain/actors/policy/policy.go @@ -0,0 +1,56 @@ +package policy + +import ( + "github.com/filecoin-project/go-state-types/abi" + + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" + power0 "github.com/filecoin-project/specs-actors/actors/builtin/power" + verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" +) + +// SetSupportedProofTypes sets supported proof types, across all actor versions. +// This should only be used for testing. +func SetSupportedProofTypes(types ...abi.RegisteredSealProof) { + newTypes := make(map[abi.RegisteredSealProof]struct{}, len(types)) + for _, t := range types { + newTypes[t] = struct{}{} + } + // Set for all miner versions. + miner0.SupportedProofTypes = newTypes +} + +// AddSupportedProofTypes sets supported proof types, across all actor versions. +// This should only be used for testing. +func AddSupportedProofTypes(types ...abi.RegisteredSealProof) { + newTypes := make(map[abi.RegisteredSealProof]struct{}, len(types)) + for _, t := range types { + newTypes[t] = struct{}{} + } + // Set for all miner versions. + miner0.SupportedProofTypes = newTypes +} + +// SetPreCommitChallengeDelay sets the pre-commit challenge delay across all +// actors versions. Use for testing. +func SetPreCommitChallengeDelay(delay abi.ChainEpoch) { + // Set for all miner versions. + miner0.PreCommitChallengeDelay = delay +} + +// TODO: this function shouldn't really exist. Instead, the API should expose the precommit delay. +func GetPreCommitChallengeDelay() abi.ChainEpoch { + return miner0.PreCommitChallengeDelay +} + +// SetConsensusMinerMinPower sets the minimum power of an individual miner must +// meet for leader election, across all actor versions. This should only be used +// for testing. +func SetConsensusMinerMinPower(p abi.StoragePower) { + power0.ConsensusMinerMinPower = p +} + +// SetMinVerifiedDealSize sets the minimum size of a verified deal. This should +// only be used for testing. +func SetMinVerifiedDealSize(size abi.StoragePower) { + verifreg0.MinVerifiedDealSize = size +} diff --git a/chain/gen/gen.go b/chain/gen/gen.go index c4ecf1d41..d05165ab1 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -14,7 +14,6 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" - miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" block "github.com/ipfs/go-block-format" "github.com/ipfs/go-blockservice" "github.com/ipfs/go-cid" @@ -28,6 +27,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/beacon" genesis2 "github.com/filecoin-project/lotus/chain/gen/genesis" "github.com/filecoin-project/lotus/chain/stmgr" @@ -121,9 +121,8 @@ var DefaultRemainderAccountActor = genesis.Actor{ } func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) { - miner0.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ - abi.RegisteredSealProof_StackedDrg2KiBV1: {}, - } + // TODO: we really shouldn't modify a global variable here. + policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg2KiBV1) mr := repo.NewMemory(nil) lr, err := mr.Lock(repo.StorageMiner) diff --git a/chain/gen/gen_test.go b/chain/gen/gen_test.go index be913f5f2..8c38328d0 100644 --- a/chain/gen/gen_test.go +++ b/chain/gen/gen_test.go @@ -4,21 +4,16 @@ import ( "testing" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/big" - miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" - power0 "github.com/filecoin-project/specs-actors/actors/builtin/power" - verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" + "github.com/filecoin-project/lotus/chain/actors/policy" _ "github.com/filecoin-project/lotus/lib/sigs/bls" _ "github.com/filecoin-project/lotus/lib/sigs/secp" ) func init() { - miner0.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ - abi.RegisteredSealProof_StackedDrg2KiBV1: {}, - } - power0.ConsensusMinerMinPower = big.NewInt(2048) - verifreg0.MinVerifiedDealSize = big.NewInt(256) + policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg2KiBV1) + policy.SetConsensusMinerMinPower(abi.NewStoragePower(2048)) + policy.SetMinVerifiedDealSize(abi.NewStoragePower(256)) } func testGeneration(t testing.TB, n int, msgs int, sectors int) { diff --git a/chain/stmgr/forks_test.go b/chain/stmgr/forks_test.go index 8c6d2ce40..516058a81 100644 --- a/chain/stmgr/forks_test.go +++ b/chain/stmgr/forks_test.go @@ -8,18 +8,15 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/specs-actors/actors/builtin" init_ "github.com/filecoin-project/specs-actors/actors/builtin/init" - miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" - power0 "github.com/filecoin-project/specs-actors/actors/builtin/power" - verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" "github.com/filecoin-project/specs-actors/actors/runtime" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/aerrors" lotusinit "github.com/filecoin-project/lotus/chain/actors/builtin/init" + "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/stmgr" . "github.com/filecoin-project/lotus/chain/stmgr" @@ -34,11 +31,9 @@ import ( ) func init() { - miner0.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ - abi.RegisteredSealProof_StackedDrg2KiBV1: {}, - } - power0.ConsensusMinerMinPower = big.NewInt(2048) - verifreg0.MinVerifiedDealSize = big.NewInt(256) + policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg2KiBV1) + policy.SetConsensusMinerMinPower(abi.NewStoragePower(2048)) + policy.SetMinVerifiedDealSize(abi.NewStoragePower(256)) } const testForkHeight = 40 diff --git a/chain/store/store_test.go b/chain/store/store_test.go index d5e092559..b7adfb595 100644 --- a/chain/store/store_test.go +++ b/chain/store/store_test.go @@ -8,12 +8,9 @@ import ( datastore "github.com/ipfs/go-datastore" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" - miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" - power0 "github.com/filecoin-project/specs-actors/actors/builtin/power" - verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" + "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" @@ -22,11 +19,9 @@ import ( ) func init() { - miner0.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ - abi.RegisteredSealProof_StackedDrg2KiBV1: {}, - } - power0.ConsensusMinerMinPower = big.NewInt(2048) - verifreg0.MinVerifiedDealSize = big.NewInt(256) + policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg2KiBV1) + policy.SetConsensusMinerMinPower(abi.NewStoragePower(2048)) + policy.SetMinVerifiedDealSize(abi.NewStoragePower(256)) } func BenchmarkGetRandomness(b *testing.B) { diff --git a/chain/sync_test.go b/chain/sync_test.go index 53001caf8..7a839be2b 100644 --- a/chain/sync_test.go +++ b/chain/sync_test.go @@ -19,13 +19,10 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/big" - miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" - power0 "github.com/filecoin-project/specs-actors/actors/builtin/power" - verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/gen/slashfilter" "github.com/filecoin-project/lotus/chain/store" @@ -43,11 +40,9 @@ func init() { if err != nil { panic(err) } - miner0.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ - abi.RegisteredSealProof_StackedDrg2KiBV1: {}, - } - power0.ConsensusMinerMinPower = big.NewInt(2048) - verifreg0.MinVerifiedDealSize = big.NewInt(256) + policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg2KiBV1) + policy.SetConsensusMinerMinPower(abi.NewStoragePower(2048)) + policy.SetMinVerifiedDealSize(abi.NewStoragePower(256)) } const source = 0 diff --git a/cli/paych_test.go b/cli/paych_test.go index 47a84fe5e..35f56d90e 100644 --- a/cli/paych_test.go +++ b/cli/paych_test.go @@ -12,39 +12,31 @@ import ( "testing" "time" - "github.com/filecoin-project/lotus/build" - - "github.com/filecoin-project/go-state-types/big" - miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" - power0 "github.com/filecoin-project/specs-actors/actors/builtin/power" - verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" - - "github.com/multiformats/go-multiaddr" - - "github.com/filecoin-project/lotus/chain/events" - - "github.com/filecoin-project/lotus/api/apibstore" - "github.com/filecoin-project/lotus/chain/actors/adt" - "github.com/filecoin-project/lotus/chain/actors/builtin/paych" - cbor "github.com/ipfs/go-ipld-cbor" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/types" - - "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/api/test" - "github.com/filecoin-project/lotus/chain/wallet" - builder "github.com/filecoin-project/lotus/node/test" "github.com/stretchr/testify/require" "github.com/urfave/cli/v2" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + cbor "github.com/ipfs/go-ipld-cbor" + "github.com/multiformats/go-multiaddr" + + "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/chain/actors/builtin/paych" + "github.com/filecoin-project/lotus/chain/actors/policy" + + "github.com/filecoin-project/lotus/api/apibstore" + "github.com/filecoin-project/lotus/api/test" + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/events" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/wallet" + builder "github.com/filecoin-project/lotus/node/test" ) func init() { - power0.ConsensusMinerMinPower = big.NewInt(2048) - miner0.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ - abi.RegisteredSealProof_StackedDrg2KiBV1: {}, - } - verifreg0.MinVerifiedDealSize = big.NewInt(256) + policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg2KiBV1) + policy.SetConsensusMinerMinPower(abi.NewStoragePower(2048)) + policy.SetMinVerifiedDealSize(abi.NewStoragePower(256)) } // TestPaymentChannels does a basic test to exercise the payment channel CLI diff --git a/cmd/lotus-bench/main.go b/cmd/lotus-bench/main.go index 7da754415..2516bfd26 100644 --- a/cmd/lotus-bench/main.go +++ b/cmd/lotus-bench/main.go @@ -27,11 +27,11 @@ import ( "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper/basicfs" "github.com/filecoin-project/lotus/extern/sector-storage/stores" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/specs-storage/storage" lapi "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/genesis" ) @@ -76,7 +76,7 @@ func main() { log.Info("Starting lotus-bench") - miner.SupportedProofTypes[abi.RegisteredSealProof_StackedDrg2KiBV1] = struct{}{} + policy.AddSupportedProofTypes(abi.RegisteredSealProof_StackedDrg2KiBV1) app := &cli.App{ Name: "lotus-bench", diff --git a/extern/storage-sealing/checks.go b/extern/storage-sealing/checks.go index 3d9aedeb4..49994024f 100644 --- a/extern/storage-sealing/checks.go +++ b/extern/storage-sealing/checks.go @@ -4,7 +4,7 @@ import ( "bytes" "context" - "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/build" miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" @@ -156,8 +156,8 @@ func (m *Sealing) checkCommit(ctx context.Context, si SectorInfo, proof []byte, return &ErrNoPrecommit{xerrors.Errorf("precommit info not found on-chain")} } - if pci.PreCommitEpoch+miner.PreCommitChallengeDelay != si.SeedEpoch { - return &ErrBadSeed{xerrors.Errorf("seed epoch doesn't match on chain info: %d != %d", pci.PreCommitEpoch+miner.PreCommitChallengeDelay, si.SeedEpoch)} + if pci.PreCommitEpoch+policy.GetPreCommitChallengeDelay() != si.SeedEpoch { + return &ErrBadSeed{xerrors.Errorf("seed epoch doesn't match on chain info: %d != %d", pci.PreCommitEpoch+policy.GetPreCommitChallengeDelay(), si.SeedEpoch)} } buf := new(bytes.Buffer) diff --git a/extern/storage-sealing/states_sealing.go b/extern/storage-sealing/states_sealing.go index 443365160..a4e852454 100644 --- a/extern/storage-sealing/states_sealing.go +++ b/extern/storage-sealing/states_sealing.go @@ -6,6 +6,7 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + "github.com/filecoin-project/lotus/chain/actors/policy" miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" "golang.org/x/xerrors" @@ -281,7 +282,7 @@ func (m *Sealing) handleWaitSeed(ctx statemachine.Context, sector SectorInfo) er return ctx.Send(SectorChainPreCommitFailed{error: xerrors.Errorf("precommit info not found on chain")}) } - randHeight := pci.PreCommitEpoch + miner.PreCommitChallengeDelay + randHeight := pci.PreCommitEpoch + policy.GetPreCommitChallengeDelay() err = m.events.ChainAt(func(ectx context.Context, _ TipSetToken, curH abi.ChainEpoch) error { // in case of null blocks the randomness can land after the tipset we diff --git a/lotuspond/spawn.go b/lotuspond/spawn.go index 5fa82a865..8b2e8661d 100644 --- a/lotuspond/spawn.go +++ b/lotuspond/spawn.go @@ -11,6 +11,7 @@ import ( "sync/atomic" "time" + "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/types" "golang.org/x/xerrors" @@ -18,7 +19,6 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" genesis2 "github.com/filecoin-project/lotus/chain/gen/genesis" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/cmd/lotus-seed/seed" @@ -26,9 +26,7 @@ import ( ) func init() { - miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ - abi.RegisteredSealProof_StackedDrg2KiBV1: {}, - } + policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg2KiBV1) } func (api *api) Spawn() (nodeInfo, error) { diff --git a/node/node_test.go b/node/node_test.go index 54498eec3..001b99c04 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -5,29 +5,22 @@ import ( "testing" "time" - "github.com/filecoin-project/lotus/chain/actors/builtin/miner" - builder "github.com/filecoin-project/lotus/node/test" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/lotus/lib/lotuslog" - miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" - power0 "github.com/filecoin-project/specs-actors/actors/builtin/power" - verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" logging "github.com/ipfs/go-log/v2" "github.com/filecoin-project/lotus/api/test" + "github.com/filecoin-project/lotus/chain/actors/policy" ) func init() { _ = logging.SetLogLevel("*", "INFO") - power0.ConsensusMinerMinPower = big.NewInt(2048) - miner0.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{ - abi.RegisteredSealProof_StackedDrg2KiBV1: {}, - } - verifreg0.MinVerifiedDealSize = big.NewInt(256) + policy.SetConsensusMinerMinPower(abi.NewStoragePower(2048)) + policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg2KiBV1) + policy.SetMinVerifiedDealSize(abi.NewStoragePower(256)) } func TestAPI(t *testing.T) { @@ -70,9 +63,12 @@ func TestAPIDealFlowReal(t *testing.T) { logging.SetLogLevel("sub", "ERROR") logging.SetLogLevel("storageminer", "ERROR") - // TODO: Do this better. - miner.PreCommitChallengeDelay = 5 - miner0.PreCommitChallengeDelay = 5 + // TODO: just set this globally? + oldDelay := policy.GetPreCommitChallengeDelay() + policy.SetPreCommitChallengeDelay(5) + t.Cleanup(func() { + policy.SetPreCommitChallengeDelay(oldDelay) + }) t.Run("basic", func(t *testing.T) { test.TestDealFlow(t, builder.Builder, time.Second, false, false) From 22fafc7054c37c80e243139f783fea3c6ed28468 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Wed, 23 Sep 2020 18:09:01 -0400 Subject: [PATCH 309/885] Lotus version 0.7.2 --- CHANGELOG.md | 36 ++++++++++++++++++++++++++++++++++++ build/version.go | 8 ++++---- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06c0959b7..7b9756da3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,41 @@ # Lotus changelog +# 0.7.2 / 2020-09-23 + +This optional release of Lotus introduces a major refactor around how a Lotus node interacts with code from the specs-actors repo. We now use interfaces to read the state of actors, which is required to be able to reason about different versions of actors code at the same time. + +Additionally, this release introduces various + +## Changes + +#### Core Lotus + +- Network upgrade support (https://github.com/filecoin-project/lotus/pull/3781) +- Upgrade markets to `v0.6.2` (https://github.com/filecoin-project/lotus/pull/3974) +- Validate chain sync response indices when fetching messages (https://github.com/filecoin-project/lotus/pull/3939) +- Add height diff to sync wait (https://github.com/filecoin-project/lotus/pull/3926) +- Replace Requires with Wants (https://github.com/filecoin-project/lotus/pull/3898) +- Update state diffing for market actor (https://github.com/filecoin-project/lotus/pull/3889) +- Parallel fetch for sync (https://github.com/filecoin-project/lotus/pull/3887) +- Fix SectorState (https://github.com/filecoin-project/lotus/pull/3881) + +#### User Experience + +- Fix out-of-bounds when loading all sector infos (https://github.com/filecoin-project/lotus/pull/3976 +- Add basic deal stats api server for spacerace slingshot (https://github.com/filecoin-project/lotus/pull/3963) +- When doing `sectors update-state`, show a list of existing states if user inputs an invalid one (https://github.com/filecoin-project/lotus/pull/3944) +- Fix `lotus-miner storage find` error (https://github.com/filecoin-project/lotus/pull/3927) +- Log shutdown method for lotus daemon and miner (https://github.com/filecoin-project/lotus/pull/3925) +- Update build and setup instruction link (https://github.com/filecoin-project/lotus/pull/3919) +- Add an option to hide sectors in Removed from `sectors list` (https://github.com/filecoin-project/lotus/pull/3903) + +#### Testing and validation + +- Add init.State#Remove() for testing (https://github.com/filecoin-project/lotus/pull/3971) +- lotus-shed: add consensus check command (https://github.com/filecoin-project/lotus/pull/3933) +- Add keyinfo verify and jwt token command to lotus-shed (https://github.com/filecoin-project/lotus/pull/3914) +- Fix conformance gen (https://github.com/filecoin-project/lotus/pull/3892) + # 0.7.1 / 2020-09-17 This optional release of Lotus introduces some critical fixes to the window PoSt process. It also upgrades some core dependencies, and introduces many improvements to the mining process, deal-making cycle, and overall User Experience. diff --git a/build/version.go b/build/version.go index 933cfca70..cfc8c3ab9 100644 --- a/build/version.go +++ b/build/version.go @@ -29,7 +29,7 @@ func buildType() string { } // BuildVersion is the local build version, set by build system -const BuildVersion = "0.7.1" +const BuildVersion = "0.7.2" func UserVersion() string { return BuildVersion + buildType() + CurrentCommit @@ -83,9 +83,9 @@ func VersionForType(nodeType NodeType) (Version, error) { // semver versions of the rpc api exposed var ( - FullAPIVersion = newVer(0, 15, 0) - MinerAPIVersion = newVer(0, 14, 0) - WorkerAPIVersion = newVer(0, 14, 0) + FullAPIVersion = newVer(0, 16, 0) + MinerAPIVersion = newVer(0, 15, 0) + WorkerAPIVersion = newVer(0, 15, 0) ) //nolint:varcheck,deadcode From f729c6b7f5920b2ffd46e23b60a8ad9e7b1ca566 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Wed, 23 Sep 2020 18:25:45 -0400 Subject: [PATCH 310/885] Update docs --- documentation/en/api-methods.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index 944751dae..22aa8e474 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -214,7 +214,7 @@ Response: ```json { "Version": "string value", - "APIVersion": 3840, + "APIVersion": 4096, "BlockDelay": 42 } ``` From 44748775ff7ae6924f315c69d0b0101eb8ddbb3b Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Wed, 23 Sep 2020 20:49:31 -0400 Subject: [PATCH 311/885] Fixup changelog: --- CHANGELOG.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b9756da3..16ced709b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ This optional release of Lotus introduces a major refactor around how a Lotus node interacts with code from the specs-actors repo. We now use interfaces to read the state of actors, which is required to be able to reason about different versions of actors code at the same time. -Additionally, this release introduces various +Additionally, this release introduces various improvements to the sync process, as well as changes to better the overall UX experience. ## Changes @@ -21,13 +21,12 @@ Additionally, this release introduces various #### User Experience -- Fix out-of-bounds when loading all sector infos (https://github.com/filecoin-project/lotus/pull/3976 - Add basic deal stats api server for spacerace slingshot (https://github.com/filecoin-project/lotus/pull/3963) - When doing `sectors update-state`, show a list of existing states if user inputs an invalid one (https://github.com/filecoin-project/lotus/pull/3944) - Fix `lotus-miner storage find` error (https://github.com/filecoin-project/lotus/pull/3927) - Log shutdown method for lotus daemon and miner (https://github.com/filecoin-project/lotus/pull/3925) - Update build and setup instruction link (https://github.com/filecoin-project/lotus/pull/3919) -- Add an option to hide sectors in Removed from `sectors list` (https://github.com/filecoin-project/lotus/pull/3903) +- Add an option to hide removed sectors from `sectors list` output (https://github.com/filecoin-project/lotus/pull/3903) #### Testing and validation From b4e03d1759813af0fdc154972948ba445a56af47 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Wed, 23 Sep 2020 18:53:28 -0700 Subject: [PATCH 312/885] batch blockstore copies after block validation --- chain/sync.go | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/chain/sync.go b/chain/sync.go index be6c3595c..7562e233d 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -17,6 +17,7 @@ import ( "github.com/Gurpartap/async" "github.com/hashicorp/go-multierror" + blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" logging "github.com/ipfs/go-log/v2" @@ -374,21 +375,28 @@ func (syncer *Syncer) InformNewBlock(from peer.ID, blk *types.FullBlock) bool { return syncer.InformNewHead(from, fts) } -func copyBlockstore(from, to bstore.Blockstore) error { - cids, err := from.AllKeysChan(context.TODO()) +func copyBlockstore(ctx context.Context, from, to bstore.Blockstore) error { + ctx, span := trace.StartSpan(ctx, "copyBlockstore") + defer span.End() + + cids, err := from.AllKeysChan(ctx) if err != nil { return err } + // TODO: should probably expose better methods on the blockstore for this operation + var blks []blocks.Block for c := range cids { b, err := from.Get(c) if err != nil { return err } - if err := to.Put(b); err != nil { - return err - } + blks = append(blks, b) + } + + if err := to.PutMany(blks); err != nil { + return err } return nil @@ -1515,11 +1523,11 @@ func (syncer *Syncer) iterFullTipsets(ctx context.Context, headers []*types.TipS return err } - if err := persistMessages(bs, bstip); err != nil { + if err := persistMessages(ctx, bs, bstip); err != nil { return err } - if err := copyBlockstore(bs, syncer.store.Blockstore()); err != nil { + if err := copyBlockstore(ctx, bs, syncer.store.Blockstore()); err != nil { return xerrors.Errorf("message processing failed: %w", err) } } @@ -1596,7 +1604,10 @@ func (syncer *Syncer) fetchMessages(ctx context.Context, headers []*types.TipSet return batch, nil } -func persistMessages(bs bstore.Blockstore, bst *exchange.CompactedMessages) error { +func persistMessages(ctx context.Context, bs bstore.Blockstore, bst *exchange.CompactedMessages) error { + _, span := trace.StartSpan(ctx, "persistMessages") + defer span.End() + for _, m := range bst.Bls { //log.Infof("putting BLS message: %s", m.Cid()) if _, err := store.PutMessage(bs, m); err != nil { From 38e256cecefd3bf92ce79b9a815151a5165117cd Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Wed, 23 Sep 2020 20:19:20 -0700 Subject: [PATCH 313/885] add some tracing to the vm's blockstore copy --- chain/vm/vm.go | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/chain/vm/vm.go b/chain/vm/vm.go index 35ece7c57..beab8a5fb 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -546,7 +546,7 @@ func (vm *VM) Flush(ctx context.Context) (cid.Cid, error) { return cid.Undef, xerrors.Errorf("flushing vm: %w", err) } - if err := Copy(from, to, root); err != nil { + if err := Copy(ctx, from, to, root); err != nil { return cid.Undef, xerrors.Errorf("copying tree: %w", err) } @@ -600,9 +600,18 @@ func linksForObj(blk block.Block, cb func(cid.Cid)) error { } } -func Copy(from, to blockstore.Blockstore, root cid.Cid) error { +func Copy(ctx context.Context, from, to blockstore.Blockstore, root cid.Cid) error { + ctx, span := trace.StartSpan(ctx, "vm.Copy") + defer span.End() + + var numBlocks int + var totalCopySize int + var batch []block.Block batchCp := func(blk block.Block) error { + numBlocks++ + totalCopySize += len(blk.RawData()) + batch = append(batch, blk) if len(batch) > 100 { if err := to.PutMany(batch); err != nil { @@ -623,6 +632,11 @@ func Copy(from, to blockstore.Blockstore, root cid.Cid) error { } } + span.AddAttributes( + trace.Int64Attribute("numBlocks", int64(numBlocks)), + trace.Int64Attribute("copySize", int64(totalCopySize)), + ) + return nil } From ded3a30f55940b5e468045347716bc63e8cbfd87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 24 Sep 2020 11:56:54 +0200 Subject: [PATCH 314/885] fix lint --- chain/vm/vm.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chain/vm/vm.go b/chain/vm/vm.go index beab8a5fb..3bafbe090 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -601,7 +601,7 @@ func linksForObj(blk block.Block, cb func(cid.Cid)) error { } func Copy(ctx context.Context, from, to blockstore.Blockstore, root cid.Cid) error { - ctx, span := trace.StartSpan(ctx, "vm.Copy") + ctx, span := trace.StartSpan(ctx, "vm.Copy") // nolint defer span.End() var numBlocks int From 15eddf0c96313e53913e768bb7e20d27b401be25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 24 Sep 2020 13:35:45 +0200 Subject: [PATCH 315/885] Make sync wait nicer --- api/api_full.go | 2 ++ chain/vm/vm.go | 11 +++++++++++ cli/sync.go | 33 +++++++++++++++++++++++++++++++-- documentation/en/api-methods.md | 3 ++- node/impl/full/sync.go | 6 +++++- 5 files changed, 51 insertions(+), 4 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 772ce9250..6d2d0c7b5 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -725,6 +725,8 @@ type ActiveSync struct { type SyncState struct { ActiveSyncs []ActiveSync + + VMApplied uint64 } type SyncStateStage int diff --git a/chain/vm/vm.go b/chain/vm/vm.go index 3bafbe090..54ea47698 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -5,6 +5,7 @@ import ( "context" "fmt" "reflect" + "sync/atomic" "time" block "github.com/ipfs/go-block-format" @@ -40,6 +41,12 @@ var log = logging.Logger("vm") var actorLog = logging.Logger("actors") var gasOnActorExec = newGasCharge("OnActorExec", 0, 0) +// stat counters +var ( + StatSends uint64 + StatApplied uint64 +) + // ResolveToKeyAddr returns the public key type of address (`BLS`/`SECP256K1`) of an account actor identified by `addr`. func ResolveToKeyAddr(state types.StateTree, cst cbor.IpldStore, addr address.Address) (address.Address, error) { if addr.Protocol() == address.BLS || addr.Protocol() == address.SECP256K1 { @@ -204,6 +211,8 @@ type ApplyRet struct { func (vm *VM) send(ctx context.Context, msg *types.Message, parent *Runtime, gasCharge *GasCharge, start time.Time) ([]byte, aerrors.ActorError, *Runtime) { + defer atomic.AddUint64(&StatSends, 1) + st := vm.cstate origin := msg.From @@ -312,6 +321,7 @@ func checkMessage(msg *types.Message) error { func (vm *VM) ApplyImplicitMessage(ctx context.Context, msg *types.Message) (*ApplyRet, error) { start := build.Clock.Now() + defer atomic.AddUint64(&StatApplied, 1) ret, actorErr, rt := vm.send(ctx, msg, nil, nil, start) rt.finilizeGasTracing() return &ApplyRet{ @@ -331,6 +341,7 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, start := build.Clock.Now() ctx, span := trace.StartSpan(ctx, "vm.ApplyMessage") defer span.End() + defer atomic.AddUint64(&StatApplied, 1) msg := cmsg.VMMessage() if span.IsRecordingEvents() { span.AddAttributes( diff --git a/cli/sync.go b/cli/sync.go index 3b4e2e9fb..bee87cf70 100644 --- a/cli/sync.go +++ b/cli/sync.go @@ -225,6 +225,16 @@ var syncCheckpointCmd = &cli.Command{ } func SyncWait(ctx context.Context, napi api.FullNode) error { + tick := time.Second / 4 + + lastLines := 0 + ticker := time.NewTicker(tick) + defer ticker.Stop() + + samples := 8 + i := 0 + var app, lastApp uint64 + for { state, err := napi.SyncState(ctx) if err != nil { @@ -266,7 +276,24 @@ func SyncWait(ctx context.Context, napi api.FullNode) error { heightDiff = 0 } - fmt.Printf("\r\x1b[2KWorker %d: Base Height: %d\tTarget Height: %d\t Height diff: %d\tTarget: %s\tState: %s\tHeight: %d", working, baseHeight, theight, heightDiff, target, ss.Stage, ss.Height) + for i := 0; i < lastLines; i++ { + fmt.Print("\r\x1b[2K\x1b[A") + } + + fmt.Printf("Worker: %d; Base: %d; Target: %d (diff: %d)\n", working, baseHeight, theight, heightDiff) + fmt.Printf("State: %s; Current Epoch: %d; Todo: %d\n", ss.Stage, ss.Height, theight-ss.Height) + lastLines = 2 + + if i%samples == 0 { + lastApp = app + app = state.VMApplied + } + if i > 0 { + fmt.Printf("Validated %d messages (%d per second)\n", state.VMApplied, (app-lastApp)*uint64(time.Second/tick)/uint64(samples)) + lastLines++ + } + + _ = target // todo: maybe print? (creates a bunch of line wrapping issues with most tipsets) if time.Now().Unix()-int64(head.MinTimestamp()) < int64(build.BlockDelaySecs) { fmt.Println("\nDone!") @@ -277,7 +304,9 @@ func SyncWait(ctx context.Context, napi api.FullNode) error { case <-ctx.Done(): fmt.Println("\nExit by user") return nil - case <-build.Clock.After(1 * time.Second): + case <-ticker.C: } + + i++ } } diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index 22aa8e474..e489fcb0f 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -4303,7 +4303,8 @@ Inputs: `null` Response: ```json { - "ActiveSyncs": null + "ActiveSyncs": null, + "VMApplied": 42 } ``` diff --git a/node/impl/full/sync.go b/node/impl/full/sync.go index 31a707b90..dc3bfe230 100644 --- a/node/impl/full/sync.go +++ b/node/impl/full/sync.go @@ -2,6 +2,7 @@ package full import ( "context" + "sync/atomic" cid "github.com/ipfs/go-cid" pubsub "github.com/libp2p/go-libp2p-pubsub" @@ -13,6 +14,7 @@ import ( "github.com/filecoin-project/lotus/chain" "github.com/filecoin-project/lotus/chain/gen/slashfilter" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/vm" "github.com/filecoin-project/lotus/node/modules/dtypes" ) @@ -28,7 +30,9 @@ type SyncAPI struct { func (a *SyncAPI) SyncState(ctx context.Context) (*api.SyncState, error) { states := a.Syncer.State() - out := &api.SyncState{} + out := &api.SyncState{ + VMApplied: atomic.LoadUint64(&vm.StatApplied), + } for i := range states { ss := &states[i] From 3c524ac0e0d806468e1e3a28437e94115089d477 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Wed, 23 Sep 2020 18:28:11 +0200 Subject: [PATCH 316/885] refactor: move nonce generation from mpool to wallet --- chain/messagepool/messagepool.go | 94 ------------- chain/messagesigner/messagesigner.go | 116 ++++++++++++++++ chain/messagesigner/messagesigner_test.go | 159 ++++++++++++++++++++++ node/builder.go | 3 + node/impl/full/mpool.go | 66 +++++---- 5 files changed, 310 insertions(+), 128 deletions(-) create mode 100644 chain/messagesigner/messagesigner.go create mode 100644 chain/messagesigner/messagesigner_test.go diff --git a/chain/messagepool/messagepool.go b/chain/messagepool/messagepool.go index 96900925f..d54ea7164 100644 --- a/chain/messagepool/messagepool.go +++ b/chain/messagepool/messagepool.go @@ -75,8 +75,6 @@ var ( ErrRBFTooLowPremium = errors.New("replace by fee has too low GasPremium") ErrTooManyPendingMessages = errors.New("too many pending messages for actor") ErrNonceGap = errors.New("unfulfilled nonce gap") - - ErrTryAgain = errors.New("state inconsistency while pushing message; please try again") ) const ( @@ -795,98 +793,6 @@ func (mp *MessagePool) getStateBalance(addr address.Address, ts *types.TipSet) ( return act.Balance, nil } -func (mp *MessagePool) PushWithNonce(ctx context.Context, addr address.Address, cb func(address.Address, uint64) (*types.SignedMessage, error)) (*types.SignedMessage, error) { - // serialize push access to reduce lock contention - mp.addSema <- struct{}{} - defer func() { - <-mp.addSema - }() - - mp.curTsLk.Lock() - mp.lk.Lock() - - curTs := mp.curTs - - fromKey := addr - if fromKey.Protocol() == address.ID { - var err error - fromKey, err = mp.api.StateAccountKey(ctx, fromKey, mp.curTs) - if err != nil { - mp.lk.Unlock() - mp.curTsLk.Unlock() - return nil, xerrors.Errorf("resolving sender key: %w", err) - } - } - - nonce, err := mp.getNonceLocked(fromKey, mp.curTs) - if err != nil { - mp.lk.Unlock() - mp.curTsLk.Unlock() - return nil, xerrors.Errorf("get nonce locked failed: %w", err) - } - - // release the locks for signing - mp.lk.Unlock() - mp.curTsLk.Unlock() - - msg, err := cb(fromKey, nonce) - if err != nil { - return nil, err - } - - err = mp.checkMessage(msg) - if err != nil { - return nil, err - } - - msgb, err := msg.Serialize() - if err != nil { - return nil, err - } - - // reacquire the locks and check state for consistency - mp.curTsLk.Lock() - defer mp.curTsLk.Unlock() - - if mp.curTs != curTs { - return nil, ErrTryAgain - } - - mp.lk.Lock() - defer mp.lk.Unlock() - - nonce2, err := mp.getNonceLocked(fromKey, mp.curTs) - if err != nil { - return nil, xerrors.Errorf("get nonce locked failed: %w", err) - } - - if nonce2 != nonce { - return nil, ErrTryAgain - } - - publish, err := mp.verifyMsgBeforeAdd(msg, curTs, true) - if err != nil { - return nil, err - } - - if err := mp.checkBalance(msg, curTs); err != nil { - return nil, err - } - - if err := mp.addLocked(msg, false); err != nil { - return nil, xerrors.Errorf("add locked failed: %w", err) - } - if err := mp.addLocal(msg, msgb); err != nil { - log.Errorf("addLocal failed: %+v", err) - } - - if publish { - err = mp.api.PubSubPublish(build.MessagesTopic(mp.netName), msgb) - } - - return msg, err -} - func (mp *MessagePool) Remove(from address.Address, nonce uint64, applied bool) { mp.lk.Lock() defer mp.lk.Unlock() diff --git a/chain/messagesigner/messagesigner.go b/chain/messagesigner/messagesigner.go new file mode 100644 index 000000000..41b0edee9 --- /dev/null +++ b/chain/messagesigner/messagesigner.go @@ -0,0 +1,116 @@ +package messagesigner + +import ( + "bytes" + "context" + + "github.com/filecoin-project/lotus/chain/wallet" + + "github.com/filecoin-project/lotus/chain/messagepool" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/node/modules/dtypes" + "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/namespace" + cbg "github.com/whyrusleeping/cbor-gen" + "golang.org/x/xerrors" +) + +const dsKeyActorNonce = "ActorNonce" + +type mpoolAPI interface { + GetNonce(address.Address) (uint64, error) +} + +// MessageSigner keeps track of nonces per address, and increments the nonce +// when signing a message +type MessageSigner struct { + wallet *wallet.Wallet + mpool mpoolAPI + ds datastore.Batching +} + +func NewMessageSigner(wallet *wallet.Wallet, mpool *messagepool.MessagePool, ds dtypes.MetadataDS) *MessageSigner { + return newMessageSigner(wallet, mpool, ds) +} + +func newMessageSigner(wallet *wallet.Wallet, mpool mpoolAPI, ds dtypes.MetadataDS) *MessageSigner { + ds = namespace.Wrap(ds, datastore.NewKey("/message-signer/")) + return &MessageSigner{ + wallet: wallet, + mpool: mpool, + ds: ds, + } +} + +// SignMessage increments the nonce for the message From address, and signs +// the message +func (ms *MessageSigner) SignMessage(ctx context.Context, msg *types.Message) (*types.SignedMessage, error) { + nonce, err := ms.nextNonce(msg.From) + if err != nil { + return nil, xerrors.Errorf("failed to create nonce: %w", err) + } + + msg.Nonce = nonce + sig, err := ms.wallet.Sign(ctx, msg.From, msg.Cid().Bytes()) + if err != nil { + return nil, xerrors.Errorf("failed to sign message: %w", err) + } + + return &types.SignedMessage{ + Message: *msg, + Signature: *sig, + }, nil +} + +// nextNonce increments the nonce. +// If there is no nonce in the datastore, gets the nonce from the message pool. +func (ms *MessageSigner) nextNonce(addr address.Address) (uint64, error) { + addrNonceKey := datastore.KeyWithNamespaces([]string{dsKeyActorNonce, addr.String()}) + + // Get the nonce for this address from the datastore + nonceBytes, err := ms.ds.Get(addrNonceKey) + + var nonce uint64 + switch { + case xerrors.Is(err, datastore.ErrNotFound): + // If a nonce for this address hasn't yet been created in the + // datastore, check the mempool - nonces used to be created by + // the mempool so we need to support nodes that still have mempool + // nonces. Note that the mempool returns the actor state's nonce by + // default. + nonce, err = ms.mpool.GetNonce(addr) + if err != nil { + return 0, xerrors.Errorf("failed to get nonce from mempool: %w", err) + } + + case err != nil: + return 0, xerrors.Errorf("failed to get nonce from datastore: %w", err) + + default: + // There is a nonce in the mempool, so unmarshall and increment it + maj, val, err := cbg.CborReadHeader(bytes.NewReader(nonceBytes)) + if err != nil { + return 0, xerrors.Errorf("failed to parse nonce from datastore: %w", err) + } + if maj != cbg.MajUnsignedInt { + return 0, xerrors.Errorf("bad cbor type parsing nonce from datastore") + } + + nonce = val + 1 + } + + // Write the nonce for this address to the datastore + buf := bytes.Buffer{} + _, err = buf.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, nonce)) + if err != nil { + return 0, xerrors.Errorf("failed to marshall nonce: %w", err) + } + err = ms.ds.Put(addrNonceKey, buf.Bytes()) + if err != nil { + return 0, xerrors.Errorf("failed to write nonce to datastore: %w", err) + } + + return nonce, nil +} diff --git a/chain/messagesigner/messagesigner_test.go b/chain/messagesigner/messagesigner_test.go new file mode 100644 index 000000000..e52137892 --- /dev/null +++ b/chain/messagesigner/messagesigner_test.go @@ -0,0 +1,159 @@ +package messagesigner + +import ( + "context" + "sync" + "testing" + + "github.com/filecoin-project/lotus/chain/wallet" + + "github.com/filecoin-project/go-state-types/crypto" + "github.com/stretchr/testify/require" + + ds_sync "github.com/ipfs/go-datastore/sync" + + "github.com/filecoin-project/go-address" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-datastore" +) + +type mockMpool struct { + lk sync.RWMutex + nonces map[address.Address]uint64 +} + +func newMockMpool() *mockMpool { + return &mockMpool{nonces: make(map[address.Address]uint64)} +} + +func (mp *mockMpool) setNonce(addr address.Address, nonce uint64) { + mp.lk.Lock() + defer mp.lk.Unlock() + + mp.nonces[addr] = nonce +} + +func (mp *mockMpool) GetNonce(addr address.Address) (uint64, error) { + mp.lk.RLock() + defer mp.lk.RUnlock() + + return mp.nonces[addr], nil +} + +func TestMessageSignerSignMessage(t *testing.T) { + ctx := context.Background() + + w, _ := wallet.NewWallet(wallet.NewMemKeyStore()) + from1, err := w.GenerateKey(crypto.SigTypeSecp256k1) + require.NoError(t, err) + from2, err := w.GenerateKey(crypto.SigTypeSecp256k1) + require.NoError(t, err) + to1, err := w.GenerateKey(crypto.SigTypeSecp256k1) + require.NoError(t, err) + to2, err := w.GenerateKey(crypto.SigTypeSecp256k1) + require.NoError(t, err) + + type msgSpec struct { + msg *types.Message + mpoolNonce [1]uint64 + expNonce uint64 + } + tests := []struct { + name string + msgs []msgSpec + }{{ + // No nonce yet in datastore + name: "no nonce yet", + msgs: []msgSpec{{ + msg: &types.Message{ + To: to1, + From: from1, + }, + expNonce: 0, + }}, + }, { + // Get nonce value of zero from mpool + name: "mpool nonce zero", + msgs: []msgSpec{{ + msg: &types.Message{ + To: to1, + From: from1, + }, + mpoolNonce: [1]uint64{0}, + expNonce: 0, + }}, + }, { + // Get non-zero nonce value from mpool + name: "mpool nonce set", + msgs: []msgSpec{{ + msg: &types.Message{ + To: to1, + From: from1, + }, + mpoolNonce: [1]uint64{5}, + expNonce: 5, + }, { + msg: &types.Message{ + To: to1, + From: from1, + }, + // Should ignore mpool nonce because after the first message nonce + // will come from the datastore + mpoolNonce: [1]uint64{10}, + expNonce: 6, + }}, + }, { + // Nonce should increment independently for each address + name: "nonce increments per address", + msgs: []msgSpec{{ + msg: &types.Message{ + To: to1, + From: from1, + }, + expNonce: 0, + }, { + msg: &types.Message{ + To: to1, + From: from1, + }, + expNonce: 1, + }, { + msg: &types.Message{ + To: to2, + From: from2, + }, + mpoolNonce: [1]uint64{5}, + expNonce: 5, + }, { + msg: &types.Message{ + To: to2, + From: from2, + }, + expNonce: 6, + }, { + msg: &types.Message{ + To: to1, + From: from1, + }, + expNonce: 2, + }}, + }} + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + mpool := newMockMpool() + ds := ds_sync.MutexWrap(datastore.NewMapDatastore()) + ms := newMessageSigner(w, mpool, ds) + + for _, m := range tt.msgs { + if len(m.mpoolNonce) == 1 { + mpool.setNonce(m.msg.From, m.mpoolNonce[0]) + } + smsg, err := ms.SignMessage(ctx, m.msg) + require.NoError(t, err) + require.Equal(t, m.expNonce, smsg.Message.Nonce) + } + }) + } +} diff --git a/node/builder.go b/node/builder.go index c37a5db58..c49789a6a 100644 --- a/node/builder.go +++ b/node/builder.go @@ -6,6 +6,8 @@ import ( "os" "time" + "github.com/filecoin-project/lotus/chain/messagesigner" + logging "github.com/ipfs/go-log" ci "github.com/libp2p/go-libp2p-core/crypto" "github.com/libp2p/go-libp2p-core/host" @@ -259,6 +261,7 @@ func Online() Option { Override(new(*store.ChainStore), modules.ChainStore), Override(new(*stmgr.StateManager), stmgr.NewStateManager), Override(new(*wallet.Wallet), wallet.NewWallet), + Override(new(*messagesigner.MessageSigner), messagesigner.NewMessageSigner), Override(new(dtypes.ChainGCLocker), blockstore.NewGCLocker), Override(new(dtypes.ChainGCBlockstore), modules.ChainGCBlockstore), diff --git a/node/impl/full/mpool.go b/node/impl/full/mpool.go index 6acb17990..003260496 100644 --- a/node/impl/full/mpool.go +++ b/node/impl/full/mpool.go @@ -4,14 +4,14 @@ import ( "context" "encoding/json" + "github.com/filecoin-project/lotus/chain/messagesigner" + "github.com/filecoin-project/go-address" "github.com/ipfs/go-cid" "go.uber.org/fx" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/messagepool" - "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/modules/dtypes" ) @@ -22,9 +22,7 @@ type MpoolAPI struct { WalletAPI GasAPI - Chain *store.ChainStore - - Mpool *messagepool.MessagePool + MessageSigner *messagesigner.MessageSigner PushLocks *dtypes.MpoolLocker } @@ -114,12 +112,14 @@ func (a *MpoolAPI) MpoolPush(ctx context.Context, smsg *types.SignedMessage) (ci } func (a *MpoolAPI) MpoolPushMessage(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec) (*types.SignedMessage, error) { + cp := *msg + msg = &cp inMsg := *msg + fromA, err := a.Stmgr.ResolveToKeyAddress(ctx, msg.From, nil) + if err != nil { + return nil, xerrors.Errorf("getting key address: %w", err) + } { - fromA, err := a.Stmgr.ResolveToKeyAddress(ctx, msg.From, nil) - if err != nil { - return nil, xerrors.Errorf("getting key address: %w", err) - } done, err := a.PushLocks.TakeLock(ctx, fromA) if err != nil { return nil, xerrors.Errorf("taking lock: %w", err) @@ -131,7 +131,7 @@ func (a *MpoolAPI) MpoolPushMessage(ctx context.Context, msg *types.Message, spe return nil, xerrors.Errorf("MpoolPushMessage expects message nonce to be 0, was %d", msg.Nonce) } - msg, err := a.GasAPI.GasEstimateMessageGas(ctx, msg, spec, types.EmptyTSK) + msg, err = a.GasAPI.GasEstimateMessageGas(ctx, msg, spec, types.EmptyTSK) if err != nil { return nil, xerrors.Errorf("GasEstimateMessageGas error: %w", err) } @@ -143,33 +143,31 @@ func (a *MpoolAPI) MpoolPushMessage(ctx context.Context, msg *types.Message, spe inJson, outJson) } - sign := func(from address.Address, nonce uint64) (*types.SignedMessage, error) { - msg.Nonce = nonce - if msg.From.Protocol() == address.ID { - log.Warnf("Push from ID address (%s), adjusting to %s", msg.From, from) - msg.From = from - } - - b, err := a.WalletBalance(ctx, msg.From) - if err != nil { - return nil, xerrors.Errorf("mpool push: getting origin balance: %w", err) - } - - if b.LessThan(msg.Value) { - return nil, xerrors.Errorf("mpool push: not enough funds: %s < %s", b, msg.Value) - } - - return a.WalletSignMessage(ctx, from, msg) + if msg.From.Protocol() == address.ID { + log.Warnf("Push from ID address (%s), adjusting to %s", msg.From, fromA) + msg.From = fromA } - var m *types.SignedMessage -again: - m, err = a.Mpool.PushWithNonce(ctx, msg.From, sign) - if err == messagepool.ErrTryAgain { - log.Debugf("temporary failure while pushing message: %s; retrying", err) - goto again + b, err := a.WalletBalance(ctx, msg.From) + if err != nil { + return nil, xerrors.Errorf("mpool push: getting origin balance: %w", err) } - return m, err + + if b.LessThan(msg.Value) { + return nil, xerrors.Errorf("mpool push: not enough funds: %s < %s", b, msg.Value) + } + + smsg, err := a.MessageSigner.SignMessage(ctx, msg) + if err != nil { + return nil, xerrors.Errorf("mpool push: failed to sign message: %w", err) + } + + _, err = a.Mpool.Push(smsg) + if err != nil { + return nil, xerrors.Errorf("mpool push: failed to push message: %w", err) + } + + return smsg, err } func (a *MpoolAPI) MpoolGetNonce(ctx context.Context, addr address.Address) (uint64, error) { From 7eb9bec13f6d351e192b0e8319ccf1bf0c3c077e Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Fri, 18 Sep 2020 18:03:59 +0200 Subject: [PATCH 317/885] feat: dont recompute post on submit redux --- storage/wdpost_changehandler.go | 533 ++++++++++++ storage/wdpost_changehandler_test.go | 1173 ++++++++++++++++++++++++++ storage/wdpost_nextdl_test.go | 38 + storage/wdpost_run.go | 193 +++-- storage/wdpost_run_test.go | 6 +- storage/wdpost_sched.go | 142 +--- 6 files changed, 1913 insertions(+), 172 deletions(-) create mode 100644 storage/wdpost_changehandler.go create mode 100644 storage/wdpost_changehandler_test.go create mode 100644 storage/wdpost_nextdl_test.go diff --git a/storage/wdpost_changehandler.go b/storage/wdpost_changehandler.go new file mode 100644 index 000000000..e65b7a7fc --- /dev/null +++ b/storage/wdpost_changehandler.go @@ -0,0 +1,533 @@ +package storage + +import ( + "context" + "sync" + + "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/specs-actors/actors/builtin/miner" + + "github.com/filecoin-project/go-state-types/dline" + "github.com/filecoin-project/lotus/chain/types" +) + +const SubmitConfidence = 4 + +type CompleteGeneratePoSTCb func(posts []miner.SubmitWindowedPoStParams, err error) +type CompleteSubmitPoSTCb func(err error) + +type changeHandlerAPI interface { + StateMinerProvingDeadline(context.Context, address.Address, types.TipSetKey) (*dline.Info, error) + startGeneratePoST(ctx context.Context, ts *types.TipSet, deadline *dline.Info, onComplete CompleteGeneratePoSTCb) context.CancelFunc + startSubmitPoST(ctx context.Context, ts *types.TipSet, deadline *dline.Info, posts []miner.SubmitWindowedPoStParams, onComplete CompleteSubmitPoSTCb) context.CancelFunc + onAbort(ts *types.TipSet, deadline *dline.Info) + failPost(err error, ts *types.TipSet, deadline *dline.Info) +} + +type changeHandler struct { + api changeHandlerAPI + actor address.Address + proveHdlr *proveHandler + submitHdlr *submitHandler +} + +func newChangeHandler(api changeHandlerAPI, actor address.Address) *changeHandler { + posts := newPostsCache() + p := newProver(api, posts) + s := newSubmitter(api, posts) + return &changeHandler{api: api, actor: actor, proveHdlr: p, submitHdlr: s} +} + +func (ch *changeHandler) start() { + go ch.proveHdlr.run() + go ch.submitHdlr.run() +} + +func (ch *changeHandler) update(ctx context.Context, revert *types.TipSet, advance *types.TipSet) error { + // Get the current deadline period + di, err := ch.api.StateMinerProvingDeadline(ctx, ch.actor, advance.Key()) + if err != nil { + return err + } + + if !di.PeriodStarted() { + return nil // not proving anything yet + } + + hc := &headChange{ + ctx: ctx, + revert: revert, + advance: advance, + di: di, + } + + select { + case ch.proveHdlr.hcs <- hc: + case <-ch.proveHdlr.shutdownCtx.Done(): + case <-ctx.Done(): + } + + select { + case ch.submitHdlr.hcs <- hc: + case <-ch.submitHdlr.shutdownCtx.Done(): + case <-ctx.Done(): + } + + return nil +} + +func (ch *changeHandler) shutdown() { + ch.proveHdlr.shutdown() + ch.submitHdlr.shutdown() +} + +func (ch *changeHandler) currentTSDI() (*types.TipSet, *dline.Info) { + return ch.submitHdlr.currentTSDI() +} + +// postsCache keeps a cache of PoSTs for each proving window +type postsCache struct { + added chan *postInfo + lk sync.RWMutex + cache map[abi.ChainEpoch][]miner.SubmitWindowedPoStParams +} + +func newPostsCache() *postsCache { + return &postsCache{ + added: make(chan *postInfo, 16), + cache: make(map[abi.ChainEpoch][]miner.SubmitWindowedPoStParams), + } +} + +func (c *postsCache) add(di *dline.Info, posts []miner.SubmitWindowedPoStParams) { + c.lk.Lock() + defer c.lk.Unlock() + + // TODO: clear cache entries older than chain finality + c.cache[di.Open] = posts + + c.added <- &postInfo{ + di: di, + posts: posts, + } +} + +func (c *postsCache) get(di *dline.Info) ([]miner.SubmitWindowedPoStParams, bool) { + c.lk.RLock() + defer c.lk.RUnlock() + + posts, ok := c.cache[di.Open] + return posts, ok +} + +type headChange struct { + ctx context.Context + revert *types.TipSet + advance *types.TipSet + di *dline.Info +} + +type currentPost struct { + di *dline.Info + abort context.CancelFunc +} + +type postResult struct { + ts *types.TipSet + currPost *currentPost + posts []miner.SubmitWindowedPoStParams + err error +} + +// proveHandler generates proofs +type proveHandler struct { + api changeHandlerAPI + posts *postsCache + + postResults chan *postResult + hcs chan *headChange + + current *currentPost + + shutdownCtx context.Context + shutdown context.CancelFunc + + // Used for testing + processedHeadChanges chan *headChange + processedPostResults chan *postResult +} + +func newProver( + api changeHandlerAPI, + posts *postsCache, +) *proveHandler { + ctx, cancel := context.WithCancel(context.Background()) + return &proveHandler{ + api: api, + posts: posts, + postResults: make(chan *postResult), + hcs: make(chan *headChange), + shutdownCtx: ctx, + shutdown: cancel, + } +} + +func (p *proveHandler) run() { + // Abort proving on shutdown + defer func() { + if p.current != nil { + p.current.abort() + } + }() + + for p.shutdownCtx.Err() == nil { + select { + case <-p.shutdownCtx.Done(): + return + + case hc := <-p.hcs: + // Head changed + p.processHeadChange(hc.ctx, hc.advance, hc.di) + if p.processedHeadChanges != nil { + p.processedHeadChanges <- hc + } + + case res := <-p.postResults: + // Proof generation complete + p.processPostResult(res) + if p.processedPostResults != nil { + p.processedPostResults <- res + } + } + } +} + +func (p *proveHandler) processHeadChange(ctx context.Context, newTS *types.TipSet, di *dline.Info) { + // If the post window has expired, abort the current proof + if p.current != nil && newTS.Height() >= p.current.di.Close { + // Cancel the context on the current proof + p.current.abort() + + // Clear out the reference to the proof so that we can immediately + // start generating a new proof, without having to worry about state + // getting clobbered when the abort completes + p.current = nil + } + + // Only generate one proof at a time + if p.current != nil { + return + } + + // If the proof for the current post window has been generated, check the + // next post window + _, complete := p.posts.get(di) + for complete { + di = nextDeadline(di) + _, complete = p.posts.get(di) + } + + // Check if the chain is above the Challenge height for the post window + if newTS.Height() < di.Challenge { + return + } + + p.current = ¤tPost{di: di} + curr := p.current + p.current.abort = p.api.startGeneratePoST(ctx, newTS, di, func(posts []miner.SubmitWindowedPoStParams, err error) { + p.postResults <- &postResult{ts: newTS, currPost: curr, posts: posts, err: err} + }) +} + +func (p *proveHandler) processPostResult(res *postResult) { + di := res.currPost.di + if res.err != nil { + // Proving failed so inform the API + p.api.failPost(res.err, res.ts, di) + log.Warnf("Aborted window post Proving (Deadline: %+v)", di) + p.api.onAbort(res.ts, di) + + // Check if the current post has already been aborted + if p.current == res.currPost { + // If the current post was not already aborted, setting it to nil + // marks it as complete so that a new post can be started + p.current = nil + } + return + } + + // Completed processing this proving window + p.current = nil + + // Add the proofs to the cache + p.posts.add(di, res.posts) +} + +type submitResult struct { + pw *postWindow + err error +} + +type SubmitState string + +const ( + SubmitStateStart SubmitState = "SubmitStateStart" + SubmitStateSubmitting SubmitState = "SubmitStateSubmitting" + SubmitStateComplete SubmitState = "SubmitStateComplete" +) + +type postWindow struct { + ts *types.TipSet + di *dline.Info + submitState SubmitState + abort context.CancelFunc +} + +type postInfo struct { + di *dline.Info + posts []miner.SubmitWindowedPoStParams +} + +// submitHandler submits proofs on-chain +type submitHandler struct { + api changeHandlerAPI + posts *postsCache + + submitResults chan *submitResult + hcs chan *headChange + + postWindows map[abi.ChainEpoch]*postWindow + getPostWindowReqs chan *getPWReq + + shutdownCtx context.Context + shutdown context.CancelFunc + + currentCtx context.Context + currentTS *types.TipSet + currentDI *dline.Info + getTSDIReq chan chan *tsdi + + // Used for testing + processedHeadChanges chan *headChange + processedSubmitResults chan *submitResult + processedPostReady chan *postInfo +} + +func newSubmitter( + api changeHandlerAPI, + posts *postsCache, +) *submitHandler { + ctx, cancel := context.WithCancel(context.Background()) + return &submitHandler{ + api: api, + posts: posts, + submitResults: make(chan *submitResult), + hcs: make(chan *headChange), + postWindows: make(map[abi.ChainEpoch]*postWindow), + getPostWindowReqs: make(chan *getPWReq), + getTSDIReq: make(chan chan *tsdi), + shutdownCtx: ctx, + shutdown: cancel, + } +} + +func (s *submitHandler) run() { + // On shutdown, abort in-progress submits + defer func() { + for _, pw := range s.postWindows { + if pw.abort != nil { + pw.abort() + } + } + }() + + for s.shutdownCtx.Err() == nil { + select { + case <-s.shutdownCtx.Done(): + return + + case hc := <-s.hcs: + // Head change + s.processHeadChange(hc.ctx, hc.revert, hc.advance, hc.di) + if s.processedHeadChanges != nil { + s.processedHeadChanges <- hc + } + + case pi := <-s.posts.added: + // Proof generated + s.processPostReady(pi) + if s.processedPostReady != nil { + s.processedPostReady <- pi + } + + case res := <-s.submitResults: + // Submit complete + s.processSubmitResult(res) + if s.processedSubmitResults != nil { + s.processedSubmitResults <- res + } + + case pwreq := <-s.getPostWindowReqs: + // used by getPostWindow() to sync with run loop + pwreq.out <- s.postWindows[pwreq.di.Open] + + case out := <-s.getTSDIReq: + // used by currentTSDI() to sync with run loop + out <- &tsdi{ts: s.currentTS, di: s.currentDI} + } + } +} + +// processHeadChange is called when the chain head changes +func (s *submitHandler) processHeadChange(ctx context.Context, revert *types.TipSet, advance *types.TipSet, di *dline.Info) { + s.currentCtx = ctx + s.currentTS = advance + s.currentDI = di + + // Start tracking the current post window if we're not already + // TODO: clear post windows older than chain finality + if _, ok := s.postWindows[di.Open]; !ok { + s.postWindows[di.Open] = &postWindow{ + di: di, + ts: advance, + submitState: SubmitStateStart, + } + } + + // Apply the change to all post windows + for _, pw := range s.postWindows { + s.processHeadChangeForPW(ctx, revert, advance, pw) + } +} + +func (s *submitHandler) processHeadChangeForPW(ctx context.Context, revert *types.TipSet, advance *types.TipSet, pw *postWindow) { + revertedToPrevDL := revert != nil && revert.Height() < pw.di.Open + expired := advance.Height() >= pw.di.Close + + // If the chain was reverted back to the previous deadline, or if the post + // window has expired, abort submit + if pw.submitState == SubmitStateSubmitting && (revertedToPrevDL || expired) { + // Replace the aborted postWindow with a new one so that we can + // submit again at any time without the state getting clobbered + // when the abort completes + abort := pw.abort + if abort != nil { + pw = &postWindow{ + di: pw.di, + ts: advance, + submitState: SubmitStateStart, + } + s.postWindows[pw.di.Open] = pw + + // Abort the current submit + abort() + } + } else if pw.submitState == SubmitStateComplete && revertedToPrevDL { + // If submit for this deadline has completed, but the chain was + // reverted back to the previous deadline, reset the submit state to the + // starting state, so that it can be resubmitted + pw.submitState = SubmitStateStart + } + + // Submit the proof to chain if the proof has been generated and the chain + // height is above confidence + s.submitIfReady(ctx, advance, pw) +} + +// processPostReady is called when a proof generation completes +func (s *submitHandler) processPostReady(pi *postInfo) { + pw, ok := s.postWindows[pi.di.Open] + if ok { + s.submitIfReady(s.currentCtx, s.currentTS, pw) + } +} + +// submitIfReady submits a proof if the chain is high enough and the proof +// has been generated for this deadline +func (s *submitHandler) submitIfReady(ctx context.Context, advance *types.TipSet, pw *postWindow) { + // If the window has expired, there's nothing more to do. + if advance.Height() >= pw.di.Close { + return + } + + // Check if we're already submitting, or already completed submit + if pw.submitState != SubmitStateStart { + return + } + + // Check if we've reached the confidence height to submit + if advance.Height() < pw.di.Open+SubmitConfidence { + return + } + + // Check if the proofs have been generated for this deadline + posts, ok := s.posts.get(pw.di) + if !ok { + return + } + + // If there was nothing to prove, move straight to the complete state + if len(posts) == 0 { + pw.submitState = SubmitStateComplete + return + } + + // Start submitting post + pw.submitState = SubmitStateSubmitting + pw.abort = s.api.startSubmitPoST(ctx, advance, pw.di, posts, func(err error) { + s.submitResults <- &submitResult{pw: pw, err: err} + }) +} + +// processSubmitResult is called with the response to a submit +func (s *submitHandler) processSubmitResult(res *submitResult) { + if res.err != nil { + // Submit failed so inform the API and go back to the start state + s.api.failPost(res.err, res.pw.ts, res.pw.di) + log.Warnf("Aborted window post Submitting (Deadline: %+v)", res.pw.di) + s.api.onAbort(res.pw.ts, res.pw.di) + + res.pw.submitState = SubmitStateStart + return + } + + // Submit succeeded so move to complete state + res.pw.submitState = SubmitStateComplete +} + +type tsdi struct { + ts *types.TipSet + di *dline.Info +} + +func (s *submitHandler) currentTSDI() (*types.TipSet, *dline.Info) { + out := make(chan *tsdi) + s.getTSDIReq <- out + res := <-out + return res.ts, res.di +} + +type getPWReq struct { + di *dline.Info + out chan *postWindow +} + +func (s *submitHandler) getPostWindow(di *dline.Info) *postWindow { + out := make(chan *postWindow) + s.getPostWindowReqs <- &getPWReq{di: di, out: out} + return <-out +} + +// nextDeadline gets deadline info for the subsequent deadline +func nextDeadline(currentDeadline *dline.Info) *dline.Info { + periodStart := currentDeadline.PeriodStart + newDeadline := currentDeadline.Index + 1 + if newDeadline == miner.WPoStPeriodDeadlines { + newDeadline = 0 + periodStart = periodStart + miner.WPoStProvingPeriod + } + + return miner.NewDeadlineInfo(periodStart, newDeadline, currentDeadline.CurrentEpoch) +} diff --git a/storage/wdpost_changehandler_test.go b/storage/wdpost_changehandler_test.go new file mode 100644 index 000000000..d2a4779e6 --- /dev/null +++ b/storage/wdpost_changehandler_test.go @@ -0,0 +1,1173 @@ +package storage + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + tutils "github.com/filecoin-project/specs-actors/support/testing" + + "github.com/filecoin-project/go-state-types/crypto" + + "github.com/ipfs/go-cid" + "github.com/stretchr/testify/require" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/dline" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/actors/builtin/miner" +) + +var dummyCid cid.Cid + +func init() { + dummyCid, _ = cid.Parse("bafkqaaa") +} + +type proveRes struct { + posts []miner.SubmitWindowedPoStParams + err error +} + +type postStatus string + +const ( + postStatusStart postStatus = "postStatusStart" + postStatusProving postStatus = "postStatusProving" + postStatusComplete postStatus = "postStatusComplete" +) + +type mockAPI struct { + ch *changeHandler + deadline *dline.Info + proveResult chan *proveRes + submitResult chan error + onStateChange chan struct{} + + tsLock sync.RWMutex + ts map[types.TipSetKey]*types.TipSet + + abortCalledLock sync.RWMutex + abortCalled bool + + statesLk sync.RWMutex + postStates map[abi.ChainEpoch]postStatus +} + +func newMockAPI() *mockAPI { + return &mockAPI{ + proveResult: make(chan *proveRes), + onStateChange: make(chan struct{}), + submitResult: make(chan error), + postStates: make(map[abi.ChainEpoch]postStatus), + ts: make(map[types.TipSetKey]*types.TipSet), + } +} + +func (m *mockAPI) makeTs(t *testing.T, h abi.ChainEpoch) *types.TipSet { + m.tsLock.Lock() + defer m.tsLock.Unlock() + + ts := makeTs(t, h) + m.ts[ts.Key()] = ts + return ts +} + +func (m *mockAPI) setDeadline(di *dline.Info) { + m.tsLock.Lock() + defer m.tsLock.Unlock() + + m.deadline = di +} + +func (m *mockAPI) getDeadline(currentEpoch abi.ChainEpoch) *dline.Info { + close := miner.WPoStChallengeWindow - 1 + dlIdx := uint64(0) + for close < currentEpoch { + close += miner.WPoStChallengeWindow + dlIdx++ + } + return miner.NewDeadlineInfo(0, dlIdx, currentEpoch) +} + +func (m *mockAPI) StateMinerProvingDeadline(ctx context.Context, address address.Address, key types.TipSetKey) (*dline.Info, error) { + m.tsLock.RLock() + defer m.tsLock.RUnlock() + + ts, ok := m.ts[key] + if !ok { + panic(fmt.Sprintf("unexpected tipset key %s", key)) + } + + if m.deadline != nil { + m.deadline.CurrentEpoch = ts.Height() + return m.deadline, nil + } + + return m.getDeadline(ts.Height()), nil +} + +func (m *mockAPI) startGeneratePoST( + ctx context.Context, + ts *types.TipSet, + deadline *dline.Info, + completeGeneratePoST CompleteGeneratePoSTCb, +) context.CancelFunc { + ctx, cancel := context.WithCancel(ctx) + + m.statesLk.Lock() + defer m.statesLk.Unlock() + m.postStates[deadline.Open] = postStatusProving + + go func() { + defer cancel() + + select { + case psRes := <-m.proveResult: + m.statesLk.Lock() + { + if psRes.err == nil { + m.postStates[deadline.Open] = postStatusComplete + } else { + m.postStates[deadline.Open] = postStatusStart + } + } + m.statesLk.Unlock() + completeGeneratePoST(psRes.posts, psRes.err) + case <-ctx.Done(): + completeGeneratePoST(nil, ctx.Err()) + } + }() + + return cancel +} + +func (m *mockAPI) getPostStatus(di *dline.Info) postStatus { + m.statesLk.RLock() + defer m.statesLk.RUnlock() + + status, ok := m.postStates[di.Open] + if ok { + return status + } + return postStatusStart +} + +func (m *mockAPI) startSubmitPoST( + ctx context.Context, + ts *types.TipSet, + deadline *dline.Info, + posts []miner.SubmitWindowedPoStParams, + completeSubmitPoST CompleteSubmitPoSTCb, +) context.CancelFunc { + ctx, cancel := context.WithCancel(ctx) + + go func() { + defer cancel() + + select { + case err := <-m.submitResult: + completeSubmitPoST(err) + case <-ctx.Done(): + completeSubmitPoST(ctx.Err()) + } + }() + + return cancel +} + +func (m *mockAPI) onAbort(ts *types.TipSet, deadline *dline.Info) { + m.abortCalledLock.Lock() + defer m.abortCalledLock.Unlock() + m.abortCalled = true +} + +func (m *mockAPI) wasAbortCalled() bool { + m.abortCalledLock.RLock() + defer m.abortCalledLock.RUnlock() + return m.abortCalled +} + +func (m *mockAPI) failPost(err error, ts *types.TipSet, deadline *dline.Info) { +} + +func (m *mockAPI) setChangeHandler(ch *changeHandler) { + m.ch = ch +} + +// TestChangeHandlerBasic verifies we can generate a proof and submit it +func TestChangeHandlerBasic(t *testing.T) { + s := makeScaffolding(t) + mock := s.mock + + defer s.ch.shutdown() + s.ch.start() + + // Trigger a head change + currentEpoch := abi.ChainEpoch(1) + go triggerHeadAdvance(t, s, currentEpoch) + + // Should start proving + <-s.ch.proveHdlr.processedHeadChanges + di := mock.getDeadline(currentEpoch) + require.Equal(t, postStatusProving, s.mock.getPostStatus(di)) + + // Submitter doesn't have anything to do yet + <-s.ch.submitHdlr.processedHeadChanges + require.Equal(t, SubmitStateStart, s.submitState(di)) + + // Send a response to the call to generate proofs + posts := []miner.SubmitWindowedPoStParams{{Deadline: di.Index}} + mock.proveResult <- &proveRes{posts: posts} + + // Should move to proving complete + <-s.ch.proveHdlr.processedPostResults + require.Equal(t, postStatusComplete, s.mock.getPostStatus(di)) + + // Move to the correct height to submit the proof + currentEpoch = 1 + SubmitConfidence + go triggerHeadAdvance(t, s, currentEpoch) + + // Should move to submitting state + <-s.ch.submitHdlr.processedHeadChanges + di = mock.getDeadline(currentEpoch) + require.Equal(t, SubmitStateSubmitting, s.submitState(di)) + + // Send a response to the submit call + mock.submitResult <- nil + + // Should move to the complete state + <-s.ch.submitHdlr.processedSubmitResults + require.Equal(t, SubmitStateComplete, s.submitState(di)) +} + +// TestChangeHandlerFromProvingToSubmittingNoHeadChange tests that when the +// chain is already advanced past the confidence interval, we should move from +// proving to submitting without a head change in between. +func TestChangeHandlerFromProvingToSubmittingNoHeadChange(t *testing.T) { + s := makeScaffolding(t) + mock := s.mock + + // Monitor submit handler's processing of incoming postInfo + s.ch.submitHdlr.processedPostReady = make(chan *postInfo) + + defer s.ch.shutdown() + s.ch.start() + + // Trigger a head change + currentEpoch := abi.ChainEpoch(1) + go triggerHeadAdvance(t, s, currentEpoch) + + // Should start proving + <-s.ch.proveHdlr.processedHeadChanges + di := mock.getDeadline(currentEpoch) + require.Equal(t, postStatusProving, s.mock.getPostStatus(di)) + + // Submitter doesn't have anything to do yet + <-s.ch.submitHdlr.processedHeadChanges + require.Equal(t, SubmitStateStart, s.submitState(di)) + + // Trigger a head change that advances the chain beyond the submit + // confidence + currentEpoch = 1 + SubmitConfidence + go triggerHeadAdvance(t, s, currentEpoch) + + // Should be no change to state yet + <-s.ch.proveHdlr.processedHeadChanges + require.Equal(t, postStatusProving, s.mock.getPostStatus(di)) + <-s.ch.submitHdlr.processedHeadChanges + require.Equal(t, SubmitStateStart, s.submitState(di)) + + // Send a response to the call to generate proofs + posts := []miner.SubmitWindowedPoStParams{{Deadline: di.Index}} + mock.proveResult <- &proveRes{posts: posts} + + // Should move to proving complete + <-s.ch.proveHdlr.processedPostResults + di = mock.getDeadline(currentEpoch) + require.Equal(t, postStatusComplete, s.mock.getPostStatus(di)) + + // Should move directly to submitting state with no further head changes + <-s.ch.submitHdlr.processedPostReady + require.Equal(t, SubmitStateSubmitting, s.submitState(di)) +} + +// TestChangeHandlerFromProvingEmptyProofsToComplete tests that when there are no +// proofs generated we should not submit anything to chain but submit state +// should move to completed +func TestChangeHandlerFromProvingEmptyProofsToComplete(t *testing.T) { + s := makeScaffolding(t) + mock := s.mock + + // Monitor submit handler's processing of incoming postInfo + s.ch.submitHdlr.processedPostReady = make(chan *postInfo) + + defer s.ch.shutdown() + s.ch.start() + + // Trigger a head change + currentEpoch := abi.ChainEpoch(1) + go triggerHeadAdvance(t, s, currentEpoch) + + // Should start proving + <-s.ch.proveHdlr.processedHeadChanges + di := mock.getDeadline(currentEpoch) + require.Equal(t, postStatusProving, s.mock.getPostStatus(di)) + + // Submitter doesn't have anything to do yet + <-s.ch.submitHdlr.processedHeadChanges + require.Equal(t, SubmitStateStart, s.submitState(di)) + + // Trigger a head change that advances the chain beyond the submit + // confidence + currentEpoch = 1 + SubmitConfidence + go triggerHeadAdvance(t, s, currentEpoch) + + // Should be no change to state yet + <-s.ch.proveHdlr.processedHeadChanges + require.Equal(t, postStatusProving, s.mock.getPostStatus(di)) + <-s.ch.submitHdlr.processedHeadChanges + require.Equal(t, SubmitStateStart, s.submitState(di)) + + // Send a response to the call to generate proofs with an empty proofs array + posts := []miner.SubmitWindowedPoStParams{} + mock.proveResult <- &proveRes{posts: posts} + + // Should move to proving complete + <-s.ch.proveHdlr.processedPostResults + di = mock.getDeadline(currentEpoch) + require.Equal(t, postStatusComplete, s.mock.getPostStatus(di)) + + // Should move directly to submitting complete state + <-s.ch.submitHdlr.processedPostReady + require.Equal(t, SubmitStateComplete, s.submitState(di)) +} + +// TestChangeHandlerDontStartUntilProvingPeriod tests that the handler +// ignores updates until the proving period has been reached. +func TestChangeHandlerDontStartUntilProvingPeriod(t *testing.T) { + s := makeScaffolding(t) + mock := s.mock + + periodStart := miner.WPoStProvingPeriod + dlIdx := uint64(1) + currentEpoch := abi.ChainEpoch(10) + di := miner.NewDeadlineInfo(periodStart, dlIdx, currentEpoch) + mock.setDeadline(di) + + defer s.ch.shutdown() + s.ch.start() + + // Trigger a head change + go triggerHeadAdvance(t, s, currentEpoch) + + // Nothing should happen because the proving period has not started + select { + case <-s.ch.proveHdlr.processedHeadChanges: + require.Fail(t, "unexpected prove change") + case <-s.ch.submitHdlr.processedHeadChanges: + require.Fail(t, "unexpected submit change") + case <-time.After(10 * time.Millisecond): + } + + // Advance the head to the next proving period's first epoch + currentEpoch = periodStart + miner.WPoStChallengeWindow + di = miner.NewDeadlineInfo(periodStart, dlIdx, currentEpoch) + mock.setDeadline(di) + go triggerHeadAdvance(t, s, currentEpoch) + + // Should start proving + <-s.ch.proveHdlr.processedHeadChanges + require.Equal(t, postStatusProving, s.mock.getPostStatus(di)) +} + +// TestChangeHandlerStartProvingNextDeadline verifies that the proof handler +// starts proving the next deadline after the current one +func TestChangeHandlerStartProvingNextDeadline(t *testing.T) { + s := makeScaffolding(t) + mock := s.mock + + defer s.ch.shutdown() + s.ch.start() + + // Trigger a head change + currentEpoch := abi.ChainEpoch(1) + go triggerHeadAdvance(t, s, currentEpoch) + + // Should start proving + <-s.ch.proveHdlr.processedHeadChanges + di := mock.getDeadline(currentEpoch) + require.Equal(t, postStatusProving, s.mock.getPostStatus(di)) + + // Trigger a head change that advances the chain beyond the submit + // confidence + currentEpoch = 1 + SubmitConfidence + go triggerHeadAdvance(t, s, currentEpoch) + + // Should be no change to state yet + <-s.ch.proveHdlr.processedHeadChanges + require.Equal(t, postStatusProving, s.mock.getPostStatus(di)) + + // Send a response to the call to generate proofs + posts := []miner.SubmitWindowedPoStParams{{Deadline: di.Index}} + mock.proveResult <- &proveRes{posts: posts} + + // Should move to proving complete + <-s.ch.proveHdlr.processedPostResults + di = mock.getDeadline(currentEpoch) + require.Equal(t, postStatusComplete, s.mock.getPostStatus(di)) + + // Trigger head change that advances the chain to the Challenge epoch for + // the next deadline + go func() { + di = nextDeadline(di) + currentEpoch = di.Challenge + triggerHeadAdvance(t, s, currentEpoch) + }() + + // Should start generating next window's proof + <-s.ch.proveHdlr.processedHeadChanges + require.Equal(t, postStatusProving, s.mock.getPostStatus(di)) +} + +// TestChangeHandlerProvingRounds verifies we can generate several rounds of +// proofs as the chain head advances +func TestChangeHandlerProvingRounds(t *testing.T) { + s := makeScaffolding(t) + mock := s.mock + + defer s.ch.shutdown() + s.ch.start() + + completeProofIndex := abi.ChainEpoch(10) + for currentEpoch := abi.ChainEpoch(1); currentEpoch < miner.WPoStChallengeWindow*5; currentEpoch++ { + // Trigger a head change + di := mock.getDeadline(currentEpoch) + go triggerHeadAdvance(t, s, currentEpoch) + + // Wait for prover to process head change + <-s.ch.proveHdlr.processedHeadChanges + + completeProofEpoch := di.Open + completeProofIndex + next := nextDeadline(di) + //fmt.Println("epoch", currentEpoch, s.mock.getPostStatus(di), "next", s.mock.getPostStatus(next)) + if currentEpoch >= next.Challenge { + require.Equal(t, postStatusComplete, s.mock.getPostStatus(di)) + // At the next deadline's challenge epoch, should start proving + // for that epoch + require.Equal(t, postStatusProving, s.mock.getPostStatus(next)) + } else if currentEpoch > completeProofEpoch { + // After proving for the round is complete, should be in complete state + require.Equal(t, postStatusComplete, s.mock.getPostStatus(di)) + require.Equal(t, postStatusStart, s.mock.getPostStatus(next)) + } else { + // Until proving completes, should be in the proving state + require.Equal(t, postStatusProving, s.mock.getPostStatus(di)) + require.Equal(t, postStatusStart, s.mock.getPostStatus(next)) + } + + // Wait for submitter to process head change + <-s.ch.submitHdlr.processedHeadChanges + + completeSubmitEpoch := completeProofEpoch + 1 + //fmt.Println("epoch", currentEpoch, s.submitState(di)) + if currentEpoch > completeSubmitEpoch { + require.Equal(t, SubmitStateComplete, s.submitState(di)) + } else if currentEpoch > completeProofEpoch { + require.Equal(t, SubmitStateSubmitting, s.submitState(di)) + } else { + require.Equal(t, SubmitStateStart, s.submitState(di)) + } + + if currentEpoch == completeProofEpoch { + // Send a response to the call to generate proofs + posts := []miner.SubmitWindowedPoStParams{{Deadline: di.Index}} + mock.proveResult <- &proveRes{posts: posts} + + // Should move to proving complete + <-s.ch.proveHdlr.processedPostResults + require.Equal(t, postStatusComplete, s.mock.getPostStatus(di)) + } + + if currentEpoch == completeSubmitEpoch { + // Send a response to the submit call + mock.submitResult <- nil + + // Should move to the complete state + <-s.ch.submitHdlr.processedSubmitResults + require.Equal(t, SubmitStateComplete, s.submitState(di)) + } + } +} + +// TestChangeHandlerProvingErrorRecovery verifies that the proof handler +// recovers correctly from an error +func TestChangeHandlerProvingErrorRecovery(t *testing.T) { + s := makeScaffolding(t) + mock := s.mock + + defer s.ch.shutdown() + s.ch.start() + + // Trigger a head change + currentEpoch := abi.ChainEpoch(1) + go triggerHeadAdvance(t, s, currentEpoch) + + // Should start proving + <-s.ch.proveHdlr.processedHeadChanges + di := mock.getDeadline(currentEpoch) + require.Equal(t, postStatusProving, s.mock.getPostStatus(di)) + + // Send an error response to the call to generate proofs + mock.proveResult <- &proveRes{err: fmt.Errorf("err")} + + // Should abort and then move to start state + <-s.ch.proveHdlr.processedPostResults + require.Equal(t, postStatusStart, s.mock.getPostStatus(di)) + + // Trigger a head change + go triggerHeadAdvance(t, s, currentEpoch) + + // Should start proving + <-s.ch.proveHdlr.processedHeadChanges + require.Equal(t, postStatusProving, s.mock.getPostStatus(di)) + + // Send a success response to the call to generate proofs + posts := []miner.SubmitWindowedPoStParams{{Deadline: di.Index}} + mock.proveResult <- &proveRes{posts: posts} + + // Should move to proving complete + <-s.ch.proveHdlr.processedPostResults + require.Equal(t, postStatusComplete, s.mock.getPostStatus(di)) +} + +// TestChangeHandlerSubmitErrorRecovery verifies that the submit handler +// recovers correctly from an error +func TestChangeHandlerSubmitErrorRecovery(t *testing.T) { + s := makeScaffolding(t) + mock := s.mock + + defer s.ch.shutdown() + s.ch.start() + + // Trigger a head change + currentEpoch := abi.ChainEpoch(1) + go triggerHeadAdvance(t, s, currentEpoch) + + // Should start proving + <-s.ch.proveHdlr.processedHeadChanges + di := mock.getDeadline(currentEpoch) + require.Equal(t, postStatusProving, s.mock.getPostStatus(di)) + + // Submitter doesn't have anything to do yet + <-s.ch.submitHdlr.processedHeadChanges + require.Equal(t, SubmitStateStart, s.submitState(di)) + + // Send a response to the call to generate proofs + posts := []miner.SubmitWindowedPoStParams{{Deadline: di.Index}} + mock.proveResult <- &proveRes{posts: posts} + + // Should move to proving complete + <-s.ch.proveHdlr.processedPostResults + require.Equal(t, postStatusComplete, s.mock.getPostStatus(di)) + + // Move to the correct height to submit the proof + currentEpoch = 1 + SubmitConfidence + go triggerHeadAdvance(t, s, currentEpoch) + + // Read from prover incoming channel (so as not to block) + <-s.ch.proveHdlr.processedHeadChanges + + // Should move to submitting state + <-s.ch.submitHdlr.processedHeadChanges + di = mock.getDeadline(currentEpoch) + require.Equal(t, SubmitStateSubmitting, s.submitState(di)) + + // Send an error response to the call to submit + mock.submitResult <- fmt.Errorf("err") + + // Should abort and then move back to the start state + <-s.ch.submitHdlr.processedSubmitResults + require.Equal(t, SubmitStateStart, s.submitState(di)) + require.True(t, mock.wasAbortCalled()) + + // Trigger another head change + go triggerHeadAdvance(t, s, currentEpoch) + + // Read from prover incoming channel (so as not to block) + <-s.ch.proveHdlr.processedHeadChanges + + // Should move to submitting state + <-s.ch.submitHdlr.processedHeadChanges + di = mock.getDeadline(currentEpoch) + require.Equal(t, SubmitStateSubmitting, s.submitState(di)) + + // Send a response to the submit call + mock.submitResult <- nil + + // Should move to the complete state + <-s.ch.submitHdlr.processedSubmitResults + require.Equal(t, SubmitStateComplete, s.submitState(di)) +} + +// TestChangeHandlerProveExpiry verifies that the prove handler +// behaves correctly on expiry +func TestChangeHandlerProveExpiry(t *testing.T) { + s := makeScaffolding(t) + mock := s.mock + + defer s.ch.shutdown() + s.ch.start() + + // Trigger a head change + currentEpoch := abi.ChainEpoch(1) + go triggerHeadAdvance(t, s, currentEpoch) + + // Should start proving + <-s.ch.proveHdlr.processedHeadChanges + di := mock.getDeadline(currentEpoch) + require.Equal(t, postStatusProving, s.mock.getPostStatus(di)) + + // Move to a height that expires the current proof + currentEpoch = miner.WPoStChallengeWindow + di = mock.getDeadline(currentEpoch) + go triggerHeadAdvance(t, s, currentEpoch) + + // Should trigger an abort and start proving for the new deadline + <-s.ch.proveHdlr.processedHeadChanges + require.Equal(t, postStatusProving, s.mock.getPostStatus(di)) + <-s.ch.proveHdlr.processedPostResults + require.True(t, mock.wasAbortCalled()) + + // Send a response to the call to generate proofs + posts := []miner.SubmitWindowedPoStParams{{Deadline: di.Index}} + mock.proveResult <- &proveRes{posts: posts} + + // Should move to proving complete + <-s.ch.proveHdlr.processedPostResults + require.Equal(t, postStatusComplete, s.mock.getPostStatus(di)) +} + +// TestChangeHandlerSubmitExpiry verifies that the submit handler +// behaves correctly on expiry +func TestChangeHandlerSubmitExpiry(t *testing.T) { + s := makeScaffolding(t) + mock := s.mock + + // Ignore prove handler head change processing for this test + s.ch.proveHdlr.processedHeadChanges = nil + + defer s.ch.shutdown() + s.ch.start() + + // Trigger a head change + currentEpoch := abi.ChainEpoch(1) + go triggerHeadAdvance(t, s, currentEpoch) + + // Submitter doesn't have anything to do yet + <-s.ch.submitHdlr.processedHeadChanges + di := mock.getDeadline(currentEpoch) + require.Equal(t, SubmitStateStart, s.submitState(di)) + + // Send a response to the call to generate proofs + posts := []miner.SubmitWindowedPoStParams{{Deadline: di.Index}} + mock.proveResult <- &proveRes{posts: posts} + + // Should move to proving complete + <-s.ch.proveHdlr.processedPostResults + require.Equal(t, postStatusComplete, s.mock.getPostStatus(di)) + + // Move to the correct height to submit the proof + currentEpoch = 1 + SubmitConfidence + go triggerHeadAdvance(t, s, currentEpoch) + + // Should move to submitting state + <-s.ch.submitHdlr.processedHeadChanges + di = mock.getDeadline(currentEpoch) + require.Equal(t, SubmitStateSubmitting, s.submitState(di)) + + // Move to a height that expires the submit + currentEpoch = miner.WPoStChallengeWindow + di = mock.getDeadline(currentEpoch) + go triggerHeadAdvance(t, s, currentEpoch) + + // Should trigger an abort and move back to start state + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + + <-s.ch.submitHdlr.processedSubmitResults + require.True(t, mock.wasAbortCalled()) + }() + + go func() { + defer wg.Done() + + <-s.ch.submitHdlr.processedHeadChanges + require.Equal(t, SubmitStateStart, s.submitState(di)) + }() + + wg.Wait() +} + +// TestChangeHandlerProveRevert verifies that the prove handler +// behaves correctly on revert +func TestChangeHandlerProveRevert(t *testing.T) { + s := makeScaffolding(t) + mock := s.mock + + defer s.ch.shutdown() + s.ch.start() + + // Trigger a head change + currentEpoch := miner.WPoStChallengeWindow + go triggerHeadAdvance(t, s, currentEpoch) + + // Should start proving + <-s.ch.proveHdlr.processedHeadChanges + di := mock.getDeadline(currentEpoch) + require.Equal(t, postStatusProving, s.mock.getPostStatus(di)) + + // Trigger a revert to the previous epoch + revertEpoch := di.Open - 5 + go triggerHeadChange(t, s, revertEpoch, currentEpoch) + + // Should be no change + <-s.ch.proveHdlr.processedHeadChanges + require.Equal(t, postStatusProving, s.mock.getPostStatus(di)) + + // Send a response to the call to generate proofs + posts := []miner.SubmitWindowedPoStParams{{Deadline: di.Index}} + mock.proveResult <- &proveRes{posts: posts} + + // Should move to proving complete + <-s.ch.proveHdlr.processedPostResults + require.Equal(t, postStatusComplete, s.mock.getPostStatus(di)) + require.False(t, mock.wasAbortCalled()) +} + +// TestChangeHandlerSubmittingRevert verifies that the submit handler +// behaves correctly when there's a revert from the submitting state +func TestChangeHandlerSubmittingRevert(t *testing.T) { + s := makeScaffolding(t) + mock := s.mock + + // Ignore prove handler head change processing for this test + s.ch.proveHdlr.processedHeadChanges = nil + + defer s.ch.shutdown() + s.ch.start() + + // Trigger a head change + currentEpoch := miner.WPoStChallengeWindow + go triggerHeadAdvance(t, s, currentEpoch) + + // Submitter doesn't have anything to do yet + <-s.ch.submitHdlr.processedHeadChanges + di := mock.getDeadline(currentEpoch) + require.Equal(t, SubmitStateStart, s.submitState(di)) + + // Send a response to the call to generate proofs + posts := []miner.SubmitWindowedPoStParams{{Deadline: di.Index}} + mock.proveResult <- &proveRes{posts: posts} + + // Should move to proving complete + <-s.ch.proveHdlr.processedPostResults + require.Equal(t, postStatusComplete, s.mock.getPostStatus(di)) + + // Move to the correct height to submit the proof + currentEpoch = currentEpoch + 1 + SubmitConfidence + go triggerHeadAdvance(t, s, currentEpoch) + + // Should move to submitting state + <-s.ch.submitHdlr.processedHeadChanges + di = mock.getDeadline(currentEpoch) + require.Equal(t, SubmitStateSubmitting, s.submitState(di)) + + // Trigger a revert to the previous epoch + revertEpoch := di.Open - 5 + go triggerHeadChange(t, s, revertEpoch, currentEpoch) + + var wg sync.WaitGroup + wg.Add(2) + + // Should trigger an abort + go func() { + defer wg.Done() + + <-s.ch.submitHdlr.processedSubmitResults + require.True(t, mock.wasAbortCalled()) + }() + + // Should resubmit current epoch + go func() { + defer wg.Done() + + <-s.ch.submitHdlr.processedHeadChanges + require.Equal(t, SubmitStateSubmitting, s.submitState(di)) + }() + + wg.Wait() + + // Send a response to the resubmit call + mock.submitResult <- nil + + // Should move to the complete state + <-s.ch.submitHdlr.processedSubmitResults + require.Equal(t, SubmitStateComplete, s.submitState(di)) +} + +// TestChangeHandlerSubmitCompleteRevert verifies that the submit handler +// behaves correctly when there's a revert from the submit complete state +func TestChangeHandlerSubmitCompleteRevert(t *testing.T) { + s := makeScaffolding(t) + mock := s.mock + + // Ignore prove handler head change processing for this test + s.ch.proveHdlr.processedHeadChanges = nil + + defer s.ch.shutdown() + s.ch.start() + + // Trigger a head change + currentEpoch := miner.WPoStChallengeWindow + go triggerHeadAdvance(t, s, currentEpoch) + + // Submitter doesn't have anything to do yet + <-s.ch.submitHdlr.processedHeadChanges + di := mock.getDeadline(currentEpoch) + require.Equal(t, SubmitStateStart, s.submitState(di)) + + // Send a response to the call to generate proofs + posts := []miner.SubmitWindowedPoStParams{{Deadline: di.Index}} + mock.proveResult <- &proveRes{posts: posts} + + // Should move to proving complete + <-s.ch.proveHdlr.processedPostResults + require.Equal(t, postStatusComplete, s.mock.getPostStatus(di)) + + // Move to the correct height to submit the proof + currentEpoch = currentEpoch + 1 + SubmitConfidence + go triggerHeadAdvance(t, s, currentEpoch) + + // Should move to submitting state + <-s.ch.submitHdlr.processedHeadChanges + di = mock.getDeadline(currentEpoch) + require.Equal(t, SubmitStateSubmitting, s.submitState(di)) + + // Send a response to the resubmit call + mock.submitResult <- nil + + // Should move to the complete state + <-s.ch.submitHdlr.processedSubmitResults + require.Equal(t, SubmitStateComplete, s.submitState(di)) + + // Trigger a revert to the previous epoch + revertEpoch := di.Open - 5 + go triggerHeadChange(t, s, revertEpoch, currentEpoch) + + // Should resubmit current epoch + <-s.ch.submitHdlr.processedHeadChanges + require.Equal(t, SubmitStateSubmitting, s.submitState(di)) + + // Send a response to the resubmit call + mock.submitResult <- nil + + // Should move to the complete state + <-s.ch.submitHdlr.processedSubmitResults + require.Equal(t, SubmitStateComplete, s.submitState(di)) +} + +// TestChangeHandlerSubmitRevertTwoEpochs verifies that the submit handler +// behaves correctly when the revert is two epochs deep +func TestChangeHandlerSubmitRevertTwoEpochs(t *testing.T) { + s := makeScaffolding(t) + mock := s.mock + + // Ignore prove handler head change processing for this test + s.ch.proveHdlr.processedHeadChanges = nil + + defer s.ch.shutdown() + s.ch.start() + + // Trigger a head change + currentEpoch := miner.WPoStChallengeWindow + go triggerHeadAdvance(t, s, currentEpoch) + + // Submitter doesn't have anything to do yet + <-s.ch.submitHdlr.processedHeadChanges + diE1 := mock.getDeadline(currentEpoch) + require.Equal(t, SubmitStateStart, s.submitState(diE1)) + + // Send a response to the call to generate proofs + posts := []miner.SubmitWindowedPoStParams{{Deadline: diE1.Index}} + mock.proveResult <- &proveRes{posts: posts} + + // Should move to proving complete + <-s.ch.proveHdlr.processedPostResults + require.Equal(t, postStatusComplete, s.mock.getPostStatus(diE1)) + + // Move to the challenge epoch for the next deadline + diE2 := nextDeadline(diE1) + currentEpoch = diE2.Challenge + go triggerHeadAdvance(t, s, currentEpoch) + + // Should move to submitting state for epoch 1 + <-s.ch.submitHdlr.processedHeadChanges + diE1 = mock.getDeadline(currentEpoch) + require.Equal(t, SubmitStateSubmitting, s.submitState(diE1)) + + // Send a response to the submit call for epoch 1 + mock.submitResult <- nil + + // Should move to the complete state for epoch 1 + <-s.ch.submitHdlr.processedSubmitResults + require.Equal(t, SubmitStateComplete, s.submitState(diE1)) + + // Should start proving epoch 2 + // Send a response to the call to generate proofs + postsE2 := []miner.SubmitWindowedPoStParams{{Deadline: diE2.Index}} + mock.proveResult <- &proveRes{posts: postsE2} + + // Should move to proving complete for epoch 2 + <-s.ch.proveHdlr.processedPostResults + require.Equal(t, postStatusComplete, s.mock.getPostStatus(diE2)) + + // Move to the correct height to submit the proof for epoch 2 + currentEpoch = diE2.Open + 1 + SubmitConfidence + go triggerHeadAdvance(t, s, currentEpoch) + + // Should move to submitting state for epoch 2 + <-s.ch.submitHdlr.processedHeadChanges + diE2 = mock.getDeadline(currentEpoch) + require.Equal(t, SubmitStateSubmitting, s.submitState(diE2)) + + // Trigger a revert through two epochs (from epoch 2 to epoch 0) + revertEpoch := diE1.Open - 5 + go triggerHeadChange(t, s, revertEpoch, currentEpoch) + + var wg sync.WaitGroup + wg.Add(2) + + // Should trigger an abort + go func() { + defer wg.Done() + + <-s.ch.submitHdlr.processedSubmitResults + require.True(t, mock.wasAbortCalled()) + }() + + go func() { + defer wg.Done() + + <-s.ch.submitHdlr.processedHeadChanges + + // Should reset epoch 1 (that is expired) to start state + require.Equal(t, SubmitStateStart, s.submitState(diE1)) + // Should resubmit epoch 2 + require.Equal(t, SubmitStateSubmitting, s.submitState(diE2)) + }() + + wg.Wait() + + // Send a response to the resubmit call for epoch 2 + mock.submitResult <- nil + + // Should move to the complete state for epoch 2 + <-s.ch.submitHdlr.processedSubmitResults + require.Equal(t, SubmitStateComplete, s.submitState(diE2)) +} + +// TestChangeHandlerSubmitRevertAdvanceLess verifies that the submit handler +// behaves correctly when the revert is two epochs deep and the advance is +// to a lower height than before +func TestChangeHandlerSubmitRevertAdvanceLess(t *testing.T) { + s := makeScaffolding(t) + mock := s.mock + + // Ignore prove handler head change processing for this test + s.ch.proveHdlr.processedHeadChanges = nil + + defer s.ch.shutdown() + s.ch.start() + + // Trigger a head change + currentEpoch := miner.WPoStChallengeWindow + go triggerHeadAdvance(t, s, currentEpoch) + + // Submitter doesn't have anything to do yet + <-s.ch.submitHdlr.processedHeadChanges + diE1 := mock.getDeadline(currentEpoch) + require.Equal(t, SubmitStateStart, s.submitState(diE1)) + + // Send a response to the call to generate proofs + posts := []miner.SubmitWindowedPoStParams{{Deadline: diE1.Index}} + mock.proveResult <- &proveRes{posts: posts} + + // Should move to proving complete + <-s.ch.proveHdlr.processedPostResults + require.Equal(t, postStatusComplete, s.mock.getPostStatus(diE1)) + + // Move to the challenge epoch for the next deadline + diE2 := nextDeadline(diE1) + currentEpoch = diE2.Challenge + go triggerHeadAdvance(t, s, currentEpoch) + + // Should move to submitting state for epoch 1 + <-s.ch.submitHdlr.processedHeadChanges + diE1 = mock.getDeadline(currentEpoch) + require.Equal(t, SubmitStateSubmitting, s.submitState(diE1)) + + // Send a response to the submit call for epoch 1 + mock.submitResult <- nil + + // Should move to the complete state for epoch 1 + <-s.ch.submitHdlr.processedSubmitResults + require.Equal(t, SubmitStateComplete, s.submitState(diE1)) + + // Should start proving epoch 2 + // Send a response to the call to generate proofs + postsE2 := []miner.SubmitWindowedPoStParams{{Deadline: diE2.Index}} + mock.proveResult <- &proveRes{posts: postsE2} + + // Should move to proving complete for epoch 2 + <-s.ch.proveHdlr.processedPostResults + require.Equal(t, postStatusComplete, s.mock.getPostStatus(diE2)) + + // Move to the correct height to submit the proof for epoch 2 + currentEpoch = diE2.Open + 1 + SubmitConfidence + go triggerHeadAdvance(t, s, currentEpoch) + + // Should move to submitting state for epoch 2 + <-s.ch.submitHdlr.processedHeadChanges + diE2 = mock.getDeadline(currentEpoch) + require.Equal(t, SubmitStateSubmitting, s.submitState(diE2)) + + // Trigger a revert through two epochs (from epoch 2 to epoch 0) + // then advance to the previous epoch (to epoch 1) + revertEpoch := diE1.Open - 5 + currentEpoch = diE2.Open - 1 + go triggerHeadChange(t, s, revertEpoch, currentEpoch) + + var wg sync.WaitGroup + wg.Add(2) + + // Should trigger an abort + go func() { + defer wg.Done() + + <-s.ch.submitHdlr.processedSubmitResults + require.True(t, mock.wasAbortCalled()) + }() + + go func() { + defer wg.Done() + + <-s.ch.submitHdlr.processedHeadChanges + + // Should resubmit epoch 1 + require.Equal(t, SubmitStateSubmitting, s.submitState(diE1)) + // Should reset epoch 2 to start state + require.Equal(t, SubmitStateStart, s.submitState(diE2)) + }() + + wg.Wait() + + // Send a response to the resubmit call for epoch 1 + mock.submitResult <- nil + + // Should move to the complete state for epoch 1 + <-s.ch.submitHdlr.processedSubmitResults + require.Equal(t, SubmitStateComplete, s.submitState(diE1)) +} + +type smScaffolding struct { + ctx context.Context + mock *mockAPI + ch *changeHandler +} + +func makeScaffolding(t *testing.T) *smScaffolding { + ctx := context.Background() + actor := tutils.NewActorAddr(t, "actor") + mock := newMockAPI() + ch := newChangeHandler(mock, actor) + mock.setChangeHandler(ch) + + ch.proveHdlr.processedHeadChanges = make(chan *headChange) + ch.proveHdlr.processedPostResults = make(chan *postResult) + + ch.submitHdlr.processedHeadChanges = make(chan *headChange) + ch.submitHdlr.processedSubmitResults = make(chan *submitResult) + + return &smScaffolding{ + ctx: ctx, + mock: mock, + ch: ch, + } +} + +func triggerHeadAdvance(t *testing.T, s *smScaffolding, height abi.ChainEpoch) { + ts := s.mock.makeTs(t, height) + err := s.ch.update(s.ctx, nil, ts) + require.NoError(t, err) +} + +func triggerHeadChange(t *testing.T, s *smScaffolding, revertHeight, advanceHeight abi.ChainEpoch) { + tsRev := s.mock.makeTs(t, revertHeight) + tsAdv := s.mock.makeTs(t, advanceHeight) + err := s.ch.update(s.ctx, tsRev, tsAdv) + require.NoError(t, err) +} + +func (s *smScaffolding) submitState(di *dline.Info) SubmitState { + return s.ch.submitHdlr.getPostWindow(di).submitState +} + +func makeTs(t *testing.T, h abi.ChainEpoch) *types.TipSet { + var parents []cid.Cid + msgcid := dummyCid + + a, _ := address.NewFromString("t00") + b, _ := address.NewFromString("t02") + var ts, err = types.NewTipSet([]*types.BlockHeader{ + { + Height: h, + Miner: a, + + Parents: parents, + + Ticket: &types.Ticket{VRFProof: []byte{byte(h % 2)}}, + + ParentStateRoot: dummyCid, + Messages: msgcid, + ParentMessageReceipts: dummyCid, + + BlockSig: &crypto.Signature{Type: crypto.SigTypeBLS}, + BLSAggregate: &crypto.Signature{Type: crypto.SigTypeBLS}, + }, + { + Height: h, + Miner: b, + + Parents: parents, + + Ticket: &types.Ticket{VRFProof: []byte{byte((h + 1) % 2)}}, + + ParentStateRoot: dummyCid, + Messages: msgcid, + ParentMessageReceipts: dummyCid, + + BlockSig: &crypto.Signature{Type: crypto.SigTypeBLS}, + BLSAggregate: &crypto.Signature{Type: crypto.SigTypeBLS}, + }, + }) + + require.NoError(t, err) + + return ts +} diff --git a/storage/wdpost_nextdl_test.go b/storage/wdpost_nextdl_test.go new file mode 100644 index 000000000..ad4b1fdeb --- /dev/null +++ b/storage/wdpost_nextdl_test.go @@ -0,0 +1,38 @@ +package storage + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/specs-actors/actors/builtin/miner" +) + +func TestNextDeadline(t *testing.T) { + periodStart := abi.ChainEpoch(0) + deadlineIdx := 0 + currentEpoch := abi.ChainEpoch(10) + + di := miner.NewDeadlineInfo(periodStart, uint64(deadlineIdx), currentEpoch) + require.EqualValues(t, 0, di.Index) + require.EqualValues(t, 0, di.PeriodStart) + require.EqualValues(t, -20, di.Challenge) + require.EqualValues(t, 0, di.Open) + require.EqualValues(t, 60, di.Close) + + for i := 1; i < 1+int(miner.WPoStPeriodDeadlines)*2; i++ { + di = nextDeadline(di) + deadlineIdx = i % int(miner.WPoStPeriodDeadlines) + expPeriodStart := int(miner.WPoStProvingPeriod) * (i / int(miner.WPoStPeriodDeadlines)) + expOpen := expPeriodStart + deadlineIdx*int(miner.WPoStChallengeWindow) + expClose := expOpen + int(miner.WPoStChallengeWindow) + expChallenge := expOpen - int(miner.WPoStChallengeLookback) + //fmt.Printf("%d: %d@%d %d-%d (%d)\n", i, expPeriodStart, deadlineIdx, expOpen, expClose, expChallenge) + require.EqualValues(t, deadlineIdx, di.Index) + require.EqualValues(t, expPeriodStart, di.PeriodStart) + require.EqualValues(t, expOpen, di.Open) + require.EqualValues(t, expClose, di.Close) + require.EqualValues(t, expChallenge, di.Challenge) + } +} diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 9a497f879..35fdfc6d1 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -29,15 +29,21 @@ import ( "github.com/filecoin-project/lotus/journal" ) -func (s *WindowPoStScheduler) failPost(err error, deadline *dline.Info) { +func (s *WindowPoStScheduler) failPost(err error, ts *types.TipSet, deadline *dline.Info) { journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} { + c := evtCommon{Error: err} + if ts != nil { + c.Deadline = deadline + c.Height = ts.Height() + c.TipSet = ts.Cids() + } return WdPoStSchedulerEvt{ - evtCommon: s.getEvtCommon(err), + evtCommon: c, State: SchedulerStateFaulted, } }) - log.Errorf("TODO") + log.Errorf("Got err %w - TODO handle errors", err) /*s.failLk.Lock() if eps > s.failed { s.failed = eps @@ -45,67 +51,134 @@ func (s *WindowPoStScheduler) failPost(err error, deadline *dline.Info) { s.failLk.Unlock()*/ } -func (s *WindowPoStScheduler) doPost(ctx context.Context, deadline *dline.Info, ts *types.TipSet) { - ctx, abort := context.WithCancel(ctx) - - s.abort = abort - s.activeDeadline = deadline - - journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} { - return WdPoStSchedulerEvt{ - evtCommon: s.getEvtCommon(nil), - State: SchedulerStateStarted, +// recordProofsEvent records a successful proofs_processed event in the +// journal, even if it was a noop (no partitions). +func (s *WindowPoStScheduler) recordProofsEvent(partitions []miner.PoStPartition, mcid cid.Cid) { + journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStProofs], func() interface{} { + return &WdPoStProofsProcessedEvt{ + evtCommon: s.getEvtCommon(nil), + Partitions: partitions, + MessageCID: mcid, } }) +} +// startGeneratePoST kicks off the process of generating a PoST +func (s *WindowPoStScheduler) startGeneratePoST( + ctx context.Context, + ts *types.TipSet, + deadline *dline.Info, + completeGeneratePoST CompleteGeneratePoSTCb, +) context.CancelFunc { + ctx, abort := context.WithCancel(ctx) go func() { defer abort() - ctx, span := trace.StartSpan(ctx, "WindowPoStScheduler.doPost") - defer span.End() - - // recordProofsEvent records a successful proofs_processed event in the - // journal, even if it was a noop (no partitions). - recordProofsEvent := func(partitions []miner.PoStPartition, mcid cid.Cid) { - journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStProofs], func() interface{} { - return &WdPoStProofsProcessedEvt{ - evtCommon: s.getEvtCommon(nil), - Partitions: partitions, - MessageCID: mcid, - } - }) - } - - posts, err := s.runPost(ctx, *deadline, ts) - if err != nil { - log.Errorf("run window post failed: %+v", err) - s.failPost(err, deadline) - return - } - - if len(posts) == 0 { - recordProofsEvent(nil, cid.Undef) - return - } - - for i := range posts { - post := &posts[i] - sm, err := s.submitPost(ctx, post) - if err != nil { - log.Errorf("submit window post failed: %+v", err) - s.failPost(err, deadline) - } else { - recordProofsEvent(post.Partitions, sm.Cid()) - } - } - journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} { return WdPoStSchedulerEvt{ evtCommon: s.getEvtCommon(nil), - State: SchedulerStateSucceeded, + State: SchedulerStateStarted, } }) + + posts, err := s.runGeneratePoST(ctx, ts, deadline) + completeGeneratePoST(posts, err) }() + + return abort +} + +// runGeneratePoST generates the PoST +func (s *WindowPoStScheduler) runGeneratePoST( + ctx context.Context, + ts *types.TipSet, + deadline *dline.Info, +) ([]miner.SubmitWindowedPoStParams, error) { + ctx, span := trace.StartSpan(ctx, "WindowPoStScheduler.generatePoST") + defer span.End() + + posts, err := s.runPost(ctx, *deadline, ts) + if err != nil { + log.Errorf("runPost failed: %+v", err) + return nil, err + } + + if len(posts) == 0 { + s.recordProofsEvent(nil, cid.Undef) + } + + return posts, nil +} + +// startSubmitPoST kicks of the process of submitting PoST +func (s *WindowPoStScheduler) startSubmitPoST( + ctx context.Context, + ts *types.TipSet, + deadline *dline.Info, + posts []miner.SubmitWindowedPoStParams, + completeSubmitPoST CompleteSubmitPoSTCb, +) context.CancelFunc { + + ctx, abort := context.WithCancel(ctx) + go func() { + defer abort() + + err := s.runSubmitPoST(ctx, ts, deadline, posts) + if err == nil { + journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} { + return WdPoStSchedulerEvt{ + evtCommon: s.getEvtCommon(nil), + State: SchedulerStateSucceeded, + } + }) + } + completeSubmitPoST(err) + }() + + return abort +} + +// runSubmitPoST submits PoST +func (s *WindowPoStScheduler) runSubmitPoST( + ctx context.Context, + ts *types.TipSet, + deadline *dline.Info, + posts []miner.SubmitWindowedPoStParams, +) error { + if len(posts) == 0 { + return nil + } + + ctx, span := trace.StartSpan(ctx, "WindowPoStScheduler.submitPoST") + defer span.End() + + // Get randomness from tickets + commEpoch := deadline.Open + commRand, err := s.api.ChainGetRandomnessFromTickets(ctx, ts.Key(), crypto.DomainSeparationTag_PoStChainCommit, commEpoch, nil) + if err != nil { + err = xerrors.Errorf("failed to get chain randomness from tickets for windowPost (ts=%d; deadline=%d): %w", ts.Height(), commEpoch, err) + log.Errorf("submitPost failed: %+v", err) + + return err + } + + var submitErr error + for i := range posts { + // Add randomness to PoST + post := &posts[i] + post.ChainCommitEpoch = commEpoch + post.ChainCommitRand = commRand + + // Submit PoST + sm, submitErr := s.submitPost(ctx, post) + if submitErr != nil { + log.Errorf("submit window post failed: %+v", submitErr) + } else { + s.recordProofsEvent(post.Partitions, sm.Cid()) + } + } + + return submitErr } func (s *WindowPoStScheduler) checkSectors(ctx context.Context, check bitfield.BitField) (bitfield.BitField, error) { @@ -392,7 +465,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty rand, err := s.api.ChainGetRandomnessFromBeacon(ctx, ts.Key(), crypto.DomainSeparationTag_WindowedPoStChallengeSeed, di.Challenge, buf.Bytes()) if err != nil { - return nil, xerrors.Errorf("failed to get chain randomness for window post (ts=%d; deadline=%d): %w", ts.Height(), di, err) + return nil, xerrors.Errorf("failed to get chain randomness from beacon for window post (ts=%d; deadline=%d): %w", ts.Height(), di, err) } // Get the partitions for the given deadline @@ -536,19 +609,6 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty posts = append(posts, params) } - // Compute randomness after generating proofs so as to reduce the impact - // of chain reorgs (which change randomness) - commEpoch := di.Open - commRand, err := s.api.ChainGetRandomnessFromTickets(ctx, ts.Key(), crypto.DomainSeparationTag_PoStChainCommit, commEpoch, nil) - if err != nil { - return nil, xerrors.Errorf("failed to get chain randomness for window post (ts=%d; deadline=%d): %w", ts.Height(), commEpoch, err) - } - - for i := range posts { - posts[i].ChainCommitEpoch = commEpoch - posts[i].ChainCommitRand = commRand - } - return posts, nil } @@ -589,6 +649,7 @@ func (s *WindowPoStScheduler) batchPartitions(partitions []api.Partition) ([][]a } batches = append(batches, partitions[i:end]) } + return batches, nil } diff --git a/storage/wdpost_run_test.go b/storage/wdpost_run_test.go index 10be2fbcd..09b9aee5c 100644 --- a/storage/wdpost_run_test.go +++ b/storage/wdpost_run_test.go @@ -11,6 +11,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-bitfield" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" @@ -177,7 +178,10 @@ func TestWDPostDoPost(t *testing.T) { FaultDeclarationCutoff: miner0.FaultDeclarationCutoff, } ts := mockTipSet(t) - scheduler.doPost(ctx, di, ts) + + scheduler.startGeneratePoST(ctx, ts, di, func(posts []miner.SubmitWindowedPoStParams, err error) { + scheduler.startSubmitPoST(ctx, ts, di, posts, func(err error) {}) + }) // Read the window PoST messages for i := 0; i < expectedMsgCount; i++ { diff --git a/storage/wdpost_sched.go b/storage/wdpost_sched.go index 7e60fd9ee..3a76a219f 100644 --- a/storage/wdpost_sched.go +++ b/storage/wdpost_sched.go @@ -22,8 +22,6 @@ import ( "go.opencensus.io/trace" ) -const StartConfidence = 4 // TODO: config - type WindowPoStScheduler struct { api storageMinerApi feeCfg config.MinerFeeConfig @@ -31,16 +29,11 @@ type WindowPoStScheduler struct { faultTracker sectorstorage.FaultTracker proofType abi.RegisteredPoStProof partitionSectors uint64 + ch *changeHandler actor address.Address worker address.Address - cur *types.TipSet - - // if a post is in progress, this indicates for which ElectionPeriodStart - activeDeadline *dline.Info - abort context.CancelFunc - evtTypes [4]journal.EventType // failed abi.ChainEpoch // eps @@ -77,16 +70,17 @@ func NewWindowedPoStScheduler(api storageMinerApi, fc config.MinerFeeConfig, sb }, nil } -func deadlineEquals(a, b *dline.Info) bool { - if a == nil || b == nil { - return b == a - } - - return a.PeriodStart == b.PeriodStart && a.Index == b.Index && a.Challenge == b.Challenge +type changeHandlerAPIImpl struct { + storageMinerApi + *WindowPoStScheduler } func (s *WindowPoStScheduler) Run(ctx context.Context) { - defer s.abortActivePoSt() + // Initialize change handler + chImpl := &changeHandlerAPIImpl{storageMinerApi: s.api, WindowPoStScheduler: s} + s.ch = newChangeHandler(chImpl, s.actor) + defer s.ch.shutdown() + s.ch.start() var notifs <-chan []*api.HeadChange var err error @@ -125,9 +119,7 @@ func (s *WindowPoStScheduler) Run(ctx context.Context) { continue } - if err := s.update(ctx, chg.Val); err != nil { - log.Errorf("%+v", err) - } + s.update(ctx, nil, chg.Val) gotCur = true continue @@ -135,7 +127,7 @@ func (s *WindowPoStScheduler) Run(ctx context.Context) { ctx, span := trace.StartSpan(ctx, "WindowPoStScheduler.headChange") - var lowest, highest *types.TipSet = s.cur, nil + var lowest, highest *types.TipSet = nil, nil for _, change := range changes { if change.Val == nil { @@ -149,12 +141,7 @@ func (s *WindowPoStScheduler) Run(ctx context.Context) { } } - if err := s.revert(ctx, lowest); err != nil { - log.Error("handling head reverts in window post sched: %+v", err) - } - if err := s.update(ctx, highest); err != nil { - log.Error("handling head updates in window post sched: %+v", err) - } + s.update(ctx, lowest, highest) span.End() case <-ctx.Done(): @@ -163,95 +150,40 @@ func (s *WindowPoStScheduler) Run(ctx context.Context) { } } -func (s *WindowPoStScheduler) revert(ctx context.Context, newLowest *types.TipSet) error { - if s.cur == newLowest { - return nil +func (s *WindowPoStScheduler) update(ctx context.Context, revert, apply *types.TipSet) { + if apply == nil { + log.Error("no new tipset in window post WindowPoStScheduler.update") + return } - s.cur = newLowest - - newDeadline, err := s.api.StateMinerProvingDeadline(ctx, s.actor, newLowest.Key()) + err := s.ch.update(ctx, revert, apply) if err != nil { - return err + log.Errorf("handling head updates in window post sched: %+v", err) } - - if !deadlineEquals(s.activeDeadline, newDeadline) { - s.abortActivePoSt() - } - - return nil } -func (s *WindowPoStScheduler) update(ctx context.Context, new *types.TipSet) error { - if new == nil { - return xerrors.Errorf("no new tipset in window post sched update") - } - - di, err := s.api.StateMinerProvingDeadline(ctx, s.actor, new.Key()) - if err != nil { - return err - } - - if deadlineEquals(s.activeDeadline, di) { - return nil // already working on this deadline - } - - if !di.PeriodStarted() { - return nil // not proving anything yet - } - - s.abortActivePoSt() - - // TODO: wait for di.Challenge here, will give us ~10min more to compute windowpost - // (Need to get correct deadline above, which is tricky) - - if di.Open+StartConfidence >= new.Height() { - log.Info("not starting window post yet, waiting for startconfidence", di.Open, di.Open+StartConfidence, new.Height()) - return nil - } - - /*s.failLk.Lock() - if s.failed > 0 { - s.failed = 0 - s.activeEPS = 0 - } - s.failLk.Unlock()*/ - log.Infof("at %d, do window post for P %d, dd %d", new.Height(), di.PeriodStart, di.Index) - - s.doPost(ctx, di, new) - - return nil +// onAbort is called when generating proofs or submitting proofs is aborted +func (s *WindowPoStScheduler) onAbort(ts *types.TipSet, deadline *dline.Info) { + journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} { + c := evtCommon{} + if ts != nil { + c.Deadline = deadline + c.Height = ts.Height() + c.TipSet = ts.Cids() + } + return WdPoStSchedulerEvt{ + evtCommon: c, + State: SchedulerStateAborted, + } + }) } -func (s *WindowPoStScheduler) abortActivePoSt() { - if s.activeDeadline == nil { - return // noop - } - - if s.abort != nil { - s.abort() - - journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} { - return WdPoStSchedulerEvt{ - evtCommon: s.getEvtCommon(nil), - State: SchedulerStateAborted, - } - }) - - log.Warnf("Aborting window post (Deadline: %+v)", s.activeDeadline) - } - - s.activeDeadline = nil - s.abort = nil -} - -// getEvtCommon populates and returns common attributes from state, for a -// WdPoSt journal event. func (s *WindowPoStScheduler) getEvtCommon(err error) evtCommon { c := evtCommon{Error: err} - if s.cur != nil { - c.Deadline = s.activeDeadline - c.Height = s.cur.Height() - c.TipSet = s.cur.Cids() + currentTS, currentDeadline := s.ch.currentTSDI() + if currentTS != nil { + c.Deadline = currentDeadline + c.Height = currentTS.Height() + c.TipSet = currentTS.Cids() } return c } From 43323b21887cb3a64a8d8680839d3f7679206dcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 24 Sep 2020 16:03:24 +0200 Subject: [PATCH 318/885] Use continue instead of goto --- miner/miner.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/miner/miner.go b/miner/miner.go index 3b1e65a72..af566d142 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -171,7 +171,7 @@ minerLoop: if err != nil { log.Errorf("failed to get best mining candidate: %s", err) if !m.niceSleep(time.Second * 5) { - goto minerLoop + continue minerLoop } continue } @@ -203,7 +203,7 @@ minerLoop: if err != nil { log.Errorf("failed getting beacon entry: %s", err) if !m.niceSleep(time.Second) { - goto minerLoop + continue minerLoop } continue } @@ -214,7 +214,7 @@ minerLoop: if base.TipSet.Equals(lastBase.TipSet) && lastBase.NullRounds == base.NullRounds { log.Warnf("BestMiningCandidate from the previous round: %s (nulls:%d)", lastBase.TipSet.Cids(), lastBase.NullRounds) if !m.niceSleep(time.Duration(build.BlockDelaySecs) * time.Second) { - goto minerLoop + continue minerLoop } continue } @@ -225,7 +225,7 @@ minerLoop: if err != nil { log.Errorf("mining block failed: %+v", err) if !m.niceSleep(time.Second) { - goto minerLoop + continue minerLoop } onDone(false, 0, err) continue From f8f335df5562dcb22a46ffeb9f466667a058c3ed Mon Sep 17 00:00:00 2001 From: jennijuju Date: Wed, 23 Sep 2020 12:53:03 -0400 Subject: [PATCH 319/885] 3941: Added cmd `lotus-miner proving deadline " - To show the current proving period deadline information of given deadline index. - It outputs the following: - number of partitions in this deadline - partitions numbers has submitted PoSt since the current proving period started - if the deadline is the current proving deadline - for each patition, shows the amount of the sectors in this partition, and their numbers. Also, shows the number of fault sectors and corresponding sector numbers. --- cmd/lotus-storage-miner/proving.go | 92 ++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/cmd/lotus-storage-miner/proving.go b/cmd/lotus-storage-miner/proving.go index 4656c2263..377b81d32 100644 --- a/cmd/lotus-storage-miner/proving.go +++ b/cmd/lotus-storage-miner/proving.go @@ -3,6 +3,7 @@ package main import ( "fmt" "os" + "strconv" "text/tabwriter" "github.com/fatih/color" @@ -22,6 +23,7 @@ var provingCmd = &cli.Command{ Subcommands: []*cli.Command{ provingInfoCmd, provingDeadlinesCmd, + provingDeadlineInfoCmd, provingFaultsCmd, }, } @@ -279,3 +281,93 @@ var provingDeadlinesCmd = &cli.Command{ return tw.Flush() }, } + +var provingDeadlineInfoCmd = &cli.Command{ + Name: "deadline", + Usage: "View the current proving period deadline information by its index ", + ArgsUsage: "", + Action: func(cctx *cli.Context) error { + + if cctx.Args().Len() != 1 { + return xerrors.Errorf("must pass deadline index") + } + + dlIdx, err := strconv.ParseUint(cctx.Args().Get(0), 10, 64) + if err != nil { + return xerrors.Errorf("could not parse deadline index: %w", err) + } + + nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return err + } + defer closer() + + api, acloser, err := lcli.GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer acloser() + + ctx := lcli.ReqContext(cctx) + + maddr, err := getActorAddress(ctx, nodeApi, cctx.String("actor")) + if err != nil { + return err + } + + deadlines, err := api.StateMinerDeadlines(ctx, maddr, types.EmptyTSK) + if err != nil { + return xerrors.Errorf("getting deadlines: %w", err) + } + + di, err := api.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK) + if err != nil { + return xerrors.Errorf("getting deadlines: %w", err) + } + + partitions, err := api.StateMinerPartitions(ctx, maddr, dlIdx, types.EmptyTSK) + if err != nil { + return xerrors.Errorf("getting partitions for deadline %d: %w", dlIdx, err) + } + + provenPartitions, err := deadlines[dlIdx].PostSubmissions.Count() + if err != nil { + return err + } + + fmt.Printf("Deadline Index: %d\n", dlIdx) + fmt.Printf("Partitions: %d\n", len(partitions)) + fmt.Printf("Proven Partitions: %d\n", provenPartitions) + fmt.Printf("Current: %t\n\n", di.Index == dlIdx) + + for pIdx, partition := range partitions { + sectorCount, err := partition.AllSectors.Count() + if err != nil { + return err + } + + sectorNumbers, err := partition.AllSectors.All(sectorCount) + if err != nil { + return err + } + + faultsCount, err := partition.FaultySectors.Count() + if err != nil { + return err + } + + fn, err := partition.FaultySectors.All(faultsCount) + if err != nil { + return err + } + + fmt.Printf("Partition Index: %d\n", pIdx) + fmt.Printf("Sectors: %d\n", sectorCount) + fmt.Printf("Sector Numbers: %v\n", sectorNumbers) + fmt.Printf("Faults: %d\n", faultsCount) + fmt.Printf("Faulty Sectors: %d\n", fn) + } + return nil + }, +} From 4eaa05db5227f214fdd1188f480b4e7826bed596 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 24 Sep 2020 09:40:49 -0700 Subject: [PATCH 320/885] add some more big pictures stats to stateroot stat --- cmd/lotus-shed/stateroot-stats.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/cmd/lotus-shed/stateroot-stats.go b/cmd/lotus-shed/stateroot-stats.go index c02e0202a..023f782bd 100644 --- a/cmd/lotus-shed/stateroot-stats.go +++ b/cmd/lotus-shed/stateroot-stats.go @@ -182,6 +182,11 @@ var staterootStatCmd = &cli.Command{ return infos[i].Stat.Size > infos[j].Stat.Size }) + var totalActorsSize uint64 + for _, info := range infos { + totalActorsSize += info.Stat.Size + } + outcap := 10 if cctx.Args().Len() > outcap { outcap = cctx.Args().Len() @@ -190,6 +195,15 @@ var staterootStatCmd = &cli.Command{ outcap = len(infos) } + totalStat, err := api.ChainStatObj(ctx, ts.ParentState(), cid.Undef) + if err != nil { + return err + } + + fmt.Println("Total state tree size: ", totalStat.Size) + fmt.Println("Sum of actor state size: ", totalActorsSize) + fmt.Println("State tree structure size: ", totalStat.Size-totalActorsSize) + fmt.Print("Addr\tType\tSize\n") for _, inf := range infos[:outcap] { cmh, err := multihash.Decode(inf.Actor.Code.Hash()) From 130ae3ccb34fad93454faa2061129915fd1b6192 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 24 Sep 2020 18:46:21 +0200 Subject: [PATCH 321/885] Multisig: Fix from flag descriptions --- cli/multisig.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/multisig.go b/cli/multisig.go index 944f67b2e..145f5ac41 100644 --- a/cli/multisig.go +++ b/cli/multisig.go @@ -968,7 +968,7 @@ var msigLockProposeCmd = &cli.Command{ Flags: []cli.Flag{ &cli.StringFlag{ Name: "from", - Usage: "account to send the approve message from", + Usage: "account to send the propose message from", }, }, Action: func(cctx *cli.Context) error { @@ -1152,7 +1152,7 @@ var msigLockCancelCmd = &cli.Command{ Flags: []cli.Flag{ &cli.StringFlag{ Name: "from", - Usage: "account to send the approve message from", + Usage: "account to send the cancel message from", }, }, Action: func(cctx *cli.Context) error { From 87b48c94a63437c54596942d3c121f6ff6caa3d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Thu, 24 Sep 2020 17:58:34 +0100 Subject: [PATCH 322/885] conformance: supply network version to driver. --- conformance/driver.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/conformance/driver.go b/conformance/driver.go index 66b6d0f8a..f43a8739d 100644 --- a/conformance/driver.go +++ b/conformance/driver.go @@ -122,6 +122,8 @@ func (d *Driver) ExecuteTipset(bs blockstore.Blockstore, ds ds.Batching, preroot // ExecuteMessage executes a conformance test vector message in a temporary VM. func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, preroot cid.Cid, epoch abi.ChainEpoch, msg *types.Message) (*vm.ApplyRet, cid.Cid, error) { + // dummy state manager; only to reference the GetNetworkVersion method, which does not depend on state. + sm := new(stmgr.StateManager) vmOpts := &vm.VMOpts{ StateBase: preroot, Epoch: epoch, @@ -130,6 +132,7 @@ func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, preroot cid.Cid, epoch Syscalls: mkFakedSigSyscalls(vm.Syscalls(ffiwrapper.ProofVerifier)), // TODO always succeeds; need more flexibility. CircSupplyCalc: nil, BaseFee: BaseFee, + NtwkVersion: sm.GetNtwkVersion, } lvm, err := vm.NewVM(context.TODO(), vmOpts) From 68663060dc19c67d09bc1544527b724b1accbe68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Thu, 24 Sep 2020 17:58:49 +0100 Subject: [PATCH 323/885] add state.StateTree#Version() accessor. --- chain/state/statetree.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/chain/state/statetree.go b/chain/state/statetree.go index a654d224b..fe932bfa1 100644 --- a/chain/state/statetree.go +++ b/chain/state/statetree.go @@ -395,6 +395,11 @@ func (st *StateTree) ForEach(f func(address.Address, *types.Actor) error) error }) } +// Version returns the version of the StateTree data structure in use. +func (st *StateTree) Version() builtin.Version { + return st.version +} + func Diff(oldTree, newTree *StateTree) (map[string]types.Actor, error) { out := map[string]types.Actor{} From cf9dfdb972f97251f604e9286f34450b2254085d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 24 Sep 2020 19:01:12 +0200 Subject: [PATCH 324/885] Multisig: fix build --- cli/multisig.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cli/multisig.go b/cli/multisig.go index 145f5ac41..5b0977382 100644 --- a/cli/multisig.go +++ b/cli/multisig.go @@ -20,6 +20,7 @@ import ( "github.com/urfave/cli/v2" "golang.org/x/xerrors" + "github.com/filecoin-project/specs-actors/actors/builtin" init0 "github.com/filecoin-project/specs-actors/actors/builtin/init" msig0 "github.com/filecoin-project/specs-actors/actors/builtin/multisig" @@ -1018,7 +1019,7 @@ var msigLockProposeCmd = &cli.Command{ from = defaddr } - params, actErr := actors.SerializeParams(&samsig.LockBalanceParams{ + params, actErr := actors.SerializeParams(&msig0.LockBalanceParams{ StartEpoch: abi.ChainEpoch(start), UnlockDuration: abi.ChainEpoch(duration), Amount: abi.NewTokenAmount(amount.Int64()), @@ -1115,7 +1116,7 @@ var msigLockApproveCmd = &cli.Command{ from = defaddr } - params, actErr := actors.SerializeParams(&samsig.LockBalanceParams{ + params, actErr := actors.SerializeParams(&msig0.LockBalanceParams{ StartEpoch: abi.ChainEpoch(start), UnlockDuration: abi.ChainEpoch(duration), Amount: abi.NewTokenAmount(amount.Int64()), @@ -1207,7 +1208,7 @@ var msigLockCancelCmd = &cli.Command{ from = defaddr } - params, actErr := actors.SerializeParams(&samsig.LockBalanceParams{ + params, actErr := actors.SerializeParams(&msig0.LockBalanceParams{ StartEpoch: abi.ChainEpoch(start), UnlockDuration: abi.ChainEpoch(duration), Amount: abi.NewTokenAmount(amount.Int64()), From e48c525053ada752041c4917a5c22d9610e8746d Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 24 Sep 2020 19:41:14 +0200 Subject: [PATCH 325/885] Re-add jaeger-tracing --- documentation/en/jaeger-tracing.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 documentation/en/jaeger-tracing.md diff --git a/documentation/en/jaeger-tracing.md b/documentation/en/jaeger-tracing.md new file mode 100644 index 000000000..bbe4d3052 --- /dev/null +++ b/documentation/en/jaeger-tracing.md @@ -0,0 +1,26 @@ +# Jaeger Tracing + +Lotus has tracing built into many of its internals. To view the traces, first download [Jaeger](https://www.jaegertracing.io/download/) (Choose the 'all-in-one' binary). Then run it somewhere, start up the lotus daemon, and open up localhost:16686 in your browser. + +## Open Census + +Lotus uses [OpenCensus](https://opencensus.io/) for tracing application flow. This generates spans through the execution of annotated code paths. + +Currently it is set up to use Jaeger, though other tracing backends should be fairly easy to swap in. + +## Running Locally + +To easily run and view tracing locally, first, install jaeger. The easiest way to do this is to [download the binaries](https://www.jaegertracing.io/download/) and then run the `jaeger-all-in-one` binary. This will start up jaeger, listen for spans on `localhost:6831`, and expose a web UI for viewing traces on `http://localhost:16686/`. + +Now, to start sending traces from Lotus to Jaeger, set the environment variable `LOTUS_JAEGER` to `localhost:6831`, and start the `lotus daemon`. + +Now, to view any generated traces, open up `http://localhost:16686/` in your browser. + +## Adding Spans + +To annotate a new codepath with spans, add the following lines to the top of the function you wish to trace: + +```go +ctx, span := trace.StartSpan(ctx, "put function name here") +defer span.End() +``` From 6db37b72a85bf056630d757f591d11c92f8c65c9 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 24 Sep 2020 11:05:21 -0700 Subject: [PATCH 326/885] update ffi to code with blst fixes --- extern/filecoin-ffi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/filecoin-ffi b/extern/filecoin-ffi index f640612a1..57e38efe4 160000 --- a/extern/filecoin-ffi +++ b/extern/filecoin-ffi @@ -1 +1 @@ -Subproject commit f640612a1a1f7a2dd8b3a49e1531db0aa0f63447 +Subproject commit 57e38efe4943f09d3127dcf6f0edd614e6acf68e From 306c098d304cb853351a9fe7d8ccf8ccb2bb1caa Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 24 Sep 2020 11:32:38 -0700 Subject: [PATCH 327/885] also update our vendored blst repo --- extern/fil-blst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/fil-blst b/extern/fil-blst index 5f93488fc..8609119cf 160000 --- a/extern/fil-blst +++ b/extern/fil-blst @@ -1 +1 @@ -Subproject commit 5f93488fc0dbfb450f2355269f18fc67010d59bb +Subproject commit 8609119cf4595d1741139c24378fcd8bc4f1c475 From 585b8cc51dee241dd0d340bd3df0502b10e324a6 Mon Sep 17 00:00:00 2001 From: Arsenii Petrovich Date: Fri, 25 Sep 2020 03:54:29 +0300 Subject: [PATCH 328/885] Add glif node to bootstrap peers --- build/bootstrap/bootstrappers.pi | 1 + 1 file changed, 1 insertion(+) diff --git a/build/bootstrap/bootstrappers.pi b/build/bootstrap/bootstrappers.pi index 465f3b5e9..481ede6ab 100644 --- a/build/bootstrap/bootstrappers.pi +++ b/build/bootstrap/bootstrappers.pi @@ -4,3 +4,4 @@ /dns4/bootstrap-4.testnet.fildev.network/tcp/1347/p2p/12D3KooWPkL9LrKRQgHtq7kn9ecNhGU9QaziG8R5tX8v9v7t3h34 /dns4/bootstrap-3.testnet.fildev.network/tcp/1347/p2p/12D3KooWKYSsbpgZ3HAjax5M1BXCwXLa6gVkUARciz7uN3FNtr7T /dns4/bootstrap-5.testnet.fildev.network/tcp/1347/p2p/12D3KooWQYzqnLASJAabyMpPb1GcWZvNSe7JDcRuhdRqonFoiK9W +/dns4/node.glif.io/tcp/1235/p2p/12D3KooWBF8cpp65hp2u9LK5mh19x67ftAam84z9LsfaquTDSBpt From 89bfe84f36bc8264a50cc6f9982774e06feac7e8 Mon Sep 17 00:00:00 2001 From: MaoBisheng-IPFSUNION Date: Fri, 25 Sep 2020 10:18:12 +0800 Subject: [PATCH 329/885] add new booststrappers we hope to add two booststrappers for the filecoin network to improve the network synchronization --- build/bootstrap/bootstrappers.pi | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build/bootstrap/bootstrappers.pi b/build/bootstrap/bootstrappers.pi index 9c619b1bb..e64bd9dae 100644 --- a/build/bootstrap/bootstrappers.pi +++ b/build/bootstrap/bootstrappers.pi @@ -4,3 +4,6 @@ /dns4/lotus-bootstrap-1.fra.fil-test.net/tcp/1347/p2p/12D3KooWLLpNYoKdf9NgcWudBhXLdTcXncqAsTzozw1scMMu6nS5 /dns4/lotus-bootstrap-0.sin.fil-test.net/tcp/1347/p2p/12D3KooWCNL9vXaXwNs3Bu8uRAJK4pxpCyPeM7jZLSDpJma1wrV8 /dns4/lotus-bootstrap-1.sin.fil-test.net/tcp/1347/p2p/12D3KooWNGGxFda1eC5U2YKAgs4ypoFHn3Z3xHCsjmFdrCcytoxm + +/dns4/bootstrap-0.starpool.in/tcp/12757/p2p/12D3KooWGHpBMeZbestVEWkfdnC9u7p6uFHXL1n7m1ZBqsEmiUzz +/dns4/bootstrap-1.starpool.in/tcp/12757/p2p/12D3KooWQZrGH1PxSNZPum99M1zNvjNFM33d1AAu5DcvdHptuU7u From 8fbbebbaf14ce6726bfaea328a6899b63f6fcc0b Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Fri, 25 Sep 2020 00:11:10 -0400 Subject: [PATCH 330/885] Correct helptext around set ask --- cmd/lotus-storage-miner/market.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/lotus-storage-miner/market.go b/cmd/lotus-storage-miner/market.go index 1c9eac1b7..0e7be3dd3 100644 --- a/cmd/lotus-storage-miner/market.go +++ b/cmd/lotus-storage-miner/market.go @@ -156,12 +156,12 @@ var setAskCmd = &cli.Command{ Flags: []cli.Flag{ &cli.Uint64Flag{ Name: "price", - Usage: "Set the price of the ask for unverified deals (specified as FIL / GiB / Epoch) to `PRICE`", + Usage: "Set the price of the ask for unverified deals (specified as attoFIL / GiB / Epoch) to `PRICE`", Required: true, }, &cli.Uint64Flag{ Name: "verified-price", - Usage: "Set the price of the ask for verified deals (specified as FIL / GiB / Epoch) to `PRICE`", + Usage: "Set the price of the ask for verified deals (specified as attoFIL / GiB / Epoch) to `PRICE`", Required: true, }, &cli.StringFlag{ From 4ff38aa8564a3c86391bc6886e5e37d0c7ca286e Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Fri, 25 Sep 2020 02:35:34 -0700 Subject: [PATCH 331/885] feat(markets): update to 0.6.3 --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index cfdf4cb1d..ca7e0760d 100644 --- a/go.mod +++ b/go.mod @@ -25,9 +25,9 @@ require ( github.com/filecoin-project/go-bitfield v0.2.0 github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 - github.com/filecoin-project/go-data-transfer v0.6.5 + github.com/filecoin-project/go-data-transfer v0.6.6 github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f - github.com/filecoin-project/go-fil-markets v0.6.2 + github.com/filecoin-project/go-fil-markets v0.6.3 github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200822201400-474f4fdccc52 github.com/filecoin-project/go-multistore v0.0.3 github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 @@ -60,7 +60,7 @@ require ( github.com/ipfs/go-ds-measure v0.1.0 github.com/ipfs/go-filestore v1.0.0 github.com/ipfs/go-fs-lock v0.0.6 - github.com/ipfs/go-graphsync v0.2.0 + github.com/ipfs/go-graphsync v0.2.1 github.com/ipfs/go-ipfs-blockstore v1.0.1 github.com/ipfs/go-ipfs-chunker v0.0.5 github.com/ipfs/go-ipfs-ds-help v1.0.0 diff --git a/go.sum b/go.sum index 3b31e6c3b..1ca615b59 100644 --- a/go.sum +++ b/go.sum @@ -222,12 +222,12 @@ github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:a github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= -github.com/filecoin-project/go-data-transfer v0.6.5 h1:oP20la8Z0CLrw0uqvt6xVgw6rOevZeGJ9GNQeC0OCSU= -github.com/filecoin-project/go-data-transfer v0.6.5/go.mod h1:I9Ylb/UiZyqnI41wUoCXq/le0nDLhlwpFQCtNPxEPOA= +github.com/filecoin-project/go-data-transfer v0.6.6 h1:2TccLSxPYJENcYRdov2WvpTvQ1qUMrPkWe8sBrfj36g= +github.com/filecoin-project/go-data-transfer v0.6.6/go.mod h1:C++k1U6+jMQODOaen5OPDo9XQbth9Yq3ie94vNjBJbk= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f h1:GxJzR3oRIMTPtpZ0b7QF8FKPK6/iPAc7trhlL5k/g+s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= -github.com/filecoin-project/go-fil-markets v0.6.2 h1:9Z57KeaQSa1liCmT1pH6SIjrn9mGTDFJXmR2WQVuaiY= -github.com/filecoin-project/go-fil-markets v0.6.2/go.mod h1:wtN4Hc/1hoVCpWhSWYxwUxH3PQtjSkWWuC1nQjiIWog= +github.com/filecoin-project/go-fil-markets v0.6.3 h1:3kTxfquGvk3zQY+hJH1kEA28tRQ47phqSRqOI4+YcQM= +github.com/filecoin-project/go-fil-markets v0.6.3/go.mod h1:Ug1yhGhzTYC6qrpKsR2QpU8QRCeBpwkTA9RICVKuOMM= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24= @@ -502,8 +502,8 @@ github.com/ipfs/go-filestore v1.0.0/go.mod h1:/XOCuNtIe2f1YPbiXdYvD0BKLA0JR1MgPi github.com/ipfs/go-fs-lock v0.0.6 h1:sn3TWwNVQqSeNjlWy6zQ1uUGAZrV3hPOyEA6y1/N2a0= github.com/ipfs/go-fs-lock v0.0.6/go.mod h1:OTR+Rj9sHiRubJh3dRhD15Juhd/+w6VPOY28L7zESmM= github.com/ipfs/go-graphsync v0.1.0/go.mod h1:jMXfqIEDFukLPZHqDPp8tJMbHO9Rmeb9CEGevngQbmE= -github.com/ipfs/go-graphsync v0.2.0 h1:x94MvHLNuRwBlZzVal7tR1RYK7T7H6bqQLPopxDbIF0= -github.com/ipfs/go-graphsync v0.2.0/go.mod h1:gEBvJUNelzMkaRPJTpg/jaKN4AQW/7wDWu0K92D8o10= +github.com/ipfs/go-graphsync v0.2.1 h1:MdehhqBSuTI2LARfKLkpYnt0mUrqHs/mtuDnESXHBfU= +github.com/ipfs/go-graphsync v0.2.1/go.mod h1:gEBvJUNelzMkaRPJTpg/jaKN4AQW/7wDWu0K92D8o10= github.com/ipfs/go-hamt-ipld v0.1.1/go.mod h1:1EZCr2v0jlCnhpa+aZ0JZYp8Tt2w16+JJOAVz17YcDk= github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw= From 9dfce400a7a5234ace883ae61648f94eb915516f Mon Sep 17 00:00:00 2001 From: MaoBisheng-IPFSUNION Date: Fri, 25 Sep 2020 17:36:57 +0800 Subject: [PATCH 332/885] delate --- build/bootstrap/bootstrappers.pi | 1 - 1 file changed, 1 deletion(-) diff --git a/build/bootstrap/bootstrappers.pi b/build/bootstrap/bootstrappers.pi index e64bd9dae..802f1471a 100644 --- a/build/bootstrap/bootstrappers.pi +++ b/build/bootstrap/bootstrappers.pi @@ -4,6 +4,5 @@ /dns4/lotus-bootstrap-1.fra.fil-test.net/tcp/1347/p2p/12D3KooWLLpNYoKdf9NgcWudBhXLdTcXncqAsTzozw1scMMu6nS5 /dns4/lotus-bootstrap-0.sin.fil-test.net/tcp/1347/p2p/12D3KooWCNL9vXaXwNs3Bu8uRAJK4pxpCyPeM7jZLSDpJma1wrV8 /dns4/lotus-bootstrap-1.sin.fil-test.net/tcp/1347/p2p/12D3KooWNGGxFda1eC5U2YKAgs4ypoFHn3Z3xHCsjmFdrCcytoxm - /dns4/bootstrap-0.starpool.in/tcp/12757/p2p/12D3KooWGHpBMeZbestVEWkfdnC9u7p6uFHXL1n7m1ZBqsEmiUzz /dns4/bootstrap-1.starpool.in/tcp/12757/p2p/12D3KooWQZrGH1PxSNZPum99M1zNvjNFM33d1AAu5DcvdHptuU7u From 13b38815c2e6df6d7d16cb024f3fedad1b2a4967 Mon Sep 17 00:00:00 2001 From: zgfzgf <1901989065@qq.com> Date: Fri, 25 Sep 2020 20:31:07 +0800 Subject: [PATCH 333/885] add trace wdpost --- storage/wdpost_sched.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/storage/wdpost_sched.go b/storage/wdpost_sched.go index 7e60fd9ee..9477e0306 100644 --- a/storage/wdpost_sched.go +++ b/storage/wdpost_sched.go @@ -125,10 +125,13 @@ func (s *WindowPoStScheduler) Run(ctx context.Context) { continue } + ctx, span := trace.StartSpan(ctx, "WindowPoStScheduler.headChange") + if err := s.update(ctx, chg.Val); err != nil { log.Errorf("%+v", err) } + span.End() gotCur = true continue } From 80a7ed811638853987a3124175606d7db5269bf2 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Fri, 25 Sep 2020 15:54:27 +0200 Subject: [PATCH 334/885] refactor: use abstract types instead of specs-actors --- chain/actors/builtin/miner/miner.go | 4 ++++ storage/wdpost_changehandler.go | 8 ++++++-- storage/wdpost_changehandler_test.go | 8 ++++---- storage/wdpost_nextdl_test.go | 4 ++-- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/chain/actors/builtin/miner/miner.go b/chain/actors/builtin/miner/miner.go index 50a0fc5ca..e09cac587 100644 --- a/chain/actors/builtin/miner/miner.go +++ b/chain/actors/builtin/miner/miner.go @@ -21,6 +21,10 @@ import ( // Unchanged between v0 and v1 actors var PreCommitChallengeDelay = miner0.PreCommitChallengeDelay var WPoStProvingPeriod = miner0.WPoStProvingPeriod +var WPoStPeriodDeadlines = miner0.WPoStPeriodDeadlines +var WPoStChallengeWindow = miner0.WPoStChallengeWindow +var WPoStChallengeLookback = miner0.WPoStChallengeLookback +var FaultDeclarationCutoff = miner0.FaultDeclarationCutoff const MinSectorExpiration = miner0.MinSectorExpiration diff --git a/storage/wdpost_changehandler.go b/storage/wdpost_changehandler.go index e65b7a7fc..285995757 100644 --- a/storage/wdpost_changehandler.go +++ b/storage/wdpost_changehandler.go @@ -7,7 +7,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/go-state-types/dline" "github.com/filecoin-project/lotus/chain/types" @@ -529,5 +529,9 @@ func nextDeadline(currentDeadline *dline.Info) *dline.Info { periodStart = periodStart + miner.WPoStProvingPeriod } - return miner.NewDeadlineInfo(periodStart, newDeadline, currentDeadline.CurrentEpoch) + return NewDeadlineInfo(periodStart, newDeadline, currentDeadline.CurrentEpoch) +} + +func NewDeadlineInfo(periodStart abi.ChainEpoch, deadlineIdx uint64, currEpoch abi.ChainEpoch) *dline.Info { + return dline.NewInfo(periodStart, deadlineIdx, currEpoch, miner.WPoStPeriodDeadlines, miner.WPoStProvingPeriod, miner.WPoStChallengeWindow, miner.WPoStChallengeLookback, miner.FaultDeclarationCutoff) } diff --git a/storage/wdpost_changehandler_test.go b/storage/wdpost_changehandler_test.go index d2a4779e6..6479c0d7e 100644 --- a/storage/wdpost_changehandler_test.go +++ b/storage/wdpost_changehandler_test.go @@ -17,8 +17,8 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/dline" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" ) var dummyCid cid.Cid @@ -90,7 +90,7 @@ func (m *mockAPI) getDeadline(currentEpoch abi.ChainEpoch) *dline.Info { close += miner.WPoStChallengeWindow dlIdx++ } - return miner.NewDeadlineInfo(0, dlIdx, currentEpoch) + return NewDeadlineInfo(0, dlIdx, currentEpoch) } func (m *mockAPI) StateMinerProvingDeadline(ctx context.Context, address address.Address, key types.TipSetKey) (*dline.Info, error) { @@ -355,7 +355,7 @@ func TestChangeHandlerDontStartUntilProvingPeriod(t *testing.T) { periodStart := miner.WPoStProvingPeriod dlIdx := uint64(1) currentEpoch := abi.ChainEpoch(10) - di := miner.NewDeadlineInfo(periodStart, dlIdx, currentEpoch) + di := NewDeadlineInfo(periodStart, dlIdx, currentEpoch) mock.setDeadline(di) defer s.ch.shutdown() @@ -375,7 +375,7 @@ func TestChangeHandlerDontStartUntilProvingPeriod(t *testing.T) { // Advance the head to the next proving period's first epoch currentEpoch = periodStart + miner.WPoStChallengeWindow - di = miner.NewDeadlineInfo(periodStart, dlIdx, currentEpoch) + di = NewDeadlineInfo(periodStart, dlIdx, currentEpoch) mock.setDeadline(di) go triggerHeadAdvance(t, s, currentEpoch) diff --git a/storage/wdpost_nextdl_test.go b/storage/wdpost_nextdl_test.go index ad4b1fdeb..4a23bad65 100644 --- a/storage/wdpost_nextdl_test.go +++ b/storage/wdpost_nextdl_test.go @@ -6,7 +6,7 @@ import ( "github.com/stretchr/testify/require" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" ) func TestNextDeadline(t *testing.T) { @@ -14,7 +14,7 @@ func TestNextDeadline(t *testing.T) { deadlineIdx := 0 currentEpoch := abi.ChainEpoch(10) - di := miner.NewDeadlineInfo(periodStart, uint64(deadlineIdx), currentEpoch) + di := NewDeadlineInfo(periodStart, uint64(deadlineIdx), currentEpoch) require.EqualValues(t, 0, di.Index) require.EqualValues(t, 0, di.PeriodStart) require.EqualValues(t, -20, di.Challenge) From bdc782617f1547b1e9631ba34ba8fba917ce767e Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 25 Sep 2020 11:27:30 -0700 Subject: [PATCH 335/885] return an error when we fail to find a sector when checking sector expiration Returning nil, nil is a footgun. fix: https://github.com/filecoin-project/lotus/issues/3984 --- chain/actors/builtin/miner/v0.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/chain/actors/builtin/miner/v0.go b/chain/actors/builtin/miner/v0.go index 9cdfc25bc..f5aa7849d 100644 --- a/chain/actors/builtin/miner/v0.go +++ b/chain/actors/builtin/miner/v0.go @@ -6,6 +6,7 @@ import ( "github.com/libp2p/go-libp2p-core/peer" cbg "github.com/whyrusleeping/cbor-gen" + "golang.org/x/xerrors" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-bitfield" @@ -96,9 +97,7 @@ func (s *state0) NumLiveSectors() (uint64, error) { // GetSectorExpiration returns the effective expiration of the given sector. // -// If the sector isn't found or has already been terminated, this method returns -// nil and no error. If the sector does not expire early, the Early expiration -// field is 0. +// If the sector does not expire early, the Early expiration field is 0. func (s *state0) GetSectorExpiration(num abi.SectorNumber) (*SectorExpiration, error) { dls, err := s.State.LoadDeadlines(s.store) if err != nil { @@ -161,7 +160,7 @@ func (s *state0) GetSectorExpiration(num abi.SectorNumber) (*SectorExpiration, e return nil, err } if out.Early == 0 && out.OnTime == 0 { - return nil, nil + return nil, xerrors.Errorf("failed to find sector %d", num) } return &out, nil } From ddcbcdea4809eb2e0c153207f9c8d1116d9cd1e8 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 25 Sep 2020 12:16:35 -0700 Subject: [PATCH 336/885] test sector status on expiring sectors --- api/test/ccupgrade.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/api/test/ccupgrade.go b/api/test/ccupgrade.go index b281f30a0..f58f1ff6e 100644 --- a/api/test/ccupgrade.go +++ b/api/test/ccupgrade.go @@ -94,6 +94,22 @@ func TestCCUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration) { require.Less(t, 50000, int(exp.OnTime)) } + dlInfo, err := client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK) + require.NoError(t, err) + + // Sector should expire. + for { + // Wait for the sector to expire. + status, err := miner.SectorsStatus(ctx, CC, true) + require.NoError(t, err) + if status.OnTime == 0 && status.Early == 0 { + break + } + t.Log("waiting for sector to expire") + // wait one deadline per loop. + time.Sleep(time.Duration(dlInfo.WPoStChallengeWindow) * blocktime) + } + fmt.Println("shutting down mining") atomic.AddInt64(&mine, -1) <-done From 60e43ccbb13a88473f91b80cf645758523658ad2 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Fri, 25 Sep 2020 15:45:27 -0400 Subject: [PATCH 337/885] Add an envvar to set address network version --- build/params_shared_funcs.go | 6 ++++++ build/params_shared_vals.go | 12 +++++++++++ chain/types_test.go | 40 ++++++++++++++++++++++++++++++++++++ go.mod | 2 +- go.sum | 2 ++ 5 files changed, 61 insertions(+), 1 deletion(-) diff --git a/build/params_shared_funcs.go b/build/params_shared_funcs.go index 0e9739914..28567d3d1 100644 --- a/build/params_shared_funcs.go +++ b/build/params_shared_funcs.go @@ -3,6 +3,8 @@ package build import ( "sort" + "github.com/filecoin-project/go-address" + "github.com/libp2p/go-libp2p-core/protocol" "github.com/filecoin-project/go-state-types/abi" @@ -44,3 +46,7 @@ func UseNewestNetwork() bool { } return false } + +func SetAddressNetwork(n address.Network) { + address.CurrentNetwork = n +} diff --git a/build/params_shared_vals.go b/build/params_shared_vals.go index 3ee9f52ec..5dd23de44 100644 --- a/build/params_shared_vals.go +++ b/build/params_shared_vals.go @@ -4,6 +4,9 @@ package build import ( "math/big" + "os" + + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/network" @@ -60,6 +63,11 @@ const TicketRandomnessLookback = abi.ChainEpoch(1) const WinningPoStSectorSetLookback = abi.ChainEpoch(10) +// ///// +// Address + +const AddressMainnetEnvVar = "_mainnet_" + // ///// // Devnet settings @@ -75,6 +83,10 @@ var InitialRewardBalance *big.Int func init() { InitialRewardBalance = big.NewInt(int64(FilAllocStorageMining)) InitialRewardBalance = InitialRewardBalance.Mul(InitialRewardBalance, big.NewInt(int64(FilecoinPrecision))) + + if os.Getenv("LOTUS_ADDRESS_TYPE") == AddressMainnetEnvVar { + SetAddressNetwork(address.Mainnet) + } } // Sync diff --git a/chain/types_test.go b/chain/types_test.go index 7d68da68d..b47471c9d 100644 --- a/chain/types_test.go +++ b/chain/types_test.go @@ -1,9 +1,12 @@ package chain import ( + "crypto/rand" "encoding/json" "testing" + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" ) @@ -35,3 +38,40 @@ func TestSignedMessageJsonRoundtrip(t *testing.T) { t.Fatal(err) } } + +func TestAddressType(t *testing.T) { + build.SetAddressNetwork(address.Testnet) + addr, err := makeRandomAddress() + if err != nil { + t.Fatal(err) + } + + if string(addr[0]) != address.TestnetPrefix { + t.Fatalf("address should start with %s", address.TestnetPrefix) + } + + build.SetAddressNetwork(address.Mainnet) + addr, err = makeRandomAddress() + if err != nil { + t.Fatal(err) + } + + if string(addr[0]) != address.MainnetPrefix { + t.Fatalf("address should start with %s", address.MainnetPrefix) + } +} + +func makeRandomAddress() (string, error) { + bytes := make([]byte, 32) + _, err := rand.Read(bytes) + if err != nil { + return "", err + } + + addr, err := address.NewActorAddress(bytes) + if err != nil { + return "", err + } + + return addr.String(), nil +} diff --git a/go.mod b/go.mod index ca7e0760d..cfa6bf47f 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/elastic/go-sysinfo v1.3.0 github.com/fatih/color v1.8.0 github.com/filecoin-project/filecoin-ffi v0.30.4-0.20200716204036-cddc56607e1d - github.com/filecoin-project/go-address v0.0.3 + github.com/filecoin-project/go-address v0.0.4 github.com/filecoin-project/go-bitfield v0.2.0 github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 diff --git a/go.sum b/go.sum index 1ca615b59..7e50fc200 100644 --- a/go.sum +++ b/go.sum @@ -214,6 +214,8 @@ github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGj github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E= github.com/filecoin-project/go-address v0.0.3 h1:eVfbdjEbpbzIrbiSa+PiGUY+oDK9HnUn+M1R/ggoHf8= github.com/filecoin-project/go-address v0.0.3/go.mod h1:jr8JxKsYx+lQlQZmF5i2U0Z+cGQ59wMIps/8YW/lDj8= +github.com/filecoin-project/go-address v0.0.4 h1:gSNMv0qWwH16fGQs7ycOUrDjY6YCSsgLUl0I0KLjo8w= +github.com/filecoin-project/go-address v0.0.4/go.mod h1:jr8JxKsYx+lQlQZmF5i2U0Z+cGQ59wMIps/8YW/lDj8= github.com/filecoin-project/go-amt-ipld/v2 v2.1.0 h1:t6qDiuGYYngDqaLc2ZUvdtAg4UNxPeOYaXhBWSNsVaM= github.com/filecoin-project/go-amt-ipld/v2 v2.1.0/go.mod h1:nfFPoGyX0CU9SkXX8EoCcSuHN1XcbN0c6KBh7yvP5fs= github.com/filecoin-project/go-bitfield v0.2.0 h1:gCtLcjskIPtdg4NfN7gQZSQF9yrBQ7mkT0qCJxzGI2Q= From ebad0ded3d394e9e1c87d8e5f700628d3ad70c7a Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Fri, 18 Sep 2020 23:46:03 -0400 Subject: [PATCH 338/885] Introduce v1 actors --- chain/actors/builtin/account/account.go | 8 + chain/actors/builtin/account/v1.go | 18 + chain/actors/builtin/builtin.go | 5 + chain/actors/builtin/init/init.go | 8 + chain/actors/builtin/init/v1.go | 47 +++ chain/actors/builtin/market/market.go | 13 +- chain/actors/builtin/market/v1.go | 186 ++++++++++ chain/actors/builtin/miner/v1.go | 403 ++++++++++++++++++++++ chain/actors/builtin/multisig/multisig.go | 8 + chain/actors/builtin/multisig/v1.go | 30 ++ chain/actors/builtin/paych/paych.go | 9 + chain/actors/builtin/paych/v1.go | 91 +++++ chain/actors/builtin/power/v1.go | 85 +++++ chain/actors/builtin/reward/reward.go | 8 + chain/actors/builtin/reward/v1.go | 74 ++++ chain/actors/builtin/verifreg/v1.go | 39 +++ chain/actors/builtin/verifreg/verifreg.go | 8 + chain/stmgr/utils.go | 26 ++ chain/vm/invoker.go | 26 +- go.mod | 1 + go.sum | 6 + 21 files changed, 1096 insertions(+), 3 deletions(-) create mode 100644 chain/actors/builtin/account/v1.go create mode 100644 chain/actors/builtin/init/v1.go create mode 100644 chain/actors/builtin/market/v1.go create mode 100644 chain/actors/builtin/miner/v1.go create mode 100644 chain/actors/builtin/multisig/v1.go create mode 100644 chain/actors/builtin/paych/v1.go create mode 100644 chain/actors/builtin/power/v1.go create mode 100644 chain/actors/builtin/reward/v1.go create mode 100644 chain/actors/builtin/verifreg/v1.go diff --git a/chain/actors/builtin/account/account.go b/chain/actors/builtin/account/account.go index 5b90580ec..07f4c4fc7 100644 --- a/chain/actors/builtin/account/account.go +++ b/chain/actors/builtin/account/account.go @@ -6,6 +6,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/cbor" builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" + builtin1 "github.com/filecoin-project/specs-actors/v2/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/types" @@ -20,6 +21,13 @@ func Load(store adt.Store, act *types.Actor) (State, error) { return nil, err } return &out, nil + case builtin1.AccountActorCodeID: + out := state1{store: store} + err := store.Get(store.Context(), act.Head, &out) + if err != nil { + return nil, err + } + return &out, nil } return nil, xerrors.Errorf("unknown actor code %s", act.Code) } diff --git a/chain/actors/builtin/account/v1.go b/chain/actors/builtin/account/v1.go new file mode 100644 index 000000000..ed0fb0104 --- /dev/null +++ b/chain/actors/builtin/account/v1.go @@ -0,0 +1,18 @@ +package account + +import ( + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/specs-actors/v2/actors/builtin/account" +) + +var _ State = (*state1)(nil) + +type state1 struct { + account.State + store adt.Store +} + +func (s *state1) PubkeyAddress() (address.Address, error) { + return s.Address, nil +} diff --git a/chain/actors/builtin/builtin.go b/chain/actors/builtin/builtin.go index 4079e694a..e25cfcf36 100644 --- a/chain/actors/builtin/builtin.go +++ b/chain/actors/builtin/builtin.go @@ -8,6 +8,7 @@ import ( proof0 "github.com/filecoin-project/specs-actors/actors/runtime/proof" smoothing0 "github.com/filecoin-project/specs-actors/actors/util/smoothing" + smoothing1 "github.com/filecoin-project/specs-actors/v2/actors/util/smoothing" "github.com/filecoin-project/go-state-types/network" ) @@ -41,3 +42,7 @@ func FromV0FilterEstimate(v0 smoothing0.FilterEstimate) FilterEstimate { func QAPowerForWeight(size abi.SectorSize, duration abi.ChainEpoch, dealWeight, verifiedWeight abi.DealWeight) abi.StoragePower { return miner0.QAPowerForWeight(size, duration, dealWeight, verifiedWeight) } + +func FromV1FilterEstimate(v1 smoothing1.FilterEstimate) FilterEstimate { + return (FilterEstimate)(v1) +} diff --git a/chain/actors/builtin/init/init.go b/chain/actors/builtin/init/init.go index f235450c2..a9fd25a1a 100644 --- a/chain/actors/builtin/init/init.go +++ b/chain/actors/builtin/init/init.go @@ -7,6 +7,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/cbor" builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" + builtin1 "github.com/filecoin-project/specs-actors/v2/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/types" @@ -24,6 +25,13 @@ func Load(store adt.Store, act *types.Actor) (State, error) { return nil, err } return &out, nil + case builtin1.InitActorCodeID: + out := state1{store: store} + err := store.Get(store.Context(), act.Head, &out) + if err != nil { + return nil, err + } + return &out, nil } return nil, xerrors.Errorf("unknown actor code %s", act.Code) } diff --git a/chain/actors/builtin/init/v1.go b/chain/actors/builtin/init/v1.go new file mode 100644 index 000000000..d11071bdd --- /dev/null +++ b/chain/actors/builtin/init/v1.go @@ -0,0 +1,47 @@ +package init + +import ( + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + cbg "github.com/whyrusleeping/cbor-gen" + + "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/node/modules/dtypes" + + init_ "github.com/filecoin-project/specs-actors/v2/actors/builtin/init" + adt1 "github.com/filecoin-project/specs-actors/v2/actors/util/adt" +) + +var _ State = (*state1)(nil) + +type state1 struct { + init_.State + store adt.Store +} + +func (s *state1) ResolveAddress(address address.Address) (address.Address, bool, error) { + return s.State.ResolveAddress(s.store, address) +} + +func (s *state1) MapAddressToNewID(address address.Address) (address.Address, error) { + return s.State.MapAddressToNewID(s.store, address) +} + +func (s *state1) ForEachActor(cb func(id abi.ActorID, address address.Address) error) error { + addrs, err := adt1.AsMap(s.store, s.State.AddressMap) + if err != nil { + return err + } + var actorID cbg.CborInt + return addrs.ForEach(&actorID, func(key string) error { + addr, err := address.NewFromBytes([]byte(key)) + if err != nil { + return err + } + return cb(abi.ActorID(actorID), addr) + }) +} + +func (s *state1) NetworkName() (dtypes.NetworkName, error) { + return dtypes.NetworkName(s.State.NetworkName), nil +} diff --git a/chain/actors/builtin/market/market.go b/chain/actors/builtin/market/market.go index fef0c03f9..b1c9d7a7e 100644 --- a/chain/actors/builtin/market/market.go +++ b/chain/actors/builtin/market/market.go @@ -6,11 +6,13 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/cbor" - builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" - market0 "github.com/filecoin-project/specs-actors/actors/builtin/market" "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" + market0 "github.com/filecoin-project/specs-actors/actors/builtin/market" + builtin1 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/types" ) @@ -26,6 +28,13 @@ func Load(store adt.Store, act *types.Actor) (st State, err error) { return nil, err } return &out, nil + case builtin1.StorageMarketActorCodeID: + out := state1{store: store} + err := store.Get(store.Context(), act.Head, &out) + if err != nil { + return nil, err + } + return &out, nil } return nil, xerrors.Errorf("unknown actor code %s", act.Code) } diff --git a/chain/actors/builtin/market/v1.go b/chain/actors/builtin/market/v1.go new file mode 100644 index 000000000..8a634ddd2 --- /dev/null +++ b/chain/actors/builtin/market/v1.go @@ -0,0 +1,186 @@ +package market + +import ( + "bytes" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v2/actors/builtin/market" + adt1 "github.com/filecoin-project/specs-actors/v2/actors/util/adt" + cbg "github.com/whyrusleeping/cbor-gen" +) + +var _ State = (*state1)(nil) + +type state1 struct { + market.State + store adt.Store +} + +func (s *state1) TotalLocked() (abi.TokenAmount, error) { + fml := types.BigAdd(s.TotalClientLockedCollateral, s.TotalProviderLockedCollateral) + fml = types.BigAdd(fml, s.TotalClientStorageFee) + return fml, nil +} + +func (s *state1) BalancesChanged(otherState State) bool { + otherState1, ok := otherState.(*state1) + if !ok { + // there's no way to compare different versions of the state, so let's + // just say that means the state of balances has changed + return true + } + return !s.State.EscrowTable.Equals(otherState1.State.EscrowTable) || !s.State.LockedTable.Equals(otherState1.State.LockedTable) +} + +func (s *state1) StatesChanged(otherState State) bool { + otherState1, ok := otherState.(*state1) + if !ok { + // there's no way to compare different versions of the state, so let's + // just say that means the state of balances has changed + return true + } + return !s.State.States.Equals(otherState1.State.States) +} + +func (s *state1) States() (DealStates, error) { + stateArray, err := adt1.AsArray(s.store, s.State.States) + if err != nil { + return nil, err + } + return &dealStates1{stateArray}, nil +} + +func (s *state1) ProposalsChanged(otherState State) bool { + otherState1, ok := otherState.(*state1) + if !ok { + // there's no way to compare different versions of the state, so let's + // just say that means the state of balances has changed + return true + } + return !s.State.Proposals.Equals(otherState1.State.Proposals) +} + +func (s *state1) Proposals() (DealProposals, error) { + proposalArray, err := adt1.AsArray(s.store, s.State.Proposals) + if err != nil { + return nil, err + } + return &dealProposals1{proposalArray}, nil +} + +func (s *state1) EscrowTable() (BalanceTable, error) { + bt, err := adt1.AsBalanceTable(s.store, s.State.EscrowTable) + if err != nil { + return nil, err + } + return &balanceTable1{bt}, nil +} + +func (s *state1) LockedTable() (BalanceTable, error) { + bt, err := adt1.AsBalanceTable(s.store, s.State.LockedTable) + if err != nil { + return nil, err + } + return &balanceTable1{bt}, nil +} + +func (s *state1) VerifyDealsForActivation( + minerAddr address.Address, deals []abi.DealID, currEpoch, sectorExpiry abi.ChainEpoch, +) (weight, verifiedWeight abi.DealWeight, err error) { + w, vw, _, err := market.ValidateDealsForActivation(&s.State, s.store, deals, minerAddr, sectorExpiry, currEpoch) + return w, vw, err +} + +type balanceTable1 struct { + *adt1.BalanceTable +} + +func (bt *balanceTable1) ForEach(cb func(address.Address, abi.TokenAmount) error) error { + asMap := (*adt1.Map)(bt.BalanceTable) + var ta abi.TokenAmount + return asMap.ForEach(&ta, func(key string) error { + a, err := address.NewFromBytes([]byte(key)) + if err != nil { + return err + } + return cb(a, ta) + }) +} + +type dealStates1 struct { + adt.Array +} + +func (s *dealStates1) Get(dealID abi.DealID) (*DealState, bool, error) { + var deal1 market.DealState + found, err := s.Array.Get(uint64(dealID), &deal1) + if err != nil { + return nil, false, err + } + if !found { + return nil, false, nil + } + deal := fromV1DealState(deal1) + return &deal, true, nil +} + +func (s *dealStates1) decode(val *cbg.Deferred) (*DealState, error) { + var ds1 market.DealState + if err := ds1.UnmarshalCBOR(bytes.NewReader(val.Raw)); err != nil { + return nil, err + } + ds := fromV1DealState(ds1) + return &ds, nil +} + +func (s *dealStates1) array() adt.Array { + return s.Array +} + +func fromV1DealState(v1 market.DealState) DealState { + return (DealState)(v1) +} + +type dealProposals1 struct { + adt.Array +} + +func (s *dealProposals1) Get(dealID abi.DealID) (*DealProposal, bool, error) { + var proposal1 market.DealProposal + found, err := s.Array.Get(uint64(dealID), &proposal1) + if err != nil { + return nil, false, err + } + if !found { + return nil, false, nil + } + proposal := fromV1DealProposal(proposal1) + return &proposal, true, nil +} + +func (s *dealProposals1) ForEach(cb func(dealID abi.DealID, dp DealProposal) error) error { + var dp1 market.DealProposal + return s.Array.ForEach(&dp1, func(idx int64) error { + return cb(abi.DealID(idx), fromV1DealProposal(dp1)) + }) +} + +func (s *dealProposals1) decode(val *cbg.Deferred) (*DealProposal, error) { + var dp1 market.DealProposal + if err := dp1.UnmarshalCBOR(bytes.NewReader(val.Raw)); err != nil { + return nil, err + } + dp := fromV1DealProposal(dp1) + return &dp, nil +} + +func (s *dealProposals1) array() adt.Array { + return s.Array +} + +func fromV1DealProposal(v1 market.DealProposal) DealProposal { + return (DealProposal)(v1) +} diff --git a/chain/actors/builtin/miner/v1.go b/chain/actors/builtin/miner/v1.go new file mode 100644 index 000000000..9c0486eb0 --- /dev/null +++ b/chain/actors/builtin/miner/v1.go @@ -0,0 +1,403 @@ +package miner + +import ( + "bytes" + "errors" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-bitfield" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/dline" + "github.com/filecoin-project/lotus/chain/actors/adt" + adt1 "github.com/filecoin-project/specs-actors/v2/actors/util/adt" + "github.com/libp2p/go-libp2p-core/peer" + cbg "github.com/whyrusleeping/cbor-gen" + "golang.org/x/xerrors" + + miner1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" +) + +var _ State = (*state1)(nil) + +type state1 struct { + miner1.State + store adt.Store +} + +type deadline1 struct { + miner1.Deadline + store adt.Store +} + +type partition1 struct { + miner1.Partition + store adt.Store +} + +func (s *state1) AvailableBalance(bal abi.TokenAmount) (abi.TokenAmount, error) { + return s.GetAvailableBalance(bal), nil +} + +func (s *state1) VestedFunds(epoch abi.ChainEpoch) (abi.TokenAmount, error) { + return s.CheckVestedFunds(s.store, epoch) +} + +func (s *state1) LockedFunds() (LockedFunds, error) { + return LockedFunds{ + VestingFunds: s.State.LockedFunds, + InitialPledgeRequirement: s.State.InitialPledge, + PreCommitDeposits: s.State.PreCommitDeposits, + }, nil +} + +func (s *state1) InitialPledge() (abi.TokenAmount, error) { + return s.State.InitialPledge, nil +} + +func (s *state1) PreCommitDeposits() (abi.TokenAmount, error) { + return s.State.PreCommitDeposits, nil +} + +func (s *state1) GetSector(num abi.SectorNumber) (*SectorOnChainInfo, error) { + info, ok, err := s.State.GetSector(s.store, num) + if !ok || err != nil { + return nil, err + } + + ret := fromV1SectorOnChainInfo(*info) + return &ret, nil +} + +func (s *state1) FindSector(num abi.SectorNumber) (*SectorLocation, error) { + dlIdx, partIdx, err := s.State.FindSector(s.store, num) + if err != nil { + return nil, err + } + return &SectorLocation{ + Deadline: dlIdx, + Partition: partIdx, + }, nil +} + +// GetSectorExpiration returns the effective expiration of the given sector. +// +// If the sector isn't found or has already been terminated, this method returns +// nil and no error. If the sector does not expire early, the Early expiration +// field is 0. +func (s *state1) GetSectorExpiration(num abi.SectorNumber) (*SectorExpiration, error) { + dls, err := s.State.LoadDeadlines(s.store) + if err != nil { + return nil, err + } + // NOTE: this can be optimized significantly. + // 1. If the sector is non-faulty, it will either expire on-time (can be + // learned from the sector info), or in the next quantized expiration + // epoch (i.e., the first element in the partition's expiration queue. + // 2. If it's faulty, it will expire early within the first 14 entries + // of the expiration queue. + stopErr := errors.New("stop") + out := SectorExpiration{} + err = dls.ForEach(s.store, func(dlIdx uint64, dl *miner1.Deadline) error { + partitions, err := dl.PartitionsArray(s.store) + if err != nil { + return err + } + quant := s.State.QuantSpecForDeadline(dlIdx) + var part miner1.Partition + return partitions.ForEach(&part, func(partIdx int64) error { + if found, err := part.Sectors.IsSet(uint64(num)); err != nil { + return err + } else if !found { + return nil + } + if found, err := part.Terminated.IsSet(uint64(num)); err != nil { + return err + } else if found { + // already terminated + return stopErr + } + + q, err := miner1.LoadExpirationQueue(s.store, part.ExpirationsEpochs, quant) + if err != nil { + return err + } + var exp miner1.ExpirationSet + return q.ForEach(&exp, func(epoch int64) error { + if early, err := exp.EarlySectors.IsSet(uint64(num)); err != nil { + return err + } else if early { + out.Early = abi.ChainEpoch(epoch) + return nil + } + if onTime, err := exp.OnTimeSectors.IsSet(uint64(num)); err != nil { + return err + } else if onTime { + out.OnTime = abi.ChainEpoch(epoch) + return stopErr + } + return nil + }) + }) + }) + if err == stopErr { + err = nil + } + if err != nil { + return nil, err + } + if out.Early == 0 && out.OnTime == 0 { + return nil, nil + } + return &out, nil +} + +func (s *state1) GetPrecommittedSector(num abi.SectorNumber) (*SectorPreCommitOnChainInfo, error) { + info, ok, err := s.State.GetPrecommittedSector(s.store, num) + if !ok || err != nil { + return nil, err + } + + ret := fromV1SectorPreCommitOnChainInfo(*info) + + return &ret, nil +} + +func (s *state1) LoadSectorsFromSet(filter *bitfield.BitField, filterOut bool) (adt.Array, error) { + a, err := adt1.AsArray(s.store, s.State.Sectors) + if err != nil { + return nil, err + } + + ret := adt1.MakeEmptyArray(s.store) + var v cbg.Deferred + if err := a.ForEach(&v, func(i int64) error { + include := true + if filter != nil { + set, err := filter.IsSet(uint64(i)) + if err != nil { + return xerrors.Errorf("filter check error: %w", err) + } + if set == filterOut { + include = false + } + } + + if include { + var oci miner1.SectorOnChainInfo + if err := oci.UnmarshalCBOR(bytes.NewReader(v.Raw)); err != nil { + return err + } + + noci := SectorOnChainInfo{ + SectorNumber: oci.SectorNumber, + SealProof: oci.SealProof, + SealedCID: oci.SealedCID, + DealIDs: oci.DealIDs, + Activation: oci.Activation, + Expiration: oci.Expiration, + DealWeight: oci.DealWeight, + VerifiedDealWeight: oci.VerifiedDealWeight, + InitialPledge: oci.InitialPledge, + ExpectedDayReward: oci.ExpectedDayReward, + ExpectedStoragePledge: oci.ExpectedStoragePledge, + } + + if err := ret.Set(uint64(i), &noci); err != nil { + return err + } + } + return nil + }); err != nil { + return nil, err + } + + return ret, nil +} + +func (s *state1) LoadPreCommittedSectors() (adt.Map, error) { + return adt1.AsMap(s.store, s.State.PreCommittedSectors) +} + +func (s *state1) IsAllocated(num abi.SectorNumber) (bool, error) { + var allocatedSectors bitfield.BitField + if err := s.store.Get(s.store.Context(), s.State.AllocatedSectors, &allocatedSectors); err != nil { + return false, err + } + + return allocatedSectors.IsSet(uint64(num)) +} + +func (s *state1) LoadDeadline(idx uint64) (Deadline, error) { + dls, err := s.State.LoadDeadlines(s.store) + if err != nil { + return nil, err + } + dl, err := dls.LoadDeadline(s.store, idx) + if err != nil { + return nil, err + } + return &deadline1{*dl, s.store}, nil +} + +func (s *state1) ForEachDeadline(cb func(uint64, Deadline) error) error { + dls, err := s.State.LoadDeadlines(s.store) + if err != nil { + return err + } + return dls.ForEach(s.store, func(i uint64, dl *miner1.Deadline) error { + return cb(i, &deadline1{*dl, s.store}) + }) +} + +func (s *state1) NumDeadlines() (uint64, error) { + return miner1.WPoStPeriodDeadlines, nil +} + +func (s *state1) DeadlinesChanged(other State) bool { + other1, ok := other.(*state1) + if !ok { + // treat an upgrade as a change, always + return true + } + + return s.State.Deadlines.Equals(other1.Deadlines) +} + +func (s *state1) Info() (MinerInfo, error) { + info, err := s.State.GetInfo(s.store) + if err != nil { + return MinerInfo{}, err + } + + var pid *peer.ID + if peerID, err := peer.IDFromBytes(info.PeerId); err == nil { + pid = &peerID + } + + mi := MinerInfo{ + Owner: info.Owner, + Worker: info.Worker, + ControlAddresses: info.ControlAddresses, + + NewWorker: address.Undef, + WorkerChangeEpoch: -1, + + PeerId: pid, + Multiaddrs: info.Multiaddrs, + SealProofType: info.SealProofType, + SectorSize: info.SectorSize, + WindowPoStPartitionSectors: info.WindowPoStPartitionSectors, + } + + if info.PendingWorkerKey != nil { + mi.NewWorker = info.PendingWorkerKey.NewWorker + mi.WorkerChangeEpoch = info.PendingWorkerKey.EffectiveAt + } + + return mi, nil +} + +func (s *state1) DeadlineInfo(epoch abi.ChainEpoch) *dline.Info { + return s.State.DeadlineInfo(epoch) +} + +func (s *state1) sectors() (adt.Array, error) { + return adt1.AsArray(s.store, s.Sectors) +} + +func (s *state1) decodeSectorOnChainInfo(val *cbg.Deferred) (SectorOnChainInfo, error) { + var si miner1.SectorOnChainInfo + err := si.UnmarshalCBOR(bytes.NewReader(val.Raw)) + if err != nil { + return SectorOnChainInfo{}, err + } + + ret := fromV1SectorOnChainInfo(si) + return ret, nil +} + +func (s *state1) precommits() (adt.Map, error) { + return adt1.AsMap(s.store, s.PreCommittedSectors) +} + +func (s *state1) decodeSectorPreCommitOnChainInfo(val *cbg.Deferred) (SectorPreCommitOnChainInfo, error) { + var sp miner1.SectorPreCommitOnChainInfo + err := sp.UnmarshalCBOR(bytes.NewReader(val.Raw)) + if err != nil { + return SectorPreCommitOnChainInfo{}, err + } + + ret := fromV1SectorPreCommitOnChainInfo(sp) + return ret, nil +} + +func (d *deadline1) LoadPartition(idx uint64) (Partition, error) { + p, err := d.Deadline.LoadPartition(d.store, idx) + if err != nil { + return nil, err + } + return &partition1{*p, d.store}, nil +} + +func (d *deadline1) ForEachPartition(cb func(uint64, Partition) error) error { + ps, err := d.Deadline.PartitionsArray(d.store) + if err != nil { + return err + } + var part miner1.Partition + return ps.ForEach(&part, func(i int64) error { + return cb(uint64(i), &partition1{part, d.store}) + }) +} + +func (d *deadline1) PartitionsChanged(other Deadline) bool { + other1, ok := other.(*deadline1) + if !ok { + // treat an upgrade as a change, always + return true + } + + return d.Deadline.Partitions.Equals(other1.Deadline.Partitions) +} + +func (d *deadline1) PostSubmissions() (bitfield.BitField, error) { + return d.Deadline.PostSubmissions, nil +} + +func (p *partition1) AllSectors() (bitfield.BitField, error) { + return p.Partition.Sectors, nil +} + +func (p *partition1) FaultySectors() (bitfield.BitField, error) { + return p.Partition.Faults, nil +} + +func (p *partition1) RecoveringSectors() (bitfield.BitField, error) { + return p.Partition.Recoveries, nil +} + +func fromV1SectorOnChainInfo(v1 miner1.SectorOnChainInfo) SectorOnChainInfo { + return SectorOnChainInfo{ + SectorNumber: v1.SectorNumber, + SealProof: v1.SealProof, + SealedCID: v1.SealedCID, + DealIDs: v1.DealIDs, + Activation: v1.Activation, + Expiration: v1.Expiration, + DealWeight: v1.DealWeight, + VerifiedDealWeight: v1.VerifiedDealWeight, + InitialPledge: v1.InitialPledge, + ExpectedDayReward: v1.ExpectedDayReward, + ExpectedStoragePledge: v1.ExpectedStoragePledge, + } +} + +func fromV1SectorPreCommitOnChainInfo(v1 miner1.SectorPreCommitOnChainInfo) SectorPreCommitOnChainInfo { + return SectorPreCommitOnChainInfo{ + Info: (SectorPreCommitInfo)(v1.Info), + PreCommitDeposit: v1.PreCommitDeposit, + PreCommitEpoch: v1.PreCommitEpoch, + DealWeight: v1.DealWeight, + VerifiedDealWeight: v1.VerifiedDealWeight, + } +} diff --git a/chain/actors/builtin/multisig/multisig.go b/chain/actors/builtin/multisig/multisig.go index 884b6f493..a30082824 100644 --- a/chain/actors/builtin/multisig/multisig.go +++ b/chain/actors/builtin/multisig/multisig.go @@ -9,6 +9,7 @@ import ( builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" msig0 "github.com/filecoin-project/specs-actors/actors/builtin/multisig" + builtin1 "github.com/filecoin-project/specs-actors/v2/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/types" @@ -23,6 +24,13 @@ func Load(store adt.Store, act *types.Actor) (State, error) { return nil, err } return &out, nil + case builtin1.MultisigActorCodeID: + out := state1{store: store} + err := store.Get(store.Context(), act.Head, &out) + if err != nil { + return nil, err + } + return &out, nil } return nil, xerrors.Errorf("unknown actor code %s", act.Code) } diff --git a/chain/actors/builtin/multisig/v1.go b/chain/actors/builtin/multisig/v1.go new file mode 100644 index 000000000..cf2d67453 --- /dev/null +++ b/chain/actors/builtin/multisig/v1.go @@ -0,0 +1,30 @@ +package multisig + +import ( + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/specs-actors/v2/actors/builtin/multisig" +) + +var _ State = (*state1)(nil) + +type state1 struct { + multisig.State + store adt.Store +} + +func (s *state1) LockedBalance(currEpoch abi.ChainEpoch) (abi.TokenAmount, error) { + return s.State.AmountLocked(currEpoch - s.State.StartEpoch), nil +} + +func (s *state1) StartEpoch() abi.ChainEpoch { + return s.State.StartEpoch +} + +func (s *state1) UnlockDuration() abi.ChainEpoch { + return s.State.UnlockDuration +} + +func (s *state1) InitialBalance() abi.TokenAmount { + return s.State.InitialBalance +} diff --git a/chain/actors/builtin/paych/paych.go b/chain/actors/builtin/paych/paych.go index dad54163f..36cf56e63 100644 --- a/chain/actors/builtin/paych/paych.go +++ b/chain/actors/builtin/paych/paych.go @@ -7,8 +7,10 @@ import ( "github.com/filecoin-project/go-state-types/abi" big "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/cbor" + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" paych0 "github.com/filecoin-project/specs-actors/actors/builtin/paych" + builtin1 "github.com/filecoin-project/specs-actors/v2/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/types" @@ -24,6 +26,13 @@ func Load(store adt.Store, act *types.Actor) (State, error) { return nil, err } return &out, nil + case builtin1.PaymentChannelActorCodeID: + out := state1{store: store} + err := store.Get(store.Context(), act.Head, &out) + if err != nil { + return nil, err + } + return &out, nil } return nil, xerrors.Errorf("unknown actor code %s", act.Code) } diff --git a/chain/actors/builtin/paych/v1.go b/chain/actors/builtin/paych/v1.go new file mode 100644 index 000000000..bbe3bbebf --- /dev/null +++ b/chain/actors/builtin/paych/v1.go @@ -0,0 +1,91 @@ +package paych + +import ( + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/specs-actors/v2/actors/builtin/paych" + adt1 "github.com/filecoin-project/specs-actors/v2/actors/util/adt" +) + +var _ State = (*state1)(nil) + +type state1 struct { + paych.State + store adt.Store + lsAmt *adt1.Array +} + +// Channel owner, who has funded the actor +func (s *state1) From() address.Address { + return s.State.From +} + +// Recipient of payouts from channel +func (s *state1) To() address.Address { + return s.State.To +} + +// Height at which the channel can be `Collected` +func (s *state1) SettlingAt() abi.ChainEpoch { + return s.State.SettlingAt +} + +// Amount successfully redeemed through the payment channel, paid out on `Collect()` +func (s *state1) ToSend() abi.TokenAmount { + return s.State.ToSend +} + +func (s *state1) getOrLoadLsAmt() (*adt1.Array, error) { + if s.lsAmt != nil { + return s.lsAmt, nil + } + + // Get the lane state from the chain + lsamt, err := adt1.AsArray(s.store, s.State.LaneStates) + if err != nil { + return nil, err + } + + s.lsAmt = lsamt + return lsamt, nil +} + +// Get total number of lanes +func (s *state1) LaneCount() (uint64, error) { + lsamt, err := s.getOrLoadLsAmt() + if err != nil { + return 0, err + } + return lsamt.Length(), nil +} + +// Iterate lane states +func (s *state1) ForEachLaneState(cb func(idx uint64, dl LaneState) error) error { + // Get the lane state from the chain + lsamt, err := s.getOrLoadLsAmt() + if err != nil { + return err + } + + // Note: we use a map instead of an array to store laneStates because the + // client sets the lane ID (the index) and potentially they could use a + // very large index. + var ls paych.LaneState + return lsamt.ForEach(&ls, func(i int64) error { + return cb(uint64(i), &laneState1{ls}) + }) +} + +type laneState1 struct { + paych.LaneState +} + +func (ls *laneState1) Redeemed() big.Int { + return ls.LaneState.Redeemed +} + +func (ls *laneState1) Nonce() uint64 { + return ls.LaneState.Nonce +} diff --git a/chain/actors/builtin/power/v1.go b/chain/actors/builtin/power/v1.go new file mode 100644 index 000000000..85defd053 --- /dev/null +++ b/chain/actors/builtin/power/v1.go @@ -0,0 +1,85 @@ +package power + +import ( + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/actors/builtin" + "github.com/filecoin-project/specs-actors/actors/util/adt" + power1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/power" +) + +var _ State = (*state1)(nil) + +type state1 struct { + power1.State + store adt.Store +} + +func (s *state1) TotalLocked() (abi.TokenAmount, error) { + return s.TotalPledgeCollateral, nil +} + +func (s *state1) TotalPower() (Claim, error) { + return Claim{ + RawBytePower: s.TotalRawBytePower, + QualityAdjPower: s.TotalQualityAdjPower, + }, nil +} + +// Committed power to the network. Includes miners below the minimum threshold. +func (s *state1) TotalCommitted() (Claim, error) { + return Claim{ + RawBytePower: s.TotalBytesCommitted, + QualityAdjPower: s.TotalQABytesCommitted, + }, nil +} + +func (s *state1) MinerPower(addr address.Address) (Claim, bool, error) { + claims, err := adt.AsMap(s.store, s.Claims) + if err != nil { + return Claim{}, false, err + } + var claim power1.Claim + ok, err := claims.Get(abi.AddrKey(addr), &claim) + if err != nil { + return Claim{}, false, err + } + return Claim{ + RawBytePower: claim.RawBytePower, + QualityAdjPower: claim.QualityAdjPower, + }, ok, nil +} + +func (s *state1) MinerNominalPowerMeetsConsensusMinimum(a address.Address) (bool, error) { + return s.State.MinerNominalPowerMeetsConsensusMinimum(s.store, a) +} + +func (s *state1) TotalPowerSmoothed() (builtin.FilterEstimate, error) { + return builtin.FromV1FilterEstimate(s.State.ThisEpochQAPowerSmoothed), nil +} + +func (s *state1) MinerCounts() (uint64, uint64, error) { + return uint64(s.State.MinerAboveMinPowerCount), uint64(s.State.MinerCount), nil +} + +func (s *state1) ListAllMiners() ([]address.Address, error) { + claims, err := adt.AsMap(s.store, s.Claims) + if err != nil { + return nil, err + } + + var miners []address.Address + err = claims.ForEach(nil, func(k string) error { + a, err := address.NewFromBytes([]byte(k)) + if err != nil { + return err + } + miners = append(miners, a) + return nil + }) + if err != nil { + return nil, err + } + + return miners, nil +} diff --git a/chain/actors/builtin/reward/reward.go b/chain/actors/builtin/reward/reward.go index cfa82c774..324bc95dd 100644 --- a/chain/actors/builtin/reward/reward.go +++ b/chain/actors/builtin/reward/reward.go @@ -7,6 +7,7 @@ import ( "github.com/filecoin-project/go-state-types/cbor" builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" + builtin1 "github.com/filecoin-project/specs-actors/v2/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/actors/builtin" @@ -24,6 +25,13 @@ func Load(store adt.Store, act *types.Actor) (st State, err error) { return nil, err } return &out, nil + case builtin1.RewardActorCodeID: + out := state1{store: store} + err := store.Get(store.Context(), act.Head, &out) + if err != nil { + return nil, err + } + return &out, nil } return nil, xerrors.Errorf("unknown actor code %s", act.Code) } diff --git a/chain/actors/builtin/reward/v1.go b/chain/actors/builtin/reward/v1.go new file mode 100644 index 000000000..162dc13a1 --- /dev/null +++ b/chain/actors/builtin/reward/v1.go @@ -0,0 +1,74 @@ +package reward + +import ( + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/actors/builtin" + "github.com/filecoin-project/specs-actors/actors/util/adt" + miner1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" + "github.com/filecoin-project/specs-actors/v2/actors/builtin/reward" + "github.com/filecoin-project/specs-actors/v2/actors/util/smoothing" +) + +var _ State = (*state1)(nil) + +type state1 struct { + reward.State + store adt.Store +} + +func (s *state1) ThisEpochReward() (abi.StoragePower, error) { + return s.State.ThisEpochReward, nil +} + +func (s *state1) ThisEpochRewardSmoothed() (builtin.FilterEstimate, error) { + return builtin.FilterEstimate{ + PositionEstimate: s.State.ThisEpochRewardSmoothed.PositionEstimate, + VelocityEstimate: s.State.ThisEpochRewardSmoothed.VelocityEstimate, + }, nil +} + +func (s *state1) ThisEpochBaselinePower() (abi.StoragePower, error) { + return s.State.ThisEpochBaselinePower, nil +} + +func (s *state1) TotalStoragePowerReward() (abi.TokenAmount, error) { + return s.State.TotalStoragePowerReward, nil +} + +func (s *state1) EffectiveBaselinePower() (abi.StoragePower, error) { + return s.State.EffectiveBaselinePower, nil +} + +func (s *state1) EffectiveNetworkTime() (abi.ChainEpoch, error) { + return s.State.EffectiveNetworkTime, nil +} + +func (s *state1) CumsumBaseline() (abi.StoragePower, error) { + return s.State.CumsumBaseline, nil +} + +func (s *state1) CumsumRealized() (abi.StoragePower, error) { + return s.State.CumsumBaseline, nil +} + +func (s *state1) InitialPledgeForPower(qaPower abi.StoragePower, networkTotalPledge abi.TokenAmount, networkQAPower *builtin.FilterEstimate, circSupply abi.TokenAmount) (abi.TokenAmount, error) { + return miner1.InitialPledgeForPower( + qaPower, + s.State.ThisEpochBaselinePower, + s.State.ThisEpochRewardSmoothed, + smoothing.FilterEstimate{ + PositionEstimate: networkQAPower.PositionEstimate, + VelocityEstimate: networkQAPower.VelocityEstimate, + }, + circSupply, + ), nil +} + +func (s *state1) PreCommitDepositForPower(networkQAPower builtin.FilterEstimate, sectorWeight abi.StoragePower) (abi.TokenAmount, error) { + return miner1.PreCommitDepositForPower(s.State.ThisEpochRewardSmoothed, + smoothing.FilterEstimate{ + PositionEstimate: networkQAPower.PositionEstimate, + VelocityEstimate: networkQAPower.VelocityEstimate, + }, + sectorWeight), nil +} diff --git a/chain/actors/builtin/verifreg/v1.go b/chain/actors/builtin/verifreg/v1.go new file mode 100644 index 000000000..4fb013e1b --- /dev/null +++ b/chain/actors/builtin/verifreg/v1.go @@ -0,0 +1,39 @@ +package verifreg + +import ( + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" + adt1 "github.com/filecoin-project/specs-actors/actors/util/adt" + verifreg1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/verifreg" + "golang.org/x/xerrors" + + "github.com/filecoin-project/lotus/chain/actors/adt" +) + +var _ State = (*state1)(nil) + +type state1 struct { + verifreg1.State + store adt.Store +} + +func (s *state1) VerifiedClientDataCap(addr address.Address) (bool, abi.StoragePower, error) { + if addr.Protocol() != address.ID { + return false, big.Zero(), xerrors.Errorf("can only look up ID addresses") + } + + vh, err := adt1.AsMap(s.store, s.VerifiedClients) + if err != nil { + return false, big.Zero(), xerrors.Errorf("loading verified clients: %w", err) + } + + var dcap abi.StoragePower + if found, err := vh.Get(abi.AddrKey(addr), &dcap); err != nil { + return false, big.Zero(), xerrors.Errorf("looking up verified clients: %w", err) + } else if !found { + return false, big.Zero(), nil + } + + return true, dcap, nil +} diff --git a/chain/actors/builtin/verifreg/verifreg.go b/chain/actors/builtin/verifreg/verifreg.go index c861f862f..4255e8855 100644 --- a/chain/actors/builtin/verifreg/verifreg.go +++ b/chain/actors/builtin/verifreg/verifreg.go @@ -7,6 +7,7 @@ import ( "github.com/filecoin-project/go-state-types/cbor" builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" + builtin1 "github.com/filecoin-project/specs-actors/v2/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/types" @@ -23,6 +24,13 @@ func Load(store adt.Store, act *types.Actor) (State, error) { return nil, err } return &out, nil + case builtin1.VerifiedRegistryActorCodeID: + out := state1{store: store} + err := store.Get(store.Context(), act.Head, &out) + if err != nil { + return nil, err + } + return &out, nil } return nil, xerrors.Errorf("unknown actor code %s", act.Code) } diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index 3493afca3..3c0a895d3 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -31,6 +31,17 @@ import ( verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" proof0 "github.com/filecoin-project/specs-actors/actors/runtime/proof" + builtin1 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + account1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/account" + init1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/init" + market1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/market" + miner1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" + msig1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/multisig" + paych1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/paych" + power1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/power" + reward1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/reward" + verifreg1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/verifreg" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init" @@ -545,6 +556,18 @@ func init() { builtin0.MultisigActorCodeID: {builtin0.MethodsMultisig, msig0.Actor{}}, builtin0.RewardActorCodeID: {builtin0.MethodsReward, reward0.Actor{}}, builtin0.VerifiedRegistryActorCodeID: {builtin0.MethodsVerifiedRegistry, verifreg0.Actor{}}, + + // builtin1.SystemActorCodeID: {builtin1.MethodsSystem, system.Actor{} }- apparently it doesn't have methods + builtin1.InitActorCodeID: {builtin1.MethodsInit, init1.Actor{}}, + builtin1.CronActorCodeID: {builtin1.MethodsCron, cron.Actor{}}, + builtin1.AccountActorCodeID: {builtin1.MethodsAccount, account1.Actor{}}, + builtin1.StoragePowerActorCodeID: {builtin1.MethodsPower, power1.Actor{}}, + builtin1.StorageMinerActorCodeID: {builtin1.MethodsMiner, miner1.Actor{}}, + builtin1.StorageMarketActorCodeID: {builtin1.MethodsMarket, market1.Actor{}}, + builtin1.PaymentChannelActorCodeID: {builtin1.MethodsPaych, paych1.Actor{}}, + builtin1.MultisigActorCodeID: {builtin1.MethodsMultisig, msig1.Actor{}}, + builtin1.RewardActorCodeID: {builtin1.MethodsReward, reward1.Actor{}}, + builtin1.VerifiedRegistryActorCodeID: {builtin1.MethodsVerifiedRegistry, verifreg1.Actor{}}, } for c, m := range cidToMethods { @@ -552,6 +575,7 @@ func init() { methods := make(map[abi.MethodNum]MethodMeta, len(exports)) // Explicitly add send, it's special. + // Note that builtin1.MethodSend = builtin0.MethodSend = 0. methods[builtin0.MethodSend] = MethodMeta{ Name: "Send", Params: reflect.TypeOf(new(abi.EmptyValue)), @@ -592,8 +616,10 @@ func init() { } switch abi.MethodNum(number) { + // Note that builtin1.MethodSend = builtin0.MethodSend = 0. case builtin0.MethodSend: panic("method 0 is reserved for Send") + // Note that builtin1.MethodConstructor = builtin0.MethodConstructor = 1. case builtin0.MethodConstructor: if fnName != "Constructor" { panic("method 1 is reserved for Constructor") diff --git a/chain/vm/invoker.go b/chain/vm/invoker.go index 0a83e273d..bf71a4f74 100644 --- a/chain/vm/invoker.go +++ b/chain/vm/invoker.go @@ -22,7 +22,20 @@ import ( reward0 "github.com/filecoin-project/specs-actors/actors/builtin/reward" system0 "github.com/filecoin-project/specs-actors/actors/builtin/system" verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" + vmr "github.com/filecoin-project/specs-actors/actors/runtime" + builtin1 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + account1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/account" + cron1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/cron" + init1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/init" + market1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/market" + miner1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" + msig1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/multisig" + paych1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/paych" + power1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/power" + reward1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/reward" + system1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/system" + verifreg1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/verifreg" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/exitcode" @@ -45,7 +58,6 @@ func NewInvoker() *Invoker { } // add builtInCode using: register(cid, singleton) - // NETUPGRADE: register code IDs for v2, etc. inv.Register(builtin0.SystemActorCodeID, system0.Actor{}, abi.EmptyValue{}) inv.Register(builtin0.InitActorCodeID, init0.Actor{}, init0.State{}) inv.Register(builtin0.RewardActorCodeID, reward0.Actor{}, reward0.State{}) @@ -58,6 +70,18 @@ func NewInvoker() *Invoker { inv.Register(builtin0.VerifiedRegistryActorCodeID, verifreg0.Actor{}, verifreg0.State{}) inv.Register(builtin0.AccountActorCodeID, account0.Actor{}, account0.State{}) + inv.Register(builtin1.SystemActorCodeID, system1.Actor{}, abi.EmptyValue{}) + inv.Register(builtin1.InitActorCodeID, init1.Actor{}, init1.State{}) + inv.Register(builtin1.RewardActorCodeID, reward1.Actor{}, reward1.State{}) + inv.Register(builtin1.CronActorCodeID, cron1.Actor{}, cron1.State{}) + inv.Register(builtin1.StoragePowerActorCodeID, power1.Actor{}, power1.State{}) + inv.Register(builtin1.StorageMarketActorCodeID, market1.Actor{}, market1.State{}) + inv.Register(builtin1.StorageMinerActorCodeID, miner1.Actor{}, miner1.State{}) + inv.Register(builtin1.MultisigActorCodeID, msig1.Actor{}, msig1.State{}) + inv.Register(builtin1.PaymentChannelActorCodeID, paych1.Actor{}, paych1.State{}) + inv.Register(builtin1.VerifiedRegistryActorCodeID, verifreg1.Actor{}, verifreg1.State{}) + inv.Register(builtin1.AccountActorCodeID, account1.Actor{}, account1.State{}) + return inv } diff --git a/go.mod b/go.mod index cfdf4cb1d..b19f5c053 100644 --- a/go.mod +++ b/go.mod @@ -37,6 +37,7 @@ require ( github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b github.com/filecoin-project/specs-actors v0.9.10 + github.com/filecoin-project/specs-actors/v2 v2.0.0-20200918035954-4caac0a9b252 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 github.com/filecoin-project/test-vectors/schema v0.0.1 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 diff --git a/go.sum b/go.sum index 3b31e6c3b..6f5ab862d 100644 --- a/go.sum +++ b/go.sum @@ -232,6 +232,8 @@ github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3 github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24= github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24= +github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 h1:b3UDemBYN2HNfk3KOXNuxgTTxlWi3xVvbQP0IT38fvM= +github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0/go.mod h1:7aWZdaQ1b16BVoQUYR+eEvrDCGJoPLxFpDynFjYfBjI= github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200822201400-474f4fdccc52 h1:FXtCp0ybqdQL9knb3OGDpkNTaBbPxgkqPeWKotUwkH0= github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200822201400-474f4fdccc52/go.mod h1:XBBpuKIMaXIIzeqzO1iucq4GvbF8CxmXRFoezRh+Cx4= github.com/filecoin-project/go-multistore v0.0.3 h1:vaRBY4YiA2UZFPK57RNuewypB8u0DzzQwqsL0XarpnI= @@ -254,8 +256,11 @@ github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/go.mod h1:Q0GQOBtKf1oE10eSXSlhN45kDBdGvEcVOqMiffqX+N8= github.com/filecoin-project/specs-actors v0.9.4/go.mod h1:BStZQzx5x7TmCkLv0Bpa07U6cPKol6fd3w9KjMPZ6Z4= github.com/filecoin-project/specs-actors v0.9.7/go.mod h1:wM2z+kwqYgXn5Z7scV1YHLyd1Q1cy0R8HfTIWQ0BFGU= +github.com/filecoin-project/specs-actors v0.9.9/go.mod h1:czlvLQGEX0fjLLfdNHD7xLymy6L3n7aQzRWzsYGf+ys= github.com/filecoin-project/specs-actors v0.9.10 h1:gU0TrRhgkCsBEOP42sGDE7RQuR0Cov9hJhBqq+RJmjU= github.com/filecoin-project/specs-actors v0.9.10/go.mod h1:czlvLQGEX0fjLLfdNHD7xLymy6L3n7aQzRWzsYGf+ys= +github.com/filecoin-project/specs-actors/v2 v2.0.0-20200918035954-4caac0a9b252 h1:0vOZo6xlVDyPhuRS3ArrAy1fml7H2FEY65IFx6rwp3o= +github.com/filecoin-project/specs-actors/v2 v2.0.0-20200918035954-4caac0a9b252/go.mod h1:vV1UOOKlmdKg4dy3YKixtJtVf15WMuZsS6KgKRDxkWg= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= github.com/filecoin-project/test-vectors/schema v0.0.1 h1:5fNF76nl4qolEvcIsjc0kUADlTMVHO73tW4kXXPnsus= @@ -1367,6 +1372,7 @@ github.com/whyrusleeping/cbor-gen v0.0.0-20200414195334-429a0b5e922e/go.mod h1:X github.com/whyrusleeping/cbor-gen v0.0.0-20200504204219-64967432584d/go.mod h1:W5MvapuoHRP8rz4vxjwCK1pDqF1aQcWsV5PZ+AHbqdg= github.com/whyrusleeping/cbor-gen v0.0.0-20200710004633-5379fc63235d/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200715143311-227fab5a2377/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= +github.com/whyrusleeping/cbor-gen v0.0.0-20200806213330-63aa96ca5488/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200810223238-211df3b9e24c/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200812213548-958ddffe352c/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200814224545-656e08ce49ee h1:U7zWWvvAjT76EiuWPSOiZlQDnaQYPxPoxugTtTAcJK0= From 35562bd2f9b884b21ed64172b8ae85a05770a194 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 23 Sep 2020 15:36:56 -0700 Subject: [PATCH 339/885] fixup v1 actors for new methods Also, correctly handle multiple ADT versions. --- chain/actors/adt/adt.go | 10 +++ chain/actors/builtin/builtin.go | 1 + chain/actors/builtin/init/v0.go | 3 +- chain/actors/builtin/init/v1.go | 19 +++++ chain/actors/builtin/market/v1.go | 25 +++--- chain/actors/builtin/miner/v1.go | 109 ++++++++++++-------------- chain/actors/builtin/multisig/v1.go | 45 +++++++++-- chain/actors/builtin/paych/v1.go | 24 +++--- chain/actors/builtin/power/v0.go | 8 +- chain/actors/builtin/power/v1.go | 8 +- chain/actors/builtin/reward/v0.go | 13 +-- chain/actors/builtin/reward/v1.go | 13 +-- chain/actors/builtin/verifreg/util.go | 46 +++++++++++ chain/actors/builtin/verifreg/v0.go | 51 ++---------- chain/actors/builtin/verifreg/v1.go | 31 +++----- 15 files changed, 235 insertions(+), 171 deletions(-) create mode 100644 chain/actors/builtin/verifreg/util.go diff --git a/chain/actors/adt/adt.go b/chain/actors/adt/adt.go index 39dd5cebc..cf2ede0ae 100644 --- a/chain/actors/adt/adt.go +++ b/chain/actors/adt/adt.go @@ -8,7 +8,9 @@ import ( "github.com/filecoin-project/go-state-types/cbor" "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/chain/actors/builtin" + adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" + adt1 "github.com/filecoin-project/specs-actors/v2/actors/util/adt" ) type Map interface { @@ -25,6 +27,8 @@ func AsMap(store Store, root cid.Cid, version builtin.Version) (Map, error) { switch version { case builtin.Version0: return adt0.AsMap(store, root) + case builtin.Version1: + return adt1.AsMap(store, root) } return nil, xerrors.Errorf("unknown network version: %d", version) } @@ -33,6 +37,8 @@ func NewMap(store Store, version builtin.Version) (Map, error) { switch version { case builtin.Version0: return adt0.MakeEmptyMap(store), nil + case builtin.Version1: + return adt1.MakeEmptyMap(store), nil } return nil, xerrors.Errorf("unknown network version: %d", version) } @@ -52,6 +58,8 @@ func AsArray(store Store, root cid.Cid, version network.Version) (Array, error) switch builtin.VersionForNetwork(version) { case builtin.Version0: return adt0.AsArray(store, root) + case builtin.Version1: + return adt1.AsArray(store, root) } return nil, xerrors.Errorf("unknown network version: %d", version) } @@ -60,6 +68,8 @@ func NewArray(store Store, version builtin.Version) (Array, error) { switch version { case builtin.Version0: return adt0.MakeEmptyArray(store), nil + case builtin.Version1: + return adt1.MakeEmptyArray(store), nil } return nil, xerrors.Errorf("unknown network version: %d", version) } diff --git a/chain/actors/builtin/builtin.go b/chain/actors/builtin/builtin.go index e25cfcf36..ae007d731 100644 --- a/chain/actors/builtin/builtin.go +++ b/chain/actors/builtin/builtin.go @@ -17,6 +17,7 @@ type Version int const ( Version0 = iota + Version1 ) // Converts a network version into a specs-actors version. diff --git a/chain/actors/builtin/init/v0.go b/chain/actors/builtin/init/v0.go index 425ba654c..b82b85aee 100644 --- a/chain/actors/builtin/init/v0.go +++ b/chain/actors/builtin/init/v0.go @@ -6,11 +6,10 @@ import ( cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" - init_ "github.com/filecoin-project/specs-actors/actors/builtin/init" - "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/node/modules/dtypes" + init_ "github.com/filecoin-project/specs-actors/actors/builtin/init" adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" ) diff --git a/chain/actors/builtin/init/v1.go b/chain/actors/builtin/init/v1.go index d11071bdd..d65d5b89d 100644 --- a/chain/actors/builtin/init/v1.go +++ b/chain/actors/builtin/init/v1.go @@ -4,6 +4,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" cbg "github.com/whyrusleeping/cbor-gen" + "golang.org/x/xerrors" "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/node/modules/dtypes" @@ -45,3 +46,21 @@ func (s *state1) ForEachActor(cb func(id abi.ActorID, address address.Address) e func (s *state1) NetworkName() (dtypes.NetworkName, error) { return dtypes.NetworkName(s.State.NetworkName), nil } + +func (s *state1) Remove(addrs ...address.Address) (err error) { + m, err := adt1.AsMap(s.store, s.State.AddressMap) + if err != nil { + return err + } + for _, addr := range addrs { + if err = m.Delete(abi.AddrKey(addr)); err != nil { + return xerrors.Errorf("failed to delete entry for address: %s; err: %w", addr, err) + } + } + amr, err := m.Root() + if err != nil { + return xerrors.Errorf("failed to get address map root: %w", err) + } + s.State.AddressMap = amr + return nil +} diff --git a/chain/actors/builtin/market/v1.go b/chain/actors/builtin/market/v1.go index 8a634ddd2..dc84c2633 100644 --- a/chain/actors/builtin/market/v1.go +++ b/chain/actors/builtin/market/v1.go @@ -25,24 +25,24 @@ func (s *state1) TotalLocked() (abi.TokenAmount, error) { return fml, nil } -func (s *state1) BalancesChanged(otherState State) bool { +func (s *state1) BalancesChanged(otherState State) (bool, error) { otherState1, ok := otherState.(*state1) if !ok { // there's no way to compare different versions of the state, so let's // just say that means the state of balances has changed - return true + return true, nil } - return !s.State.EscrowTable.Equals(otherState1.State.EscrowTable) || !s.State.LockedTable.Equals(otherState1.State.LockedTable) + return !s.State.EscrowTable.Equals(otherState1.State.EscrowTable) || !s.State.LockedTable.Equals(otherState1.State.LockedTable), nil } -func (s *state1) StatesChanged(otherState State) bool { +func (s *state1) StatesChanged(otherState State) (bool, error) { otherState1, ok := otherState.(*state1) if !ok { // there's no way to compare different versions of the state, so let's // just say that means the state of balances has changed - return true + return true, nil } - return !s.State.States.Equals(otherState1.State.States) + return !s.State.States.Equals(otherState1.State.States), nil } func (s *state1) States() (DealStates, error) { @@ -53,14 +53,14 @@ func (s *state1) States() (DealStates, error) { return &dealStates1{stateArray}, nil } -func (s *state1) ProposalsChanged(otherState State) bool { +func (s *state1) ProposalsChanged(otherState State) (bool, error) { otherState1, ok := otherState.(*state1) if !ok { // there's no way to compare different versions of the state, so let's // just say that means the state of balances has changed - return true + return true, nil } - return !s.State.Proposals.Equals(otherState1.State.Proposals) + return !s.State.Proposals.Equals(otherState1.State.Proposals), nil } func (s *state1) Proposals() (DealProposals, error) { @@ -127,6 +127,13 @@ func (s *dealStates1) Get(dealID abi.DealID) (*DealState, bool, error) { return &deal, true, nil } +func (s *dealStates1) ForEach(cb func(dealID abi.DealID, ds DealState) error) error { + var ds1 market.DealState + return s.Array.ForEach(&ds1, func(idx int64) error { + return cb(abi.DealID(idx), fromV1DealState(ds1)) + }) +} + func (s *dealStates1) decode(val *cbg.Deferred) (*DealState, error) { var ds1 market.DealState if err := ds1.UnmarshalCBOR(bytes.NewReader(val.Raw)); err != nil { diff --git a/chain/actors/builtin/miner/v1.go b/chain/actors/builtin/miner/v1.go index 9c0486eb0..4c991b4f3 100644 --- a/chain/actors/builtin/miner/v1.go +++ b/chain/actors/builtin/miner/v1.go @@ -12,7 +12,6 @@ import ( adt1 "github.com/filecoin-project/specs-actors/v2/actors/util/adt" "github.com/libp2p/go-libp2p-core/peer" cbg "github.com/whyrusleeping/cbor-gen" - "golang.org/x/xerrors" miner1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" ) @@ -79,6 +78,21 @@ func (s *state1) FindSector(num abi.SectorNumber) (*SectorLocation, error) { }, nil } +func (s *state1) NumLiveSectors() (uint64, error) { + dls, err := s.State.LoadDeadlines(s.store) + if err != nil { + return 0, err + } + var total uint64 + if err := dls.ForEach(s.store, func(dlIdx uint64, dl *miner1.Deadline) error { + total += dl.LiveSectors + return nil + }); err != nil { + return 0, err + } + return total, nil +} + // GetSectorExpiration returns the effective expiration of the given sector. // // If the sector isn't found or has already been terminated, this method returns @@ -162,60 +176,37 @@ func (s *state1) GetPrecommittedSector(num abi.SectorNumber) (*SectorPreCommitOn return &ret, nil } -func (s *state1) LoadSectorsFromSet(filter *bitfield.BitField, filterOut bool) (adt.Array, error) { - a, err := adt1.AsArray(s.store, s.State.Sectors) +func (s *state1) LoadSectors(snos *bitfield.BitField) ([]*SectorOnChainInfo, error) { + sectors, err := miner1.LoadSectors(s.store, s.State.Sectors) if err != nil { return nil, err } - ret := adt1.MakeEmptyArray(s.store) - var v cbg.Deferred - if err := a.ForEach(&v, func(i int64) error { - include := true - if filter != nil { - set, err := filter.IsSet(uint64(i)) - if err != nil { - return xerrors.Errorf("filter check error: %w", err) - } - if set == filterOut { - include = false - } + // If no sector numbers are specified, load all. + if snos == nil { + infos := make([]*SectorOnChainInfo, 0, sectors.Length()) + var info1 miner1.SectorOnChainInfo + if err := sectors.ForEach(&info1, func(_ int64) error { + info := fromV1SectorOnChainInfo(info1) + infos = append(infos, &info) + return nil + }); err != nil { + return nil, err } - - if include { - var oci miner1.SectorOnChainInfo - if err := oci.UnmarshalCBOR(bytes.NewReader(v.Raw)); err != nil { - return err - } - - noci := SectorOnChainInfo{ - SectorNumber: oci.SectorNumber, - SealProof: oci.SealProof, - SealedCID: oci.SealedCID, - DealIDs: oci.DealIDs, - Activation: oci.Activation, - Expiration: oci.Expiration, - DealWeight: oci.DealWeight, - VerifiedDealWeight: oci.VerifiedDealWeight, - InitialPledge: oci.InitialPledge, - ExpectedDayReward: oci.ExpectedDayReward, - ExpectedStoragePledge: oci.ExpectedStoragePledge, - } - - if err := ret.Set(uint64(i), &noci); err != nil { - return err - } - } - return nil - }); err != nil { - return nil, err + return infos, nil } - return ret, nil -} - -func (s *state1) LoadPreCommittedSectors() (adt.Map, error) { - return adt1.AsMap(s.store, s.State.PreCommittedSectors) + // Otherwise, load selected. + infos1, err := sectors.Load(*snos) + if err != nil { + return nil, err + } + infos := make([]*SectorOnChainInfo, len(infos1)) + for i, info1 := range infos1 { + info := fromV1SectorOnChainInfo(*info1) + infos[i] = &info + } + return infos, nil } func (s *state1) IsAllocated(num abi.SectorNumber) (bool, error) { @@ -253,14 +244,14 @@ func (s *state1) NumDeadlines() (uint64, error) { return miner1.WPoStPeriodDeadlines, nil } -func (s *state1) DeadlinesChanged(other State) bool { +func (s *state1) DeadlinesChanged(other State) (bool, error) { other1, ok := other.(*state1) if !ok { // treat an upgrade as a change, always - return true + return true, nil } - return s.State.Deadlines.Equals(other1.Deadlines) + return s.State.Deadlines.Equals(other1.Deadlines), nil } func (s *state1) Info() (MinerInfo, error) { @@ -297,8 +288,8 @@ func (s *state1) Info() (MinerInfo, error) { return mi, nil } -func (s *state1) DeadlineInfo(epoch abi.ChainEpoch) *dline.Info { - return s.State.DeadlineInfo(epoch) +func (s *state1) DeadlineInfo(epoch abi.ChainEpoch) (*dline.Info, error) { + return s.State.DeadlineInfo(epoch), nil } func (s *state1) sectors() (adt.Array, error) { @@ -312,8 +303,7 @@ func (s *state1) decodeSectorOnChainInfo(val *cbg.Deferred) (SectorOnChainInfo, return SectorOnChainInfo{}, err } - ret := fromV1SectorOnChainInfo(si) - return ret, nil + return fromV1SectorOnChainInfo(si), nil } func (s *state1) precommits() (adt.Map, error) { @@ -327,8 +317,7 @@ func (s *state1) decodeSectorPreCommitOnChainInfo(val *cbg.Deferred) (SectorPreC return SectorPreCommitOnChainInfo{}, err } - ret := fromV1SectorPreCommitOnChainInfo(sp) - return ret, nil + return fromV1SectorPreCommitOnChainInfo(sp), nil } func (d *deadline1) LoadPartition(idx uint64) (Partition, error) { @@ -350,14 +339,14 @@ func (d *deadline1) ForEachPartition(cb func(uint64, Partition) error) error { }) } -func (d *deadline1) PartitionsChanged(other Deadline) bool { +func (d *deadline1) PartitionsChanged(other Deadline) (bool, error) { other1, ok := other.(*deadline1) if !ok { // treat an upgrade as a change, always - return true + return true, nil } - return d.Deadline.Partitions.Equals(other1.Deadline.Partitions) + return d.Deadline.Partitions.Equals(other1.Deadline.Partitions), nil } func (d *deadline1) PostSubmissions() (bitfield.BitField, error) { diff --git a/chain/actors/builtin/multisig/v1.go b/chain/actors/builtin/multisig/v1.go index cf2d67453..97b9ae4a3 100644 --- a/chain/actors/builtin/multisig/v1.go +++ b/chain/actors/builtin/multisig/v1.go @@ -1,15 +1,21 @@ package multisig import ( + "encoding/binary" + + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/actors/adt" - "github.com/filecoin-project/specs-actors/v2/actors/builtin/multisig" + "golang.org/x/xerrors" + + msig1 "github.com/filecoin-project/specs-actors/actors/builtin/multisig" + adt1 "github.com/filecoin-project/specs-actors/actors/util/adt" ) var _ State = (*state1)(nil) type state1 struct { - multisig.State + msig1.State store adt.Store } @@ -17,14 +23,37 @@ func (s *state1) LockedBalance(currEpoch abi.ChainEpoch) (abi.TokenAmount, error return s.State.AmountLocked(currEpoch - s.State.StartEpoch), nil } -func (s *state1) StartEpoch() abi.ChainEpoch { - return s.State.StartEpoch +func (s *state1) StartEpoch() (abi.ChainEpoch, error) { + return s.State.StartEpoch, nil } -func (s *state1) UnlockDuration() abi.ChainEpoch { - return s.State.UnlockDuration +func (s *state1) UnlockDuration() (abi.ChainEpoch, error) { + return s.State.UnlockDuration, nil } -func (s *state1) InitialBalance() abi.TokenAmount { - return s.State.InitialBalance +func (s *state1) InitialBalance() (abi.TokenAmount, error) { + return s.State.InitialBalance, nil +} + +func (s *state1) Threshold() (uint64, error) { + return s.State.NumApprovalsThreshold, nil +} + +func (s *state1) Signers() ([]address.Address, error) { + return s.State.Signers, nil +} + +func (s *state1) ForEachPendingTxn(cb func(id int64, txn Transaction) error) error { + arr, err := adt1.AsMap(s.store, s.State.PendingTxns) + if err != nil { + return err + } + var out msig1.Transaction + return arr.ForEach(&out, func(key string) error { + txid, n := binary.Varint([]byte(key)) + if n <= 0 { + return xerrors.Errorf("invalid pending transaction key: %v", key) + } + return cb(txid, (Transaction)(out)) + }) } diff --git a/chain/actors/builtin/paych/v1.go b/chain/actors/builtin/paych/v1.go index bbe3bbebf..7a960067c 100644 --- a/chain/actors/builtin/paych/v1.go +++ b/chain/actors/builtin/paych/v1.go @@ -18,23 +18,23 @@ type state1 struct { } // Channel owner, who has funded the actor -func (s *state1) From() address.Address { - return s.State.From +func (s *state1) From() (address.Address, error) { + return s.State.From, nil } // Recipient of payouts from channel -func (s *state1) To() address.Address { - return s.State.To +func (s *state1) To() (address.Address, error) { + return s.State.To, nil } // Height at which the channel can be `Collected` -func (s *state1) SettlingAt() abi.ChainEpoch { - return s.State.SettlingAt +func (s *state1) SettlingAt() (abi.ChainEpoch, error) { + return s.State.SettlingAt, nil } // Amount successfully redeemed through the payment channel, paid out on `Collect()` -func (s *state1) ToSend() abi.TokenAmount { - return s.State.ToSend +func (s *state1) ToSend() (abi.TokenAmount, error) { + return s.State.ToSend, nil } func (s *state1) getOrLoadLsAmt() (*adt1.Array, error) { @@ -82,10 +82,10 @@ type laneState1 struct { paych.LaneState } -func (ls *laneState1) Redeemed() big.Int { - return ls.LaneState.Redeemed +func (ls *laneState1) Redeemed() (big.Int, error) { + return ls.LaneState.Redeemed, nil } -func (ls *laneState1) Nonce() uint64 { - return ls.LaneState.Nonce +func (ls *laneState1) Nonce() (uint64, error) { + return ls.LaneState.Nonce, nil } diff --git a/chain/actors/builtin/power/v0.go b/chain/actors/builtin/power/v0.go index f2fe96dad..657de7ed0 100644 --- a/chain/actors/builtin/power/v0.go +++ b/chain/actors/builtin/power/v0.go @@ -3,9 +3,11 @@ package power import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/actors/builtin" + power0 "github.com/filecoin-project/specs-actors/actors/builtin/power" - "github.com/filecoin-project/specs-actors/actors/util/adt" + adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" ) var _ State = (*state0)(nil) @@ -35,7 +37,7 @@ func (s *state0) TotalCommitted() (Claim, error) { } func (s *state0) MinerPower(addr address.Address) (Claim, bool, error) { - claims, err := adt.AsMap(s.store, s.Claims) + claims, err := adt0.AsMap(s.store, s.Claims) if err != nil { return Claim{}, false, err } @@ -63,7 +65,7 @@ func (s *state0) MinerCounts() (uint64, uint64, error) { } func (s *state0) ListAllMiners() ([]address.Address, error) { - claims, err := adt.AsMap(s.store, s.Claims) + claims, err := adt0.AsMap(s.store, s.Claims) if err != nil { return nil, err } diff --git a/chain/actors/builtin/power/v1.go b/chain/actors/builtin/power/v1.go index 85defd053..8c1a052b3 100644 --- a/chain/actors/builtin/power/v1.go +++ b/chain/actors/builtin/power/v1.go @@ -3,8 +3,10 @@ package power import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/util/adt" + + adt1 "github.com/filecoin-project/specs-actors/actors/util/adt" power1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/power" ) @@ -35,7 +37,7 @@ func (s *state1) TotalCommitted() (Claim, error) { } func (s *state1) MinerPower(addr address.Address) (Claim, bool, error) { - claims, err := adt.AsMap(s.store, s.Claims) + claims, err := adt1.AsMap(s.store, s.Claims) if err != nil { return Claim{}, false, err } @@ -63,7 +65,7 @@ func (s *state1) MinerCounts() (uint64, uint64, error) { } func (s *state1) ListAllMiners() ([]address.Address, error) { - claims, err := adt.AsMap(s.store, s.Claims) + claims, err := adt1.AsMap(s.store, s.Claims) if err != nil { return nil, err } diff --git a/chain/actors/builtin/reward/v0.go b/chain/actors/builtin/reward/v0.go index df7117b67..57084899d 100644 --- a/chain/actors/builtin/reward/v0.go +++ b/chain/actors/builtin/reward/v0.go @@ -2,17 +2,18 @@ package reward import ( "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/actors/builtin" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" - "github.com/filecoin-project/specs-actors/actors/builtin/reward" - "github.com/filecoin-project/specs-actors/actors/util/adt" - "github.com/filecoin-project/specs-actors/actors/util/smoothing" + reward0 "github.com/filecoin-project/specs-actors/actors/builtin/reward" + smoothing0 "github.com/filecoin-project/specs-actors/actors/util/smoothing" ) var _ State = (*state0)(nil) type state0 struct { - reward.State + reward0.State store adt.Store } @@ -54,7 +55,7 @@ func (s *state0) InitialPledgeForPower(sectorWeight abi.StoragePower, networkTot s.State.ThisEpochBaselinePower, networkTotalPledge, s.State.ThisEpochRewardSmoothed, - &smoothing.FilterEstimate{ + &smoothing0.FilterEstimate{ PositionEstimate: networkQAPower.PositionEstimate, VelocityEstimate: networkQAPower.VelocityEstimate, }, @@ -63,7 +64,7 @@ func (s *state0) InitialPledgeForPower(sectorWeight abi.StoragePower, networkTot func (s *state0) PreCommitDepositForPower(networkQAPower builtin.FilterEstimate, sectorWeight abi.StoragePower) (abi.TokenAmount, error) { return miner0.PreCommitDepositForPower(s.State.ThisEpochRewardSmoothed, - &smoothing.FilterEstimate{ + &smoothing0.FilterEstimate{ PositionEstimate: networkQAPower.PositionEstimate, VelocityEstimate: networkQAPower.VelocityEstimate, }, diff --git a/chain/actors/builtin/reward/v1.go b/chain/actors/builtin/reward/v1.go index 162dc13a1..f16256a00 100644 --- a/chain/actors/builtin/reward/v1.go +++ b/chain/actors/builtin/reward/v1.go @@ -2,17 +2,18 @@ package reward import ( "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/util/adt" + miner1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" - "github.com/filecoin-project/specs-actors/v2/actors/builtin/reward" - "github.com/filecoin-project/specs-actors/v2/actors/util/smoothing" + reward1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/reward" + smoothing1 "github.com/filecoin-project/specs-actors/v2/actors/util/smoothing" ) var _ State = (*state1)(nil) type state1 struct { - reward.State + reward1.State store adt.Store } @@ -56,7 +57,7 @@ func (s *state1) InitialPledgeForPower(qaPower abi.StoragePower, networkTotalPle qaPower, s.State.ThisEpochBaselinePower, s.State.ThisEpochRewardSmoothed, - smoothing.FilterEstimate{ + smoothing1.FilterEstimate{ PositionEstimate: networkQAPower.PositionEstimate, VelocityEstimate: networkQAPower.VelocityEstimate, }, @@ -66,7 +67,7 @@ func (s *state1) InitialPledgeForPower(qaPower abi.StoragePower, networkTotalPle func (s *state1) PreCommitDepositForPower(networkQAPower builtin.FilterEstimate, sectorWeight abi.StoragePower) (abi.TokenAmount, error) { return miner1.PreCommitDepositForPower(s.State.ThisEpochRewardSmoothed, - smoothing.FilterEstimate{ + smoothing1.FilterEstimate{ PositionEstimate: networkQAPower.PositionEstimate, VelocityEstimate: networkQAPower.VelocityEstimate, }, diff --git a/chain/actors/builtin/verifreg/util.go b/chain/actors/builtin/verifreg/util.go new file mode 100644 index 000000000..413a68a15 --- /dev/null +++ b/chain/actors/builtin/verifreg/util.go @@ -0,0 +1,46 @@ +package verifreg + +import ( + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/chain/actors/builtin" + "github.com/ipfs/go-cid" + "golang.org/x/xerrors" +) + +func getDataCap(store adt.Store, ver builtin.Version, root cid.Cid, addr address.Address) (bool, abi.StoragePower, error) { + if addr.Protocol() != address.ID { + return false, big.Zero(), xerrors.Errorf("can only look up ID addresses") + } + + vh, err := adt.AsMap(store, root, ver) + if err != nil { + return false, big.Zero(), xerrors.Errorf("loading verifreg: %w", err) + } + + var dcap abi.StoragePower + if found, err := vh.Get(abi.AddrKey(addr), &dcap); err != nil { + return false, big.Zero(), xerrors.Errorf("looking up addr: %w", err) + } else if !found { + return false, big.Zero(), nil + } + + return true, dcap, nil +} + +func forEachCap(store adt.Store, ver builtin.Version, root cid.Cid, cb func(addr address.Address, dcap abi.StoragePower) error) error { + vh, err := adt.AsMap(store, root, ver) + if err != nil { + return xerrors.Errorf("loading verified clients: %w", err) + } + var dcap abi.StoragePower + return vh.ForEach(&dcap, func(key string) error { + a, err := address.NewFromBytes([]byte(key)) + if err != nil { + return err + } + return cb(a, dcap) + }) +} diff --git a/chain/actors/builtin/verifreg/v0.go b/chain/actors/builtin/verifreg/v0.go index c59a58811..33e6757e7 100644 --- a/chain/actors/builtin/verifreg/v0.go +++ b/chain/actors/builtin/verifreg/v0.go @@ -3,13 +3,11 @@ package verifreg import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/big" - verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" - adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" - "github.com/ipfs/go-cid" - "golang.org/x/xerrors" "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/chain/actors/builtin" + + verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" ) var _ State = (*state0)(nil) @@ -19,53 +17,18 @@ type state0 struct { store adt.Store } -func getDataCap(store adt.Store, root cid.Cid, addr address.Address) (bool, abi.StoragePower, error) { - if addr.Protocol() != address.ID { - return false, big.Zero(), xerrors.Errorf("can only look up ID addresses") - } - - vh, err := adt0.AsMap(store, root) - if err != nil { - return false, big.Zero(), xerrors.Errorf("loading verifreg: %w", err) - } - - var dcap abi.StoragePower - if found, err := vh.Get(abi.AddrKey(addr), &dcap); err != nil { - return false, big.Zero(), xerrors.Errorf("looking up addr: %w", err) - } else if !found { - return false, big.Zero(), nil - } - - return true, dcap, nil -} - func (s *state0) VerifiedClientDataCap(addr address.Address) (bool, abi.StoragePower, error) { - return getDataCap(s.store, s.State.VerifiedClients, addr) + return getDataCap(s.store, builtin.Version0, s.State.VerifiedClients, addr) } func (s *state0) VerifierDataCap(addr address.Address) (bool, abi.StoragePower, error) { - return getDataCap(s.store, s.State.Verifiers, addr) -} - -func forEachCap(store adt.Store, root cid.Cid, cb func(addr address.Address, dcap abi.StoragePower) error) error { - vh, err := adt0.AsMap(store, root) - if err != nil { - return xerrors.Errorf("loading verified clients: %w", err) - } - var dcap abi.StoragePower - return vh.ForEach(&dcap, func(key string) error { - a, err := address.NewFromBytes([]byte(key)) - if err != nil { - return err - } - return cb(a, dcap) - }) + return getDataCap(s.store, builtin.Version0, s.State.Verifiers, addr) } func (s *state0) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error { - return forEachCap(s.store, s.State.Verifiers, cb) + return forEachCap(s.store, builtin.Version0, s.State.Verifiers, cb) } func (s *state0) ForEachClient(cb func(addr address.Address, dcap abi.StoragePower) error) error { - return forEachCap(s.store, s.State.VerifiedClients, cb) + return forEachCap(s.store, builtin.Version0, s.State.VerifiedClients, cb) } diff --git a/chain/actors/builtin/verifreg/v1.go b/chain/actors/builtin/verifreg/v1.go index 4fb013e1b..ae78e260b 100644 --- a/chain/actors/builtin/verifreg/v1.go +++ b/chain/actors/builtin/verifreg/v1.go @@ -3,12 +3,11 @@ package verifreg import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/big" - adt1 "github.com/filecoin-project/specs-actors/actors/util/adt" - verifreg1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/verifreg" - "golang.org/x/xerrors" "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/chain/actors/builtin" + + verifreg1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/verifreg" ) var _ State = (*state1)(nil) @@ -19,21 +18,17 @@ type state1 struct { } func (s *state1) VerifiedClientDataCap(addr address.Address) (bool, abi.StoragePower, error) { - if addr.Protocol() != address.ID { - return false, big.Zero(), xerrors.Errorf("can only look up ID addresses") - } + return getDataCap(s.store, builtin.Version1, s.State.VerifiedClients, addr) +} - vh, err := adt1.AsMap(s.store, s.VerifiedClients) - if err != nil { - return false, big.Zero(), xerrors.Errorf("loading verified clients: %w", err) - } +func (s *state1) VerifierDataCap(addr address.Address) (bool, abi.StoragePower, error) { + return getDataCap(s.store, builtin.Version1, s.State.Verifiers, addr) +} - var dcap abi.StoragePower - if found, err := vh.Get(abi.AddrKey(addr), &dcap); err != nil { - return false, big.Zero(), xerrors.Errorf("looking up verified clients: %w", err) - } else if !found { - return false, big.Zero(), nil - } +func (s *state1) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error { + return forEachCap(s.store, builtin.Version1, s.State.Verifiers, cb) +} - return true, dcap, nil +func (s *state1) ForEachClient(cb func(addr address.Address, dcap abi.StoragePower) error) error { + return forEachCap(s.store, builtin.Version1, s.State.VerifiedClients, cb) } From 858f11992f93d037c53d7d895a8df9e47a5c830e Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 23 Sep 2020 16:02:17 -0700 Subject: [PATCH 340/885] compile fix --- chain/stmgr/utils.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index 3c0a895d3..0d7be7adf 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -33,6 +33,7 @@ import ( builtin1 "github.com/filecoin-project/specs-actors/v2/actors/builtin" account1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/account" + cron1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/cron" init1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/init" market1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/market" miner1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" @@ -559,7 +560,7 @@ func init() { // builtin1.SystemActorCodeID: {builtin1.MethodsSystem, system.Actor{} }- apparently it doesn't have methods builtin1.InitActorCodeID: {builtin1.MethodsInit, init1.Actor{}}, - builtin1.CronActorCodeID: {builtin1.MethodsCron, cron.Actor{}}, + builtin1.CronActorCodeID: {builtin1.MethodsCron, cron1.Actor{}}, builtin1.AccountActorCodeID: {builtin1.MethodsAccount, account1.Actor{}}, builtin1.StoragePowerActorCodeID: {builtin1.MethodsPower, power1.Actor{}}, builtin1.StorageMinerActorCodeID: {builtin1.MethodsMiner, miner1.Actor{}}, From 8806f2722045ff240a52a6301ec11ef900f73755 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 23 Sep 2020 16:15:37 -0700 Subject: [PATCH 341/885] fix runtime reflection for upgrade --- chain/vm/invoker.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/chain/vm/invoker.go b/chain/vm/invoker.go index bf71a4f74..7465074b4 100644 --- a/chain/vm/invoker.go +++ b/chain/vm/invoker.go @@ -23,7 +23,6 @@ import ( system0 "github.com/filecoin-project/specs-actors/actors/builtin/system" verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" - vmr "github.com/filecoin-project/specs-actors/actors/runtime" builtin1 "github.com/filecoin-project/specs-actors/v2/actors/builtin" account1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/account" cron1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/cron" @@ -36,6 +35,7 @@ import ( reward1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/reward" system1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/system" verifreg1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/verifreg" + vmr "github.com/filecoin-project/specs-actors/v2/actors/runtime" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/exitcode" @@ -115,6 +115,7 @@ type Invokee interface { func (*Invoker) transform(instance Invokee) (nativeCode, error) { itype := reflect.TypeOf(instance) exports := instance.Exports() + runtimeType := reflect.TypeOf((*vmr.Runtime)(nil)).Elem() for i, m := range exports { i := i newErr := func(format string, args ...interface{}) error { @@ -135,7 +136,7 @@ func (*Invoker) transform(instance Invokee) (nativeCode, error) { return nil, newErr("wrong number of inputs should be: " + "vmr.Runtime, ") } - if t.In(0) != reflect.TypeOf((*vmr.Runtime)(nil)).Elem() { + if !runtimeType.Implements(t.In(0)) { return nil, newErr("first arguemnt should be vmr.Runtime") } if t.In(1).Kind() != reflect.Ptr { From edb31e606a05a7624c8bd3cde922b883cbbb93dc Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 23 Sep 2020 16:50:34 -0700 Subject: [PATCH 342/885] correctly load new actors --- chain/actors/builtin/miner/miner.go | 13 +++++++++++-- chain/actors/builtin/power/power.go | 11 ++++++++++- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/chain/actors/builtin/miner/miner.go b/chain/actors/builtin/miner/miner.go index 1a4c466b9..11f65fd49 100644 --- a/chain/actors/builtin/miner/miner.go +++ b/chain/actors/builtin/miner/miner.go @@ -11,11 +11,13 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/cbor" "github.com/filecoin-project/go-state-types/dline" - builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" - miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/types" + + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" + builtin1 "github.com/filecoin-project/specs-actors/v2/actors/builtin" ) // Unchanged between v0 and v1 actors @@ -32,6 +34,13 @@ func Load(store adt.Store, act *types.Actor) (st State, err error) { return nil, err } return &out, nil + case builtin1.StorageMinerActorCodeID: + out := state1{store: store} + err := store.Get(store.Context(), act.Head, &out) + if err != nil { + return nil, err + } + return &out, nil } return nil, xerrors.Errorf("unknown actor code %s", act.Code) } diff --git a/chain/actors/builtin/power/power.go b/chain/actors/builtin/power/power.go index e4bb52d44..7aa86fe00 100644 --- a/chain/actors/builtin/power/power.go +++ b/chain/actors/builtin/power/power.go @@ -6,11 +6,13 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/cbor" - builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/types" + + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" + builtin1 "github.com/filecoin-project/specs-actors/v2/actors/builtin" ) var Address = builtin0.StoragePowerActorAddr @@ -24,6 +26,13 @@ func Load(store adt.Store, act *types.Actor) (st State, err error) { return nil, err } return &out, nil + case builtin1.StoragePowerActorCodeID: + out := state1{store: store} + err := store.Get(store.Context(), act.Head, &out) + if err != nil { + return nil, err + } + return &out, nil } return nil, xerrors.Errorf("unknown actor code %s", act.Code) } From 87351fa35cbd1be5796803d23ea1a1f9704d1bc4 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 23 Sep 2020 17:01:40 -0700 Subject: [PATCH 343/885] move version to actors, from builtin Otherwise, we're going to end up with an import cycle between the adt and this version. --- chain/actors/adt/adt.go | 27 ++++++++++++++------------- chain/actors/builtin/builtin.go | 21 --------------------- chain/actors/builtin/verifreg/util.go | 6 +++--- chain/actors/builtin/verifreg/v0.go | 10 +++++----- chain/actors/builtin/verifreg/v1.go | 10 +++++----- chain/actors/version.go | 24 ++++++++++++++++++++++++ chain/gen/genesis/genesis.go | 4 ++-- chain/state/statetree.go | 18 +++++++++--------- chain/state/statetree_test.go | 14 +++++++------- chain/stmgr/stmgr.go | 3 +-- 10 files changed, 70 insertions(+), 67 deletions(-) create mode 100644 chain/actors/version.go diff --git a/chain/actors/adt/adt.go b/chain/actors/adt/adt.go index cf2ede0ae..778ba9fcf 100644 --- a/chain/actors/adt/adt.go +++ b/chain/actors/adt/adt.go @@ -7,7 +7,8 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/cbor" "github.com/filecoin-project/go-state-types/network" - "github.com/filecoin-project/lotus/chain/actors/builtin" + + "github.com/filecoin-project/lotus/chain/actors" adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" adt1 "github.com/filecoin-project/specs-actors/v2/actors/util/adt" @@ -23,21 +24,21 @@ type Map interface { ForEach(v cbor.Unmarshaler, fn func(key string) error) error } -func AsMap(store Store, root cid.Cid, version builtin.Version) (Map, error) { +func AsMap(store Store, root cid.Cid, version actors.Version) (Map, error) { switch version { - case builtin.Version0: + case actors.Version0: return adt0.AsMap(store, root) - case builtin.Version1: + case actors.Version1: return adt1.AsMap(store, root) } return nil, xerrors.Errorf("unknown network version: %d", version) } -func NewMap(store Store, version builtin.Version) (Map, error) { +func NewMap(store Store, version actors.Version) (Map, error) { switch version { - case builtin.Version0: + case actors.Version0: return adt0.MakeEmptyMap(store), nil - case builtin.Version1: + case actors.Version1: return adt1.MakeEmptyMap(store), nil } return nil, xerrors.Errorf("unknown network version: %d", version) @@ -55,20 +56,20 @@ type Array interface { } func AsArray(store Store, root cid.Cid, version network.Version) (Array, error) { - switch builtin.VersionForNetwork(version) { - case builtin.Version0: + switch actors.VersionForNetwork(version) { + case actors.Version0: return adt0.AsArray(store, root) - case builtin.Version1: + case actors.Version1: return adt1.AsArray(store, root) } return nil, xerrors.Errorf("unknown network version: %d", version) } -func NewArray(store Store, version builtin.Version) (Array, error) { +func NewArray(store Store, version actors.Version) (Array, error) { switch version { - case builtin.Version0: + case actors.Version0: return adt0.MakeEmptyArray(store), nil - case builtin.Version1: + case actors.Version1: return adt1.MakeEmptyArray(store), nil } return nil, xerrors.Errorf("unknown network version: %d", version) diff --git a/chain/actors/builtin/builtin.go b/chain/actors/builtin/builtin.go index ae007d731..53dfa8669 100644 --- a/chain/actors/builtin/builtin.go +++ b/chain/actors/builtin/builtin.go @@ -1,35 +1,14 @@ package builtin import ( - "fmt" - "github.com/filecoin-project/go-state-types/abi" miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" proof0 "github.com/filecoin-project/specs-actors/actors/runtime/proof" smoothing0 "github.com/filecoin-project/specs-actors/actors/util/smoothing" smoothing1 "github.com/filecoin-project/specs-actors/v2/actors/util/smoothing" - - "github.com/filecoin-project/go-state-types/network" ) -type Version int - -const ( - Version0 = iota - Version1 -) - -// Converts a network version into a specs-actors version. -func VersionForNetwork(version network.Version) Version { - switch version { - case network.Version0, network.Version1, network.Version2: - return Version0 - default: - panic(fmt.Sprintf("unsupported network version %d", version)) - } -} - // TODO: Why does actors have 2 different versions of this? type SectorInfo = proof0.SectorInfo type PoStProof = proof0.PoStProof diff --git a/chain/actors/builtin/verifreg/util.go b/chain/actors/builtin/verifreg/util.go index 413a68a15..4136c0c30 100644 --- a/chain/actors/builtin/verifreg/util.go +++ b/chain/actors/builtin/verifreg/util.go @@ -4,13 +4,13 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/adt" - "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/ipfs/go-cid" "golang.org/x/xerrors" ) -func getDataCap(store adt.Store, ver builtin.Version, root cid.Cid, addr address.Address) (bool, abi.StoragePower, error) { +func getDataCap(store adt.Store, ver actors.Version, root cid.Cid, addr address.Address) (bool, abi.StoragePower, error) { if addr.Protocol() != address.ID { return false, big.Zero(), xerrors.Errorf("can only look up ID addresses") } @@ -30,7 +30,7 @@ func getDataCap(store adt.Store, ver builtin.Version, root cid.Cid, addr address return true, dcap, nil } -func forEachCap(store adt.Store, ver builtin.Version, root cid.Cid, cb func(addr address.Address, dcap abi.StoragePower) error) error { +func forEachCap(store adt.Store, ver actors.Version, root cid.Cid, cb func(addr address.Address, dcap abi.StoragePower) error) error { vh, err := adt.AsMap(store, root, ver) if err != nil { return xerrors.Errorf("loading verified clients: %w", err) diff --git a/chain/actors/builtin/verifreg/v0.go b/chain/actors/builtin/verifreg/v0.go index 33e6757e7..5cb053c79 100644 --- a/chain/actors/builtin/verifreg/v0.go +++ b/chain/actors/builtin/verifreg/v0.go @@ -4,8 +4,8 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/adt" - "github.com/filecoin-project/lotus/chain/actors/builtin" verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" ) @@ -18,17 +18,17 @@ type state0 struct { } func (s *state0) VerifiedClientDataCap(addr address.Address) (bool, abi.StoragePower, error) { - return getDataCap(s.store, builtin.Version0, s.State.VerifiedClients, addr) + return getDataCap(s.store, actors.Version0, s.State.VerifiedClients, addr) } func (s *state0) VerifierDataCap(addr address.Address) (bool, abi.StoragePower, error) { - return getDataCap(s.store, builtin.Version0, s.State.Verifiers, addr) + return getDataCap(s.store, actors.Version0, s.State.Verifiers, addr) } func (s *state0) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error { - return forEachCap(s.store, builtin.Version0, s.State.Verifiers, cb) + return forEachCap(s.store, actors.Version0, s.State.Verifiers, cb) } func (s *state0) ForEachClient(cb func(addr address.Address, dcap abi.StoragePower) error) error { - return forEachCap(s.store, builtin.Version0, s.State.VerifiedClients, cb) + return forEachCap(s.store, actors.Version0, s.State.VerifiedClients, cb) } diff --git a/chain/actors/builtin/verifreg/v1.go b/chain/actors/builtin/verifreg/v1.go index ae78e260b..ec998f9e5 100644 --- a/chain/actors/builtin/verifreg/v1.go +++ b/chain/actors/builtin/verifreg/v1.go @@ -4,8 +4,8 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/adt" - "github.com/filecoin-project/lotus/chain/actors/builtin" verifreg1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/verifreg" ) @@ -18,17 +18,17 @@ type state1 struct { } func (s *state1) VerifiedClientDataCap(addr address.Address) (bool, abi.StoragePower, error) { - return getDataCap(s.store, builtin.Version1, s.State.VerifiedClients, addr) + return getDataCap(s.store, actors.Version1, s.State.VerifiedClients, addr) } func (s *state1) VerifierDataCap(addr address.Address) (bool, abi.StoragePower, error) { - return getDataCap(s.store, builtin.Version1, s.State.Verifiers, addr) + return getDataCap(s.store, actors.Version1, s.State.Verifiers, addr) } func (s *state1) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error { - return forEachCap(s.store, builtin.Version1, s.State.Verifiers, cb) + return forEachCap(s.store, actors.Version1, s.State.Verifiers, cb) } func (s *state1) ForEachClient(cb func(addr address.Address, dcap abi.StoragePower) error) error { - return forEachCap(s.store, builtin.Version1, s.State.VerifiedClients, cb) + return forEachCap(s.store, actors.Version1, s.State.VerifiedClients, cb) } diff --git a/chain/actors/version.go b/chain/actors/version.go new file mode 100644 index 000000000..8d256cf7d --- /dev/null +++ b/chain/actors/version.go @@ -0,0 +1,24 @@ +package actors + +import ( + "fmt" + + "github.com/filecoin-project/go-state-types/network" +) + +type Version int + +const ( + Version0 = iota + Version1 +) + +// Converts a network version into an actors adt version. +func VersionForNetwork(version network.Version) Version { + switch version { + case network.Version0, network.Version1, network.Version2: + return Version0 + default: + panic(fmt.Sprintf("unsupported network version %d", version)) + } +} diff --git a/chain/gen/genesis/genesis.go b/chain/gen/genesis/genesis.go index 039e284cd..bb1056e2e 100644 --- a/chain/gen/genesis/genesis.go +++ b/chain/gen/genesis/genesis.go @@ -24,7 +24,7 @@ import ( adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/actors/builtin" + "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" @@ -115,7 +115,7 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge return nil, nil, xerrors.Errorf("putting empty object: %w", err) } - state, err := state.NewStateTree(cst, builtin.Version0) + state, err := state.NewStateTree(cst, actors.Version0) if err != nil { return nil, nil, xerrors.Errorf("making new state tree: %w", err) } diff --git a/chain/state/statetree.go b/chain/state/statetree.go index fe932bfa1..acf67e977 100644 --- a/chain/state/statetree.go +++ b/chain/state/statetree.go @@ -13,7 +13,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/chain/actors/builtin" + "github.com/filecoin-project/lotus/chain/actors" init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init" cbg "github.com/whyrusleeping/cbor-gen" @@ -26,7 +26,7 @@ var log = logging.Logger("statetree") // StateTree stores actors state by their ID. type StateTree struct { root adt.Map - version builtin.Version // TODO + version actors.Version // TODO info cid.Cid Store cbor.IpldStore @@ -120,10 +120,10 @@ func (ss *stateSnaps) deleteActor(addr address.Address) { ss.layers[len(ss.layers)-1].actors[addr] = streeOp{Delete: true} } -func NewStateTree(cst cbor.IpldStore, version builtin.Version) (*StateTree, error) { +func NewStateTree(cst cbor.IpldStore, version actors.Version) (*StateTree, error) { var info cid.Cid switch version { - case builtin.Version0: + case actors.Version0: // info is undefined default: return nil, xerrors.Errorf("unsupported state tree version: %d", version) @@ -148,18 +148,18 @@ func LoadStateTree(cst cbor.IpldStore, c cid.Cid) (*StateTree, error) { if err := cst.Get(context.TODO(), c, &root); err != nil { // We failed to decode as the new version, must be an old version. root.Actors = c - root.Version = builtin.Version0 + root.Version = actors.Version0 } // If that fails, load as an old-style state-tree (direct hampt, version 0. - nd, err := adt.AsMap(adt.WrapStore(context.TODO(), cst), root.Actors, builtin.Version(root.Version)) + nd, err := adt.AsMap(adt.WrapStore(context.TODO(), cst), root.Actors, actors.Version(root.Version)) if err != nil { log.Errorf("loading hamt node %s failed: %s", c, err) return nil, err } switch root.Version { - case builtin.Version0: + case actors.Version0: // supported default: return nil, xerrors.Errorf("unsupported state tree version: %d", root.Version) @@ -168,7 +168,7 @@ func LoadStateTree(cst cbor.IpldStore, c cid.Cid) (*StateTree, error) { return &StateTree{ root: nd, info: root.Info, - version: builtin.Version(root.Version), + version: actors.Version(root.Version), Store: cst, snaps: newStateSnaps(), }, nil @@ -305,7 +305,7 @@ func (st *StateTree) Flush(ctx context.Context) (cid.Cid, error) { return cid.Undef, xerrors.Errorf("failed to flush state-tree hamt: %w", err) } // If we're version 0, return a raw tree. - if st.version == builtin.Version0 { + if st.version == actors.Version0 { return root, nil } // Otherwise, return a versioned tree. diff --git a/chain/state/statetree_test.go b/chain/state/statetree_test.go index 79ab20606..554ba2697 100644 --- a/chain/state/statetree_test.go +++ b/chain/state/statetree_test.go @@ -12,13 +12,13 @@ import ( "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/lotus/build" - builtin2 "github.com/filecoin-project/lotus/chain/actors/builtin" + "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/types" ) func BenchmarkStateTreeSet(b *testing.B) { cst := cbor.NewMemCborStore() - st, err := NewStateTree(cst, builtin2.VersionForNetwork(build.NewestNetworkVersion)) + st, err := NewStateTree(cst, actors.VersionForNetwork(build.NewestNetworkVersion)) if err != nil { b.Fatal(err) } @@ -45,7 +45,7 @@ func BenchmarkStateTreeSet(b *testing.B) { func BenchmarkStateTreeSetFlush(b *testing.B) { cst := cbor.NewMemCborStore() - st, err := NewStateTree(cst, builtin2.VersionForNetwork(build.NewestNetworkVersion)) + st, err := NewStateTree(cst, actors.VersionForNetwork(build.NewestNetworkVersion)) if err != nil { b.Fatal(err) } @@ -75,7 +75,7 @@ func BenchmarkStateTreeSetFlush(b *testing.B) { func BenchmarkStateTree10kGetActor(b *testing.B) { cst := cbor.NewMemCborStore() - st, err := NewStateTree(cst, builtin2.VersionForNetwork(build.NewestNetworkVersion)) + st, err := NewStateTree(cst, actors.VersionForNetwork(build.NewestNetworkVersion)) if err != nil { b.Fatal(err) } @@ -117,7 +117,7 @@ func BenchmarkStateTree10kGetActor(b *testing.B) { func TestSetCache(t *testing.T) { cst := cbor.NewMemCborStore() - st, err := NewStateTree(cst, builtin2.VersionForNetwork(build.NewestNetworkVersion)) + st, err := NewStateTree(cst, actors.VersionForNetwork(build.NewestNetworkVersion)) if err != nil { t.Fatal(err) } @@ -154,7 +154,7 @@ func TestSetCache(t *testing.T) { func TestSnapshots(t *testing.T) { ctx := context.Background() cst := cbor.NewMemCborStore() - st, err := NewStateTree(cst, builtin2.VersionForNetwork(build.NewestNetworkVersion)) + st, err := NewStateTree(cst, actors.VersionForNetwork(build.NewestNetworkVersion)) if err != nil { t.Fatal(err) } @@ -237,7 +237,7 @@ func assertNotHas(t *testing.T, st *StateTree, addr address.Address) { func TestStateTreeConsistency(t *testing.T) { cst := cbor.NewMemCborStore() - st, err := NewStateTree(cst, builtin2.VersionForNetwork(build.NewestNetworkVersion)) + st, err := NewStateTree(cst, actors.VersionForNetwork(build.NewestNetworkVersion)) if err != nil { t.Fatal(err) } diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index eaf9215db..b93de3b07 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -24,7 +24,6 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/adt" - "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/builtin/market" "github.com/filecoin-project/lotus/chain/actors/builtin/multisig" "github.com/filecoin-project/lotus/chain/actors/builtin/paych" @@ -291,7 +290,7 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp } // XXX: Is the height correct? Or should it be epoch-1? - rectarr, err := adt.NewArray(sm.cs.Store(ctx), builtin.VersionForNetwork(sm.GetNtwkVersion(ctx, epoch))) + rectarr, err := adt.NewArray(sm.cs.Store(ctx), actors.VersionForNetwork(sm.GetNtwkVersion(ctx, epoch))) if err != nil { return cid.Undef, cid.Undef, xerrors.Errorf("failed to create receipts amt: %w", err) } From d9656f5220a1744966417ac5d881378447c3861c Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 23 Sep 2020 17:40:29 -0700 Subject: [PATCH 344/885] add a generic load method for actor state This will make it easier to load arbitrary actors. We can: * Type switch (sort of unsafe, may want marker methods?) * Use this with `vm.MutateActorState`. --- chain/actors/builtin/account/account.go | 30 ++++++++++--------- chain/actors/builtin/account/v0.go | 16 ++++++++-- chain/actors/builtin/account/v1.go | 16 ++++++++-- chain/actors/builtin/builtin.go | 25 +++++++++++++++- chain/actors/builtin/init/init.go | 30 ++++++++++--------- chain/actors/builtin/init/v0.go | 14 +++++++-- chain/actors/builtin/init/v1.go | 14 +++++++-- chain/actors/builtin/market/market.go | 24 +++++++-------- chain/actors/builtin/market/v0.go | 36 +++++++++++++++-------- chain/actors/builtin/market/v1.go | 36 +++++++++++++++-------- chain/actors/builtin/miner/miner.go | 24 +++++++-------- chain/actors/builtin/miner/v0.go | 20 +++++++++---- chain/actors/builtin/miner/v1.go | 15 ++++++++-- chain/actors/builtin/multisig/multisig.go | 25 ++++++++-------- chain/actors/builtin/multisig/v0.go | 13 +++++++- chain/actors/builtin/multisig/v1.go | 13 +++++++- chain/actors/builtin/paych/paych.go | 25 ++++++++-------- chain/actors/builtin/paych/v0.go | 23 +++++++++++---- chain/actors/builtin/paych/v1.go | 21 ++++++++++--- chain/actors/builtin/power/power.go | 24 +++++++-------- chain/actors/builtin/power/v0.go | 11 +++++++ chain/actors/builtin/power/v1.go | 11 +++++++ chain/actors/builtin/reward/reward.go | 24 +++++++-------- chain/actors/builtin/reward/v0.go | 11 +++++++ chain/actors/builtin/reward/v1.go | 11 +++++++ chain/actors/builtin/verifreg/v0.go | 10 +++++++ chain/actors/builtin/verifreg/v1.go | 10 +++++++ chain/actors/builtin/verifreg/verifreg.go | 25 ++++++++-------- 28 files changed, 394 insertions(+), 163 deletions(-) diff --git a/chain/actors/builtin/account/account.go b/chain/actors/builtin/account/account.go index 07f4c4fc7..2754607e2 100644 --- a/chain/actors/builtin/account/account.go +++ b/chain/actors/builtin/account/account.go @@ -5,29 +5,31 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/cbor" - builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" - builtin1 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + "github.com/ipfs/go-cid" "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/types" + + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" + builtin1 "github.com/filecoin-project/specs-actors/v2/actors/builtin" ) +func init() { + builtin.RegisterActorState(builtin0.AccountActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) { + return load0(store, root) + }) + builtin.RegisterActorState(builtin1.AccountActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) { + return load1(store, root) + }) +} + func Load(store adt.Store, act *types.Actor) (State, error) { switch act.Code { case builtin0.AccountActorCodeID: - out := state0{store: store} - err := store.Get(store.Context(), act.Head, &out) - if err != nil { - return nil, err - } - return &out, nil + return load0(store, act.Head) case builtin1.AccountActorCodeID: - out := state1{store: store} - err := store.Get(store.Context(), act.Head, &out) - if err != nil { - return nil, err - } - return &out, nil + return load1(store, act.Head) } return nil, xerrors.Errorf("unknown actor code %s", act.Code) } diff --git a/chain/actors/builtin/account/v0.go b/chain/actors/builtin/account/v0.go index 30bafbfd3..67c555c5d 100644 --- a/chain/actors/builtin/account/v0.go +++ b/chain/actors/builtin/account/v0.go @@ -2,14 +2,26 @@ package account import ( "github.com/filecoin-project/go-address" + "github.com/ipfs/go-cid" + "github.com/filecoin-project/lotus/chain/actors/adt" - "github.com/filecoin-project/specs-actors/actors/builtin/account" + + account0 "github.com/filecoin-project/specs-actors/actors/builtin/account" ) var _ State = (*state0)(nil) +func load0(store adt.Store, root cid.Cid) (State, error) { + out := state0{store: store} + err := store.Get(store.Context(), root, &out) + if err != nil { + return nil, err + } + return &out, nil +} + type state0 struct { - account.State + account0.State store adt.Store } diff --git a/chain/actors/builtin/account/v1.go b/chain/actors/builtin/account/v1.go index ed0fb0104..993c0e397 100644 --- a/chain/actors/builtin/account/v1.go +++ b/chain/actors/builtin/account/v1.go @@ -2,14 +2,26 @@ package account import ( "github.com/filecoin-project/go-address" + "github.com/ipfs/go-cid" + "github.com/filecoin-project/lotus/chain/actors/adt" - "github.com/filecoin-project/specs-actors/v2/actors/builtin/account" + + account1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/account" ) var _ State = (*state1)(nil) +func load1(store adt.Store, root cid.Cid) (State, error) { + out := state1{store: store} + err := store.Get(store.Context(), root, &out) + if err != nil { + return nil, err + } + return &out, nil +} + type state1 struct { - account.State + account1.State store adt.Store } diff --git a/chain/actors/builtin/builtin.go b/chain/actors/builtin/builtin.go index 53dfa8669..49a07951b 100644 --- a/chain/actors/builtin/builtin.go +++ b/chain/actors/builtin/builtin.go @@ -1,10 +1,17 @@ package builtin import ( + "github.com/ipfs/go-cid" + "golang.org/x/xerrors" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/cbor" + + "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/chain/types" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" proof0 "github.com/filecoin-project/specs-actors/actors/runtime/proof" - smoothing0 "github.com/filecoin-project/specs-actors/actors/util/smoothing" smoothing1 "github.com/filecoin-project/specs-actors/v2/actors/util/smoothing" ) @@ -26,3 +33,19 @@ func QAPowerForWeight(size abi.SectorSize, duration abi.ChainEpoch, dealWeight, func FromV1FilterEstimate(v1 smoothing1.FilterEstimate) FilterEstimate { return (FilterEstimate)(v1) } + +type ActorStateLoader func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) + +var ActorStateLoaders = make(map[cid.Cid]ActorStateLoader) + +func RegisterActorState(code cid.Cid, loader ActorStateLoader) { + ActorStateLoaders[code] = loader +} + +func Load(store adt.Store, act *types.Actor) (cbor.Marshaler, error) { + loader, found := ActorStateLoaders[act.Code] + if !found { + return nil, xerrors.Errorf("unknown actor code %s", act.Code) + } + return loader(store, act.Head) +} diff --git a/chain/actors/builtin/init/init.go b/chain/actors/builtin/init/init.go index a9fd25a1a..622921ee9 100644 --- a/chain/actors/builtin/init/init.go +++ b/chain/actors/builtin/init/init.go @@ -6,32 +6,34 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/cbor" - builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" - builtin1 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + "github.com/ipfs/go-cid" "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/modules/dtypes" + + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" + builtin1 "github.com/filecoin-project/specs-actors/v2/actors/builtin" ) +func init() { + builtin.RegisterActorState(builtin0.InitActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) { + return load0(store, root) + }) + builtin.RegisterActorState(builtin1.InitActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) { + return load1(store, root) + }) +} + var Address = builtin0.InitActorAddr func Load(store adt.Store, act *types.Actor) (State, error) { switch act.Code { case builtin0.InitActorCodeID: - out := state0{store: store} - err := store.Get(store.Context(), act.Head, &out) - if err != nil { - return nil, err - } - return &out, nil + return load0(store, act.Head) case builtin1.InitActorCodeID: - out := state1{store: store} - err := store.Get(store.Context(), act.Head, &out) - if err != nil { - return nil, err - } - return &out, nil + return load1(store, act.Head) } return nil, xerrors.Errorf("unknown actor code %s", act.Code) } diff --git a/chain/actors/builtin/init/v0.go b/chain/actors/builtin/init/v0.go index b82b85aee..de0d6c9e1 100644 --- a/chain/actors/builtin/init/v0.go +++ b/chain/actors/builtin/init/v0.go @@ -3,20 +3,30 @@ package init import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/node/modules/dtypes" - init_ "github.com/filecoin-project/specs-actors/actors/builtin/init" + init0 "github.com/filecoin-project/specs-actors/actors/builtin/init" adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" ) var _ State = (*state0)(nil) +func load0(store adt.Store, root cid.Cid) (State, error) { + out := state0{store: store} + err := store.Get(store.Context(), root, &out) + if err != nil { + return nil, err + } + return &out, nil +} + type state0 struct { - init_.State + init0.State store adt.Store } diff --git a/chain/actors/builtin/init/v1.go b/chain/actors/builtin/init/v1.go index d65d5b89d..dcaa5a9ed 100644 --- a/chain/actors/builtin/init/v1.go +++ b/chain/actors/builtin/init/v1.go @@ -3,20 +3,30 @@ package init import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/node/modules/dtypes" - init_ "github.com/filecoin-project/specs-actors/v2/actors/builtin/init" + init1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/init" adt1 "github.com/filecoin-project/specs-actors/v2/actors/util/adt" ) var _ State = (*state1)(nil) +func load1(store adt.Store, root cid.Cid) (State, error) { + out := state1{store: store} + err := store.Get(store.Context(), root, &out) + if err != nil { + return nil, err + } + return &out, nil +} + type state1 struct { - init_.State + init1.State store adt.Store } diff --git a/chain/actors/builtin/market/market.go b/chain/actors/builtin/market/market.go index b1c9d7a7e..d0798560b 100644 --- a/chain/actors/builtin/market/market.go +++ b/chain/actors/builtin/market/market.go @@ -14,27 +14,27 @@ import ( builtin1 "github.com/filecoin-project/specs-actors/v2/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/types" ) +func init() { + builtin.RegisterActorState(builtin0.StorageMarketActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) { + return load0(store, root) + }) + builtin.RegisterActorState(builtin1.StorageMarketActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) { + return load1(store, root) + }) +} + var Address = builtin0.StorageMarketActorAddr func Load(store adt.Store, act *types.Actor) (st State, err error) { switch act.Code { case builtin0.StorageMarketActorCodeID: - out := state0{store: store} - err := store.Get(store.Context(), act.Head, &out) - if err != nil { - return nil, err - } - return &out, nil + return load0(store, act.Head) case builtin1.StorageMarketActorCodeID: - out := state1{store: store} - err := store.Get(store.Context(), act.Head, &out) - if err != nil { - return nil, err - } - return &out, nil + return load1(store, act.Head) } return nil, xerrors.Errorf("unknown actor code %s", act.Code) } diff --git a/chain/actors/builtin/market/v0.go b/chain/actors/builtin/market/v0.go index 2727f513d..20d38b5f1 100644 --- a/chain/actors/builtin/market/v0.go +++ b/chain/actors/builtin/market/v0.go @@ -5,17 +5,29 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/specs-actors/actors/builtin/market" + + market0 "github.com/filecoin-project/specs-actors/actors/builtin/market" adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" - cbg "github.com/whyrusleeping/cbor-gen" ) var _ State = (*state0)(nil) +func load0(store adt.Store, root cid.Cid) (State, error) { + out := state0{store: store} + err := store.Get(store.Context(), root, &out) + if err != nil { + return nil, err + } + return &out, nil +} + type state0 struct { - market.State + market0.State store adt.Store } @@ -90,7 +102,7 @@ func (s *state0) LockedTable() (BalanceTable, error) { func (s *state0) VerifyDealsForActivation( minerAddr address.Address, deals []abi.DealID, currEpoch, sectorExpiry abi.ChainEpoch, ) (weight, verifiedWeight abi.DealWeight, err error) { - return market.ValidateDealsForActivation(&s.State, s.store, deals, minerAddr, sectorExpiry, currEpoch) + return market0.ValidateDealsForActivation(&s.State, s.store, deals, minerAddr, sectorExpiry, currEpoch) } type balanceTable0 struct { @@ -114,7 +126,7 @@ type dealStates0 struct { } func (s *dealStates0) Get(dealID abi.DealID) (*DealState, bool, error) { - var deal0 market.DealState + var deal0 market0.DealState found, err := s.Array.Get(uint64(dealID), &deal0) if err != nil { return nil, false, err @@ -127,14 +139,14 @@ func (s *dealStates0) Get(dealID abi.DealID) (*DealState, bool, error) { } func (s *dealStates0) ForEach(cb func(dealID abi.DealID, ds DealState) error) error { - var ds0 market.DealState + var ds0 market0.DealState return s.Array.ForEach(&ds0, func(idx int64) error { return cb(abi.DealID(idx), fromV0DealState(ds0)) }) } func (s *dealStates0) decode(val *cbg.Deferred) (*DealState, error) { - var ds0 market.DealState + var ds0 market0.DealState if err := ds0.UnmarshalCBOR(bytes.NewReader(val.Raw)); err != nil { return nil, err } @@ -146,7 +158,7 @@ func (s *dealStates0) array() adt.Array { return s.Array } -func fromV0DealState(v0 market.DealState) DealState { +func fromV0DealState(v0 market0.DealState) DealState { return (DealState)(v0) } @@ -155,7 +167,7 @@ type dealProposals0 struct { } func (s *dealProposals0) Get(dealID abi.DealID) (*DealProposal, bool, error) { - var proposal0 market.DealProposal + var proposal0 market0.DealProposal found, err := s.Array.Get(uint64(dealID), &proposal0) if err != nil { return nil, false, err @@ -168,14 +180,14 @@ func (s *dealProposals0) Get(dealID abi.DealID) (*DealProposal, bool, error) { } func (s *dealProposals0) ForEach(cb func(dealID abi.DealID, dp DealProposal) error) error { - var dp0 market.DealProposal + var dp0 market0.DealProposal return s.Array.ForEach(&dp0, func(idx int64) error { return cb(abi.DealID(idx), fromV0DealProposal(dp0)) }) } func (s *dealProposals0) decode(val *cbg.Deferred) (*DealProposal, error) { - var dp0 market.DealProposal + var dp0 market0.DealProposal if err := dp0.UnmarshalCBOR(bytes.NewReader(val.Raw)); err != nil { return nil, err } @@ -187,6 +199,6 @@ func (s *dealProposals0) array() adt.Array { return s.Array } -func fromV0DealProposal(v0 market.DealProposal) DealProposal { +func fromV0DealProposal(v0 market0.DealProposal) DealProposal { return (DealProposal)(v0) } diff --git a/chain/actors/builtin/market/v1.go b/chain/actors/builtin/market/v1.go index dc84c2633..3ad0cded6 100644 --- a/chain/actors/builtin/market/v1.go +++ b/chain/actors/builtin/market/v1.go @@ -5,17 +5,29 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/specs-actors/v2/actors/builtin/market" + + market1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/market" adt1 "github.com/filecoin-project/specs-actors/v2/actors/util/adt" - cbg "github.com/whyrusleeping/cbor-gen" ) var _ State = (*state1)(nil) +func load1(store adt.Store, root cid.Cid) (State, error) { + out := state1{store: store} + err := store.Get(store.Context(), root, &out) + if err != nil { + return nil, err + } + return &out, nil +} + type state1 struct { - market.State + market1.State store adt.Store } @@ -90,7 +102,7 @@ func (s *state1) LockedTable() (BalanceTable, error) { func (s *state1) VerifyDealsForActivation( minerAddr address.Address, deals []abi.DealID, currEpoch, sectorExpiry abi.ChainEpoch, ) (weight, verifiedWeight abi.DealWeight, err error) { - w, vw, _, err := market.ValidateDealsForActivation(&s.State, s.store, deals, minerAddr, sectorExpiry, currEpoch) + w, vw, _, err := market1.ValidateDealsForActivation(&s.State, s.store, deals, minerAddr, sectorExpiry, currEpoch) return w, vw, err } @@ -115,7 +127,7 @@ type dealStates1 struct { } func (s *dealStates1) Get(dealID abi.DealID) (*DealState, bool, error) { - var deal1 market.DealState + var deal1 market1.DealState found, err := s.Array.Get(uint64(dealID), &deal1) if err != nil { return nil, false, err @@ -128,14 +140,14 @@ func (s *dealStates1) Get(dealID abi.DealID) (*DealState, bool, error) { } func (s *dealStates1) ForEach(cb func(dealID abi.DealID, ds DealState) error) error { - var ds1 market.DealState + var ds1 market1.DealState return s.Array.ForEach(&ds1, func(idx int64) error { return cb(abi.DealID(idx), fromV1DealState(ds1)) }) } func (s *dealStates1) decode(val *cbg.Deferred) (*DealState, error) { - var ds1 market.DealState + var ds1 market1.DealState if err := ds1.UnmarshalCBOR(bytes.NewReader(val.Raw)); err != nil { return nil, err } @@ -147,7 +159,7 @@ func (s *dealStates1) array() adt.Array { return s.Array } -func fromV1DealState(v1 market.DealState) DealState { +func fromV1DealState(v1 market1.DealState) DealState { return (DealState)(v1) } @@ -156,7 +168,7 @@ type dealProposals1 struct { } func (s *dealProposals1) Get(dealID abi.DealID) (*DealProposal, bool, error) { - var proposal1 market.DealProposal + var proposal1 market1.DealProposal found, err := s.Array.Get(uint64(dealID), &proposal1) if err != nil { return nil, false, err @@ -169,14 +181,14 @@ func (s *dealProposals1) Get(dealID abi.DealID) (*DealProposal, bool, error) { } func (s *dealProposals1) ForEach(cb func(dealID abi.DealID, dp DealProposal) error) error { - var dp1 market.DealProposal + var dp1 market1.DealProposal return s.Array.ForEach(&dp1, func(idx int64) error { return cb(abi.DealID(idx), fromV1DealProposal(dp1)) }) } func (s *dealProposals1) decode(val *cbg.Deferred) (*DealProposal, error) { - var dp1 market.DealProposal + var dp1 market1.DealProposal if err := dp1.UnmarshalCBOR(bytes.NewReader(val.Raw)); err != nil { return nil, err } @@ -188,6 +200,6 @@ func (s *dealProposals1) array() adt.Array { return s.Array } -func fromV1DealProposal(v1 market.DealProposal) DealProposal { +func fromV1DealProposal(v1 market1.DealProposal) DealProposal { return (DealProposal)(v1) } diff --git a/chain/actors/builtin/miner/miner.go b/chain/actors/builtin/miner/miner.go index 11f65fd49..00687a76f 100644 --- a/chain/actors/builtin/miner/miner.go +++ b/chain/actors/builtin/miner/miner.go @@ -13,6 +13,7 @@ import ( "github.com/filecoin-project/go-state-types/dline" "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/types" builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" @@ -20,6 +21,15 @@ import ( builtin1 "github.com/filecoin-project/specs-actors/v2/actors/builtin" ) +func init() { + builtin.RegisterActorState(builtin0.StorageMinerActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) { + return load0(store, root) + }) + builtin.RegisterActorState(builtin1.StorageMinerActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) { + return load1(store, root) + }) +} + // Unchanged between v0 and v1 actors var WPoStProvingPeriod = miner0.WPoStProvingPeriod @@ -28,19 +38,9 @@ const MinSectorExpiration = miner0.MinSectorExpiration func Load(store adt.Store, act *types.Actor) (st State, err error) { switch act.Code { case builtin0.StorageMinerActorCodeID: - out := state0{store: store} - err := store.Get(store.Context(), act.Head, &out) - if err != nil { - return nil, err - } - return &out, nil + return load0(store, act.Head) case builtin1.StorageMinerActorCodeID: - out := state1{store: store} - err := store.Get(store.Context(), act.Head, &out) - if err != nil { - return nil, err - } - return &out, nil + return load1(store, act.Head) } return nil, xerrors.Errorf("unknown actor code %s", act.Code) } diff --git a/chain/actors/builtin/miner/v0.go b/chain/actors/builtin/miner/v0.go index 9cdfc25bc..c0bd6f5dc 100644 --- a/chain/actors/builtin/miner/v0.go +++ b/chain/actors/builtin/miner/v0.go @@ -4,21 +4,31 @@ import ( "bytes" "errors" - "github.com/libp2p/go-libp2p-core/peer" - cbg "github.com/whyrusleeping/cbor-gen" - "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-bitfield" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/dline" - miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" - adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" + "github.com/ipfs/go-cid" + "github.com/libp2p/go-libp2p-core/peer" + cbg "github.com/whyrusleeping/cbor-gen" "github.com/filecoin-project/lotus/chain/actors/adt" + + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" + adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" ) var _ State = (*state0)(nil) +func load0(store adt.Store, root cid.Cid) (State, error) { + out := state0{store: store} + err := store.Get(store.Context(), root, &out) + if err != nil { + return nil, err + } + return &out, nil +} + type state0 struct { miner0.State store adt.Store diff --git a/chain/actors/builtin/miner/v1.go b/chain/actors/builtin/miner/v1.go index 4c991b4f3..9259353f3 100644 --- a/chain/actors/builtin/miner/v1.go +++ b/chain/actors/builtin/miner/v1.go @@ -8,16 +8,27 @@ import ( "github.com/filecoin-project/go-bitfield" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/dline" - "github.com/filecoin-project/lotus/chain/actors/adt" - adt1 "github.com/filecoin-project/specs-actors/v2/actors/util/adt" + "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p-core/peer" cbg "github.com/whyrusleeping/cbor-gen" + "github.com/filecoin-project/lotus/chain/actors/adt" + miner1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" + adt1 "github.com/filecoin-project/specs-actors/v2/actors/util/adt" ) var _ State = (*state1)(nil) +func load1(store adt.Store, root cid.Cid) (State, error) { + out := state1{store: store} + err := store.Get(store.Context(), root, &out) + if err != nil { + return nil, err + } + return &out, nil +} + type state1 struct { miner1.State store adt.Store diff --git a/chain/actors/builtin/multisig/multisig.go b/chain/actors/builtin/multisig/multisig.go index a30082824..022e7e2c5 100644 --- a/chain/actors/builtin/multisig/multisig.go +++ b/chain/actors/builtin/multisig/multisig.go @@ -6,31 +6,32 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/cbor" + "github.com/ipfs/go-cid" builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" msig0 "github.com/filecoin-project/specs-actors/actors/builtin/multisig" builtin1 "github.com/filecoin-project/specs-actors/v2/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/types" ) +func init() { + builtin.RegisterActorState(builtin0.MultisigActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) { + return load0(store, root) + }) + builtin.RegisterActorState(builtin1.MultisigActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) { + return load1(store, root) + }) +} + func Load(store adt.Store, act *types.Actor) (State, error) { switch act.Code { case builtin0.MultisigActorCodeID: - out := state0{store: store} - err := store.Get(store.Context(), act.Head, &out) - if err != nil { - return nil, err - } - return &out, nil + return load0(store, act.Head) case builtin1.MultisigActorCodeID: - out := state1{store: store} - err := store.Get(store.Context(), act.Head, &out) - if err != nil { - return nil, err - } - return &out, nil + return load1(store, act.Head) } return nil, xerrors.Errorf("unknown actor code %s", act.Code) } diff --git a/chain/actors/builtin/multisig/v0.go b/chain/actors/builtin/multisig/v0.go index ae0a7ac0e..c934343e7 100644 --- a/chain/actors/builtin/multisig/v0.go +++ b/chain/actors/builtin/multisig/v0.go @@ -5,15 +5,26 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/ipfs/go-cid" "golang.org/x/xerrors" + "github.com/filecoin-project/lotus/chain/actors/adt" + msig0 "github.com/filecoin-project/specs-actors/actors/builtin/multisig" adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" ) var _ State = (*state0)(nil) +func load0(store adt.Store, root cid.Cid) (State, error) { + out := state0{store: store} + err := store.Get(store.Context(), root, &out) + if err != nil { + return nil, err + } + return &out, nil +} + type state0 struct { msig0.State store adt.Store diff --git a/chain/actors/builtin/multisig/v1.go b/chain/actors/builtin/multisig/v1.go index 97b9ae4a3..7bfde404a 100644 --- a/chain/actors/builtin/multisig/v1.go +++ b/chain/actors/builtin/multisig/v1.go @@ -5,15 +5,26 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/ipfs/go-cid" "golang.org/x/xerrors" + "github.com/filecoin-project/lotus/chain/actors/adt" + msig1 "github.com/filecoin-project/specs-actors/actors/builtin/multisig" adt1 "github.com/filecoin-project/specs-actors/actors/util/adt" ) var _ State = (*state1)(nil) +func load1(store adt.Store, root cid.Cid) (State, error) { + out := state1{store: store} + err := store.Get(store.Context(), root, &out) + if err != nil { + return nil, err + } + return &out, nil +} + type state1 struct { msig1.State store adt.Store diff --git a/chain/actors/builtin/paych/paych.go b/chain/actors/builtin/paych/paych.go index 36cf56e63..959968f22 100644 --- a/chain/actors/builtin/paych/paych.go +++ b/chain/actors/builtin/paych/paych.go @@ -7,32 +7,33 @@ import ( "github.com/filecoin-project/go-state-types/abi" big "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/cbor" + "github.com/ipfs/go-cid" builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" paych0 "github.com/filecoin-project/specs-actors/actors/builtin/paych" builtin1 "github.com/filecoin-project/specs-actors/v2/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/types" ) +func init() { + builtin.RegisterActorState(builtin0.PaymentChannelActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) { + return load0(store, root) + }) + builtin.RegisterActorState(builtin1.PaymentChannelActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) { + return load1(store, root) + }) +} + // Load returns an abstract copy of payment channel state, irregardless of actor version func Load(store adt.Store, act *types.Actor) (State, error) { switch act.Code { case builtin0.PaymentChannelActorCodeID: - out := state0{store: store} - err := store.Get(store.Context(), act.Head, &out) - if err != nil { - return nil, err - } - return &out, nil + return load0(store, act.Head) case builtin1.PaymentChannelActorCodeID: - out := state1{store: store} - err := store.Get(store.Context(), act.Head, &out) - if err != nil { - return nil, err - } - return &out, nil + return load1(store, act.Head) } return nil, xerrors.Errorf("unknown actor code %s", act.Code) } diff --git a/chain/actors/builtin/paych/v0.go b/chain/actors/builtin/paych/v0.go index c0eea1000..8e0e3434e 100644 --- a/chain/actors/builtin/paych/v0.go +++ b/chain/actors/builtin/paych/v0.go @@ -1,18 +1,31 @@ package paych import ( + "github.com/ipfs/go-cid" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" - big "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/lotus/chain/actors/adt" - "github.com/filecoin-project/specs-actors/actors/builtin/paych" + + paych0 "github.com/filecoin-project/specs-actors/actors/builtin/paych" adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" ) var _ State = (*state0)(nil) +func load0(store adt.Store, root cid.Cid) (State, error) { + out := state0{store: store} + err := store.Get(store.Context(), root, &out) + if err != nil { + return nil, err + } + return &out, nil +} + type state0 struct { - paych.State + paych0.State store adt.Store lsAmt *adt0.Array } @@ -72,14 +85,14 @@ func (s *state0) ForEachLaneState(cb func(idx uint64, dl LaneState) error) error // Note: we use a map instead of an array to store laneStates because the // client sets the lane ID (the index) and potentially they could use a // very large index. - var ls paych.LaneState + var ls paych0.LaneState return lsamt.ForEach(&ls, func(i int64) error { return cb(uint64(i), &laneState0{ls}) }) } type laneState0 struct { - paych.LaneState + paych0.LaneState } func (ls *laneState0) Redeemed() (big.Int, error) { diff --git a/chain/actors/builtin/paych/v1.go b/chain/actors/builtin/paych/v1.go index 7a960067c..c1b47c120 100644 --- a/chain/actors/builtin/paych/v1.go +++ b/chain/actors/builtin/paych/v1.go @@ -1,18 +1,31 @@ package paych import ( + "github.com/ipfs/go-cid" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/lotus/chain/actors/adt" - "github.com/filecoin-project/specs-actors/v2/actors/builtin/paych" + + paych1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/paych" adt1 "github.com/filecoin-project/specs-actors/v2/actors/util/adt" ) var _ State = (*state1)(nil) +func load1(store adt.Store, root cid.Cid) (State, error) { + out := state1{store: store} + err := store.Get(store.Context(), root, &out) + if err != nil { + return nil, err + } + return &out, nil +} + type state1 struct { - paych.State + paych1.State store adt.Store lsAmt *adt1.Array } @@ -72,14 +85,14 @@ func (s *state1) ForEachLaneState(cb func(idx uint64, dl LaneState) error) error // Note: we use a map instead of an array to store laneStates because the // client sets the lane ID (the index) and potentially they could use a // very large index. - var ls paych.LaneState + var ls paych1.LaneState return lsamt.ForEach(&ls, func(i int64) error { return cb(uint64(i), &laneState1{ls}) }) } type laneState1 struct { - paych.LaneState + paych1.LaneState } func (ls *laneState1) Redeemed() (big.Int, error) { diff --git a/chain/actors/builtin/power/power.go b/chain/actors/builtin/power/power.go index 7aa86fe00..9dd61c07e 100644 --- a/chain/actors/builtin/power/power.go +++ b/chain/actors/builtin/power/power.go @@ -2,6 +2,7 @@ package power import ( "github.com/filecoin-project/go-address" + "github.com/ipfs/go-cid" "golang.org/x/xerrors" "github.com/filecoin-project/go-state-types/abi" @@ -15,24 +16,23 @@ import ( builtin1 "github.com/filecoin-project/specs-actors/v2/actors/builtin" ) +func init() { + builtin.RegisterActorState(builtin0.StoragePowerActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) { + return load0(store, root) + }) + builtin.RegisterActorState(builtin1.StoragePowerActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) { + return load1(store, root) + }) +} + var Address = builtin0.StoragePowerActorAddr func Load(store adt.Store, act *types.Actor) (st State, err error) { switch act.Code { case builtin0.StoragePowerActorCodeID: - out := state0{store: store} - err := store.Get(store.Context(), act.Head, &out) - if err != nil { - return nil, err - } - return &out, nil + return load0(store, act.Head) case builtin1.StoragePowerActorCodeID: - out := state1{store: store} - err := store.Get(store.Context(), act.Head, &out) - if err != nil { - return nil, err - } - return &out, nil + return load1(store, act.Head) } return nil, xerrors.Errorf("unknown actor code %s", act.Code) } diff --git a/chain/actors/builtin/power/v0.go b/chain/actors/builtin/power/v0.go index 657de7ed0..e2a9cf382 100644 --- a/chain/actors/builtin/power/v0.go +++ b/chain/actors/builtin/power/v0.go @@ -3,6 +3,8 @@ package power import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + "github.com/ipfs/go-cid" + "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/actors/builtin" @@ -12,6 +14,15 @@ import ( var _ State = (*state0)(nil) +func load0(store adt.Store, root cid.Cid) (State, error) { + out := state0{store: store} + err := store.Get(store.Context(), root, &out) + if err != nil { + return nil, err + } + return &out, nil +} + type state0 struct { power0.State store adt.Store diff --git a/chain/actors/builtin/power/v1.go b/chain/actors/builtin/power/v1.go index 8c1a052b3..38df11916 100644 --- a/chain/actors/builtin/power/v1.go +++ b/chain/actors/builtin/power/v1.go @@ -3,6 +3,8 @@ package power import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + "github.com/ipfs/go-cid" + "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/actors/builtin" @@ -12,6 +14,15 @@ import ( var _ State = (*state1)(nil) +func load1(store adt.Store, root cid.Cid) (State, error) { + out := state1{store: store} + err := store.Get(store.Context(), root, &out) + if err != nil { + return nil, err + } + return &out, nil +} + type state1 struct { power1.State store adt.Store diff --git a/chain/actors/builtin/reward/reward.go b/chain/actors/builtin/reward/reward.go index 324bc95dd..e6d05de46 100644 --- a/chain/actors/builtin/reward/reward.go +++ b/chain/actors/builtin/reward/reward.go @@ -3,6 +3,7 @@ package reward import ( "github.com/filecoin-project/go-state-types/abi" reward0 "github.com/filecoin-project/specs-actors/actors/builtin/reward" + "github.com/ipfs/go-cid" "golang.org/x/xerrors" "github.com/filecoin-project/go-state-types/cbor" @@ -14,24 +15,23 @@ import ( "github.com/filecoin-project/lotus/chain/types" ) +func init() { + builtin.RegisterActorState(builtin0.RewardActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) { + return load0(store, root) + }) + builtin.RegisterActorState(builtin1.RewardActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) { + return load1(store, root) + }) +} + var Address = builtin0.RewardActorAddr func Load(store adt.Store, act *types.Actor) (st State, err error) { switch act.Code { case builtin0.RewardActorCodeID: - out := state0{store: store} - err := store.Get(store.Context(), act.Head, &out) - if err != nil { - return nil, err - } - return &out, nil + return load0(store, act.Head) case builtin1.RewardActorCodeID: - out := state1{store: store} - err := store.Get(store.Context(), act.Head, &out) - if err != nil { - return nil, err - } - return &out, nil + return load1(store, act.Head) } return nil, xerrors.Errorf("unknown actor code %s", act.Code) } diff --git a/chain/actors/builtin/reward/v0.go b/chain/actors/builtin/reward/v0.go index 57084899d..0efd0b482 100644 --- a/chain/actors/builtin/reward/v0.go +++ b/chain/actors/builtin/reward/v0.go @@ -2,6 +2,8 @@ package reward import ( "github.com/filecoin-project/go-state-types/abi" + "github.com/ipfs/go-cid" + "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/actors/builtin" @@ -12,6 +14,15 @@ import ( var _ State = (*state0)(nil) +func load0(store adt.Store, root cid.Cid) (State, error) { + out := state0{store: store} + err := store.Get(store.Context(), root, &out) + if err != nil { + return nil, err + } + return &out, nil +} + type state0 struct { reward0.State store adt.Store diff --git a/chain/actors/builtin/reward/v1.go b/chain/actors/builtin/reward/v1.go index f16256a00..9f94bbf1d 100644 --- a/chain/actors/builtin/reward/v1.go +++ b/chain/actors/builtin/reward/v1.go @@ -2,6 +2,8 @@ package reward import ( "github.com/filecoin-project/go-state-types/abi" + "github.com/ipfs/go-cid" + "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/actors/builtin" @@ -12,6 +14,15 @@ import ( var _ State = (*state1)(nil) +func load1(store adt.Store, root cid.Cid) (State, error) { + out := state1{store: store} + err := store.Get(store.Context(), root, &out) + if err != nil { + return nil, err + } + return &out, nil +} + type state1 struct { reward1.State store adt.Store diff --git a/chain/actors/builtin/verifreg/v0.go b/chain/actors/builtin/verifreg/v0.go index 5cb053c79..c203cef39 100644 --- a/chain/actors/builtin/verifreg/v0.go +++ b/chain/actors/builtin/verifreg/v0.go @@ -3,6 +3,7 @@ package verifreg import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + "github.com/ipfs/go-cid" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/adt" @@ -12,6 +13,15 @@ import ( var _ State = (*state0)(nil) +func load0(store adt.Store, root cid.Cid) (State, error) { + out := state0{store: store} + err := store.Get(store.Context(), root, &out) + if err != nil { + return nil, err + } + return &out, nil +} + type state0 struct { verifreg0.State store adt.Store diff --git a/chain/actors/builtin/verifreg/v1.go b/chain/actors/builtin/verifreg/v1.go index ec998f9e5..0c9e1b139 100644 --- a/chain/actors/builtin/verifreg/v1.go +++ b/chain/actors/builtin/verifreg/v1.go @@ -3,6 +3,7 @@ package verifreg import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + "github.com/ipfs/go-cid" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/adt" @@ -12,6 +13,15 @@ import ( var _ State = (*state1)(nil) +func load1(store adt.Store, root cid.Cid) (State, error) { + out := state1{store: store} + err := store.Get(store.Context(), root, &out) + if err != nil { + return nil, err + } + return &out, nil +} + type state1 struct { verifreg1.State store adt.Store diff --git a/chain/actors/builtin/verifreg/verifreg.go b/chain/actors/builtin/verifreg/verifreg.go index 4255e8855..86b1c5939 100644 --- a/chain/actors/builtin/verifreg/verifreg.go +++ b/chain/actors/builtin/verifreg/verifreg.go @@ -3,6 +3,7 @@ package verifreg import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + "github.com/ipfs/go-cid" "golang.org/x/xerrors" "github.com/filecoin-project/go-state-types/cbor" @@ -10,27 +11,27 @@ import ( builtin1 "github.com/filecoin-project/specs-actors/v2/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/types" ) +func init() { + builtin.RegisterActorState(builtin0.VerifiedRegistryActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) { + return load0(store, root) + }) + builtin.RegisterActorState(builtin1.VerifiedRegistryActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) { + return load1(store, root) + }) +} + var Address = builtin0.VerifiedRegistryActorAddr func Load(store adt.Store, act *types.Actor) (State, error) { switch act.Code { case builtin0.VerifiedRegistryActorCodeID: - out := state0{store: store} - err := store.Get(store.Context(), act.Head, &out) - if err != nil { - return nil, err - } - return &out, nil + return load0(store, act.Head) case builtin1.VerifiedRegistryActorCodeID: - out := state1{store: store} - err := store.Get(store.Context(), act.Head, &out) - if err != nil { - return nil, err - } - return &out, nil + return load1(store, act.Head) } return nil, xerrors.Errorf("unknown actor code %s", act.Code) } From 8b35f480c4f4c3dc7379f1767e2af8eeee4559f9 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 24 Sep 2020 17:51:34 -0700 Subject: [PATCH 345/885] initial vm conversion We're probably going to want to change some of these design decisions down the road, but this is a good starting point. * We may want to use a more general test for "is actor valid at epoch". Maybe just a function? * I'd like to push some of the actor metadata down into the actor types themselves. Ideally, we'd be able to register actors with a simple `Register(validation, manyActors...)` call. --- chain/stmgr/forks_test.go | 4 +- chain/vm/gas.go | 3 +- chain/vm/gas_v0.go | 6 +- chain/vm/invoker.go | 126 ++++++++++++++++++++++++-------------- chain/vm/invoker_test.go | 2 +- chain/vm/mkactor.go | 40 ++++++------ chain/vm/runtime.go | 18 ++---- chain/vm/vm.go | 10 +-- conformance/driver.go | 5 +- node/impl/full/chain.go | 2 +- node/impl/full/state.go | 2 +- 11 files changed, 121 insertions(+), 97 deletions(-) diff --git a/chain/stmgr/forks_test.go b/chain/stmgr/forks_test.go index 516058a81..8ec00f95f 100644 --- a/chain/stmgr/forks_test.go +++ b/chain/stmgr/forks_test.go @@ -106,7 +106,7 @@ func TestForkHeightTriggers(t *testing.T) { sm := NewStateManager(cg.ChainStore()) - inv := vm.NewInvoker() + inv := vm.NewActorRegistry() // predicting the address here... may break if other assumptions change taddr, err := address.NewIDAddress(1002) @@ -143,7 +143,7 @@ func TestForkHeightTriggers(t *testing.T) { return nil } - inv.Register(builtin.PaymentChannelActorCodeID, &testActor{}, &testActorState{}) + inv.Register(actors.Version0, builtin.PaymentChannelActorCodeID, &testActor{}, &testActorState{}, false) sm.SetVMConstructor(func(ctx context.Context, vmopt *vm.VMOpts) (*vm.VM, error) { nvm, err := vm.NewVM(ctx, vmopt) if err != nil { diff --git a/chain/vm/gas.go b/chain/vm/gas.go index 12acf6a21..6802013e5 100644 --- a/chain/vm/gas.go +++ b/chain/vm/gas.go @@ -9,7 +9,6 @@ import ( addr "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/crypto" - "github.com/filecoin-project/specs-actors/actors/runtime" vmr "github.com/filecoin-project/specs-actors/actors/runtime" "github.com/ipfs/go-cid" ) @@ -210,7 +209,7 @@ func (ps pricedSyscalls) VerifyPoSt(vi proof.WindowPoStVerifyInfo) error { // the "parent grinding fault", in which case it must be the sibling of h1 (same parent tipset) and one of the // blocks in the parent of h2 (i.e. h2's grandparent). // Returns nil and an error if the headers don't prove a fault. -func (ps pricedSyscalls) VerifyConsensusFault(h1 []byte, h2 []byte, extra []byte) (*runtime.ConsensusFault, error) { +func (ps pricedSyscalls) VerifyConsensusFault(h1 []byte, h2 []byte, extra []byte) (*vmr.ConsensusFault, error) { ps.chargeGas(ps.pl.OnVerifyConsensusFault()) defer ps.chargeGas(gasOnActorExec) diff --git a/chain/vm/gas_v0.go b/chain/vm/gas_v0.go index e5ded440e..bfb49c345 100644 --- a/chain/vm/gas_v0.go +++ b/chain/vm/gas_v0.go @@ -8,7 +8,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" - "github.com/filecoin-project/specs-actors/actors/builtin" + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" ) type scalingCost struct { @@ -112,14 +112,14 @@ func (pl *pricelistV0) OnMethodInvocation(value abi.TokenAmount, methodNum abi.M if big.Cmp(value, abi.NewTokenAmount(0)) != 0 { ret += pl.sendTransferFunds - if methodNum == builtin.MethodSend { + if methodNum == builtin0.MethodSend { // transfer only ret += pl.sendTransferOnlyPremium } extra += "t" } - if methodNum != builtin.MethodSend { + if methodNum != builtin0.MethodSend { extra += "i" // running actors is cheaper becase we hand over to actors ret += pl.sendInvokeMethod diff --git a/chain/vm/invoker.go b/chain/vm/invoker.go index 7465074b4..48a574f9d 100644 --- a/chain/vm/invoker.go +++ b/chain/vm/invoker.go @@ -40,79 +40,113 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/exitcode" + "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/aerrors" + "github.com/filecoin-project/lotus/chain/types" ) -type Invoker struct { - builtInCode map[cid.Cid]nativeCode - builtInState map[cid.Cid]reflect.Type +type ActorRegistry struct { + actors map[cid.Cid]*actorInfo } type invokeFunc func(rt vmr.Runtime, params []byte) ([]byte, aerrors.ActorError) type nativeCode []invokeFunc -func NewInvoker() *Invoker { - inv := &Invoker{ - builtInCode: make(map[cid.Cid]nativeCode), - builtInState: make(map[cid.Cid]reflect.Type), - } +type actorInfo struct { + methods nativeCode + stateType reflect.Type + // TODO: consider making this a network version range? + version actors.Version + singleton bool +} + +func NewActorRegistry() *ActorRegistry { + inv := &ActorRegistry{actors: make(map[cid.Cid]*actorInfo)} + + // TODO: define all these properties on the actors themselves, in specs-actors. // add builtInCode using: register(cid, singleton) - inv.Register(builtin0.SystemActorCodeID, system0.Actor{}, abi.EmptyValue{}) - inv.Register(builtin0.InitActorCodeID, init0.Actor{}, init0.State{}) - inv.Register(builtin0.RewardActorCodeID, reward0.Actor{}, reward0.State{}) - inv.Register(builtin0.CronActorCodeID, cron0.Actor{}, cron0.State{}) - inv.Register(builtin0.StoragePowerActorCodeID, power0.Actor{}, power0.State{}) - inv.Register(builtin0.StorageMarketActorCodeID, market0.Actor{}, market0.State{}) - inv.Register(builtin0.StorageMinerActorCodeID, miner0.Actor{}, miner0.State{}) - inv.Register(builtin0.MultisigActorCodeID, msig0.Actor{}, msig0.State{}) - inv.Register(builtin0.PaymentChannelActorCodeID, paych0.Actor{}, paych0.State{}) - inv.Register(builtin0.VerifiedRegistryActorCodeID, verifreg0.Actor{}, verifreg0.State{}) - inv.Register(builtin0.AccountActorCodeID, account0.Actor{}, account0.State{}) + inv.Register(actors.Version0, builtin0.SystemActorCodeID, system0.Actor{}, abi.EmptyValue{}, true) + inv.Register(actors.Version0, builtin0.InitActorCodeID, init0.Actor{}, init0.State{}, true) + inv.Register(actors.Version0, builtin0.RewardActorCodeID, reward0.Actor{}, reward0.State{}, true) + inv.Register(actors.Version0, builtin0.CronActorCodeID, cron0.Actor{}, cron0.State{}, true) + inv.Register(actors.Version0, builtin0.StoragePowerActorCodeID, power0.Actor{}, power0.State{}, true) + inv.Register(actors.Version0, builtin0.StorageMarketActorCodeID, market0.Actor{}, market0.State{}, true) + inv.Register(actors.Version0, builtin0.VerifiedRegistryActorCodeID, verifreg0.Actor{}, verifreg0.State{}, true) + inv.Register(actors.Version0, builtin0.StorageMinerActorCodeID, miner0.Actor{}, miner0.State{}, false) + inv.Register(actors.Version0, builtin0.MultisigActorCodeID, msig0.Actor{}, msig0.State{}, false) + inv.Register(actors.Version0, builtin0.PaymentChannelActorCodeID, paych0.Actor{}, paych0.State{}, false) + inv.Register(actors.Version0, builtin0.AccountActorCodeID, account0.Actor{}, account0.State{}, false) - inv.Register(builtin1.SystemActorCodeID, system1.Actor{}, abi.EmptyValue{}) - inv.Register(builtin1.InitActorCodeID, init1.Actor{}, init1.State{}) - inv.Register(builtin1.RewardActorCodeID, reward1.Actor{}, reward1.State{}) - inv.Register(builtin1.CronActorCodeID, cron1.Actor{}, cron1.State{}) - inv.Register(builtin1.StoragePowerActorCodeID, power1.Actor{}, power1.State{}) - inv.Register(builtin1.StorageMarketActorCodeID, market1.Actor{}, market1.State{}) - inv.Register(builtin1.StorageMinerActorCodeID, miner1.Actor{}, miner1.State{}) - inv.Register(builtin1.MultisigActorCodeID, msig1.Actor{}, msig1.State{}) - inv.Register(builtin1.PaymentChannelActorCodeID, paych1.Actor{}, paych1.State{}) - inv.Register(builtin1.VerifiedRegistryActorCodeID, verifreg1.Actor{}, verifreg1.State{}) - inv.Register(builtin1.AccountActorCodeID, account1.Actor{}, account1.State{}) + inv.Register(actors.Version0, builtin1.SystemActorCodeID, system1.Actor{}, abi.EmptyValue{}, true) + inv.Register(actors.Version0, builtin1.InitActorCodeID, init1.Actor{}, init1.State{}, true) + inv.Register(actors.Version0, builtin1.RewardActorCodeID, reward1.Actor{}, reward1.State{}, true) + inv.Register(actors.Version0, builtin1.CronActorCodeID, cron1.Actor{}, cron1.State{}, true) + inv.Register(actors.Version0, builtin1.StoragePowerActorCodeID, power1.Actor{}, power1.State{}, true) + inv.Register(actors.Version0, builtin1.StorageMarketActorCodeID, market1.Actor{}, market1.State{}, true) + inv.Register(actors.Version0, builtin1.VerifiedRegistryActorCodeID, verifreg1.Actor{}, verifreg1.State{}, true) + inv.Register(actors.Version0, builtin1.StorageMinerActorCodeID, miner1.Actor{}, miner1.State{}, false) + inv.Register(actors.Version0, builtin1.MultisigActorCodeID, msig1.Actor{}, msig1.State{}, false) + inv.Register(actors.Version0, builtin1.PaymentChannelActorCodeID, paych1.Actor{}, paych1.State{}, false) + inv.Register(actors.Version0, builtin1.AccountActorCodeID, account1.Actor{}, account1.State{}, false) return inv } -func (inv *Invoker) Invoke(codeCid cid.Cid, rt vmr.Runtime, method abi.MethodNum, params []byte) ([]byte, aerrors.ActorError) { - - code, ok := inv.builtInCode[codeCid] +func (ar *ActorRegistry) Invoke(codeCid cid.Cid, rt vmr.Runtime, method abi.MethodNum, params []byte) ([]byte, aerrors.ActorError) { + act, ok := ar.actors[codeCid] if !ok { log.Errorf("no code for actor %s (Addr: %s)", codeCid, rt.Receiver()) return nil, aerrors.Newf(exitcode.SysErrorIllegalActor, "no code for actor %s(%d)(%s)", codeCid, method, hex.EncodeToString(params)) } - if method >= abi.MethodNum(len(code)) || code[method] == nil { + if method >= abi.MethodNum(len(act.methods)) || act.methods[method] == nil { return nil, aerrors.Newf(exitcode.SysErrInvalidMethod, "no method %d on actor", method) } - return code[method](rt, params) + if curVer := actors.VersionForNetwork(rt.NetworkVersion()); curVer != act.version { + return nil, aerrors.Newf(exitcode.SysErrInvalidMethod, "unsupported actors code version %d, expected %d", act.version, curVer) + } + return act.methods[method](rt, params) } -func (inv *Invoker) Register(c cid.Cid, instance Invokee, state interface{}) { - code, err := inv.transform(instance) +func (ar *ActorRegistry) Register(version actors.Version, c cid.Cid, instance Invokee, state interface{}, singleton bool) { + code, err := ar.transform(instance) if err != nil { panic(xerrors.Errorf("%s: %w", string(c.Hash()), err)) } - inv.builtInCode[c] = code - inv.builtInState[c] = reflect.TypeOf(state) + ar.actors[c] = &actorInfo{ + methods: code, + version: version, + stateType: reflect.TypeOf(state), + singleton: singleton, + } +} + +func (ar *ActorRegistry) Create(codeCid cid.Cid, rt vmr.Runtime) (*types.Actor, aerrors.ActorError) { + act, ok := ar.actors[codeCid] + if !ok { + return nil, aerrors.Newf(exitcode.SysErrorIllegalArgument, "Can only create built-in actors.") + } + if version := actors.VersionForNetwork(rt.NetworkVersion()); act.version != version { + return nil, aerrors.Newf(exitcode.SysErrorIllegalArgument, "Can only create version %d actors, attempted to create version %d actor", version, act.version) + } + + if act.singleton { + return nil, aerrors.Newf(exitcode.SysErrorIllegalArgument, "Can only have one instance of singleton actors.") + } + return &types.Actor{ + Code: codeCid, + Head: EmptyObjectCid, + Nonce: 0, + Balance: abi.NewTokenAmount(0), + }, nil } type Invokee interface { Exports() []interface{} } -func (*Invoker) transform(instance Invokee) (nativeCode, error) { +func (*ActorRegistry) transform(instance Invokee) (nativeCode, error) { itype := reflect.TypeOf(instance) exports := instance.Exports() runtimeType := reflect.TypeOf((*vmr.Runtime)(nil)).Elem() @@ -201,19 +235,19 @@ func DecodeParams(b []byte, out interface{}) error { return um.UnmarshalCBOR(bytes.NewReader(b)) } -func DumpActorState(code cid.Cid, b []byte) (interface{}, error) { - if code == builtin0.AccountActorCodeID { // Account code special case +func DumpActorState(act *types.Actor, b []byte) (interface{}, error) { + if act.IsAccountActor() { // Account code special case return nil, nil } - i := NewInvoker() // TODO: register builtins in init block + i := NewActorRegistry() // TODO: register builtins in init block - typ, ok := i.builtInState[code] + actInfo, ok := i.actors[act.Code] if !ok { - return nil, xerrors.Errorf("state type for actor %s not found", code) + return nil, xerrors.Errorf("state type for actor %s not found", act.Code) } - rv := reflect.New(typ) + rv := reflect.New(actInfo.stateType) um, ok := rv.Interface().(cbg.CBORUnmarshaler) if !ok { return nil, xerrors.New("state type does not implement CBORUnmarshaler") diff --git a/chain/vm/invoker_test.go b/chain/vm/invoker_test.go index 3744aa8d2..4005dd42f 100644 --- a/chain/vm/invoker_test.go +++ b/chain/vm/invoker_test.go @@ -77,7 +77,7 @@ func (basicContract) InvokeSomething10(rt runtime.Runtime, params *basicParams) } func TestInvokerBasic(t *testing.T) { - inv := Invoker{} + inv := ActorRegistry{} code, err := inv.transform(basicContract{}) assert.NoError(t, err) diff --git a/chain/vm/mkactor.go b/chain/vm/mkactor.go index 43d2f9431..26e8c3ba1 100644 --- a/chain/vm/mkactor.go +++ b/chain/vm/mkactor.go @@ -6,11 +6,13 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/exitcode" "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" + builtin1 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/actors/aerrors" "github.com/filecoin-project/lotus/chain/types" @@ -39,7 +41,7 @@ func TryCreateAccountActor(rt *Runtime, addr address.Address) (*types.Actor, aer return nil, aerrors.Escalate(err, "registering actor address") } - act, aerr := makeActor(addr) + act, aerr := makeActor(actors.VersionForNetwork(rt.NetworkVersion()), addr) if aerr != nil { return nil, aerr } @@ -54,7 +56,7 @@ func TryCreateAccountActor(rt *Runtime, addr address.Address) (*types.Actor, aer } // call constructor on account - _, aerr = rt.internalSend(builtin.SystemActorAddr, addrID, builtin.MethodsAccount.Constructor, big.Zero(), p) + _, aerr = rt.internalSend(builtin0.SystemActorAddr, addrID, builtin0.MethodsAccount.Constructor, big.Zero(), p) if aerr != nil { return nil, aerrors.Wrap(aerr, "failed to invoke account constructor") } @@ -66,12 +68,10 @@ func TryCreateAccountActor(rt *Runtime, addr address.Address) (*types.Actor, aer return act, nil } -func makeActor(addr address.Address) (*types.Actor, aerrors.ActorError) { +func makeActor(ver actors.Version, addr address.Address) (*types.Actor, aerrors.ActorError) { switch addr.Protocol() { - case address.BLS: - return NewBLSAccountActor(), nil - case address.SECP256K1: - return NewSecp256k1AccountActor(), nil + case address.BLS, address.SECP256K1: + return newAccountActor(ver), nil case address.ID: return nil, aerrors.Newf(exitcode.SysErrInvalidReceiver, "no actor with given ID: %s", addr) case address.Actor: @@ -81,19 +81,19 @@ func makeActor(addr address.Address) (*types.Actor, aerrors.ActorError) { } } -func NewBLSAccountActor() *types.Actor { +func newAccountActor(ver actors.Version) *types.Actor { + // TODO: ActorsUpgrade use a global actor registry? + var code cid.Cid + switch ver { + case actors.Version0: + code = builtin0.AccountActorCodeID + case actors.Version1: + code = builtin1.AccountActorCodeID + default: + panic("unsupported actors version") + } nact := &types.Actor{ - Code: builtin.AccountActorCodeID, - Balance: types.NewInt(0), - Head: EmptyObjectCid, - } - - return nact -} - -func NewSecp256k1AccountActor() *types.Actor { - nact := &types.Actor{ - Code: builtin.AccountActorCodeID, + Code: code, Balance: types.NewInt(0), Head: EmptyObjectCid, } diff --git a/chain/vm/runtime.go b/chain/vm/runtime.go index 156d57282..3dcd269fa 100644 --- a/chain/vm/runtime.go +++ b/chain/vm/runtime.go @@ -10,13 +10,11 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/cbor" "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/go-state-types/exitcode" "github.com/filecoin-project/go-state-types/network" rtt "github.com/filecoin-project/go-state-types/rt" - "github.com/filecoin-project/specs-actors/actors/builtin" rt0 "github.com/filecoin-project/specs-actors/actors/runtime" "github.com/ipfs/go-cid" ipldcbor "github.com/ipfs/go-ipld-cbor" @@ -216,12 +214,9 @@ func (rt *Runtime) NewActorAddress() address.Address { } func (rt *Runtime) CreateActor(codeID cid.Cid, address address.Address) { - if !builtin.IsBuiltinActor(codeID) { - rt.Abortf(exitcode.SysErrorIllegalArgument, "Can only create built-in actors.") - } - - if builtin.IsSingletonActor(codeID) { - rt.Abortf(exitcode.SysErrorIllegalArgument, "Can only have one instance of singleton actors.") + act, aerr := rt.vm.areg.Create(codeID, rt) + if aerr != nil { + rt.Abortf(aerr.RetCode(), aerr.Error()) } _, err := rt.state.GetActor(address) @@ -231,12 +226,7 @@ func (rt *Runtime) CreateActor(codeID cid.Cid, address address.Address) { rt.chargeGas(rt.Pricelist().OnCreateActor()) - err = rt.state.SetActor(address, &types.Actor{ - Code: codeID, - Head: EmptyObjectCid, - Nonce: 0, - Balance: big.Zero(), - }) + err = rt.state.SetActor(address, act) if err != nil { panic(aerrors.Fatalf("creating actor entry: %v", err)) } diff --git a/chain/vm/vm.go b/chain/vm/vm.go index 54ea47698..d92d16310 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -152,7 +152,7 @@ type VM struct { cst *cbor.BasicIpldStore buf *bufbstore.BufferedBS blockHeight abi.ChainEpoch - inv *Invoker + areg *ActorRegistry rand Rand circSupplyCalc CircSupplyCalculator ntwkVersion NtwkVersionGetter @@ -186,7 +186,7 @@ func NewVM(ctx context.Context, opts *VMOpts) (*VM, error) { cst: cst, buf: buf, blockHeight: opts.Epoch, - inv: NewInvoker(), + areg: NewActorRegistry(), rand: opts.Rand, // TODO: Probably should be a syscall circSupplyCalc: opts.CircSupplyCalc, ntwkVersion: opts.NtwkVersion, @@ -734,15 +734,15 @@ func (vm *VM) Invoke(act *types.Actor, rt *Runtime, method abi.MethodNum, params defer func() { rt.ctx = oldCtx }() - ret, err := vm.inv.Invoke(act.Code, rt, method, params) + ret, err := vm.areg.Invoke(act.Code, rt, method, params) if err != nil { return nil, err } return ret, nil } -func (vm *VM) SetInvoker(i *Invoker) { - vm.inv = i +func (vm *VM) SetInvoker(i *ActorRegistry) { + vm.areg = i } func (vm *VM) GetNtwkVersion(ctx context.Context, ce abi.ChainEpoch) network.Version { diff --git a/conformance/driver.go b/conformance/driver.go index f43a8739d..1a63aaff3 100644 --- a/conformance/driver.go +++ b/conformance/driver.go @@ -5,6 +5,7 @@ import ( "github.com/filecoin-project/go-state-types/crypto" + "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" @@ -140,11 +141,11 @@ func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, preroot cid.Cid, epoch return nil, cid.Undef, err } - invoker := vm.NewInvoker() + invoker := vm.NewActorRegistry() // register the chaos actor if required by the vector. if chaosOn, ok := d.selector["chaos_actor"]; ok && chaosOn == "true" { - invoker.Register(chaos.ChaosActorCodeCID, chaos.Actor{}, chaos.State{}) + invoker.Register(actors.Version0, chaos.ChaosActorCodeCID, chaos.Actor{}, chaos.State{}, true) } lvm.SetInvoker(invoker) diff --git a/node/impl/full/chain.go b/node/impl/full/chain.go index 9606a023a..ea715c66a 100644 --- a/node/impl/full/chain.go +++ b/node/impl/full/chain.go @@ -422,7 +422,7 @@ func resolveOnce(bs blockstore.Blockstore) func(ctx context.Context, ds ipld.Nod return nil, nil, xerrors.Errorf("getting actor head for @state: %w", err) } - m, err := vm.DumpActorState(act.Code, head.RawData()) + m, err := vm.DumpActorState(&act, head.RawData()) if err != nil { return nil, nil, err } diff --git a/node/impl/full/state.go b/node/impl/full/state.go index f8bf92a92..d2bf5cf25 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -411,7 +411,7 @@ func (a *StateAPI) StateReadState(ctx context.Context, actor address.Address, ts return nil, xerrors.Errorf("getting actor head: %w", err) } - oif, err := vm.DumpActorState(act.Code, blk.RawData()) + oif, err := vm.DumpActorState(act, blk.RawData()) if err != nil { return nil, xerrors.Errorf("dumping actor state (a:%s): %w", actor, err) } From 7d3bd146e663674e449737acc70eaf6feda8bbf8 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 25 Sep 2020 12:51:02 -0700 Subject: [PATCH 346/885] rebase fixup --- chain/state/statetree.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chain/state/statetree.go b/chain/state/statetree.go index acf67e977..179e6dd02 100644 --- a/chain/state/statetree.go +++ b/chain/state/statetree.go @@ -396,7 +396,7 @@ func (st *StateTree) ForEach(f func(address.Address, *types.Actor) error) error } // Version returns the version of the StateTree data structure in use. -func (st *StateTree) Version() builtin.Version { +func (st *StateTree) Version() actors.Version { return st.version } From b8d9d7894fad1662ff29e12ac947ad89d0dc6ca0 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 25 Sep 2020 12:52:16 -0700 Subject: [PATCH 347/885] fix vm version --- chain/vm/invoker.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/chain/vm/invoker.go b/chain/vm/invoker.go index 48a574f9d..e137e71e4 100644 --- a/chain/vm/invoker.go +++ b/chain/vm/invoker.go @@ -78,17 +78,17 @@ func NewActorRegistry() *ActorRegistry { inv.Register(actors.Version0, builtin0.PaymentChannelActorCodeID, paych0.Actor{}, paych0.State{}, false) inv.Register(actors.Version0, builtin0.AccountActorCodeID, account0.Actor{}, account0.State{}, false) - inv.Register(actors.Version0, builtin1.SystemActorCodeID, system1.Actor{}, abi.EmptyValue{}, true) - inv.Register(actors.Version0, builtin1.InitActorCodeID, init1.Actor{}, init1.State{}, true) - inv.Register(actors.Version0, builtin1.RewardActorCodeID, reward1.Actor{}, reward1.State{}, true) - inv.Register(actors.Version0, builtin1.CronActorCodeID, cron1.Actor{}, cron1.State{}, true) - inv.Register(actors.Version0, builtin1.StoragePowerActorCodeID, power1.Actor{}, power1.State{}, true) - inv.Register(actors.Version0, builtin1.StorageMarketActorCodeID, market1.Actor{}, market1.State{}, true) - inv.Register(actors.Version0, builtin1.VerifiedRegistryActorCodeID, verifreg1.Actor{}, verifreg1.State{}, true) - inv.Register(actors.Version0, builtin1.StorageMinerActorCodeID, miner1.Actor{}, miner1.State{}, false) - inv.Register(actors.Version0, builtin1.MultisigActorCodeID, msig1.Actor{}, msig1.State{}, false) - inv.Register(actors.Version0, builtin1.PaymentChannelActorCodeID, paych1.Actor{}, paych1.State{}, false) - inv.Register(actors.Version0, builtin1.AccountActorCodeID, account1.Actor{}, account1.State{}, false) + inv.Register(actors.Version1, builtin1.SystemActorCodeID, system1.Actor{}, abi.EmptyValue{}, true) + inv.Register(actors.Version1, builtin1.InitActorCodeID, init1.Actor{}, init1.State{}, true) + inv.Register(actors.Version1, builtin1.RewardActorCodeID, reward1.Actor{}, reward1.State{}, true) + inv.Register(actors.Version1, builtin1.CronActorCodeID, cron1.Actor{}, cron1.State{}, true) + inv.Register(actors.Version1, builtin1.StoragePowerActorCodeID, power1.Actor{}, power1.State{}, true) + inv.Register(actors.Version1, builtin1.StorageMarketActorCodeID, market1.Actor{}, market1.State{}, true) + inv.Register(actors.Version1, builtin1.VerifiedRegistryActorCodeID, verifreg1.Actor{}, verifreg1.State{}, true) + inv.Register(actors.Version1, builtin1.StorageMinerActorCodeID, miner1.Actor{}, miner1.State{}, false) + inv.Register(actors.Version1, builtin1.MultisigActorCodeID, msig1.Actor{}, msig1.State{}, false) + inv.Register(actors.Version1, builtin1.PaymentChannelActorCodeID, paych1.Actor{}, paych1.State{}, false) + inv.Register(actors.Version1, builtin1.AccountActorCodeID, account1.Actor{}, account1.State{}, false) return inv } From 271ceb968a3b8d0d494ee8c867f102e0c6e5c3f4 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 25 Sep 2020 12:55:10 -0700 Subject: [PATCH 348/885] add v1 support to the state tree --- chain/state/statetree.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/chain/state/statetree.go b/chain/state/statetree.go index 179e6dd02..7a2e70cdc 100644 --- a/chain/state/statetree.go +++ b/chain/state/statetree.go @@ -125,6 +125,12 @@ func NewStateTree(cst cbor.IpldStore, version actors.Version) (*StateTree, error switch version { case actors.Version0: // info is undefined + case actors.Version1: + var err error + info, err = cst.Put(context.TODO(), new(types.StateInfo)) + if err != nil { + return nil, err + } default: return nil, xerrors.Errorf("unsupported state tree version: %d", version) } @@ -159,7 +165,7 @@ func LoadStateTree(cst cbor.IpldStore, c cid.Cid) (*StateTree, error) { } switch root.Version { - case actors.Version0: + case actors.Version0, actors.Version1: // supported default: return nil, xerrors.Errorf("unsupported state tree version: %d", root.Version) From a13d10e42d70018652e9c4926bad9dd155514950 Mon Sep 17 00:00:00 2001 From: Travis Person Date: Fri, 25 Sep 2020 20:08:28 +0000 Subject: [PATCH 349/885] add logging to chain export --- chain/store/store.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/chain/store/store.go b/chain/store/store.go index fce8a650f..1dbf69547 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -15,9 +15,11 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/vm" "github.com/filecoin-project/lotus/journal" bstore "github.com/filecoin-project/lotus/lib/blockstore" @@ -1183,6 +1185,7 @@ func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, inclRecentRo } blocksToWalk := ts.Cids() + currentMinHeight := ts.Height() walkChain := func(blk cid.Cid) error { if !seen.Visit(blk) { @@ -1203,6 +1206,13 @@ func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, inclRecentRo return xerrors.Errorf("unmarshaling block header (cid=%s): %w", blk, err) } + if currentMinHeight > b.Height { + currentMinHeight = b.Height + if currentMinHeight%builtin.EpochsInDay == 0 { + log.Infow("export", "height", currentMinHeight) + } + } + var cids []cid.Cid if !skipOldMsgs || b.Height > ts.Height()-inclRecentRoots { mcids, err := recurseLinks(cs.bs, walked, b.Messages, []cid.Cid{b.Messages}) @@ -1251,6 +1261,9 @@ func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, inclRecentRo return nil } + log.Infow("export started") + exportStart := build.Clock.Now() + for len(blocksToWalk) > 0 { next := blocksToWalk[0] blocksToWalk = blocksToWalk[1:] @@ -1259,6 +1272,8 @@ func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, inclRecentRo } } + log.Infow("export finished", "duration", build.Clock.Now().Sub(exportStart).Seconds()) + return nil } From 4eec4a01410c19a5687fec620fca418551ce9803 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 25 Sep 2020 23:28:12 +0200 Subject: [PATCH 350/885] Move policy change to seal bench Signed-off-by: Jakub Sztandera --- cmd/lotus-bench/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/lotus-bench/main.go b/cmd/lotus-bench/main.go index 2516bfd26..e409dfe5a 100644 --- a/cmd/lotus-bench/main.go +++ b/cmd/lotus-bench/main.go @@ -76,8 +76,6 @@ func main() { log.Info("Starting lotus-bench") - policy.AddSupportedProofTypes(abi.RegisteredSealProof_StackedDrg2KiBV1) - app := &cli.App{ Name: "lotus-bench", Usage: "Benchmark performance of lotus on your hardware", @@ -147,6 +145,8 @@ var sealBenchCmd = &cli.Command{ }, }, Action: func(c *cli.Context) error { + policy.AddSupportedProofTypes(abi.RegisteredSealProof_StackedDrg2KiBV1) + if c.Bool("no-gpu") { err := os.Setenv("BELLMAN_NO_GPU", "1") if err != nil { From d8431ff611be8ae5d47a4b05504e10ffd48c1673 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Fri, 25 Sep 2020 17:33:25 -0400 Subject: [PATCH 351/885] Fix AddSupportedProofTypes --- chain/actors/policy/policy.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/chain/actors/policy/policy.go b/chain/actors/policy/policy.go index eec52e855..b8205177e 100644 --- a/chain/actors/policy/policy.go +++ b/chain/actors/policy/policy.go @@ -22,12 +22,10 @@ func SetSupportedProofTypes(types ...abi.RegisteredSealProof) { // AddSupportedProofTypes sets supported proof types, across all actor versions. // This should only be used for testing. func AddSupportedProofTypes(types ...abi.RegisteredSealProof) { - newTypes := make(map[abi.RegisteredSealProof]struct{}, len(types)) for _, t := range types { - newTypes[t] = struct{}{} + // Set for all miner versions. + miner0.SupportedProofTypes[t] = struct{}{} } - // Set for all miner versions. - miner0.SupportedProofTypes = newTypes } // SetPreCommitChallengeDelay sets the pre-commit challenge delay across all From a5f13a5b31cb555eca3d1e9a63fb69fb93ef5023 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 25 Sep 2020 14:36:36 -0700 Subject: [PATCH 352/885] Test supported proof types. --- chain/actors/policy/policy_test.go | 36 ++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 chain/actors/policy/policy_test.go diff --git a/chain/actors/policy/policy_test.go b/chain/actors/policy/policy_test.go new file mode 100644 index 000000000..be64362a2 --- /dev/null +++ b/chain/actors/policy/policy_test.go @@ -0,0 +1,36 @@ +package policy + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/filecoin-project/go-state-types/abi" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" +) + +func TestSupportedProofTypes(t *testing.T) { + var oldTypes []abi.RegisteredSealProof + for t := range miner0.SupportedProofTypes { + oldTypes = append(oldTypes, t) + } + t.Cleanup(func() { + SetSupportedProofTypes(oldTypes...) + }) + + SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg2KiBV1) + require.EqualValues(t, + miner0.SupportedProofTypes, + map[abi.RegisteredSealProof]struct{}{ + abi.RegisteredSealProof_StackedDrg2KiBV1: {}, + }, + ) + AddSupportedProofTypes(abi.RegisteredSealProof_StackedDrg8MiBV1) + require.EqualValues(t, + miner0.SupportedProofTypes, + map[abi.RegisteredSealProof]struct{}{ + abi.RegisteredSealProof_StackedDrg2KiBV1: {}, + abi.RegisteredSealProof_StackedDrg8MiBV1: {}, + }, + ) +} From 8545c08f30bdb25add71604f5f6b5f12ed862966 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Fri, 25 Sep 2020 23:47:59 -0700 Subject: [PATCH 353/885] add json output to state compute --- cli/state.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/cli/state.go b/cli/state.go index d96c93c54..3d37e6fb7 100644 --- a/cli/state.go +++ b/cli/state.go @@ -818,6 +818,10 @@ var stateComputeStateCmd = &cli.Command{ Name: "html", Usage: "generate html report", }, + &cli.BoolFlag{ + Name: "json", + Usage: "generate json output", + }, }, Action: func(cctx *cli.Context) error { api, closer, err := GetFullNodeAPI(cctx) @@ -862,6 +866,15 @@ var stateComputeStateCmd = &cli.Command{ return err } + if cctx.Bool("json") { + out, err := json.Marshal(stout) + if err != nil { + return err + } + fmt.Println(string(out)) + return nil + } + if cctx.Bool("html") { codeCache := map[address.Address]cid.Cid{} getCode := func(addr address.Address) (cid.Cid, error) { From ef28ebb14a9e92833d6eb9cfab5f6f59279e516a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 24 Sep 2020 23:30:11 +0200 Subject: [PATCH 354/885] Ignition upgrades, much excite. --- build/params_2k.go | 2 + build/params_shared_vals.go | 6 +- build/params_testnet.go | 8 + chain/actors/builtin/builtin.go | 2 +- chain/stmgr/forks.go | 377 +++++++++++++++++++++++++++++--- chain/stmgr/forks_test.go | 18 +- chain/stmgr/stmgr.go | 194 +++++++++++++--- chain/stmgr/utils.go | 20 +- cli/state.go | 24 +- cmd/lotus-shed/balances.go | 138 ++++++++++++ go.mod | 2 +- go.sum | 4 +- node/modules/chain.go | 5 + node/modules/storageminer.go | 3 + storage/wdpost_run.go | 4 + 15 files changed, 708 insertions(+), 99 deletions(-) diff --git a/build/params_2k.go b/build/params_2k.go index 3edd0fb82..3682f7be1 100644 --- a/build/params_2k.go +++ b/build/params_2k.go @@ -12,6 +12,8 @@ const UpgradeBreezeHeight = -1 const BreezeGasTampingDuration = 0 const UpgradeSmokeHeight = -1 +const UpgradeIgnitionHeight = -2 +const UpgradeLiftoffHeight = -3 var DrandSchedule = map[abi.ChainEpoch]DrandEnum{ 0: DrandMainnet, diff --git a/build/params_shared_vals.go b/build/params_shared_vals.go index 3ee9f52ec..95c347281 100644 --- a/build/params_shared_vals.go +++ b/build/params_shared_vals.go @@ -22,8 +22,8 @@ const UnixfsLinksPerLevel = 1024 // Consensus / Network const AllowableClockDriftSecs = uint64(1) -const NewestNetworkVersion = network.Version2 -const ActorUpgradeNetworkVersion = network.Version3 +const NewestNetworkVersion = network.Version3 +const ActorUpgradeNetworkVersion = network.Version4 // Epochs const ForkLengthThreshold = Finality @@ -63,6 +63,8 @@ const WinningPoStSectorSetLookback = abi.ChainEpoch(10) // ///// // Devnet settings +var Devnet = true + const FilBase = uint64(2_000_000_000) const FilAllocStorageMining = uint64(1_100_000_000) diff --git a/build/params_testnet.go b/build/params_testnet.go index 13d2ff62e..960f3a9b6 100644 --- a/build/params_testnet.go +++ b/build/params_testnet.go @@ -21,12 +21,20 @@ const BreezeGasTampingDuration = 120 const UpgradeSmokeHeight = 51000 +const UpgradeIgnitionHeight = 94000 + +// This signals our tentative epoch for mainnet launch. Can make it later, but not earlier. +// Miners, clients, developers, custodians all need time to prepare. +// We still have upgrades and state changes to do, but can happen after signaling timing here. +const UpgradeLiftoffHeight = 148888 + func init() { policy.SetConsensusMinerMinPower(abi.NewStoragePower(10 << 40)) policy.SetSupportedProofTypes( abi.RegisteredSealProof_StackedDrg32GiBV1, abi.RegisteredSealProof_StackedDrg64GiBV1, ) + Devnet = false } const BlockDelaySecs = uint64(builtin0.EpochDurationSeconds) diff --git a/chain/actors/builtin/builtin.go b/chain/actors/builtin/builtin.go index 4079e694a..a85b4da65 100644 --- a/chain/actors/builtin/builtin.go +++ b/chain/actors/builtin/builtin.go @@ -21,7 +21,7 @@ const ( // Converts a network version into a specs-actors version. func VersionForNetwork(version network.Version) Version { switch version { - case network.Version0, network.Version1, network.Version2: + case network.Version0, network.Version1, network.Version2, network.Version3: return Version0 default: panic(fmt.Sprintf("unsupported network version %d", version)) diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index f9418c4d8..872c70b1e 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -1,44 +1,63 @@ package stmgr import ( + "bytes" "context" + "encoding/binary" + "math" + + multisig0 "github.com/filecoin-project/specs-actors/actors/builtin/multisig" + + "github.com/filecoin-project/lotus/chain/actors/builtin/multisig" + + "github.com/filecoin-project/lotus/chain/state" + + "github.com/filecoin-project/specs-actors/actors/migration/nv3" + + "github.com/ipfs/go-cid" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" + init0 "github.com/filecoin-project/specs-actors/actors/builtin/init" miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" power0 "github.com/filecoin-project/specs-actors/actors/builtin/power" adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/vm" cbor "github.com/ipfs/go-ipld-cbor" "golang.org/x/xerrors" ) -var ForksAtHeight = map[abi.ChainEpoch]func(context.Context, *StateManager, types.StateTree, *types.TipSet) error{ - build.UpgradeBreezeHeight: UpgradeFaucetBurnRecovery, +var ForksAtHeight = map[abi.ChainEpoch]func(context.Context, *StateManager, ExecCallback, cid.Cid, *types.TipSet) (cid.Cid, error){ + build.UpgradeBreezeHeight: UpgradeFaucetBurnRecovery, + build.UpgradeIgnitionHeight: UpgradeIgnition, + build.UpgradeLiftoffHeight: UpgradeLiftoff, } -func (sm *StateManager) handleStateForks(ctx context.Context, st types.StateTree, height abi.ChainEpoch, ts *types.TipSet) (err error) { +func (sm *StateManager) handleStateForks(ctx context.Context, root cid.Cid, height abi.ChainEpoch, cb ExecCallback, ts *types.TipSet) (cid.Cid, error) { + retCid := root + var err error f, ok := ForksAtHeight[height] if ok { - err := f(ctx, sm, st, ts) + retCid, err = f(ctx, sm, cb, root, ts) if err != nil { - return err + return cid.Undef, err } } - return nil + return retCid, nil } type forEachTree interface { ForEach(func(address.Address, *types.Actor) error) error } -func doTransfer(tree types.StateTree, from, to address.Address, amt abi.TokenAmount) error { +func doTransfer(cb ExecCallback, tree types.StateTree, from, to address.Address, amt abi.TokenAmount) error { fromAct, err := tree.GetActor(from) if err != nil { return xerrors.Errorf("failed to get 'from' actor for transfer: %w", err) @@ -64,10 +83,43 @@ func doTransfer(tree types.StateTree, from, to address.Address, amt abi.TokenAmo return xerrors.Errorf("failed to persist to actor: %w", err) } + if cb != nil { + // record the transfer in execution traces + + fakeMsg := &types.Message{ + From: from, + To: to, + Value: amt, + Nonce: math.MaxUint64, + } + fakeRct := &types.MessageReceipt{ + ExitCode: 0, + Return: nil, + GasUsed: 0, + } + + if err := cb(fakeMsg.Cid(), fakeMsg, &vm.ApplyRet{ + MessageReceipt: *fakeRct, + ActorErr: nil, + ExecutionTrace: types.ExecutionTrace{ + Msg: fakeMsg, + MsgRct: fakeRct, + Error: "", + Duration: 0, + GasCharges: nil, + Subcalls: nil, + }, + Duration: 0, + GasCosts: vm.ZeroGasOutputs(), + }); err != nil { + return xerrors.Errorf("recording transfer: %w", err) + } + } + return nil } -func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, tree types.StateTree, ts *types.TipSet) error { +func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, ts *types.TipSet) (cid.Cid, error) { // Some initial parameters FundsForMiners := types.FromFil(1_000_000) LookbackEpoch := abi.ChainEpoch(32000) @@ -94,22 +146,22 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, tree types // Grab lookback state for account checks lbts, err := sm.ChainStore().GetTipsetByHeight(ctx, LookbackEpoch, ts, false) if err != nil { - return xerrors.Errorf("failed to get tipset at lookback height: %w", err) + return cid.Undef, xerrors.Errorf("failed to get tipset at lookback height: %w", err) } lbtree, err := sm.ParentState(lbts) if err != nil { - return xerrors.Errorf("loading state tree failed: %w", err) + return cid.Undef, xerrors.Errorf("loading state tree failed: %w", err) } ReserveAddress, err := address.NewFromString("t090") if err != nil { - return xerrors.Errorf("failed to parse reserve address: %w", err) + return cid.Undef, xerrors.Errorf("failed to parse reserve address: %w", err) } - fetree, ok := tree.(forEachTree) - if !ok { - return xerrors.Errorf("fork transition state tree doesnt support ForEach (%T)", tree) + tree, err := sm.StateTree(root) + if err != nil { + return cid.Undef, xerrors.Errorf("getting state tree: %w", err) } type transfer struct { @@ -121,7 +173,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, tree types var transfers []transfer // Take all excess funds away, put them into the reserve account - err = fetree.ForEach(func(addr address.Address, act *types.Actor) error { + err = tree.ForEach(func(addr address.Address, act *types.Actor) error { switch act.Code { case builtin0.AccountActorCodeID, builtin0.MultisigActorCodeID, builtin0.PaymentChannelActorCodeID: sysAcc, err := isSystemAccount(addr) @@ -163,13 +215,13 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, tree types return nil }) if err != nil { - return xerrors.Errorf("foreach over state tree failed: %w", err) + return cid.Undef, xerrors.Errorf("foreach over state tree failed: %w", err) } // Execute transfers from previous step for _, t := range transfers { - if err := doTransfer(tree, t.From, t.To, t.Amt); err != nil { - return xerrors.Errorf("transfer %s %s->%s failed: %w", t.Amt, t.From, t.To, err) + if err := doTransfer(cb, tree, t.From, t.To, t.Amt); err != nil { + return cid.Undef, xerrors.Errorf("transfer %s %s->%s failed: %w", t.Amt, t.From, t.To, err) } } @@ -177,19 +229,19 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, tree types var ps power0.State powAct, err := tree.GetActor(builtin0.StoragePowerActorAddr) if err != nil { - return xerrors.Errorf("failed to load power actor: %w", err) + return cid.Undef, xerrors.Errorf("failed to load power actor: %w", err) } cst := cbor.NewCborStore(sm.ChainStore().Blockstore()) if err := cst.Get(ctx, powAct.Head, &ps); err != nil { - return xerrors.Errorf("failed to get power actor state: %w", err) + return cid.Undef, xerrors.Errorf("failed to get power actor state: %w", err) } totalPower := ps.TotalBytesCommitted var transfersBack []transfer // Now, we return some funds to places where they are needed - err = fetree.ForEach(func(addr address.Address, act *types.Actor) error { + err = tree.ForEach(func(addr address.Address, act *types.Actor) error { lbact, err := lbtree.GetActor(addr) if err != nil { if !xerrors.Is(err, types.ErrActorNotFound) { @@ -267,53 +319,310 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, tree types return nil }) if err != nil { - return xerrors.Errorf("foreach over state tree failed: %w", err) + return cid.Undef, xerrors.Errorf("foreach over state tree failed: %w", err) } for _, t := range transfersBack { - if err := doTransfer(tree, t.From, t.To, t.Amt); err != nil { - return xerrors.Errorf("transfer %s %s->%s failed: %w", t.Amt, t.From, t.To, err) + if err := doTransfer(cb, tree, t.From, t.To, t.Amt); err != nil { + return cid.Undef, xerrors.Errorf("transfer %s %s->%s failed: %w", t.Amt, t.From, t.To, err) } } // transfer all burnt funds back to the reserve account burntAct, err := tree.GetActor(builtin0.BurntFundsActorAddr) if err != nil { - return xerrors.Errorf("failed to load burnt funds actor: %w", err) + return cid.Undef, xerrors.Errorf("failed to load burnt funds actor: %w", err) } - if err := doTransfer(tree, builtin0.BurntFundsActorAddr, ReserveAddress, burntAct.Balance); err != nil { - return xerrors.Errorf("failed to unburn funds: %w", err) + if err := doTransfer(cb, tree, builtin0.BurntFundsActorAddr, ReserveAddress, burntAct.Balance); err != nil { + return cid.Undef, xerrors.Errorf("failed to unburn funds: %w", err) } // Top up the reimbursement service reimbAddr, err := address.NewFromString("t0111") if err != nil { - return xerrors.Errorf("failed to parse reimbursement service address") + return cid.Undef, xerrors.Errorf("failed to parse reimbursement service address") } reimb, err := tree.GetActor(reimbAddr) if err != nil { - return xerrors.Errorf("failed to load reimbursement account actor: %w", err) + return cid.Undef, xerrors.Errorf("failed to load reimbursement account actor: %w", err) } difference := types.BigSub(DesiredReimbursementBalance, reimb.Balance) - if err := doTransfer(tree, ReserveAddress, reimbAddr, difference); err != nil { - return xerrors.Errorf("failed to top up reimbursement account: %w", err) + if err := doTransfer(cb, tree, ReserveAddress, reimbAddr, difference); err != nil { + return cid.Undef, xerrors.Errorf("failed to top up reimbursement account: %w", err) } // Now, a final sanity check to make sure the balances all check out total := abi.NewTokenAmount(0) - err = fetree.ForEach(func(addr address.Address, act *types.Actor) error { + err = tree.ForEach(func(addr address.Address, act *types.Actor) error { total = types.BigAdd(total, act.Balance) return nil }) if err != nil { - return xerrors.Errorf("checking final state balance failed: %w", err) + return cid.Undef, xerrors.Errorf("checking final state balance failed: %w", err) } exp := types.FromFil(build.FilBase) if !exp.Equals(total) { - return xerrors.Errorf("resultant state tree account balance was not correct: %s", total) + return cid.Undef, xerrors.Errorf("resultant state tree account balance was not correct: %s", total) + } + + return tree.Flush(ctx) +} + +func UpgradeIgnition(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, ts *types.TipSet) (cid.Cid, error) { + store := sm.cs.Store(ctx) + + nst, err := nv3.MigrateStateTree(ctx, store, root, build.UpgradeIgnitionHeight) + if err != nil { + return cid.Undef, xerrors.Errorf("migrating actors state: %w", err) + } + + tree, err := sm.StateTree(nst) + if err != nil { + return cid.Undef, xerrors.Errorf("getting state tree: %w", err) + } + + err = setNetworkName(ctx, store, tree, "ignition") + if err != nil { + return cid.Undef, xerrors.Errorf("setting network name: %w", err) + } + + split1, err := address.NewFromString("t0115") + if err != nil { + return cid.Undef, xerrors.Errorf("first split address: %w", err) + } + + split2, err := address.NewFromString("t0116") + if err != nil { + return cid.Undef, xerrors.Errorf("second split address: %w", err) + } + + err = resetGenesisMsigs(ctx, sm, store, tree) + if err != nil { + return cid.Undef, xerrors.Errorf("resetting genesis msig start epochs: %w", err) + } + + err = splitGenesisMultisig(ctx, cb, split1, store, tree, 50) + if err != nil { + return cid.Undef, xerrors.Errorf("splitting first msig: %w", err) + } + + err = splitGenesisMultisig(ctx, cb, split2, store, tree, 50) + if err != nil { + return cid.Undef, xerrors.Errorf("splitting second msig: %w", err) + } + + err = nv3.CheckStateTree(ctx, store, nst, build.UpgradeIgnitionHeight, builtin0.TotalFilecoin) + if err != nil { + return cid.Undef, xerrors.Errorf("sanity check after ignition upgrade failed: %w", err) + } + + return tree.Flush(ctx) +} + +func UpgradeLiftoff(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, ts *types.TipSet) (cid.Cid, error) { + tree, err := sm.StateTree(root) + if err != nil { + return cid.Undef, xerrors.Errorf("getting state tree: %w", err) + } + + err = setNetworkName(ctx, sm.cs.Store(ctx), tree, "mainnet") + if err != nil { + return cid.Undef, xerrors.Errorf("setting network name: %w", err) + } + + return tree.Flush(ctx) +} + +func setNetworkName(ctx context.Context, store adt0.Store, tree *state.StateTree, name string) error { + ia, err := tree.GetActor(builtin0.InitActorAddr) + if err != nil { + return xerrors.Errorf("getting init actor: %w", err) + } + + var initState init0.State + if err := store.Get(ctx, ia.Head, &initState); err != nil { + return xerrors.Errorf("reading init state: %w", err) + } + + initState.NetworkName = name + + ia.Head, err = store.Put(ctx, &initState) + if err != nil { + return xerrors.Errorf("writing new init state: %w", err) + } + + if err := tree.SetActor(builtin0.InitActorAddr, ia); err != nil { + return xerrors.Errorf("setting init actor: %w", err) + } + + return nil +} + +func splitGenesisMultisig(ctx context.Context, cb ExecCallback, addr address.Address, store adt0.Store, tree *state.StateTree, portions uint64) error { + if portions < 1 { + return xerrors.Errorf("cannot split into 0 portions") + } + + mact, err := tree.GetActor(addr) + if err != nil { + return xerrors.Errorf("getting msig actor: %w", err) + } + + mst, err := multisig.Load(store, mact) + if err != nil { + return xerrors.Errorf("getting msig state: %w", err) + } + + signers, err := mst.Signers() + if err != nil { + return xerrors.Errorf("getting msig signers: %w", err) + } + + thresh, err := mst.Threshold() + if err != nil { + return xerrors.Errorf("getting msig threshold: %w", err) + } + + ibal, err := mst.InitialBalance() + if err != nil { + return xerrors.Errorf("getting msig initial balance: %w", err) + } + + se, err := mst.StartEpoch() + if err != nil { + return xerrors.Errorf("getting msig start epoch: %w", err) + } + + ud, err := mst.UnlockDuration() + if err != nil { + return xerrors.Errorf("getting msig unlock duration: %w", err) + } + + pending, err := adt0.MakeEmptyMap(store).Root() + if err != nil { + return xerrors.Errorf("failed to create empty map: %w", err) + } + + newIbal := big.Div(ibal, types.NewInt(portions)) + newState := &multisig0.State{ + Signers: signers, + NumApprovalsThreshold: thresh, + NextTxnID: 0, + InitialBalance: newIbal, + StartEpoch: se, + UnlockDuration: ud, + PendingTxns: pending, + } + + scid, err := store.Put(ctx, newState) + if err != nil { + return xerrors.Errorf("storing new state: %w", err) + } + + newActor := types.Actor{ + Code: builtin0.MultisigActorCodeID, + Head: scid, + Nonce: 0, + Balance: big.Zero(), + } + + i := uint64(0) + for i < portions { + keyAddr, err := makeKeyAddr(addr, i) + if err != nil { + return xerrors.Errorf("creating key address: %w", err) + } + + idAddr, err := tree.RegisterNewAddress(keyAddr) + if err != nil { + return xerrors.Errorf("registering new address: %w", err) + } + + err = tree.SetActor(idAddr, &newActor) + if err != nil { + return xerrors.Errorf("setting new msig actor state: %w", err) + } + + if err := doTransfer(cb, tree, addr, idAddr, newIbal); err != nil { + return xerrors.Errorf("transferring split msig balance: %w", err) + } + + i++ + } + + return nil +} + +func makeKeyAddr(splitAddr address.Address, count uint64) (address.Address, error) { + var b bytes.Buffer + if err := splitAddr.MarshalCBOR(&b); err != nil { + return address.Undef, xerrors.Errorf("marshalling split address: %w", err) + } + + if err := binary.Write(&b, binary.BigEndian, count); err != nil { + return address.Undef, xerrors.Errorf("writing count into a buffer: %w", err) + } + + if err := binary.Write(&b, binary.BigEndian, []byte("Ignition upgrade")); err != nil { + return address.Undef, xerrors.Errorf("writing fork name into a buffer: %w", err) + } + + addr, err := address.NewActorAddress(b.Bytes()) + if err != nil { + return address.Undef, xerrors.Errorf("create actor address: %w", err) + } + + return addr, nil +} + +func resetGenesisMsigs(ctx context.Context, sm *StateManager, store adt0.Store, tree *state.StateTree) error { + gb, err := sm.cs.GetGenesis() + if err != nil { + return xerrors.Errorf("getting genesis block: %w", err) + } + + gts, err := types.NewTipSet([]*types.BlockHeader{gb}) + if err != nil { + return xerrors.Errorf("getting genesis tipset: %w", err) + } + + cst := cbor.NewCborStore(sm.cs.Blockstore()) + genesisTree, err := state.LoadStateTree(cst, gts.ParentState()) + if err != nil { + return xerrors.Errorf("loading state tree: %w", err) + } + + err = genesisTree.ForEach(func(addr address.Address, genesisActor *types.Actor) error { + if genesisActor.Code == builtin0.MultisigActorCodeID { + currActor, err := tree.GetActor(addr) + if err != nil { + return xerrors.Errorf("loading actor: %w", err) + } + + var currState multisig0.State + if err := store.Get(ctx, currActor.Head, &currState); err != nil { + return xerrors.Errorf("reading multisig state: %w", err) + } + + currState.StartEpoch = build.UpgradeLiftoffHeight + + currActor.Head, err = store.Put(ctx, &currState) + if err != nil { + return xerrors.Errorf("writing new multisig state: %w", err) + } + + if err := tree.SetActor(addr, currActor); err != nil { + return xerrors.Errorf("setting multisig actor: %w", err) + } + } + return nil + }) + + if err != nil { + return xerrors.Errorf("iterating over genesis actors: %w", err) } return nil diff --git a/chain/stmgr/forks_test.go b/chain/stmgr/forks_test.go index 516058a81..a3423ccdd 100644 --- a/chain/stmgr/forks_test.go +++ b/chain/stmgr/forks_test.go @@ -25,6 +25,7 @@ import ( _ "github.com/filecoin-project/lotus/lib/sigs/bls" _ "github.com/filecoin-project/lotus/lib/sigs/secp" + "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" logging "github.com/ipfs/go-log" cbg "github.com/whyrusleeping/cbor-gen" @@ -114,33 +115,38 @@ func TestForkHeightTriggers(t *testing.T) { t.Fatal(err) } - stmgr.ForksAtHeight[testForkHeight] = func(ctx context.Context, sm *StateManager, st types.StateTree, ts *types.TipSet) error { + stmgr.ForksAtHeight[testForkHeight] = func(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, ts *types.TipSet) (cid.Cid, error) { cst := cbor.NewCborStore(sm.ChainStore().Blockstore()) + st, err := sm.StateTree(root) + if err != nil { + return cid.Undef, xerrors.Errorf("getting state tree: %w", err) + } + act, err := st.GetActor(taddr) if err != nil { - return err + return cid.Undef, err } var tas testActorState if err := cst.Get(ctx, act.Head, &tas); err != nil { - return xerrors.Errorf("in fork handler, failed to run get: %w", err) + return cid.Undef, xerrors.Errorf("in fork handler, failed to run get: %w", err) } tas.HasUpgraded = 55 ns, err := cst.Put(ctx, &tas) if err != nil { - return err + return cid.Undef, err } act.Head = ns if err := st.SetActor(taddr, act); err != nil { - return err + return cid.Undef, err } - return nil + return st.Flush(ctx) } inv.Register(builtin.PaymentChannelActorCodeID, &testActor{}, &testActorState{}) diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index eaf9215db..e800ce665 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -41,12 +41,13 @@ var log = logging.Logger("statemgr") type StateManager struct { cs *store.ChainStore - stCache map[string][]cid.Cid - compWait map[string]chan struct{} - stlk sync.Mutex - genesisMsigLk sync.Mutex - newVM func(context.Context, *vm.VMOpts) (*vm.VM, error) - genInfo *genesisInfo + stCache map[string][]cid.Cid + compWait map[string]chan struct{} + stlk sync.Mutex + genesisMsigLk sync.Mutex + newVM func(context.Context, *vm.VMOpts) (*vm.VM, error) + preIgnitionGenInfos *genesisInfo + postIgnitionGenInfos *genesisInfo } func NewStateManager(cs *store.ChainStore) *StateManager { @@ -123,9 +124,8 @@ func (sm *StateManager) TipSetState(ctx context.Context, ts *types.TipSet) (st c return st, rec, nil } -func (sm *StateManager) ExecutionTrace(ctx context.Context, ts *types.TipSet) (cid.Cid, []*api.InvocResult, error) { - var trace []*api.InvocResult - st, _, err := sm.computeTipSetState(ctx, ts, func(mcid cid.Cid, msg *types.Message, ret *vm.ApplyRet) error { +func traceFunc(trace *[]*api.InvocResult) func(mcid cid.Cid, msg *types.Message, ret *vm.ApplyRet) error { + return func(mcid cid.Cid, msg *types.Message, ret *vm.ApplyRet) error { ir := &api.InvocResult{ Msg: msg, MsgRct: &ret.MessageReceipt, @@ -135,9 +135,14 @@ func (sm *StateManager) ExecutionTrace(ctx context.Context, ts *types.TipSet) (c if ret.ActorErr != nil { ir.Error = ret.ActorErr.Error() } - trace = append(trace, ir) + *trace = append(*trace, ir) return nil - }) + } +} + +func (sm *StateManager) ExecutionTrace(ctx context.Context, ts *types.TipSet) (cid.Cid, []*api.InvocResult, error) { + var trace []*api.InvocResult + st, _, err := sm.computeTipSetState(ctx, ts, traceFunc(&trace)) if err != nil { return cid.Undef, nil, err } @@ -149,20 +154,24 @@ type ExecCallback func(cid.Cid, *types.Message, *vm.ApplyRet) error func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEpoch, pstate cid.Cid, bms []store.BlockMessages, epoch abi.ChainEpoch, r vm.Rand, cb ExecCallback, baseFee abi.TokenAmount, ts *types.TipSet) (cid.Cid, cid.Cid, error) { - vmopt := &vm.VMOpts{ - StateBase: pstate, - Epoch: epoch, - Rand: r, - Bstore: sm.cs.Blockstore(), - Syscalls: sm.cs.VMSys(), - CircSupplyCalc: sm.GetCirculatingSupply, - NtwkVersion: sm.GetNtwkVersion, - BaseFee: baseFee, + makeVmWithBaseState := func(base cid.Cid) (*vm.VM, error) { + vmopt := &vm.VMOpts{ + StateBase: base, + Epoch: epoch, + Rand: r, + Bstore: sm.cs.Blockstore(), + Syscalls: sm.cs.VMSys(), + CircSupplyCalc: sm.GetCirculatingSupply, + NtwkVersion: sm.GetNtwkVersion, + BaseFee: baseFee, + } + + return sm.newVM(ctx, vmopt) } - vmi, err := sm.newVM(ctx, vmopt) + vmi, err := makeVmWithBaseState(pstate) if err != nil { - return cid.Undef, cid.Undef, xerrors.Errorf("instantiating VM failed: %w", err) + return cid.Undef, cid.Undef, xerrors.Errorf("making vm: %w", err) } runCron := func() error { @@ -202,19 +211,32 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp for i := parentEpoch; i < epoch; i++ { // handle state forks // XXX: The state tree - err = sm.handleStateForks(ctx, vmi.StateTree(), i, ts) + newState, err := sm.handleStateForks(ctx, pstate, i, cb, ts) if err != nil { return cid.Undef, cid.Undef, xerrors.Errorf("error handling state forks: %w", err) } + if pstate != newState { + vmi, err = makeVmWithBaseState(newState) + if err != nil { + return cid.Undef, cid.Undef, xerrors.Errorf("making vm: %w", err) + } + } + if i > parentEpoch { // run cron for null rounds if any if err := runCron(); err != nil { return cid.Cid{}, cid.Cid{}, err } + + newState, err = vmi.Flush(ctx) + if err != nil { + return cid.Undef, cid.Undef, xerrors.Errorf("flushing vm: %w", err) + } } vmi.SetBlockHeight(i + 1) + pstate = newState } var receipts []cbg.CBORMarshaler @@ -904,7 +926,7 @@ func (sm *StateManager) setupGenesisActors(ctx context.Context) error { gi.genesisMsigs = append(gi.genesisMsigs, ns) } - sm.genInfo = &gi + sm.preIgnitionGenInfos = &gi return nil } @@ -912,7 +934,7 @@ func (sm *StateManager) setupGenesisActors(ctx context.Context) error { // sets up information about the actors in the genesis state // For testnet we use a hardcoded set of multisig states, instead of what's actually in the genesis multisigs // We also do not consider ANY account actors (including the faucet) -func (sm *StateManager) setupGenesisActorsTestnet(ctx context.Context) error { +func (sm *StateManager) setupPreIgnitionGenesisActorsTestnet(ctx context.Context) error { gi := genesisInfo{} @@ -981,7 +1003,87 @@ func (sm *StateManager) setupGenesisActorsTestnet(ctx context.Context) error { gi.genesisMsigs = append(gi.genesisMsigs, ns) } - sm.genInfo = &gi + sm.preIgnitionGenInfos = &gi + + return nil +} + +// sets up information about the actors in the genesis state, post the ignition fork +func (sm *StateManager) setupPostIgnitionGenesisActors(ctx context.Context) error { + + gi := genesisInfo{} + + gb, err := sm.cs.GetGenesis() + if err != nil { + return xerrors.Errorf("getting genesis block: %w", err) + } + + gts, err := types.NewTipSet([]*types.BlockHeader{gb}) + if err != nil { + return xerrors.Errorf("getting genesis tipset: %w", err) + } + + st, _, err := sm.TipSetState(ctx, gts) + if err != nil { + return xerrors.Errorf("getting genesis tipset state: %w", err) + } + + cst := cbor.NewCborStore(sm.cs.Blockstore()) + sTree, err := state.LoadStateTree(cst, st) + if err != nil { + return xerrors.Errorf("loading state tree: %w", err) + } + + // Unnecessary, should be removed + gi.genesisMarketFunds, err = getFilMarketLocked(ctx, sTree) + if err != nil { + return xerrors.Errorf("setting up genesis market funds: %w", err) + } + + // Unnecessary, should be removed + gi.genesisPledge, err = getFilPowerLocked(ctx, sTree) + if err != nil { + return xerrors.Errorf("setting up genesis pledge: %w", err) + } + + totalsByEpoch := make(map[abi.ChainEpoch]abi.TokenAmount) + + // 6 months + sixMonths := abi.ChainEpoch(183 * builtin0.EpochsInDay) + totalsByEpoch[sixMonths] = big.NewInt(49_929_341) + totalsByEpoch[sixMonths] = big.Add(totalsByEpoch[sixMonths], big.NewInt(32_787_700)) + + // 1 year + oneYear := abi.ChainEpoch(365 * builtin0.EpochsInDay) + totalsByEpoch[oneYear] = big.NewInt(22_421_712) + + // 2 years + twoYears := abi.ChainEpoch(2 * 365 * builtin0.EpochsInDay) + totalsByEpoch[twoYears] = big.NewInt(7_223_364) + + // 3 years + threeYears := abi.ChainEpoch(3 * 365 * builtin0.EpochsInDay) + totalsByEpoch[threeYears] = big.NewInt(87_637_883) + + // 6 years + sixYears := abi.ChainEpoch(6 * 365 * builtin0.EpochsInDay) + totalsByEpoch[sixYears] = big.NewInt(100_000_000) + totalsByEpoch[sixYears] = big.Add(totalsByEpoch[sixYears], big.NewInt(300_000_000)) + + gi.genesisMsigs = make([]msig0.State, 0, len(totalsByEpoch)) + for k, v := range totalsByEpoch { + ns := msig0.State{ + // In the pre-ignition logic, we incorrectly set this value in Fil, not attoFil, an off-by-10^18 error + InitialBalance: big.Mul(v, big.NewInt(int64(build.FilecoinPrecision))), + UnlockDuration: k, + PendingTxns: cid.Undef, + // In the pre-ignition logic, the start epoch was 0. This changes in the fork logic of the Ignition upgrade itself. + StartEpoch: build.UpgradeLiftoffHeight, + } + gi.genesisMsigs = append(gi.genesisMsigs, ns) + } + + sm.postIgnitionGenInfos = &gi return nil } @@ -991,13 +1093,23 @@ func (sm *StateManager) setupGenesisActorsTestnet(ctx context.Context) error { // - For Accounts, it counts max(currentBalance - genesisBalance, 0). func (sm *StateManager) GetFilVested(ctx context.Context, height abi.ChainEpoch, st *state.StateTree) (abi.TokenAmount, error) { vf := big.Zero() - for _, v := range sm.genInfo.genesisMsigs { - au := big.Sub(v.InitialBalance, v.AmountLocked(height)) - vf = big.Add(vf, au) + if height <= build.UpgradeIgnitionHeight { + for _, v := range sm.preIgnitionGenInfos.genesisMsigs { + au := big.Sub(v.InitialBalance, v.AmountLocked(height)) + vf = big.Add(vf, au) + } + } else { + for _, v := range sm.postIgnitionGenInfos.genesisMsigs { + // In the pre-ignition logic, we simply called AmountLocked(height), assuming startEpoch was 0. + // The start epoch changed in the Ignition upgrade. + au := big.Sub(v.InitialBalance, v.AmountLocked(height-v.StartEpoch)) + vf = big.Add(vf, au) + } } // there should not be any such accounts in testnet (and also none in mainnet?) - for _, v := range sm.genInfo.genesisActors { + // continue to use preIgnitionGenInfos, nothing changed at the Ignition epoch + for _, v := range sm.preIgnitionGenInfos.genesisActors { act, err := st.GetActor(v.addr) if err != nil { return big.Zero(), xerrors.Errorf("failed to get actor: %w", err) @@ -1009,8 +1121,10 @@ func (sm *StateManager) GetFilVested(ctx context.Context, height abi.ChainEpoch, } } - vf = big.Add(vf, sm.genInfo.genesisPledge) - vf = big.Add(vf, sm.genInfo.genesisMarketFunds) + // continue to use preIgnitionGenInfos, nothing changed at the Ignition epoch + vf = big.Add(vf, sm.preIgnitionGenInfos.genesisPledge) + // continue to use preIgnitionGenInfos, nothing changed at the Ignition epoch + vf = big.Add(vf, sm.preIgnitionGenInfos.genesisMarketFunds) return vf, nil } @@ -1084,10 +1198,16 @@ func GetFilBurnt(ctx context.Context, st *state.StateTree) (abi.TokenAmount, err func (sm *StateManager) GetCirculatingSupplyDetailed(ctx context.Context, height abi.ChainEpoch, st *state.StateTree) (api.CirculatingSupply, error) { sm.genesisMsigLk.Lock() defer sm.genesisMsigLk.Unlock() - if sm.genInfo == nil { - err := sm.setupGenesisActorsTestnet(ctx) + if sm.preIgnitionGenInfos == nil { + err := sm.setupPreIgnitionGenesisActorsTestnet(ctx) if err != nil { - return api.CirculatingSupply{}, xerrors.Errorf("failed to setup genesis information: %w", err) + return api.CirculatingSupply{}, xerrors.Errorf("failed to setup pre-ignition genesis information: %w", err) + } + } + if sm.postIgnitionGenInfos == nil { + err := sm.setupPostIgnitionGenesisActors(ctx) + if err != nil { + return api.CirculatingSupply{}, xerrors.Errorf("failed to setup post-ignition genesis information: %w", err) } } @@ -1152,6 +1272,10 @@ func (sm *StateManager) GetNtwkVersion(ctx context.Context, height abi.ChainEpoc return network.Version1 } + if height <= build.UpgradeIgnitionHeight { + return network.Version2 + } + return build.NewestNetworkVersion } diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index 3493afca3..58e7f480f 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -368,6 +368,16 @@ func ComputeState(ctx context.Context, sm *StateManager, height abi.ChainEpoch, return cid.Undef, nil, err } + for i := ts.Height(); i < height; i++ { + // handle state forks + base, err = sm.handleStateForks(ctx, base, i, traceFunc(&trace), ts) + if err != nil { + return cid.Undef, nil, xerrors.Errorf("error handling state forks: %w", err) + } + + // TODO: should we also run cron here? + } + r := store.NewChainRand(sm.cs, ts.Cids()) vmopt := &vm.VMOpts{ StateBase: base, @@ -384,16 +394,6 @@ func ComputeState(ctx context.Context, sm *StateManager, height abi.ChainEpoch, return cid.Undef, nil, err } - for i := ts.Height(); i < height; i++ { - // handle state forks - err = sm.handleStateForks(ctx, vmi.StateTree(), i, ts) - if err != nil { - return cid.Undef, nil, xerrors.Errorf("error handling state forks: %w", err) - } - - // TODO: should we also run cron here? - } - for i, msg := range msgs { // TODO: Use the signed message length for secp messages ret, err := vmi.ApplyMessage(ctx, msg) diff --git a/cli/state.go b/cli/state.go index d96c93c54..b25a5e4a7 100644 --- a/cli/state.go +++ b/cli/state.go @@ -19,6 +19,7 @@ import ( "github.com/multiformats/go-multiaddr" "github.com/ipfs/go-cid" + cbor "github.com/ipfs/go-ipld-cbor" "github.com/libp2p/go-libp2p-core/peer" "github.com/multiformats/go-multihash" "github.com/urfave/cli/v2" @@ -33,7 +34,9 @@ import ( "github.com/filecoin-project/specs-actors/actors/builtin/exported" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/api/apibstore" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/types" ) @@ -834,14 +837,14 @@ var stateComputeStateCmd = &cli.Command{ } h := abi.ChainEpoch(cctx.Uint64("vm-height")) - if h == 0 { - if ts == nil { - head, err := api.ChainHead(ctx) - if err != nil { - return err - } - ts = head + if ts == nil { + head, err := api.ChainHead(ctx) + if err != nil { + return err } + ts = head + } + if h == 0 { h = ts.Height() } @@ -863,13 +866,18 @@ var stateComputeStateCmd = &cli.Command{ } if cctx.Bool("html") { + st, err := state.LoadStateTree(cbor.NewCborStore(apibstore.NewAPIBlockstore(api)), stout.Root) + if err != nil { + return xerrors.Errorf("loading state tree: %w", err) + } + codeCache := map[address.Address]cid.Cid{} getCode := func(addr address.Address) (cid.Cid, error) { if c, found := codeCache[addr]; found { return c, nil } - c, err := api.StateGetActor(ctx, addr, ts.Key()) + c, err := st.GetActor(addr) if err != nil { return cid.Cid{}, err } diff --git a/cmd/lotus-shed/balances.go b/cmd/lotus-shed/balances.go index c156de931..1c89a00cf 100644 --- a/cmd/lotus-shed/balances.go +++ b/cmd/lotus-shed/balances.go @@ -3,9 +3,16 @@ package main import ( "context" "fmt" + "strconv" + + "github.com/docker/go-units" + lotusbuiltin "github.com/filecoin-project/lotus/chain/actors/builtin" + "github.com/filecoin-project/lotus/chain/actors/builtin/power" + "github.com/filecoin-project/lotus/chain/actors/builtin/reward" "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" + logging "github.com/ipfs/go-log/v2" "github.com/urfave/cli/v2" "golang.org/x/xerrors" @@ -44,6 +51,7 @@ var auditsCmd = &cli.Command{ Subcommands: []*cli.Command{ chainBalanceCmd, chainBalanceStateCmd, + chainPledgeCmd, }, } @@ -248,3 +256,133 @@ var chainBalanceStateCmd = &cli.Command{ return nil }, } + +var chainPledgeCmd = &cli.Command{ + Name: "stateroot-pledge", + Description: "Calculate sector pledge numbers", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "repo", + Value: "~/.lotus", + }, + }, + ArgsUsage: "[stateroot epoch]", + Action: func(cctx *cli.Context) error { + logging.SetLogLevel("badger", "ERROR") + ctx := context.TODO() + + if !cctx.Args().Present() { + return fmt.Errorf("must pass state root") + } + + sroot, err := cid.Decode(cctx.Args().First()) + if err != nil { + return fmt.Errorf("failed to parse input: %w", err) + } + + epoch, err := strconv.ParseInt(cctx.Args().Get(1), 10, 64) + if err != nil { + return xerrors.Errorf("parsing epoch arg: %w", err) + } + + fsrepo, err := repo.NewFS(cctx.String("repo")) + if err != nil { + return err + } + + lkrepo, err := fsrepo.Lock(repo.FullNode) + if err != nil { + return err + } + + defer lkrepo.Close() //nolint:errcheck + + ds, err := lkrepo.Datastore("/chain") + if err != nil { + return err + } + + mds, err := lkrepo.Datastore("/metadata") + if err != nil { + return err + } + + bs := blockstore.NewBlockstore(ds) + + cs := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier)) + + cst := cbor.NewCborStore(bs) + store := adt.WrapStore(ctx, cst) + + sm := stmgr.NewStateManager(cs) + + state, err := state.LoadStateTree(cst, sroot) + if err != nil { + return err + } + + var ( + powerSmoothed lotusbuiltin.FilterEstimate + pledgeCollateral abi.TokenAmount + ) + if act, err := state.GetActor(power.Address); err != nil { + return xerrors.Errorf("loading miner actor: %w", err) + } else if s, err := power.Load(store, act); err != nil { + return xerrors.Errorf("loading power actor state: %w", err) + } else if p, err := s.TotalPowerSmoothed(); err != nil { + return xerrors.Errorf("failed to determine total power: %w", err) + } else if c, err := s.TotalLocked(); err != nil { + return xerrors.Errorf("failed to determine pledge collateral: %w", err) + } else { + powerSmoothed = p + pledgeCollateral = c + } + + circ, err := sm.GetCirculatingSupplyDetailed(ctx, abi.ChainEpoch(epoch), state) + if err != nil { + return err + } + + fmt.Println("(real) circulating supply: ", types.FIL(circ.FilCirculating)) + if circ.FilCirculating.LessThan(big.Zero()) { + circ.FilCirculating = big.Zero() + } + + rewardActor, err := state.GetActor(reward.Address) + if err != nil { + return xerrors.Errorf("loading miner actor: %w", err) + } + + rewardState, err := reward.Load(store, rewardActor) + if err != nil { + return xerrors.Errorf("loading reward actor state: %w", err) + } + + fmt.Println("FilVested", types.FIL(circ.FilVested)) + fmt.Println("FilMined", types.FIL(circ.FilMined)) + fmt.Println("FilBurnt", types.FIL(circ.FilBurnt)) + fmt.Println("FilLocked", types.FIL(circ.FilLocked)) + fmt.Println("FilCirculating", types.FIL(circ.FilCirculating)) + + for _, sectorWeight := range []abi.StoragePower{ + types.NewInt(32 << 30), + types.NewInt(64 << 30), + types.NewInt(32 << 30 * 10), + types.NewInt(64 << 30 * 10), + } { + initialPledge, err := rewardState.InitialPledgeForPower( + sectorWeight, + pledgeCollateral, + &powerSmoothed, + circ.FilCirculating, + ) + if err != nil { + return xerrors.Errorf("calculating initial pledge: %w", err) + } + + fmt.Println("IP ", units.HumanSize(float64(sectorWeight.Uint64())), types.FIL(initialPledge)) + } + + return nil + }, +} diff --git a/go.mod b/go.mod index ca7e0760d..b0de7dfd6 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/filecoin-project/go-statemachine v0.0.0-20200813232949-df9b130df370 github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b - github.com/filecoin-project/specs-actors v0.9.10 + github.com/filecoin-project/specs-actors v0.9.11 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 github.com/filecoin-project/test-vectors/schema v0.0.1 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 diff --git a/go.sum b/go.sum index 1ca615b59..6412fe743 100644 --- a/go.sum +++ b/go.sum @@ -254,8 +254,8 @@ github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/go.mod h1:Q0GQOBtKf1oE10eSXSlhN45kDBdGvEcVOqMiffqX+N8= github.com/filecoin-project/specs-actors v0.9.4/go.mod h1:BStZQzx5x7TmCkLv0Bpa07U6cPKol6fd3w9KjMPZ6Z4= github.com/filecoin-project/specs-actors v0.9.7/go.mod h1:wM2z+kwqYgXn5Z7scV1YHLyd1Q1cy0R8HfTIWQ0BFGU= -github.com/filecoin-project/specs-actors v0.9.10 h1:gU0TrRhgkCsBEOP42sGDE7RQuR0Cov9hJhBqq+RJmjU= -github.com/filecoin-project/specs-actors v0.9.10/go.mod h1:czlvLQGEX0fjLLfdNHD7xLymy6L3n7aQzRWzsYGf+ys= +github.com/filecoin-project/specs-actors v0.9.11 h1:TnpG7HAeiUrfj0mJM7UaPW0P2137H62RGof7ftT5Mas= +github.com/filecoin-project/specs-actors v0.9.11/go.mod h1:czlvLQGEX0fjLLfdNHD7xLymy6L3n7aQzRWzsYGf+ys= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= github.com/filecoin-project/test-vectors/schema v0.0.1 h1:5fNF76nl4qolEvcIsjc0kUADlTMVHO73tW4kXXPnsus= diff --git a/node/modules/chain.go b/node/modules/chain.go index 5eda51078..66f54a76a 100644 --- a/node/modules/chain.go +++ b/node/modules/chain.go @@ -18,6 +18,7 @@ import ( "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain" "github.com/filecoin-project/lotus/chain/beacon" "github.com/filecoin-project/lotus/chain/exchange" @@ -157,6 +158,10 @@ func SetGenesis(cs *store.ChainStore, g Genesis) (dtypes.AfterGenesisSet, error) } func NetworkName(mctx helpers.MetricsCtx, lc fx.Lifecycle, cs *store.ChainStore, _ dtypes.AfterGenesisSet) (dtypes.NetworkName, error) { + if !build.Devnet { + return "testnetnet", nil + } + ctx := helpers.LifecycleCtx(mctx, lc) netName, err := stmgr.GetNetworkName(ctx, stmgr.NewStateManager(cs), cs.GetHeaviestTipSet().ParentState()) diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 9a94a56a5..de466b004 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -109,6 +109,9 @@ func MinerID(ma dtypes.MinerAddress) (dtypes.MinerID, error) { } func StorageNetworkName(ctx helpers.MetricsCtx, a lapi.FullNode) (dtypes.NetworkName, error) { + if !build.Devnet { + return "testnetnet", nil + } return a.StateNetworkName(ctx) } diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 9a497f879..59ffcb74c 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -371,6 +371,10 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty return j }) + if ts.Height() > build.UpgradeIgnitionHeight { + return // FORK: declaring faults after ignition upgrade makes no sense + } + if faults, sigmsg, err = s.checkNextFaults(context.TODO(), declDeadline, partitions); err != nil { // TODO: This is also potentially really bad, but we try to post anyways log.Errorf("checking sector faults: %v", err) From 12e97dbea743c02031ffbcd4f9800cc2d414e2f1 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Sat, 26 Sep 2020 02:59:24 -0400 Subject: [PATCH 355/885] Fix docs and linter --- chain/stmgr/forks.go | 4 ---- documentation/en/api-methods.md | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index 872c70b1e..252b731d7 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -53,10 +53,6 @@ func (sm *StateManager) handleStateForks(ctx context.Context, root cid.Cid, heig return retCid, nil } -type forEachTree interface { - ForEach(func(address.Address, *types.Actor) error) error -} - func doTransfer(cb ExecCallback, tree types.StateTree, from, to address.Address, amt abi.TokenAmount) error { fromAct, err := tree.GetActor(from) if err != nil { diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index e489fcb0f..ed082ccbf 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -3825,7 +3825,7 @@ Inputs: ] ``` -Response: `2` +Response: `3` ### StateReadState StateReadState returns the indicated actor's state. From 567261e2c7bc5b6cb764ab0ef3c9f1f30600448d Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Fri, 25 Sep 2020 23:59:59 -0700 Subject: [PATCH 356/885] set upgrade heights for testground builds --- build/params_testground.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/build/params_testground.go b/build/params_testground.go index 954b5ccfd..07cc88688 100644 --- a/build/params_testground.go +++ b/build/params_testground.go @@ -74,7 +74,9 @@ var ( UpgradeBreezeHeight abi.ChainEpoch = -1 BreezeGasTampingDuration abi.ChainEpoch = 0 - UpgradeSmokeHeight abi.ChainEpoch = -1 + UpgradeSmokeHeight abi.ChainEpoch = -1 + UpgradeIgnitionHeight abi.ChainEpoch = -2 + UpgradeLiftoffHeight abi.ChainEpoch = -3 DrandSchedule = map[abi.ChainEpoch]DrandEnum{ 0: DrandMainnet, @@ -82,4 +84,6 @@ var ( NewestNetworkVersion = network.Version2 ActorUpgradeNetworkVersion = network.Version3 + + Devnet = true ) From 45eadc1b3aeddf794b8f29186863e43e6e25034b Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Sat, 26 Sep 2020 01:45:22 -0400 Subject: [PATCH 357/885] Lotus version 0.8.0 --- CHANGELOG.md | 57 ++++++++++++++++++++++++++++++++++++++++++++++++ build/version.go | 2 +- 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 16ced709b..ac687675e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,62 @@ # Lotus changelog +# 0.8.0 / 2020-09-26 + +This consensus-breaking release of Lotus introduces an upgrade to the network. The changes that break consensus are: + +- Upgrading to specs-actors v0.9.11, which reduces WindowPoSt faults per [FIP 0002](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0002.md) to reduce cost for honest miners with occasional faults (see https://github.com/filecoin-project/specs-actors/pull/1181) +- Revisions to some cryptoeconomics and network params + +This release also updates go-fil-markets to fix an incompatibility issue between v0.7.2 and earlier versions. + +## Changes + +#### Dependencies + +- Update spec actors to 0.9.11 (https://github.com/filecoin-project/lotus/pull/4039) +- Update markets to 0.6.3 (https://github.com/filecoin-project/lotus/pull/4013) + +#### Core Lotus + +- Network upgrade (https://github.com/filecoin-project/lotus/pull/4039) +- Fix AddSupportedProofTypes (https://github.com/filecoin-project/lotus/pull/4033) +- Return an error when we fail to find a sector when checking sector expiration (https://github.com/filecoin-project/lotus/pull/4026) +- Batch blockstore copies after block validation (https://github.com/filecoin-project/lotus/pull/3980) +- Remove a misleading miner actor abstraction (https://github.com/filecoin-project/lotus/pull/3977) +- Fix out-of-bounds when loading all sector infos (https://github.com/filecoin-project/lotus/pull/3976) +- Fix break condition in the miner (https://github.com/filecoin-project/lotus/pull/3953) + +#### UX + +- Correct helptext around miners setting ask (https://github.com/filecoin-project/lotus/pull/4009) +- Make sync wait nicer (https://github.com/filecoin-project/lotus/pull/3991) + +#### Tooling and validation + +- Small adjustments following network upgradability changes (https://github.com/filecoin-project/lotus/pull/3996) +- Add some more big pictures stats to stateroot stat (https://github.com/filecoin-project/lotus/pull/3995) +- Add some actors policy setters for testing (https://github.com/filecoin-project/lotus/pull/3975) + +## Contributors + +The following contributors had 5 or more commits go into this release. +We are grateful for every contribution! + +| Contributor | Commits | Lines ± | +|--------------------|---------|---------------| +| arajasek | 66 | +3140/-1261 | +| Stebalien | 64 | +3797/-3434 | +| magik6k | 48 | +1892/-976 | +| raulk | 40 | +2412/-1549 | +| vyzo | 22 | +287/-196 | +| alanshaw | 15 | +761/-146 | +| whyrusleeping | 15 | +736/-52 | +| hannahhoward | 14 | +1237/837- | +| anton | 6 | +32/-8 | +| travisperson | 5 | +502/-6 | +| Frank | 5 | +78/-39 | +| Jennifer | 5 | +148/-41 | + # 0.7.2 / 2020-09-23 This optional release of Lotus introduces a major refactor around how a Lotus node interacts with code from the specs-actors repo. We now use interfaces to read the state of actors, which is required to be able to reason about different versions of actors code at the same time. diff --git a/build/version.go b/build/version.go index cfc8c3ab9..77b98f008 100644 --- a/build/version.go +++ b/build/version.go @@ -29,7 +29,7 @@ func buildType() string { } // BuildVersion is the local build version, set by build system -const BuildVersion = "0.7.2" +const BuildVersion = "0.8.0" func UserVersion() string { return BuildVersion + buildType() + CurrentCommit From 81a30cbf062fb0e031dfdfc0826f61faeacc6b22 Mon Sep 17 00:00:00 2001 From: Steven Li Date: Sat, 26 Sep 2020 16:01:10 +0800 Subject: [PATCH 358/885] Add one more node located in China --- build/bootstrap/bootstrappers.pi | 1 + 1 file changed, 1 insertion(+) diff --git a/build/bootstrap/bootstrappers.pi b/build/bootstrap/bootstrappers.pi index 465f3b5e9..1c8b77709 100644 --- a/build/bootstrap/bootstrappers.pi +++ b/build/bootstrap/bootstrappers.pi @@ -4,3 +4,4 @@ /dns4/bootstrap-4.testnet.fildev.network/tcp/1347/p2p/12D3KooWPkL9LrKRQgHtq7kn9ecNhGU9QaziG8R5tX8v9v7t3h34 /dns4/bootstrap-3.testnet.fildev.network/tcp/1347/p2p/12D3KooWKYSsbpgZ3HAjax5M1BXCwXLa6gVkUARciz7uN3FNtr7T /dns4/bootstrap-5.testnet.fildev.network/tcp/1347/p2p/12D3KooWQYzqnLASJAabyMpPb1GcWZvNSe7JDcRuhdRqonFoiK9W +/dns4/lotus-bootstrap.forceup.cn/tcp/41778/p2p/12D3KooWFQsv3nRMUevZNWWsY1Wu6NUzUbawnWU5NcRhgKuJA37C From 650725d71e909089e5087169d8d694e3f4076e4c Mon Sep 17 00:00:00 2001 From: zgfzgf <1901989065@qq.com> Date: Sat, 26 Sep 2020 17:30:58 +0800 Subject: [PATCH 359/885] add printf --- api/test/window_post.go | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/api/test/window_post.go b/api/test/window_post.go index 683489a91..958c91816 100644 --- a/api/test/window_post.go +++ b/api/test/window_post.go @@ -153,18 +153,16 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector require.NoError(t, err) fmt.Printf("Running one proving period\n") + fmt.Printf("End for head.Height > %d\n", di.PeriodStart+di.WPoStProvingPeriod+2) for { head, err := client.ChainHead(ctx) require.NoError(t, err) if head.Height() > di.PeriodStart+(di.WPoStProvingPeriod)+2 { + fmt.Printf("Now head.Height = %d\n", head.Height()) break } - - if head.Height()%100 == 0 { - fmt.Printf("@%d\n", head.Height()) - } build.Clock.Sleep(blocktime) } @@ -186,7 +184,6 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector require.Greater(t, len(parts), 0) secs := parts[0].AllSectors - require.NoError(t, err) n, err := secs.Count() require.NoError(t, err) require.Equal(t, uint64(2), n) @@ -210,7 +207,6 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector require.Greater(t, len(parts), 0) secs := parts[0].AllSectors - require.NoError(t, err) n, err := secs.Count() require.NoError(t, err) require.Equal(t, uint64(2), n) @@ -236,18 +232,17 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector require.NoError(t, err) fmt.Printf("Go through another PP, wait for sectors to become faulty\n") + fmt.Printf("End for head.Height > %d\n", di.PeriodStart+di.WPoStProvingPeriod+2) for { head, err := client.ChainHead(ctx) require.NoError(t, err) if head.Height() > di.PeriodStart+(di.WPoStProvingPeriod)+2 { + fmt.Printf("Now head.Height = %d\n", head.Height()) break } - if head.Height()%100 == 0 { - fmt.Printf("@%d\n", head.Height()) - } build.Clock.Sleep(blocktime) } @@ -267,17 +262,17 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector di, err = client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK) require.NoError(t, err) + fmt.Printf("End for head.Height > %d\n", di.PeriodStart+di.WPoStProvingPeriod+2) + for { head, err := client.ChainHead(ctx) require.NoError(t, err) if head.Height() > di.PeriodStart+di.WPoStProvingPeriod+2 { + fmt.Printf("Now head.Height = %d\n", head.Height()) break } - if head.Height()%100 == 0 { - fmt.Printf("@%d\n", head.Height()) - } build.Clock.Sleep(blocktime) } @@ -300,12 +295,14 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector require.NoError(t, err) waitUntil := head.Height() + 10 + fmt.Printf("End for head.Height > %d\n", waitUntil) for { head, err := client.ChainHead(ctx) require.NoError(t, err) if head.Height() > waitUntil { + fmt.Printf("Now head.Height = %d\n", head.Height()) break } } From bddd6dd8a8aaa89d38a5ccebc3b066224ca6b510 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Sat, 26 Sep 2020 11:06:16 -0500 Subject: [PATCH 360/885] fix GetPower with no miner address --- chain/stmgr/utils.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index 58e7f480f..bac5a31f5 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -102,6 +102,7 @@ func GetPowerRaw(ctx context.Context, sm *StateManager, st cid.Cid, maddr addres } var mpow power.Claim + var minpow bool if maddr != address.Undef { var found bool mpow, found, err = pas.MinerPower(maddr) @@ -109,11 +110,11 @@ func GetPowerRaw(ctx context.Context, sm *StateManager, st cid.Cid, maddr addres // TODO: return an error when not found? return power.Claim{}, power.Claim{}, false, err } - } - minpow, err := pas.MinerNominalPowerMeetsConsensusMinimum(maddr) - if err != nil { - return power.Claim{}, power.Claim{}, false, err + minpow, err = pas.MinerNominalPowerMeetsConsensusMinimum(maddr) + if err != nil { + return power.Claim{}, power.Claim{}, false, err + } } return mpow, tpow, minpow, nil From 1c5cb50da340b19f287b211a40238e8782378cad Mon Sep 17 00:00:00 2001 From: Travis Person Date: Sat, 26 Sep 2020 17:09:16 +0000 Subject: [PATCH 361/885] Add back network power to stats --- tools/stats/metrics.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tools/stats/metrics.go b/tools/stats/metrics.go index 22069c3d0..dd51ee69f 100644 --- a/tools/stats/metrics.go +++ b/tools/stats/metrics.go @@ -10,6 +10,7 @@ import ( "strings" "time" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors/builtin/reward" @@ -252,6 +253,14 @@ func RecordTipsetStatePoints(ctx context.Context, api api.FullNode, pl *PointLis p := NewPoint("network.balance", netBalFilFloat) pl.AddPoint(p) + totalPower, err := api.StateMinerPower(ctx, address.Address{}, tipset.Key()) + if err != nil { + return err + } + + p = NewPoint("chain.power", totalPower.TotalPower.QualityAdjPower.Int64()) + pl.AddPoint(p) + miners, err := api.StateListMiners(ctx, tipset.Key()) if err != nil { return err From 8955b8d8a7ae3110acd82f072e9d739be7f73ac1 Mon Sep 17 00:00:00 2001 From: Peter Rabbitson Date: Sat, 26 Sep 2020 21:16:28 +0200 Subject: [PATCH 362/885] Centralize filtering, output wallet addresses --- cmd/lotus-shed/dealtracker.go | 186 ++++++++++++++++++---------------- 1 file changed, 97 insertions(+), 89 deletions(-) diff --git a/cmd/lotus-shed/dealtracker.go b/cmd/lotus-shed/dealtracker.go index d39f51bd1..a21923009 100644 --- a/cmd/lotus-shed/dealtracker.go +++ b/cmd/lotus-shed/dealtracker.go @@ -5,8 +5,7 @@ import ( "encoding/json" "net" "net/http" - "os" - "strings" + "sync" "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/api" @@ -19,23 +18,27 @@ type dealStatsServer struct { api api.FullNode } -var filteredClients map[address.Address]bool +// these lists grow continuously with the network +// TODO: need to switch this to an LRU of sorts, to ensure refreshes +var knownFiltered = new(sync.Map) +var resolvedWallets = new(sync.Map) func init() { - fc := []string{"t0112", "t0113", "t0114", "t010089"} - - filtered, set := os.LookupEnv("FILTERED_CLIENTS") - if set { - fc = strings.Split(filtered, ":") - } - - filteredClients = make(map[address.Address]bool) - for _, a := range fc { - addr, err := address.NewFromString(a) + for _, a := range []string{ + "t0100", // client for genesis miner + "t0112", // client for genesis miner + "t0113", // client for genesis miner + "t0114", // client for genesis miner + "t1nslxql4pck5pq7hddlzym3orxlx35wkepzjkm3i", // SR1 dealbot wallet + "t1stghxhdp2w53dym2nz2jtbpk6ccd4l2lxgmezlq", // SR1 dealbot wallet + "t1mcr5xkgv4jdl3rnz77outn6xbmygb55vdejgbfi", // SR1 dealbot wallet + "t1qiqdbbmrdalbntnuapriirduvxu5ltsc5mhy7si", // SR1 dealbot wallet + } { + a, err := address.NewFromString(a) if err != nil { panic(err) } - filteredClients[addr] = true + knownFiltered.Store(a, true) } } @@ -45,32 +48,16 @@ type dealCountResp struct { } func (dss *dealStatsServer) handleStorageDealCount(w http.ResponseWriter, r *http.Request) { - ctx := context.Background() - head, err := dss.api.ChainHead(ctx) - if err != nil { - log.Warnf("failed to get chain head: %s", err) + epoch, deals := dss.filteredDealList() + if epoch == 0 { w.WriteHeader(500) return } - deals, err := dss.api.StateMarketDeals(ctx, head.Key()) - if err != nil { - log.Warnf("failed to get market deals: %s", err) - w.WriteHeader(500) - return - } - - var count int64 - for _, d := range deals { - if !filteredClients[d.Proposal.Client] { - count++ - } - } - if err := json.NewEncoder(w).Encode(&dealCountResp{ - Total: count, - Epoch: int64(head.Height()), + Total: int64(len(deals)), + Epoch: epoch, }); err != nil { log.Warnf("failed to write back deal count response: %s", err) return @@ -83,34 +70,21 @@ type dealAverageResp struct { } func (dss *dealStatsServer) handleStorageDealAverageSize(w http.ResponseWriter, r *http.Request) { - ctx := context.Background() - head, err := dss.api.ChainHead(ctx) - if err != nil { - log.Warnf("failed to get chain head: %s", err) + epoch, deals := dss.filteredDealList() + if epoch == 0 { w.WriteHeader(500) return } - deals, err := dss.api.StateMarketDeals(ctx, head.Key()) - if err != nil { - log.Warnf("failed to get market deals: %s", err) - w.WriteHeader(500) - return - } - - var count int64 var totalBytes int64 for _, d := range deals { - if !filteredClients[d.Proposal.Client] { - count++ - totalBytes += int64(d.Proposal.PieceSize.Unpadded()) - } + totalBytes += int64(d.deal.Proposal.PieceSize.Unpadded()) } if err := json.NewEncoder(w).Encode(&dealAverageResp{ - AverageSize: totalBytes / count, - Epoch: int64(head.Height()), + AverageSize: totalBytes / int64(len(deals)), + Epoch: epoch, }); err != nil { log.Warnf("failed to write back deal average response: %s", err) return @@ -123,32 +97,20 @@ type dealTotalResp struct { } func (dss *dealStatsServer) handleStorageDealTotalReal(w http.ResponseWriter, r *http.Request) { - ctx := context.Background() - - head, err := dss.api.ChainHead(ctx) - if err != nil { - log.Warnf("failed to get chain head: %s", err) - w.WriteHeader(500) - return - } - - deals, err := dss.api.StateMarketDeals(ctx, head.Key()) - if err != nil { - log.Warnf("failed to get market deals: %s", err) + epoch, deals := dss.filteredDealList() + if epoch == 0 { w.WriteHeader(500) return } var totalBytes int64 for _, d := range deals { - if !filteredClients[d.Proposal.Client] { - totalBytes += int64(d.Proposal.PieceSize.Unpadded()) - } + totalBytes += int64(d.deal.Proposal.PieceSize.Unpadded()) } if err := json.NewEncoder(w).Encode(&dealTotalResp{ TotalBytes: totalBytes, - Epoch: int64(head.Height()), + Epoch: epoch, }); err != nil { log.Warnf("failed to write back deal average response: %s", err) return @@ -168,18 +130,8 @@ type clientStatsOutput struct { } func (dss *dealStatsServer) handleStorageClientStats(w http.ResponseWriter, r *http.Request) { - ctx := context.Background() - - head, err := dss.api.ChainHead(ctx) - if err != nil { - log.Warnf("failed to get chain head: %s", err) - w.WriteHeader(500) - return - } - - deals, err := dss.api.StateMarketDeals(ctx, head.Key()) - if err != nil { - log.Warnf("failed to get market deals: %s", err) + epoch, deals := dss.filteredDealList() + if epoch == 0 { w.WriteHeader(500) return } @@ -187,23 +139,20 @@ func (dss *dealStatsServer) handleStorageClientStats(w http.ResponseWriter, r *h stats := make(map[address.Address]*clientStatsOutput) for _, d := range deals { - if filteredClients[d.Proposal.Client] { - continue - } - st, ok := stats[d.Proposal.Client] + st, ok := stats[d.deal.Proposal.Client] if !ok { st = &clientStatsOutput{ - Client: d.Proposal.Client, + Client: d.resolvedWallet, cids: make(map[cid.Cid]bool), providers: make(map[address.Address]bool), } - stats[d.Proposal.Client] = st + stats[d.deal.Proposal.Client] = st } - st.DataSize += int64(d.Proposal.PieceSize.Unpadded()) - st.cids[d.Proposal.PieceCID] = true - st.providers[d.Proposal.Provider] = true + st.DataSize += int64(d.deal.Proposal.PieceSize.Unpadded()) + st.cids[d.deal.Proposal.PieceCID] = true + st.providers[d.deal.Proposal.Provider] = true st.NumDeals++ } @@ -221,6 +170,65 @@ func (dss *dealStatsServer) handleStorageClientStats(w http.ResponseWriter, r *h } } +type dealInfo struct { + deal api.MarketDeal + resolvedWallet address.Address +} + +// filteredDealList returns the current epoch and a list of filtered deals +// on error returns an epoch of 0 +func (dss *dealStatsServer) filteredDealList() (int64, map[string]dealInfo) { + ctx := context.Background() + + head, err := dss.api.ChainHead(ctx) + if err != nil { + log.Warnf("failed to get chain head: %s", err) + return 0, nil + } + + deals, err := dss.api.StateMarketDeals(ctx, head.Key()) + if err != nil { + log.Warnf("failed to get market deals: %s", err) + return 0, nil + } + + ret := make(map[string]dealInfo, len(deals)) + for dealKey, d := range deals { + + // Counting no-longer-active deals as per Pooja's request + // // https://github.com/filecoin-project/specs-actors/blob/v0.9.9/actors/builtin/market/deal.go#L81-L85 + // if d.State.SectorStartEpoch < 0 { + // continue + // } + + if _, isFiltered := knownFiltered.Load(d.Proposal.Client); isFiltered { + continue + } + + if _, wasSeen := resolvedWallets.Load(d.Proposal.Client); !wasSeen { + w, err := dss.api.StateAccountKey(ctx, d.Proposal.Client, head.Key()) + if err != nil { + log.Warnf("failed to resolve id '%s' to wallet address: %s", d.Proposal.Client, err) + continue + } else { + resolvedWallets.Store(d.Proposal.Client, w) + } + } + + w, _ := resolvedWallets.Load(d.Proposal.Client) + if _, isFiltered := knownFiltered.Load(w); isFiltered { + continue + } + + ret[dealKey] = dealInfo{ + deal: d, + resolvedWallet: w.(address.Address), + } + } + + return int64(head.Height()), ret +} + var serveDealStatsCmd = &cli.Command{ Name: "serve-deal-stats", Flags: []cli.Flag{}, From 10cdbadd82158cc36959877733ad008188b6aa14 Mon Sep 17 00:00:00 2001 From: Peter Rabbitson Date: Sat, 26 Sep 2020 21:29:11 +0200 Subject: [PATCH 363/885] Arrange json as the frontend expects it --- cmd/lotus-shed/dealtracker.go | 55 ++++++++++++++++++++++------------- 1 file changed, 35 insertions(+), 20 deletions(-) diff --git a/cmd/lotus-shed/dealtracker.go b/cmd/lotus-shed/dealtracker.go index a21923009..1340dc9c6 100644 --- a/cmd/lotus-shed/dealtracker.go +++ b/cmd/lotus-shed/dealtracker.go @@ -43,8 +43,9 @@ func init() { } type dealCountResp struct { - Total int64 `json:"total"` - Epoch int64 `json:"epoch"` + Epoch int64 `json:"epoch"` + Endpoint string `json:"endpoint"` + Payload int64 `json:"payload"` } func (dss *dealStatsServer) handleStorageDealCount(w http.ResponseWriter, r *http.Request) { @@ -56,8 +57,9 @@ func (dss *dealStatsServer) handleStorageDealCount(w http.ResponseWriter, r *htt } if err := json.NewEncoder(w).Encode(&dealCountResp{ - Total: int64(len(deals)), - Epoch: epoch, + Endpoint: "COUNT_DEALS", + Payload: int64(len(deals)), + Epoch: epoch, }); err != nil { log.Warnf("failed to write back deal count response: %s", err) return @@ -65,8 +67,9 @@ func (dss *dealStatsServer) handleStorageDealCount(w http.ResponseWriter, r *htt } type dealAverageResp struct { - AverageSize int64 `json:"average_size"` - Epoch int64 `json:"epoch"` + Epoch int64 `json:"epoch"` + Endpoint string `json:"endpoint"` + Payload int64 `json:"payload"` } func (dss *dealStatsServer) handleStorageDealAverageSize(w http.ResponseWriter, r *http.Request) { @@ -83,8 +86,9 @@ func (dss *dealStatsServer) handleStorageDealAverageSize(w http.ResponseWriter, } if err := json.NewEncoder(w).Encode(&dealAverageResp{ - AverageSize: totalBytes / int64(len(deals)), - Epoch: epoch, + Endpoint: "AVERAGE_DEAL_SIZE", + Payload: totalBytes / int64(len(deals)), + Epoch: epoch, }); err != nil { log.Warnf("failed to write back deal average response: %s", err) return @@ -92,8 +96,9 @@ func (dss *dealStatsServer) handleStorageDealAverageSize(w http.ResponseWriter, } type dealTotalResp struct { - TotalBytes int64 `json:"total_size"` - Epoch int64 `json:"epoch"` + Epoch int64 `json:"epoch"` + Endpoint string `json:"endpoint"` + Payload int64 `json:"payload"` } func (dss *dealStatsServer) handleStorageDealTotalReal(w http.ResponseWriter, r *http.Request) { @@ -109,8 +114,9 @@ func (dss *dealStatsServer) handleStorageDealTotalReal(w http.ResponseWriter, r } if err := json.NewEncoder(w).Encode(&dealTotalResp{ - TotalBytes: totalBytes, - Epoch: epoch, + Endpoint: "DEAL_BYTES", + Payload: totalBytes, + Epoch: epoch, }); err != nil { log.Warnf("failed to write back deal average response: %s", err) return @@ -119,6 +125,12 @@ func (dss *dealStatsServer) handleStorageDealTotalReal(w http.ResponseWriter, r } type clientStatsOutput struct { + Epoch int64 `json:"epoch"` + Endpoint string `json:"endpoint"` + Payload []*clientStats `json:"payload"` +} + +type clientStats struct { Client address.Address `json:"client"` DataSize int64 `json:"data_size"` NumCids int `json:"num_cids"` @@ -136,13 +148,13 @@ func (dss *dealStatsServer) handleStorageClientStats(w http.ResponseWriter, r *h return } - stats := make(map[address.Address]*clientStatsOutput) + stats := make(map[address.Address]*clientStats) for _, d := range deals { st, ok := stats[d.deal.Proposal.Client] if !ok { - st = &clientStatsOutput{ + st = &clientStats{ Client: d.resolvedWallet, cids: make(map[cid.Cid]bool), providers: make(map[address.Address]bool), @@ -156,12 +168,15 @@ func (dss *dealStatsServer) handleStorageClientStats(w http.ResponseWriter, r *h st.NumDeals++ } - out := make([]*clientStatsOutput, 0, len(stats)) - for _, cso := range stats { - cso.NumCids = len(cso.cids) - cso.NumMiners = len(cso.providers) - - out = append(out, cso) + out := clientStatsOutput{ + Epoch: epoch, + Endpoint: "CLIENT_DEAL_STATS", + Payload: make([]*clientStats, 0, len(stats)), + } + for _, cs := range stats { + cs.NumCids = len(cs.cids) + cs.NumMiners = len(cs.providers) + out.Payload = append(out.Payload, cs) } if err := json.NewEncoder(w).Encode(out); err != nil { From 1483f1e59adcbd91ab90ca01a3b8591e8f37c1fc Mon Sep 17 00:00:00 2001 From: Peter Rabbitson Date: Sat, 26 Sep 2020 21:43:49 +0200 Subject: [PATCH 364/885] Add filtering of addresses associated with miners --- cmd/lotus-shed/dealtracker.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/cmd/lotus-shed/dealtracker.go b/cmd/lotus-shed/dealtracker.go index 1340dc9c6..719e821a5 100644 --- a/cmd/lotus-shed/dealtracker.go +++ b/cmd/lotus-shed/dealtracker.go @@ -201,6 +201,26 @@ func (dss *dealStatsServer) filteredDealList() (int64, map[string]dealInfo) { return 0, nil } + // Exclude any address associated with a miner + miners, err := dss.api.StateListMiners(ctx, head.Key()) + if err != nil { + log.Warnf("failed to get miner list: %s", err) + return 0, nil + } + for _, m := range miners { + info, err := dss.api.StateMinerInfo(ctx, m, head.Key()) + if err != nil { + log.Warnf("failed to get info for known miner '%s': %s", m, err) + continue + } + + knownFiltered.Store(info.Owner, true) + knownFiltered.Store(info.Worker, true) + for _, a := range info.ControlAddresses { + knownFiltered.Store(a, true) + } + } + deals, err := dss.api.StateMarketDeals(ctx, head.Key()) if err != nil { log.Warnf("failed to get market deals: %s", err) From fb3bcc4ce527961fa172898a180d1fbb865fc133 Mon Sep 17 00:00:00 2001 From: Peter Rabbitson Date: Sat, 26 Sep 2020 21:49:05 +0200 Subject: [PATCH 365/885] Add startup warning --- cmd/lotus-shed/dealtracker.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/lotus-shed/dealtracker.go b/cmd/lotus-shed/dealtracker.go index 719e821a5..8a9d0d6f3 100644 --- a/cmd/lotus-shed/dealtracker.go +++ b/cmd/lotus-shed/dealtracker.go @@ -303,6 +303,8 @@ var serveDealStatsCmd = &cli.Command{ panic(err) } + log.Warnf("deal-stat server listening on %s\n== NOTE: QUERIES ARE EXPENSIVE - YOU MUST FRONT-CACHE THIS SERVICE\n", list.Addr().String()) + return s.Serve(list) }, } From 70071e273d10f07010e70673f0d878fbb46f2262 Mon Sep 17 00:00:00 2001 From: zgfzgf <1901989065@qq.com> Date: Sun, 27 Sep 2020 09:58:37 +0800 Subject: [PATCH 366/885] optimize tipset Equals func --- chain/types/tipset.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/chain/types/tipset.go b/chain/types/tipset.go index 44d41c29d..5d34ec89d 100644 --- a/chain/types/tipset.go +++ b/chain/types/tipset.go @@ -167,6 +167,10 @@ func (ts *TipSet) Equals(ots *TipSet) bool { return false } + if ts.height != ots.height { + return false + } + if len(ts.blks) != len(ots.blks) { return false } From 04876c663e16eb381adb6695f32db1c7f99493ab Mon Sep 17 00:00:00 2001 From: zgfzgf <1901989065@qq.com> Date: Sun, 27 Sep 2020 10:17:06 +0800 Subject: [PATCH 367/885] modify tipset Equals --- chain/types/tipset.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/chain/types/tipset.go b/chain/types/tipset.go index 5d34ec89d..07eff3734 100644 --- a/chain/types/tipset.go +++ b/chain/types/tipset.go @@ -171,12 +171,12 @@ func (ts *TipSet) Equals(ots *TipSet) bool { return false } - if len(ts.blks) != len(ots.blks) { + if len(ts.cids) != len(ots.cids) { return false } - for i, b := range ts.blks { - if b.Cid() != ots.blks[i].Cid() { + for i, cid := range ts.cids { + if cid != ots.cids[i] { return false } } From 7a14455ac8253e0a9fd23fd358baf541efe3ca8e Mon Sep 17 00:00:00 2001 From: zgfzgf <1901989065@qq.com> Date: Sun, 27 Sep 2020 15:01:42 +0800 Subject: [PATCH 368/885] miner debug where injectNulls != 0 --- miner/miner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/miner/miner.go b/miner/miner.go index 1b79f5245..d4e7b2317 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -211,6 +211,8 @@ minerLoop: base = prebase } + base.NullRounds += injectNulls // testing + if base.TipSet.Equals(lastBase.TipSet) && lastBase.NullRounds == base.NullRounds { log.Warnf("BestMiningCandidate from the previous round: %s (nulls:%d)", lastBase.TipSet.Cids(), lastBase.NullRounds) if !m.niceSleep(time.Duration(build.BlockDelaySecs) * time.Second) { @@ -219,8 +221,6 @@ minerLoop: continue } - base.NullRounds += injectNulls // testing - b, err := m.mineOne(ctx, base) if err != nil { log.Errorf("mining block failed: %+v", err) From e4c1f090af73cc0e9f997b3de5e9a9443b493ad5 Mon Sep 17 00:00:00 2001 From: Peter Rabbitson Date: Sun, 27 Sep 2020 20:44:50 +0200 Subject: [PATCH 369/885] Disable exclusion of miner-associated addresses --- cmd/lotus-shed/dealtracker.go | 40 +++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/cmd/lotus-shed/dealtracker.go b/cmd/lotus-shed/dealtracker.go index 8a9d0d6f3..083db8ecb 100644 --- a/cmd/lotus-shed/dealtracker.go +++ b/cmd/lotus-shed/dealtracker.go @@ -26,6 +26,8 @@ var resolvedWallets = new(sync.Map) func init() { for _, a := range []string{ "t0100", // client for genesis miner + "t0101", // client for genesis miner + "t0102", // client for genesis miner "t0112", // client for genesis miner "t0113", // client for genesis miner "t0114", // client for genesis miner @@ -201,25 +203,27 @@ func (dss *dealStatsServer) filteredDealList() (int64, map[string]dealInfo) { return 0, nil } - // Exclude any address associated with a miner - miners, err := dss.api.StateListMiners(ctx, head.Key()) - if err != nil { - log.Warnf("failed to get miner list: %s", err) - return 0, nil - } - for _, m := range miners { - info, err := dss.api.StateMinerInfo(ctx, m, head.Key()) - if err != nil { - log.Warnf("failed to get info for known miner '%s': %s", m, err) - continue - } + // Disabled as per @pooja's request + // + // // Exclude any address associated with a miner + // miners, err := dss.api.StateListMiners(ctx, head.Key()) + // if err != nil { + // log.Warnf("failed to get miner list: %s", err) + // return 0, nil + // } + // for _, m := range miners { + // info, err := dss.api.StateMinerInfo(ctx, m, head.Key()) + // if err != nil { + // log.Warnf("failed to get info for known miner '%s': %s", m, err) + // continue + // } - knownFiltered.Store(info.Owner, true) - knownFiltered.Store(info.Worker, true) - for _, a := range info.ControlAddresses { - knownFiltered.Store(a, true) - } - } + // knownFiltered.Store(info.Owner, true) + // knownFiltered.Store(info.Worker, true) + // for _, a := range info.ControlAddresses { + // knownFiltered.Store(a, true) + // } + // } deals, err := dss.api.StateMarketDeals(ctx, head.Key()) if err != nil { From be5dc2c57fb8dfde934ddaa63b22d25dd5ca0356 Mon Sep 17 00:00:00 2001 From: Peter Rabbitson Date: Sun, 27 Sep 2020 20:45:45 +0200 Subject: [PATCH 370/885] Walk back 10 epochs for stat generation --- cmd/lotus-shed/dealtracker.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/cmd/lotus-shed/dealtracker.go b/cmd/lotus-shed/dealtracker.go index 083db8ecb..8ded6bf4a 100644 --- a/cmd/lotus-shed/dealtracker.go +++ b/cmd/lotus-shed/dealtracker.go @@ -8,6 +8,7 @@ import ( "sync" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/api" lcli "github.com/filecoin-project/lotus/cli" "github.com/ipfs/go-cid" @@ -18,6 +19,10 @@ type dealStatsServer struct { api api.FullNode } +// Requested by @jbenet +// How many epochs back to look at for dealstats +var epochLookback = abi.ChainEpoch(10) + // these lists grow continuously with the network // TODO: need to switch this to an LRU of sorts, to ensure refreshes var knownFiltered = new(sync.Map) @@ -203,6 +208,12 @@ func (dss *dealStatsServer) filteredDealList() (int64, map[string]dealInfo) { return 0, nil } + head, err = dss.api.ChainGetTipSetByHeight(ctx, head.Height()-epochLookback, head.Key()) + if err != nil { + log.Warnf("failed to walk back %s epochs: %s", epochLookback, err) + return 0, nil + } + // Disabled as per @pooja's request // // // Exclude any address associated with a miner From e5c56da3217217362cffef257b1433034882c228 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Sun, 27 Sep 2020 20:10:05 +0100 Subject: [PATCH 371/885] move conformance tvx tool to lotus. --- .github/CODEOWNERS | 1 + cmd/tvx/exec.go | 92 +++++++++ cmd/tvx/extract.go | 380 +++++++++++++++++++++++++++++++++++++ cmd/tvx/main.go | 92 +++++++++ cmd/tvx/state.go | 293 ++++++++++++++++++++++++++++ cmd/tvx/stores.go | 143 ++++++++++++++ conformance/corpus_test.go | 133 +++++++++++++ conformance/driver.go | 8 +- conformance/reporter.go | 62 ++++++ conformance/runner.go | 253 ++++++++++++++++++++++++ conformance/runner_test.go | 376 ------------------------------------ go.sum | 1 + 12 files changed, 1457 insertions(+), 377 deletions(-) create mode 100644 cmd/tvx/exec.go create mode 100644 cmd/tvx/extract.go create mode 100644 cmd/tvx/main.go create mode 100644 cmd/tvx/state.go create mode 100644 cmd/tvx/stores.go create mode 100644 conformance/corpus_test.go create mode 100644 conformance/reporter.go create mode 100644 conformance/runner.go delete mode 100644 conformance/runner_test.go diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 49e461d00..6d717b44d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -13,3 +13,4 @@ ### Conformance testing. conformance/ @raulk extern/test-vectors @raulk +cmd/tvx @raulk \ No newline at end of file diff --git a/cmd/tvx/exec.go b/cmd/tvx/exec.go new file mode 100644 index 000000000..9ec6f9e2b --- /dev/null +++ b/cmd/tvx/exec.go @@ -0,0 +1,92 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "log" + "os" + + "github.com/fatih/color" + "github.com/urfave/cli/v2" + + "github.com/filecoin-project/lotus/conformance" + + "github.com/filecoin-project/test-vectors/schema" +) + +var execFlags struct { + file string +} + +var execCmd = &cli.Command{ + Name: "exec", + Description: "execute one or many test vectors against Lotus; supplied as a single JSON file, or a ndjson stdin stream", + Action: runExecLotus, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "file", + Usage: "input file; if not supplied, the vector will be read from stdin", + TakesFile: true, + Destination: &execFlags.file, + }, + }, +} + +func runExecLotus(_ *cli.Context) error { + if file := execFlags.file; file != "" { + // we have a single test vector supplied as a file. + file, err := os.Open(file) + if err != nil { + return fmt.Errorf("failed to open test vector: %w", err) + } + + var ( + dec = json.NewDecoder(file) + tv schema.TestVector + ) + + if err = dec.Decode(&tv); err != nil { + return fmt.Errorf("failed to decode test vector: %w", err) + } + + return executeTestVector(tv) + } + + for dec := json.NewDecoder(os.Stdin); ; { + var tv schema.TestVector + switch err := dec.Decode(&tv); err { + case nil: + if err = executeTestVector(tv); err != nil { + return err + } + case io.EOF: + // we're done. + return nil + default: + // something bad happened. + return err + } + } +} + +func executeTestVector(tv schema.TestVector) error { + log.Println("executing test vector:", tv.Meta.ID) + r := new(conformance.LogReporter) + switch class := tv.Class; class { + case "message": + conformance.ExecuteMessageVector(r, &tv) + case "tipset": + conformance.ExecuteTipsetVector(r, &tv) + default: + return fmt.Errorf("test vector class %s not supported", class) + } + + if r.Failed() { + log.Println(color.HiRedString("❌ test vector failed")) + } else { + log.Println(color.GreenString("✅ test vector succeeded")) + } + + return nil +} diff --git a/cmd/tvx/extract.go b/cmd/tvx/extract.go new file mode 100644 index 000000000..81fd3efbf --- /dev/null +++ b/cmd/tvx/extract.go @@ -0,0 +1,380 @@ +package main + +import ( + "bytes" + "compress/gzip" + "context" + "encoding/json" + "fmt" + "io" + "log" + "os" + + "github.com/filecoin-project/lotus/api" + init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init" + "github.com/filecoin-project/lotus/chain/actors/builtin/reward" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/vm" + "github.com/filecoin-project/lotus/conformance" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/specs-actors/actors/builtin" + "github.com/filecoin-project/test-vectors/schema" + + "github.com/ipfs/go-cid" + "github.com/urfave/cli/v2" +) + +var extractFlags struct { + id string + class string + cid string + file string + retain string +} + +var extractCmd = &cli.Command{ + Name: "extract", + Description: "generate a message-class test vector by extracting it from a live chain", + Action: runExtract, + Flags: []cli.Flag{ + &apiFlag, + &cli.StringFlag{ + Name: "class", + Usage: "class of vector to extract; other required flags depend on the; values: 'message'", + Value: "message", + Destination: &extractFlags.class, + }, + &cli.StringFlag{ + Name: "id", + Usage: "identifier to name this test vector with", + Value: "", + Destination: &extractFlags.id, + }, + &cli.StringFlag{ + Name: "cid", + Usage: "message CID to generate test vector from", + Required: true, + Destination: &extractFlags.cid, + }, + &cli.StringFlag{ + Name: "out", + Aliases: []string{"o"}, + Usage: "file to write test vector to", + Destination: &extractFlags.file, + }, + &cli.StringFlag{ + Name: "state-retain", + Usage: "state retention policy; values: 'accessed-cids', 'accessed-actors'", + Value: "accessed-cids", + Destination: &extractFlags.retain, + }, + }, +} + +func runExtract(_ *cli.Context) error { + // LOTUS_DISABLE_VM_BUF disables what's called "VM state tree buffering", + // which stashes write operations in a BufferedBlockstore + // (https://github.com/filecoin-project/lotus/blob/b7a4dbb07fd8332b4492313a617e3458f8003b2a/lib/bufbstore/buf_bstore.go#L21) + // such that they're not written until the VM is actually flushed. + // + // For some reason, the standard behaviour was not working for me (raulk), + // and disabling it (such that the state transformations are written immediately + // to the blockstore) worked. + _ = os.Setenv("LOTUS_DISABLE_VM_BUF", "iknowitsabadidea") + + ctx := context.Background() + + mcid, err := cid.Decode(extractFlags.cid) + if err != nil { + return err + } + + // Make the API client. + api, closer, err := makeAPIClient() + if err != nil { + return err + } + defer closer() + + log.Printf("locating message with CID: %s", mcid) + + // Locate the message. + msgInfo, err := api.StateSearchMsg(ctx, mcid) + if err != nil { + return fmt.Errorf("failed to locate message: %w", err) + } + + log.Printf("located message at tipset %s (height: %d) with exit code: %s", msgInfo.TipSet, msgInfo.Height, msgInfo.Receipt.ExitCode) + + // Extract the full message. + msg, err := api.ChainGetMessage(ctx, mcid) + if err != nil { + return err + } + + log.Printf("full message: %+v", msg) + + execTs, incTs, err := fetchThisAndPrevTipset(ctx, api, msgInfo.TipSet) + if err != nil { + return err + } + + log.Printf("message was executed in tipset: %s", execTs.Key()) + log.Printf("message was included in tipset: %s", incTs.Key()) + log.Printf("finding precursor messages") + + // Iterate through blocks, finding the one that contains the message and its + // precursors, if any. + var allmsgs []*types.Message + for _, b := range incTs.Blocks() { + messages, err := api.ChainGetBlockMessages(ctx, b.Cid()) + if err != nil { + return err + } + + related, found, err := findMsgAndPrecursors(messages, msg) + if err != nil { + return fmt.Errorf("invariant failed while scanning messages in block %s: %w", b.Cid(), err) + } + + if found { + var mcids []cid.Cid + for _, m := range related { + mcids = append(mcids, m.Cid()) + } + log.Printf("found message in block %s; precursors: %v", b.Cid(), mcids[:len(mcids)-1]) + allmsgs = related + break + } + + log.Printf("message not found in block %s; precursors found: %v; ignoring block", b.Cid(), related) + } + + if allmsgs == nil { + // Message was not found; abort. + return fmt.Errorf("did not find a block containing the message") + } + + precursors := allmsgs[:len(allmsgs)-1] + + var ( + // create a read-through store that uses ChainGetObject to fetch unknown CIDs. + pst = NewProxyingStores(ctx, api) + g = NewSurgeon(ctx, api, pst) + ) + + driver := conformance.NewDriver(ctx, schema.Selector{}) + + // this is the root of the state tree we start with. + root := incTs.ParentState() + log.Printf("base state tree root CID: %s", root) + + // on top of that state tree, we apply all precursors. + log.Printf("number of precursors to apply: %d", len(precursors)) + for i, m := range precursors { + log.Printf("applying precursor %d, cid: %s", i, m.Cid()) + _, root, err = driver.ExecuteMessage(pst.Blockstore, root, execTs.Height(), m) + if err != nil { + return fmt.Errorf("failed to execute precursor message: %w", err) + } + } + + var ( + preroot cid.Cid + postroot cid.Cid + applyret *vm.ApplyRet + carWriter func(w io.Writer) error + retention = extractFlags.retain + ) + + log.Printf("using state retention strategy: %s", retention) + switch retention { + case "accessed-cids": + tbs, ok := pst.Blockstore.(TracingBlockstore) + if !ok { + return fmt.Errorf("requested 'accessed-cids' state retention, but no tracing blockstore was present") + } + + tbs.StartTracing() + + preroot = root + applyret, postroot, err = driver.ExecuteMessage(pst.Blockstore, preroot, execTs.Height(), msg) + if err != nil { + return fmt.Errorf("failed to execute message: %w", err) + } + accessed := tbs.FinishTracing() + carWriter = func(w io.Writer) error { + return g.WriteCARIncluding(w, accessed, preroot, postroot) + } + + case "accessed-actors": + log.Printf("calculating accessed actors") + // get actors accessed by message. + retain, err := g.GetAccessedActors(ctx, api, mcid) + if err != nil { + return fmt.Errorf("failed to calculate accessed actors: %w", err) + } + // also append the reward actor and the burnt funds actor. + retain = append(retain, reward.Address, builtin.BurntFundsActorAddr, init_.Address) + log.Printf("calculated accessed actors: %v", retain) + + // get the masked state tree from the root, + preroot, err = g.GetMaskedStateTree(root, retain) + if err != nil { + return err + } + applyret, postroot, err = driver.ExecuteMessage(pst.Blockstore, preroot, execTs.Height(), msg) + if err != nil { + return fmt.Errorf("failed to execute message: %w", err) + } + carWriter = func(w io.Writer) error { + return g.WriteCAR(w, preroot, postroot) + } + + default: + return fmt.Errorf("unknown state retention option: %s", retention) + } + + msgBytes, err := msg.Serialize() + if err != nil { + return err + } + + var ( + out = new(bytes.Buffer) + gw = gzip.NewWriter(out) + ) + if err := carWriter(gw); err != nil { + return err + } + if err = gw.Flush(); err != nil { + return err + } + if err = gw.Close(); err != nil { + return err + } + + version, err := api.Version(ctx) + if err != nil { + return err + } + + ntwkName, err := api.StateNetworkName(ctx) + if err != nil { + return err + } + + // Write out the test vector. + vector := schema.TestVector{ + Class: schema.ClassMessage, + Meta: &schema.Metadata{ + ID: extractFlags.id, + Gen: []schema.GenerationData{ + {Source: fmt.Sprintf("message:%s:%s", ntwkName, msg.Cid().String())}, + {Source: "github.com/filecoin-project/lotus", Version: version.String()}}, + }, + CAR: out.Bytes(), + Pre: &schema.Preconditions{ + Epoch: int64(execTs.Height()), + StateTree: &schema.StateTree{ + RootCID: preroot, + }, + }, + ApplyMessages: []schema.Message{{Bytes: msgBytes}}, + Post: &schema.Postconditions{ + StateTree: &schema.StateTree{ + RootCID: postroot, + }, + Receipts: []*schema.Receipt{ + { + ExitCode: int64(applyret.ExitCode), + ReturnValue: applyret.Return, + GasUsed: applyret.GasUsed, + }, + }, + }, + } + + output := io.WriteCloser(os.Stdout) + if extractFlags.file != "" { + output, err = os.Create(extractFlags.file) + if err != nil { + return err + } + defer output.Close() + } + + enc := json.NewEncoder(output) + enc.SetIndent("", " ") + if err := enc.Encode(&vector); err != nil { + return err + } + + return nil +} + +// fetchThisAndPrevTipset returns the full tipset identified by the key, as well +// as the previous tipset. In the context of vector generation, the target +// tipset is the one where a message was executed, and the previous tipset is +// the one where the message was included. +func fetchThisAndPrevTipset(ctx context.Context, api api.FullNode, target types.TipSetKey) (targetTs *types.TipSet, prevTs *types.TipSet, err error) { + // get the tipset on which this message was "executed" on. + // https://github.com/filecoin-project/lotus/issues/2847 + targetTs, err = api.ChainGetTipSet(ctx, target) + if err != nil { + return nil, nil, err + } + // get the previous tipset, on which this message was mined, + // i.e. included on-chain. + prevTs, err = api.ChainGetTipSet(ctx, targetTs.Parents()) + if err != nil { + return nil, nil, err + } + return targetTs, prevTs, nil +} + +// findMsgAndPrecursors scans the messages in a block to locate the supplied +// message, looking into the BLS or SECP section depending on the sender's +// address type. +// +// It returns any precursors (if they exist), and the found message (if found), +// in a slice. +// +// It also returns a boolean indicating whether the message was actually found. +// +// This function also asserts invariants, and if those fail, it returns an error. +func findMsgAndPrecursors(messages *api.BlockMessages, target *types.Message) (related []*types.Message, found bool, err error) { + // Decide which block of messages to process, depending on whether the + // sender is a BLS or a SECP account. + input := messages.BlsMessages + if senderKind := target.From.Protocol(); senderKind == address.SECP256K1 { + input = make([]*types.Message, 0, len(messages.SecpkMessages)) + for _, sm := range messages.SecpkMessages { + input = append(input, &sm.Message) + } + } + + for _, other := range input { + if other.From != target.From { + continue + } + + // this message is from the same sender, so it's related. + related = append(related, other) + + if other.Nonce > target.Nonce { + return nil, false, fmt.Errorf("a message with nonce higher than the target was found before the target; offending mcid: %s", other.Cid()) + } + + // this message is the target; we're done. + if other.Cid() == target.Cid() { + return related, true, nil + } + } + + // this could happen because a block contained related messages, but not + // the target (that is, messages with a lower nonce, but ultimately not the + // target). + return related, false, nil +} diff --git a/cmd/tvx/main.go b/cmd/tvx/main.go new file mode 100644 index 000000000..183c2fbe1 --- /dev/null +++ b/cmd/tvx/main.go @@ -0,0 +1,92 @@ +package main + +import ( + "context" + "fmt" + "log" + "net/http" + "os" + "sort" + "strings" + + "github.com/filecoin-project/go-jsonrpc" + "github.com/multiformats/go-multiaddr" + manet "github.com/multiformats/go-multiaddr/net" + + "github.com/urfave/cli/v2" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/api/client" +) + +var apiEndpoint string + +var apiFlag = cli.StringFlag{ + Name: "api", + Usage: "json-rpc api endpoint, formatted as token:multiaddr", + EnvVars: []string{"FULLNODE_API_INFO"}, + DefaultText: "", + Destination: &apiEndpoint, +} + +func main() { + app := &cli.App{ + Name: "tvx", + Description: `tvx is a tool for extracting and executing test vectors. It has two subcommands. + + tvx extract extracts a test vector from a live network. It requires access to + a Filecoin client that exposes the standard JSON-RPC API endpoint. Set the API + endpoint on the FULLNODE_API_INFO env variable, or through the --api flag. The + format is token:multiaddr. Only message class test vectors are supported + for now. + + tvx exec executes test vectors against Lotus. Either you can supply one in a + file, or many as an ndjson stdin stream.`, + Usage: "tvx is a tool for extracting and executing test vectors", + Commands: []*cli.Command{ + extractCmd, + execCmd, + }, + } + + sort.Sort(cli.CommandsByName(app.Commands)) + for _, c := range app.Commands { + sort.Sort(cli.FlagsByName(c.Flags)) + } + + if err := app.Run(os.Args); err != nil { + log.Fatal(err) + } +} + +func makeAPIClient() (api.FullNode, jsonrpc.ClientCloser, error) { + sp := strings.SplitN(apiEndpoint, ":", 2) + if len(sp) != 2 { + return nil, nil, fmt.Errorf("invalid api value, missing token or address: %s", apiEndpoint) + } + + token := sp[0] + ma, err := multiaddr.NewMultiaddr(sp[1]) + if err != nil { + return nil, nil, fmt.Errorf("could not parse provided multiaddr: %w", err) + } + + _, dialAddr, err := manet.DialArgs(ma) + if err != nil { + return nil, nil, fmt.Errorf("invalid api multiAddr: %w", err) + } + + var ( + addr = "ws://" + dialAddr + "/rpc/v0" + headers = make(http.Header, 1) + ) + if len(token) != 0 { + headers.Add("Authorization", "Bearer "+token) + } + + node, closer, err := client.NewFullNodeRPC(context.Background(), addr, headers) + if err != nil { + return nil, nil, fmt.Errorf("could not connect to api: %w", err) + } + return node, closer, nil +} diff --git a/cmd/tvx/state.go b/cmd/tvx/state.go new file mode 100644 index 000000000..cef5c5494 --- /dev/null +++ b/cmd/tvx/state.go @@ -0,0 +1,293 @@ +package main + +import ( + "context" + "fmt" + "io" + "log" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-ipld-format" + "github.com/ipld/go-car" + cbg "github.com/whyrusleeping/cbor-gen" + + "github.com/filecoin-project/lotus/api" + init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init" + "github.com/filecoin-project/lotus/chain/state" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/vm" +) + +// StateSurgeon is an object used to fetch and manipulate state. +type StateSurgeon struct { + ctx context.Context + api api.FullNode + stores *Stores +} + +// NewSurgeon returns a state surgeon, an object used to fetch and manipulate +// state. +func NewSurgeon(ctx context.Context, api api.FullNode, stores *Stores) *StateSurgeon { + return &StateSurgeon{ + ctx: ctx, + api: api, + stores: stores, + } +} + +// GetMaskedStateTree trims the state tree at the supplied tipset to contain +// only the state of the actors in the retain set. It also "dives" into some +// singleton system actors, like the init actor, to trim the state so as to +// compute a minimal state tree. In the future, thid method will dive into +// other system actors like the power actor and the market actor. +func (sg *StateSurgeon) GetMaskedStateTree(previousRoot cid.Cid, retain []address.Address) (cid.Cid, error) { + // TODO: this will need to be parameterized on network version. + st, err := state.LoadStateTree(sg.stores.CBORStore, previousRoot) + if err != nil { + return cid.Undef, err + } + + initActor, initState, err := sg.loadInitActor(st) + if err != nil { + return cid.Undef, err + } + + err = sg.retainInitEntries(initState, retain) + if err != nil { + return cid.Undef, err + } + + err = sg.saveInitActor(initActor, initState, st) + if err != nil { + return cid.Undef, err + } + + // resolve all addresses to ID addresses. + resolved, err := sg.resolveAddresses(retain, initState) + if err != nil { + return cid.Undef, err + } + + st, err = sg.transplantActors(st, resolved) + if err != nil { + return cid.Undef, err + } + + root, err := st.Flush(sg.ctx) + if err != nil { + return cid.Undef, err + } + + return root, nil +} + +// GetAccessedActors identifies the actors that were accessed during the +// execution of a message. +func (sg *StateSurgeon) GetAccessedActors(ctx context.Context, a api.FullNode, mid cid.Cid) ([]address.Address, error) { + log.Printf("calculating accessed actors during execution of message: %s", mid) + msgInfo, err := a.StateSearchMsg(ctx, mid) + if err != nil { + return nil, err + } + if msgInfo == nil { + return nil, fmt.Errorf("message info is nil") + } + + msgObj, err := a.ChainGetMessage(ctx, mid) + if err != nil { + return nil, err + } + + ts, err := a.ChainGetTipSet(ctx, msgInfo.TipSet) + if err != nil { + return nil, err + } + + trace, err := a.StateCall(ctx, msgObj, ts.Parents()) + if err != nil { + return nil, fmt.Errorf("could not replay msg: %w", err) + } + + accessed := make(map[address.Address]struct{}) + + var recur func(trace *types.ExecutionTrace) + recur = func(trace *types.ExecutionTrace) { + accessed[trace.Msg.To] = struct{}{} + accessed[trace.Msg.From] = struct{}{} + for _, s := range trace.Subcalls { + recur(&s) + } + } + recur(&trace.ExecutionTrace) + + ret := make([]address.Address, 0, len(accessed)) + for k := range accessed { + ret = append(ret, k) + } + + return ret, nil +} + +// WriteCAR recursively writes the tree referenced by the root as a CAR into the +// supplied io.Writer. +func (sg *StateSurgeon) WriteCAR(w io.Writer, roots ...cid.Cid) error { + carWalkFn := func(nd format.Node) (out []*format.Link, err error) { + for _, link := range nd.Links() { + if link.Cid.Prefix().Codec == cid.FilCommitmentSealed || link.Cid.Prefix().Codec == cid.FilCommitmentUnsealed { + continue + } + out = append(out, link) + } + return out, nil + } + return car.WriteCarWithWalker(sg.ctx, sg.stores.DAGService, roots, w, carWalkFn) +} + +// WriteCARIncluding writes a CAR including only the CIDs that are listed in +// the include set. This leads to an intentially sparse tree with dangling links. +func (sg *StateSurgeon) WriteCARIncluding(w io.Writer, include map[cid.Cid]struct{}, roots ...cid.Cid) error { + carWalkFn := func(nd format.Node) (out []*format.Link, err error) { + for _, link := range nd.Links() { + if _, ok := include[link.Cid]; !ok { + continue + } + if link.Cid.Prefix().Codec == cid.FilCommitmentSealed || link.Cid.Prefix().Codec == cid.FilCommitmentUnsealed { + continue + } + out = append(out, link) + } + return out, nil + } + return car.WriteCarWithWalker(sg.ctx, sg.stores.DAGService, roots, w, carWalkFn) +} + +// transplantActors plucks the state from the supplied actors at the given +// tipset, and places it into the supplied state map. +func (sg *StateSurgeon) transplantActors(src *state.StateTree, pluck []address.Address) (*state.StateTree, error) { + log.Printf("transplanting actor states: %v", pluck) + + dst, err := state.NewStateTree(sg.stores.CBORStore, src.Version()) + if err != nil { + return nil, err + } + + for _, a := range pluck { + actor, err := src.GetActor(a) + if err != nil { + return nil, fmt.Errorf("get actor %s failed: %w", a, err) + } + + err = dst.SetActor(a, actor) + if err != nil { + return nil, err + } + + // recursive copy of the actor state. + err = vm.Copy(context.TODO(), sg.stores.Blockstore, sg.stores.Blockstore, actor.Head) + if err != nil { + return nil, err + } + + actorState, err := sg.api.ChainReadObj(sg.ctx, actor.Head) + if err != nil { + return nil, err + } + + cid, err := sg.stores.CBORStore.Put(sg.ctx, &cbg.Deferred{Raw: actorState}) + if err != nil { + return nil, err + } + + if cid != actor.Head { + panic("mismatched cids") + } + } + + return dst, nil +} + +// saveInitActor saves the state of the init actor to the provided state map. +func (sg *StateSurgeon) saveInitActor(initActor *types.Actor, initState init_.State, st *state.StateTree) error { + log.Printf("saving init actor into state tree") + + // Store the state of the init actor. + cid, err := sg.stores.CBORStore.Put(sg.ctx, initState) + if err != nil { + return err + } + actor := *initActor + actor.Head = cid + + err = st.SetActor(init_.Address, &actor) + if err != nil { + return err + } + + cid, _ = st.Flush(sg.ctx) + log.Printf("saved init actor into state tree; new root: %s", cid) + return nil +} + +// retainInitEntries takes an old init actor state, and retains only the +// entries in the retain set, returning a new init actor state. +func (sg *StateSurgeon) retainInitEntries(state init_.State, retain []address.Address) error { + log.Printf("retaining init actor entries for addresses: %v", retain) + + m := make(map[address.Address]struct{}, len(retain)) + for _, a := range retain { + m[a] = struct{}{} + } + + var remove []address.Address + _ = state.ForEachActor(func(id abi.ActorID, address address.Address) error { + if _, ok := m[address]; !ok { + remove = append(remove, address) + } + return nil + }) + + err := state.Remove(remove...) + log.Printf("new init actor state: %+v", state) + return err +} + +// resolveAddresses resolved the requested addresses from the provided +// InitActor state, returning a slice of length len(orig), where each index +// contains the resolved address. +func (sg *StateSurgeon) resolveAddresses(orig []address.Address, ist init_.State) (ret []address.Address, err error) { + log.Printf("resolving addresses: %v", orig) + + ret = make([]address.Address, len(orig)) + for i, addr := range orig { + resolved, found, err := ist.ResolveAddress(addr) + if err != nil { + return nil, err + } + if !found { + return nil, fmt.Errorf("address not found: %s", addr) + } + ret[i] = resolved + } + + log.Printf("resolved addresses: %v", ret) + return ret, nil +} + +// loadInitActor loads the init actor state from a given tipset. +func (sg *StateSurgeon) loadInitActor(st *state.StateTree) (*types.Actor, init_.State, error) { + actor, err := st.GetActor(init_.Address) + if err != nil { + return nil, nil, err + } + + initState, err := init_.Load(sg.stores.ADTStore, actor) + if err != nil { + return nil, nil, err + } + + log.Printf("loaded init actor state: %+v", initState) + + return actor, initState, nil +} diff --git a/cmd/tvx/stores.go b/cmd/tvx/stores.go new file mode 100644 index 000000000..7d3fd5e3a --- /dev/null +++ b/cmd/tvx/stores.go @@ -0,0 +1,143 @@ +package main + +import ( + "context" + "log" + "sync" + + "github.com/fatih/color" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/lib/blockstore" + + "github.com/filecoin-project/specs-actors/actors/util/adt" + + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-blockservice" + "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + exchange "github.com/ipfs/go-ipfs-exchange-interface" + offline "github.com/ipfs/go-ipfs-exchange-offline" + cbor "github.com/ipfs/go-ipld-cbor" + format "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-merkledag" +) + +// Stores is a collection of the different stores and services that are needed +// to deal with the data layer of Filecoin, conveniently interlinked with one +// another. +type Stores struct { + CBORStore cbor.IpldStore + ADTStore adt.Store + Datastore ds.Batching + Blockstore blockstore.Blockstore + BlockService blockservice.BlockService + Exchange exchange.Interface + DAGService format.DAGService +} + +// NewProxyingStores is a set of Stores backed by a proxying Blockstore that +// proxies Get requests for unknown CIDs to a Filecoin node, via the +// ChainReadObj RPC. +func NewProxyingStores(ctx context.Context, api api.FullNode) *Stores { + ds := ds.NewMapDatastore() + + bs := &proxyingBlockstore{ + ctx: ctx, + api: api, + Blockstore: blockstore.NewBlockstore(ds), + } + + return NewStores(ctx, ds, bs) +} + +// NewStores creates a non-proxying set of Stores. +func NewStores(ctx context.Context, ds ds.Batching, bs blockstore.Blockstore) *Stores { + var ( + cborstore = cbor.NewCborStore(bs) + offl = offline.Exchange(bs) + blkserv = blockservice.New(bs, offl) + dserv = merkledag.NewDAGService(blkserv) + ) + + return &Stores{ + CBORStore: cborstore, + ADTStore: adt.WrapStore(ctx, cborstore), + Datastore: ds, + Blockstore: bs, + Exchange: offl, + BlockService: blkserv, + DAGService: dserv, + } +} + +// TracingBlockstore is a Blockstore trait that records CIDs that were accessed +// through Get. +type TracingBlockstore interface { + // StartTracing starts tracing CIDs accessed through the this Blockstore. + StartTracing() + + // FinishTracing finishes tracing accessed CIDs, and returns a map of the + // CIDs that were traced. + FinishTracing() map[cid.Cid]struct{} +} + +// proxyingBlockstore is a Blockstore wrapper that fetches unknown CIDs from +// a Filecoin node via JSON-RPC. +type proxyingBlockstore struct { + ctx context.Context + api api.FullNode + + lk sync.RWMutex + tracing bool + traced map[cid.Cid]struct{} + + blockstore.Blockstore +} + +var _ TracingBlockstore = (*proxyingBlockstore)(nil) + +func (pb *proxyingBlockstore) StartTracing() { + pb.lk.Lock() + pb.tracing = true + pb.traced = map[cid.Cid]struct{}{} + pb.lk.Unlock() +} + +func (pb *proxyingBlockstore) FinishTracing() map[cid.Cid]struct{} { + pb.lk.Lock() + ret := pb.traced + pb.tracing = false + pb.traced = map[cid.Cid]struct{}{} + pb.lk.Unlock() + return ret +} + +func (pb *proxyingBlockstore) Get(cid cid.Cid) (blocks.Block, error) { + pb.lk.RLock() + if pb.tracing { + pb.traced[cid] = struct{}{} + } + pb.lk.RUnlock() + + if block, err := pb.Blockstore.Get(cid); err == nil { + return block, err + } + + log.Println(color.CyanString("fetching cid via rpc: %v", cid)) + item, err := pb.api.ChainReadObj(pb.ctx, cid) + if err != nil { + return nil, err + } + block, err := blocks.NewBlockWithCid(item, cid) + if err != nil { + return nil, err + } + + err = pb.Blockstore.Put(block) + if err != nil { + return nil, err + } + + return block, nil +} diff --git a/conformance/corpus_test.go b/conformance/corpus_test.go new file mode 100644 index 000000000..3d447570d --- /dev/null +++ b/conformance/corpus_test.go @@ -0,0 +1,133 @@ +package conformance + +import ( + "encoding/json" + "io/ioutil" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/filecoin-project/test-vectors/schema" +) + +const ( + // EnvSkipConformance, if 1, skips the conformance test suite. + EnvSkipConformance = "SKIP_CONFORMANCE" + + // EnvCorpusRootDir is the name of the environment variable where the path + // to an alternative corpus location can be provided. + // + // The default is defaultCorpusRoot. + EnvCorpusRootDir = "CORPUS_DIR" + + // defaultCorpusRoot is the directory where the test vector corpus is hosted. + // It is mounted on the Lotus repo as a git submodule. + // + // When running this test, the corpus root can be overridden through the + // -conformance.corpus CLI flag to run an alternate corpus. + defaultCorpusRoot = "../extern/test-vectors/corpus" +) + +// ignore is a set of paths relative to root to skip. +var ignore = map[string]struct{}{ + ".git": {}, + "schema.json": {}, +} + +// TestConformance is the entrypoint test that runs all test vectors found +// in the corpus root directory. +// +// It locates all json files via a recursive walk, skipping over the ignore set, +// as well as files beginning with _. It parses each file as a test vector, and +// runs it via the Driver. +func TestConformance(t *testing.T) { + if skip := strings.TrimSpace(os.Getenv(EnvSkipConformance)); skip == "1" { + t.SkipNow() + } + // corpusRoot is the effective corpus root path, taken from the `-conformance.corpus` CLI flag, + // falling back to defaultCorpusRoot if not provided. + corpusRoot := defaultCorpusRoot + if dir := strings.TrimSpace(os.Getenv(EnvCorpusRootDir)); dir != "" { + corpusRoot = dir + } + + var vectors []string + err := filepath.Walk(corpusRoot+"/", func(path string, info os.FileInfo, err error) error { + if err != nil { + t.Fatal(err) + } + + filename := filepath.Base(path) + rel, err := filepath.Rel(corpusRoot, path) + if err != nil { + t.Fatal(err) + } + + if _, ok := ignore[rel]; ok { + // skip over using the right error. + if info.IsDir() { + return filepath.SkipDir + } + return nil + } + if info.IsDir() { + // dive into directories. + return nil + } + if filepath.Ext(path) != ".json" { + // skip if not .json. + return nil + } + if ignored := strings.HasPrefix(filename, "_"); ignored { + // ignore files starting with _. + t.Logf("ignoring: %s", rel) + return nil + } + vectors = append(vectors, rel) + return nil + }) + + if err != nil { + t.Fatal(err) + } + + if len(vectors) == 0 { + t.Fatalf("no test vectors found") + } + + // Run a test for each vector. + for _, v := range vectors { + path := filepath.Join(corpusRoot, v) + raw, err := ioutil.ReadFile(path) + if err != nil { + t.Fatalf("failed to read test raw file: %s", path) + } + + var vector schema.TestVector + err = json.Unmarshal(raw, &vector) + if err != nil { + t.Errorf("failed to parse test vector %s: %s; skipping", path, err) + continue + } + + t.Run(v, func(t *testing.T) { + for _, h := range vector.Hints { + if h == schema.HintIncorrect { + t.Logf("skipping vector marked as incorrect: %s", vector.Meta.ID) + t.SkipNow() + } + } + + // dispatch the execution depending on the vector class. + switch vector.Class { + case "message": + ExecuteMessageVector(t, &vector) + case "tipset": + ExecuteTipsetVector(t, &vector) + default: + t.Fatalf("test vector class not supported: %s", vector.Class) + } + }) + } +} diff --git a/conformance/driver.go b/conformance/driver.go index f43a8739d..ee9727cae 100644 --- a/conformance/driver.go +++ b/conformance/driver.go @@ -5,6 +5,7 @@ import ( "github.com/filecoin-project/go-state-types/crypto" + "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" @@ -154,7 +155,12 @@ func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, preroot cid.Cid, epoch return nil, cid.Undef, err } - root, err := lvm.Flush(d.ctx) + // do not flush the VM, as this forces a recursive copy to the blockstore, + // walking the full state tree, which we don't require. + // root, err := lvm.Flush(d.ctx) + // + // instead, flush the pending writes on the state tree. + root, err := lvm.StateTree().(*state.StateTree).Flush(d.ctx) return ret, root, err } diff --git a/conformance/reporter.go b/conformance/reporter.go new file mode 100644 index 000000000..747caae32 --- /dev/null +++ b/conformance/reporter.go @@ -0,0 +1,62 @@ +package conformance + +import ( + "log" + "os" + "sync/atomic" + "testing" + + "github.com/fatih/color" +) + +// Reporter is a contains a subset of the testing.T methods, so that the +// Execute* functions in this package can be used inside or outside of +// go test runs. +type Reporter interface { + Helper() + + Log(args ...interface{}) + Errorf(format string, args ...interface{}) + Fatalf(format string, args ...interface{}) + Logf(format string, args ...interface{}) + FailNow() + Failed() bool +} + +var _ Reporter = (*testing.T)(nil) + +// LogReporter wires the Reporter methods to the log package. It is appropriate +// to use when calling the Execute* functions from a standalone CLI program. +type LogReporter struct { + failed int32 +} + +var _ Reporter = (*LogReporter)(nil) + +func (_ *LogReporter) Helper() {} + +func (_ *LogReporter) Log(args ...interface{}) { + log.Println(args...) +} + +func (_ *LogReporter) Logf(format string, args ...interface{}) { + log.Printf(format, args...) +} + +func (_ *LogReporter) FailNow() { + os.Exit(1) +} + +func (l *LogReporter) Failed() bool { + return atomic.LoadInt32(&l.failed) == 1 +} + +func (l *LogReporter) Errorf(format string, args ...interface{}) { + atomic.StoreInt32(&l.failed, 1) + log.Println(color.HiRedString("❌ "+format, args...)) +} + +func (l *LogReporter) Fatalf(format string, args ...interface{}) { + atomic.StoreInt32(&l.failed, 1) + log.Fatal(color.HiRedString("❌ "+format, args...)) +} diff --git a/conformance/runner.go b/conformance/runner.go new file mode 100644 index 000000000..1fc1c1425 --- /dev/null +++ b/conformance/runner.go @@ -0,0 +1,253 @@ +package conformance + +import ( + "bytes" + "compress/gzip" + "context" + "encoding/base64" + "fmt" + "io/ioutil" + "os" + "os/exec" + "strconv" + + "github.com/fatih/color" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/exitcode" + "github.com/filecoin-project/test-vectors/schema" + "github.com/ipfs/go-blockservice" + "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + offline "github.com/ipfs/go-ipfs-exchange-offline" + format "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-merkledag" + "github.com/ipld/go-car" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/vm" + "github.com/filecoin-project/lotus/lib/blockstore" +) + +// ExecuteMessageVector executes a message-class test vector. +func ExecuteMessageVector(r Reporter, vector *schema.TestVector) { + var ( + ctx = context.Background() + epoch = vector.Pre.Epoch + root = vector.Pre.StateTree.RootCID + ) + + // Load the CAR into a new temporary Blockstore. + bs, err := LoadVectorCAR(vector.CAR) + if err != nil { + r.Fatalf("failed to load the vector CAR: %w", err) + } + + // Create a new Driver. + driver := NewDriver(ctx, vector.Selector) + + // Apply every message. + for i, m := range vector.ApplyMessages { + msg, err := types.DecodeMessage(m.Bytes) + if err != nil { + r.Fatalf("failed to deserialize message: %s", err) + } + + // add an epoch if one's set. + if m.Epoch != nil { + epoch = *m.Epoch + } + + // Execute the message. + var ret *vm.ApplyRet + ret, root, err = driver.ExecuteMessage(bs, root, abi.ChainEpoch(epoch), msg) + if err != nil { + r.Fatalf("fatal failure when executing message: %s", err) + } + + // Assert that the receipt matches what the test vector expects. + assertMsgResult(r, vector.Post.Receipts[i], ret, strconv.Itoa(i)) + } + + // Once all messages are applied, assert that the final state root matches + // the expected postcondition root. + if expected, actual := vector.Post.StateTree.RootCID, root; expected != actual { + r.Errorf("wrong post root cid; expected %v, but got %v", expected, actual) + dumpThreeWayStateDiff(r, vector, bs, root) + r.FailNow() + } +} + +// ExecuteTipsetVector executes a tipset-class test vector. +func ExecuteTipsetVector(r Reporter, vector *schema.TestVector) { + var ( + ctx = context.Background() + prevEpoch = vector.Pre.Epoch + root = vector.Pre.StateTree.RootCID + tmpds = ds.NewMapDatastore() + ) + + // Load the vector CAR into a new temporary Blockstore. + bs, err := LoadVectorCAR(vector.CAR) + if err != nil { + r.Fatalf("failed to load the vector CAR: %w", err) + } + + // Create a new Driver. + driver := NewDriver(ctx, vector.Selector) + + // Apply every tipset. + var receiptsIdx int + for i, ts := range vector.ApplyTipsets { + ts := ts // capture + ret, err := driver.ExecuteTipset(bs, tmpds, root, abi.ChainEpoch(prevEpoch), &ts) + if err != nil { + r.Fatalf("failed to apply tipset %d message: %s", i, err) + } + + for j, v := range ret.AppliedResults { + assertMsgResult(r, vector.Post.Receipts[receiptsIdx], v, fmt.Sprintf("%d of tipset %d", j, i)) + receiptsIdx++ + } + + // Compare the receipts root. + if expected, actual := vector.Post.ReceiptsRoots[i], ret.ReceiptsRoot; expected != actual { + r.Errorf("post receipts root doesn't match; expected: %s, was: %s", expected, actual) + } + + prevEpoch = ts.Epoch + root = ret.PostStateRoot + } + + // Once all messages are applied, assert that the final state root matches + // the expected postcondition root. + if expected, actual := vector.Post.StateTree.RootCID, root; expected != actual { + r.Errorf("wrong post root cid; expected %v, but got %v", expected, actual) + dumpThreeWayStateDiff(r, vector, bs, root) + r.FailNow() + } +} + +// assertMsgResult compares a message result. It takes the expected receipt +// encoded in the vector, the actual receipt returned by Lotus, and a message +// label to log in the assertion failure message to facilitate debugging. +func assertMsgResult(r Reporter, expected *schema.Receipt, actual *vm.ApplyRet, label string) { + r.Helper() + + if expected, actual := exitcode.ExitCode(expected.ExitCode), actual.ExitCode; expected != actual { + r.Errorf("exit code of msg %s did not match; expected: %s, got: %s", label, expected, actual) + } + if expected, actual := expected.GasUsed, actual.GasUsed; expected != actual { + r.Errorf("gas used of msg %s did not match; expected: %d, got: %d", label, expected, actual) + } + if expected, actual := []byte(expected.ReturnValue), actual.Return; !bytes.Equal(expected, actual) { + r.Errorf("return value of msg %s did not match; expected: %s, got: %s", label, base64.StdEncoding.EncodeToString(expected), base64.StdEncoding.EncodeToString(actual)) + } +} + +func dumpThreeWayStateDiff(r Reporter, vector *schema.TestVector, bs blockstore.Blockstore, actual cid.Cid) { + // check if statediff exists; if not, skip. + if err := exec.Command("statediff", "--help").Run(); err != nil { + r.Log("could not dump 3-way state tree diff upon test failure: statediff command not found") + r.Log("install statediff with:") + r.Log("$ git clone https://github.com/filecoin-project/statediff.git") + r.Log("$ cd statediff") + r.Log("$ go generate ./...") + r.Log("$ go install ./cmd/statediff") + return + } + + tmpCar, err := writeStateToTempCAR(bs, + vector.Pre.StateTree.RootCID, + vector.Post.StateTree.RootCID, + actual, + ) + if err != nil { + r.Fatalf("failed to write temporary state CAR: %s", err) + } + defer os.RemoveAll(tmpCar) + + color.NoColor = false // enable colouring. + + var ( + a = color.New(color.FgMagenta, color.Bold).Sprint("(A) expected final state") + b = color.New(color.FgYellow, color.Bold).Sprint("(B) actual final state") + c = color.New(color.FgCyan, color.Bold).Sprint("(C) initial state") + d1 = color.New(color.FgGreen, color.Bold).Sprint("[Δ1]") + d2 = color.New(color.FgGreen, color.Bold).Sprint("[Δ2]") + d3 = color.New(color.FgGreen, color.Bold).Sprint("[Δ3]") + ) + + printDiff := func(left, right cid.Cid) { + cmd := exec.Command("statediff", "car", "--file", tmpCar, left.String(), right.String()) + b, err := cmd.CombinedOutput() + if err != nil { + r.Fatalf("statediff failed: %s", err) + } + r.Log(string(b)) + } + + bold := color.New(color.Bold).SprintfFunc() + + // run state diffs. + r.Log(bold("=== dumping 3-way diffs between %s, %s, %s ===", a, b, c)) + + r.Log(bold("--- %s left: %s; right: %s ---", d1, a, b)) + printDiff(vector.Post.StateTree.RootCID, actual) + + r.Log(bold("--- %s left: %s; right: %s ---", d2, c, b)) + printDiff(vector.Pre.StateTree.RootCID, actual) + + r.Log(bold("--- %s left: %s; right: %s ---", d3, c, a)) + printDiff(vector.Pre.StateTree.RootCID, vector.Post.StateTree.RootCID) +} + +// writeStateToTempCAR writes the provided roots to a temporary CAR that'll be +// cleaned up via t.Cleanup(). It returns the full path of the temp file. +func writeStateToTempCAR(bs blockstore.Blockstore, roots ...cid.Cid) (string, error) { + tmp, err := ioutil.TempFile("", "lotus-tests-*.car") + if err != nil { + return "", fmt.Errorf("failed to create temp file to dump CAR for diffing: %w", err) + } + + carWalkFn := func(nd format.Node) (out []*format.Link, err error) { + for _, link := range nd.Links() { + if link.Cid.Prefix().Codec == cid.FilCommitmentSealed || link.Cid.Prefix().Codec == cid.FilCommitmentUnsealed { + continue + } + out = append(out, link) + } + return out, nil + } + + var ( + offl = offline.Exchange(bs) + blkserv = blockservice.New(bs, offl) + dserv = merkledag.NewDAGService(blkserv) + ) + + err = car.WriteCarWithWalker(context.Background(), dserv, roots, tmp, carWalkFn) + if err != nil { + return "", fmt.Errorf("failed to dump CAR for diffing: %w", err) + } + _ = tmp.Close() + return tmp.Name(), nil +} + +func LoadVectorCAR(vectorCAR schema.Base64EncodedBytes) (blockstore.Blockstore, error) { + bs := blockstore.NewTemporary() + + // Read the base64-encoded CAR from the vector, and inflate the gzip. + buf := bytes.NewReader(vectorCAR) + r, err := gzip.NewReader(buf) + if err != nil { + return nil, fmt.Errorf("failed to inflate gzipped CAR: %s", err) + } + defer r.Close() // nolint + + // Load the CAR embedded in the test vector into the Blockstore. + _, err = car.LoadCar(bs, r) + if err != nil { + return nil, fmt.Errorf("failed to load state tree car from test vector: %s", err) + } + return bs, nil +} diff --git a/conformance/runner_test.go b/conformance/runner_test.go deleted file mode 100644 index cc7ef6b3d..000000000 --- a/conformance/runner_test.go +++ /dev/null @@ -1,376 +0,0 @@ -package conformance - -import ( - "bytes" - "compress/gzip" - "context" - "encoding/base64" - "encoding/json" - "fmt" - "io/ioutil" - "os" - "os/exec" - "path/filepath" - "strconv" - "strings" - "testing" - - "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/exitcode" - "github.com/ipfs/go-blockservice" - "github.com/ipfs/go-cid" - ds "github.com/ipfs/go-datastore" - offline "github.com/ipfs/go-ipfs-exchange-offline" - format "github.com/ipfs/go-ipld-format" - "github.com/ipfs/go-merkledag" - - "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/chain/vm" - "github.com/filecoin-project/lotus/lib/blockstore" - - "github.com/filecoin-project/test-vectors/schema" - - "github.com/fatih/color" - "github.com/ipld/go-car" -) - -const ( - // EnvSkipConformance, if 1, skips the conformance test suite. - EnvSkipConformance = "SKIP_CONFORMANCE" - - // EnvCorpusRootDir is the name of the environment variable where the path - // to an alternative corpus location can be provided. - // - // The default is defaultCorpusRoot. - EnvCorpusRootDir = "CORPUS_DIR" - - // defaultCorpusRoot is the directory where the test vector corpus is hosted. - // It is mounted on the Lotus repo as a git submodule. - // - // When running this test, the corpus root can be overridden through the - // -conformance.corpus CLI flag to run an alternate corpus. - defaultCorpusRoot = "../extern/test-vectors/corpus" -) - -// ignore is a set of paths relative to root to skip. -var ignore = map[string]struct{}{ - ".git": {}, - "schema.json": {}, -} - -// TestConformance is the entrypoint test that runs all test vectors found -// in the corpus root directory. -// -// It locates all json files via a recursive walk, skipping over the ignore set, -// as well as files beginning with _. It parses each file as a test vector, and -// runs it via the Driver. -func TestConformance(t *testing.T) { - if skip := strings.TrimSpace(os.Getenv(EnvSkipConformance)); skip == "1" { - t.SkipNow() - } - // corpusRoot is the effective corpus root path, taken from the `-conformance.corpus` CLI flag, - // falling back to defaultCorpusRoot if not provided. - corpusRoot := defaultCorpusRoot - if dir := strings.TrimSpace(os.Getenv(EnvCorpusRootDir)); dir != "" { - corpusRoot = dir - } - - var vectors []string - err := filepath.Walk(corpusRoot+"/", func(path string, info os.FileInfo, err error) error { - if err != nil { - t.Fatal(err) - } - - filename := filepath.Base(path) - rel, err := filepath.Rel(corpusRoot, path) - if err != nil { - t.Fatal(err) - } - - if _, ok := ignore[rel]; ok { - // skip over using the right error. - if info.IsDir() { - return filepath.SkipDir - } - return nil - } - if info.IsDir() { - // dive into directories. - return nil - } - if filepath.Ext(path) != ".json" { - // skip if not .json. - return nil - } - if ignored := strings.HasPrefix(filename, "_"); ignored { - // ignore files starting with _. - t.Logf("ignoring: %s", rel) - return nil - } - vectors = append(vectors, rel) - return nil - }) - - if err != nil { - t.Fatal(err) - } - - if len(vectors) == 0 { - t.Fatalf("no test vectors found") - } - - // Run a test for each vector. - for _, v := range vectors { - path := filepath.Join(corpusRoot, v) - raw, err := ioutil.ReadFile(path) - if err != nil { - t.Fatalf("failed to read test raw file: %s", path) - } - - var vector schema.TestVector - err = json.Unmarshal(raw, &vector) - if err != nil { - t.Errorf("failed to parse test vector %s: %s; skipping", path, err) - continue - } - - t.Run(v, func(t *testing.T) { - for _, h := range vector.Hints { - if h == schema.HintIncorrect { - t.Logf("skipping vector marked as incorrect: %s", vector.Meta.ID) - t.SkipNow() - } - } - - // dispatch the execution depending on the vector class. - switch vector.Class { - case "message": - executeMessageVector(t, &vector) - case "tipset": - executeTipsetVector(t, &vector) - default: - t.Fatalf("test vector class not supported: %s", vector.Class) - } - }) - } -} - -// executeMessageVector executes a message-class test vector. -func executeMessageVector(t *testing.T, vector *schema.TestVector) { - var ( - ctx = context.Background() - epoch = vector.Pre.Epoch - root = vector.Pre.StateTree.RootCID - ) - - // Load the CAR into a new temporary Blockstore. - bs := loadCAR(t, vector.CAR) - - // Create a new Driver. - driver := NewDriver(ctx, vector.Selector) - - // Apply every message. - for i, m := range vector.ApplyMessages { - msg, err := types.DecodeMessage(m.Bytes) - if err != nil { - t.Fatalf("failed to deserialize message: %s", err) - } - - // add an epoch if one's set. - if m.Epoch != nil { - epoch = *m.Epoch - } - - // Execute the message. - var ret *vm.ApplyRet - ret, root, err = driver.ExecuteMessage(bs, root, abi.ChainEpoch(epoch), msg) - if err != nil { - t.Fatalf("fatal failure when executing message: %s", err) - } - - // Assert that the receipt matches what the test vector expects. - assertMsgResult(t, vector.Post.Receipts[i], ret, strconv.Itoa(i)) - } - - // Once all messages are applied, assert that the final state root matches - // the expected postcondition root. - if expected, actual := vector.Post.StateTree.RootCID, root; expected != actual { - t.Logf("actual state root CID doesn't match expected one; expected: %s, actual: %s", expected, actual) - dumpThreeWayStateDiff(t, vector, bs, root) - t.FailNow() - } -} - -// executeTipsetVector executes a tipset-class test vector. -func executeTipsetVector(t *testing.T, vector *schema.TestVector) { - var ( - ctx = context.Background() - prevEpoch = vector.Pre.Epoch - root = vector.Pre.StateTree.RootCID - tmpds = ds.NewMapDatastore() - ) - - // Load the CAR into a new temporary Blockstore. - bs := loadCAR(t, vector.CAR) - - // Create a new Driver. - driver := NewDriver(ctx, vector.Selector) - - // Apply every tipset. - var receiptsIdx int - for i, ts := range vector.ApplyTipsets { - ts := ts // capture - ret, err := driver.ExecuteTipset(bs, tmpds, root, abi.ChainEpoch(prevEpoch), &ts) - if err != nil { - t.Fatalf("failed to apply tipset %d message: %s", i, err) - } - - for j, v := range ret.AppliedResults { - assertMsgResult(t, vector.Post.Receipts[receiptsIdx], v, fmt.Sprintf("%d of tipset %d", j, i)) - receiptsIdx++ - } - - // Compare the receipts root. - if expected, actual := vector.Post.ReceiptsRoots[i], ret.ReceiptsRoot; expected != actual { - t.Errorf("post receipts root doesn't match; expected: %s, was: %s", expected, actual) - } - - prevEpoch = ts.Epoch - root = ret.PostStateRoot - } - - // Once all messages are applied, assert that the final state root matches - // the expected postcondition root. - if expected, actual := vector.Post.StateTree.RootCID, root; expected != actual { - t.Logf("actual state root CID doesn't match expected one; expected: %s, actual: %s", expected, actual) - dumpThreeWayStateDiff(t, vector, bs, root) - t.FailNow() - } -} - -// assertMsgResult compares a message result. It takes the expected receipt -// encoded in the vector, the actual receipt returned by Lotus, and a message -// label to log in the assertion failure message to facilitate debugging. -func assertMsgResult(t *testing.T, expected *schema.Receipt, actual *vm.ApplyRet, label string) { - t.Helper() - - if expected, actual := exitcode.ExitCode(expected.ExitCode), actual.ExitCode; expected != actual { - t.Errorf("exit code of msg %s did not match; expected: %s, got: %s", label, expected, actual) - } - if expected, actual := expected.GasUsed, actual.GasUsed; expected != actual { - t.Errorf("gas used of msg %s did not match; expected: %d, got: %d", label, expected, actual) - } - if expected, actual := []byte(expected.ReturnValue), actual.Return; !bytes.Equal(expected, actual) { - t.Errorf("return value of msg %s did not match; expected: %s, got: %s", label, base64.StdEncoding.EncodeToString(expected), base64.StdEncoding.EncodeToString(actual)) - } -} - -func dumpThreeWayStateDiff(t *testing.T, vector *schema.TestVector, bs blockstore.Blockstore, actual cid.Cid) { - // check if statediff exists; if not, skip. - if err := exec.Command("statediff", "--help").Run(); err != nil { - t.Log("could not dump 3-way state tree diff upon test failure: statediff command not found") - t.Log("install statediff with:") - t.Log("$ git clone https://github.com/filecoin-project/statediff.git") - t.Log("$ cd statediff") - t.Log("$ go generate ./...") - t.Log("$ go install ./cmd/statediff") - return - } - - tmpCar := writeStateToTempCAR(t, bs, - vector.Pre.StateTree.RootCID, - vector.Post.StateTree.RootCID, - actual, - ) - - color.NoColor = false // enable colouring. - - t.Errorf("wrong post root cid; expected %v, but got %v", vector.Post.StateTree.RootCID, actual) - - var ( - a = color.New(color.FgMagenta, color.Bold).Sprint("(A) expected final state") - b = color.New(color.FgYellow, color.Bold).Sprint("(B) actual final state") - c = color.New(color.FgCyan, color.Bold).Sprint("(C) initial state") - d1 = color.New(color.FgGreen, color.Bold).Sprint("[Δ1]") - d2 = color.New(color.FgGreen, color.Bold).Sprint("[Δ2]") - d3 = color.New(color.FgGreen, color.Bold).Sprint("[Δ3]") - ) - - printDiff := func(left, right cid.Cid) { - cmd := exec.Command("statediff", "car", "--file", tmpCar, left.String(), right.String()) - b, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("statediff failed: %s", err) - } - t.Log(string(b)) - } - - bold := color.New(color.Bold).SprintfFunc() - - // run state diffs. - t.Log(bold("=== dumping 3-way diffs between %s, %s, %s ===", a, b, c)) - - t.Log(bold("--- %s left: %s; right: %s ---", d1, a, b)) - printDiff(vector.Post.StateTree.RootCID, actual) - - t.Log(bold("--- %s left: %s; right: %s ---", d2, c, b)) - printDiff(vector.Pre.StateTree.RootCID, actual) - - t.Log(bold("--- %s left: %s; right: %s ---", d3, c, a)) - printDiff(vector.Pre.StateTree.RootCID, vector.Post.StateTree.RootCID) -} - -// writeStateToTempCAR writes the provided roots to a temporary CAR that'll be -// cleaned up via t.Cleanup(). It returns the full path of the temp file. -func writeStateToTempCAR(t *testing.T, bs blockstore.Blockstore, roots ...cid.Cid) string { - tmp, err := ioutil.TempFile("", "lotus-tests-*.car") - if err != nil { - t.Fatalf("failed to create temp file to dump CAR for diffing: %s", err) - } - // register a cleanup function to delete the CAR. - t.Cleanup(func() { - _ = os.Remove(tmp.Name()) - }) - - carWalkFn := func(nd format.Node) (out []*format.Link, err error) { - for _, link := range nd.Links() { - if link.Cid.Prefix().Codec == cid.FilCommitmentSealed || link.Cid.Prefix().Codec == cid.FilCommitmentUnsealed { - continue - } - out = append(out, link) - } - return out, nil - } - - var ( - offl = offline.Exchange(bs) - blkserv = blockservice.New(bs, offl) - dserv = merkledag.NewDAGService(blkserv) - ) - - err = car.WriteCarWithWalker(context.Background(), dserv, roots, tmp, carWalkFn) - if err != nil { - t.Fatalf("failed to dump CAR for diffing: %s", err) - } - _ = tmp.Close() - return tmp.Name() -} - -func loadCAR(t *testing.T, vectorCAR schema.Base64EncodedBytes) blockstore.Blockstore { - bs := blockstore.NewTemporary() - - // Read the base64-encoded CAR from the vector, and inflate the gzip. - buf := bytes.NewReader(vectorCAR) - r, err := gzip.NewReader(buf) - if err != nil { - t.Fatalf("failed to inflate gzipped CAR: %s", err) - } - defer r.Close() // nolint - - // Load the CAR embedded in the test vector into the Blockstore. - _, err = car.LoadCar(bs, r) - if err != nil { - t.Fatalf("failed to load state tree car from test vector: %s", err) - } - return bs -} diff --git a/go.sum b/go.sum index 6412fe743..bdb80aa83 100644 --- a/go.sum +++ b/go.sum @@ -504,6 +504,7 @@ github.com/ipfs/go-fs-lock v0.0.6/go.mod h1:OTR+Rj9sHiRubJh3dRhD15Juhd/+w6VPOY28 github.com/ipfs/go-graphsync v0.1.0/go.mod h1:jMXfqIEDFukLPZHqDPp8tJMbHO9Rmeb9CEGevngQbmE= github.com/ipfs/go-graphsync v0.2.1 h1:MdehhqBSuTI2LARfKLkpYnt0mUrqHs/mtuDnESXHBfU= github.com/ipfs/go-graphsync v0.2.1/go.mod h1:gEBvJUNelzMkaRPJTpg/jaKN4AQW/7wDWu0K92D8o10= +github.com/ipfs/go-hamt-ipld v0.1.1 h1:0IQdvwnAAUKmDE+PMJa5y1QiwOPHpI9+eAbQEEEYthk= github.com/ipfs/go-hamt-ipld v0.1.1/go.mod h1:1EZCr2v0jlCnhpa+aZ0JZYp8Tt2w16+JJOAVz17YcDk= github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw= From a712c109d804c09988d69a741a10b17fd2312f2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Sun, 27 Sep 2020 20:30:32 +0100 Subject: [PATCH 372/885] tvx/extract: print confirmation. --- cmd/tvx/extract.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cmd/tvx/extract.go b/cmd/tvx/extract.go index 81fd3efbf..62314c6f6 100644 --- a/cmd/tvx/extract.go +++ b/cmd/tvx/extract.go @@ -48,7 +48,7 @@ var extractCmd = &cli.Command{ &cli.StringFlag{ Name: "id", Usage: "identifier to name this test vector with", - Value: "", + Value: "(undefined)", Destination: &extractFlags.id, }, &cli.StringFlag{ @@ -297,12 +297,13 @@ func runExtract(_ *cli.Context) error { } output := io.WriteCloser(os.Stdout) - if extractFlags.file != "" { - output, err = os.Create(extractFlags.file) + if file := extractFlags.file; file != "" { + output, err = os.Create(file) if err != nil { return err } defer output.Close() + defer log.Printf("wrote test vector to file: %s", file) } enc := json.NewEncoder(output) From f5f23f7291ab83c021e86164d775c896d261ea52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Sun, 27 Sep 2020 20:55:09 +0100 Subject: [PATCH 373/885] driver: option for VM flushing. --- cmd/tvx/extract.go | 4 +++- conformance/driver.go | 35 +++++++++++++++++++++++++++-------- conformance/runner.go | 4 ++-- 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/cmd/tvx/extract.go b/cmd/tvx/extract.go index 62314c6f6..06b3d6fe8 100644 --- a/cmd/tvx/extract.go +++ b/cmd/tvx/extract.go @@ -164,7 +164,9 @@ func runExtract(_ *cli.Context) error { g = NewSurgeon(ctx, api, pst) ) - driver := conformance.NewDriver(ctx, schema.Selector{}) + driver := conformance.NewDriver(ctx, schema.Selector{}, conformance.DriverOpts{ + DisableVMFlush: true, + }) // this is the root of the state tree we start with. root := incTs.ParentState() diff --git a/conformance/driver.go b/conformance/driver.go index ee9727cae..fdd4fe3a9 100644 --- a/conformance/driver.go +++ b/conformance/driver.go @@ -33,10 +33,24 @@ var ( type Driver struct { ctx context.Context selector schema.Selector + vmFlush bool } -func NewDriver(ctx context.Context, selector schema.Selector) *Driver { - return &Driver{ctx: ctx, selector: selector} +type DriverOpts struct { + // DisableVMFlush, when true, avoids calling VM.Flush(), forces a blockstore + // recursive copy, from the temporary buffer blockstore, to the real + // system's blockstore. Disabling VM flushing is useful when extracting test + // vectors and trimming state, as we don't want to force an accidental + // deep copy of the state tree. + // + // Disabling VM flushing almost always should go hand-in-hand with + // LOTUS_DISABLE_VM_BUF=iknowitsabadidea. That way, state tree writes are + // immediately committed to the blockstore. + DisableVMFlush bool +} + +func NewDriver(ctx context.Context, selector schema.Selector, opts DriverOpts) *Driver { + return &Driver{ctx: ctx, selector: selector, vmFlush: !opts.DisableVMFlush} } type ExecuteTipsetResult struct { @@ -155,12 +169,17 @@ func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, preroot cid.Cid, epoch return nil, cid.Undef, err } - // do not flush the VM, as this forces a recursive copy to the blockstore, - // walking the full state tree, which we don't require. - // root, err := lvm.Flush(d.ctx) - // - // instead, flush the pending writes on the state tree. - root, err := lvm.StateTree().(*state.StateTree).Flush(d.ctx) + var root cid.Cid + if d.vmFlush { + // flush the VM, committing the state tree changes and forcing a + // recursive copoy from the temporary blcokstore to the real blockstore. + root, err = lvm.Flush(d.ctx) + } else { + // do not flush the VM, just the state tree; this should be used with + // LOTUS_DISABLE_VM_BUF enabled, so writes will anyway be visible. + root, err = lvm.StateTree().(*state.StateTree).Flush(d.ctx) + } + return ret, root, err } diff --git a/conformance/runner.go b/conformance/runner.go index 1fc1c1425..6ffdcd2eb 100644 --- a/conformance/runner.go +++ b/conformance/runner.go @@ -43,7 +43,7 @@ func ExecuteMessageVector(r Reporter, vector *schema.TestVector) { } // Create a new Driver. - driver := NewDriver(ctx, vector.Selector) + driver := NewDriver(ctx, vector.Selector, DriverOpts{}) // Apply every message. for i, m := range vector.ApplyMessages { @@ -93,7 +93,7 @@ func ExecuteTipsetVector(r Reporter, vector *schema.TestVector) { } // Create a new Driver. - driver := NewDriver(ctx, vector.Selector) + driver := NewDriver(ctx, vector.Selector, DriverOpts{}) // Apply every tipset. var receiptsIdx int From fe869c9c223e148b44efddca794a570145c1c3fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Sun, 27 Sep 2020 21:06:07 +0100 Subject: [PATCH 374/885] address review comments; lint. --- cmd/tvx/extract.go | 2 +- cmd/tvx/main.go | 4 +++- cmd/tvx/state.go | 2 +- cmd/tvx/stores.go | 4 +++- conformance/driver.go | 2 +- conformance/reporter.go | 8 ++++---- conformance/runner.go | 2 +- 7 files changed, 14 insertions(+), 10 deletions(-) diff --git a/cmd/tvx/extract.go b/cmd/tvx/extract.go index 06b3d6fe8..d3faa6d6d 100644 --- a/cmd/tvx/extract.go +++ b/cmd/tvx/extract.go @@ -304,7 +304,7 @@ func runExtract(_ *cli.Context) error { if err != nil { return err } - defer output.Close() + defer output.Close() //nolint:errcheck defer log.Printf("wrote test vector to file: %s", file) } diff --git a/cmd/tvx/main.go b/cmd/tvx/main.go index 183c2fbe1..2ab217953 100644 --- a/cmd/tvx/main.go +++ b/cmd/tvx/main.go @@ -23,7 +23,9 @@ var apiEndpoint string var apiFlag = cli.StringFlag{ Name: "api", - Usage: "json-rpc api endpoint, formatted as token:multiaddr", + Usage: "json-rpc api endpoint, formatted as [token]:multiaddr;" + + "tvx uses unpriviliged operations, so the token may be omitted," + + "but permissions may change in the future", EnvVars: []string{"FULLNODE_API_INFO"}, DefaultText: "", Destination: &apiEndpoint, diff --git a/cmd/tvx/state.go b/cmd/tvx/state.go index cef5c5494..5bf1cf153 100644 --- a/cmd/tvx/state.go +++ b/cmd/tvx/state.go @@ -9,7 +9,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/ipfs/go-cid" - "github.com/ipfs/go-ipld-format" + format "github.com/ipfs/go-ipld-format" "github.com/ipld/go-car" cbg "github.com/whyrusleeping/cbor-gen" diff --git a/cmd/tvx/stores.go b/cmd/tvx/stores.go index 7d3fd5e3a..c389f8c88 100644 --- a/cmd/tvx/stores.go +++ b/cmd/tvx/stores.go @@ -6,6 +6,7 @@ import ( "sync" "github.com/fatih/color" + dssync "github.com/ipfs/go-datastore/sync" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/lib/blockstore" @@ -40,7 +41,8 @@ type Stores struct { // proxies Get requests for unknown CIDs to a Filecoin node, via the // ChainReadObj RPC. func NewProxyingStores(ctx context.Context, api api.FullNode) *Stores { - ds := ds.NewMapDatastore() + ds := dssync.MutexWrap(ds.NewMapDatastore()) + ds = dssync.MutexWrap(ds) bs := &proxyingBlockstore{ ctx: ctx, diff --git a/conformance/driver.go b/conformance/driver.go index fdd4fe3a9..90d05ae88 100644 --- a/conformance/driver.go +++ b/conformance/driver.go @@ -33,7 +33,7 @@ var ( type Driver struct { ctx context.Context selector schema.Selector - vmFlush bool + vmFlush bool } type DriverOpts struct { diff --git a/conformance/reporter.go b/conformance/reporter.go index 747caae32..1cd2d389d 100644 --- a/conformance/reporter.go +++ b/conformance/reporter.go @@ -33,17 +33,17 @@ type LogReporter struct { var _ Reporter = (*LogReporter)(nil) -func (_ *LogReporter) Helper() {} +func (*LogReporter) Helper() {} -func (_ *LogReporter) Log(args ...interface{}) { +func (*LogReporter) Log(args ...interface{}) { log.Println(args...) } -func (_ *LogReporter) Logf(format string, args ...interface{}) { +func (*LogReporter) Logf(format string, args ...interface{}) { log.Printf(format, args...) } -func (_ *LogReporter) FailNow() { +func (*LogReporter) FailNow() { os.Exit(1) } diff --git a/conformance/runner.go b/conformance/runner.go index 6ffdcd2eb..456955b25 100644 --- a/conformance/runner.go +++ b/conformance/runner.go @@ -164,7 +164,7 @@ func dumpThreeWayStateDiff(r Reporter, vector *schema.TestVector, bs blockstore. if err != nil { r.Fatalf("failed to write temporary state CAR: %s", err) } - defer os.RemoveAll(tmpCar) + defer os.RemoveAll(tmpCar) //nolint:errcheck color.NoColor = false // enable colouring. From 9a355c4bc5b10bd24c55f819e2a32ae882312dcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Sun, 27 Sep 2020 21:11:32 +0100 Subject: [PATCH 375/885] fix lint errors. --- cmd/tvx/main.go | 4 ++-- cmd/tvx/state.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/tvx/main.go b/cmd/tvx/main.go index 2ab217953..6d5c64101 100644 --- a/cmd/tvx/main.go +++ b/cmd/tvx/main.go @@ -22,8 +22,8 @@ import ( var apiEndpoint string var apiFlag = cli.StringFlag{ - Name: "api", - Usage: "json-rpc api endpoint, formatted as [token]:multiaddr;" + + Name: "api", + Usage: "json-rpc api endpoint, formatted as [token]:multiaddr;" + "tvx uses unpriviliged operations, so the token may be omitted," + "but permissions may change in the future", EnvVars: []string{"FULLNODE_API_INFO"}, diff --git a/cmd/tvx/state.go b/cmd/tvx/state.go index 5bf1cf153..bff5cbd6e 100644 --- a/cmd/tvx/state.go +++ b/cmd/tvx/state.go @@ -116,8 +116,8 @@ func (sg *StateSurgeon) GetAccessedActors(ctx context.Context, a api.FullNode, m recur = func(trace *types.ExecutionTrace) { accessed[trace.Msg.To] = struct{}{} accessed[trace.Msg.From] = struct{}{} - for _, s := range trace.Subcalls { - recur(&s) + for i := range trace.Subcalls { + recur(&trace.Subcalls[i]) } } recur(&trace.ExecutionTrace) From a2d24b5b33a18418edc63bea9518e9dbed72bfef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=9E=97=E6=AC=A3?= Date: Mon, 28 Sep 2020 13:53:29 +0800 Subject: [PATCH 376/885] add ipfsmain bootstrapper --- build/bootstrap/bootstrappers.pi | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build/bootstrap/bootstrappers.pi b/build/bootstrap/bootstrappers.pi index 465f3b5e9..51adb3cc7 100644 --- a/build/bootstrap/bootstrappers.pi +++ b/build/bootstrap/bootstrappers.pi @@ -4,3 +4,6 @@ /dns4/bootstrap-4.testnet.fildev.network/tcp/1347/p2p/12D3KooWPkL9LrKRQgHtq7kn9ecNhGU9QaziG8R5tX8v9v7t3h34 /dns4/bootstrap-3.testnet.fildev.network/tcp/1347/p2p/12D3KooWKYSsbpgZ3HAjax5M1BXCwXLa6gVkUARciz7uN3FNtr7T /dns4/bootstrap-5.testnet.fildev.network/tcp/1347/p2p/12D3KooWQYzqnLASJAabyMpPb1GcWZvNSe7JDcRuhdRqonFoiK9W + +/dns4/bootstrap-0.ipfsmain.cn/tcp/34721/p2p/12D3KooWQnwEGNqcM2nAcPtRR9rAX8Hrg4k9kJLCHoTR5chJfz6d +/dns4/bootstrap-1.ipfsmain.cn/tcp/34723/p2p/12D3KooWMKxMkD5DMpSWsW7dBddKxKT7L2GgbNuckz9otxvkvByP \ No newline at end of file From a3145bae07f3658183d95d9a6e56d2f3ae38ce76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 28 Sep 2020 12:17:54 +0200 Subject: [PATCH 377/885] daemon cmd: Add progress bar to chain import --- cmd/lotus/daemon.go | 18 +++++++++++++++++- go.mod | 1 + 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index e0fee6564..93f6e4b8c 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -23,6 +23,7 @@ import ( "go.opencensus.io/stats/view" "go.opencensus.io/tag" "golang.org/x/xerrors" + "gopkg.in/cheggaaa/pb.v1" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" @@ -333,6 +334,11 @@ func ImportChain(r repo.Repo, fname string, snapshot bool) error { } defer fi.Close() //nolint:errcheck + st, err := os.Stat(fname) + if err != nil { + return err + } + lr, err := r.Lock(repo.FullNode) if err != nil { return err @@ -354,7 +360,17 @@ func ImportChain(r repo.Repo, fname string, snapshot bool) error { cst := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier)) log.Info("importing chain from file...") - ts, err := cst.Import(fi) + + bar := pb.New64(st.Size()) + br := bar.NewProxyReader(fi) + bar.ShowTimeLeft = true + bar.ShowPercent = true + bar.Units = pb.U_BYTES + + bar.Start() + ts, err := cst.Import(br) + bar.Finish() + if err != nil { return xerrors.Errorf("importing chain failed: %w", err) } diff --git a/go.mod b/go.mod index 67d87347f..2c0322ecc 100644 --- a/go.mod +++ b/go.mod @@ -125,6 +125,7 @@ require ( golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 golang.org/x/time v0.0.0-20191024005414-555d28b269f0 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 + gopkg.in/cheggaaa/pb.v1 v1.0.28 gotest.tools v2.2.0+incompatible launchpad.net/gocheck v0.0.0-20140225173054-000000000087 // indirect ) From f8095296997f6d5d8ba05d66acf291cfa048ae50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 28 Sep 2020 12:28:12 +0200 Subject: [PATCH 378/885] daemon cmd: Support imports straight from http --- cmd/lotus/daemon.go | 64 +++++++++++++++++++++++++++++++-------------- 1 file changed, 45 insertions(+), 19 deletions(-) diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 93f6e4b8c..b976fde79 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -3,11 +3,14 @@ package main import ( + "bufio" "context" "encoding/hex" "encoding/json" "fmt" + "io" "io/ioutil" + "net/http" "os" "runtime/pprof" "strings" @@ -101,11 +104,11 @@ var DaemonCmd = &cli.Command{ }, &cli.StringFlag{ Name: "import-chain", - Usage: "on first run, load chain from given file and validate", + Usage: "on first run, load chain from given file or url and validate", }, &cli.StringFlag{ Name: "import-snapshot", - Usage: "import chain state from a given chain export file", + Usage: "import chain state from a given chain export file or url", }, &cli.BoolFlag{ Name: "halt-after-import", @@ -207,11 +210,6 @@ var DaemonCmd = &cli.Command{ issnapshot = true } - chainfile, err := homedir.Expand(chainfile) - if err != nil { - return err - } - if err := ImportChain(r, chainfile, issnapshot); err != nil { return err } @@ -327,16 +325,41 @@ func importKey(ctx context.Context, api api.FullNode, f string) error { return nil } -func ImportChain(r repo.Repo, fname string, snapshot bool) error { - fi, err := os.Open(fname) - if err != nil { - return err - } - defer fi.Close() //nolint:errcheck +func ImportChain(r repo.Repo, fname string, snapshot bool) (err error) { + var rd io.Reader + var l int64 + if strings.HasPrefix(fname, "http://") || strings.HasPrefix(fname, "https://") { + resp, err := http.Get(fname) //nolint:gosec + if err != nil { + return err + } + defer resp.Body.Close() //nolint:errcheck - st, err := os.Stat(fname) - if err != nil { - return err + if resp.StatusCode != http.StatusOK { + return xerrors.Errorf("non-200 response: %d", resp.StatusCode) + } + + rd = resp.Body + l = resp.ContentLength + } else { + fname, err = homedir.Expand(fname) + if err != nil { + return err + } + + fi, err := os.Open(fname) + if err != nil { + return err + } + defer fi.Close() //nolint:errcheck + + st, err := os.Stat(fname) + if err != nil { + return err + } + + rd = fi + l = st.Size() } lr, err := r.Lock(repo.FullNode) @@ -359,12 +382,15 @@ func ImportChain(r repo.Repo, fname string, snapshot bool) error { cst := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier)) - log.Info("importing chain from file...") + log.Infof("importing chain from %s...", fname) - bar := pb.New64(st.Size()) - br := bar.NewProxyReader(fi) + bufr := bufio.NewReaderSize(rd, 1<<20) + + bar := pb.New64(l) + br := bar.NewProxyReader(bufr) bar.ShowTimeLeft = true bar.ShowPercent = true + bar.ShowSpeed = true bar.Units = pb.U_BYTES bar.Start() From f05a40feed0ebd475ad5845ad751c449a01fb199 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Mon, 28 Sep 2020 12:27:42 +0100 Subject: [PATCH 379/885] use lotus CLI package; document API endpoint setting in usage. --- cmd/tvx/extract.go | 6 ++-- cmd/tvx/main.go | 80 ++++++++++++---------------------------------- 2 files changed, 23 insertions(+), 63 deletions(-) diff --git a/cmd/tvx/extract.go b/cmd/tvx/extract.go index d3faa6d6d..d8053e5c2 100644 --- a/cmd/tvx/extract.go +++ b/cmd/tvx/extract.go @@ -15,6 +15,7 @@ import ( "github.com/filecoin-project/lotus/chain/actors/builtin/reward" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" + lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/conformance" "github.com/filecoin-project/go-address" @@ -38,7 +39,6 @@ var extractCmd = &cli.Command{ Description: "generate a message-class test vector by extracting it from a live chain", Action: runExtract, Flags: []cli.Flag{ - &apiFlag, &cli.StringFlag{ Name: "class", Usage: "class of vector to extract; other required flags depend on the; values: 'message'", @@ -72,7 +72,7 @@ var extractCmd = &cli.Command{ }, } -func runExtract(_ *cli.Context) error { +func runExtract(c *cli.Context) error { // LOTUS_DISABLE_VM_BUF disables what's called "VM state tree buffering", // which stashes write operations in a BufferedBlockstore // (https://github.com/filecoin-project/lotus/blob/b7a4dbb07fd8332b4492313a617e3458f8003b2a/lib/bufbstore/buf_bstore.go#L21) @@ -91,7 +91,7 @@ func runExtract(_ *cli.Context) error { } // Make the API client. - api, closer, err := makeAPIClient() + api, closer, err := lcli.GetFullNodeAPI(c) if err != nil { return err } diff --git a/cmd/tvx/main.go b/cmd/tvx/main.go index 6d5c64101..a74a029ce 100644 --- a/cmd/tvx/main.go +++ b/cmd/tvx/main.go @@ -1,49 +1,41 @@ package main import ( - "context" - "fmt" "log" - "net/http" "os" "sort" - "strings" - - "github.com/filecoin-project/go-jsonrpc" - "github.com/multiformats/go-multiaddr" - manet "github.com/multiformats/go-multiaddr/net" "github.com/urfave/cli/v2" - - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/api/client" ) -var apiEndpoint string - -var apiFlag = cli.StringFlag{ - Name: "api", - Usage: "json-rpc api endpoint, formatted as [token]:multiaddr;" + - "tvx uses unpriviliged operations, so the token may be omitted," + - "but permissions may change in the future", - EnvVars: []string{"FULLNODE_API_INFO"}, - DefaultText: "", - Destination: &apiEndpoint, -} - func main() { app := &cli.App{ Name: "tvx", Description: `tvx is a tool for extracting and executing test vectors. It has two subcommands. tvx extract extracts a test vector from a live network. It requires access to - a Filecoin client that exposes the standard JSON-RPC API endpoint. Set the API - endpoint on the FULLNODE_API_INFO env variable, or through the --api flag. The - format is token:multiaddr. Only message class test vectors are supported - for now. + a Filecoin client that exposes the standard JSON-RPC API endpoint. Only + message class test vectors are supported at this time. tvx exec executes test vectors against Lotus. Either you can supply one in a - file, or many as an ndjson stdin stream.`, + file, or many as an ndjson stdin stream. + + SETTING THE JSON-RPC API ENDPOINT + + You can set the JSON-RPC API endpoint through one of the following approaches. + + 1. Directly set the API endpoint on the FULLNODE_API_INFO env variable. + The format is [token]:multiaddr, where token is optional for commands not + accessing privileged operations. + + 2. If you're running tvx against a local Lotus client, you can set the REPO + env variable to have the API endpoint and token extracted from the repo. + + 3. Rely on the default fallback, which inspects ~/.lotus and extracts the + API endpoint string if the location is a Lotus repo. + + tvx will apply these approaches in the same order of precedence they're listed. +`, Usage: "tvx is a tool for extracting and executing test vectors", Commands: []*cli.Command{ extractCmd, @@ -60,35 +52,3 @@ func main() { log.Fatal(err) } } - -func makeAPIClient() (api.FullNode, jsonrpc.ClientCloser, error) { - sp := strings.SplitN(apiEndpoint, ":", 2) - if len(sp) != 2 { - return nil, nil, fmt.Errorf("invalid api value, missing token or address: %s", apiEndpoint) - } - - token := sp[0] - ma, err := multiaddr.NewMultiaddr(sp[1]) - if err != nil { - return nil, nil, fmt.Errorf("could not parse provided multiaddr: %w", err) - } - - _, dialAddr, err := manet.DialArgs(ma) - if err != nil { - return nil, nil, fmt.Errorf("invalid api multiAddr: %w", err) - } - - var ( - addr = "ws://" + dialAddr + "/rpc/v0" - headers = make(http.Header, 1) - ) - if len(token) != 0 { - headers.Add("Authorization", "Bearer "+token) - } - - node, closer, err := client.NewFullNodeRPC(context.Background(), addr, headers) - if err != nil { - return nil, nil, fmt.Errorf("could not connect to api: %w", err) - } - return node, closer, nil -} From dfdcbd184d7679cb32dfefe7be4ce2ffd6159d6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Mon, 28 Sep 2020 12:35:01 +0100 Subject: [PATCH 380/885] add --repo flag. --- cmd/tvx/extract.go | 3 ++- cmd/tvx/main.go | 17 +++++++++++++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/cmd/tvx/extract.go b/cmd/tvx/extract.go index d8053e5c2..3408cea12 100644 --- a/cmd/tvx/extract.go +++ b/cmd/tvx/extract.go @@ -36,9 +36,10 @@ var extractFlags struct { var extractCmd = &cli.Command{ Name: "extract", - Description: "generate a message-class test vector by extracting it from a live chain", + Description: "generate a test vector by extracting it from a live chain", Action: runExtract, Flags: []cli.Flag{ + &repoFlag, &cli.StringFlag{ Name: "class", Usage: "class of vector to extract; other required flags depend on the; values: 'message'", diff --git a/cmd/tvx/main.go b/cmd/tvx/main.go index a74a029ce..361ba41c3 100644 --- a/cmd/tvx/main.go +++ b/cmd/tvx/main.go @@ -8,6 +8,18 @@ import ( "github.com/urfave/cli/v2" ) +// DefaultLotusRepoPath is where the fallback path where to look for a Lotus +// client repo. It is expanded with mitchellh/go-homedir, so it'll work with all +// OSes despite the Unix twiddle notation. +const DefaultLotusRepoPath = "~/.lotus" + +var repoFlag = cli.StringFlag{ + Name: "repo", + EnvVars: []string{"LOTUS_PATH"}, + Value: DefaultLotusRepoPath, + TakesFile: true, +} + func main() { app := &cli.App{ Name: "tvx", @@ -22,7 +34,7 @@ func main() { SETTING THE JSON-RPC API ENDPOINT - You can set the JSON-RPC API endpoint through one of the following approaches. + You can set the JSON-RPC API endpoint through one of the following methods. 1. Directly set the API endpoint on the FULLNODE_API_INFO env variable. The format is [token]:multiaddr, where token is optional for commands not @@ -30,11 +42,12 @@ func main() { 2. If you're running tvx against a local Lotus client, you can set the REPO env variable to have the API endpoint and token extracted from the repo. + Alternatively, you can pass the --repo CLI flag. 3. Rely on the default fallback, which inspects ~/.lotus and extracts the API endpoint string if the location is a Lotus repo. - tvx will apply these approaches in the same order of precedence they're listed. + tvx will apply these methods in the same order of precedence they're listed. `, Usage: "tvx is a tool for extracting and executing test vectors", Commands: []*cli.Command{ From 8f3be7866765ce5054dd2ffe20c55dee82a4fe2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Mon, 28 Sep 2020 13:00:07 +0100 Subject: [PATCH 381/885] tvx/extract: allow passing in block to speed things up. --- cmd/tvx/extract.go | 71 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 56 insertions(+), 15 deletions(-) diff --git a/cmd/tvx/extract.go b/cmd/tvx/extract.go index 3408cea12..09465d356 100644 --- a/cmd/tvx/extract.go +++ b/cmd/tvx/extract.go @@ -28,6 +28,7 @@ import ( var extractFlags struct { id string + block string class string cid string file string @@ -52,6 +53,11 @@ var extractCmd = &cli.Command{ Value: "(undefined)", Destination: &extractFlags.id, }, + &cli.StringFlag{ + Name: "block", + Usage: "optionally, the block CID the message was included in, to avoid expensive chain scanning", + Destination: &extractFlags.block, + }, &cli.StringFlag{ Name: "cid", Usage: "message CID to generate test vector from", @@ -98,27 +104,59 @@ func runExtract(c *cli.Context) error { } defer closer() - log.Printf("locating message with CID: %s", mcid) - - // Locate the message. - msgInfo, err := api.StateSearchMsg(ctx, mcid) - if err != nil { - return fmt.Errorf("failed to locate message: %w", err) - } - - log.Printf("located message at tipset %s (height: %d) with exit code: %s", msgInfo.TipSet, msgInfo.Height, msgInfo.Receipt.ExitCode) + var ( + msg *types.Message + incTs *types.TipSet + execTs *types.TipSet + ) // Extract the full message. - msg, err := api.ChainGetMessage(ctx, mcid) + msg, err = api.ChainGetMessage(ctx, mcid) if err != nil { return err } - log.Printf("full message: %+v", msg) + log.Printf("found message with CID %s: %+v", mcid, msg) - execTs, incTs, err := fetchThisAndPrevTipset(ctx, api, msgInfo.TipSet) - if err != nil { - return err + if block := extractFlags.block; block == "" { + log.Printf("locating message in blockchain") + + // Locate the message. + msgInfo, err := api.StateSearchMsg(ctx, mcid) + if err != nil { + return fmt.Errorf("failed to locate message: %w", err) + } + + log.Printf("located message at tipset %s (height: %d) with exit code: %s", msgInfo.TipSet, msgInfo.Height, msgInfo.Receipt.ExitCode) + + execTs, incTs, err = fetchThisAndPrevTipset(ctx, api, msgInfo.TipSet) + if err != nil { + return err + } + } else { + bcid, err := cid.Decode(block) + if err != nil { + return err + } + + log.Printf("message inclusion block CID was provided; scanning around it: %s", bcid) + + blk, err := api.ChainGetBlock(ctx, bcid) + if err != nil { + return fmt.Errorf("failed to get block: %w", err) + } + + // types.EmptyTSK hints to use the HEAD. + execTs, err = api.ChainGetTipSetByHeight(ctx, blk.Height+1, types.EmptyTSK) + if err != nil { + return fmt.Errorf("failed to get message execution tipset: %w", err) + } + + // walk back from the execTs instead of HEAD, to save time. + incTs, err = api.ChainGetTipSetByHeight(ctx, blk.Height, execTs.Key()) + if err != nil { + return fmt.Errorf("failed to get message inclusion tipset: %w", err) + } } log.Printf("message was executed in tipset: %s", execTs.Key()) @@ -274,7 +312,10 @@ func runExtract(c *cli.Context) error { Meta: &schema.Metadata{ ID: extractFlags.id, Gen: []schema.GenerationData{ - {Source: fmt.Sprintf("message:%s:%s", ntwkName, msg.Cid().String())}, + {Source: fmt.Sprintf("network:%s", ntwkName)}, + {Source: fmt.Sprintf("msg:%s", msg.Cid().String())}, + {Source: fmt.Sprintf("inc_ts:%s", incTs.Key().String())}, + {Source: fmt.Sprintf("exec_ts:%s", execTs.Key().String())}, {Source: "github.com/filecoin-project/lotus", Version: version.String()}}, }, CAR: out.Bytes(), From a0dffb44d31595a51857516a58439fd3964f044e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Mon, 28 Sep 2020 13:11:34 +0100 Subject: [PATCH 382/885] tvx/extract: perform sanity check on receipt. --- cmd/tvx/extract.go | 31 ++++++++++++++++++++++++++++--- conformance/runner.go | 8 ++++---- 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/cmd/tvx/extract.go b/cmd/tvx/extract.go index 09465d356..8dbaf45d5 100644 --- a/cmd/tvx/extract.go +++ b/cmd/tvx/extract.go @@ -10,6 +10,8 @@ import ( "log" "os" + "github.com/fatih/color" + "github.com/filecoin-project/lotus/api" init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init" "github.com/filecoin-project/lotus/chain/actors/builtin/reward" @@ -277,6 +279,25 @@ func runExtract(c *cli.Context) error { return fmt.Errorf("unknown state retention option: %s", retention) } + log.Printf("message applied; preroot: %s, postroot: %s", preroot, postroot) + log.Printf("performing sanity check on receipt") + + receipt := &schema.Receipt{ + ExitCode: int64(applyret.ExitCode), + ReturnValue: applyret.Return, + GasUsed: applyret.GasUsed, + } + + reporter := new(conformance.LogReporter) + conformance.AssertMsgResult(reporter, receipt, applyret, "as locally executed") + if reporter.Failed() { + log.Printf(color.RedString("receipt sanity check failed; aborting")) + return fmt.Errorf("vector generation aborted") + } else { + log.Printf(color.GreenString("receipt sanity check succeeded")) + } + + log.Printf("generating vector") msgBytes, err := msg.Serialize() if err != nil { return err @@ -311,11 +332,15 @@ func runExtract(c *cli.Context) error { Class: schema.ClassMessage, Meta: &schema.Metadata{ ID: extractFlags.id, + // TODO need to replace schema.GenerationData with a more flexible + // data structure that makes no assumption about the traceability + // data that's being recorded; a flexible map[string]string + // would do. Gen: []schema.GenerationData{ {Source: fmt.Sprintf("network:%s", ntwkName)}, - {Source: fmt.Sprintf("msg:%s", msg.Cid().String())}, - {Source: fmt.Sprintf("inc_ts:%s", incTs.Key().String())}, - {Source: fmt.Sprintf("exec_ts:%s", execTs.Key().String())}, + {Source: fmt.Sprintf("message:%s", msg.Cid().String())}, + {Source: fmt.Sprintf("inclusion_tipset:%s", incTs.Key().String())}, + {Source: fmt.Sprintf("execution_tipset:%s", execTs.Key().String())}, {Source: "github.com/filecoin-project/lotus", Version: version.String()}}, }, CAR: out.Bytes(), diff --git a/conformance/runner.go b/conformance/runner.go index 456955b25..0fc4b13fc 100644 --- a/conformance/runner.go +++ b/conformance/runner.go @@ -65,7 +65,7 @@ func ExecuteMessageVector(r Reporter, vector *schema.TestVector) { } // Assert that the receipt matches what the test vector expects. - assertMsgResult(r, vector.Post.Receipts[i], ret, strconv.Itoa(i)) + AssertMsgResult(r, vector.Post.Receipts[i], ret, strconv.Itoa(i)) } // Once all messages are applied, assert that the final state root matches @@ -105,7 +105,7 @@ func ExecuteTipsetVector(r Reporter, vector *schema.TestVector) { } for j, v := range ret.AppliedResults { - assertMsgResult(r, vector.Post.Receipts[receiptsIdx], v, fmt.Sprintf("%d of tipset %d", j, i)) + AssertMsgResult(r, vector.Post.Receipts[receiptsIdx], v, fmt.Sprintf("%d of tipset %d", j, i)) receiptsIdx++ } @@ -127,10 +127,10 @@ func ExecuteTipsetVector(r Reporter, vector *schema.TestVector) { } } -// assertMsgResult compares a message result. It takes the expected receipt +// AssertMsgResult compares a message result. It takes the expected receipt // encoded in the vector, the actual receipt returned by Lotus, and a message // label to log in the assertion failure message to facilitate debugging. -func assertMsgResult(r Reporter, expected *schema.Receipt, actual *vm.ApplyRet, label string) { +func AssertMsgResult(r Reporter, expected *schema.Receipt, actual *vm.ApplyRet, label string) { r.Helper() if expected, actual := exitcode.ExitCode(expected.ExitCode), actual.ExitCode; expected != actual { From 4c9717187f8995dd5b9cf647ac003815dd645a80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Mon, 28 Sep 2020 13:19:45 +0100 Subject: [PATCH 383/885] tvx/extract: small refactor. --- cmd/tvx/extract.go | 108 +++++++++++++++++++++++---------------------- 1 file changed, 56 insertions(+), 52 deletions(-) diff --git a/cmd/tvx/extract.go b/cmd/tvx/extract.go index 8dbaf45d5..29330daaa 100644 --- a/cmd/tvx/extract.go +++ b/cmd/tvx/extract.go @@ -106,59 +106,9 @@ func runExtract(c *cli.Context) error { } defer closer() - var ( - msg *types.Message - incTs *types.TipSet - execTs *types.TipSet - ) - - // Extract the full message. - msg, err = api.ChainGetMessage(ctx, mcid) + msg, execTs, incTs, err := resolveFromChain(ctx, api, mcid) if err != nil { - return err - } - - log.Printf("found message with CID %s: %+v", mcid, msg) - - if block := extractFlags.block; block == "" { - log.Printf("locating message in blockchain") - - // Locate the message. - msgInfo, err := api.StateSearchMsg(ctx, mcid) - if err != nil { - return fmt.Errorf("failed to locate message: %w", err) - } - - log.Printf("located message at tipset %s (height: %d) with exit code: %s", msgInfo.TipSet, msgInfo.Height, msgInfo.Receipt.ExitCode) - - execTs, incTs, err = fetchThisAndPrevTipset(ctx, api, msgInfo.TipSet) - if err != nil { - return err - } - } else { - bcid, err := cid.Decode(block) - if err != nil { - return err - } - - log.Printf("message inclusion block CID was provided; scanning around it: %s", bcid) - - blk, err := api.ChainGetBlock(ctx, bcid) - if err != nil { - return fmt.Errorf("failed to get block: %w", err) - } - - // types.EmptyTSK hints to use the HEAD. - execTs, err = api.ChainGetTipSetByHeight(ctx, blk.Height+1, types.EmptyTSK) - if err != nil { - return fmt.Errorf("failed to get message execution tipset: %w", err) - } - - // walk back from the execTs instead of HEAD, to save time. - incTs, err = api.ChainGetTipSetByHeight(ctx, blk.Height, execTs.Key()) - if err != nil { - return fmt.Errorf("failed to get message inclusion tipset: %w", err) - } + return fmt.Errorf("failed to resolve message and tipsets from chain: %w", err) } log.Printf("message was executed in tipset: %s", execTs.Key()) @@ -384,6 +334,60 @@ func runExtract(c *cli.Context) error { return nil } +// resolveFromChain queries the chain for the provided message, using the block CID to +// speed up the query, if provided +func resolveFromChain(ctx context.Context, api api.FullNode, mcid cid.Cid) (msg *types.Message, execTs *types.TipSet, incTs *types.TipSet, err error) { + // Extract the full message. + msg, err = api.ChainGetMessage(ctx, mcid) + if err != nil { + return nil, nil, nil, err + } + + log.Printf("found message with CID %s: %+v", mcid, msg) + + block := extractFlags.block + if block == "" { + log.Printf("locating message in blockchain") + + // Locate the message. + msgInfo, err := api.StateSearchMsg(ctx, mcid) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to locate message: %w", err) + } + + log.Printf("located message at tipset %s (height: %d) with exit code: %s", msgInfo.TipSet, msgInfo.Height, msgInfo.Receipt.ExitCode) + + execTs, incTs, err = fetchThisAndPrevTipset(ctx, api, msgInfo.TipSet) + return msg, execTs, incTs, err + } + + bcid, err := cid.Decode(block) + if err != nil { + return nil, nil, nil, err + } + + log.Printf("message inclusion block CID was provided; scanning around it: %s", bcid) + + blk, err := api.ChainGetBlock(ctx, bcid) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to get block: %w", err) + } + + // types.EmptyTSK hints to use the HEAD. + execTs, err = api.ChainGetTipSetByHeight(ctx, blk.Height+1, types.EmptyTSK) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to get message execution tipset: %w", err) + } + + // walk back from the execTs instead of HEAD, to save time. + incTs, err = api.ChainGetTipSetByHeight(ctx, blk.Height, execTs.Key()) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to get message inclusion tipset: %w", err) + } + + return msg, execTs, incTs, nil +} + // fetchThisAndPrevTipset returns the full tipset identified by the key, as well // as the previous tipset. In the context of vector generation, the target // tipset is the one where a message was executed, and the previous tipset is From 9f6862a456e391552b6c17cd1a0c82d5a6e70965 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Mon, 28 Sep 2020 13:22:56 +0100 Subject: [PATCH 384/885] fix lint. --- cmd/tvx/extract.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/tvx/extract.go b/cmd/tvx/extract.go index 29330daaa..07c6ca7ef 100644 --- a/cmd/tvx/extract.go +++ b/cmd/tvx/extract.go @@ -230,7 +230,7 @@ func runExtract(c *cli.Context) error { } log.Printf("message applied; preroot: %s, postroot: %s", preroot, postroot) - log.Printf("performing sanity check on receipt") + log.Println("performing sanity check on receipt") receipt := &schema.Receipt{ ExitCode: int64(applyret.ExitCode), @@ -241,13 +241,13 @@ func runExtract(c *cli.Context) error { reporter := new(conformance.LogReporter) conformance.AssertMsgResult(reporter, receipt, applyret, "as locally executed") if reporter.Failed() { - log.Printf(color.RedString("receipt sanity check failed; aborting")) + log.Println(color.RedString("receipt sanity check failed; aborting")) return fmt.Errorf("vector generation aborted") - } else { - log.Printf(color.GreenString("receipt sanity check succeeded")) } - log.Printf("generating vector") + log.Println(color.GreenString("receipt sanity check succeeded")) + + log.Println("generating vector") msgBytes, err := msg.Serialize() if err != nil { return err From 6b16d48bad56544d547449e3df8972cf5feb72ee Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Mon, 28 Sep 2020 15:56:44 +0200 Subject: [PATCH 385/885] refactor: fetch tipset blocks in parallel --- chain/store/store.go | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/chain/store/store.go b/chain/store/store.go index 1dbf69547..e68857e0b 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -10,6 +10,8 @@ import ( "strconv" "sync" + "golang.org/x/sync/errgroup" + "github.com/filecoin-project/go-state-types/crypto" "github.com/minio/blake2b-simd" @@ -467,14 +469,25 @@ func (cs *ChainStore) LoadTipSet(tsk types.TipSetKey) (*types.TipSet, error) { return v.(*types.TipSet), nil } - var blks []*types.BlockHeader - for _, c := range tsk.Cids() { - b, err := cs.GetBlock(c) - if err != nil { - return nil, xerrors.Errorf("get block %s: %w", c, err) - } + // Fetch tipset block headers from blockstore in parallel + var eg errgroup.Group + cids := tsk.Cids() + blks := make([]*types.BlockHeader, 0, len(cids)) + for _, c := range cids { + c := c + eg.Go(func() error { + b, err := cs.GetBlock(c) + if err != nil { + return xerrors.Errorf("get block %s: %w", c, err) + } - blks = append(blks, b) + blks = append(blks, b) + return nil + }) + } + err := eg.Wait() + if err != nil { + return nil, err } ts, err := types.NewTipSet(blks) From cfe6f595b036c332843697413fce5b78104159af Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Mon, 28 Sep 2020 16:35:37 +0200 Subject: [PATCH 386/885] fix: unsafe append in LoadTipSet --- chain/store/store.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/chain/store/store.go b/chain/store/store.go index e68857e0b..6c93db7a0 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -472,16 +472,16 @@ func (cs *ChainStore) LoadTipSet(tsk types.TipSetKey) (*types.TipSet, error) { // Fetch tipset block headers from blockstore in parallel var eg errgroup.Group cids := tsk.Cids() - blks := make([]*types.BlockHeader, 0, len(cids)) - for _, c := range cids { - c := c + blks := make([]*types.BlockHeader, len(cids)) + for i, c := range cids { + i, c := i, c eg.Go(func() error { b, err := cs.GetBlock(c) if err != nil { return xerrors.Errorf("get block %s: %w", c, err) } - blks = append(blks, b) + blks[i] = b return nil }) } From 96f882860fdc2a9edeb92f07964d7fb5a73b0c1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Mon, 28 Sep 2020 17:02:56 +0100 Subject: [PATCH 387/885] add extract-many command. --- cmd/tvx/actor_mapping.go | 44 +++++++++ cmd/tvx/extract.go | 50 ++++++---- cmd/tvx/extract_many.go | 204 +++++++++++++++++++++++++++++++++++++++ cmd/tvx/main.go | 6 +- 4 files changed, 283 insertions(+), 21 deletions(-) create mode 100644 cmd/tvx/actor_mapping.go create mode 100644 cmd/tvx/extract_many.go diff --git a/cmd/tvx/actor_mapping.go b/cmd/tvx/actor_mapping.go new file mode 100644 index 000000000..8c306aca0 --- /dev/null +++ b/cmd/tvx/actor_mapping.go @@ -0,0 +1,44 @@ +package main + +import ( + "reflect" + + "github.com/filecoin-project/specs-actors/actors/builtin" + "github.com/ipfs/go-cid" + "github.com/multiformats/go-multihash" +) + +var ActorMethodTable = make(map[string][]string, 64) + +var Actors = map[cid.Cid]interface{}{ + builtin.InitActorCodeID: builtin.MethodsInit, + builtin.CronActorCodeID: builtin.MethodsCron, + builtin.AccountActorCodeID: builtin.MethodsAccount, + builtin.StoragePowerActorCodeID: builtin.MethodsPower, + builtin.StorageMinerActorCodeID: builtin.MethodsMiner, + builtin.StorageMarketActorCodeID: builtin.MethodsMarket, + builtin.PaymentChannelActorCodeID: builtin.MethodsPaych, + builtin.MultisigActorCodeID: builtin.MethodsMultisig, + builtin.RewardActorCodeID: builtin.MethodsReward, + builtin.VerifiedRegistryActorCodeID: builtin.MethodsVerifiedRegistry, +} + +func init() { + for code, methods := range Actors { + cmh, err := multihash.Decode(code.Hash()) // identity hash. + if err != nil { + panic(err) + } + + var ( + aname = string(cmh.Digest) + rt = reflect.TypeOf(methods) + nf = rt.NumField() + ) + + ActorMethodTable[aname] = append(ActorMethodTable[aname], "Send") + for i := 0; i < nf; i++ { + ActorMethodTable[aname] = append(ActorMethodTable[aname], rt.Field(i).Name) + } + } +} diff --git a/cmd/tvx/extract.go b/cmd/tvx/extract.go index 07c6ca7ef..e10fbad09 100644 --- a/cmd/tvx/extract.go +++ b/cmd/tvx/extract.go @@ -9,6 +9,7 @@ import ( "io" "log" "os" + "path/filepath" "github.com/fatih/color" @@ -28,7 +29,7 @@ import ( "github.com/urfave/cli/v2" ) -var extractFlags struct { +type extractOpts struct { id string block string class string @@ -37,6 +38,8 @@ var extractFlags struct { retain string } +var extractFlags extractOpts + var extractCmd = &cli.Command{ Name: "extract", Description: "generate a test vector by extracting it from a live chain", @@ -94,19 +97,23 @@ func runExtract(c *cli.Context) error { ctx := context.Background() - mcid, err := cid.Decode(extractFlags.cid) - if err != nil { - return err - } - // Make the API client. - api, closer, err := lcli.GetFullNodeAPI(c) + fapi, closer, err := lcli.GetFullNodeAPI(c) if err != nil { return err } defer closer() - msg, execTs, incTs, err := resolveFromChain(ctx, api, mcid) + return doExtract(ctx, fapi, extractFlags) +} + +func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { + mcid, err := cid.Decode(opts.cid) + if err != nil { + return err + } + + msg, execTs, incTs, err := resolveFromChain(ctx, fapi, mcid, opts.block) if err != nil { return fmt.Errorf("failed to resolve message and tipsets from chain: %w", err) } @@ -119,7 +126,7 @@ func runExtract(c *cli.Context) error { // precursors, if any. var allmsgs []*types.Message for _, b := range incTs.Blocks() { - messages, err := api.ChainGetBlockMessages(ctx, b.Cid()) + messages, err := fapi.ChainGetBlockMessages(ctx, b.Cid()) if err != nil { return err } @@ -139,7 +146,7 @@ func runExtract(c *cli.Context) error { break } - log.Printf("message not found in block %s; precursors found: %v; ignoring block", b.Cid(), related) + log.Printf("message not found in block %s; number of precursors found: %d; ignoring block", b.Cid(), len(related)) } if allmsgs == nil { @@ -151,8 +158,8 @@ func runExtract(c *cli.Context) error { var ( // create a read-through store that uses ChainGetObject to fetch unknown CIDs. - pst = NewProxyingStores(ctx, api) - g = NewSurgeon(ctx, api, pst) + pst = NewProxyingStores(ctx, fapi) + g = NewSurgeon(ctx, fapi, pst) ) driver := conformance.NewDriver(ctx, schema.Selector{}, conformance.DriverOpts{ @@ -178,7 +185,7 @@ func runExtract(c *cli.Context) error { postroot cid.Cid applyret *vm.ApplyRet carWriter func(w io.Writer) error - retention = extractFlags.retain + retention = opts.retain ) log.Printf("using state retention strategy: %s", retention) @@ -204,7 +211,7 @@ func runExtract(c *cli.Context) error { case "accessed-actors": log.Printf("calculating accessed actors") // get actors accessed by message. - retain, err := g.GetAccessedActors(ctx, api, mcid) + retain, err := g.GetAccessedActors(ctx, fapi, mcid) if err != nil { return fmt.Errorf("failed to calculate accessed actors: %w", err) } @@ -267,12 +274,12 @@ func runExtract(c *cli.Context) error { return err } - version, err := api.Version(ctx) + version, err := fapi.Version(ctx) if err != nil { return err } - ntwkName, err := api.StateNetworkName(ctx) + ntwkName, err := fapi.StateNetworkName(ctx) if err != nil { return err } @@ -281,7 +288,7 @@ func runExtract(c *cli.Context) error { vector := schema.TestVector{ Class: schema.ClassMessage, Meta: &schema.Metadata{ - ID: extractFlags.id, + ID: opts.id, // TODO need to replace schema.GenerationData with a more flexible // data structure that makes no assumption about the traceability // data that's being recorded; a flexible map[string]string @@ -316,7 +323,11 @@ func runExtract(c *cli.Context) error { } output := io.WriteCloser(os.Stdout) - if file := extractFlags.file; file != "" { + if file := opts.file; file != "" { + dir := filepath.Dir(file) + if err := os.MkdirAll(dir, 0755); err != nil { + return fmt.Errorf("unable to create directory %s: %w", dir, err) + } output, err = os.Create(file) if err != nil { return err @@ -336,7 +347,7 @@ func runExtract(c *cli.Context) error { // resolveFromChain queries the chain for the provided message, using the block CID to // speed up the query, if provided -func resolveFromChain(ctx context.Context, api api.FullNode, mcid cid.Cid) (msg *types.Message, execTs *types.TipSet, incTs *types.TipSet, err error) { +func resolveFromChain(ctx context.Context, api api.FullNode, mcid cid.Cid, block string) (msg *types.Message, execTs *types.TipSet, incTs *types.TipSet, err error) { // Extract the full message. msg, err = api.ChainGetMessage(ctx, mcid) if err != nil { @@ -345,7 +356,6 @@ func resolveFromChain(ctx context.Context, api api.FullNode, mcid cid.Cid) (msg log.Printf("found message with CID %s: %+v", mcid, msg) - block := extractFlags.block if block == "" { log.Printf("locating message in blockchain") diff --git a/cmd/tvx/extract_many.go b/cmd/tvx/extract_many.go new file mode 100644 index 000000000..8fea8df4c --- /dev/null +++ b/cmd/tvx/extract_many.go @@ -0,0 +1,204 @@ +package main + +import ( + "context" + "encoding/csv" + "fmt" + "io" + "log" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/fatih/color" + "github.com/filecoin-project/go-state-types/exitcode" + "github.com/hashicorp/go-multierror" + "github.com/urfave/cli/v2" + + lcli "github.com/filecoin-project/lotus/cli" +) + +var extractManyFlags struct { + in string + outdir string +} + +var extractManyCmd = &cli.Command{ + Name: "extract-many", + Description: `generate many test vectors by repeateadly calling tvx extract, using a csv file as input. + + The CSV file must have a format just like the following: + + message_cid,receiver_code,method_num,exit_code,height,block_cid,seq + bafy2bzacedvuvgpsnwq7i7kltfap6hnp7fdmzf6lr4w34zycjrthb3v7k6zi6,fil/1/account,0,0,67972,bafy2bzacebthpxzlk7zhlkz3jfzl4qw7mdoswcxlf3rkof3b4mbxfj3qzfk7w,1 + bafy2bzacedwicofymn4imgny2hhbmcm4o5bikwnv3qqgohyx73fbtopiqlro6,fil/1/account,0,0,67860,bafy2bzacebj7beoxyzll522o6o76mt7von4psn3tlvunokhv4zhpwmfpipgti,2 + ... + + The first row MUST be a header row. At the bare minimum, those seven fields + must appear, in the order specified. Extra fields are accepted, but always + after these compulsory six. +`, + Action: runExtractMany, + Flags: []cli.Flag{ + &repoFlag, + &cli.StringFlag{ + Name: "in", + Usage: "path to input file (csv)", + Destination: &extractManyFlags.in, + }, + &cli.StringFlag{ + Name: "out-dir", + Usage: "output directory", + Destination: &extractManyFlags.outdir, + }, + }, +} + +func runExtractMany(c *cli.Context) error { + // LOTUS_DISABLE_VM_BUF disables what's called "VM state tree buffering", + // which stashes write operations in a BufferedBlockstore + // (https://github.com/filecoin-project/lotus/blob/b7a4dbb07fd8332b4492313a617e3458f8003b2a/lib/bufbstore/buf_bstore.go#L21) + // such that they're not written until the VM is actually flushed. + // + // For some reason, the standard behaviour was not working for me (raulk), + // and disabling it (such that the state transformations are written immediately + // to the blockstore) worked. + _ = os.Setenv("LOTUS_DISABLE_VM_BUF", "iknowitsabadidea") + + ctx := context.Background() + + // Make the API client. + fapi, closer, err := lcli.GetFullNodeAPI(c) + if err != nil { + return err + } + defer closer() + + var ( + in = extractManyFlags.in + outdir = extractManyFlags.outdir + ) + + if in == "" { + return fmt.Errorf("input file not provided") + } + + if outdir == "" { + return fmt.Errorf("output dir not provided") + } + + // Open the CSV file for reading. + f, err := os.Open(in) + if err != nil { + return fmt.Errorf("could not open file %s: %w", in, err) + } + + // Ensure the output directory exists. + if err := os.MkdirAll(outdir, 0755); err != nil { + return fmt.Errorf("could not create output dir %s: %w", outdir, err) + } + + // Create a CSV reader and validate the header row. + reader := csv.NewReader(f) + if header, err := reader.Read(); err != nil { + return fmt.Errorf("failed to read header from csv: %w", err) + } else if l := len(header); l < 7 { + return fmt.Errorf("insufficient number of fields: %d", l) + } else if f := header[0]; f != "message_cid" { + return fmt.Errorf("csv sanity check failed: expected first field in header to be 'message_cid'; was: %s", f) + } else { + log.Println(color.GreenString("csv sanity check succeeded; header contains fields: %v", header)) + } + + var generated []string + merr := new(multierror.Error) + // Read each row and extract the requested message. + for { + row, err := reader.Read() + if err == io.EOF { + break + } else if err != nil { + return fmt.Errorf("failed to read row: %w", err) + } + var ( + cid = row[0] + actorcode = row[1] + methodnumstr = row[2] + exitcodestr = row[3] + _ = row[4] + block = row[5] + seq = row[6] + + exit int + methodnum int + methodname string + ) + + // Parse the exit code. + if exit, err = strconv.Atoi(exitcodestr); err != nil { + return fmt.Errorf("invalid exitcode number: %d", exit) + } + // Parse the method number. + if methodnum, err = strconv.Atoi(methodnumstr); err != nil { + return fmt.Errorf("invalid method number: %s", methodnumstr) + } + + // Lookup the method in actor method table. + if m, ok := ActorMethodTable[actorcode]; !ok { + return fmt.Errorf("unrecognized actor: %s", actorcode) + } else if methodnum >= len(m) { + return fmt.Errorf("unrecognized method number for actor %s: %d", actorcode, methodnum) + } else { + methodname = m[methodnum] + } + + // exitcode string representations are of kind ErrType(0); strip out + // the number portion. + exitcodename := strings.Split(exitcode.ExitCode(exit).String(), "(")[0] + // replace the slashes in the actor code name with underscores. + actorcodename := strings.ReplaceAll(actorcode, "/", "_") + + // Compute the ID of the vector. + id := fmt.Sprintf("extracted-msg-%s-%s-%s-%s", actorcodename, methodname, exitcodename, seq) + // Vector filename, using a base of outdir. + file := filepath.Join(outdir, actorcodename, methodname, exitcodename, id) + ".json" + + log.Println(color.YellowString("processing message id: %s", id)) + + opts := extractOpts{ + id: id, + block: block, + class: "message", + cid: cid, + file: file, + retain: "accessed-cids", + } + + if err := doExtract(ctx, fapi, opts); err != nil { + merr = multierror.Append(err, fmt.Errorf("failed to extract vector for message %s: %w", cid, err)) + continue + } + + log.Println(color.MagentaString("generated file: %s", file)) + + generated = append(generated, file) + } + + if len(generated) == 0 { + log.Println("no files generated") + } else { + log.Println("files generated:") + for _, g := range generated { + log.Println(g) + } + } + + if merr.ErrorOrNil() != nil { + log.Println(color.YellowString("done processing with errors: %s")) + } else { + log.Println(color.GreenString("done processing with no errors")) + } + + return merr.ErrorOrNil() +} diff --git a/cmd/tvx/main.go b/cmd/tvx/main.go index 361ba41c3..6c887d163 100644 --- a/cmd/tvx/main.go +++ b/cmd/tvx/main.go @@ -23,7 +23,7 @@ var repoFlag = cli.StringFlag{ func main() { app := &cli.App{ Name: "tvx", - Description: `tvx is a tool for extracting and executing test vectors. It has two subcommands. + Description: `tvx is a tool for extracting and executing test vectors. It has three subcommands. tvx extract extracts a test vector from a live network. It requires access to a Filecoin client that exposes the standard JSON-RPC API endpoint. Only @@ -32,6 +32,9 @@ func main() { tvx exec executes test vectors against Lotus. Either you can supply one in a file, or many as an ndjson stdin stream. + tvx extract-many performs a batch extraction of many messages, supplied in a + CSV file. Refer to the help of that subcommand for more info. + SETTING THE JSON-RPC API ENDPOINT You can set the JSON-RPC API endpoint through one of the following methods. @@ -53,6 +56,7 @@ func main() { Commands: []*cli.Command{ extractCmd, execCmd, + extractManyCmd, }, } From 124bd0c765b8a39c6ad7295874a48d144fd67952 Mon Sep 17 00:00:00 2001 From: jennijuju Date: Mon, 28 Sep 2020 14:58:46 -0400 Subject: [PATCH 388/885] Always use default 720h for setask duration and hide the duration param option. --- cmd/lotus-storage-miner/market.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/cmd/lotus-storage-miner/market.go b/cmd/lotus-storage-miner/market.go index 0e7be3dd3..6e3743143 100644 --- a/cmd/lotus-storage-miner/market.go +++ b/cmd/lotus-storage-miner/market.go @@ -164,12 +164,6 @@ var setAskCmd = &cli.Command{ Usage: "Set the price of the ask for verified deals (specified as attoFIL / GiB / Epoch) to `PRICE`", Required: true, }, - &cli.StringFlag{ - Name: "duration", - Usage: "Set duration of ask (a quantity of time after which the ask expires) `DURATION`", - DefaultText: "720h0m0s", - Value: "720h0m0s", - }, &cli.StringFlag{ Name: "min-piece-size", Usage: "Set minimum piece size (w/bit-padding, in bytes) in ask to `SIZE`", @@ -194,7 +188,7 @@ var setAskCmd = &cli.Command{ pri := types.NewInt(cctx.Uint64("price")) vpri := types.NewInt(cctx.Uint64("verified-price")) - dur, err := time.ParseDuration(cctx.String("duration")) + dur, err := time.ParseDuration("720h0m0s") if err != nil { return xerrors.Errorf("cannot parse duration: %w", err) } From 64f24fd276d0f964b78ece61fd2be6cdbb674b1f Mon Sep 17 00:00:00 2001 From: jennijuju Date: Mon, 28 Sep 2020 15:34:06 -0400 Subject: [PATCH 389/885] Added an option to hide sector info for `lotus-miner info` --- cmd/lotus-storage-miner/info.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/cmd/lotus-storage-miner/info.go b/cmd/lotus-storage-miner/info.go index 3ccfd67da..213d62e6e 100644 --- a/cmd/lotus-storage-miner/info.go +++ b/cmd/lotus-storage-miner/info.go @@ -33,6 +33,12 @@ var infoCmd = &cli.Command{ Subcommands: []*cli.Command{ infoAllCmd, }, + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "hide-sectors-info", + Usage: "hide sectors info", + }, + }, Action: infoCmdAct, } @@ -199,10 +205,12 @@ func infoCmdAct(cctx *cli.Context) error { fmt.Printf("Expected Seal Duration: %s\n\n", sealdur) - fmt.Println("Sectors:") - err = sectorsInfo(ctx, nodeApi) - if err != nil { - return err + if !cctx.Bool("hide-sectors-info") { + fmt.Println("Sectors:") + err = sectorsInfo(ctx, nodeApi) + if err != nil { + return err + } } // TODO: grab actr state / info From 5f3160cf5bf3fd304882a6756e605131d35dfd6c Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 28 Sep 2020 12:48:08 -0700 Subject: [PATCH 390/885] continue expanding vm abstraction layer --- chain/actors/builtin/builtin.go | 17 +++ chain/stmgr/forks_test.go | 25 +++- chain/stmgr/utils.go | 88 +++---------- chain/vm/invoker.go | 119 +++++++----------- cli/state.go | 27 ++-- .../processor/common_actors.go | 25 ++-- conformance/chaos/actor.go | 8 +- conformance/driver.go | 3 +- go.mod | 6 +- go.sum | 10 +- 10 files changed, 135 insertions(+), 193 deletions(-) diff --git a/chain/actors/builtin/builtin.go b/chain/actors/builtin/builtin.go index 49a07951b..a0567dc27 100644 --- a/chain/actors/builtin/builtin.go +++ b/chain/actors/builtin/builtin.go @@ -10,9 +10,11 @@ import ( "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/types" + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" proof0 "github.com/filecoin-project/specs-actors/actors/runtime/proof" smoothing0 "github.com/filecoin-project/specs-actors/actors/util/smoothing" + builtin1 "github.com/filecoin-project/specs-actors/v2/actors/builtin" smoothing1 "github.com/filecoin-project/specs-actors/v2/actors/util/smoothing" ) @@ -49,3 +51,18 @@ func Load(store adt.Store, act *types.Actor) (cbor.Marshaler, error) { } return loader(store, act.Head) } + +func ActorNameByCode(c cid.Cid) string { + switch { + case builtin0.IsBuiltinActor(c): + return builtin0.ActorNameByCode(c) + case builtin1.IsBuiltinActor(c): + return builtin1.ActorNameByCode(c) + default: + return "" + } +} + +func IsBuiltinActor(c cid.Cid) bool { + return builtin0.IsBuiltinActor(c) || builtin1.IsBuiltinActor(c) +} diff --git a/chain/stmgr/forks_test.go b/chain/stmgr/forks_test.go index 8ec00f95f..d2ab74bc7 100644 --- a/chain/stmgr/forks_test.go +++ b/chain/stmgr/forks_test.go @@ -8,9 +8,11 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/cbor" "github.com/filecoin-project/specs-actors/actors/builtin" init_ "github.com/filecoin-project/specs-actors/actors/builtin/init" "github.com/filecoin-project/specs-actors/actors/runtime" + "github.com/multiformats/go-multihash" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/chain/actors" @@ -25,7 +27,8 @@ import ( _ "github.com/filecoin-project/lotus/lib/sigs/bls" _ "github.com/filecoin-project/lotus/lib/sigs/secp" - cbor "github.com/ipfs/go-ipld-cbor" + "github.com/ipfs/go-cid" + ipldcbor "github.com/ipfs/go-ipld-cbor" logging "github.com/ipfs/go-log" cbg "github.com/whyrusleeping/cbor-gen" ) @@ -41,6 +44,18 @@ const testForkHeight = 40 type testActor struct { } +var testActorCodeID = func() cid.Cid { + builder := cid.V1Builder{Codec: cid.Raw, MhType: multihash.IDENTITY} + c, err := builder.Sum([]byte("fil/any/test")) + if err != nil { + panic(err) + } + return c +}() + +func (testActor) Code() cid.Cid { return testActorCodeID } +func (testActor) State() cbor.Er { return new(testActorState) } + type testActorState struct { HasUpgraded uint64 } @@ -61,7 +76,7 @@ func (tas *testActorState) UnmarshalCBOR(r io.Reader) error { return nil } -func (ta *testActor) Exports() []interface{} { +func (ta testActor) Exports() []interface{} { return []interface{}{ 1: ta.Constructor, 2: ta.TestMethod, @@ -115,7 +130,7 @@ func TestForkHeightTriggers(t *testing.T) { } stmgr.ForksAtHeight[testForkHeight] = func(ctx context.Context, sm *StateManager, st types.StateTree, ts *types.TipSet) error { - cst := cbor.NewCborStore(sm.ChainStore().Blockstore()) + cst := ipldcbor.NewCborStore(sm.ChainStore().Blockstore()) act, err := st.GetActor(taddr) if err != nil { @@ -143,7 +158,7 @@ func TestForkHeightTriggers(t *testing.T) { return nil } - inv.Register(actors.Version0, builtin.PaymentChannelActorCodeID, &testActor{}, &testActorState{}, false) + inv.Register(nil, testActor{}) sm.SetVMConstructor(func(ctx context.Context, vmopt *vm.VMOpts) (*vm.VM, error) { nvm, err := vm.NewVM(ctx, vmopt) if err != nil { @@ -157,7 +172,7 @@ func TestForkHeightTriggers(t *testing.T) { var msgs []*types.SignedMessage - enc, err := actors.SerializeParams(&init_.ExecParams{CodeCID: builtin.PaymentChannelActorCodeID}) + enc, err := actors.SerializeParams(&init_.ExecParams{CodeCID: (testActor{}).Code()}) if err != nil { t.Fatal(err) } diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index 0d7be7adf..aedd324c2 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -17,32 +17,13 @@ import ( "github.com/filecoin-project/go-bitfield" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/crypto" + "github.com/filecoin-project/go-state-types/rt" builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" - account0 "github.com/filecoin-project/specs-actors/actors/builtin/account" - cron0 "github.com/filecoin-project/specs-actors/actors/builtin/cron" - init0 "github.com/filecoin-project/specs-actors/actors/builtin/init" - market0 "github.com/filecoin-project/specs-actors/actors/builtin/market" - miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" - msig0 "github.com/filecoin-project/specs-actors/actors/builtin/multisig" - paych0 "github.com/filecoin-project/specs-actors/actors/builtin/paych" - power0 "github.com/filecoin-project/specs-actors/actors/builtin/power" - reward0 "github.com/filecoin-project/specs-actors/actors/builtin/reward" - verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" + exported0 "github.com/filecoin-project/specs-actors/actors/builtin/exported" + exported1 "github.com/filecoin-project/specs-actors/actors/builtin/exported" proof0 "github.com/filecoin-project/specs-actors/actors/runtime/proof" - builtin1 "github.com/filecoin-project/specs-actors/v2/actors/builtin" - account1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/account" - cron1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/cron" - init1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/init" - market1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/market" - miner1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" - msig1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/multisig" - paych1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/paych" - power1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/power" - reward1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/reward" - verifreg1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/verifreg" - "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init" @@ -545,34 +526,13 @@ type MethodMeta struct { var MethodsMap = map[cid.Cid]map[abi.MethodNum]MethodMeta{} func init() { - cidToMethods := map[cid.Cid][2]interface{}{ - // builtin.SystemActorCodeID: {builtin.MethodsSystem, system.Actor{} }- apparently it doesn't have methods - builtin0.InitActorCodeID: {builtin0.MethodsInit, init0.Actor{}}, - builtin0.CronActorCodeID: {builtin0.MethodsCron, cron0.Actor{}}, - builtin0.AccountActorCodeID: {builtin0.MethodsAccount, account0.Actor{}}, - builtin0.StoragePowerActorCodeID: {builtin0.MethodsPower, power0.Actor{}}, - builtin0.StorageMinerActorCodeID: {builtin0.MethodsMiner, miner0.Actor{}}, - builtin0.StorageMarketActorCodeID: {builtin0.MethodsMarket, market0.Actor{}}, - builtin0.PaymentChannelActorCodeID: {builtin0.MethodsPaych, paych0.Actor{}}, - builtin0.MultisigActorCodeID: {builtin0.MethodsMultisig, msig0.Actor{}}, - builtin0.RewardActorCodeID: {builtin0.MethodsReward, reward0.Actor{}}, - builtin0.VerifiedRegistryActorCodeID: {builtin0.MethodsVerifiedRegistry, verifreg0.Actor{}}, + // TODO: combine with the runtime actor registry. + var actors []rt.VMActor + actors = append(actors, exported0.BuiltinActors()...) + actors = append(actors, exported1.BuiltinActors()...) - // builtin1.SystemActorCodeID: {builtin1.MethodsSystem, system.Actor{} }- apparently it doesn't have methods - builtin1.InitActorCodeID: {builtin1.MethodsInit, init1.Actor{}}, - builtin1.CronActorCodeID: {builtin1.MethodsCron, cron1.Actor{}}, - builtin1.AccountActorCodeID: {builtin1.MethodsAccount, account1.Actor{}}, - builtin1.StoragePowerActorCodeID: {builtin1.MethodsPower, power1.Actor{}}, - builtin1.StorageMinerActorCodeID: {builtin1.MethodsMiner, miner1.Actor{}}, - builtin1.StorageMarketActorCodeID: {builtin1.MethodsMarket, market1.Actor{}}, - builtin1.PaymentChannelActorCodeID: {builtin1.MethodsPaych, paych1.Actor{}}, - builtin1.MultisigActorCodeID: {builtin1.MethodsMultisig, msig1.Actor{}}, - builtin1.RewardActorCodeID: {builtin1.MethodsReward, reward1.Actor{}}, - builtin1.VerifiedRegistryActorCodeID: {builtin1.MethodsVerifiedRegistry, verifreg1.Actor{}}, - } - - for c, m := range cidToMethods { - exports := m[1].(vm.Invokee).Exports() + for _, actor := range actors { + exports := actor.Exports() methods := make(map[abi.MethodNum]MethodMeta, len(exports)) // Explicitly add send, it's special. @@ -583,17 +543,6 @@ func init() { Ret: reflect.TypeOf(new(abi.EmptyValue)), } - // Learn method names from the builtin.Methods* structs. - rv := reflect.ValueOf(m[0]) - rt := rv.Type() - nf := rt.NumField() - methodToName := make([]string, len(exports)) - for i := 0; i < nf; i++ { - name := rt.Field(i).Name - number := rv.Field(i).Interface().(abi.MethodNum) - methodToName[number] = name - } - // Iterate over exported methods. Some of these _may_ be nil and // must be skipped. for number, export := range exports { @@ -604,24 +553,19 @@ func init() { ev := reflect.ValueOf(export) et := ev.Type() - // Make sure the method name is correct. - // This is just a nice sanity check. + // Extract the method names using reflection. These + // method names always match the field names in the + // `builtin.Method*` structs (tested in the specs-actors + // tests). fnName := runtime.FuncForPC(ev.Pointer()).Name() fnName = strings.TrimSuffix(fnName[strings.LastIndexByte(fnName, '.')+1:], "-fm") - mName := methodToName[number] - if mName != fnName { - panic(fmt.Sprintf( - "actor method name is %s but exported method name is %s", - fnName, mName, - )) - } switch abi.MethodNum(number) { - // Note that builtin1.MethodSend = builtin0.MethodSend = 0. case builtin0.MethodSend: + // Note that builtin1.MethodSend = builtin0.MethodSend = 0. panic("method 0 is reserved for Send") - // Note that builtin1.MethodConstructor = builtin0.MethodConstructor = 1. case builtin0.MethodConstructor: + // Note that builtin1.MethodConstructor = builtin0.MethodConstructor = 1. if fnName != "Constructor" { panic("method 1 is reserved for Constructor") } @@ -633,7 +577,7 @@ func init() { Ret: et.Out(0), } } - MethodsMap[c] = methods + MethodsMap[actor.Code()] = methods } } diff --git a/chain/vm/invoker.go b/chain/vm/invoker.go index e137e71e4..1f3db3ee9 100644 --- a/chain/vm/invoker.go +++ b/chain/vm/invoker.go @@ -10,35 +10,13 @@ import ( cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" - builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" - account0 "github.com/filecoin-project/specs-actors/actors/builtin/account" - cron0 "github.com/filecoin-project/specs-actors/actors/builtin/cron" - init0 "github.com/filecoin-project/specs-actors/actors/builtin/init" - market0 "github.com/filecoin-project/specs-actors/actors/builtin/market" - miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" - msig0 "github.com/filecoin-project/specs-actors/actors/builtin/multisig" - paych0 "github.com/filecoin-project/specs-actors/actors/builtin/paych" - power0 "github.com/filecoin-project/specs-actors/actors/builtin/power" - reward0 "github.com/filecoin-project/specs-actors/actors/builtin/reward" - system0 "github.com/filecoin-project/specs-actors/actors/builtin/system" - verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" - - builtin1 "github.com/filecoin-project/specs-actors/v2/actors/builtin" - account1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/account" - cron1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/cron" - init1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/init" - market1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/market" - miner1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" - msig1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/multisig" - paych1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/paych" - power1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/power" - reward1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/reward" - system1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/system" - verifreg1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/verifreg" + exported0 "github.com/filecoin-project/specs-actors/actors/builtin/exported" + exported1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/exported" vmr "github.com/filecoin-project/specs-actors/v2/actors/runtime" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/exitcode" + rtt "github.com/filecoin-project/go-state-types/rt" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/aerrors" @@ -49,15 +27,27 @@ type ActorRegistry struct { actors map[cid.Cid]*actorInfo } +// An ActorPredicate returns an error if the given actor is not valid for the given runtime environment (e.g., chain height, version, etc.). +type ActorPredicate func(vmr.Runtime, rtt.VMActor) error + +func ActorsVersionPredicate(ver actors.Version) ActorPredicate { + return func(rt vmr.Runtime, v rtt.VMActor) error { + nver := actors.VersionForNetwork(rt.NetworkVersion()) + if nver != ver { + return xerrors.Errorf("actor %s is a version %d actor; chain only supports actor version %d at height %d", v.Code(), ver, nver, rt.CurrEpoch()) + } + return nil + } +} + type invokeFunc func(rt vmr.Runtime, params []byte) ([]byte, aerrors.ActorError) type nativeCode []invokeFunc type actorInfo struct { - methods nativeCode - stateType reflect.Type + methods nativeCode + vmActor rtt.VMActor // TODO: consider making this a network version range? - version actors.Version - singleton bool + predicate ActorPredicate } func NewActorRegistry() *ActorRegistry { @@ -66,29 +56,8 @@ func NewActorRegistry() *ActorRegistry { // TODO: define all these properties on the actors themselves, in specs-actors. // add builtInCode using: register(cid, singleton) - inv.Register(actors.Version0, builtin0.SystemActorCodeID, system0.Actor{}, abi.EmptyValue{}, true) - inv.Register(actors.Version0, builtin0.InitActorCodeID, init0.Actor{}, init0.State{}, true) - inv.Register(actors.Version0, builtin0.RewardActorCodeID, reward0.Actor{}, reward0.State{}, true) - inv.Register(actors.Version0, builtin0.CronActorCodeID, cron0.Actor{}, cron0.State{}, true) - inv.Register(actors.Version0, builtin0.StoragePowerActorCodeID, power0.Actor{}, power0.State{}, true) - inv.Register(actors.Version0, builtin0.StorageMarketActorCodeID, market0.Actor{}, market0.State{}, true) - inv.Register(actors.Version0, builtin0.VerifiedRegistryActorCodeID, verifreg0.Actor{}, verifreg0.State{}, true) - inv.Register(actors.Version0, builtin0.StorageMinerActorCodeID, miner0.Actor{}, miner0.State{}, false) - inv.Register(actors.Version0, builtin0.MultisigActorCodeID, msig0.Actor{}, msig0.State{}, false) - inv.Register(actors.Version0, builtin0.PaymentChannelActorCodeID, paych0.Actor{}, paych0.State{}, false) - inv.Register(actors.Version0, builtin0.AccountActorCodeID, account0.Actor{}, account0.State{}, false) - - inv.Register(actors.Version1, builtin1.SystemActorCodeID, system1.Actor{}, abi.EmptyValue{}, true) - inv.Register(actors.Version1, builtin1.InitActorCodeID, init1.Actor{}, init1.State{}, true) - inv.Register(actors.Version1, builtin1.RewardActorCodeID, reward1.Actor{}, reward1.State{}, true) - inv.Register(actors.Version1, builtin1.CronActorCodeID, cron1.Actor{}, cron1.State{}, true) - inv.Register(actors.Version1, builtin1.StoragePowerActorCodeID, power1.Actor{}, power1.State{}, true) - inv.Register(actors.Version1, builtin1.StorageMarketActorCodeID, market1.Actor{}, market1.State{}, true) - inv.Register(actors.Version1, builtin1.VerifiedRegistryActorCodeID, verifreg1.Actor{}, verifreg1.State{}, true) - inv.Register(actors.Version1, builtin1.StorageMinerActorCodeID, miner1.Actor{}, miner1.State{}, false) - inv.Register(actors.Version1, builtin1.MultisigActorCodeID, msig1.Actor{}, msig1.State{}, false) - inv.Register(actors.Version1, builtin1.PaymentChannelActorCodeID, paych1.Actor{}, paych1.State{}, false) - inv.Register(actors.Version1, builtin1.AccountActorCodeID, account1.Actor{}, account1.State{}, false) + inv.Register(ActorsVersionPredicate(actors.Version0), exported0.BuiltinActors()...) + inv.Register(ActorsVersionPredicate(actors.Version1), exported1.BuiltinActors()...) return inv } @@ -102,23 +71,27 @@ func (ar *ActorRegistry) Invoke(codeCid cid.Cid, rt vmr.Runtime, method abi.Meth if method >= abi.MethodNum(len(act.methods)) || act.methods[method] == nil { return nil, aerrors.Newf(exitcode.SysErrInvalidMethod, "no method %d on actor", method) } - if curVer := actors.VersionForNetwork(rt.NetworkVersion()); curVer != act.version { - return nil, aerrors.Newf(exitcode.SysErrInvalidMethod, "unsupported actors code version %d, expected %d", act.version, curVer) + if err := act.predicate(rt, act.vmActor); err != nil { + return nil, aerrors.Newf(exitcode.SysErrInvalidMethod, "unsupported actor: %s", err) } return act.methods[method](rt, params) } -func (ar *ActorRegistry) Register(version actors.Version, c cid.Cid, instance Invokee, state interface{}, singleton bool) { - code, err := ar.transform(instance) - if err != nil { - panic(xerrors.Errorf("%s: %w", string(c.Hash()), err)) +func (ar *ActorRegistry) Register(pred ActorPredicate, actors ...rtt.VMActor) { + if pred == nil { + pred = func(vmr.Runtime, rtt.VMActor) error { return nil } } - ar.actors[c] = &actorInfo{ - methods: code, - version: version, - stateType: reflect.TypeOf(state), - singleton: singleton, + for _, a := range actors { + code, err := ar.transform(a) + if err != nil { + panic(xerrors.Errorf("%s: %w", string(a.Code().Hash()), err)) + } + ar.actors[a.Code()] = &actorInfo{ + methods: code, + vmActor: a, + predicate: pred, + } } } @@ -127,11 +100,12 @@ func (ar *ActorRegistry) Create(codeCid cid.Cid, rt vmr.Runtime) (*types.Actor, if !ok { return nil, aerrors.Newf(exitcode.SysErrorIllegalArgument, "Can only create built-in actors.") } - if version := actors.VersionForNetwork(rt.NetworkVersion()); act.version != version { - return nil, aerrors.Newf(exitcode.SysErrorIllegalArgument, "Can only create version %d actors, attempted to create version %d actor", version, act.version) + + if err := act.predicate(rt, act.vmActor); err != nil { + return nil, aerrors.Newf(exitcode.SysErrorIllegalArgument, "Cannot create actor: %w", err) } - if act.singleton { + if rtt.IsSingletonActor(act.vmActor) { return nil, aerrors.Newf(exitcode.SysErrorIllegalArgument, "Can only have one instance of singleton actors.") } return &types.Actor{ @@ -142,11 +116,11 @@ func (ar *ActorRegistry) Create(codeCid cid.Cid, rt vmr.Runtime) (*types.Actor, }, nil } -type Invokee interface { +type invokee interface { Exports() []interface{} } -func (*ActorRegistry) transform(instance Invokee) (nativeCode, error) { +func (*ActorRegistry) transform(instance invokee) (nativeCode, error) { itype := reflect.TypeOf(instance) exports := instance.Exports() runtimeType := reflect.TypeOf((*vmr.Runtime)(nil)).Elem() @@ -247,15 +221,10 @@ func DumpActorState(act *types.Actor, b []byte) (interface{}, error) { return nil, xerrors.Errorf("state type for actor %s not found", act.Code) } - rv := reflect.New(actInfo.stateType) - um, ok := rv.Interface().(cbg.CBORUnmarshaler) - if !ok { - return nil, xerrors.New("state type does not implement CBORUnmarshaler") - } - + um := actInfo.vmActor.State() if err := um.UnmarshalCBOR(bytes.NewReader(b)); err != nil { return nil, xerrors.Errorf("unmarshaling actor state: %w", err) } - return rv.Elem().Interface(), nil + return um, nil } diff --git a/cli/state.go b/cli/state.go index d96c93c54..610fcac03 100644 --- a/cli/state.go +++ b/cli/state.go @@ -14,8 +14,6 @@ import ( "strings" "time" - "github.com/filecoin-project/specs-actors/actors/runtime" - "github.com/multiformats/go-multiaddr" "github.com/ipfs/go-cid" @@ -30,7 +28,6 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/exitcode" "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/builtin/exported" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" @@ -1513,28 +1510,18 @@ func parseParamsForMethod(act cid.Cid, method uint64, args []string) ([]byte, er return nil, nil } - var target runtime.Invokee - for _, actor := range exported.BuiltinActors() { - if actor.Code() == act { - target = actor - } - } - if target == nil { + // TODO: consider moving this to a dedicated helper + actMeta, ok := stmgr.MethodsMap[act] + if !ok { return nil, fmt.Errorf("unknown actor %s", act) } - methods := target.Exports() - if uint64(len(methods)) <= method || methods[method] == nil { + + methodMeta, ok := actMeta[abi.MethodNum(method)] + if !ok { return nil, fmt.Errorf("unknown method %d for actor %s", method, act) } - f := methods[method] - - rf := reflect.TypeOf(f) - if rf.NumIn() != 2 { - return nil, fmt.Errorf("expected referenced method to have three arguments") - } - - paramObj := rf.In(1).Elem() + paramObj := methodMeta.Params if paramObj.NumField() != len(args) { return nil, fmt.Errorf("not enough arguments given to call that method (expecting %d)", paramObj.NumField()) } diff --git a/cmd/lotus-chainwatch/processor/common_actors.go b/cmd/lotus-chainwatch/processor/common_actors.go index 56520880c..4b8d124e2 100644 --- a/cmd/lotus-chainwatch/processor/common_actors.go +++ b/cmd/lotus-chainwatch/processor/common_actors.go @@ -9,12 +9,15 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + "github.com/ipfs/go-cid" + + builtin1 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + + "github.com/filecoin-project/lotus/chain/actors/builtin" _init "github.com/filecoin-project/lotus/chain/actors/builtin/init" "github.com/filecoin-project/lotus/chain/events/state" "github.com/filecoin-project/lotus/chain/types" cw_util "github.com/filecoin-project/lotus/cmd/lotus-chainwatch/util" - "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/ipfs/go-cid" ) func (p *Processor) setupCommonActors() error { @@ -135,15 +138,15 @@ func (p Processor) storeActorAddresses(ctx context.Context, actors map[cid.Cid]A addressToID := map[address.Address]address.Address{} // HACK until genesis storage is figured out: - addressToID[builtin.SystemActorAddr] = builtin.SystemActorAddr - addressToID[builtin.InitActorAddr] = builtin.InitActorAddr - addressToID[builtin.RewardActorAddr] = builtin.RewardActorAddr - addressToID[builtin.CronActorAddr] = builtin.CronActorAddr - addressToID[builtin.StoragePowerActorAddr] = builtin.StoragePowerActorAddr - addressToID[builtin.StorageMarketActorAddr] = builtin.StorageMarketActorAddr - addressToID[builtin.VerifiedRegistryActorAddr] = builtin.VerifiedRegistryActorAddr - addressToID[builtin.BurntFundsActorAddr] = builtin.BurntFundsActorAddr - initActor, err := p.node.StateGetActor(ctx, builtin.InitActorAddr, types.EmptyTSK) + addressToID[builtin1.SystemActorAddr] = builtin1.SystemActorAddr + addressToID[builtin1.InitActorAddr] = builtin1.InitActorAddr + addressToID[builtin1.RewardActorAddr] = builtin1.RewardActorAddr + addressToID[builtin1.CronActorAddr] = builtin1.CronActorAddr + addressToID[builtin1.StoragePowerActorAddr] = builtin1.StoragePowerActorAddr + addressToID[builtin1.StorageMarketActorAddr] = builtin1.StorageMarketActorAddr + addressToID[builtin1.VerifiedRegistryActorAddr] = builtin1.VerifiedRegistryActorAddr + addressToID[builtin1.BurntFundsActorAddr] = builtin1.BurntFundsActorAddr + initActor, err := p.node.StateGetActor(ctx, builtin1.InitActorAddr, types.EmptyTSK) if err != nil { return err } diff --git a/conformance/chaos/actor.go b/conformance/chaos/actor.go index 477204374..d5e0b4352 100644 --- a/conformance/chaos/actor.go +++ b/conformance/chaos/actor.go @@ -3,7 +3,9 @@ package chaos import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/cbor" "github.com/filecoin-project/go-state-types/exitcode" + "github.com/filecoin-project/go-state-types/rt" "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/runtime" "github.com/ipfs/go-cid" @@ -86,7 +88,11 @@ func (a Actor) Exports() []interface{} { } } -var _ runtime.Invokee = Actor{} +func (a Actor) Code() cid.Cid { return ChaosActorCodeCID } +func (a Actor) State() cbor.Er { return new(State) } +func (a Actor) IsSingleton() bool { return true } + +var _ rt.VMActor = Actor{} // SendArgs are the arguments for the Send method. type SendArgs struct { diff --git a/conformance/driver.go b/conformance/driver.go index 1a63aaff3..7645e11c8 100644 --- a/conformance/driver.go +++ b/conformance/driver.go @@ -5,7 +5,6 @@ import ( "github.com/filecoin-project/go-state-types/crypto" - "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" @@ -145,7 +144,7 @@ func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, preroot cid.Cid, epoch // register the chaos actor if required by the vector. if chaosOn, ok := d.selector["chaos_actor"]; ok && chaosOn == "true" { - invoker.Register(actors.Version0, chaos.ChaosActorCodeCID, chaos.Actor{}, chaos.State{}, true) + invoker.Register(nil, chaos.Actor{}) } lvm.SetInvoker(invoker) diff --git a/go.mod b/go.mod index b19f5c053..8212fa826 100644 --- a/go.mod +++ b/go.mod @@ -32,12 +32,12 @@ require ( github.com/filecoin-project/go-multistore v0.0.3 github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 github.com/filecoin-project/go-paramfetch v0.0.2-0.20200701152213-3e0f0afdc261 - github.com/filecoin-project/go-state-types v0.0.0-20200911004822-964d6c679cfc + github.com/filecoin-project/go-state-types v0.0.0-20200928172055-2df22083d8ab github.com/filecoin-project/go-statemachine v0.0.0-20200813232949-df9b130df370 github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b - github.com/filecoin-project/specs-actors v0.9.10 - github.com/filecoin-project/specs-actors/v2 v2.0.0-20200918035954-4caac0a9b252 + github.com/filecoin-project/specs-actors v0.9.12-0.20200928180918-488a087c5add + github.com/filecoin-project/specs-actors/v2 v2.0.0-20200928175842-971c8d772684 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 github.com/filecoin-project/test-vectors/schema v0.0.1 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 diff --git a/go.sum b/go.sum index 6f5ab862d..b96159c10 100644 --- a/go.sum +++ b/go.sum @@ -247,6 +247,8 @@ github.com/filecoin-project/go-state-types v0.0.0-20200904021452-1883f36ca2f4/go github.com/filecoin-project/go-state-types v0.0.0-20200905071437-95828685f9df/go.mod h1:IQ0MBPnonv35CJHtWSN3YY1Hz2gkPru1Q9qoaYLxx9I= github.com/filecoin-project/go-state-types v0.0.0-20200911004822-964d6c679cfc h1:1vr/LoqGq5m5g37Q3sNSAjfwF1uJY0zmiHcvnxY6hik= github.com/filecoin-project/go-state-types v0.0.0-20200911004822-964d6c679cfc/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= +github.com/filecoin-project/go-state-types v0.0.0-20200928172055-2df22083d8ab h1:cEDC5Ei8UuT99hPWhCjA72SM9AuRtnpvdSTIYbnzN8I= +github.com/filecoin-project/go-state-types v0.0.0-20200928172055-2df22083d8ab/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-statemachine v0.0.0-20200714194326-a77c3ae20989/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= github.com/filecoin-project/go-statemachine v0.0.0-20200813232949-df9b130df370 h1:Jbburj7Ih2iaJ/o5Q9A+EAeTabME6YII7FLi9SKUf5c= github.com/filecoin-project/go-statemachine v0.0.0-20200813232949-df9b130df370/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= @@ -257,10 +259,10 @@ github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/ github.com/filecoin-project/specs-actors v0.9.4/go.mod h1:BStZQzx5x7TmCkLv0Bpa07U6cPKol6fd3w9KjMPZ6Z4= github.com/filecoin-project/specs-actors v0.9.7/go.mod h1:wM2z+kwqYgXn5Z7scV1YHLyd1Q1cy0R8HfTIWQ0BFGU= github.com/filecoin-project/specs-actors v0.9.9/go.mod h1:czlvLQGEX0fjLLfdNHD7xLymy6L3n7aQzRWzsYGf+ys= -github.com/filecoin-project/specs-actors v0.9.10 h1:gU0TrRhgkCsBEOP42sGDE7RQuR0Cov9hJhBqq+RJmjU= -github.com/filecoin-project/specs-actors v0.9.10/go.mod h1:czlvLQGEX0fjLLfdNHD7xLymy6L3n7aQzRWzsYGf+ys= -github.com/filecoin-project/specs-actors/v2 v2.0.0-20200918035954-4caac0a9b252 h1:0vOZo6xlVDyPhuRS3ArrAy1fml7H2FEY65IFx6rwp3o= -github.com/filecoin-project/specs-actors/v2 v2.0.0-20200918035954-4caac0a9b252/go.mod h1:vV1UOOKlmdKg4dy3YKixtJtVf15WMuZsS6KgKRDxkWg= +github.com/filecoin-project/specs-actors v0.9.12-0.20200928180918-488a087c5add h1:iXQOxr8uSyZ/qnTlOZf7ALp0io+jwLxmuWsNAk/YdoQ= +github.com/filecoin-project/specs-actors v0.9.12-0.20200928180918-488a087c5add/go.mod h1:TS1AW/7LbG+615j4NsjMK1qlpAwaFsG9w0V2tg2gSao= +github.com/filecoin-project/specs-actors/v2 v2.0.0-20200928175842-971c8d772684 h1:sjWZqblOOf1RaohI9w2R2AVp5uifNdzsusy7oVi5ioU= +github.com/filecoin-project/specs-actors/v2 v2.0.0-20200928175842-971c8d772684/go.mod h1:/2Zra1BhLtpRywUhm++QP+3I5Ir+hBk/W24TpYjj43E= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= github.com/filecoin-project/test-vectors/schema v0.0.1 h1:5fNF76nl4qolEvcIsjc0kUADlTMVHO73tW4kXXPnsus= From 2049daa008b4029eaa551aaaf09a392c50a862d3 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 28 Sep 2020 13:12:41 -0700 Subject: [PATCH 391/885] fix actor imports --- chain/actors/builtin/multisig/v1.go | 4 ++-- chain/actors/builtin/power/v1.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/chain/actors/builtin/multisig/v1.go b/chain/actors/builtin/multisig/v1.go index 7bfde404a..56760cbba 100644 --- a/chain/actors/builtin/multisig/v1.go +++ b/chain/actors/builtin/multisig/v1.go @@ -10,8 +10,8 @@ import ( "github.com/filecoin-project/lotus/chain/actors/adt" - msig1 "github.com/filecoin-project/specs-actors/actors/builtin/multisig" - adt1 "github.com/filecoin-project/specs-actors/actors/util/adt" + msig1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/multisig" + adt1 "github.com/filecoin-project/specs-actors/v2/actors/util/adt" ) var _ State = (*state1)(nil) diff --git a/chain/actors/builtin/power/v1.go b/chain/actors/builtin/power/v1.go index 38df11916..63c9df547 100644 --- a/chain/actors/builtin/power/v1.go +++ b/chain/actors/builtin/power/v1.go @@ -8,8 +8,8 @@ import ( "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/actors/builtin" - adt1 "github.com/filecoin-project/specs-actors/actors/util/adt" power1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/power" + adt1 "github.com/filecoin-project/specs-actors/v2/actors/util/adt" ) var _ State = (*state1)(nil) From ca9448bc1171d02a365fe0a066c2568d826dd4dc Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 28 Sep 2020 13:13:18 -0700 Subject: [PATCH 392/885] rename actors v1 -> actors v2 The actual actors version is v2, not v1. Using Version1 internally was really confusing. --- chain/actors/adt/adt.go | 18 +- chain/actors/builtin/account/account.go | 10 +- chain/actors/builtin/account/{v1.go => v2.go} | 14 +- chain/actors/builtin/builtin.go | 12 +- chain/actors/builtin/init/init.go | 10 +- chain/actors/builtin/init/{v1.go => v2.go} | 28 +-- chain/actors/builtin/market/market.go | 10 +- chain/actors/builtin/market/v1.go | 205 ------------------ chain/actors/builtin/market/v2.go | 205 ++++++++++++++++++ chain/actors/builtin/miner/miner.go | 12 +- chain/actors/builtin/miner/{v1.go => v2.go} | 180 +++++++-------- chain/actors/builtin/multisig/multisig.go | 10 +- .../actors/builtin/multisig/{v1.go => v2.go} | 32 +-- chain/actors/builtin/paych/paych.go | 10 +- chain/actors/builtin/paych/{v1.go => v2.go} | 44 ++-- chain/actors/builtin/power/power.go | 10 +- chain/actors/builtin/power/{v1.go => v2.go} | 38 ++-- chain/actors/builtin/reward/reward.go | 10 +- chain/actors/builtin/reward/{v1.go => v2.go} | 44 ++-- chain/actors/builtin/verifreg/v1.go | 44 ---- chain/actors/builtin/verifreg/v2.go | 44 ++++ chain/actors/builtin/verifreg/verifreg.go | 10 +- chain/actors/version.go | 4 +- chain/state/statetree.go | 4 +- chain/stmgr/utils.go | 10 +- chain/vm/invoker.go | 4 +- chain/vm/mkactor.go | 6 +- .../processor/common_actors.go | 20 +- journal/registry_test.go | 10 +- journal/types.go | 2 +- 30 files changed, 530 insertions(+), 530 deletions(-) rename chain/actors/builtin/account/{v1.go => v2.go} (53%) rename chain/actors/builtin/init/{v1.go => v2.go} (65%) delete mode 100644 chain/actors/builtin/market/v1.go create mode 100644 chain/actors/builtin/market/v2.go rename chain/actors/builtin/miner/{v1.go => v2.go} (59%) rename chain/actors/builtin/multisig/{v1.go => v2.go} (55%) rename chain/actors/builtin/paych/{v1.go => v2.go} (61%) rename chain/actors/builtin/power/{v1.go => v2.go} (62%) rename chain/actors/builtin/reward/{v1.go => v2.go} (56%) delete mode 100644 chain/actors/builtin/verifreg/v1.go create mode 100644 chain/actors/builtin/verifreg/v2.go diff --git a/chain/actors/adt/adt.go b/chain/actors/adt/adt.go index 778ba9fcf..6a454ac26 100644 --- a/chain/actors/adt/adt.go +++ b/chain/actors/adt/adt.go @@ -11,7 +11,7 @@ import ( "github.com/filecoin-project/lotus/chain/actors" adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" - adt1 "github.com/filecoin-project/specs-actors/v2/actors/util/adt" + adt2 "github.com/filecoin-project/specs-actors/v2/actors/util/adt" ) type Map interface { @@ -28,8 +28,8 @@ func AsMap(store Store, root cid.Cid, version actors.Version) (Map, error) { switch version { case actors.Version0: return adt0.AsMap(store, root) - case actors.Version1: - return adt1.AsMap(store, root) + case actors.Version2: + return adt2.AsMap(store, root) } return nil, xerrors.Errorf("unknown network version: %d", version) } @@ -38,8 +38,8 @@ func NewMap(store Store, version actors.Version) (Map, error) { switch version { case actors.Version0: return adt0.MakeEmptyMap(store), nil - case actors.Version1: - return adt1.MakeEmptyMap(store), nil + case actors.Version2: + return adt2.MakeEmptyMap(store), nil } return nil, xerrors.Errorf("unknown network version: %d", version) } @@ -59,8 +59,8 @@ func AsArray(store Store, root cid.Cid, version network.Version) (Array, error) switch actors.VersionForNetwork(version) { case actors.Version0: return adt0.AsArray(store, root) - case actors.Version1: - return adt1.AsArray(store, root) + case actors.Version2: + return adt2.AsArray(store, root) } return nil, xerrors.Errorf("unknown network version: %d", version) } @@ -69,8 +69,8 @@ func NewArray(store Store, version actors.Version) (Array, error) { switch version { case actors.Version0: return adt0.MakeEmptyArray(store), nil - case actors.Version1: - return adt1.MakeEmptyArray(store), nil + case actors.Version2: + return adt2.MakeEmptyArray(store), nil } return nil, xerrors.Errorf("unknown network version: %d", version) } diff --git a/chain/actors/builtin/account/account.go b/chain/actors/builtin/account/account.go index 2754607e2..7b1b2a792 100644 --- a/chain/actors/builtin/account/account.go +++ b/chain/actors/builtin/account/account.go @@ -12,15 +12,15 @@ import ( "github.com/filecoin-project/lotus/chain/types" builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" - builtin1 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" ) func init() { builtin.RegisterActorState(builtin0.AccountActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) { return load0(store, root) }) - builtin.RegisterActorState(builtin1.AccountActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) { - return load1(store, root) + builtin.RegisterActorState(builtin2.AccountActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) { + return load2(store, root) }) } @@ -28,8 +28,8 @@ func Load(store adt.Store, act *types.Actor) (State, error) { switch act.Code { case builtin0.AccountActorCodeID: return load0(store, act.Head) - case builtin1.AccountActorCodeID: - return load1(store, act.Head) + case builtin2.AccountActorCodeID: + return load2(store, act.Head) } return nil, xerrors.Errorf("unknown actor code %s", act.Code) } diff --git a/chain/actors/builtin/account/v1.go b/chain/actors/builtin/account/v2.go similarity index 53% rename from chain/actors/builtin/account/v1.go rename to chain/actors/builtin/account/v2.go index 993c0e397..2664631bc 100644 --- a/chain/actors/builtin/account/v1.go +++ b/chain/actors/builtin/account/v2.go @@ -6,13 +6,13 @@ import ( "github.com/filecoin-project/lotus/chain/actors/adt" - account1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/account" + account2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/account" ) -var _ State = (*state1)(nil) +var _ State = (*state2)(nil) -func load1(store adt.Store, root cid.Cid) (State, error) { - out := state1{store: store} +func load2(store adt.Store, root cid.Cid) (State, error) { + out := state2{store: store} err := store.Get(store.Context(), root, &out) if err != nil { return nil, err @@ -20,11 +20,11 @@ func load1(store adt.Store, root cid.Cid) (State, error) { return &out, nil } -type state1 struct { - account1.State +type state2 struct { + account2.State store adt.Store } -func (s *state1) PubkeyAddress() (address.Address, error) { +func (s *state2) PubkeyAddress() (address.Address, error) { return s.Address, nil } diff --git a/chain/actors/builtin/builtin.go b/chain/actors/builtin/builtin.go index a0567dc27..784c004eb 100644 --- a/chain/actors/builtin/builtin.go +++ b/chain/actors/builtin/builtin.go @@ -14,8 +14,8 @@ import ( miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" proof0 "github.com/filecoin-project/specs-actors/actors/runtime/proof" smoothing0 "github.com/filecoin-project/specs-actors/actors/util/smoothing" - builtin1 "github.com/filecoin-project/specs-actors/v2/actors/builtin" - smoothing1 "github.com/filecoin-project/specs-actors/v2/actors/util/smoothing" + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + smoothing2 "github.com/filecoin-project/specs-actors/v2/actors/util/smoothing" ) // TODO: Why does actors have 2 different versions of this? @@ -32,7 +32,7 @@ func QAPowerForWeight(size abi.SectorSize, duration abi.ChainEpoch, dealWeight, return miner0.QAPowerForWeight(size, duration, dealWeight, verifiedWeight) } -func FromV1FilterEstimate(v1 smoothing1.FilterEstimate) FilterEstimate { +func FromV2FilterEstimate(v1 smoothing2.FilterEstimate) FilterEstimate { return (FilterEstimate)(v1) } @@ -56,13 +56,13 @@ func ActorNameByCode(c cid.Cid) string { switch { case builtin0.IsBuiltinActor(c): return builtin0.ActorNameByCode(c) - case builtin1.IsBuiltinActor(c): - return builtin1.ActorNameByCode(c) + case builtin2.IsBuiltinActor(c): + return builtin2.ActorNameByCode(c) default: return "" } } func IsBuiltinActor(c cid.Cid) bool { - return builtin0.IsBuiltinActor(c) || builtin1.IsBuiltinActor(c) + return builtin0.IsBuiltinActor(c) || builtin2.IsBuiltinActor(c) } diff --git a/chain/actors/builtin/init/init.go b/chain/actors/builtin/init/init.go index 622921ee9..d1a41b158 100644 --- a/chain/actors/builtin/init/init.go +++ b/chain/actors/builtin/init/init.go @@ -14,15 +14,15 @@ import ( "github.com/filecoin-project/lotus/node/modules/dtypes" builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" - builtin1 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" ) func init() { builtin.RegisterActorState(builtin0.InitActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) { return load0(store, root) }) - builtin.RegisterActorState(builtin1.InitActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) { - return load1(store, root) + builtin.RegisterActorState(builtin2.InitActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) { + return load2(store, root) }) } @@ -32,8 +32,8 @@ func Load(store adt.Store, act *types.Actor) (State, error) { switch act.Code { case builtin0.InitActorCodeID: return load0(store, act.Head) - case builtin1.InitActorCodeID: - return load1(store, act.Head) + case builtin2.InitActorCodeID: + return load2(store, act.Head) } return nil, xerrors.Errorf("unknown actor code %s", act.Code) } diff --git a/chain/actors/builtin/init/v1.go b/chain/actors/builtin/init/v2.go similarity index 65% rename from chain/actors/builtin/init/v1.go rename to chain/actors/builtin/init/v2.go index dcaa5a9ed..38b3099fd 100644 --- a/chain/actors/builtin/init/v1.go +++ b/chain/actors/builtin/init/v2.go @@ -10,14 +10,14 @@ import ( "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/node/modules/dtypes" - init1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/init" - adt1 "github.com/filecoin-project/specs-actors/v2/actors/util/adt" + init2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/init" + adt2 "github.com/filecoin-project/specs-actors/v2/actors/util/adt" ) -var _ State = (*state1)(nil) +var _ State = (*state2)(nil) -func load1(store adt.Store, root cid.Cid) (State, error) { - out := state1{store: store} +func load2(store adt.Store, root cid.Cid) (State, error) { + out := state2{store: store} err := store.Get(store.Context(), root, &out) if err != nil { return nil, err @@ -25,21 +25,21 @@ func load1(store adt.Store, root cid.Cid) (State, error) { return &out, nil } -type state1 struct { - init1.State +type state2 struct { + init2.State store adt.Store } -func (s *state1) ResolveAddress(address address.Address) (address.Address, bool, error) { +func (s *state2) ResolveAddress(address address.Address) (address.Address, bool, error) { return s.State.ResolveAddress(s.store, address) } -func (s *state1) MapAddressToNewID(address address.Address) (address.Address, error) { +func (s *state2) MapAddressToNewID(address address.Address) (address.Address, error) { return s.State.MapAddressToNewID(s.store, address) } -func (s *state1) ForEachActor(cb func(id abi.ActorID, address address.Address) error) error { - addrs, err := adt1.AsMap(s.store, s.State.AddressMap) +func (s *state2) ForEachActor(cb func(id abi.ActorID, address address.Address) error) error { + addrs, err := adt2.AsMap(s.store, s.State.AddressMap) if err != nil { return err } @@ -53,12 +53,12 @@ func (s *state1) ForEachActor(cb func(id abi.ActorID, address address.Address) e }) } -func (s *state1) NetworkName() (dtypes.NetworkName, error) { +func (s *state2) NetworkName() (dtypes.NetworkName, error) { return dtypes.NetworkName(s.State.NetworkName), nil } -func (s *state1) Remove(addrs ...address.Address) (err error) { - m, err := adt1.AsMap(s.store, s.State.AddressMap) +func (s *state2) Remove(addrs ...address.Address) (err error) { + m, err := adt2.AsMap(s.store, s.State.AddressMap) if err != nil { return err } diff --git a/chain/actors/builtin/market/market.go b/chain/actors/builtin/market/market.go index d0798560b..fd08a0119 100644 --- a/chain/actors/builtin/market/market.go +++ b/chain/actors/builtin/market/market.go @@ -11,7 +11,7 @@ import ( builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" market0 "github.com/filecoin-project/specs-actors/actors/builtin/market" - builtin1 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/actors/builtin" @@ -22,8 +22,8 @@ func init() { builtin.RegisterActorState(builtin0.StorageMarketActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) { return load0(store, root) }) - builtin.RegisterActorState(builtin1.StorageMarketActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) { - return load1(store, root) + builtin.RegisterActorState(builtin2.StorageMarketActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) { + return load2(store, root) }) } @@ -33,8 +33,8 @@ func Load(store adt.Store, act *types.Actor) (st State, err error) { switch act.Code { case builtin0.StorageMarketActorCodeID: return load0(store, act.Head) - case builtin1.StorageMarketActorCodeID: - return load1(store, act.Head) + case builtin2.StorageMarketActorCodeID: + return load2(store, act.Head) } return nil, xerrors.Errorf("unknown actor code %s", act.Code) } diff --git a/chain/actors/builtin/market/v1.go b/chain/actors/builtin/market/v1.go deleted file mode 100644 index 3ad0cded6..000000000 --- a/chain/actors/builtin/market/v1.go +++ /dev/null @@ -1,205 +0,0 @@ -package market - -import ( - "bytes" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-state-types/abi" - "github.com/ipfs/go-cid" - cbg "github.com/whyrusleeping/cbor-gen" - - "github.com/filecoin-project/lotus/chain/actors/adt" - "github.com/filecoin-project/lotus/chain/types" - - market1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/market" - adt1 "github.com/filecoin-project/specs-actors/v2/actors/util/adt" -) - -var _ State = (*state1)(nil) - -func load1(store adt.Store, root cid.Cid) (State, error) { - out := state1{store: store} - err := store.Get(store.Context(), root, &out) - if err != nil { - return nil, err - } - return &out, nil -} - -type state1 struct { - market1.State - store adt.Store -} - -func (s *state1) TotalLocked() (abi.TokenAmount, error) { - fml := types.BigAdd(s.TotalClientLockedCollateral, s.TotalProviderLockedCollateral) - fml = types.BigAdd(fml, s.TotalClientStorageFee) - return fml, nil -} - -func (s *state1) BalancesChanged(otherState State) (bool, error) { - otherState1, ok := otherState.(*state1) - if !ok { - // there's no way to compare different versions of the state, so let's - // just say that means the state of balances has changed - return true, nil - } - return !s.State.EscrowTable.Equals(otherState1.State.EscrowTable) || !s.State.LockedTable.Equals(otherState1.State.LockedTable), nil -} - -func (s *state1) StatesChanged(otherState State) (bool, error) { - otherState1, ok := otherState.(*state1) - if !ok { - // there's no way to compare different versions of the state, so let's - // just say that means the state of balances has changed - return true, nil - } - return !s.State.States.Equals(otherState1.State.States), nil -} - -func (s *state1) States() (DealStates, error) { - stateArray, err := adt1.AsArray(s.store, s.State.States) - if err != nil { - return nil, err - } - return &dealStates1{stateArray}, nil -} - -func (s *state1) ProposalsChanged(otherState State) (bool, error) { - otherState1, ok := otherState.(*state1) - if !ok { - // there's no way to compare different versions of the state, so let's - // just say that means the state of balances has changed - return true, nil - } - return !s.State.Proposals.Equals(otherState1.State.Proposals), nil -} - -func (s *state1) Proposals() (DealProposals, error) { - proposalArray, err := adt1.AsArray(s.store, s.State.Proposals) - if err != nil { - return nil, err - } - return &dealProposals1{proposalArray}, nil -} - -func (s *state1) EscrowTable() (BalanceTable, error) { - bt, err := adt1.AsBalanceTable(s.store, s.State.EscrowTable) - if err != nil { - return nil, err - } - return &balanceTable1{bt}, nil -} - -func (s *state1) LockedTable() (BalanceTable, error) { - bt, err := adt1.AsBalanceTable(s.store, s.State.LockedTable) - if err != nil { - return nil, err - } - return &balanceTable1{bt}, nil -} - -func (s *state1) VerifyDealsForActivation( - minerAddr address.Address, deals []abi.DealID, currEpoch, sectorExpiry abi.ChainEpoch, -) (weight, verifiedWeight abi.DealWeight, err error) { - w, vw, _, err := market1.ValidateDealsForActivation(&s.State, s.store, deals, minerAddr, sectorExpiry, currEpoch) - return w, vw, err -} - -type balanceTable1 struct { - *adt1.BalanceTable -} - -func (bt *balanceTable1) ForEach(cb func(address.Address, abi.TokenAmount) error) error { - asMap := (*adt1.Map)(bt.BalanceTable) - var ta abi.TokenAmount - return asMap.ForEach(&ta, func(key string) error { - a, err := address.NewFromBytes([]byte(key)) - if err != nil { - return err - } - return cb(a, ta) - }) -} - -type dealStates1 struct { - adt.Array -} - -func (s *dealStates1) Get(dealID abi.DealID) (*DealState, bool, error) { - var deal1 market1.DealState - found, err := s.Array.Get(uint64(dealID), &deal1) - if err != nil { - return nil, false, err - } - if !found { - return nil, false, nil - } - deal := fromV1DealState(deal1) - return &deal, true, nil -} - -func (s *dealStates1) ForEach(cb func(dealID abi.DealID, ds DealState) error) error { - var ds1 market1.DealState - return s.Array.ForEach(&ds1, func(idx int64) error { - return cb(abi.DealID(idx), fromV1DealState(ds1)) - }) -} - -func (s *dealStates1) decode(val *cbg.Deferred) (*DealState, error) { - var ds1 market1.DealState - if err := ds1.UnmarshalCBOR(bytes.NewReader(val.Raw)); err != nil { - return nil, err - } - ds := fromV1DealState(ds1) - return &ds, nil -} - -func (s *dealStates1) array() adt.Array { - return s.Array -} - -func fromV1DealState(v1 market1.DealState) DealState { - return (DealState)(v1) -} - -type dealProposals1 struct { - adt.Array -} - -func (s *dealProposals1) Get(dealID abi.DealID) (*DealProposal, bool, error) { - var proposal1 market1.DealProposal - found, err := s.Array.Get(uint64(dealID), &proposal1) - if err != nil { - return nil, false, err - } - if !found { - return nil, false, nil - } - proposal := fromV1DealProposal(proposal1) - return &proposal, true, nil -} - -func (s *dealProposals1) ForEach(cb func(dealID abi.DealID, dp DealProposal) error) error { - var dp1 market1.DealProposal - return s.Array.ForEach(&dp1, func(idx int64) error { - return cb(abi.DealID(idx), fromV1DealProposal(dp1)) - }) -} - -func (s *dealProposals1) decode(val *cbg.Deferred) (*DealProposal, error) { - var dp1 market1.DealProposal - if err := dp1.UnmarshalCBOR(bytes.NewReader(val.Raw)); err != nil { - return nil, err - } - dp := fromV1DealProposal(dp1) - return &dp, nil -} - -func (s *dealProposals1) array() adt.Array { - return s.Array -} - -func fromV1DealProposal(v1 market1.DealProposal) DealProposal { - return (DealProposal)(v1) -} diff --git a/chain/actors/builtin/market/v2.go b/chain/actors/builtin/market/v2.go new file mode 100644 index 000000000..a5e5c7b45 --- /dev/null +++ b/chain/actors/builtin/market/v2.go @@ -0,0 +1,205 @@ +package market + +import ( + "bytes" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + + "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/chain/types" + + market2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/market" + adt2 "github.com/filecoin-project/specs-actors/v2/actors/util/adt" +) + +var _ State = (*state2)(nil) + +func load2(store adt.Store, root cid.Cid) (State, error) { + out := state2{store: store} + err := store.Get(store.Context(), root, &out) + if err != nil { + return nil, err + } + return &out, nil +} + +type state2 struct { + market2.State + store adt.Store +} + +func (s *state2) TotalLocked() (abi.TokenAmount, error) { + fml := types.BigAdd(s.TotalClientLockedCollateral, s.TotalProviderLockedCollateral) + fml = types.BigAdd(fml, s.TotalClientStorageFee) + return fml, nil +} + +func (s *state2) BalancesChanged(otherState State) (bool, error) { + otherState2, ok := otherState.(*state2) + if !ok { + // there's no way to compare different versions of the state, so let's + // just say that means the state of balances has changed + return true, nil + } + return !s.State.EscrowTable.Equals(otherState2.State.EscrowTable) || !s.State.LockedTable.Equals(otherState2.State.LockedTable), nil +} + +func (s *state2) StatesChanged(otherState State) (bool, error) { + otherState2, ok := otherState.(*state2) + if !ok { + // there's no way to compare different versions of the state, so let's + // just say that means the state of balances has changed + return true, nil + } + return !s.State.States.Equals(otherState2.State.States), nil +} + +func (s *state2) States() (DealStates, error) { + stateArray, err := adt2.AsArray(s.store, s.State.States) + if err != nil { + return nil, err + } + return &dealStates2{stateArray}, nil +} + +func (s *state2) ProposalsChanged(otherState State) (bool, error) { + otherState2, ok := otherState.(*state2) + if !ok { + // there's no way to compare different versions of the state, so let's + // just say that means the state of balances has changed + return true, nil + } + return !s.State.Proposals.Equals(otherState2.State.Proposals), nil +} + +func (s *state2) Proposals() (DealProposals, error) { + proposalArray, err := adt2.AsArray(s.store, s.State.Proposals) + if err != nil { + return nil, err + } + return &dealProposals2{proposalArray}, nil +} + +func (s *state2) EscrowTable() (BalanceTable, error) { + bt, err := adt2.AsBalanceTable(s.store, s.State.EscrowTable) + if err != nil { + return nil, err + } + return &balanceTable2{bt}, nil +} + +func (s *state2) LockedTable() (BalanceTable, error) { + bt, err := adt2.AsBalanceTable(s.store, s.State.LockedTable) + if err != nil { + return nil, err + } + return &balanceTable2{bt}, nil +} + +func (s *state2) VerifyDealsForActivation( + minerAddr address.Address, deals []abi.DealID, currEpoch, sectorExpiry abi.ChainEpoch, +) (weight, verifiedWeight abi.DealWeight, err error) { + w, vw, _, err := market2.ValidateDealsForActivation(&s.State, s.store, deals, minerAddr, sectorExpiry, currEpoch) + return w, vw, err +} + +type balanceTable2 struct { + *adt2.BalanceTable +} + +func (bt *balanceTable2) ForEach(cb func(address.Address, abi.TokenAmount) error) error { + asMap := (*adt2.Map)(bt.BalanceTable) + var ta abi.TokenAmount + return asMap.ForEach(&ta, func(key string) error { + a, err := address.NewFromBytes([]byte(key)) + if err != nil { + return err + } + return cb(a, ta) + }) +} + +type dealStates2 struct { + adt.Array +} + +func (s *dealStates2) Get(dealID abi.DealID) (*DealState, bool, error) { + var deal2 market2.DealState + found, err := s.Array.Get(uint64(dealID), &deal2) + if err != nil { + return nil, false, err + } + if !found { + return nil, false, nil + } + deal := fromV2DealState(deal2) + return &deal, true, nil +} + +func (s *dealStates2) ForEach(cb func(dealID abi.DealID, ds DealState) error) error { + var ds1 market2.DealState + return s.Array.ForEach(&ds1, func(idx int64) error { + return cb(abi.DealID(idx), fromV2DealState(ds1)) + }) +} + +func (s *dealStates2) decode(val *cbg.Deferred) (*DealState, error) { + var ds1 market2.DealState + if err := ds1.UnmarshalCBOR(bytes.NewReader(val.Raw)); err != nil { + return nil, err + } + ds := fromV2DealState(ds1) + return &ds, nil +} + +func (s *dealStates2) array() adt.Array { + return s.Array +} + +func fromV2DealState(v1 market2.DealState) DealState { + return (DealState)(v1) +} + +type dealProposals2 struct { + adt.Array +} + +func (s *dealProposals2) Get(dealID abi.DealID) (*DealProposal, bool, error) { + var proposal2 market2.DealProposal + found, err := s.Array.Get(uint64(dealID), &proposal2) + if err != nil { + return nil, false, err + } + if !found { + return nil, false, nil + } + proposal := fromV2DealProposal(proposal2) + return &proposal, true, nil +} + +func (s *dealProposals2) ForEach(cb func(dealID abi.DealID, dp DealProposal) error) error { + var dp1 market2.DealProposal + return s.Array.ForEach(&dp1, func(idx int64) error { + return cb(abi.DealID(idx), fromV2DealProposal(dp1)) + }) +} + +func (s *dealProposals2) decode(val *cbg.Deferred) (*DealProposal, error) { + var dp1 market2.DealProposal + if err := dp1.UnmarshalCBOR(bytes.NewReader(val.Raw)); err != nil { + return nil, err + } + dp := fromV2DealProposal(dp1) + return &dp, nil +} + +func (s *dealProposals2) array() adt.Array { + return s.Array +} + +func fromV2DealProposal(v1 market2.DealProposal) DealProposal { + return (DealProposal)(v1) +} diff --git a/chain/actors/builtin/miner/miner.go b/chain/actors/builtin/miner/miner.go index 00687a76f..66a157588 100644 --- a/chain/actors/builtin/miner/miner.go +++ b/chain/actors/builtin/miner/miner.go @@ -18,19 +18,19 @@ import ( builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" - builtin1 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" ) func init() { builtin.RegisterActorState(builtin0.StorageMinerActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) { return load0(store, root) }) - builtin.RegisterActorState(builtin1.StorageMinerActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) { - return load1(store, root) + builtin.RegisterActorState(builtin2.StorageMinerActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) { + return load2(store, root) }) } -// Unchanged between v0 and v1 actors +// Unchanged between v0 and v2 actors var WPoStProvingPeriod = miner0.WPoStProvingPeriod const MinSectorExpiration = miner0.MinSectorExpiration @@ -39,8 +39,8 @@ func Load(store adt.Store, act *types.Actor) (st State, err error) { switch act.Code { case builtin0.StorageMinerActorCodeID: return load0(store, act.Head) - case builtin1.StorageMinerActorCodeID: - return load1(store, act.Head) + case builtin2.StorageMinerActorCodeID: + return load2(store, act.Head) } return nil, xerrors.Errorf("unknown actor code %s", act.Code) } diff --git a/chain/actors/builtin/miner/v1.go b/chain/actors/builtin/miner/v2.go similarity index 59% rename from chain/actors/builtin/miner/v1.go rename to chain/actors/builtin/miner/v2.go index 9259353f3..6f496a51e 100644 --- a/chain/actors/builtin/miner/v1.go +++ b/chain/actors/builtin/miner/v2.go @@ -14,14 +14,14 @@ import ( "github.com/filecoin-project/lotus/chain/actors/adt" - miner1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" - adt1 "github.com/filecoin-project/specs-actors/v2/actors/util/adt" + miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" + adt2 "github.com/filecoin-project/specs-actors/v2/actors/util/adt" ) -var _ State = (*state1)(nil) +var _ State = (*state2)(nil) -func load1(store adt.Store, root cid.Cid) (State, error) { - out := state1{store: store} +func load2(store adt.Store, root cid.Cid) (State, error) { + out := state2{store: store} err := store.Get(store.Context(), root, &out) if err != nil { return nil, err @@ -29,30 +29,30 @@ func load1(store adt.Store, root cid.Cid) (State, error) { return &out, nil } -type state1 struct { - miner1.State +type state2 struct { + miner2.State store adt.Store } -type deadline1 struct { - miner1.Deadline +type deadline2 struct { + miner2.Deadline store adt.Store } -type partition1 struct { - miner1.Partition +type partition2 struct { + miner2.Partition store adt.Store } -func (s *state1) AvailableBalance(bal abi.TokenAmount) (abi.TokenAmount, error) { +func (s *state2) AvailableBalance(bal abi.TokenAmount) (abi.TokenAmount, error) { return s.GetAvailableBalance(bal), nil } -func (s *state1) VestedFunds(epoch abi.ChainEpoch) (abi.TokenAmount, error) { +func (s *state2) VestedFunds(epoch abi.ChainEpoch) (abi.TokenAmount, error) { return s.CheckVestedFunds(s.store, epoch) } -func (s *state1) LockedFunds() (LockedFunds, error) { +func (s *state2) LockedFunds() (LockedFunds, error) { return LockedFunds{ VestingFunds: s.State.LockedFunds, InitialPledgeRequirement: s.State.InitialPledge, @@ -60,25 +60,25 @@ func (s *state1) LockedFunds() (LockedFunds, error) { }, nil } -func (s *state1) InitialPledge() (abi.TokenAmount, error) { +func (s *state2) InitialPledge() (abi.TokenAmount, error) { return s.State.InitialPledge, nil } -func (s *state1) PreCommitDeposits() (abi.TokenAmount, error) { +func (s *state2) PreCommitDeposits() (abi.TokenAmount, error) { return s.State.PreCommitDeposits, nil } -func (s *state1) GetSector(num abi.SectorNumber) (*SectorOnChainInfo, error) { +func (s *state2) GetSector(num abi.SectorNumber) (*SectorOnChainInfo, error) { info, ok, err := s.State.GetSector(s.store, num) if !ok || err != nil { return nil, err } - ret := fromV1SectorOnChainInfo(*info) + ret := fromV2SectorOnChainInfo(*info) return &ret, nil } -func (s *state1) FindSector(num abi.SectorNumber) (*SectorLocation, error) { +func (s *state2) FindSector(num abi.SectorNumber) (*SectorLocation, error) { dlIdx, partIdx, err := s.State.FindSector(s.store, num) if err != nil { return nil, err @@ -89,13 +89,13 @@ func (s *state1) FindSector(num abi.SectorNumber) (*SectorLocation, error) { }, nil } -func (s *state1) NumLiveSectors() (uint64, error) { +func (s *state2) NumLiveSectors() (uint64, error) { dls, err := s.State.LoadDeadlines(s.store) if err != nil { return 0, err } var total uint64 - if err := dls.ForEach(s.store, func(dlIdx uint64, dl *miner1.Deadline) error { + if err := dls.ForEach(s.store, func(dlIdx uint64, dl *miner2.Deadline) error { total += dl.LiveSectors return nil }); err != nil { @@ -109,7 +109,7 @@ func (s *state1) NumLiveSectors() (uint64, error) { // If the sector isn't found or has already been terminated, this method returns // nil and no error. If the sector does not expire early, the Early expiration // field is 0. -func (s *state1) GetSectorExpiration(num abi.SectorNumber) (*SectorExpiration, error) { +func (s *state2) GetSectorExpiration(num abi.SectorNumber) (*SectorExpiration, error) { dls, err := s.State.LoadDeadlines(s.store) if err != nil { return nil, err @@ -122,13 +122,13 @@ func (s *state1) GetSectorExpiration(num abi.SectorNumber) (*SectorExpiration, e // of the expiration queue. stopErr := errors.New("stop") out := SectorExpiration{} - err = dls.ForEach(s.store, func(dlIdx uint64, dl *miner1.Deadline) error { + err = dls.ForEach(s.store, func(dlIdx uint64, dl *miner2.Deadline) error { partitions, err := dl.PartitionsArray(s.store) if err != nil { return err } quant := s.State.QuantSpecForDeadline(dlIdx) - var part miner1.Partition + var part miner2.Partition return partitions.ForEach(&part, func(partIdx int64) error { if found, err := part.Sectors.IsSet(uint64(num)); err != nil { return err @@ -142,11 +142,11 @@ func (s *state1) GetSectorExpiration(num abi.SectorNumber) (*SectorExpiration, e return stopErr } - q, err := miner1.LoadExpirationQueue(s.store, part.ExpirationsEpochs, quant) + q, err := miner2.LoadExpirationQueue(s.store, part.ExpirationsEpochs, quant) if err != nil { return err } - var exp miner1.ExpirationSet + var exp miner2.ExpirationSet return q.ForEach(&exp, func(epoch int64) error { if early, err := exp.EarlySectors.IsSet(uint64(num)); err != nil { return err @@ -176,19 +176,19 @@ func (s *state1) GetSectorExpiration(num abi.SectorNumber) (*SectorExpiration, e return &out, nil } -func (s *state1) GetPrecommittedSector(num abi.SectorNumber) (*SectorPreCommitOnChainInfo, error) { +func (s *state2) GetPrecommittedSector(num abi.SectorNumber) (*SectorPreCommitOnChainInfo, error) { info, ok, err := s.State.GetPrecommittedSector(s.store, num) if !ok || err != nil { return nil, err } - ret := fromV1SectorPreCommitOnChainInfo(*info) + ret := fromV2SectorPreCommitOnChainInfo(*info) return &ret, nil } -func (s *state1) LoadSectors(snos *bitfield.BitField) ([]*SectorOnChainInfo, error) { - sectors, err := miner1.LoadSectors(s.store, s.State.Sectors) +func (s *state2) LoadSectors(snos *bitfield.BitField) ([]*SectorOnChainInfo, error) { + sectors, err := miner2.LoadSectors(s.store, s.State.Sectors) if err != nil { return nil, err } @@ -196,9 +196,9 @@ func (s *state1) LoadSectors(snos *bitfield.BitField) ([]*SectorOnChainInfo, err // If no sector numbers are specified, load all. if snos == nil { infos := make([]*SectorOnChainInfo, 0, sectors.Length()) - var info1 miner1.SectorOnChainInfo - if err := sectors.ForEach(&info1, func(_ int64) error { - info := fromV1SectorOnChainInfo(info1) + var info2 miner2.SectorOnChainInfo + if err := sectors.ForEach(&info2, func(_ int64) error { + info := fromV2SectorOnChainInfo(info2) infos = append(infos, &info) return nil }); err != nil { @@ -208,19 +208,19 @@ func (s *state1) LoadSectors(snos *bitfield.BitField) ([]*SectorOnChainInfo, err } // Otherwise, load selected. - infos1, err := sectors.Load(*snos) + infos2, err := sectors.Load(*snos) if err != nil { return nil, err } - infos := make([]*SectorOnChainInfo, len(infos1)) - for i, info1 := range infos1 { - info := fromV1SectorOnChainInfo(*info1) + infos := make([]*SectorOnChainInfo, len(infos2)) + for i, info2 := range infos2 { + info := fromV2SectorOnChainInfo(*info2) infos[i] = &info } return infos, nil } -func (s *state1) IsAllocated(num abi.SectorNumber) (bool, error) { +func (s *state2) IsAllocated(num abi.SectorNumber) (bool, error) { var allocatedSectors bitfield.BitField if err := s.store.Get(s.store.Context(), s.State.AllocatedSectors, &allocatedSectors); err != nil { return false, err @@ -229,7 +229,7 @@ func (s *state1) IsAllocated(num abi.SectorNumber) (bool, error) { return allocatedSectors.IsSet(uint64(num)) } -func (s *state1) LoadDeadline(idx uint64) (Deadline, error) { +func (s *state2) LoadDeadline(idx uint64) (Deadline, error) { dls, err := s.State.LoadDeadlines(s.store) if err != nil { return nil, err @@ -238,34 +238,34 @@ func (s *state1) LoadDeadline(idx uint64) (Deadline, error) { if err != nil { return nil, err } - return &deadline1{*dl, s.store}, nil + return &deadline2{*dl, s.store}, nil } -func (s *state1) ForEachDeadline(cb func(uint64, Deadline) error) error { +func (s *state2) ForEachDeadline(cb func(uint64, Deadline) error) error { dls, err := s.State.LoadDeadlines(s.store) if err != nil { return err } - return dls.ForEach(s.store, func(i uint64, dl *miner1.Deadline) error { - return cb(i, &deadline1{*dl, s.store}) + return dls.ForEach(s.store, func(i uint64, dl *miner2.Deadline) error { + return cb(i, &deadline2{*dl, s.store}) }) } -func (s *state1) NumDeadlines() (uint64, error) { - return miner1.WPoStPeriodDeadlines, nil +func (s *state2) NumDeadlines() (uint64, error) { + return miner2.WPoStPeriodDeadlines, nil } -func (s *state1) DeadlinesChanged(other State) (bool, error) { - other1, ok := other.(*state1) +func (s *state2) DeadlinesChanged(other State) (bool, error) { + other2, ok := other.(*state2) if !ok { // treat an upgrade as a change, always return true, nil } - return s.State.Deadlines.Equals(other1.Deadlines), nil + return s.State.Deadlines.Equals(other2.Deadlines), nil } -func (s *state1) Info() (MinerInfo, error) { +func (s *state2) Info() (MinerInfo, error) { info, err := s.State.GetInfo(s.store) if err != nil { return MinerInfo{}, err @@ -299,105 +299,105 @@ func (s *state1) Info() (MinerInfo, error) { return mi, nil } -func (s *state1) DeadlineInfo(epoch abi.ChainEpoch) (*dline.Info, error) { +func (s *state2) DeadlineInfo(epoch abi.ChainEpoch) (*dline.Info, error) { return s.State.DeadlineInfo(epoch), nil } -func (s *state1) sectors() (adt.Array, error) { - return adt1.AsArray(s.store, s.Sectors) +func (s *state2) sectors() (adt.Array, error) { + return adt2.AsArray(s.store, s.Sectors) } -func (s *state1) decodeSectorOnChainInfo(val *cbg.Deferred) (SectorOnChainInfo, error) { - var si miner1.SectorOnChainInfo +func (s *state2) decodeSectorOnChainInfo(val *cbg.Deferred) (SectorOnChainInfo, error) { + var si miner2.SectorOnChainInfo err := si.UnmarshalCBOR(bytes.NewReader(val.Raw)) if err != nil { return SectorOnChainInfo{}, err } - return fromV1SectorOnChainInfo(si), nil + return fromV2SectorOnChainInfo(si), nil } -func (s *state1) precommits() (adt.Map, error) { - return adt1.AsMap(s.store, s.PreCommittedSectors) +func (s *state2) precommits() (adt.Map, error) { + return adt2.AsMap(s.store, s.PreCommittedSectors) } -func (s *state1) decodeSectorPreCommitOnChainInfo(val *cbg.Deferred) (SectorPreCommitOnChainInfo, error) { - var sp miner1.SectorPreCommitOnChainInfo +func (s *state2) decodeSectorPreCommitOnChainInfo(val *cbg.Deferred) (SectorPreCommitOnChainInfo, error) { + var sp miner2.SectorPreCommitOnChainInfo err := sp.UnmarshalCBOR(bytes.NewReader(val.Raw)) if err != nil { return SectorPreCommitOnChainInfo{}, err } - return fromV1SectorPreCommitOnChainInfo(sp), nil + return fromV2SectorPreCommitOnChainInfo(sp), nil } -func (d *deadline1) LoadPartition(idx uint64) (Partition, error) { +func (d *deadline2) LoadPartition(idx uint64) (Partition, error) { p, err := d.Deadline.LoadPartition(d.store, idx) if err != nil { return nil, err } - return &partition1{*p, d.store}, nil + return &partition2{*p, d.store}, nil } -func (d *deadline1) ForEachPartition(cb func(uint64, Partition) error) error { +func (d *deadline2) ForEachPartition(cb func(uint64, Partition) error) error { ps, err := d.Deadline.PartitionsArray(d.store) if err != nil { return err } - var part miner1.Partition + var part miner2.Partition return ps.ForEach(&part, func(i int64) error { - return cb(uint64(i), &partition1{part, d.store}) + return cb(uint64(i), &partition2{part, d.store}) }) } -func (d *deadline1) PartitionsChanged(other Deadline) (bool, error) { - other1, ok := other.(*deadline1) +func (d *deadline2) PartitionsChanged(other Deadline) (bool, error) { + other2, ok := other.(*deadline2) if !ok { // treat an upgrade as a change, always return true, nil } - return d.Deadline.Partitions.Equals(other1.Deadline.Partitions), nil + return d.Deadline.Partitions.Equals(other2.Deadline.Partitions), nil } -func (d *deadline1) PostSubmissions() (bitfield.BitField, error) { +func (d *deadline2) PostSubmissions() (bitfield.BitField, error) { return d.Deadline.PostSubmissions, nil } -func (p *partition1) AllSectors() (bitfield.BitField, error) { +func (p *partition2) AllSectors() (bitfield.BitField, error) { return p.Partition.Sectors, nil } -func (p *partition1) FaultySectors() (bitfield.BitField, error) { +func (p *partition2) FaultySectors() (bitfield.BitField, error) { return p.Partition.Faults, nil } -func (p *partition1) RecoveringSectors() (bitfield.BitField, error) { +func (p *partition2) RecoveringSectors() (bitfield.BitField, error) { return p.Partition.Recoveries, nil } -func fromV1SectorOnChainInfo(v1 miner1.SectorOnChainInfo) SectorOnChainInfo { +func fromV2SectorOnChainInfo(v2 miner2.SectorOnChainInfo) SectorOnChainInfo { return SectorOnChainInfo{ - SectorNumber: v1.SectorNumber, - SealProof: v1.SealProof, - SealedCID: v1.SealedCID, - DealIDs: v1.DealIDs, - Activation: v1.Activation, - Expiration: v1.Expiration, - DealWeight: v1.DealWeight, - VerifiedDealWeight: v1.VerifiedDealWeight, - InitialPledge: v1.InitialPledge, - ExpectedDayReward: v1.ExpectedDayReward, - ExpectedStoragePledge: v1.ExpectedStoragePledge, + SectorNumber: v2.SectorNumber, + SealProof: v2.SealProof, + SealedCID: v2.SealedCID, + DealIDs: v2.DealIDs, + Activation: v2.Activation, + Expiration: v2.Expiration, + DealWeight: v2.DealWeight, + VerifiedDealWeight: v2.VerifiedDealWeight, + InitialPledge: v2.InitialPledge, + ExpectedDayReward: v2.ExpectedDayReward, + ExpectedStoragePledge: v2.ExpectedStoragePledge, } } -func fromV1SectorPreCommitOnChainInfo(v1 miner1.SectorPreCommitOnChainInfo) SectorPreCommitOnChainInfo { +func fromV2SectorPreCommitOnChainInfo(v2 miner2.SectorPreCommitOnChainInfo) SectorPreCommitOnChainInfo { return SectorPreCommitOnChainInfo{ - Info: (SectorPreCommitInfo)(v1.Info), - PreCommitDeposit: v1.PreCommitDeposit, - PreCommitEpoch: v1.PreCommitEpoch, - DealWeight: v1.DealWeight, - VerifiedDealWeight: v1.VerifiedDealWeight, + Info: (SectorPreCommitInfo)(v2.Info), + PreCommitDeposit: v2.PreCommitDeposit, + PreCommitEpoch: v2.PreCommitEpoch, + DealWeight: v2.DealWeight, + VerifiedDealWeight: v2.VerifiedDealWeight, } } diff --git a/chain/actors/builtin/multisig/multisig.go b/chain/actors/builtin/multisig/multisig.go index 022e7e2c5..89a7eedad 100644 --- a/chain/actors/builtin/multisig/multisig.go +++ b/chain/actors/builtin/multisig/multisig.go @@ -10,7 +10,7 @@ import ( builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" msig0 "github.com/filecoin-project/specs-actors/actors/builtin/multisig" - builtin1 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/actors/builtin" @@ -21,8 +21,8 @@ func init() { builtin.RegisterActorState(builtin0.MultisigActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) { return load0(store, root) }) - builtin.RegisterActorState(builtin1.MultisigActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) { - return load1(store, root) + builtin.RegisterActorState(builtin2.MultisigActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) { + return load2(store, root) }) } @@ -30,8 +30,8 @@ func Load(store adt.Store, act *types.Actor) (State, error) { switch act.Code { case builtin0.MultisigActorCodeID: return load0(store, act.Head) - case builtin1.MultisigActorCodeID: - return load1(store, act.Head) + case builtin2.MultisigActorCodeID: + return load2(store, act.Head) } return nil, xerrors.Errorf("unknown actor code %s", act.Code) } diff --git a/chain/actors/builtin/multisig/v1.go b/chain/actors/builtin/multisig/v2.go similarity index 55% rename from chain/actors/builtin/multisig/v1.go rename to chain/actors/builtin/multisig/v2.go index 56760cbba..a78b07d55 100644 --- a/chain/actors/builtin/multisig/v1.go +++ b/chain/actors/builtin/multisig/v2.go @@ -10,14 +10,14 @@ import ( "github.com/filecoin-project/lotus/chain/actors/adt" - msig1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/multisig" - adt1 "github.com/filecoin-project/specs-actors/v2/actors/util/adt" + msig2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/multisig" + adt2 "github.com/filecoin-project/specs-actors/v2/actors/util/adt" ) -var _ State = (*state1)(nil) +var _ State = (*state2)(nil) -func load1(store adt.Store, root cid.Cid) (State, error) { - out := state1{store: store} +func load2(store adt.Store, root cid.Cid) (State, error) { + out := state2{store: store} err := store.Get(store.Context(), root, &out) if err != nil { return nil, err @@ -25,41 +25,41 @@ func load1(store adt.Store, root cid.Cid) (State, error) { return &out, nil } -type state1 struct { - msig1.State +type state2 struct { + msig2.State store adt.Store } -func (s *state1) LockedBalance(currEpoch abi.ChainEpoch) (abi.TokenAmount, error) { +func (s *state2) LockedBalance(currEpoch abi.ChainEpoch) (abi.TokenAmount, error) { return s.State.AmountLocked(currEpoch - s.State.StartEpoch), nil } -func (s *state1) StartEpoch() (abi.ChainEpoch, error) { +func (s *state2) StartEpoch() (abi.ChainEpoch, error) { return s.State.StartEpoch, nil } -func (s *state1) UnlockDuration() (abi.ChainEpoch, error) { +func (s *state2) UnlockDuration() (abi.ChainEpoch, error) { return s.State.UnlockDuration, nil } -func (s *state1) InitialBalance() (abi.TokenAmount, error) { +func (s *state2) InitialBalance() (abi.TokenAmount, error) { return s.State.InitialBalance, nil } -func (s *state1) Threshold() (uint64, error) { +func (s *state2) Threshold() (uint64, error) { return s.State.NumApprovalsThreshold, nil } -func (s *state1) Signers() ([]address.Address, error) { +func (s *state2) Signers() ([]address.Address, error) { return s.State.Signers, nil } -func (s *state1) ForEachPendingTxn(cb func(id int64, txn Transaction) error) error { - arr, err := adt1.AsMap(s.store, s.State.PendingTxns) +func (s *state2) ForEachPendingTxn(cb func(id int64, txn Transaction) error) error { + arr, err := adt2.AsMap(s.store, s.State.PendingTxns) if err != nil { return err } - var out msig1.Transaction + var out msig2.Transaction return arr.ForEach(&out, func(key string) error { txid, n := binary.Varint([]byte(key)) if n <= 0 { diff --git a/chain/actors/builtin/paych/paych.go b/chain/actors/builtin/paych/paych.go index 959968f22..f462de6fd 100644 --- a/chain/actors/builtin/paych/paych.go +++ b/chain/actors/builtin/paych/paych.go @@ -11,7 +11,7 @@ import ( builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" paych0 "github.com/filecoin-project/specs-actors/actors/builtin/paych" - builtin1 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/actors/builtin" @@ -22,8 +22,8 @@ func init() { builtin.RegisterActorState(builtin0.PaymentChannelActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) { return load0(store, root) }) - builtin.RegisterActorState(builtin1.PaymentChannelActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) { - return load1(store, root) + builtin.RegisterActorState(builtin2.PaymentChannelActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) { + return load2(store, root) }) } @@ -32,8 +32,8 @@ func Load(store adt.Store, act *types.Actor) (State, error) { switch act.Code { case builtin0.PaymentChannelActorCodeID: return load0(store, act.Head) - case builtin1.PaymentChannelActorCodeID: - return load1(store, act.Head) + case builtin2.PaymentChannelActorCodeID: + return load2(store, act.Head) } return nil, xerrors.Errorf("unknown actor code %s", act.Code) } diff --git a/chain/actors/builtin/paych/v1.go b/chain/actors/builtin/paych/v2.go similarity index 61% rename from chain/actors/builtin/paych/v1.go rename to chain/actors/builtin/paych/v2.go index c1b47c120..fbf4b9fde 100644 --- a/chain/actors/builtin/paych/v1.go +++ b/chain/actors/builtin/paych/v2.go @@ -9,14 +9,14 @@ import ( "github.com/filecoin-project/lotus/chain/actors/adt" - paych1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/paych" - adt1 "github.com/filecoin-project/specs-actors/v2/actors/util/adt" + paych2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/paych" + adt2 "github.com/filecoin-project/specs-actors/v2/actors/util/adt" ) -var _ State = (*state1)(nil) +var _ State = (*state2)(nil) -func load1(store adt.Store, root cid.Cid) (State, error) { - out := state1{store: store} +func load2(store adt.Store, root cid.Cid) (State, error) { + out := state2{store: store} err := store.Get(store.Context(), root, &out) if err != nil { return nil, err @@ -24,39 +24,39 @@ func load1(store adt.Store, root cid.Cid) (State, error) { return &out, nil } -type state1 struct { - paych1.State +type state2 struct { + paych2.State store adt.Store - lsAmt *adt1.Array + lsAmt *adt2.Array } // Channel owner, who has funded the actor -func (s *state1) From() (address.Address, error) { +func (s *state2) From() (address.Address, error) { return s.State.From, nil } // Recipient of payouts from channel -func (s *state1) To() (address.Address, error) { +func (s *state2) To() (address.Address, error) { return s.State.To, nil } // Height at which the channel can be `Collected` -func (s *state1) SettlingAt() (abi.ChainEpoch, error) { +func (s *state2) SettlingAt() (abi.ChainEpoch, error) { return s.State.SettlingAt, nil } // Amount successfully redeemed through the payment channel, paid out on `Collect()` -func (s *state1) ToSend() (abi.TokenAmount, error) { +func (s *state2) ToSend() (abi.TokenAmount, error) { return s.State.ToSend, nil } -func (s *state1) getOrLoadLsAmt() (*adt1.Array, error) { +func (s *state2) getOrLoadLsAmt() (*adt2.Array, error) { if s.lsAmt != nil { return s.lsAmt, nil } // Get the lane state from the chain - lsamt, err := adt1.AsArray(s.store, s.State.LaneStates) + lsamt, err := adt2.AsArray(s.store, s.State.LaneStates) if err != nil { return nil, err } @@ -66,7 +66,7 @@ func (s *state1) getOrLoadLsAmt() (*adt1.Array, error) { } // Get total number of lanes -func (s *state1) LaneCount() (uint64, error) { +func (s *state2) LaneCount() (uint64, error) { lsamt, err := s.getOrLoadLsAmt() if err != nil { return 0, err @@ -75,7 +75,7 @@ func (s *state1) LaneCount() (uint64, error) { } // Iterate lane states -func (s *state1) ForEachLaneState(cb func(idx uint64, dl LaneState) error) error { +func (s *state2) ForEachLaneState(cb func(idx uint64, dl LaneState) error) error { // Get the lane state from the chain lsamt, err := s.getOrLoadLsAmt() if err != nil { @@ -85,20 +85,20 @@ func (s *state1) ForEachLaneState(cb func(idx uint64, dl LaneState) error) error // Note: we use a map instead of an array to store laneStates because the // client sets the lane ID (the index) and potentially they could use a // very large index. - var ls paych1.LaneState + var ls paych2.LaneState return lsamt.ForEach(&ls, func(i int64) error { - return cb(uint64(i), &laneState1{ls}) + return cb(uint64(i), &laneState2{ls}) }) } -type laneState1 struct { - paych1.LaneState +type laneState2 struct { + paych2.LaneState } -func (ls *laneState1) Redeemed() (big.Int, error) { +func (ls *laneState2) Redeemed() (big.Int, error) { return ls.LaneState.Redeemed, nil } -func (ls *laneState1) Nonce() (uint64, error) { +func (ls *laneState2) Nonce() (uint64, error) { return ls.LaneState.Nonce, nil } diff --git a/chain/actors/builtin/power/power.go b/chain/actors/builtin/power/power.go index 9dd61c07e..e683cfd96 100644 --- a/chain/actors/builtin/power/power.go +++ b/chain/actors/builtin/power/power.go @@ -13,15 +13,15 @@ import ( "github.com/filecoin-project/lotus/chain/types" builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" - builtin1 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" ) func init() { builtin.RegisterActorState(builtin0.StoragePowerActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) { return load0(store, root) }) - builtin.RegisterActorState(builtin1.StoragePowerActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) { - return load1(store, root) + builtin.RegisterActorState(builtin2.StoragePowerActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) { + return load2(store, root) }) } @@ -31,8 +31,8 @@ func Load(store adt.Store, act *types.Actor) (st State, err error) { switch act.Code { case builtin0.StoragePowerActorCodeID: return load0(store, act.Head) - case builtin1.StoragePowerActorCodeID: - return load1(store, act.Head) + case builtin2.StoragePowerActorCodeID: + return load2(store, act.Head) } return nil, xerrors.Errorf("unknown actor code %s", act.Code) } diff --git a/chain/actors/builtin/power/v1.go b/chain/actors/builtin/power/v2.go similarity index 62% rename from chain/actors/builtin/power/v1.go rename to chain/actors/builtin/power/v2.go index 63c9df547..6346a09b6 100644 --- a/chain/actors/builtin/power/v1.go +++ b/chain/actors/builtin/power/v2.go @@ -8,14 +8,14 @@ import ( "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/actors/builtin" - power1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/power" - adt1 "github.com/filecoin-project/specs-actors/v2/actors/util/adt" + power2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/power" + adt2 "github.com/filecoin-project/specs-actors/v2/actors/util/adt" ) -var _ State = (*state1)(nil) +var _ State = (*state2)(nil) -func load1(store adt.Store, root cid.Cid) (State, error) { - out := state1{store: store} +func load2(store adt.Store, root cid.Cid) (State, error) { + out := state2{store: store} err := store.Get(store.Context(), root, &out) if err != nil { return nil, err @@ -23,16 +23,16 @@ func load1(store adt.Store, root cid.Cid) (State, error) { return &out, nil } -type state1 struct { - power1.State +type state2 struct { + power2.State store adt.Store } -func (s *state1) TotalLocked() (abi.TokenAmount, error) { +func (s *state2) TotalLocked() (abi.TokenAmount, error) { return s.TotalPledgeCollateral, nil } -func (s *state1) TotalPower() (Claim, error) { +func (s *state2) TotalPower() (Claim, error) { return Claim{ RawBytePower: s.TotalRawBytePower, QualityAdjPower: s.TotalQualityAdjPower, @@ -40,19 +40,19 @@ func (s *state1) TotalPower() (Claim, error) { } // Committed power to the network. Includes miners below the minimum threshold. -func (s *state1) TotalCommitted() (Claim, error) { +func (s *state2) TotalCommitted() (Claim, error) { return Claim{ RawBytePower: s.TotalBytesCommitted, QualityAdjPower: s.TotalQABytesCommitted, }, nil } -func (s *state1) MinerPower(addr address.Address) (Claim, bool, error) { - claims, err := adt1.AsMap(s.store, s.Claims) +func (s *state2) MinerPower(addr address.Address) (Claim, bool, error) { + claims, err := adt2.AsMap(s.store, s.Claims) if err != nil { return Claim{}, false, err } - var claim power1.Claim + var claim power2.Claim ok, err := claims.Get(abi.AddrKey(addr), &claim) if err != nil { return Claim{}, false, err @@ -63,20 +63,20 @@ func (s *state1) MinerPower(addr address.Address) (Claim, bool, error) { }, ok, nil } -func (s *state1) MinerNominalPowerMeetsConsensusMinimum(a address.Address) (bool, error) { +func (s *state2) MinerNominalPowerMeetsConsensusMinimum(a address.Address) (bool, error) { return s.State.MinerNominalPowerMeetsConsensusMinimum(s.store, a) } -func (s *state1) TotalPowerSmoothed() (builtin.FilterEstimate, error) { - return builtin.FromV1FilterEstimate(s.State.ThisEpochQAPowerSmoothed), nil +func (s *state2) TotalPowerSmoothed() (builtin.FilterEstimate, error) { + return builtin.FromV2FilterEstimate(s.State.ThisEpochQAPowerSmoothed), nil } -func (s *state1) MinerCounts() (uint64, uint64, error) { +func (s *state2) MinerCounts() (uint64, uint64, error) { return uint64(s.State.MinerAboveMinPowerCount), uint64(s.State.MinerCount), nil } -func (s *state1) ListAllMiners() ([]address.Address, error) { - claims, err := adt1.AsMap(s.store, s.Claims) +func (s *state2) ListAllMiners() ([]address.Address, error) { + claims, err := adt2.AsMap(s.store, s.Claims) if err != nil { return nil, err } diff --git a/chain/actors/builtin/reward/reward.go b/chain/actors/builtin/reward/reward.go index e6d05de46..065f242e2 100644 --- a/chain/actors/builtin/reward/reward.go +++ b/chain/actors/builtin/reward/reward.go @@ -8,7 +8,7 @@ import ( "github.com/filecoin-project/go-state-types/cbor" builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" - builtin1 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/actors/builtin" @@ -19,8 +19,8 @@ func init() { builtin.RegisterActorState(builtin0.RewardActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) { return load0(store, root) }) - builtin.RegisterActorState(builtin1.RewardActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) { - return load1(store, root) + builtin.RegisterActorState(builtin2.RewardActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) { + return load2(store, root) }) } @@ -30,8 +30,8 @@ func Load(store adt.Store, act *types.Actor) (st State, err error) { switch act.Code { case builtin0.RewardActorCodeID: return load0(store, act.Head) - case builtin1.RewardActorCodeID: - return load1(store, act.Head) + case builtin2.RewardActorCodeID: + return load2(store, act.Head) } return nil, xerrors.Errorf("unknown actor code %s", act.Code) } diff --git a/chain/actors/builtin/reward/v1.go b/chain/actors/builtin/reward/v2.go similarity index 56% rename from chain/actors/builtin/reward/v1.go rename to chain/actors/builtin/reward/v2.go index 9f94bbf1d..ec0709c39 100644 --- a/chain/actors/builtin/reward/v1.go +++ b/chain/actors/builtin/reward/v2.go @@ -7,15 +7,15 @@ import ( "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/actors/builtin" - miner1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" - reward1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/reward" - smoothing1 "github.com/filecoin-project/specs-actors/v2/actors/util/smoothing" + miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" + reward2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/reward" + smoothing2 "github.com/filecoin-project/specs-actors/v2/actors/util/smoothing" ) -var _ State = (*state1)(nil) +var _ State = (*state2)(nil) -func load1(store adt.Store, root cid.Cid) (State, error) { - out := state1{store: store} +func load2(store adt.Store, root cid.Cid) (State, error) { + out := state2{store: store} err := store.Get(store.Context(), root, &out) if err != nil { return nil, err @@ -23,52 +23,52 @@ func load1(store adt.Store, root cid.Cid) (State, error) { return &out, nil } -type state1 struct { - reward1.State +type state2 struct { + reward2.State store adt.Store } -func (s *state1) ThisEpochReward() (abi.StoragePower, error) { +func (s *state2) ThisEpochReward() (abi.StoragePower, error) { return s.State.ThisEpochReward, nil } -func (s *state1) ThisEpochRewardSmoothed() (builtin.FilterEstimate, error) { +func (s *state2) ThisEpochRewardSmoothed() (builtin.FilterEstimate, error) { return builtin.FilterEstimate{ PositionEstimate: s.State.ThisEpochRewardSmoothed.PositionEstimate, VelocityEstimate: s.State.ThisEpochRewardSmoothed.VelocityEstimate, }, nil } -func (s *state1) ThisEpochBaselinePower() (abi.StoragePower, error) { +func (s *state2) ThisEpochBaselinePower() (abi.StoragePower, error) { return s.State.ThisEpochBaselinePower, nil } -func (s *state1) TotalStoragePowerReward() (abi.TokenAmount, error) { +func (s *state2) TotalStoragePowerReward() (abi.TokenAmount, error) { return s.State.TotalStoragePowerReward, nil } -func (s *state1) EffectiveBaselinePower() (abi.StoragePower, error) { +func (s *state2) EffectiveBaselinePower() (abi.StoragePower, error) { return s.State.EffectiveBaselinePower, nil } -func (s *state1) EffectiveNetworkTime() (abi.ChainEpoch, error) { +func (s *state2) EffectiveNetworkTime() (abi.ChainEpoch, error) { return s.State.EffectiveNetworkTime, nil } -func (s *state1) CumsumBaseline() (abi.StoragePower, error) { +func (s *state2) CumsumBaseline() (abi.StoragePower, error) { return s.State.CumsumBaseline, nil } -func (s *state1) CumsumRealized() (abi.StoragePower, error) { +func (s *state2) CumsumRealized() (abi.StoragePower, error) { return s.State.CumsumBaseline, nil } -func (s *state1) InitialPledgeForPower(qaPower abi.StoragePower, networkTotalPledge abi.TokenAmount, networkQAPower *builtin.FilterEstimate, circSupply abi.TokenAmount) (abi.TokenAmount, error) { - return miner1.InitialPledgeForPower( +func (s *state2) InitialPledgeForPower(qaPower abi.StoragePower, networkTotalPledge abi.TokenAmount, networkQAPower *builtin.FilterEstimate, circSupply abi.TokenAmount) (abi.TokenAmount, error) { + return miner2.InitialPledgeForPower( qaPower, s.State.ThisEpochBaselinePower, s.State.ThisEpochRewardSmoothed, - smoothing1.FilterEstimate{ + smoothing2.FilterEstimate{ PositionEstimate: networkQAPower.PositionEstimate, VelocityEstimate: networkQAPower.VelocityEstimate, }, @@ -76,9 +76,9 @@ func (s *state1) InitialPledgeForPower(qaPower abi.StoragePower, networkTotalPle ), nil } -func (s *state1) PreCommitDepositForPower(networkQAPower builtin.FilterEstimate, sectorWeight abi.StoragePower) (abi.TokenAmount, error) { - return miner1.PreCommitDepositForPower(s.State.ThisEpochRewardSmoothed, - smoothing1.FilterEstimate{ +func (s *state2) PreCommitDepositForPower(networkQAPower builtin.FilterEstimate, sectorWeight abi.StoragePower) (abi.TokenAmount, error) { + return miner2.PreCommitDepositForPower(s.State.ThisEpochRewardSmoothed, + smoothing2.FilterEstimate{ PositionEstimate: networkQAPower.PositionEstimate, VelocityEstimate: networkQAPower.VelocityEstimate, }, diff --git a/chain/actors/builtin/verifreg/v1.go b/chain/actors/builtin/verifreg/v1.go deleted file mode 100644 index 0c9e1b139..000000000 --- a/chain/actors/builtin/verifreg/v1.go +++ /dev/null @@ -1,44 +0,0 @@ -package verifreg - -import ( - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-state-types/abi" - "github.com/ipfs/go-cid" - - "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/actors/adt" - - verifreg1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/verifreg" -) - -var _ State = (*state1)(nil) - -func load1(store adt.Store, root cid.Cid) (State, error) { - out := state1{store: store} - err := store.Get(store.Context(), root, &out) - if err != nil { - return nil, err - } - return &out, nil -} - -type state1 struct { - verifreg1.State - store adt.Store -} - -func (s *state1) VerifiedClientDataCap(addr address.Address) (bool, abi.StoragePower, error) { - return getDataCap(s.store, actors.Version1, s.State.VerifiedClients, addr) -} - -func (s *state1) VerifierDataCap(addr address.Address) (bool, abi.StoragePower, error) { - return getDataCap(s.store, actors.Version1, s.State.Verifiers, addr) -} - -func (s *state1) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error { - return forEachCap(s.store, actors.Version1, s.State.Verifiers, cb) -} - -func (s *state1) ForEachClient(cb func(addr address.Address, dcap abi.StoragePower) error) error { - return forEachCap(s.store, actors.Version1, s.State.VerifiedClients, cb) -} diff --git a/chain/actors/builtin/verifreg/v2.go b/chain/actors/builtin/verifreg/v2.go new file mode 100644 index 000000000..72a781f26 --- /dev/null +++ b/chain/actors/builtin/verifreg/v2.go @@ -0,0 +1,44 @@ +package verifreg + +import ( + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/actors/adt" + + verifreg2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/verifreg" +) + +var _ State = (*state2)(nil) + +func load2(store adt.Store, root cid.Cid) (State, error) { + out := state2{store: store} + err := store.Get(store.Context(), root, &out) + if err != nil { + return nil, err + } + return &out, nil +} + +type state2 struct { + verifreg2.State + store adt.Store +} + +func (s *state2) VerifiedClientDataCap(addr address.Address) (bool, abi.StoragePower, error) { + return getDataCap(s.store, actors.Version2, s.State.VerifiedClients, addr) +} + +func (s *state2) VerifierDataCap(addr address.Address) (bool, abi.StoragePower, error) { + return getDataCap(s.store, actors.Version2, s.State.Verifiers, addr) +} + +func (s *state2) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error { + return forEachCap(s.store, actors.Version2, s.State.Verifiers, cb) +} + +func (s *state2) ForEachClient(cb func(addr address.Address, dcap abi.StoragePower) error) error { + return forEachCap(s.store, actors.Version2, s.State.VerifiedClients, cb) +} diff --git a/chain/actors/builtin/verifreg/verifreg.go b/chain/actors/builtin/verifreg/verifreg.go index 86b1c5939..00a492e95 100644 --- a/chain/actors/builtin/verifreg/verifreg.go +++ b/chain/actors/builtin/verifreg/verifreg.go @@ -8,7 +8,7 @@ import ( "github.com/filecoin-project/go-state-types/cbor" builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" - builtin1 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/actors/builtin" @@ -19,8 +19,8 @@ func init() { builtin.RegisterActorState(builtin0.VerifiedRegistryActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) { return load0(store, root) }) - builtin.RegisterActorState(builtin1.VerifiedRegistryActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) { - return load1(store, root) + builtin.RegisterActorState(builtin2.VerifiedRegistryActorCodeID, func(store adt.Store, root cid.Cid) (cbor.Marshaler, error) { + return load2(store, root) }) } @@ -30,8 +30,8 @@ func Load(store adt.Store, act *types.Actor) (State, error) { switch act.Code { case builtin0.VerifiedRegistryActorCodeID: return load0(store, act.Head) - case builtin1.VerifiedRegistryActorCodeID: - return load1(store, act.Head) + case builtin2.VerifiedRegistryActorCodeID: + return load2(store, act.Head) } return nil, xerrors.Errorf("unknown actor code %s", act.Code) } diff --git a/chain/actors/version.go b/chain/actors/version.go index 6242ad738..2173a320d 100644 --- a/chain/actors/version.go +++ b/chain/actors/version.go @@ -9,8 +9,8 @@ import ( type Version int const ( - Version0 = iota - Version1 + Version0 = 0 + Version2 = 2 ) // Converts a network version into an actors adt version. diff --git a/chain/state/statetree.go b/chain/state/statetree.go index 7a2e70cdc..fcb7ffbaa 100644 --- a/chain/state/statetree.go +++ b/chain/state/statetree.go @@ -125,7 +125,7 @@ func NewStateTree(cst cbor.IpldStore, version actors.Version) (*StateTree, error switch version { case actors.Version0: // info is undefined - case actors.Version1: + case actors.Version2: var err error info, err = cst.Put(context.TODO(), new(types.StateInfo)) if err != nil { @@ -165,7 +165,7 @@ func LoadStateTree(cst cbor.IpldStore, c cid.Cid) (*StateTree, error) { } switch root.Version { - case actors.Version0, actors.Version1: + case actors.Version0, actors.Version2: // supported default: return nil, xerrors.Errorf("unsupported state tree version: %d", root.Version) diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index 19a3246a7..6ed94775b 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -21,7 +21,7 @@ import ( builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" exported0 "github.com/filecoin-project/specs-actors/actors/builtin/exported" - exported1 "github.com/filecoin-project/specs-actors/actors/builtin/exported" + exported2 "github.com/filecoin-project/specs-actors/actors/builtin/exported" proof0 "github.com/filecoin-project/specs-actors/actors/runtime/proof" "github.com/filecoin-project/lotus/api" @@ -530,14 +530,14 @@ func init() { // TODO: combine with the runtime actor registry. var actors []rt.VMActor actors = append(actors, exported0.BuiltinActors()...) - actors = append(actors, exported1.BuiltinActors()...) + actors = append(actors, exported2.BuiltinActors()...) for _, actor := range actors { exports := actor.Exports() methods := make(map[abi.MethodNum]MethodMeta, len(exports)) // Explicitly add send, it's special. - // Note that builtin1.MethodSend = builtin0.MethodSend = 0. + // Note that builtin2.MethodSend = builtin0.MethodSend = 0. methods[builtin0.MethodSend] = MethodMeta{ Name: "Send", Params: reflect.TypeOf(new(abi.EmptyValue)), @@ -563,10 +563,10 @@ func init() { switch abi.MethodNum(number) { case builtin0.MethodSend: - // Note that builtin1.MethodSend = builtin0.MethodSend = 0. + // Note that builtin2.MethodSend = builtin0.MethodSend = 0. panic("method 0 is reserved for Send") case builtin0.MethodConstructor: - // Note that builtin1.MethodConstructor = builtin0.MethodConstructor = 1. + // Note that builtin2.MethodConstructor = builtin0.MethodConstructor = 1. if fnName != "Constructor" { panic("method 1 is reserved for Constructor") } diff --git a/chain/vm/invoker.go b/chain/vm/invoker.go index 1f3db3ee9..65ef73cfc 100644 --- a/chain/vm/invoker.go +++ b/chain/vm/invoker.go @@ -11,7 +11,7 @@ import ( "golang.org/x/xerrors" exported0 "github.com/filecoin-project/specs-actors/actors/builtin/exported" - exported1 "github.com/filecoin-project/specs-actors/v2/actors/builtin/exported" + exported2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/exported" vmr "github.com/filecoin-project/specs-actors/v2/actors/runtime" "github.com/filecoin-project/go-state-types/abi" @@ -57,7 +57,7 @@ func NewActorRegistry() *ActorRegistry { // add builtInCode using: register(cid, singleton) inv.Register(ActorsVersionPredicate(actors.Version0), exported0.BuiltinActors()...) - inv.Register(ActorsVersionPredicate(actors.Version1), exported1.BuiltinActors()...) + inv.Register(ActorsVersionPredicate(actors.Version2), exported2.BuiltinActors()...) return inv } diff --git a/chain/vm/mkactor.go b/chain/vm/mkactor.go index 26e8c3ba1..33884368f 100644 --- a/chain/vm/mkactor.go +++ b/chain/vm/mkactor.go @@ -11,7 +11,7 @@ import ( cbor "github.com/ipfs/go-ipld-cbor" builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" - builtin1 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/actors/aerrors" @@ -87,8 +87,8 @@ func newAccountActor(ver actors.Version) *types.Actor { switch ver { case actors.Version0: code = builtin0.AccountActorCodeID - case actors.Version1: - code = builtin1.AccountActorCodeID + case actors.Version2: + code = builtin2.AccountActorCodeID default: panic("unsupported actors version") } diff --git a/cmd/lotus-chainwatch/processor/common_actors.go b/cmd/lotus-chainwatch/processor/common_actors.go index 4b8d124e2..0f2c0d2ea 100644 --- a/cmd/lotus-chainwatch/processor/common_actors.go +++ b/cmd/lotus-chainwatch/processor/common_actors.go @@ -11,7 +11,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/ipfs/go-cid" - builtin1 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/builtin" _init "github.com/filecoin-project/lotus/chain/actors/builtin/init" @@ -138,15 +138,15 @@ func (p Processor) storeActorAddresses(ctx context.Context, actors map[cid.Cid]A addressToID := map[address.Address]address.Address{} // HACK until genesis storage is figured out: - addressToID[builtin1.SystemActorAddr] = builtin1.SystemActorAddr - addressToID[builtin1.InitActorAddr] = builtin1.InitActorAddr - addressToID[builtin1.RewardActorAddr] = builtin1.RewardActorAddr - addressToID[builtin1.CronActorAddr] = builtin1.CronActorAddr - addressToID[builtin1.StoragePowerActorAddr] = builtin1.StoragePowerActorAddr - addressToID[builtin1.StorageMarketActorAddr] = builtin1.StorageMarketActorAddr - addressToID[builtin1.VerifiedRegistryActorAddr] = builtin1.VerifiedRegistryActorAddr - addressToID[builtin1.BurntFundsActorAddr] = builtin1.BurntFundsActorAddr - initActor, err := p.node.StateGetActor(ctx, builtin1.InitActorAddr, types.EmptyTSK) + addressToID[builtin2.SystemActorAddr] = builtin2.SystemActorAddr + addressToID[builtin2.InitActorAddr] = builtin2.InitActorAddr + addressToID[builtin2.RewardActorAddr] = builtin2.RewardActorAddr + addressToID[builtin2.CronActorAddr] = builtin2.CronActorAddr + addressToID[builtin2.StoragePowerActorAddr] = builtin2.StoragePowerActorAddr + addressToID[builtin2.StorageMarketActorAddr] = builtin2.StorageMarketActorAddr + addressToID[builtin2.VerifiedRegistryActorAddr] = builtin2.VerifiedRegistryActorAddr + addressToID[builtin2.BurntFundsActorAddr] = builtin2.BurntFundsActorAddr + initActor, err := p.node.StateGetActor(ctx, builtin2.InitActorAddr, types.EmptyTSK) if err != nil { return err } diff --git a/journal/registry_test.go b/journal/registry_test.go index 9d2085c12..bce3d3f17 100644 --- a/journal/registry_test.go +++ b/journal/registry_test.go @@ -14,7 +14,7 @@ func TestDisabledEvents(t *testing.T) { registry := NewEventTypeRegistry(dis) reg1 := registry.RegisterEventType("system1", "disabled1") - reg2 := registry.RegisterEventType("system2", "disabled2") + reg2 := registry.RegisterEventType("system1", "disabled2") req.False(reg1.Enabled()) req.False(reg2.Enabled()) @@ -29,21 +29,21 @@ func TestDisabledEvents(t *testing.T) { t.Run("direct", test(DisabledEvents{ EventType{System: "system1", Event: "disabled1"}, - EventType{System: "system2", Event: "disabled2"}, + EventType{System: "system1", Event: "disabled2"}, })) - dis, err := ParseDisabledEvents("system1:disabled1,system2:disabled2") + dis, err := ParseDisabledEvents("system1:disabled1,system1:disabled2") req.NoError(err) t.Run("parsed", test(dis)) - dis, err = ParseDisabledEvents(" system1:disabled1 , system2:disabled2 ") + dis, err = ParseDisabledEvents(" system1:disabled1 , system1:disabled2 ") req.NoError(err) t.Run("parsed_spaces", test(dis)) } func TestParseDisableEvents(t *testing.T) { - _, err := ParseDisabledEvents("system1:disabled1:failed,system2:disabled2") + _, err := ParseDisabledEvents("system1:disabled1:failed,system1:disabled2") require.Error(t, err) } diff --git a/journal/types.go b/journal/types.go index 56c7d4bc0..5b51ed4c8 100644 --- a/journal/types.go +++ b/journal/types.go @@ -22,7 +22,7 @@ var ( // DisabledEvents is the set of event types whose journaling is suppressed. type DisabledEvents []EventType -// ParseDisabledEvents parses a string of the form: "system1:event1,system2:event2[,...]" +// ParseDisabledEvents parses a string of the form: "system1:event1,system1:event2[,...]" // into a DisabledEvents object, returning an error if the string failed to parse. // // It sanitizes strings via strings.TrimSpace. From 6fcf57ed1f45ab8938ed3b12e65b2a3e176ff26d Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 28 Sep 2020 13:34:14 -0700 Subject: [PATCH 393/885] use actor state abstraction in payment channel test --- api/test/paych.go | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/api/test/paych.go b/api/test/paych.go index 36eb2c256..4c3bdba1b 100644 --- a/api/test/paych.go +++ b/api/test/paych.go @@ -8,17 +8,21 @@ import ( "testing" "time" - "github.com/filecoin-project/specs-actors/actors/builtin" + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" + paych0 "github.com/filecoin-project/specs-actors/actors/builtin/paych" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" - "github.com/filecoin-project/specs-actors/actors/builtin/paych" "github.com/ipfs/go-cid" "github.com/filecoin-project/go-address" + cbor "github.com/ipfs/go-ipld-cbor" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/api/apibstore" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/chain/actors/builtin/paych" "github.com/filecoin-project/lotus/chain/events" "github.com/filecoin-project/lotus/chain/events/state" "github.com/filecoin-project/lotus/chain/types" @@ -133,17 +137,26 @@ func TestPaymentChannels(t *testing.T, b APIBuilder, blocktime time.Duration) { t.Fatal("Unable to settle payment channel") } + creatorStore := adt.WrapStore(ctx, cbor.NewCborStore(apibstore.NewAPIBlockstore(paymentCreator))) + // wait for the receiver to submit their vouchers ev := events.NewEvents(ctx, paymentCreator) preds := state.NewStatePredicates(paymentCreator) finished := make(chan struct{}) err = ev.StateChanged(func(ts *types.TipSet) (done bool, more bool, err error) { - act, err := paymentCreator.StateReadState(ctx, channel, ts.Key()) + act, err := paymentCreator.StateGetActor(ctx, channel, ts.Key()) if err != nil { return false, false, err } - state := act.State.(paych.State) - if state.ToSend.GreaterThanEqual(abi.NewTokenAmount(6000)) { + state, err := paych.Load(creatorStore, act) + if err != nil { + return false, false, err + } + toSend, err := state.ToSend() + if err != nil { + return false, false, err + } + if toSend.GreaterThanEqual(abi.NewTokenAmount(6000)) { return true, false, nil } return false, true, nil @@ -170,7 +183,7 @@ func TestPaymentChannels(t *testing.T, b APIBuilder, blocktime time.Duration) { } // wait for the settlement period to pass before collecting - waitForBlocks(ctx, t, bm, paymentReceiver, receiverAddr, paych.SettleDelay) + waitForBlocks(ctx, t, bm, paymentReceiver, receiverAddr, paych0.SettleDelay) creatorPreCollectBalance, err := paymentCreator.WalletBalance(ctx, createrAddr) if err != nil { @@ -226,7 +239,7 @@ func waitForBlocks(ctx context.Context, t *testing.T, bm *BlockMiner, paymentRec // Add a real block m, err := paymentReceiver.MpoolPushMessage(ctx, &types.Message{ - To: builtin.BurntFundsActorAddr, + To: builtin0.BurntFundsActorAddr, From: receiverAddr, Value: types.NewInt(0), }, nil) From 1a7fb06dd9629f87be2f447fe0d7240944ee3749 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 28 Sep 2020 14:09:04 -0700 Subject: [PATCH 394/885] improve lotuspond There's still some work to do here, but it should now work better with the network upgrade. --- lotuspond/front/src/chain/methodgen.go | 38 ++++++++++---------------- 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/lotuspond/front/src/chain/methodgen.go b/lotuspond/front/src/chain/methodgen.go index 3197803fe..5a00d5e6e 100644 --- a/lotuspond/front/src/chain/methodgen.go +++ b/lotuspond/front/src/chain/methodgen.go @@ -4,12 +4,11 @@ import ( "encoding/json" "io/ioutil" "os" - "reflect" - "github.com/ipfs/go-cid" "github.com/multiformats/go-multihash" - "github.com/filecoin-project/specs-actors/actors/builtin" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/stmgr" ) func main() { @@ -17,6 +16,7 @@ func main() { panic(err) // note: must run in lotuspond/front/src/chain } + // TODO: ActorUpgrade: this is going to be a problem. names := map[string]string{ "system": "fil/1/system", "init": "fil/1/init", @@ -42,33 +42,25 @@ func main() { } } - methods := map[cid.Cid]interface{}{ - // builtin.SystemActorCodeID: builtin.MethodsSystem - apparently it doesn't have methods - builtin.InitActorCodeID: builtin.MethodsInit, - builtin.CronActorCodeID: builtin.MethodsCron, - builtin.AccountActorCodeID: builtin.MethodsAccount, - builtin.StoragePowerActorCodeID: builtin.MethodsPower, - builtin.StorageMinerActorCodeID: builtin.MethodsMiner, - builtin.StorageMarketActorCodeID: builtin.MethodsMarket, - builtin.PaymentChannelActorCodeID: builtin.MethodsPaych, - builtin.MultisigActorCodeID: builtin.MethodsMultisig, - builtin.RewardActorCodeID: builtin.MethodsReward, - builtin.VerifiedRegistryActorCodeID: builtin.MethodsVerifiedRegistry, - } - out := map[string][]string{} - for c, methods := range methods { + + for c, methods := range stmgr.MethodsMap { cmh, err := multihash.Decode(c.Hash()) if err != nil { panic(err) } - rt := reflect.TypeOf(methods) - nf := rt.NumField() + name := string(cmh.Digest) + remaining := len(methods) - out[string(cmh.Digest)] = append(out[string(cmh.Digest)], "Send") - for i := 0; i < nf; i++ { - out[string(cmh.Digest)] = append(out[string(cmh.Digest)], rt.Field(i).Name) + // iterate over actor methods in order. + for i := abi.MethodNum(0); remaining > 0; i++ { + m, ok := methods[i] + if !ok { + continue + } + out[name] = append(out[name], m.Name) + remaining-- } } From 52e064dabb0361cc063e97baa3f4934405ddb331 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 28 Sep 2020 14:09:46 -0700 Subject: [PATCH 395/885] add gas numbers for v2 --- chain/messagepool/gasguess/guessgas.go | 46 +++++++++++++++++--------- 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/chain/messagepool/gasguess/guessgas.go b/chain/messagepool/gasguess/guessgas.go index af58db7d2..1a6a21755 100644 --- a/chain/messagepool/gasguess/guessgas.go +++ b/chain/messagepool/gasguess/guessgas.go @@ -10,7 +10,9 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/specs-actors/actors/builtin" + + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" ) type ActorLookup func(context.Context, address.Address, types.TipSetKey) (*types.Actor, error) @@ -27,19 +29,32 @@ type CostKey struct { } var Costs = map[CostKey]int64{ - {builtin.InitActorCodeID, 2}: 8916753, - {builtin.StorageMarketActorCodeID, 2}: 6955002, - {builtin.StorageMarketActorCodeID, 4}: 245436108, - {builtin.StorageMinerActorCodeID, 4}: 2315133, - {builtin.StorageMinerActorCodeID, 5}: 1600271356, - {builtin.StorageMinerActorCodeID, 6}: 22864493, - {builtin.StorageMinerActorCodeID, 7}: 142002419, - {builtin.StorageMinerActorCodeID, 10}: 23008274, - {builtin.StorageMinerActorCodeID, 11}: 19303178, - {builtin.StorageMinerActorCodeID, 14}: 566356835, - {builtin.StorageMinerActorCodeID, 16}: 5325185, - {builtin.StorageMinerActorCodeID, 18}: 2328637, - {builtin.StoragePowerActorCodeID, 2}: 23600956, + {builtin0.InitActorCodeID, 2}: 8916753, + {builtin0.StorageMarketActorCodeID, 2}: 6955002, + {builtin0.StorageMarketActorCodeID, 4}: 245436108, + {builtin0.StorageMinerActorCodeID, 4}: 2315133, + {builtin0.StorageMinerActorCodeID, 5}: 1600271356, + {builtin0.StorageMinerActorCodeID, 6}: 22864493, + {builtin0.StorageMinerActorCodeID, 7}: 142002419, + {builtin0.StorageMinerActorCodeID, 10}: 23008274, + {builtin0.StorageMinerActorCodeID, 11}: 19303178, + {builtin0.StorageMinerActorCodeID, 14}: 566356835, + {builtin0.StorageMinerActorCodeID, 16}: 5325185, + {builtin0.StorageMinerActorCodeID, 18}: 2328637, + {builtin0.StoragePowerActorCodeID, 2}: 23600956, + {builtin2.InitActorCodeID, 2}: 8916753, + {builtin2.StorageMarketActorCodeID, 2}: 6955002, + {builtin2.StorageMarketActorCodeID, 4}: 245436108, + {builtin2.StorageMinerActorCodeID, 4}: 2315133, + {builtin2.StorageMinerActorCodeID, 5}: 1600271356, + {builtin2.StorageMinerActorCodeID, 6}: 22864493, + {builtin2.StorageMinerActorCodeID, 7}: 142002419, + {builtin2.StorageMinerActorCodeID, 10}: 23008274, + {builtin2.StorageMinerActorCodeID, 11}: 19303178, + {builtin2.StorageMinerActorCodeID, 14}: 566356835, + {builtin2.StorageMinerActorCodeID, 16}: 5325185, + {builtin2.StorageMinerActorCodeID, 18}: 2328637, + {builtin2.StoragePowerActorCodeID, 2}: 23600956, } func failedGuess(msg *types.SignedMessage) int64 { @@ -51,7 +66,8 @@ func failedGuess(msg *types.SignedMessage) int64 { } func GuessGasUsed(ctx context.Context, tsk types.TipSetKey, msg *types.SignedMessage, al ActorLookup) (int64, error) { - if msg.Message.Method == builtin.MethodSend { + // MethodSend is the same in all versions. + if msg.Message.Method == builtin0.MethodSend { switch msg.Message.From.Protocol() { case address.BLS: return 1298450, nil From 233d8a9b7292a80ec51cf46be68d4373f9a82712 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 28 Sep 2020 14:25:58 -0700 Subject: [PATCH 396/885] update even more imports --- chain/actors/builtin/paych/paych.go | 18 ++++++++++++++++++ chain/stmgr/forks.go | 20 ++++++++------------ chain/store/store.go | 10 +++++++--- chain/sub/incoming.go | 10 ++++++---- chain/sync.go | 16 +++++++++------- chain/types/voucher.go | 22 ---------------------- cli/paych.go | 10 +++++----- extern/storage-sealing/states_failed.go | 6 +++--- extern/storage-sealing/types.go | 4 ++-- markets/retrievaladapter/client.go | 3 +-- markets/retrievaladapter/provider.go | 2 +- node/impl/full/chain.go | 2 ++ paychmgr/manager.go | 13 ++++++------- paychmgr/paych.go | 20 ++++++++++++-------- paychmgr/settler/settler.go | 10 +++++----- paychmgr/store.go | 15 +++++++-------- paychmgr/util.go | 10 +++++----- 17 files changed, 97 insertions(+), 94 deletions(-) delete mode 100644 chain/types/voucher.go diff --git a/chain/actors/builtin/paych/paych.go b/chain/actors/builtin/paych/paych.go index f462de6fd..20c7a74b7 100644 --- a/chain/actors/builtin/paych/paych.go +++ b/chain/actors/builtin/paych/paych.go @@ -1,6 +1,8 @@ package paych import ( + "encoding/base64" + "golang.org/x/xerrors" "github.com/filecoin-project/go-address" @@ -8,6 +10,7 @@ import ( big "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/cbor" "github.com/ipfs/go-cid" + ipldcbor "github.com/ipfs/go-ipld-cbor" builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" paych0 "github.com/filecoin-project/specs-actors/actors/builtin/paych" @@ -68,3 +71,18 @@ type LaneState interface { type SignedVoucher = paych0.SignedVoucher type ModVerifyParams = paych0.ModVerifyParams + +// DecodeSignedVoucher decodes base64 encoded signed voucher. +func DecodeSignedVoucher(s string) (*SignedVoucher, error) { + data, err := base64.RawURLEncoding.DecodeString(s) + if err != nil { + return nil, err + } + + var sv SignedVoucher + if err := ipldcbor.DecodeInto(data, &sv); err != nil { + return nil, err + } + + return &sv, nil +} diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index 252b731d7..9561e5ed9 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -6,31 +6,27 @@ import ( "encoding/binary" "math" - multisig0 "github.com/filecoin-project/specs-actors/actors/builtin/multisig" - - "github.com/filecoin-project/lotus/chain/actors/builtin/multisig" - - "github.com/filecoin-project/lotus/chain/state" - - "github.com/filecoin-project/specs-actors/actors/migration/nv3" - - "github.com/ipfs/go-cid" - "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" + "github.com/ipfs/go-cid" + cbor "github.com/ipfs/go-ipld-cbor" + "golang.org/x/xerrors" builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" init0 "github.com/filecoin-project/specs-actors/actors/builtin/init" miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" + multisig0 "github.com/filecoin-project/specs-actors/actors/builtin/multisig" power0 "github.com/filecoin-project/specs-actors/actors/builtin/power" adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" + "github.com/filecoin-project/specs-actors/actors/migration/nv3" + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors/builtin/multisig" + "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" - cbor "github.com/ipfs/go-ipld-cbor" - "golang.org/x/xerrors" ) var ForksAtHeight = map[abi.ChainEpoch]func(context.Context, *StateManager, ExecCallback, cid.Cid, *types.TipSet) (cid.Cid, error){ diff --git a/chain/store/store.go b/chain/store/store.go index 6c93db7a0..ab732413b 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -17,11 +17,13 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/util/adt" + adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/vm" "github.com/filecoin-project/lotus/journal" bstore "github.com/filecoin-project/lotus/lib/blockstore" @@ -747,7 +749,8 @@ func (cs *ChainStore) GetSignedMessage(c cid.Cid) (*types.SignedMessage, error) func (cs *ChainStore) readAMTCids(root cid.Cid) ([]cid.Cid, error) { ctx := context.TODO() - a, err := adt.AsArray(cs.Store(ctx), root) + // block headers use adt0, for now. + a, err := adt0.AsArray(cs.Store(ctx), root) if err != nil { return nil, xerrors.Errorf("amt load: %w", err) } @@ -940,7 +943,8 @@ func (cs *ChainStore) MessagesForBlock(b *types.BlockHeader) ([]*types.Message, func (cs *ChainStore) GetParentReceipt(b *types.BlockHeader, i int) (*types.MessageReceipt, error) { ctx := context.TODO() - a, err := adt.AsArray(cs.Store(ctx), b.ParentMessageReceipts) + // block headers use adt0, for now. + a, err := adt0.AsArray(cs.Store(ctx), b.ParentMessageReceipts) if err != nil { return nil, xerrors.Errorf("amt load: %w", err) } diff --git a/chain/sub/incoming.go b/chain/sub/incoming.go index c6e0c8b80..eb17b078d 100644 --- a/chain/sub/incoming.go +++ b/chain/sub/incoming.go @@ -10,7 +10,6 @@ import ( "golang.org/x/xerrors" address "github.com/filecoin-project/go-address" - "github.com/filecoin-project/specs-actors/actors/util/adt" lru "github.com/hashicorp/golang-lru" blocks "github.com/ipfs/go-block-format" bserv "github.com/ipfs/go-blockservice" @@ -24,6 +23,8 @@ import ( "go.opencensus.io/stats" "go.opencensus.io/tag" + adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" @@ -374,9 +375,10 @@ func (bv *BlockValidator) isChainNearSynced() bool { func (bv *BlockValidator) validateMsgMeta(ctx context.Context, msg *types.BlockMsg) error { // TODO there has to be a simpler way to do this without the blockstore dance - store := adt.WrapStore(ctx, cbor.NewCborStore(blockstore.NewTemporary())) - bmArr := adt.MakeEmptyArray(store) - smArr := adt.MakeEmptyArray(store) + // block headers use adt0 + store := adt0.WrapStore(ctx, cbor.NewCborStore(blockstore.NewTemporary())) + bmArr := adt0.MakeEmptyArray(store) + smArr := adt0.MakeEmptyArray(store) for i, m := range msg.BlsMessages { c := cbg.CborCid(m) diff --git a/chain/sync.go b/chain/sync.go index 9e098a57e..6abe43567 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -33,9 +33,10 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" - "github.com/filecoin-project/specs-actors/actors/util/adt" blst "github.com/supranational/blst/bindings/go" + adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors/builtin/power" @@ -455,9 +456,10 @@ func zipTipSetAndMessages(bs cbor.IpldStore, ts *types.TipSet, allbmsgs []*types // computeMsgMeta computes the root CID of the combined arrays of message CIDs // of both types (BLS and Secpk). func computeMsgMeta(bs cbor.IpldStore, bmsgCids, smsgCids []cid.Cid) (cid.Cid, error) { - store := adt.WrapStore(context.TODO(), bs) - bmArr := adt.MakeEmptyArray(store) - smArr := adt.MakeEmptyArray(store) + // block headers use adt0 + store := adt0.WrapStore(context.TODO(), bs) + bmArr := adt0.MakeEmptyArray(store) + smArr := adt0.MakeEmptyArray(store) for i, m := range bmsgCids { c := cbg.CborCid(m) @@ -1092,9 +1094,9 @@ func (syncer *Syncer) checkBlockMessages(ctx context.Context, b *types.FullBlock return nil } - store := adt.WrapStore(ctx, cst) + store := adt0.WrapStore(ctx, cst) - bmArr := adt.MakeEmptyArray(store) + bmArr := adt0.MakeEmptyArray(store) for i, m := range b.BlsMessages { if err := checkMsg(m); err != nil { return xerrors.Errorf("block had invalid bls message at index %d: %w", i, err) @@ -1106,7 +1108,7 @@ func (syncer *Syncer) checkBlockMessages(ctx context.Context, b *types.FullBlock } } - smArr := adt.MakeEmptyArray(store) + smArr := adt0.MakeEmptyArray(store) for i, m := range b.SecpkMessages { if err := checkMsg(m); err != nil { return xerrors.Errorf("block had invalid secpk message at index %d: %w", i, err) diff --git a/chain/types/voucher.go b/chain/types/voucher.go deleted file mode 100644 index 687109c33..000000000 --- a/chain/types/voucher.go +++ /dev/null @@ -1,22 +0,0 @@ -package types - -import ( - "encoding/base64" - - "github.com/filecoin-project/specs-actors/actors/builtin/paych" - cbor "github.com/ipfs/go-ipld-cbor" -) - -func DecodeSignedVoucher(s string) (*paych.SignedVoucher, error) { - data, err := base64.RawURLEncoding.DecodeString(s) - if err != nil { - return nil, err - } - - var sv paych.SignedVoucher - if err := cbor.DecodeInto(data, &sv); err != nil { - return nil, err - } - - return &sv, nil -} diff --git a/cli/paych.go b/cli/paych.go index dc5b54c4a..1d5e304c3 100644 --- a/cli/paych.go +++ b/cli/paych.go @@ -14,10 +14,10 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/specs-actors/actors/builtin/paych" "github.com/urfave/cli/v2" - types "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/actors/builtin/paych" + "github.com/filecoin-project/lotus/chain/types" ) var paychCmd = &cli.Command{ @@ -404,7 +404,7 @@ var paychVoucherCheckCmd = &cli.Command{ return err } - sv, err := types.DecodeSignedVoucher(cctx.Args().Get(1)) + sv, err := paych.DecodeSignedVoucher(cctx.Args().Get(1)) if err != nil { return err } @@ -440,7 +440,7 @@ var paychVoucherAddCmd = &cli.Command{ return err } - sv, err := types.DecodeSignedVoucher(cctx.Args().Get(1)) + sv, err := paych.DecodeSignedVoucher(cctx.Args().Get(1)) if err != nil { return err } @@ -598,7 +598,7 @@ var paychVoucherSubmitCmd = &cli.Command{ return err } - sv, err := types.DecodeSignedVoucher(cctx.Args().Get(1)) + sv, err := paych.DecodeSignedVoucher(cctx.Args().Get(1)) if err != nil { return err } diff --git a/extern/storage-sealing/states_failed.go b/extern/storage-sealing/states_failed.go index b026e70f9..63e64ca1b 100644 --- a/extern/storage-sealing/states_failed.go +++ b/extern/storage-sealing/states_failed.go @@ -4,14 +4,14 @@ import ( "bytes" "time" - "github.com/filecoin-project/lotus/chain/actors/builtin/miner" - "golang.org/x/xerrors" + "github.com/filecoin-project/lotus/chain/actors/builtin/market" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/exitcode" "github.com/filecoin-project/go-statemachine" - "github.com/filecoin-project/specs-actors/actors/builtin/market" "github.com/filecoin-project/lotus/extern/sector-storage/zerocomm" ) diff --git a/extern/storage-sealing/types.go b/extern/storage-sealing/types.go index c044defd3..046271a7f 100644 --- a/extern/storage-sealing/types.go +++ b/extern/storage-sealing/types.go @@ -9,7 +9,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/exitcode" - "github.com/filecoin-project/specs-actors/actors/builtin/miner" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/specs-storage/storage" sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" @@ -82,7 +82,7 @@ type SectorInfo struct { CommR *cid.Cid Proof []byte - PreCommitInfo *miner.SectorPreCommitInfo + PreCommitInfo *miner0.SectorPreCommitInfo PreCommitDeposit big.Int PreCommitMessage *cid.Cid PreCommitTipSet TipSetToken diff --git a/markets/retrievaladapter/client.go b/markets/retrievaladapter/client.go index ab81dd8c9..1bef23e12 100644 --- a/markets/retrievaladapter/client.go +++ b/markets/retrievaladapter/client.go @@ -3,8 +3,6 @@ package retrievaladapter import ( "context" - "github.com/filecoin-project/specs-actors/actors/builtin/paych" - "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/go-fil-markets/shared" @@ -12,6 +10,7 @@ import ( "github.com/ipfs/go-cid" "github.com/multiformats/go-multiaddr" + "github.com/filecoin-project/lotus/chain/actors/builtin/paych" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/impl/full" payapi "github.com/filecoin-project/lotus/node/impl/paych" diff --git a/markets/retrievaladapter/provider.go b/markets/retrievaladapter/provider.go index 443c4fb4a..674ec4793 100644 --- a/markets/retrievaladapter/provider.go +++ b/markets/retrievaladapter/provider.go @@ -5,6 +5,7 @@ import ( "io" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/actors/builtin/paych" "github.com/filecoin-project/lotus/chain/types" sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" "github.com/filecoin-project/lotus/extern/sector-storage/storiface" @@ -14,7 +15,6 @@ import ( "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/go-fil-markets/shared" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/specs-actors/actors/builtin/paych" "github.com/ipfs/go-cid" ) diff --git a/node/impl/full/chain.go b/node/impl/full/chain.go index ea715c66a..095e32ad2 100644 --- a/node/impl/full/chain.go +++ b/node/impl/full/chain.go @@ -285,6 +285,8 @@ func (s stringKey) Key() string { return (string)(s) } +// TODO: ActorUpgrade: this entire function is a problem (in theory) as we don't know the HAMT version. +// In practice, hamt v0 should work "just fine" for reading. func resolveOnce(bs blockstore.Blockstore) func(ctx context.Context, ds ipld.NodeGetter, nd ipld.Node, names []string) (*ipld.Link, []string, error) { return func(ctx context.Context, ds ipld.NodeGetter, nd ipld.Node, names []string) (*ipld.Link, []string, error) { store := adt.WrapStore(ctx, cbor.NewCborStore(bs)) diff --git a/paychmgr/manager.go b/paychmgr/manager.go index d9c568cbe..7ef1264c5 100644 --- a/paychmgr/manager.go +++ b/paychmgr/manager.go @@ -12,7 +12,6 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/crypto" - paych0 "github.com/filecoin-project/specs-actors/actors/builtin/paych" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors/adt" @@ -221,7 +220,7 @@ func (pm *Manager) GetChannelInfo(addr address.Address) (*ChannelInfo, error) { return ca.getChannelInfo(addr) } -func (pm *Manager) CreateVoucher(ctx context.Context, ch address.Address, voucher paych0.SignedVoucher) (*api.VoucherCreateResult, error) { +func (pm *Manager) CreateVoucher(ctx context.Context, ch address.Address, voucher paych.SignedVoucher) (*api.VoucherCreateResult, error) { ca, err := pm.accessorByAddress(ch) if err != nil { return nil, err @@ -233,7 +232,7 @@ func (pm *Manager) CreateVoucher(ctx context.Context, ch address.Address, vouche // CheckVoucherValid checks if the given voucher is valid (is or could become spendable at some point). // If the channel is not in the store, fetches the channel from state (and checks that // the channel To address is owned by the wallet). -func (pm *Manager) CheckVoucherValid(ctx context.Context, ch address.Address, sv *paych0.SignedVoucher) error { +func (pm *Manager) CheckVoucherValid(ctx context.Context, ch address.Address, sv *paych.SignedVoucher) error { // Get an accessor for the channel, creating it from state if necessary ca, err := pm.inboundChannelAccessor(ctx, ch) if err != nil { @@ -245,7 +244,7 @@ func (pm *Manager) CheckVoucherValid(ctx context.Context, ch address.Address, sv } // CheckVoucherSpendable checks if the given voucher is currently spendable -func (pm *Manager) CheckVoucherSpendable(ctx context.Context, ch address.Address, sv *paych0.SignedVoucher, secret []byte, proof []byte) (bool, error) { +func (pm *Manager) CheckVoucherSpendable(ctx context.Context, ch address.Address, sv *paych.SignedVoucher, secret []byte, proof []byte) (bool, error) { ca, err := pm.accessorByAddress(ch) if err != nil { return false, err @@ -256,7 +255,7 @@ func (pm *Manager) CheckVoucherSpendable(ctx context.Context, ch address.Address // AddVoucherOutbound adds a voucher for an outbound channel. // Returns an error if the channel is not already in the store. -func (pm *Manager) AddVoucherOutbound(ctx context.Context, ch address.Address, sv *paych0.SignedVoucher, proof []byte, minDelta types.BigInt) (types.BigInt, error) { +func (pm *Manager) AddVoucherOutbound(ctx context.Context, ch address.Address, sv *paych.SignedVoucher, proof []byte, minDelta types.BigInt) (types.BigInt, error) { ca, err := pm.accessorByAddress(ch) if err != nil { return types.NewInt(0), err @@ -267,7 +266,7 @@ func (pm *Manager) AddVoucherOutbound(ctx context.Context, ch address.Address, s // AddVoucherInbound adds a voucher for an inbound channel. // If the channel is not in the store, fetches the channel from state (and checks that // the channel To address is owned by the wallet). -func (pm *Manager) AddVoucherInbound(ctx context.Context, ch address.Address, sv *paych0.SignedVoucher, proof []byte, minDelta types.BigInt) (types.BigInt, error) { +func (pm *Manager) AddVoucherInbound(ctx context.Context, ch address.Address, sv *paych.SignedVoucher, proof []byte, minDelta types.BigInt) (types.BigInt, error) { // Get an accessor for the channel, creating it from state if necessary ca, err := pm.inboundChannelAccessor(ctx, ch) if err != nil { @@ -336,7 +335,7 @@ func (pm *Manager) trackInboundChannel(ctx context.Context, ch address.Address) return pm.store.TrackChannel(stateCi) } -func (pm *Manager) SubmitVoucher(ctx context.Context, ch address.Address, sv *paych0.SignedVoucher, secret []byte, proof []byte) (cid.Cid, error) { +func (pm *Manager) SubmitVoucher(ctx context.Context, ch address.Address, sv *paych.SignedVoucher, secret []byte, proof []byte) (cid.Cid, error) { ca, err := pm.accessorByAddress(ch) if err != nil { return cid.Undef, err diff --git a/paychmgr/paych.go b/paychmgr/paych.go index f856b9890..ba96c5e40 100644 --- a/paychmgr/paych.go +++ b/paychmgr/paych.go @@ -103,7 +103,7 @@ func (ca *channelAccessor) outboundActiveByFromTo(from, to address.Address) (*Ch // nonce, signing the voucher and storing it in the local datastore. // If there are not enough funds in the channel to create the voucher, returns // the shortfall in funds. -func (ca *channelAccessor) createVoucher(ctx context.Context, ch address.Address, voucher paych0.SignedVoucher) (*api.VoucherCreateResult, error) { +func (ca *channelAccessor) createVoucher(ctx context.Context, ch address.Address, voucher paych.SignedVoucher) (*api.VoucherCreateResult, error) { ca.lk.Lock() defer ca.lk.Unlock() @@ -162,14 +162,14 @@ func (ca *channelAccessor) nextNonceForLane(ci *ChannelInfo, lane uint64) uint64 return maxnonce + 1 } -func (ca *channelAccessor) checkVoucherValid(ctx context.Context, ch address.Address, sv *paych0.SignedVoucher) (map[uint64]paych.LaneState, error) { +func (ca *channelAccessor) checkVoucherValid(ctx context.Context, ch address.Address, sv *paych.SignedVoucher) (map[uint64]paych.LaneState, error) { ca.lk.Lock() defer ca.lk.Unlock() return ca.checkVoucherValidUnlocked(ctx, ch, sv) } -func (ca *channelAccessor) checkVoucherValidUnlocked(ctx context.Context, ch address.Address, sv *paych0.SignedVoucher) (map[uint64]paych.LaneState, error) { +func (ca *channelAccessor) checkVoucherValidUnlocked(ctx context.Context, ch address.Address, sv *paych.SignedVoucher) (map[uint64]paych.LaneState, error) { if sv.ChannelAddr != ch { return nil, xerrors.Errorf("voucher ChannelAddr doesn't match channel address, got %s, expected %s", sv.ChannelAddr, ch) } @@ -272,7 +272,7 @@ func (ca *channelAccessor) checkVoucherValidUnlocked(ctx context.Context, ch add return laneStates, nil } -func (ca *channelAccessor) checkVoucherSpendable(ctx context.Context, ch address.Address, sv *paych0.SignedVoucher, secret []byte, proof []byte) (bool, error) { +func (ca *channelAccessor) checkVoucherSpendable(ctx context.Context, ch address.Address, sv *paych.SignedVoucher, secret []byte, proof []byte) (bool, error) { ca.lk.Lock() defer ca.lk.Unlock() @@ -346,14 +346,14 @@ func (ca *channelAccessor) getPaychRecipient(ctx context.Context, ch address.Add return state.To() } -func (ca *channelAccessor) addVoucher(ctx context.Context, ch address.Address, sv *paych0.SignedVoucher, proof []byte, minDelta types.BigInt) (types.BigInt, error) { +func (ca *channelAccessor) addVoucher(ctx context.Context, ch address.Address, sv *paych.SignedVoucher, proof []byte, minDelta types.BigInt) (types.BigInt, error) { ca.lk.Lock() defer ca.lk.Unlock() return ca.addVoucherUnlocked(ctx, ch, sv, proof, minDelta) } -func (ca *channelAccessor) addVoucherUnlocked(ctx context.Context, ch address.Address, sv *paych0.SignedVoucher, proof []byte, minDelta types.BigInt) (types.BigInt, error) { +func (ca *channelAccessor) addVoucherUnlocked(ctx context.Context, ch address.Address, sv *paych.SignedVoucher, proof []byte, minDelta types.BigInt) (types.BigInt, error) { ci, err := ca.store.ByAddress(ch) if err != nil { return types.BigInt{}, err @@ -420,7 +420,7 @@ func (ca *channelAccessor) addVoucherUnlocked(ctx context.Context, ch address.Ad return delta, ca.store.putChannelInfo(ci) } -func (ca *channelAccessor) submitVoucher(ctx context.Context, ch address.Address, sv *paych0.SignedVoucher, secret []byte, proof []byte) (cid.Cid, error) { +func (ca *channelAccessor) submitVoucher(ctx context.Context, ch address.Address, sv *paych.SignedVoucher, secret []byte, proof []byte) (cid.Cid, error) { ca.lk.Lock() defer ca.lk.Unlock() @@ -461,6 +461,10 @@ func (ca *channelAccessor) submitVoucher(ctx context.Context, ch address.Address } } + // TODO: ActorUpgrade + // The "proof" field is going away. We will need to abstract over the + // network version here. + // Alternatively, we'd need to support the "old" method on-chain. enc, err := actors.SerializeParams(&paych0.UpdateChannelStateParams{ Sv: *sv, Secret: secret, @@ -572,7 +576,7 @@ func (ca *channelAccessor) laneState(ctx context.Context, state paych.State, ch } // Get the total redeemed amount across all lanes, after applying the voucher -func (ca *channelAccessor) totalRedeemedWithVoucher(laneStates map[uint64]paych.LaneState, sv *paych0.SignedVoucher) (big.Int, error) { +func (ca *channelAccessor) totalRedeemedWithVoucher(laneStates map[uint64]paych.LaneState, sv *paych.SignedVoucher) (big.Int, error) { // TODO: merges if len(sv.Merges) != 0 { return big.Int{}, xerrors.Errorf("dont currently support paych lane merges") diff --git a/paychmgr/settler/settler.go b/paychmgr/settler/settler.go index 654ed66cc..02fe9256e 100644 --- a/paychmgr/settler/settler.go +++ b/paychmgr/settler/settler.go @@ -15,10 +15,10 @@ import ( "github.com/filecoin-project/go-state-types/abi" builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" - paych0 "github.com/filecoin-project/specs-actors/actors/builtin/paych" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors/builtin/paych" "github.com/filecoin-project/lotus/chain/events" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/impl/full" @@ -39,9 +39,9 @@ type API struct { type settlerAPI interface { PaychList(context.Context) ([]address.Address, error) PaychStatus(context.Context, address.Address) (*api.PaychStatus, error) - PaychVoucherCheckSpendable(context.Context, address.Address, *paych0.SignedVoucher, []byte, []byte) (bool, error) - PaychVoucherList(context.Context, address.Address) ([]*paych0.SignedVoucher, error) - PaychVoucherSubmit(context.Context, address.Address, *paych0.SignedVoucher, []byte, []byte) (cid.Cid, error) + PaychVoucherCheckSpendable(context.Context, address.Address, *paych.SignedVoucher, []byte, []byte) (bool, error) + PaychVoucherList(context.Context, address.Address) ([]*paych.SignedVoucher, error) + PaychVoucherSubmit(context.Context, address.Address, *paych.SignedVoucher, []byte, []byte) (cid.Cid, error) StateWaitMsg(ctx context.Context, cid cid.Cid, confidence uint64) (*api.MsgLookup, error) } @@ -86,7 +86,7 @@ func (pcs *paymentChannelSettler) messageHandler(msg *types.Message, rec *types. if err != nil { return true, err } - go func(voucher *paych0.SignedVoucher, submitMessageCID cid.Cid) { + go func(voucher *paych.SignedVoucher, submitMessageCID cid.Cid) { defer wg.Done() msgLookup, err := pcs.api.StateWaitMsg(pcs.ctx, submitMessageCID, build.MessageConfidence) if err != nil { diff --git a/paychmgr/store.go b/paychmgr/store.go index 23916669e..dbd663832 100644 --- a/paychmgr/store.go +++ b/paychmgr/store.go @@ -17,11 +17,10 @@ import ( "github.com/ipfs/go-datastore/namespace" dsq "github.com/ipfs/go-datastore/query" - paych0 "github.com/filecoin-project/specs-actors/actors/builtin/paych" - "github.com/filecoin-project/go-address" cborrpc "github.com/filecoin-project/go-cbor-util" + "github.com/filecoin-project/lotus/chain/actors/builtin/paych" "github.com/filecoin-project/lotus/node/modules/dtypes" ) @@ -49,7 +48,7 @@ const ( ) type VoucherInfo struct { - Voucher *paych0.SignedVoucher + Voucher *paych.SignedVoucher Proof []byte Submitted bool } @@ -103,7 +102,7 @@ func (ci *ChannelInfo) to() address.Address { // infoForVoucher gets the VoucherInfo for the given voucher. // returns nil if the channel doesn't have the voucher. -func (ci *ChannelInfo) infoForVoucher(sv *paych0.SignedVoucher) (*VoucherInfo, error) { +func (ci *ChannelInfo) infoForVoucher(sv *paych.SignedVoucher) (*VoucherInfo, error) { for _, v := range ci.Vouchers { eq, err := cborutil.Equals(sv, v.Voucher) if err != nil { @@ -116,7 +115,7 @@ func (ci *ChannelInfo) infoForVoucher(sv *paych0.SignedVoucher) (*VoucherInfo, e return nil, nil } -func (ci *ChannelInfo) hasVoucher(sv *paych0.SignedVoucher) (bool, error) { +func (ci *ChannelInfo) hasVoucher(sv *paych.SignedVoucher) (bool, error) { vi, err := ci.infoForVoucher(sv) return vi != nil, err } @@ -124,7 +123,7 @@ func (ci *ChannelInfo) hasVoucher(sv *paych0.SignedVoucher) (bool, error) { // markVoucherSubmitted marks the voucher, and any vouchers of lower nonce // in the same lane, as being submitted. // Note: This method doesn't write anything to the store. -func (ci *ChannelInfo) markVoucherSubmitted(sv *paych0.SignedVoucher) error { +func (ci *ChannelInfo) markVoucherSubmitted(sv *paych.SignedVoucher) error { vi, err := ci.infoForVoucher(sv) if err != nil { return err @@ -148,7 +147,7 @@ func (ci *ChannelInfo) markVoucherSubmitted(sv *paych0.SignedVoucher) error { } // wasVoucherSubmitted returns true if the voucher has been submitted -func (ci *ChannelInfo) wasVoucherSubmitted(sv *paych0.SignedVoucher) (bool, error) { +func (ci *ChannelInfo) wasVoucherSubmitted(sv *paych.SignedVoucher) (bool, error) { vi, err := ci.infoForVoucher(sv) if err != nil { return false, err @@ -277,7 +276,7 @@ func (ps *Store) VouchersForPaych(ch address.Address) ([]*VoucherInfo, error) { return ci.Vouchers, nil } -func (ps *Store) MarkVoucherSubmitted(ci *ChannelInfo, sv *paych0.SignedVoucher) error { +func (ps *Store) MarkVoucherSubmitted(ci *ChannelInfo, sv *paych.SignedVoucher) error { err := ci.markVoucherSubmitted(sv) if err != nil { return err diff --git a/paychmgr/util.go b/paychmgr/util.go index 2a8181c15..8e5dd4fab 100644 --- a/paychmgr/util.go +++ b/paychmgr/util.go @@ -5,21 +5,21 @@ import ( "github.com/filecoin-project/go-address" - paych0 "github.com/filecoin-project/specs-actors/actors/builtin/paych" + "github.com/filecoin-project/lotus/chain/actors/builtin/paych" ) type BestSpendableAPI interface { - PaychVoucherList(context.Context, address.Address) ([]*paych0.SignedVoucher, error) - PaychVoucherCheckSpendable(context.Context, address.Address, *paych0.SignedVoucher, []byte, []byte) (bool, error) + PaychVoucherList(context.Context, address.Address) ([]*paych.SignedVoucher, error) + PaychVoucherCheckSpendable(context.Context, address.Address, *paych.SignedVoucher, []byte, []byte) (bool, error) } -func BestSpendableByLane(ctx context.Context, api BestSpendableAPI, ch address.Address) (map[uint64]*paych0.SignedVoucher, error) { +func BestSpendableByLane(ctx context.Context, api BestSpendableAPI, ch address.Address) (map[uint64]*paych.SignedVoucher, error) { vouchers, err := api.PaychVoucherList(ctx, ch) if err != nil { return nil, err } - bestByLane := make(map[uint64]*paych0.SignedVoucher) + bestByLane := make(map[uint64]*paych.SignedVoucher) for _, voucher := range vouchers { spendable, err := api.PaychVoucherCheckSpendable(ctx, ch, voucher, nil, nil) if err != nil { From c88d124954221398c7bab5f524dccbaad095c997 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 28 Sep 2020 14:43:55 -0700 Subject: [PATCH 397/885] use built-in actor ID The init actor will not allow the account actor to exec an arbitrary actor. --- chain/stmgr/forks_test.go | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/chain/stmgr/forks_test.go b/chain/stmgr/forks_test.go index ee378c983..9db6a491a 100644 --- a/chain/stmgr/forks_test.go +++ b/chain/stmgr/forks_test.go @@ -12,7 +12,6 @@ import ( "github.com/filecoin-project/specs-actors/actors/builtin" init_ "github.com/filecoin-project/specs-actors/actors/builtin/init" "github.com/filecoin-project/specs-actors/actors/runtime" - "github.com/multiformats/go-multihash" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/chain/actors" @@ -44,16 +43,8 @@ const testForkHeight = 40 type testActor struct { } -var testActorCodeID = func() cid.Cid { - builder := cid.V1Builder{Codec: cid.Raw, MhType: multihash.IDENTITY} - c, err := builder.Sum([]byte("fil/any/test")) - if err != nil { - panic(err) - } - return c -}() - -func (testActor) Code() cid.Cid { return testActorCodeID } +// must use existing actor that an account is allowed to exec. +func (testActor) Code() cid.Cid { return builtin.PaymentChannelActorCodeID } func (testActor) State() cbor.Er { return new(testActorState) } type testActorState struct { From 4b3b35c9defec5d2bfab6b527e4eeebe25109bd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Mon, 28 Sep 2020 23:04:52 +0100 Subject: [PATCH 398/885] conformance: record and feed circulating supply. --- cmd/tvx/extract.go | 18 ++++++++++++++---- conformance/driver.go | 34 ++++++++++++++++++++++------------ conformance/runner.go | 8 +++++++- go.mod | 2 +- go.sum | 4 ++-- 5 files changed, 46 insertions(+), 20 deletions(-) diff --git a/cmd/tvx/extract.go b/cmd/tvx/extract.go index e10fbad09..0dc7f6aa0 100644 --- a/cmd/tvx/extract.go +++ b/cmd/tvx/extract.go @@ -118,8 +118,17 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { return fmt.Errorf("failed to resolve message and tipsets from chain: %w", err) } + // get the circulating supply before the message was executed. + circSupplyDetail, err := fapi.StateCirculatingSupply(ctx, incTs.Key()) + if err != nil { + return fmt.Errorf("failed while fetching circulating supply: %w", err) + } + + circSupply := circSupplyDetail.FilCirculating.Int64() + log.Printf("message was executed in tipset: %s", execTs.Key()) log.Printf("message was included in tipset: %s", incTs.Key()) + log.Printf("circulating supply at inclusion tipset: %d", circSupply) log.Printf("finding precursor messages") // Iterate through blocks, finding the one that contains the message and its @@ -174,7 +183,7 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { log.Printf("number of precursors to apply: %d", len(precursors)) for i, m := range precursors { log.Printf("applying precursor %d, cid: %s", i, m.Cid()) - _, root, err = driver.ExecuteMessage(pst.Blockstore, root, execTs.Height(), m) + _, root, err = driver.ExecuteMessage(pst.Blockstore, root, execTs.Height(), m, &circSupplyDetail.FilCirculating) if err != nil { return fmt.Errorf("failed to execute precursor message: %w", err) } @@ -199,7 +208,7 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { tbs.StartTracing() preroot = root - applyret, postroot, err = driver.ExecuteMessage(pst.Blockstore, preroot, execTs.Height(), msg) + applyret, postroot, err = driver.ExecuteMessage(pst.Blockstore, preroot, execTs.Height(), msg, &circSupplyDetail.FilCirculating) if err != nil { return fmt.Errorf("failed to execute message: %w", err) } @@ -224,7 +233,7 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { if err != nil { return err } - applyret, postroot, err = driver.ExecuteMessage(pst.Blockstore, preroot, execTs.Height(), msg) + applyret, postroot, err = driver.ExecuteMessage(pst.Blockstore, preroot, execTs.Height(), msg, &circSupplyDetail.FilCirculating) if err != nil { return fmt.Errorf("failed to execute message: %w", err) } @@ -302,7 +311,8 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { }, CAR: out.Bytes(), Pre: &schema.Preconditions{ - Epoch: int64(execTs.Height()), + Epoch: int64(execTs.Height()), + CircSupply: &circSupply, StateTree: &schema.StateTree{ RootCID: preroot, }, diff --git a/conformance/driver.go b/conformance/driver.go index 90d05ae88..3f50b67a9 100644 --- a/conformance/driver.go +++ b/conformance/driver.go @@ -3,8 +3,6 @@ package conformance import ( "context" - "github.com/filecoin-project/go-state-types/crypto" - "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/store" @@ -15,6 +13,7 @@ import ( "github.com/filecoin-project/lotus/lib/blockstore" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/test-vectors/schema" @@ -24,6 +23,11 @@ import ( ds "github.com/ipfs/go-datastore" ) +// DefaultCirculatingSupply is the fallback circulating supply returned by +// the driver's CircSupplyCalculator function, used if the vector specifies +// no circulating supply. +var DefaultCirculatingSupply = types.TotalFilecoinInt + var ( // BaseFee to use in the VM. // TODO make parametrisable through vector. @@ -136,18 +140,24 @@ func (d *Driver) ExecuteTipset(bs blockstore.Blockstore, ds ds.Batching, preroot } // ExecuteMessage executes a conformance test vector message in a temporary VM. -func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, preroot cid.Cid, epoch abi.ChainEpoch, msg *types.Message) (*vm.ApplyRet, cid.Cid, error) { - // dummy state manager; only to reference the GetNetworkVersion method, which does not depend on state. +func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, preroot cid.Cid, epoch abi.ChainEpoch, msg *types.Message, circSupply *abi.TokenAmount) (*vm.ApplyRet, cid.Cid, error) { + // dummy state manager; only to reference the GetNetworkVersion method, + // which does not depend on state. sm := new(stmgr.StateManager) vmOpts := &vm.VMOpts{ - StateBase: preroot, - Epoch: epoch, - Rand: &testRand{}, // TODO always succeeds; need more flexibility. - Bstore: bs, - Syscalls: mkFakedSigSyscalls(vm.Syscalls(ffiwrapper.ProofVerifier)), // TODO always succeeds; need more flexibility. - CircSupplyCalc: nil, - BaseFee: BaseFee, - NtwkVersion: sm.GetNtwkVersion, + StateBase: preroot, + Epoch: epoch, + Rand: &testRand{}, // TODO always succeeds; need more flexibility. + Bstore: bs, + Syscalls: mkFakedSigSyscalls(vm.Syscalls(ffiwrapper.ProofVerifier)), // TODO always succeeds; need more flexibility. + CircSupplyCalc: func(_ context.Context, _ abi.ChainEpoch, _ *state.StateTree) (abi.TokenAmount, error) { + if circSupply != nil { + return *circSupply, nil + } + return DefaultCirculatingSupply, nil + }, + BaseFee: BaseFee, + NtwkVersion: sm.GetNtwkVersion, } lvm, err := vm.NewVM(context.TODO(), vmOpts) diff --git a/conformance/runner.go b/conformance/runner.go index 0fc4b13fc..812f3cc08 100644 --- a/conformance/runner.go +++ b/conformance/runner.go @@ -45,6 +45,12 @@ func ExecuteMessageVector(r Reporter, vector *schema.TestVector) { // Create a new Driver. driver := NewDriver(ctx, vector.Selector, DriverOpts{}) + var circSupply *abi.TokenAmount + if cs := vector.Pre.CircSupply; cs != nil { + ta := abi.NewTokenAmount(*cs) + circSupply = &ta + } + // Apply every message. for i, m := range vector.ApplyMessages { msg, err := types.DecodeMessage(m.Bytes) @@ -59,7 +65,7 @@ func ExecuteMessageVector(r Reporter, vector *schema.TestVector) { // Execute the message. var ret *vm.ApplyRet - ret, root, err = driver.ExecuteMessage(bs, root, abi.ChainEpoch(epoch), msg) + ret, root, err = driver.ExecuteMessage(bs, root, abi.ChainEpoch(epoch), msg, circSupply) if err != nil { r.Fatalf("fatal failure when executing message: %s", err) } diff --git a/go.mod b/go.mod index b0de7dfd6..bf3748749 100644 --- a/go.mod +++ b/go.mod @@ -38,7 +38,7 @@ require ( github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b github.com/filecoin-project/specs-actors v0.9.11 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 - github.com/filecoin-project/test-vectors/schema v0.0.1 + github.com/filecoin-project/test-vectors/schema v0.0.2 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 github.com/go-kit/kit v0.10.0 github.com/go-ole/go-ole v1.2.4 // indirect diff --git a/go.sum b/go.sum index bdb80aa83..6a310ed3d 100644 --- a/go.sum +++ b/go.sum @@ -258,8 +258,8 @@ github.com/filecoin-project/specs-actors v0.9.11 h1:TnpG7HAeiUrfj0mJM7UaPW0P2137 github.com/filecoin-project/specs-actors v0.9.11/go.mod h1:czlvLQGEX0fjLLfdNHD7xLymy6L3n7aQzRWzsYGf+ys= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= -github.com/filecoin-project/test-vectors/schema v0.0.1 h1:5fNF76nl4qolEvcIsjc0kUADlTMVHO73tW4kXXPnsus= -github.com/filecoin-project/test-vectors/schema v0.0.1/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E= +github.com/filecoin-project/test-vectors/schema v0.0.2 h1:/Pp//88WBXe0h+ksntdL2HpEgAmbwXrftAfeVG39zdY= +github.com/filecoin-project/test-vectors/schema v0.0.2/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6 h1:u/UEqS66A5ckRmS4yNpjmVH56sVtS/RfclBAYocb4as= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ= From 9b403e26e5d6c279383469cc86e1ac554b50ec75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Mon, 28 Sep 2020 23:07:45 +0100 Subject: [PATCH 399/885] fix lint. --- cmd/tvx/extract_many.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/tvx/extract_many.go b/cmd/tvx/extract_many.go index 8fea8df4c..83ec72b21 100644 --- a/cmd/tvx/extract_many.go +++ b/cmd/tvx/extract_many.go @@ -26,7 +26,7 @@ var extractManyFlags struct { var extractManyCmd = &cli.Command{ Name: "extract-many", - Description: `generate many test vectors by repeateadly calling tvx extract, using a csv file as input. + Description: `generate many test vectors by repeatedly calling tvx extract, using a csv file as input. The CSV file must have a format just like the following: From 044674487e56c6355354191ab546a5410aee789e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Mon, 28 Sep 2020 23:14:20 +0100 Subject: [PATCH 400/885] fix double mutex. --- cmd/tvx/stores.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/cmd/tvx/stores.go b/cmd/tvx/stores.go index c389f8c88..6e50e0839 100644 --- a/cmd/tvx/stores.go +++ b/cmd/tvx/stores.go @@ -42,14 +42,11 @@ type Stores struct { // ChainReadObj RPC. func NewProxyingStores(ctx context.Context, api api.FullNode) *Stores { ds := dssync.MutexWrap(ds.NewMapDatastore()) - ds = dssync.MutexWrap(ds) - bs := &proxyingBlockstore{ ctx: ctx, api: api, Blockstore: blockstore.NewBlockstore(ds), } - return NewStores(ctx, ds, bs) } From d1000e38d715890c8d7853d4c604844f8957c221 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 28 Sep 2020 15:50:54 -0700 Subject: [PATCH 401/885] wire up network upgrade logic for v2 actors upgrade --- build/params_2k.go | 1 + build/params_testground.go | 1 + build/params_testnet.go | 2 ++ chain/stmgr/forks.go | 40 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 44 insertions(+) diff --git a/build/params_2k.go b/build/params_2k.go index 3682f7be1..4428da748 100644 --- a/build/params_2k.go +++ b/build/params_2k.go @@ -14,6 +14,7 @@ const BreezeGasTampingDuration = 0 const UpgradeSmokeHeight = -1 const UpgradeIgnitionHeight = -2 const UpgradeLiftoffHeight = -3 +const UpgradeActorsV2 = 10 var DrandSchedule = map[abi.ChainEpoch]DrandEnum{ 0: DrandMainnet, diff --git a/build/params_testground.go b/build/params_testground.go index 07cc88688..718e21d68 100644 --- a/build/params_testground.go +++ b/build/params_testground.go @@ -77,6 +77,7 @@ var ( UpgradeSmokeHeight abi.ChainEpoch = -1 UpgradeIgnitionHeight abi.ChainEpoch = -2 UpgradeLiftoffHeight abi.ChainEpoch = -3 + UpgradeActorsV2 abi.ChainEpoch = 10 DrandSchedule = map[abi.ChainEpoch]DrandEnum{ 0: DrandMainnet, diff --git a/build/params_testnet.go b/build/params_testnet.go index 960f3a9b6..66f30f869 100644 --- a/build/params_testnet.go +++ b/build/params_testnet.go @@ -23,6 +23,8 @@ const UpgradeSmokeHeight = 51000 const UpgradeIgnitionHeight = 94000 +const UpgradeActorsV2 = 128888 + // This signals our tentative epoch for mainnet launch. Can make it later, but not earlier. // Miners, clients, developers, custodians all need time to prepare. // We still have upgrades and state changes to do, but can happen after signaling timing here. diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index 9561e5ed9..c49d75370 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -21,8 +21,10 @@ import ( adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/specs-actors/actors/migration/nv3" + m2 "github.com/filecoin-project/specs-actors/v2/actors/migration" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/builtin/multisig" "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/types" @@ -32,6 +34,7 @@ import ( var ForksAtHeight = map[abi.ChainEpoch]func(context.Context, *StateManager, ExecCallback, cid.Cid, *types.TipSet) (cid.Cid, error){ build.UpgradeBreezeHeight: UpgradeFaucetBurnRecovery, build.UpgradeIgnitionHeight: UpgradeIgnition, + build.UpgradeActorsV2: UpgradeActorsV2, build.UpgradeLiftoffHeight: UpgradeLiftoff, } @@ -414,6 +417,43 @@ func UpgradeIgnition(ctx context.Context, sm *StateManager, cb ExecCallback, roo return tree.Flush(ctx) } +func UpgradeActorsV2(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, ts *types.TipSet) (cid.Cid, error) { + store := sm.cs.Store(ctx) + + info, err := store.Put(ctx, new(types.StateInfo)) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to create new state info for actors v2: %w", err) + } + + newHamtRoot, err := m2.MigrateStateTree(ctx, store, root) + if err != nil { + return cid.Undef, xerrors.Errorf("upgrading to actors v2: %w", err) + } + + newRoot, err := store.Put(ctx, &types.StateRoot{ + // TODO: ActorUpgrade: should be state-tree specific, not just the actors version. + Version: actors.Version2, + Actors: newHamtRoot, + Info: info, + }) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to persist new state root: %w", err) + } + + // perform some basic sanity checks. + if newSm, err := state.LoadStateTree(store, newRoot); err != nil { + return cid.Undef, xerrors.Errorf("state tree sanity load failed: %w", err) + } else if newRoot2, err := newSm.Flush(ctx); err != nil { + return cid.Undef, xerrors.Errorf("state tree sanity flush failed: %w", err) + } else if newRoot2 != newRoot { + return cid.Undef, xerrors.Errorf("state-root mismatch: %s != %s", newRoot, newRoot2) + } else if _, err := newSm.GetActor(builtin0.InitActorAddr); err != nil { + return cid.Undef, xerrors.Errorf("failed to load init actor after upgrade: %w", err) + } + + return newRoot, nil +} + func UpgradeLiftoff(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, ts *types.TipSet) (cid.Cid, error) { tree, err := sm.StateTree(root) if err != nil { From 3e6323a503e7a05ede45c3cf2a42cccc729bad0e Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 28 Sep 2020 15:54:48 -0700 Subject: [PATCH 402/885] make setNetworkName work with any actors version --- chain/actors/builtin/init/init.go | 3 +++ chain/actors/builtin/init/v0.go | 5 +++++ chain/actors/builtin/init/v2.go | 5 +++++ chain/stmgr/forks.go | 13 +++++++------ 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/chain/actors/builtin/init/init.go b/chain/actors/builtin/init/init.go index d1a41b158..5777bb890 100644 --- a/chain/actors/builtin/init/init.go +++ b/chain/actors/builtin/init/init.go @@ -51,4 +51,7 @@ type State interface { // It should not be used in production code, as init actor entries are // immutable. Remove(addrs ...address.Address) error + + // Sets the network's name. This should only be used on upgrade/fork. + SetNetworkName(name string) error } diff --git a/chain/actors/builtin/init/v0.go b/chain/actors/builtin/init/v0.go index de0d6c9e1..ceb87f970 100644 --- a/chain/actors/builtin/init/v0.go +++ b/chain/actors/builtin/init/v0.go @@ -57,6 +57,11 @@ func (s *state0) NetworkName() (dtypes.NetworkName, error) { return dtypes.NetworkName(s.State.NetworkName), nil } +func (s *state0) SetNetworkName(name string) error { + s.State.NetworkName = name + return nil +} + func (s *state0) Remove(addrs ...address.Address) (err error) { m, err := adt0.AsMap(s.store, s.State.AddressMap) if err != nil { diff --git a/chain/actors/builtin/init/v2.go b/chain/actors/builtin/init/v2.go index 38b3099fd..5aa0ddc18 100644 --- a/chain/actors/builtin/init/v2.go +++ b/chain/actors/builtin/init/v2.go @@ -57,6 +57,11 @@ func (s *state2) NetworkName() (dtypes.NetworkName, error) { return dtypes.NetworkName(s.State.NetworkName), nil } +func (s *state2) SetNetworkName(name string) error { + s.State.NetworkName = name + return nil +} + func (s *state2) Remove(addrs ...address.Address) (err error) { m, err := adt2.AsMap(s.store, s.State.AddressMap) if err != nil { diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index c49d75370..813557c04 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -14,7 +14,6 @@ import ( "golang.org/x/xerrors" builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" - init0 "github.com/filecoin-project/specs-actors/actors/builtin/init" miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" multisig0 "github.com/filecoin-project/specs-actors/actors/builtin/multisig" power0 "github.com/filecoin-project/specs-actors/actors/builtin/power" @@ -25,6 +24,8 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/actors/adt" + init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init" "github.com/filecoin-project/lotus/chain/actors/builtin/multisig" "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/types" @@ -468,20 +469,20 @@ func UpgradeLiftoff(ctx context.Context, sm *StateManager, cb ExecCallback, root return tree.Flush(ctx) } -func setNetworkName(ctx context.Context, store adt0.Store, tree *state.StateTree, name string) error { +func setNetworkName(ctx context.Context, store adt.Store, tree *state.StateTree, name string) error { ia, err := tree.GetActor(builtin0.InitActorAddr) if err != nil { return xerrors.Errorf("getting init actor: %w", err) } - var initState init0.State - if err := store.Get(ctx, ia.Head, &initState); err != nil { + initState, err := init_.Load(store, ia) + if err != nil { return xerrors.Errorf("reading init state: %w", err) } - initState.NetworkName = name + initState.SetNetworkName(name) - ia.Head, err = store.Put(ctx, &initState) + ia.Head, err = store.Put(ctx, initState) if err != nil { return xerrors.Errorf("writing new init state: %w", err) } From c0b316fd9ca0b0fc6cfe9051cacb7ff83d817c55 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 28 Sep 2020 15:58:07 -0700 Subject: [PATCH 403/885] finish wiring up actors v2 --- build/params_shared_funcs.go | 2 +- build/params_shared_vals.go | 2 +- chain/stmgr/stmgr.go | 4 ++++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/build/params_shared_funcs.go b/build/params_shared_funcs.go index 28567d3d1..95daa45e7 100644 --- a/build/params_shared_funcs.go +++ b/build/params_shared_funcs.go @@ -41,7 +41,7 @@ func DhtProtocolName(netName dtypes.NetworkName) protocol.ID { func UseNewestNetwork() bool { // TODO: Put these in a container we can iterate over - if UpgradeBreezeHeight <= 0 && UpgradeSmokeHeight <= 0 { + if UpgradeBreezeHeight <= 0 && UpgradeSmokeHeight <= 0 && UpgradeActorsV2 <= 0 { return true } return false diff --git a/build/params_shared_vals.go b/build/params_shared_vals.go index 33828f954..9655711fb 100644 --- a/build/params_shared_vals.go +++ b/build/params_shared_vals.go @@ -25,7 +25,7 @@ const UnixfsLinksPerLevel = 1024 // Consensus / Network const AllowableClockDriftSecs = uint64(1) -const NewestNetworkVersion = network.Version3 +const NewestNetworkVersion = network.Version4 const ActorUpgradeNetworkVersion = network.Version4 // Epochs diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index 409b2de29..aa0439467 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -1275,6 +1275,10 @@ func (sm *StateManager) GetNtwkVersion(ctx context.Context, height abi.ChainEpoc return network.Version2 } + if height <= build.UpgradeActorsV2 { + return network.Version3 + } + return build.NewestNetworkVersion } From a314cfd2372e1a80e361a53910664c708ea303f4 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 28 Sep 2020 15:59:11 -0700 Subject: [PATCH 404/885] process error when setting network name --- chain/stmgr/forks.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index 813557c04..772a1a9a3 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -480,7 +480,9 @@ func setNetworkName(ctx context.Context, store adt.Store, tree *state.StateTree, return xerrors.Errorf("reading init state: %w", err) } - initState.SetNetworkName(name) + if err := initState.SetNetworkName(name); err != nil { + return xerrors.Errorf("setting network name: %w", err) + } ia.Head, err = store.Put(ctx, initState) if err != nil { From b95b871943e39784a93d0c3d815f1035c3b5cd5d Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 28 Sep 2020 16:14:51 -0700 Subject: [PATCH 405/885] actually upgrade to actors v2 --- chain/actors/version.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/chain/actors/version.go b/chain/actors/version.go index 2173a320d..385ff592f 100644 --- a/chain/actors/version.go +++ b/chain/actors/version.go @@ -18,6 +18,8 @@ func VersionForNetwork(version network.Version) Version { switch version { case network.Version0, network.Version1, network.Version2, network.Version3: return Version0 + case network.Version4: + return Version2 default: panic(fmt.Sprintf("unsupported network version %d", version)) } From 01272f850d5fefc27925ce4e6d2c252b9dec60d8 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 28 Sep 2020 16:23:46 -0700 Subject: [PATCH 406/885] fix state test --- chain/state/statetree_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/chain/state/statetree_test.go b/chain/state/statetree_test.go index 554ba2697..3b08a4b53 100644 --- a/chain/state/statetree_test.go +++ b/chain/state/statetree_test.go @@ -9,6 +9,7 @@ import ( cbor "github.com/ipfs/go-ipld-cbor" address "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/lotus/build" @@ -237,7 +238,8 @@ func assertNotHas(t *testing.T, st *StateTree, addr address.Address) { func TestStateTreeConsistency(t *testing.T) { cst := cbor.NewMemCborStore() - st, err := NewStateTree(cst, actors.VersionForNetwork(build.NewestNetworkVersion)) + // TODO: ActorUpgrade: this test tests pre actors v2 + st, err := NewStateTree(cst, actors.VersionForNetwork(network.Version3)) if err != nil { t.Fatal(err) } From 9344bc4e9bd94c4505ce593f9cea0b5e2d707c91 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 28 Sep 2020 16:27:47 -0700 Subject: [PATCH 407/885] update api docs for new network version --- documentation/en/api-methods.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index ed082ccbf..c39840382 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -3825,7 +3825,7 @@ Inputs: ] ``` -Response: `3` +Response: `4` ### StateReadState StateReadState returns the indicated actor's state. From 2315db161bed787776136c4f9ee30fef12ae80d5 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 28 Sep 2020 16:46:29 -0700 Subject: [PATCH 408/885] cleanup state-tree loading And remove incorrect comment. --- chain/state/statetree.go | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/chain/state/statetree.go b/chain/state/statetree.go index fcb7ffbaa..3f9597420 100644 --- a/chain/state/statetree.go +++ b/chain/state/statetree.go @@ -157,27 +157,25 @@ func LoadStateTree(cst cbor.IpldStore, c cid.Cid) (*StateTree, error) { root.Version = actors.Version0 } - // If that fails, load as an old-style state-tree (direct hampt, version 0. - nd, err := adt.AsMap(adt.WrapStore(context.TODO(), cst), root.Actors, actors.Version(root.Version)) - if err != nil { - log.Errorf("loading hamt node %s failed: %s", c, err) - return nil, err - } - switch root.Version { case actors.Version0, actors.Version2: - // supported + // Load the actual state-tree HAMT. + nd, err := adt.AsMap(adt.WrapStore(context.TODO(), cst), root.Actors, actors.Version(root.Version)) + if err != nil { + log.Errorf("loading hamt node %s failed: %s", c, err) + return nil, err + } + + return &StateTree{ + root: nd, + info: root.Info, + version: actors.Version(root.Version), + Store: cst, + snaps: newStateSnaps(), + }, nil default: return nil, xerrors.Errorf("unsupported state tree version: %d", root.Version) } - - return &StateTree{ - root: nd, - info: root.Info, - version: actors.Version(root.Version), - Store: cst, - snaps: newStateSnaps(), - }, nil } func (st *StateTree) SetActor(addr address.Address, act *types.Actor) error { From 1c0386285476cb6e51a451c5114f640688cd4c91 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 28 Sep 2020 17:28:16 -0700 Subject: [PATCH 409/885] fix policy for specs-actors update --- api/test/paych.go | 2 +- build/params_shared_vals.go | 13 ++++-------- build/params_testground.go | 9 ++++---- chain/actors/policy/policy.go | 27 ++++++++++++++++++++++++ chain/actors/policy/policy_test.go | 11 ++++++++++ extern/storage-sealing/checks.go | 16 ++++---------- extern/storage-sealing/constants.go | 7 ------ extern/storage-sealing/states_sealing.go | 13 +++--------- 8 files changed, 54 insertions(+), 44 deletions(-) diff --git a/api/test/paych.go b/api/test/paych.go index 4c3bdba1b..c13303593 100644 --- a/api/test/paych.go +++ b/api/test/paych.go @@ -169,7 +169,7 @@ func TestPaymentChannels(t *testing.T, b APIBuilder, blocktime time.Duration) { return true, nil }, func(ctx context.Context, ts *types.TipSet) error { return nil - }, int(build.MessageConfidence)+1, build.SealRandomnessLookbackLimit, func(oldTs, newTs *types.TipSet) (bool, events.StateChange, error) { + }, int(build.MessageConfidence)+1, build.Finality, func(oldTs, newTs *types.TipSet) (bool, events.StateChange, error) { return preds.OnPaymentChannelActorChanged(channel, preds.OnToSendAmountChanges())(ctx, oldTs.Key(), newTs.Key()) }) if err != nil { diff --git a/build/params_shared_vals.go b/build/params_shared_vals.go index 9655711fb..4734a15ed 100644 --- a/build/params_shared_vals.go +++ b/build/params_shared_vals.go @@ -7,12 +7,12 @@ import ( "os" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/specs-actors/actors/builtin" - miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" ) // ///// @@ -35,7 +35,7 @@ const ForkLengthThreshold = Finality var BlocksPerEpoch = uint64(builtin.ExpectedLeadersPerEpoch) // Epochs -const Finality = miner0.ChainFinality +const Finality = policy.ChainFinality const MessageConfidence = uint64(5) // constants for Weight calculation @@ -47,13 +47,8 @@ const WRatioDen = uint64(2) // Proofs // Epochs -const SealRandomnessLookback = Finality - -// Epochs -const SealRandomnessLookbackLimit = SealRandomnessLookback + 2000 // TODO: Get from spec specs-actors - -// Maximum lookback that randomness can be sourced from for a seal proof submission -const MaxSealLookback = SealRandomnessLookbackLimit + 2000 // TODO: Get from specs-actors +// TODO: unused +const SealRandomnessLookback = policy.SealRandomnessLookback // ///// // Mining diff --git a/build/params_testground.go b/build/params_testground.go index 718e21d68..975c26599 100644 --- a/build/params_testground.go +++ b/build/params_testground.go @@ -13,7 +13,8 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/specs-actors/actors/builtin" - miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" + + "github.com/filecoin-project/lotus/chain/actors/policy" ) var ( @@ -32,7 +33,7 @@ var ( AllowableClockDriftSecs = uint64(1) - Finality = miner0.ChainFinality + Finality = policy.ChainFinality ForkLengthThreshold = Finality SlashablePowerDelay = 20 @@ -47,9 +48,7 @@ var ( BlsSignatureCacheSize = 40000 VerifSigCacheSize = 32000 - SealRandomnessLookback = Finality - SealRandomnessLookbackLimit = SealRandomnessLookback + 2000 - MaxSealLookback = SealRandomnessLookbackLimit + 2000 + SealRandomnessLookback = policy.SealRandomnessLookback TicketRandomnessLookback = abi.ChainEpoch(1) WinningPoStSectorSetLookback = abi.ChainEpoch(10) diff --git a/chain/actors/policy/policy.go b/chain/actors/policy/policy.go index b8205177e..2101cfdcf 100644 --- a/chain/actors/policy/policy.go +++ b/chain/actors/policy/policy.go @@ -2,10 +2,19 @@ package policy import ( "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/actors" miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" power0 "github.com/filecoin-project/specs-actors/actors/builtin/power" verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" + verifreg2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/verifreg" +) + +const ( + ChainFinality = miner0.ChainFinality + SealRandomnessLookback = ChainFinality ) // SetSupportedProofTypes sets supported proof types, across all actor versions. @@ -17,6 +26,7 @@ func SetSupportedProofTypes(types ...abi.RegisteredSealProof) { } // Set for all miner versions. miner0.SupportedProofTypes = newTypes + miner2.SupportedProofTypes = newTypes } // AddSupportedProofTypes sets supported proof types, across all actor versions. @@ -25,6 +35,7 @@ func AddSupportedProofTypes(types ...abi.RegisteredSealProof) { for _, t := range types { // Set for all miner versions. miner0.SupportedProofTypes[t] = struct{}{} + miner2.SupportedProofTypes[t] = struct{}{} } } @@ -33,6 +44,7 @@ func AddSupportedProofTypes(types ...abi.RegisteredSealProof) { func SetPreCommitChallengeDelay(delay abi.ChainEpoch) { // Set for all miner versions. miner0.PreCommitChallengeDelay = delay + miner2.PreCommitChallengeDelay = delay } // TODO: this function shouldn't really exist. Instead, the API should expose the precommit delay. @@ -45,10 +57,25 @@ func GetPreCommitChallengeDelay() abi.ChainEpoch { // for testing. func SetConsensusMinerMinPower(p abi.StoragePower) { power0.ConsensusMinerMinPower = p + for _, policy := range builtin2.SealProofPolicies { + policy.ConsensusMinerMinPower = p + } } // SetMinVerifiedDealSize sets the minimum size of a verified deal. This should // only be used for testing. func SetMinVerifiedDealSize(size abi.StoragePower) { verifreg0.MinVerifiedDealSize = size + verifreg2.MinVerifiedDealSize = size +} + +func GetMaxProveCommitDuration(ver actors.Version, t abi.RegisteredSealProof) abi.ChainEpoch { + switch ver { + case actors.Version0: + return miner0.MaxSealDuration[t] + case actors.Version2: + return miner2.MaxProveCommitDuration[t] + default: + panic("unsupported actors version") + } } diff --git a/chain/actors/policy/policy_test.go b/chain/actors/policy/policy_test.go index be64362a2..9ba5b4030 100644 --- a/chain/actors/policy/policy_test.go +++ b/chain/actors/policy/policy_test.go @@ -7,6 +7,9 @@ import ( "github.com/filecoin-project/go-state-types/abi" miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" + verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" + miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" + verifreg2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/verifreg" ) func TestSupportedProofTypes(t *testing.T) { @@ -34,3 +37,11 @@ func TestSupportedProofTypes(t *testing.T) { }, ) } + +// Tests assumptions about policies being the same between actor versions. +func TestAssumptions(t *testing.T) { + require.EqualValues(t, miner0.SupportedProofTypes, miner2.SupportedProofTypes) + require.Equal(t, miner0.PreCommitChallengeDelay, miner2.PreCommitChallengeDelay) + require.Equal(t, miner0.ChainFinality, miner2.ChainFinality) + require.Equal(t, verifreg0.MinVerifiedDealSize, verifreg2.MinVerifiedDealSize) +} diff --git a/extern/storage-sealing/checks.go b/extern/storage-sealing/checks.go index 49994024f..1010d31b2 100644 --- a/extern/storage-sealing/checks.go +++ b/extern/storage-sealing/checks.go @@ -4,11 +4,9 @@ import ( "bytes" "context" + "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/policy" - "github.com/filecoin-project/lotus/build" - miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" - proof0 "github.com/filecoin-project/specs-actors/actors/runtime/proof" "golang.org/x/xerrors" @@ -102,16 +100,10 @@ func checkPrecommit(ctx context.Context, maddr address.Address, si SectorInfo, t return &ErrApi{xerrors.Errorf("calling StateNetworkVersion: %w", err)} } - var msd abi.ChainEpoch - if nv < build.ActorUpgradeNetworkVersion { - msd = miner0.MaxSealDuration[si.SectorType] - } else { - // TODO: ActorUpgrade(use MaxProveCommitDuration) - msd = 0 - } + msd := policy.GetMaxProveCommitDuration(actors.VersionForNetwork(nv), si.SectorType) - if height-(si.TicketEpoch+SealRandomnessLookback) > msd { - return &ErrExpiredTicket{xerrors.Errorf("ticket expired: seal height: %d, head: %d", si.TicketEpoch+SealRandomnessLookback, height)} + if height-(si.TicketEpoch+policy.SealRandomnessLookback) > msd { + return &ErrExpiredTicket{xerrors.Errorf("ticket expired: seal height: %d, head: %d", si.TicketEpoch+policy.SealRandomnessLookback, height)} } pci, err := api.StateSectorPreCommitInfo(ctx, maddr, si.SectorNumber, tok) diff --git a/extern/storage-sealing/constants.go b/extern/storage-sealing/constants.go index 8c7fa5abc..d6aba1814 100644 --- a/extern/storage-sealing/constants.go +++ b/extern/storage-sealing/constants.go @@ -1,11 +1,4 @@ package sealing -import ( - miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" -) - -// Epochs -const SealRandomnessLookback = miner0.ChainFinality - // Epochs const InteractivePoRepConfidence = 6 diff --git a/extern/storage-sealing/states_sealing.go b/extern/storage-sealing/states_sealing.go index a4e852454..96589bcd2 100644 --- a/extern/storage-sealing/states_sealing.go +++ b/extern/storage-sealing/states_sealing.go @@ -4,10 +4,9 @@ import ( "bytes" "context" - "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/actors/policy" - miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" "golang.org/x/xerrors" @@ -60,7 +59,7 @@ func (m *Sealing) getTicket(ctx statemachine.Context, sector SectorInfo) (abi.Se return nil, 0, nil } - ticketEpoch := epoch - SealRandomnessLookback + ticketEpoch := epoch - policy.SealRandomnessLookback buf := new(bytes.Buffer) if err := m.maddr.MarshalCBOR(buf); err != nil { return nil, 0, err @@ -189,13 +188,7 @@ func (m *Sealing) handlePreCommitting(ctx statemachine.Context, sector SectorInf return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("failed to get network version: %w", err)}) } - var msd abi.ChainEpoch - if nv < build.ActorUpgradeNetworkVersion { - msd = miner0.MaxSealDuration[sector.SectorType] - } else { - // TODO: ActorUpgrade(use MaxProveCommitDuration) - msd = 0 - } + msd := policy.GetMaxProveCommitDuration(actors.VersionForNetwork(nv), sector.SectorType) if minExpiration := height + msd + miner.MinSectorExpiration + 10; expiration < minExpiration { expiration = minExpiration From c0e190344d6e9df2a13d071ed0a571b50dd0e0cb Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 28 Sep 2020 17:42:29 -0700 Subject: [PATCH 410/885] check state invariants after upgrade --- chain/stmgr/forks.go | 16 +++++++++++++++- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index 772a1a9a3..96873b675 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -21,6 +21,7 @@ import ( "github.com/filecoin-project/specs-actors/actors/migration/nv3" m2 "github.com/filecoin-project/specs-actors/v2/actors/migration" + states2 "github.com/filecoin-project/specs-actors/v2/actors/states" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" @@ -431,6 +432,19 @@ func UpgradeActorsV2(ctx context.Context, sm *StateManager, cb ExecCallback, roo return cid.Undef, xerrors.Errorf("upgrading to actors v2: %w", err) } + newStateTree, err := states2.LoadTree(store, newHamtRoot) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to load new state tree: %w", err) + } + + // Check all state-tree invariants. + if msgs, err := states2.CheckStateInvariants(newStateTree, types.TotalFilecoinInt); err != nil { + return cid.Undef, xerrors.Errorf("failed to check new state tree: %w", err) + } else if !msgs.IsEmpty() { + // This error is going to be really nasty. + return cid.Undef, xerrors.Errorf("network upgrade failed: %v", msgs.Messages()) + } + newRoot, err := store.Put(ctx, &types.StateRoot{ // TODO: ActorUpgrade: should be state-tree specific, not just the actors version. Version: actors.Version2, @@ -441,7 +455,7 @@ func UpgradeActorsV2(ctx context.Context, sm *StateManager, cb ExecCallback, roo return cid.Undef, xerrors.Errorf("failed to persist new state root: %w", err) } - // perform some basic sanity checks. + // perform some basic sanity checks to make sure everything still works. if newSm, err := state.LoadStateTree(store, newRoot); err != nil { return cid.Undef, xerrors.Errorf("state tree sanity load failed: %w", err) } else if newRoot2, err := newSm.Flush(ctx); err != nil { diff --git a/go.mod b/go.mod index 4908523fc..55f2fa308 100644 --- a/go.mod +++ b/go.mod @@ -37,7 +37,7 @@ require ( github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b github.com/filecoin-project/specs-actors v0.9.12-0.20200928180918-488a087c5add - github.com/filecoin-project/specs-actors/v2 v2.0.0-20200928175842-971c8d772684 + github.com/filecoin-project/specs-actors/v2 v2.0.0-20200929004132-ea2d270c9f47 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 github.com/filecoin-project/test-vectors/schema v0.0.1 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 diff --git a/go.sum b/go.sum index 042b0d855..3413d730a 100644 --- a/go.sum +++ b/go.sum @@ -263,8 +263,8 @@ github.com/filecoin-project/specs-actors v0.9.7/go.mod h1:wM2z+kwqYgXn5Z7scV1YHL github.com/filecoin-project/specs-actors v0.9.9/go.mod h1:czlvLQGEX0fjLLfdNHD7xLymy6L3n7aQzRWzsYGf+ys= github.com/filecoin-project/specs-actors v0.9.12-0.20200928180918-488a087c5add h1:iXQOxr8uSyZ/qnTlOZf7ALp0io+jwLxmuWsNAk/YdoQ= github.com/filecoin-project/specs-actors v0.9.12-0.20200928180918-488a087c5add/go.mod h1:TS1AW/7LbG+615j4NsjMK1qlpAwaFsG9w0V2tg2gSao= -github.com/filecoin-project/specs-actors/v2 v2.0.0-20200928175842-971c8d772684 h1:sjWZqblOOf1RaohI9w2R2AVp5uifNdzsusy7oVi5ioU= -github.com/filecoin-project/specs-actors/v2 v2.0.0-20200928175842-971c8d772684/go.mod h1:/2Zra1BhLtpRywUhm++QP+3I5Ir+hBk/W24TpYjj43E= +github.com/filecoin-project/specs-actors/v2 v2.0.0-20200929004132-ea2d270c9f47 h1:bt/CdGbF1Rq5rDVaMV+XWfHBRCxN7IxZfw6lC1v6KQM= +github.com/filecoin-project/specs-actors/v2 v2.0.0-20200929004132-ea2d270c9f47/go.mod h1:/2Zra1BhLtpRywUhm++QP+3I5Ir+hBk/W24TpYjj43E= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= github.com/filecoin-project/test-vectors/schema v0.0.1 h1:5fNF76nl4qolEvcIsjc0kUADlTMVHO73tW4kXXPnsus= From e658ab32872e7cf026287de127c0ee192d2aefa3 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 28 Sep 2020 17:44:34 -0700 Subject: [PATCH 411/885] fix policy test --- chain/actors/policy/policy_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chain/actors/policy/policy_test.go b/chain/actors/policy/policy_test.go index 9ba5b4030..8d6aac09f 100644 --- a/chain/actors/policy/policy_test.go +++ b/chain/actors/policy/policy_test.go @@ -43,5 +43,5 @@ func TestAssumptions(t *testing.T) { require.EqualValues(t, miner0.SupportedProofTypes, miner2.SupportedProofTypes) require.Equal(t, miner0.PreCommitChallengeDelay, miner2.PreCommitChallengeDelay) require.Equal(t, miner0.ChainFinality, miner2.ChainFinality) - require.Equal(t, verifreg0.MinVerifiedDealSize, verifreg2.MinVerifiedDealSize) + require.True(t, verifreg0.MinVerifiedDealSize.Equals(verifreg2.MinVerifiedDealSize)) } From dd65e57d4b6335d3882fda8ea8fb042e5813a63b Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 28 Sep 2020 17:53:11 -0700 Subject: [PATCH 412/885] set policy correctly from test vectors --- chain/vectors/gen/main.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/chain/vectors/gen/main.go b/chain/vectors/gen/main.go index 2b1c6f340..51d907648 100644 --- a/chain/vectors/gen/main.go +++ b/chain/vectors/gen/main.go @@ -6,28 +6,25 @@ import ( "math/rand" "os" - power0 "github.com/filecoin-project/specs-actors/actors/builtin/power" - "github.com/filecoin-project/go-address" "golang.org/x/xerrors" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" + "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/types/mock" "github.com/filecoin-project/lotus/chain/vectors" "github.com/filecoin-project/lotus/chain/wallet" - verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" _ "github.com/filecoin-project/lotus/lib/sigs/bls" _ "github.com/filecoin-project/lotus/lib/sigs/secp" ) func init() { - verifreg0.MinVerifiedDealSize = big.NewInt(2048) - power0.ConsensusMinerMinPower = big.NewInt(2048) + policy.SetMinVerifiedDealSize(abi.NewStoragePower(2048)) + policy.SetConsensusMinerMinPower(abi.NewStoragePower(2048)) } func MakeHeaderVectors() []vectors.HeaderVector { From be9d23b32981b660d3b5fdc392b4e5d2cdd70458 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Tue, 29 Sep 2020 00:24:38 -0400 Subject: [PATCH 413/885] Centralize some params in builtin --- chain/actors/builtin/builtin.go | 20 ++++++++++++++++++++ chain/stmgr/stmgr.go | 6 ++++-- chain/sync.go | 4 +++- chain/types/actor.go | 18 ------------------ chain/vm/invoker.go | 4 +++- chain/vm/vm.go | 5 +++-- cli/multisig.go | 13 ++++++++----- cli/state.go | 3 ++- cmd/lotus-pcr/main.go | 10 ++++++---- cmd/lotus-shed/balances.go | 8 ++++---- cmd/lotus-shed/genesis-verify.go | 8 +++++--- cmd/lotus-shed/mempool-stats.go | 3 ++- node/impl/full/gas.go | 4 +++- 13 files changed, 63 insertions(+), 43 deletions(-) diff --git a/chain/actors/builtin/builtin.go b/chain/actors/builtin/builtin.go index 784c004eb..d49164486 100644 --- a/chain/actors/builtin/builtin.go +++ b/chain/actors/builtin/builtin.go @@ -18,6 +18,9 @@ import ( smoothing2 "github.com/filecoin-project/specs-actors/v2/actors/util/smoothing" ) +var SystemActorAddr = builtin0.SystemActorAddr +var BurntFundsActorAddr = builtin0.BurntFundsActorAddr + // TODO: Why does actors have 2 different versions of this? type SectorInfo = proof0.SectorInfo type PoStProof = proof0.PoStProof @@ -66,3 +69,20 @@ func ActorNameByCode(c cid.Cid) string { func IsBuiltinActor(c cid.Cid) bool { return builtin0.IsBuiltinActor(c) || builtin2.IsBuiltinActor(c) } + +func IsAccountActor(c cid.Cid) bool { + return c == builtin0.AccountActorCodeID || c == builtin2.AccountActorCodeID +} + +func IsStorageMinerActor(c cid.Cid) bool { + return c == builtin0.StorageMinerActorCodeID || c == builtin2.StorageMinerActorCodeID +} + +func IsMultisigActor(c cid.Cid) bool { + return c == builtin0.MultisigActorCodeID || c == builtin2.MultisigActorCodeID + +} + +func IsPaymentChannelActor(c cid.Cid) bool { + return c == builtin0.PaymentChannelActorCodeID || c == builtin2.PaymentChannelActorCodeID +} diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index aa0439467..c5968c4a4 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -5,6 +5,8 @@ import ( "fmt" "sync" + "github.com/filecoin-project/lotus/chain/actors/builtin" + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" msig0 "github.com/filecoin-project/specs-actors/actors/builtin/multisig" @@ -858,7 +860,7 @@ func (sm *StateManager) setupGenesisActors(ctx context.Context) error { totalsByEpoch := make(map[abi.ChainEpoch]abi.TokenAmount) err = sTree.ForEach(func(kaddr address.Address, act *types.Actor) error { - if act.IsMultisigActor() { + if builtin.IsMultisigActor(act.Code) { s, err := multisig.Load(sm.cs.Store(ctx), act) if err != nil { return err @@ -890,7 +892,7 @@ func (sm *StateManager) setupGenesisActors(ctx context.Context) error { totalsByEpoch[ud] = ib } - } else if act.IsAccountActor() { + } else if builtin.IsAccountActor(act.Code) { // should exclude burnt funds actor and "remainder account actor" // should only ever be "faucet" accounts in testnets if kaddr == builtin0.BurntFundsActorAddr { diff --git a/chain/sync.go b/chain/sync.go index 6abe43567..eba10e95e 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -11,6 +11,8 @@ import ( "sync" "time" + "github.com/filecoin-project/lotus/chain/actors/builtin" + "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/specs-actors/actors/runtime/proof" @@ -1080,7 +1082,7 @@ func (syncer *Syncer) checkBlockMessages(ctx context.Context, b *types.FullBlock return xerrors.Errorf("failed to get actor: %w", err) } - if !act.IsAccountActor() { + if !builtin.IsAccountActor(act.Code) { return xerrors.New("Sender must be an account actor") } nonces[m.From] = act.Nonce diff --git a/chain/types/actor.go b/chain/types/actor.go index eb8e05c49..a9974a01f 100644 --- a/chain/types/actor.go +++ b/chain/types/actor.go @@ -4,8 +4,6 @@ import ( "errors" "github.com/ipfs/go-cid" - - builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" ) var ErrActorNotFound = errors.New("actor not found") @@ -17,19 +15,3 @@ type Actor struct { Nonce uint64 Balance BigInt } - -func (a *Actor) IsAccountActor() bool { - return a.Code == builtin0.AccountActorCodeID -} - -func (a *Actor) IsStorageMinerActor() bool { - return a.Code == builtin0.StorageMinerActorCodeID -} - -func (a *Actor) IsMultisigActor() bool { - return a.Code == builtin0.MultisigActorCodeID -} - -func (a *Actor) IsPaymentChannelActor() bool { - return a.Code == builtin0.PaymentChannelActorCodeID -} diff --git a/chain/vm/invoker.go b/chain/vm/invoker.go index 65ef73cfc..1e9f04081 100644 --- a/chain/vm/invoker.go +++ b/chain/vm/invoker.go @@ -6,6 +6,8 @@ import ( "fmt" "reflect" + "github.com/filecoin-project/lotus/chain/actors/builtin" + "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" @@ -210,7 +212,7 @@ func DecodeParams(b []byte, out interface{}) error { } func DumpActorState(act *types.Actor, b []byte) (interface{}, error) { - if act.IsAccountActor() { // Account code special case + if builtin.IsAccountActor(act.Code) { // Account code special case return nil, nil } diff --git a/chain/vm/vm.go b/chain/vm/vm.go index d92d16310..1161178ac 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -8,6 +8,8 @@ import ( "sync/atomic" "time" + "github.com/filecoin-project/lotus/chain/actors/builtin" + block "github.com/ipfs/go-block-format" cid "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" @@ -23,7 +25,6 @@ import ( "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/go-state-types/exitcode" "github.com/filecoin-project/go-state-types/network" - "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors/adt" @@ -396,7 +397,7 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, } // this should never happen, but is currently still exercised by some tests - if !fromActor.IsAccountActor() { + if !builtin.IsAccountActor(fromActor.Code) { gasOutputs := ZeroGasOutputs() gasOutputs.MinerPenalty = minerPenaltyAmount return &ApplyRet{ diff --git a/cli/multisig.go b/cli/multisig.go index 5b0977382..6b3867cb3 100644 --- a/cli/multisig.go +++ b/cli/multisig.go @@ -9,6 +9,10 @@ import ( "strconv" "text/tabwriter" + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + + "github.com/filecoin-project/lotus/chain/actors/builtin" + "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/go-state-types/big" @@ -20,7 +24,6 @@ import ( "github.com/urfave/cli/v2" "golang.org/x/xerrors" - "github.com/filecoin-project/specs-actors/actors/builtin" init0 "github.com/filecoin-project/specs-actors/actors/builtin/init" msig0 "github.com/filecoin-project/specs-actors/actors/builtin/multisig" @@ -356,7 +359,7 @@ var msigProposeCmd = &cli.Command{ return fmt.Errorf("failed to look up multisig %s: %w", msig, err) } - if !act.IsMultisigActor() { + if !builtin.IsMultisigActor(act.Code) { return fmt.Errorf("actor %s is not a multisig actor", msig) } @@ -1029,7 +1032,7 @@ var msigLockProposeCmd = &cli.Command{ return actErr } - msgCid, err := api.MsigPropose(ctx, msig, msig, big.Zero(), from, uint64(builtin.MethodsMultisig.LockBalance), params) + msgCid, err := api.MsigPropose(ctx, msig, msig, big.Zero(), from, uint64(builtin2.MethodsMultisig.LockBalance), params) if err != nil { return err } @@ -1126,7 +1129,7 @@ var msigLockApproveCmd = &cli.Command{ return actErr } - msgCid, err := api.MsigApprove(ctx, msig, txid, prop, msig, big.Zero(), from, uint64(builtin.MethodsMultisig.LockBalance), params) + msgCid, err := api.MsigApprove(ctx, msig, txid, prop, msig, big.Zero(), from, uint64(builtin2.MethodsMultisig.LockBalance), params) if err != nil { return err } @@ -1218,7 +1221,7 @@ var msigLockCancelCmd = &cli.Command{ return actErr } - msgCid, err := api.MsigCancel(ctx, msig, txid, msig, big.Zero(), from, uint64(builtin.MethodsMultisig.LockBalance), params) + msgCid, err := api.MsigCancel(ctx, msig, txid, msig, big.Zero(), from, uint64(builtin2.MethodsMultisig.LockBalance), params) if err != nil { return err } diff --git a/cli/state.go b/cli/state.go index 94ffa63fe..cb1c974a7 100644 --- a/cli/state.go +++ b/cli/state.go @@ -14,6 +14,8 @@ import ( "strings" "time" + "github.com/filecoin-project/lotus/chain/actors/builtin" + "github.com/multiformats/go-multiaddr" "github.com/ipfs/go-cid" @@ -28,7 +30,6 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/exitcode" - "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/apibstore" diff --git a/cmd/lotus-pcr/main.go b/cmd/lotus-pcr/main.go index 36961a663..8acaa3c71 100644 --- a/cmd/lotus-pcr/main.go +++ b/cmd/lotus-pcr/main.go @@ -12,7 +12,9 @@ import ( "strconv" "time" - "github.com/filecoin-project/specs-actors/actors/builtin" + "github.com/filecoin-project/lotus/chain/actors/builtin" + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/go-state-types/network" @@ -375,14 +377,14 @@ func (r *refunder) ProcessTipset(ctx context.Context, tipset *types.TipSet, refu continue } - if !a.IsStorageMinerActor() { + if !builtin.IsStorageMinerActor(a.Code) { continue } var messageMethod string switch m.Method { - case builtin.MethodsMiner.ProveCommitSector: + case builtin0.MethodsMiner.ProveCommitSector: if !r.proveCommitEnabled { continue } @@ -429,7 +431,7 @@ func (r *refunder) ProcessTipset(ctx context.Context, tipset *types.TipSet, refu } refundValue = collateral - case builtin.MethodsMiner.PreCommitSector: + case builtin0.MethodsMiner.PreCommitSector: if !r.preCommitEnabled { continue } diff --git a/cmd/lotus-shed/balances.go b/cmd/lotus-shed/balances.go index 1c89a00cf..fba6d7724 100644 --- a/cmd/lotus-shed/balances.go +++ b/cmd/lotus-shed/balances.go @@ -6,7 +6,7 @@ import ( "strconv" "github.com/docker/go-units" - lotusbuiltin "github.com/filecoin-project/lotus/chain/actors/builtin" + "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/builtin/power" "github.com/filecoin-project/lotus/chain/actors/builtin/reward" @@ -97,7 +97,7 @@ var chainBalanceCmd = &cli.Command{ Type: string(act.Code.Hash()[2:]), } - if act.IsStorageMinerActor() { + if builtin.IsStorageMinerActor(act.Code) { pow, err := api.StateMinerPower(ctx, addr, tsk) if err != nil { return xerrors.Errorf("failed to get power: %w", err) @@ -198,7 +198,7 @@ var chainBalanceStateCmd = &cli.Command{ PreCommits: types.FIL(big.NewInt(0)), } - if minerInfo && act.IsStorageMinerActor() { + if minerInfo && builtin.IsStorageMinerActor(act.Code) { pow, _, _, err := stmgr.GetPowerRaw(ctx, sm, sroot, addr) if err != nil { return xerrors.Errorf("failed to get power: %w", err) @@ -322,7 +322,7 @@ var chainPledgeCmd = &cli.Command{ } var ( - powerSmoothed lotusbuiltin.FilterEstimate + powerSmoothed builtin.FilterEstimate pledgeCollateral abi.TokenAmount ) if act, err := state.GetActor(power.Address); err != nil { diff --git a/cmd/lotus-shed/genesis-verify.go b/cmd/lotus-shed/genesis-verify.go index da2c82359..9a47d6561 100644 --- a/cmd/lotus-shed/genesis-verify.go +++ b/cmd/lotus-shed/genesis-verify.go @@ -6,6 +6,8 @@ import ( "os" "sort" + "github.com/filecoin-project/lotus/chain/actors/builtin" + "github.com/fatih/color" "github.com/ipfs/go-datastore" cbor "github.com/ipfs/go-ipld-cbor" @@ -95,14 +97,14 @@ var genesisVerifyCmd = &cli.Command{ if err := stree.ForEach(func(addr address.Address, act *types.Actor) error { switch { - case act.IsStorageMinerActor(): + case builtin.IsStorageMinerActor(act.Code): _, err := miner.Load(store, act) if err != nil { return xerrors.Errorf("miner actor: %w", err) } // TODO: actually verify something here? kminers[addr] = minerInfo{} - case act.IsMultisigActor(): + case builtin.IsMultisigActor(act.Code): st, err := multisig.Load(store, act) if err != nil { return xerrors.Errorf("multisig actor: %w", err) @@ -123,7 +125,7 @@ var genesisVerifyCmd = &cli.Command{ Threshold: threshold, } msigAddrs = append(msigAddrs, addr) - case act.IsAccountActor(): + case builtin.IsAccountActor(act.Code): st, err := account.Load(store, act) if err != nil { // TODO: magik6k: this _used_ to log instead of failing, why? diff --git a/cmd/lotus-shed/mempool-stats.go b/cmd/lotus-shed/mempool-stats.go index 165c01432..d70cd4b71 100644 --- a/cmd/lotus-shed/mempool-stats.go +++ b/cmd/lotus-shed/mempool-stats.go @@ -18,6 +18,7 @@ import ( "github.com/filecoin-project/go-address" lapi "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" ) @@ -122,7 +123,7 @@ var mpoolStatsCmd = &cli.Command{ return false, err } - ism := act.IsStorageMinerActor() + ism := builtin.IsStorageMinerActor(act.Code) mcache[addr] = ism return ism, nil } diff --git a/node/impl/full/gas.go b/node/impl/full/gas.go index f03807c80..c912c7a8c 100644 --- a/node/impl/full/gas.go +++ b/node/impl/full/gas.go @@ -6,6 +6,8 @@ import ( "math/rand" "sort" + "github.com/filecoin-project/lotus/chain/actors/builtin" + "go.uber.org/fx" "golang.org/x/xerrors" @@ -182,7 +184,7 @@ func (a *GasAPI) GasEstimateGasLimit(ctx context.Context, msgIn *types.Message, return res.MsgRct.GasUsed, nil } - if !act.IsPaymentChannelActor() { + if !builtin.IsPaymentChannelActor(act.Code) { return res.MsgRct.GasUsed, nil } if msgIn.Method != builtin0.MethodsPaych.Collect { From eaece306b6c16efe642160f52d5e73a6a687d976 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 29 Sep 2020 11:17:23 +0200 Subject: [PATCH 414/885] wallet list cli: Print balances/nonces --- cli/wallet.go | 69 +++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 62 insertions(+), 7 deletions(-) diff --git a/cli/wallet.go b/cli/wallet.go index 27993a1ba..0d69673f9 100644 --- a/cli/wallet.go +++ b/cli/wallet.go @@ -9,13 +9,16 @@ import ( "os" "strings" - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-state-types/crypto" - types "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/chain/wallet" + "github.com/urfave/cli/v2" "golang.org/x/xerrors" - "github.com/urfave/cli/v2" + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/go-state-types/crypto" + + types "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/wallet" + "github.com/filecoin-project/lotus/lib/tablewriter" ) var walletCmd = &cli.Command{ @@ -66,6 +69,13 @@ var walletNew = &cli.Command{ var walletList = &cli.Command{ Name: "list", Usage: "List wallet address", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "addr-only", + Usage: "Only print addresses", + Aliases: []string{"a"}, + }, + }, Action: func(cctx *cli.Context) error { api, closer, err := GetFullNodeAPI(cctx) if err != nil { @@ -79,9 +89,54 @@ var walletList = &cli.Command{ return err } - for _, addr := range addrs { - fmt.Println(addr.String()) + def, err := api.WalletDefaultAddress(ctx) + if err != nil { + return err } + + tw := tablewriter.New( + tablewriter.Col("Address"), + tablewriter.Col("Balance"), + tablewriter.Col("Nonce"), + tablewriter.Col("Default"), + tablewriter.NewLineCol("Error")) + + for _, addr := range addrs { + if cctx.Bool("addr-only") { + fmt.Println(addr.String()) + } else { + a, err := api.StateGetActor(ctx, addr, types.EmptyTSK) + if err != nil { + if !strings.Contains(err.Error(), "actor not found") { + tw.Write(map[string]interface{}{ + "Address": addr, + "Error": err, + }) + continue + } + + a = &types.Actor{ + Balance: big.Zero(), + } + } + + row := map[string]interface{}{ + "Address": addr, + "Balance": types.FIL(a.Balance), + "Nonce": a.Nonce, + } + if addr == def { + row["Default"] = "X" + } + + tw.Write(row) + } + } + + if !cctx.Bool("addr-only") { + return tw.Flush(os.Stdout) + } + return nil }, } From d1c10a61dd61a38a90c7d49dcd1cfeb74318eba4 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Tue, 29 Sep 2020 12:19:04 +0200 Subject: [PATCH 415/885] fix: message signer - always compare with mpool nonce --- chain/messagesigner/messagesigner.go | 44 +++++++++++++---------- chain/messagesigner/messagesigner_test.go | 5 ++- node/builder.go | 3 +- node/impl/full/mpool.go | 3 +- 4 files changed, 30 insertions(+), 25 deletions(-) diff --git a/chain/messagesigner/messagesigner.go b/chain/messagesigner/messagesigner.go index 41b0edee9..1ad83543b 100644 --- a/chain/messagesigner/messagesigner.go +++ b/chain/messagesigner/messagesigner.go @@ -4,21 +4,22 @@ import ( "bytes" "context" - "github.com/filecoin-project/lotus/chain/wallet" - - "github.com/filecoin-project/lotus/chain/messagepool" - "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/messagepool" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/wallet" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" + logging "github.com/ipfs/go-log/v2" cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" ) const dsKeyActorNonce = "ActorNonce" +var log = logging.Logger("messagesigner") + type mpoolAPI interface { GetNonce(address.Address) (uint64, error) } @@ -67,30 +68,30 @@ func (ms *MessageSigner) SignMessage(ctx context.Context, msg *types.Message) (* // nextNonce increments the nonce. // If there is no nonce in the datastore, gets the nonce from the message pool. func (ms *MessageSigner) nextNonce(addr address.Address) (uint64, error) { - addrNonceKey := datastore.KeyWithNamespaces([]string{dsKeyActorNonce, addr.String()}) + // Nonces used to be created by the mempool and we need to support nodes + // that have mempool nonces, so first check the mempool for a nonce for + // this address. Note that the mempool returns the actor state's nonce + // by default. + nonce, err := ms.mpool.GetNonce(addr) + if err != nil { + return 0, xerrors.Errorf("failed to get nonce from mempool: %w", err) + } // Get the nonce for this address from the datastore - nonceBytes, err := ms.ds.Get(addrNonceKey) + addrNonceKey := datastore.KeyWithNamespaces([]string{dsKeyActorNonce, addr.String()}) + dsNonceBytes, err := ms.ds.Get(addrNonceKey) - var nonce uint64 switch { case xerrors.Is(err, datastore.ErrNotFound): // If a nonce for this address hasn't yet been created in the - // datastore, check the mempool - nonces used to be created by - // the mempool so we need to support nodes that still have mempool - // nonces. Note that the mempool returns the actor state's nonce by - // default. - nonce, err = ms.mpool.GetNonce(addr) - if err != nil { - return 0, xerrors.Errorf("failed to get nonce from mempool: %w", err) - } + // datastore, just use the nonce from the mempool case err != nil: return 0, xerrors.Errorf("failed to get nonce from datastore: %w", err) default: - // There is a nonce in the mempool, so unmarshall and increment it - maj, val, err := cbg.CborReadHeader(bytes.NewReader(nonceBytes)) + // There is a nonce in the datastore, so unmarshall and increment it + maj, val, err := cbg.CborReadHeader(bytes.NewReader(dsNonceBytes)) if err != nil { return 0, xerrors.Errorf("failed to parse nonce from datastore: %w", err) } @@ -98,7 +99,14 @@ func (ms *MessageSigner) nextNonce(addr address.Address) (uint64, error) { return 0, xerrors.Errorf("bad cbor type parsing nonce from datastore") } - nonce = val + 1 + dsNonce := val + 1 + + // The message pool nonce should be <= than the datastore nonce + if nonce <= dsNonce { + nonce = dsNonce + } else { + log.Warnf("mempool nonce was larger than datastore nonce (%d > %d)", nonce, dsNonce) + } } // Write the nonce for this address to the datastore diff --git a/chain/messagesigner/messagesigner_test.go b/chain/messagesigner/messagesigner_test.go index e52137892..55676b258 100644 --- a/chain/messagesigner/messagesigner_test.go +++ b/chain/messagesigner/messagesigner_test.go @@ -98,10 +98,9 @@ func TestMessageSignerSignMessage(t *testing.T) { To: to1, From: from1, }, - // Should ignore mpool nonce because after the first message nonce - // will come from the datastore + // Should adjust datastore nonce because mpool nonce is higher mpoolNonce: [1]uint64{10}, - expNonce: 6, + expNonce: 10, }}, }, { // Nonce should increment independently for each address diff --git a/node/builder.go b/node/builder.go index c49789a6a..da2924338 100644 --- a/node/builder.go +++ b/node/builder.go @@ -6,8 +6,6 @@ import ( "os" "time" - "github.com/filecoin-project/lotus/chain/messagesigner" - logging "github.com/ipfs/go-log" ci "github.com/libp2p/go-libp2p-core/crypto" "github.com/libp2p/go-libp2p-core/host" @@ -37,6 +35,7 @@ import ( "github.com/filecoin-project/lotus/chain/gen/slashfilter" "github.com/filecoin-project/lotus/chain/market" "github.com/filecoin-project/lotus/chain/messagepool" + "github.com/filecoin-project/lotus/chain/messagesigner" "github.com/filecoin-project/lotus/chain/metrics" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/store" diff --git a/node/impl/full/mpool.go b/node/impl/full/mpool.go index 003260496..066aafdc5 100644 --- a/node/impl/full/mpool.go +++ b/node/impl/full/mpool.go @@ -4,14 +4,13 @@ import ( "context" "encoding/json" - "github.com/filecoin-project/lotus/chain/messagesigner" - "github.com/filecoin-project/go-address" "github.com/ipfs/go-cid" "go.uber.org/fx" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/messagesigner" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/modules/dtypes" ) From 09e5cc90a40791b3139e5eac8912532246ca8058 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Tue, 29 Sep 2020 14:45:55 +0200 Subject: [PATCH 416/885] Add README to documentation/en with explanations --- documentation/en/README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 documentation/en/README.md diff --git a/documentation/en/README.md b/documentation/en/README.md new file mode 100644 index 000000000..76f11ed90 --- /dev/null +++ b/documentation/en/README.md @@ -0,0 +1,16 @@ +# Lotus documentation + +This folder contains some Lotus documentation mostly intended for Lotus developers. + +User documentation (including documentation for miners) has been moved to specific Lotus sections in https://docs.filecoin.io: + +- https://docs.filecoin.io/get-started/lotus +- https://docs.filecoin.io/store/lotus +- https://docs.filecoin.io/mine/lotus +- https://docs.filecoin.io/build/lotus + +## The Lotu.sh site + +The https://lotu.sh and https://docs.lotu.sh sites are generated from this folder based on the index provided by [.library.json](.library.json). This is done at the [lotus-docs repository](https://github.com/filecoin-project/lotus-docs), which contains Lotus as a git submodule. + +To update the site, the lotus-docs repository should be updated with the desired version for the lotus git submodule. Once pushed to master, it will be auto-deployed. From 96193c20448a530e3f0c542700fb78fa928bc880 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 16 Sep 2020 16:34:54 +0200 Subject: [PATCH 417/885] Implement bench-cache Signed-off-by: Jakub Sztandera --- cmd/lotus-bench/caching_verifier.go | 33 +++++++++++++++ cmd/lotus-bench/import.go | 63 +++++++++++++++++++++-------- 2 files changed, 80 insertions(+), 16 deletions(-) create mode 100644 cmd/lotus-bench/caching_verifier.go diff --git a/cmd/lotus-bench/caching_verifier.go b/cmd/lotus-bench/caching_verifier.go new file mode 100644 index 000000000..cd794e647 --- /dev/null +++ b/cmd/lotus-bench/caching_verifier.go @@ -0,0 +1,33 @@ +package main + +import ( + "context" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" + "github.com/filecoin-project/specs-actors/actors/runtime/proof" + "github.com/ipfs/go-datastore" +) + +type cachingVerifier struct { + ds datastore.Datastore + backend ffiwrapper.Verifier +} + +func (cv *cachingVerifier) VerifySeal(svi proof.SealVerifyInfo) (bool, error) { + svi.MarshalCBOR(nil) + return cv.backend.VerifySeal(svi) +} +func (cv *cachingVerifier) VerifyWinningPoSt(ctx context.Context, info proof.WinningPoStVerifyInfo) (bool, error) { + info.MarshalCBOR(nil) + return cv.backend.VerifyWinningPoSt(ctx, info) +} +func (cv *cachingVerifier) VerifyWindowPoSt(ctx context.Context, info proof.WindowPoStVerifyInfo) (bool, error) { + info.MarshalCBOR(nil) + return cv.backend.VerifyWindowPoSt(ctx, info) +} +func (cv *cachingVerifier) GenerateWinningPoStSectorChallenge(ctx context.Context, proofType abi.RegisteredPoStProof, a abi.ActorID, rnd abi.PoStRandomness, u uint64) ([]uint64, error) { + return cv.backend.GenerateWinningPoStSectorChallenge(ctx, proofType, a, rnd, u) +} + +var _ ffiwrapper.Verifier = (*cachingVerifier)(nil) diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index f2845ba20..fc81c600e 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -60,6 +60,15 @@ var importBenchCmd = &cli.Command{ Name: "repodir", Usage: "set the repo directory for the lotus bench run (defaults to /tmp)", }, + &cli.StringFlag{ + Name: "syscall-cache", + Usage: "read and write syscall results from datastore", + }, + &cli.BoolFlag{ + Name: "export-traces", + Usage: "should we export execution traces", + Value: true, + }, }, Action: func(cctx *cli.Context) error { vm.BatchSealVerifyParallelism = cctx.Int("batch-seal-verify-threads") @@ -85,7 +94,10 @@ var importBenchCmd = &cli.Command{ tdir = tmp } - bds, err := badger.NewDatastore(tdir, nil) + bdgOpt := badger.DefaultOptions + bdgOpt.GcInterval = 0 + + bds, err := badger.NewDatastore(tdir, &bdgOpt) if err != nil { return err } @@ -96,7 +108,21 @@ var importBenchCmd = &cli.Command{ } bs = cbs ds := datastore.NewMapDatastore() - cs := store.NewChainStore(bs, ds, vm.Syscalls(ffiwrapper.ProofVerifier)) + + var verifier ffiwrapper.Verifier = ffiwrapper.ProofVerifier + if cctx.IsSet("syscall-cache") { + + scds, err := badger.NewDatastore(cctx.String("syscall-cache"), &bdgOpt) + if err != nil { + return xerrors.Errorf("opening syscall-cache datastore: %w", err) + } + verifier = &cachingVerifier{ + ds: scds, + backend: verifier, + } + } + + cs := store.NewChainStore(bs, ds, vm.Syscalls(verifier)) stm := stmgr.NewStateManager(cs) prof, err := os.Create("import-bench.prof") @@ -144,13 +170,16 @@ var importBenchCmd = &cli.Command{ ts = next } - ibj, err := os.Create("import-bench.json") - if err != nil { - return err - } - defer ibj.Close() //nolint:errcheck + var enc *json.Encoder + if cctx.Bool("export-traces") { + ibj, err := os.Create("import-bench.json") + if err != nil { + return err + } + defer ibj.Close() //nolint:errcheck - enc := json.NewEncoder(ibj) + enc = json.NewEncoder(ibj) + } var lastTse *TipSetExec @@ -173,17 +202,19 @@ var importBenchCmd = &cli.Command{ if err != nil { return err } - stripCallers(trace) + if enc != nil { + stripCallers(trace) - lastTse = &TipSetExec{ - TipSet: cur.Key(), - Trace: trace, - Duration: time.Since(start), + lastTse = &TipSetExec{ + TipSet: cur.Key(), + Trace: trace, + Duration: time.Since(start), + } + if err := enc.Encode(lastTse); err != nil { + return xerrors.Errorf("failed to write out tipsetexec: %w", err) + } } lastState = st - if err := enc.Encode(lastTse); err != nil { - return xerrors.Errorf("failed to write out tipsetexec: %w", err) - } } pprof.StopCPUProfile() From 79ba4598d6f85bdc3e95faeb0e394af61e5e0666 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 16 Sep 2020 17:54:22 +0200 Subject: [PATCH 418/885] Implement cache Signed-off-by: Jakub Sztandera --- cmd/lotus-bench/caching_verifier.go | 75 +++++++++++++++++++++++++++-- 1 file changed, 70 insertions(+), 5 deletions(-) diff --git a/cmd/lotus-bench/caching_verifier.go b/cmd/lotus-bench/caching_verifier.go index cd794e647..28897071a 100644 --- a/cmd/lotus-bench/caching_verifier.go +++ b/cmd/lotus-bench/caching_verifier.go @@ -1,12 +1,16 @@ package main import ( + "bufio" "context" + "errors" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/specs-actors/actors/runtime/proof" "github.com/ipfs/go-datastore" + "github.com/minio/blake2b-simd" + cbg "github.com/whyrusleeping/cbor-gen" ) type cachingVerifier struct { @@ -14,17 +18,78 @@ type cachingVerifier struct { backend ffiwrapper.Verifier } +const bufsize = 128 + +func (cv cachingVerifier) withCache(execute func() (bool, error), param cbg.CBORMarshaler) (bool, error) { + hasher := blake2b.New256() + wr := bufio.NewWriterSize(hasher, bufsize) + err := param.MarshalCBOR(wr) + if err != nil { + log.Errorf("could not marshal call info: %+v", err) + return execute() + } + err = wr.Flush() + if err != nil { + log.Errorf("could not flush: %+v", err) + return execute() + } + hash := hasher.Sum(nil) + key := datastore.NewKey(string(hash)) + fromDs, err := cv.ds.Get(key) + if err == nil { + switch fromDs[0] { + case 's': + return true, nil + case 'f': + return false, nil + case 'e': + return false, errors.New(string(fromDs[1:])) + default: + log.Errorf("bad cached result in cache %s(%x)", fromDs[0], fromDs[0]) + return execute() + } + } else if errors.Is(err, datastore.ErrNotFound) { + // recalc + ok, err := execute() + var save []byte + if err != nil { + if ok { + log.Errorf("sucess with an error: %+v", err) + } else { + save = append([]byte{'e'}, []byte(err.Error())...) + } + } else if ok { + save = []byte{'s'} + } else { + save = []byte{'f'} + } + + if len(save) != 0 { + errSave := cv.ds.Put(key, save) + if errSave != nil { + log.Errorf("error saving result: %+v", errSave) + } + } + + return ok, err + } else { + log.Errorf("could not get data from cache: %+v", err) + return execute() + } +} + func (cv *cachingVerifier) VerifySeal(svi proof.SealVerifyInfo) (bool, error) { - svi.MarshalCBOR(nil) - return cv.backend.VerifySeal(svi) + return cv.withCache(func() (bool, error) { + return cv.backend.VerifySeal(svi) + }, &svi) } func (cv *cachingVerifier) VerifyWinningPoSt(ctx context.Context, info proof.WinningPoStVerifyInfo) (bool, error) { - info.MarshalCBOR(nil) return cv.backend.VerifyWinningPoSt(ctx, info) } func (cv *cachingVerifier) VerifyWindowPoSt(ctx context.Context, info proof.WindowPoStVerifyInfo) (bool, error) { - info.MarshalCBOR(nil) - return cv.backend.VerifyWindowPoSt(ctx, info) + return cv.withCache(func() (bool, error) { + return cv.backend.VerifyWindowPoSt(ctx, info) + }, &info) } func (cv *cachingVerifier) GenerateWinningPoStSectorChallenge(ctx context.Context, proofType abi.RegisteredPoStProof, a abi.ActorID, rnd abi.PoStRandomness, u uint64) ([]uint64, error) { return cv.backend.GenerateWinningPoStSectorChallenge(ctx, proofType, a, rnd, u) From 53ab17cf50f5d7a69b81a6945bbe317a747bd10d Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 16 Sep 2020 20:10:00 +0200 Subject: [PATCH 419/885] Add no import to import-bench Signed-off-by: Jakub Sztandera --- cmd/lotus-bench/import.go | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index fc81c600e..c16796a13 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -24,6 +24,7 @@ import ( "github.com/filecoin-project/lotus/lib/blockstore" _ "github.com/filecoin-project/lotus/lib/sigs/bls" _ "github.com/filecoin-project/lotus/lib/sigs/secp" + "github.com/ipld/go-car" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" @@ -69,6 +70,10 @@ var importBenchCmd = &cli.Command{ Usage: "should we export execution traces", Value: true, }, + &cli.BoolFlag{ + Name: "no-import", + Usage: "should we import the chain? if set to true chain has to be previously imported", + }, }, Action: func(cctx *cli.Context) error { vm.BatchSealVerifyParallelism = cctx.Int("batch-seal-verify-threads") @@ -111,7 +116,6 @@ var importBenchCmd = &cli.Command{ var verifier ffiwrapper.Verifier = ffiwrapper.ProofVerifier if cctx.IsSet("syscall-cache") { - scds, err := badger.NewDatastore(cctx.String("syscall-cache"), &bdgOpt) if err != nil { return xerrors.Errorf("opening syscall-cache datastore: %w", err) @@ -135,9 +139,21 @@ var importBenchCmd = &cli.Command{ return err } - head, err := cs.Import(cfi) - if err != nil { - return err + var head *types.TipSet + if !cctx.Bool("no-import") { + head, err = cs.Import(cfi) + if err != nil { + return err + } + } else { + cr, err := car.NewCarReader(cfi) + if err != nil { + return err + } + head, err = cs.LoadTipSet(types.NewTipSetKey(cr.Header.Roots...)) + if err != nil { + return err + } } gb, err := cs.GetTipsetByHeight(context.TODO(), 0, head, true) @@ -188,6 +204,7 @@ var importBenchCmd = &cli.Command{ cur := tschain[i] log.Infof("computing state (height: %d, ts=%s)", cur.Height(), cur.Cids()) if cur.ParentState() != lastState { + stripCallers(lastTse.Trace) lastTrace := lastTse.Trace d, err := json.MarshalIndent(lastTrace, "", " ") if err != nil { @@ -202,14 +219,14 @@ var importBenchCmd = &cli.Command{ if err != nil { return err } + lastTse = &TipSetExec{ + TipSet: cur.Key(), + Trace: trace, + Duration: time.Since(start), + } if enc != nil { - stripCallers(trace) + stripCallers(lastTse.Trace) - lastTse = &TipSetExec{ - TipSet: cur.Key(), - Trace: trace, - Duration: time.Since(start), - } if err := enc.Encode(lastTse); err != nil { return xerrors.Errorf("failed to write out tipsetexec: %w", err) } From 1f4d1dcc58fc2da3574c479159775e4b19b1a720 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 16 Sep 2020 20:38:28 +0200 Subject: [PATCH 420/885] Do not sync Signed-off-by: Jakub Sztandera --- cmd/lotus-bench/import.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index c16796a13..443e67950 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -101,6 +101,8 @@ var importBenchCmd = &cli.Command{ bdgOpt := badger.DefaultOptions bdgOpt.GcInterval = 0 + bdgOpt.Options.SyncWrites = false + bdgOpt.Options.Truncate = true bds, err := badger.NewDatastore(tdir, &bdgOpt) if err != nil { From 12a0dd3d0a0a6b80c1e11c2e05d39b5024d55139 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 17 Sep 2020 00:06:20 +0200 Subject: [PATCH 421/885] <3 to linter Signed-off-by: Jakub Sztandera --- cmd/lotus-bench/caching_verifier.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/lotus-bench/caching_verifier.go b/cmd/lotus-bench/caching_verifier.go index 28897071a..51ab696f7 100644 --- a/cmd/lotus-bench/caching_verifier.go +++ b/cmd/lotus-bench/caching_verifier.go @@ -54,7 +54,7 @@ func (cv cachingVerifier) withCache(execute func() (bool, error), param cbg.CBOR var save []byte if err != nil { if ok { - log.Errorf("sucess with an error: %+v", err) + log.Errorf("success with an error: %+v", err) } else { save = append([]byte{'e'}, []byte(err.Error())...) } From 108fe7823c9dadf3d7e3812d79a66f4e6769ef5e Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 18 Sep 2020 13:39:38 +0200 Subject: [PATCH 422/885] Add command to trigger gc Signed-off-by: Jakub Sztandera --- cmd/lotus-bench/import.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index 443e67950..72ac5d60b 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -74,6 +74,9 @@ var importBenchCmd = &cli.Command{ Name: "no-import", Usage: "should we import the chain? if set to true chain has to be previously imported", }, + &cli.BoolFlag{ + Name: "only-gc", + }, }, Action: func(cctx *cli.Context) error { vm.BatchSealVerifyParallelism = cctx.Int("batch-seal-verify-threads") @@ -103,11 +106,15 @@ var importBenchCmd = &cli.Command{ bdgOpt.GcInterval = 0 bdgOpt.Options.SyncWrites = false bdgOpt.Options.Truncate = true + bdgOpt.Options.DetectConflicts = false + bdgOpt.Options.MaxTableSize = 64 << 20 bds, err := badger.NewDatastore(tdir, &bdgOpt) if err != nil { return err } + + bds.CollectGarbage() bs := blockstore.NewBlockstore(bds) cbs, err := blockstore.CachedBlockstore(context.TODO(), bs, blockstore.DefaultCacheOpts()) if err != nil { @@ -122,11 +129,15 @@ var importBenchCmd = &cli.Command{ if err != nil { return xerrors.Errorf("opening syscall-cache datastore: %w", err) } + scds.CollectGarbage() verifier = &cachingVerifier{ ds: scds, backend: verifier, } } + if cctx.Bool("only-gc") { + return nil + } cs := store.NewChainStore(bs, ds, vm.Syscalls(verifier)) stm := stmgr.NewStateManager(cs) From 782717948ae9926e4afa7a42f758e8c70abe13d8 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 18 Sep 2020 13:54:20 +0200 Subject: [PATCH 423/885] Add logs Signed-off-by: Jakub Sztandera --- cmd/lotus-bench/import.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index 72ac5d60b..c6d49a78c 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -114,7 +114,11 @@ var importBenchCmd = &cli.Command{ return err } - bds.CollectGarbage() + if cctx.Bool("only-gc") { + log.Info("calling CollectGarbage on main ds") + bds.CollectGarbage() + log.Info("done calling CollectGarbage on main ds") + } bs := blockstore.NewBlockstore(bds) cbs, err := blockstore.CachedBlockstore(context.TODO(), bs, blockstore.DefaultCacheOpts()) if err != nil { @@ -129,7 +133,12 @@ var importBenchCmd = &cli.Command{ if err != nil { return xerrors.Errorf("opening syscall-cache datastore: %w", err) } - scds.CollectGarbage() + + if cctx.Bool("only-gc") { + log.Info("calling CollectGarbage on syscall ds") + scds.CollectGarbage() + log.Info("done calling CollectGarbage on syscall ds") + } verifier = &cachingVerifier{ ds: scds, backend: verifier, From 3858309368bfa069f86f71b3558ef4532d6f422f Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 18 Sep 2020 13:55:37 +0200 Subject: [PATCH 424/885] Add http to import bench Signed-off-by: Jakub Sztandera --- cmd/lotus-bench/import.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index c6d49a78c..8c874206f 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -91,6 +91,10 @@ var importBenchCmd = &cli.Command{ } defer cfi.Close() //nolint:errcheck // read only file + go func() { + http.ListenAndServe("localhost:6060", nil) //nolint:errcheck + }() + var tdir string if rdir := cctx.String("repodir"); rdir != "" { tdir = rdir From 01386a206c0c8c9d9007d35a80307cae93b0781b Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 18 Sep 2020 15:17:13 +0200 Subject: [PATCH 425/885] Update options Signed-off-by: Jakub Sztandera --- cmd/lotus-bench/import.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index 8c874206f..db87ebeb6 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -29,8 +29,10 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" + bdg "github.com/dgraph-io/badger/v2" "github.com/ipfs/go-datastore" badger "github.com/ipfs/go-ds-badger2" + "github.com/urfave/cli/v2" "golang.org/x/xerrors" ) @@ -108,10 +110,10 @@ var importBenchCmd = &cli.Command{ bdgOpt := badger.DefaultOptions bdgOpt.GcInterval = 0 + bdgOpt.Options = bdg.DefaultOptions("") bdgOpt.Options.SyncWrites = false bdgOpt.Options.Truncate = true bdgOpt.Options.DetectConflicts = false - bdgOpt.Options.MaxTableSize = 64 << 20 bds, err := badger.NewDatastore(tdir, &bdgOpt) if err != nil { From f21c5cbbe28fe382c1670298e11819cf4949e7af Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Sat, 19 Sep 2020 19:11:37 +0200 Subject: [PATCH 426/885] Add start-at Signed-off-by: Jakub Sztandera --- cmd/lotus-bench/import.go | 69 ++++++++++++++++++++++++--------------- go.mod | 9 ++--- go.sum | 31 ++++++++++++++++++ 3 files changed, 78 insertions(+), 31 deletions(-) diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index db87ebeb6..3c5bd0cda 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -28,6 +28,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" + "github.com/filecoin-project/statediff" bdg "github.com/dgraph-io/badger/v2" "github.com/ipfs/go-datastore" @@ -79,6 +80,9 @@ var importBenchCmd = &cli.Command{ &cli.BoolFlag{ Name: "only-gc", }, + &cli.Int64Flag{ + Name: "start-at", + }, }, Action: func(cctx *cli.Context) error { vm.BatchSealVerifyParallelism = cctx.Int("batch-seal-verify-threads") @@ -194,6 +198,20 @@ var importBenchCmd = &cli.Command{ return err } + startEpoch := abi.ChainEpoch(1) + if cctx.IsSet("start-at") { + startEpoch = abi.ChainEpoch(cctx.Int64("start-at")) + start, err := cs.GetTipsetByHeight(context.TODO(), abi.ChainEpoch(cctx.Int64("start-at")), head, true) + if err != nil { + return err + } + + err = cs.SetHead(start) + if err != nil { + return err + } + } + if h := cctx.Int64("height"); h != 0 { tsh, err := cs.GetTipsetByHeight(context.TODO(), abi.ChainEpoch(h), head, true) if err != nil { @@ -204,7 +222,7 @@ var importBenchCmd = &cli.Command{ ts := head tschain := []*types.TipSet{ts} - for ts.Height() != 0 { + for ts.Height() > startEpoch { next, err := cs.LoadTipSet(ts.Parents()) if err != nil { return err @@ -225,41 +243,38 @@ var importBenchCmd = &cli.Command{ enc = json.NewEncoder(ibj) } - var lastTse *TipSetExec - - lastState := tschain[len(tschain)-1].ParentState() - for i := len(tschain) - 2; i >= 0; i-- { + for i := len(tschain) - 1; i >= 1; i-- { cur := tschain[i] + start := time.Now() log.Infof("computing state (height: %d, ts=%s)", cur.Height(), cur.Cids()) - if cur.ParentState() != lastState { - stripCallers(lastTse.Trace) - lastTrace := lastTse.Trace + st, trace, err := stm.ExecutionTrace(context.TODO(), cur) + if err != nil { + return err + } + tse := &TipSetExec{ + TipSet: cur.Key(), + Trace: trace, + Duration: time.Since(start), + } + if enc != nil { + stripCallers(tse.Trace) + + if err := enc.Encode(tse); err != nil { + return xerrors.Errorf("failed to write out tipsetexec: %w", err) + } + } + if tschain[i-1].ParentState() != st { + stripCallers(tse.Trace) + lastTrace := tse.Trace d, err := json.MarshalIndent(lastTrace, "", " ") if err != nil { panic(err) } fmt.Println("TRACE") fmt.Println(string(d)) - return xerrors.Errorf("tipset chain had state mismatch at height %d (%s != %s)", cur.Height(), cur.ParentState(), lastState) + fmt.Println(statediff.Diff(context.Background(), bs, tschain[i-1].ParentState(), st, statediff.ExpandActors)) + return xerrors.Errorf("tipset chain had state mismatch at height %d (%s != %s)", cur.Height(), cur.ParentState(), st) } - start := time.Now() - st, trace, err := stm.ExecutionTrace(context.TODO(), cur) - if err != nil { - return err - } - lastTse = &TipSetExec{ - TipSet: cur.Key(), - Trace: trace, - Duration: time.Since(start), - } - if enc != nil { - stripCallers(lastTse.Trace) - - if err := enc.Encode(lastTse); err != nil { - return xerrors.Errorf("failed to write out tipsetexec: %w", err) - } - } - lastState = st } pprof.StopCPUProfile() diff --git a/go.mod b/go.mod index 2c0322ecc..83137d7c8 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,6 @@ require ( github.com/BurntSushi/toml v0.3.1 github.com/GeertJohan/go.rice v1.0.0 github.com/Gurpartap/async v0.0.0-20180927173644-4f7f499dd9ee - github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d github.com/buger/goterm v0.0.0-20200322175922-2f3e71b85129 github.com/coreos/go-systemd/v22 v22.0.0 @@ -38,10 +37,10 @@ require ( github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b github.com/filecoin-project/specs-actors v0.9.11 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 + github.com/filecoin-project/statediff v0.0.6-0.20200918150628-da86dd0d264c github.com/filecoin-project/test-vectors/schema v0.0.1 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 github.com/go-kit/kit v0.10.0 - github.com/go-ole/go-ole v1.2.4 // indirect github.com/google/uuid v1.1.1 github.com/gorilla/mux v1.7.4 github.com/gorilla/websocket v1.4.2 @@ -117,7 +116,6 @@ require ( github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542 go.opencensus.io v0.22.4 - go.uber.org/dig v1.10.0 // indirect go.uber.org/fx v1.9.0 go.uber.org/multierr v1.5.0 go.uber.org/zap v1.15.0 @@ -127,9 +125,12 @@ require ( golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 gopkg.in/cheggaaa/pb.v1 v1.0.28 gotest.tools v2.2.0+incompatible - launchpad.net/gocheck v0.0.0-20140225173054-000000000087 // indirect ) +replace github.com/filecoin-project/lotus => ./ + +replace github.com/filecoin-project/statediff => ./../statediff + replace github.com/golangci/golangci-lint => github.com/golangci/golangci-lint v1.18.0 replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi diff --git a/go.sum b/go.sum index 05e643708..2455766b1 100644 --- a/go.sum +++ b/go.sum @@ -159,6 +159,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018/go.mod h1:rQYf4tfk5sSwFsnDg3qYaBxSjsD9S8+59vW0dKUgme4= github.com/davidlazar/go-crypto v0.0.0-20190912175916-7055855a373f h1:BOaYiTvg8p9vBUXpklC22XSK/mifLF7lG9jtmYYi3Tc= github.com/davidlazar/go-crypto v0.0.0-20190912175916-7055855a373f/go.mod h1:rQYf4tfk5sSwFsnDg3qYaBxSjsD9S8+59vW0dKUgme4= +github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c h1:pFUpOrbxDR6AkioZ1ySsx5yxlDQZ8stG2b88gTPxgJU= +github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c/go.mod h1:6UhI8N9EjYm1c2odKpFpAYeR8dsBeM7PtzQhRgxRr9U= github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e h1:lj77EKYUpYXTd8CD/+QMIf8b6OIOTsfEBSXiAzuEHTU= github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e/go.mod h1:3ZQK6DMPSz/QZ73jlWxBtUhNA8xZx7LzUFSq/OfP8vk= github.com/dgraph-io/badger v1.5.5-0.20190226225317-8115aed38f8f/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ= @@ -206,6 +208,7 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanw/esbuild v0.6.28/go.mod h1:mptxmSXIzBIKKCe4jo9A5SToEd1G+AKZ9JmY85dYRJ0= github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 h1:BBso6MBKW8ncyZLv37o+KNyy0HrrHgfnOaGQC2qvN+A= github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5/go.mod h1:JpoxHjuQauoxiFMl1ie8Xc/7TfLuMZ5eOCONd1sUBHg= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -218,6 +221,8 @@ github.com/filecoin-project/go-address v0.0.4 h1:gSNMv0qWwH16fGQs7ycOUrDjY6YCSsg github.com/filecoin-project/go-address v0.0.4/go.mod h1:jr8JxKsYx+lQlQZmF5i2U0Z+cGQ59wMIps/8YW/lDj8= github.com/filecoin-project/go-amt-ipld/v2 v2.1.0 h1:t6qDiuGYYngDqaLc2ZUvdtAg4UNxPeOYaXhBWSNsVaM= github.com/filecoin-project/go-amt-ipld/v2 v2.1.0/go.mod h1:nfFPoGyX0CU9SkXX8EoCcSuHN1XcbN0c6KBh7yvP5fs= +github.com/filecoin-project/go-amt-ipld/v2 v2.1.1-0.20200731171407-e559a0579161 h1:K6t4Hrs+rwUxBz2xg88Bdqeh4k5/rycQFdPseZhRyfE= +github.com/filecoin-project/go-amt-ipld/v2 v2.1.1-0.20200731171407-e559a0579161/go.mod h1:vgmwKBkx+ca5OIeEvstiQgzAZnb7R6QaqE1oEDSqa6g= github.com/filecoin-project/go-bitfield v0.2.0 h1:gCtLcjskIPtdg4NfN7gQZSQF9yrBQ7mkT0qCJxzGI2Q= github.com/filecoin-project/go-bitfield v0.2.0/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM= github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:av5fw6wmm58FYMgJeoB/lK9XXrgdugYiTqkdxjTy9k8= @@ -280,6 +285,7 @@ github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclK github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.10.0 h1:dXFJfIHVvUcpSgDOV+Ne6t7jXri8Tfv2uOLHUZ2XNuo= @@ -506,6 +512,11 @@ github.com/ipfs/go-fs-lock v0.0.6/go.mod h1:OTR+Rj9sHiRubJh3dRhD15Juhd/+w6VPOY28 github.com/ipfs/go-graphsync v0.1.0/go.mod h1:jMXfqIEDFukLPZHqDPp8tJMbHO9Rmeb9CEGevngQbmE= github.com/ipfs/go-graphsync v0.2.1 h1:MdehhqBSuTI2LARfKLkpYnt0mUrqHs/mtuDnESXHBfU= github.com/ipfs/go-graphsync v0.2.1/go.mod h1:gEBvJUNelzMkaRPJTpg/jaKN4AQW/7wDWu0K92D8o10= +github.com/ipfs/go-graphsync v0.2.0 h1:x94MvHLNuRwBlZzVal7tR1RYK7T7H6bqQLPopxDbIF0= +github.com/ipfs/go-graphsync v0.2.0/go.mod h1:gEBvJUNelzMkaRPJTpg/jaKN4AQW/7wDWu0K92D8o10= +github.com/ipfs/go-graphsync v0.1.2 h1:25Ll9kIXCE+DY0dicvfS3KMw+U5sd01b/FJbA7KAbhg= +github.com/ipfs/go-graphsync v0.1.2/go.mod h1:sLXVXm1OxtE2XYPw62MuXCdAuNwkAdsbnfrmos5odbA= +github.com/ipfs/go-hamt-ipld v0.1.1 h1:0IQdvwnAAUKmDE+PMJa5y1QiwOPHpI9+eAbQEEEYthk= github.com/ipfs/go-hamt-ipld v0.1.1/go.mod h1:1EZCr2v0jlCnhpa+aZ0JZYp8Tt2w16+JJOAVz17YcDk= github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw= @@ -688,6 +699,7 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lib/pq v1.7.0 h1:h93mCPfUSkaul3Ka/VG8uZdmW1uMHDGxzu0NWHuJmHY= github.com/lib/pq v1.7.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ= @@ -1015,6 +1027,8 @@ github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= @@ -1317,6 +1331,8 @@ github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5J github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -1369,6 +1385,7 @@ github.com/whyrusleeping/cbor-gen v0.0.0-20200414195334-429a0b5e922e/go.mod h1:X github.com/whyrusleeping/cbor-gen v0.0.0-20200504204219-64967432584d/go.mod h1:W5MvapuoHRP8rz4vxjwCK1pDqF1aQcWsV5PZ+AHbqdg= github.com/whyrusleeping/cbor-gen v0.0.0-20200710004633-5379fc63235d/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200715143311-227fab5a2377/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= +github.com/whyrusleeping/cbor-gen v0.0.0-20200723185710-6a3894a6352b/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200810223238-211df3b9e24c/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200812213548-958ddffe352c/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200814224545-656e08ce49ee h1:U7zWWvvAjT76EiuWPSOiZlQDnaQYPxPoxugTtTAcJK0= @@ -1395,6 +1412,8 @@ github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d/go.mod h1:g7c github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee h1:lYbXeSvJi5zk5GLKVuid9TVjS9a0OmLIDKTfoZBL6Ow= github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee/go.mod h1:m2aV4LZI4Aez7dP5PMyVKEHhUyEJ/RjmPEDOpDvudHg= github.com/whyrusleeping/yamux v1.1.5/go.mod h1:E8LnQQ8HKx5KD29HZFUwM1PxCOdPRzGwur1mcYhXcD8= +github.com/willscott/go-cmp v0.5.2-0.20200812183318-8affb9542345 h1:IJVAwIctqDFOrO0C2qzksXmANviyHJzrklU27e1ltzE= +github.com/willscott/go-cmp v0.5.2-0.20200812183318-8affb9542345/go.mod h1:D7hA8H5pyQx7Y5Em7IWx1R4vNJzfon3gpG9nxjkITjQ= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/c-for-go v0.0.0-20200718154222-87b0065af829 h1:wb7xrDzfkLgPHsSEBm+VSx6aDdi64VtV0xvP0E6j8bk= @@ -1483,6 +1502,7 @@ golang.org/x/crypto v0.0.0-20200128174031-69ecbb4d6d5d/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -1495,6 +1515,8 @@ golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd h1:zkO/Lhoka23X63N9OSzpSeROEUQ5ODw47tM3YWjygbs= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 h1:QE6XYQK6naiK1EPAe1g/ILLxN5RBoH5xkJk3CqlMI/Y= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1508,11 +1530,14 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367 h1:0IiAsCRByjO2QjX7ZPkw5oU9x+n1YqRL802rjC0c3Aw= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -1614,6 +1639,7 @@ golang.org/x/sys v0.0.0-20190902133755-9109b7679e13/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191025021431-6c3a3bfe00ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1623,6 +1649,7 @@ golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1631,6 +1658,7 @@ golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200509044756-6aff5f38e54f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 h1:OjiUf46hAmXblsZdnoSXsEUSKU8r1UEzcL5RVZ4gO9Y= @@ -1674,6 +1702,7 @@ golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -1802,6 +1831,8 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqpnJdvzuiHsl/dnxl11M= howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= launchpad.net/gocheck v0.0.0-20140225173054-000000000087 h1:Izowp2XBH6Ya6rv+hqbceQyw/gSGoXfH/UPoTGduL54= From ebc8489ff183859cc0d567bd95b5904183b25152 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Sat, 19 Sep 2020 20:27:24 +0200 Subject: [PATCH 427/885] Add global-profile option Signed-off-by: Jakub Sztandera --- cmd/lotus-bench/import.go | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index 3c5bd0cda..e201baa85 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -80,6 +80,10 @@ var importBenchCmd = &cli.Command{ &cli.BoolFlag{ Name: "only-gc", }, + &cli.BoolFlag{ + Name: "global-profile", + Value: true, + }, &cli.Int64Flag{ Name: "start-at", }, @@ -161,14 +165,16 @@ var importBenchCmd = &cli.Command{ cs := store.NewChainStore(bs, ds, vm.Syscalls(verifier)) stm := stmgr.NewStateManager(cs) - prof, err := os.Create("import-bench.prof") - if err != nil { - return err - } - defer prof.Close() //nolint:errcheck + if cctx.Bool("global-profile") { + prof, err := os.Create("import-bench.prof") + if err != nil { + return err + } + defer prof.Close() //nolint:errcheck - if err := pprof.StartCPUProfile(prof); err != nil { - return err + if err := pprof.StartCPUProfile(prof); err != nil { + return err + } } var head *types.TipSet From 35cf69ae646574557acfaaa8bd65a4693560b352 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Sat, 19 Sep 2020 20:49:40 +0200 Subject: [PATCH 428/885] Disable bloomcache Signed-off-by: Jakub Sztandera --- cmd/lotus-bench/import.go | 3 +++ go.mod | 1 + 2 files changed, 4 insertions(+) diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index e201baa85..5da4f2d3d 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -134,6 +134,9 @@ var importBenchCmd = &cli.Command{ log.Info("done calling CollectGarbage on main ds") } bs := blockstore.NewBlockstore(bds) + cacheOpts := blockstore.DefaultCacheOpts() + cacheOpts.HasBloomFilterSize = 0 + cbs, err := blockstore.CachedBlockstore(context.TODO(), bs, blockstore.DefaultCacheOpts()) if err != nil { return err diff --git a/go.mod b/go.mod index 83137d7c8..d2e552ffc 100644 --- a/go.mod +++ b/go.mod @@ -116,6 +116,7 @@ require ( github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542 go.opencensus.io v0.22.4 + go.uber.org/dig v1.10.0 // indirect go.uber.org/fx v1.9.0 go.uber.org/multierr v1.5.0 go.uber.org/zap v1.15.0 From 242a77b391c001ddf3fad53c3f5c8160e157c4c3 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Sat, 19 Sep 2020 20:50:06 +0200 Subject: [PATCH 429/885] go mod tidy Signed-off-by: Jakub Sztandera --- cmd/lotus-bench/import.go | 2 +- go.mod | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index 5da4f2d3d..428fd4212 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -137,7 +137,7 @@ var importBenchCmd = &cli.Command{ cacheOpts := blockstore.DefaultCacheOpts() cacheOpts.HasBloomFilterSize = 0 - cbs, err := blockstore.CachedBlockstore(context.TODO(), bs, blockstore.DefaultCacheOpts()) + cbs, err := blockstore.CachedBlockstore(context.TODO(), bs, cacheOpts) if err != nil { return err } diff --git a/go.mod b/go.mod index d2e552ffc..83137d7c8 100644 --- a/go.mod +++ b/go.mod @@ -116,7 +116,6 @@ require ( github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542 go.opencensus.io v0.22.4 - go.uber.org/dig v1.10.0 // indirect go.uber.org/fx v1.9.0 go.uber.org/multierr v1.5.0 go.uber.org/zap v1.15.0 From b7f18b460147f43d59203f43f5845164d9f81066 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Sat, 19 Sep 2020 21:00:38 +0200 Subject: [PATCH 430/885] Disable callers Signed-off-by: Jakub Sztandera --- chain/vm/runtime.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chain/vm/runtime.go b/chain/vm/runtime.go index 156d57282..eb4476718 100644 --- a/chain/vm/runtime.go +++ b/chain/vm/runtime.go @@ -5,7 +5,6 @@ import ( "context" "encoding/binary" "fmt" - gruntime "runtime" "time" "github.com/filecoin-project/go-address" @@ -493,7 +492,8 @@ func (rt *Runtime) chargeGasFunc(skip int) func(GasCharge) { func (rt *Runtime) chargeGasInternal(gas GasCharge, skip int) aerrors.ActorError { toUse := gas.Total() var callers [10]uintptr - cout := gruntime.Callers(2+skip, callers[:]) + + cout := 0 //gruntime.Callers(2+skip, callers[:]) now := build.Clock.Now() if rt.lastGasCharge != nil { From 1c6214b76d1cb1022afeb8ac66e475c046ec4a4f Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Sun, 20 Sep 2020 03:22:41 +0200 Subject: [PATCH 431/885] Usage go-bitfield with buffer pool Signed-off-by: Jakub Sztandera --- go.mod | 2 +- go.sum | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 83137d7c8..afa317614 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/fatih/color v1.8.0 github.com/filecoin-project/filecoin-ffi v0.30.4-0.20200716204036-cddc56607e1d github.com/filecoin-project/go-address v0.0.4 - github.com/filecoin-project/go-bitfield v0.2.0 + github.com/filecoin-project/go-bitfield v0.2.1-0.20200920172649-837cbe6a1ed3 github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 github.com/filecoin-project/go-data-transfer v0.6.6 diff --git a/go.sum b/go.sum index 2455766b1..28606d17a 100644 --- a/go.sum +++ b/go.sum @@ -225,6 +225,10 @@ github.com/filecoin-project/go-amt-ipld/v2 v2.1.1-0.20200731171407-e559a0579161 github.com/filecoin-project/go-amt-ipld/v2 v2.1.1-0.20200731171407-e559a0579161/go.mod h1:vgmwKBkx+ca5OIeEvstiQgzAZnb7R6QaqE1oEDSqa6g= github.com/filecoin-project/go-bitfield v0.2.0 h1:gCtLcjskIPtdg4NfN7gQZSQF9yrBQ7mkT0qCJxzGI2Q= github.com/filecoin-project/go-bitfield v0.2.0/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM= +github.com/filecoin-project/go-bitfield v0.2.1-0.20200920171219-7c2059195a8c h1:eEmdVMWo7AngX9fGZSSAm/V6+7tqiBawFfHRjW35JwU= +github.com/filecoin-project/go-bitfield v0.2.1-0.20200920171219-7c2059195a8c/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM= +github.com/filecoin-project/go-bitfield v0.2.1-0.20200920172649-837cbe6a1ed3 h1:HQa4+yCYsLq1TLM0kopeAhSCLbtZ541cWEi5N5rO+9g= +github.com/filecoin-project/go-bitfield v0.2.1-0.20200920172649-837cbe6a1ed3/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM= github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:av5fw6wmm58FYMgJeoB/lK9XXrgdugYiTqkdxjTy9k8= github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= From 0771c23fb02d2c6cb456f81c297cdb441345e7a6 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Mon, 21 Sep 2020 22:47:03 +0200 Subject: [PATCH 432/885] Use pebble Signed-off-by: Jakub Sztandera --- cmd/lotus-bench/import.go | 26 ++++++++++++++++++++++++-- go.mod | 4 +++- go.sum | 23 +++++++++++++++++++++-- 3 files changed, 48 insertions(+), 5 deletions(-) diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index 428fd4212..b535ed96c 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -16,6 +16,8 @@ import ( "sort" "time" + "github.com/cockroachdb/pebble" + "github.com/cockroachdb/pebble/bloom" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/store" @@ -33,6 +35,7 @@ import ( bdg "github.com/dgraph-io/badger/v2" "github.com/ipfs/go-datastore" badger "github.com/ipfs/go-ds-badger2" + pebbleds "github.com/ipfs/go-ds-pebble" "github.com/urfave/cli/v2" "golang.org/x/xerrors" @@ -123,14 +126,33 @@ var importBenchCmd = &cli.Command{ bdgOpt.Options.Truncate = true bdgOpt.Options.DetectConflicts = false - bds, err := badger.NewDatastore(tdir, &bdgOpt) + cache := 512 + bds, err := pebbleds.NewDatastore(tdir, &pebble.Options{ + // Pebble has a single combined cache area and the write + // buffers are taken from this too. Assign all available + // memory allowance for cache. + Cache: pebble.NewCache(int64(cache * 1024 * 1024)), + // The size of memory table(as well as the write buffer). + // Note, there may have more than two memory tables in the system. + // MemTableStopWritesThreshold can be configured to avoid the memory abuse. + MemTableSize: cache * 1024 * 1024 / 4, + // The default compaction concurrency(1 thread), + // Here use all available CPUs for faster compaction. + MaxConcurrentCompactions: runtime.NumCPU(), + // Per-level options. Options for at least one level must be specified. The + // options for the last level are used for all subsequent levels. + Levels: []pebble.LevelOptions{ + {TargetFileSize: 2 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)}, + }, + Logger: log, + }) if err != nil { return err } if cctx.Bool("only-gc") { log.Info("calling CollectGarbage on main ds") - bds.CollectGarbage() + //bds.CollectGarbage() log.Info("done calling CollectGarbage on main ds") } bs := blockstore.NewBlockstore(bds) diff --git a/go.mod b/go.mod index afa317614..3b3b4fed7 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/Gurpartap/async v0.0.0-20180927173644-4f7f499dd9ee github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d github.com/buger/goterm v0.0.0-20200322175922-2f3e71b85129 + github.com/cockroachdb/pebble v0.0.0-20200916222308-4e219a90ba5b github.com/coreos/go-systemd/v22 v22.0.0 github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e github.com/dgraph-io/badger/v2 v2.2007.2 @@ -53,10 +54,11 @@ require ( github.com/ipfs/go-blockservice v0.1.4-0.20200624145336-a978cec6e834 github.com/ipfs/go-cid v0.0.7 github.com/ipfs/go-cidutil v0.0.2 - github.com/ipfs/go-datastore v0.4.4 + github.com/ipfs/go-datastore v0.4.5 github.com/ipfs/go-ds-badger2 v0.1.1-0.20200708190120-187fc06f714e github.com/ipfs/go-ds-leveldb v0.4.2 github.com/ipfs/go-ds-measure v0.1.0 + github.com/ipfs/go-ds-pebble v0.0.2-0.20200921211847-f1ffb3128b61 github.com/ipfs/go-filestore v1.0.0 github.com/ipfs/go-fs-lock v0.0.6 github.com/ipfs/go-graphsync v0.2.1 diff --git a/go.sum b/go.sum index 28606d17a..19e33d294 100644 --- a/go.sum +++ b/go.sum @@ -112,6 +112,8 @@ github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7 github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894 h1:JLaf/iINcLyjwbtTsCJjc6rtlASgHeIJPrB6QmwURnA= +github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= @@ -125,6 +127,14 @@ github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/cockroachdb/errors v1.2.4 h1:Lap807SXTH5tri2TivECb/4abUkMZC9zRoLarvcKDqs= +github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= +github.com/cockroachdb/pebble v0.0.0-20200916222308-4e219a90ba5b h1:OKALTB609+19AM7wsO0k8yMwAqjEIppcnYvyIhA+ZlQ= +github.com/cockroachdb/pebble v0.0.0-20200916222308-4e219a90ba5b/go.mod h1:hU7vhtrqonEphNF+xt8/lHdaBprxmV1h8BOGrd9XwmQ= +github.com/cockroachdb/redact v0.0.0-20200622112456-cd282804bbd3 h1:2+dpIJzYMSbLi0587YXpi8tOJT52qCOI/1I0UNThc/I= +github.com/cockroachdb/redact v0.0.0-20200622112456-cd282804bbd3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -225,8 +235,6 @@ github.com/filecoin-project/go-amt-ipld/v2 v2.1.1-0.20200731171407-e559a0579161 github.com/filecoin-project/go-amt-ipld/v2 v2.1.1-0.20200731171407-e559a0579161/go.mod h1:vgmwKBkx+ca5OIeEvstiQgzAZnb7R6QaqE1oEDSqa6g= github.com/filecoin-project/go-bitfield v0.2.0 h1:gCtLcjskIPtdg4NfN7gQZSQF9yrBQ7mkT0qCJxzGI2Q= github.com/filecoin-project/go-bitfield v0.2.0/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM= -github.com/filecoin-project/go-bitfield v0.2.1-0.20200920171219-7c2059195a8c h1:eEmdVMWo7AngX9fGZSSAm/V6+7tqiBawFfHRjW35JwU= -github.com/filecoin-project/go-bitfield v0.2.1-0.20200920171219-7c2059195a8c/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM= github.com/filecoin-project/go-bitfield v0.2.1-0.20200920172649-837cbe6a1ed3 h1:HQa4+yCYsLq1TLM0kopeAhSCLbtZ541cWEi5N5rO+9g= github.com/filecoin-project/go-bitfield v0.2.1-0.20200920172649-837cbe6a1ed3/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM= github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:av5fw6wmm58FYMgJeoB/lK9XXrgdugYiTqkdxjTy9k8= @@ -283,6 +291,9 @@ github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 h1:EzDjxMg43q1tA2c0MV3tNbaontnHLplHyFF6M5KiVP0= github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1/go.mod h1:0eHX/BVySxPc6SE2mZRoppGq7qcEagxdmQnA3dzork8= +github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JYMGs= +github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= +github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9/go.mod h1:106OIgooyS7OzLDOpUGgm9fA3bQENb/cFSyyBmMoJDs= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= @@ -351,6 +362,8 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf h1:gFVkHXmVAhEbxZVDln5V9GKrLaluNoFHDbrZwAWZgws= +github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -491,6 +504,8 @@ github.com/ipfs/go-datastore v0.4.1/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13X github.com/ipfs/go-datastore v0.4.2/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= github.com/ipfs/go-datastore v0.4.4 h1:rjvQ9+muFaJ+QZ7dN5B1MSDNQ0JVZKkkES/rMZmA8X8= github.com/ipfs/go-datastore v0.4.4/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= +github.com/ipfs/go-datastore v0.4.5 h1:cwOUcGMLdLPWgu3SlrCckCMznaGADbPqE0r8h768/Dg= +github.com/ipfs/go-datastore v0.4.5/go.mod h1:eXTcaaiN6uOlVCLS9GjJUJtlvJfM3xk23w3fyfrmmJs= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= github.com/ipfs/go-ds-badger v0.0.2/go.mod h1:Y3QpeSFWQf6MopLTiZD+VT6IC1yZqaGmjvRcKeSGij8= @@ -509,6 +524,8 @@ github.com/ipfs/go-ds-leveldb v0.4.2 h1:QmQoAJ9WkPMUfBLnu1sBVy0xWWlJPg0m4kRAiJL9 github.com/ipfs/go-ds-leveldb v0.4.2/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s= github.com/ipfs/go-ds-measure v0.1.0 h1:vE4TyY4aeLeVgnnPBC5QzKIjKrqzha0NCujTfgvVbVQ= github.com/ipfs/go-ds-measure v0.1.0/go.mod h1:1nDiFrhLlwArTME1Ees2XaBOl49OoCgd2A3f8EchMSY= +github.com/ipfs/go-ds-pebble v0.0.2-0.20200921211847-f1ffb3128b61 h1:2wNNdpETSZgnsgy7wx7O6ueu+LCSZRedWrAsIPiOeFE= +github.com/ipfs/go-ds-pebble v0.0.2-0.20200921211847-f1ffb3128b61/go.mod h1:oh4liWHulKcDKVhCska5NLelE3MatWl+1FwSz3tY91g= github.com/ipfs/go-filestore v1.0.0 h1:QR7ekKH+q2AGiWDc7W2Q0qHuYSRZGUJqUn0GsegEPb0= github.com/ipfs/go-filestore v1.0.0/go.mod h1:/XOCuNtIe2f1YPbiXdYvD0BKLA0JR1MgPiFOdcuu9SM= github.com/ipfs/go-fs-lock v0.0.6 h1:sn3TWwNVQqSeNjlWy6zQ1uUGAZrV3hPOyEA6y1/N2a0= @@ -1521,6 +1538,8 @@ golang.org/x/exp v0.0.0-20200207192155-f17229e696bd h1:zkO/Lhoka23X63N9OSzpSeROE golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 h1:QE6XYQK6naiK1EPAe1g/ILLxN5RBoH5xkJk3CqlMI/Y= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20200513190911-00229845015e h1:rMqLP+9XLy+LdbCXHjJHAmTfXCr93W7oruWA6Hq1Alc= +golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= From 55c6b88537dd9f7b5db9aa601127634b9d13c96b Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Tue, 22 Sep 2020 00:46:31 +0200 Subject: [PATCH 433/885] Add toggle for badger, flag out gas tracing Signed-off-by: Jakub Sztandera --- chain/vm/runtime.go | 56 ++++++++++++++++++++----------------- chain/vm/vm.go | 19 +++++++++---- cmd/lotus-bench/import.go | 58 +++++++++++++++++++++------------------ 3 files changed, 75 insertions(+), 58 deletions(-) diff --git a/chain/vm/runtime.go b/chain/vm/runtime.go index eb4476718..72dd413ed 100644 --- a/chain/vm/runtime.go +++ b/chain/vm/runtime.go @@ -459,8 +459,10 @@ func (rt *Runtime) stateCommit(oldh, newh cid.Cid) aerrors.ActorError { } func (rt *Runtime) finilizeGasTracing() { - if rt.lastGasCharge != nil { - rt.lastGasCharge.TimeTaken = time.Since(rt.lastGasChargeTime) + if enableTracing { + if rt.lastGasCharge != nil { + rt.lastGasCharge.TimeTaken = time.Since(rt.lastGasChargeTime) + } } } @@ -489,35 +491,39 @@ func (rt *Runtime) chargeGasFunc(skip int) func(GasCharge) { } +var enableTracing = false + func (rt *Runtime) chargeGasInternal(gas GasCharge, skip int) aerrors.ActorError { toUse := gas.Total() - var callers [10]uintptr + if enableTracing { + var callers [10]uintptr - cout := 0 //gruntime.Callers(2+skip, callers[:]) + cout := 0 //gruntime.Callers(2+skip, callers[:]) - now := build.Clock.Now() - if rt.lastGasCharge != nil { - rt.lastGasCharge.TimeTaken = now.Sub(rt.lastGasChargeTime) + now := build.Clock.Now() + if rt.lastGasCharge != nil { + rt.lastGasCharge.TimeTaken = now.Sub(rt.lastGasChargeTime) + } + + gasTrace := types.GasTrace{ + Name: gas.Name, + Extra: gas.Extra, + + TotalGas: toUse, + ComputeGas: gas.ComputeGas, + StorageGas: gas.StorageGas, + + TotalVirtualGas: gas.VirtualCompute*GasComputeMulti + gas.VirtualStorage*GasStorageMulti, + VirtualComputeGas: gas.VirtualCompute, + VirtualStorageGas: gas.VirtualStorage, + + Callers: callers[:cout], + } + rt.executionTrace.GasCharges = append(rt.executionTrace.GasCharges, &gasTrace) + rt.lastGasChargeTime = now + rt.lastGasCharge = &gasTrace } - gasTrace := types.GasTrace{ - Name: gas.Name, - Extra: gas.Extra, - - TotalGas: toUse, - ComputeGas: gas.ComputeGas, - StorageGas: gas.StorageGas, - - TotalVirtualGas: gas.VirtualCompute*GasComputeMulti + gas.VirtualStorage*GasStorageMulti, - VirtualComputeGas: gas.VirtualCompute, - VirtualStorageGas: gas.VirtualStorage, - - Callers: callers[:cout], - } - rt.executionTrace.GasCharges = append(rt.executionTrace.GasCharges, &gasTrace) - rt.lastGasChargeTime = now - rt.lastGasCharge = &gasTrace - // overflow safe if rt.gasUsed > rt.gasAvailable-toUse { rt.gasUsed = rt.gasAvailable diff --git a/chain/vm/vm.go b/chain/vm/vm.go index 54ea47698..c566ec1eb 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -227,14 +227,21 @@ func (vm *VM) send(ctx context.Context, msg *types.Message, parent *Runtime, } rt := vm.makeRuntime(ctx, msg, origin, on, gasUsed, nac) - rt.lastGasChargeTime = start + if enableTracing { + rt.lastGasChargeTime = start + if parent != nil { + rt.lastGasChargeTime = parent.lastGasChargeTime + rt.lastGasCharge = parent.lastGasCharge + defer func() { + parent.lastGasChargeTime = rt.lastGasChargeTime + parent.lastGasCharge = rt.lastGasCharge + }() + } + } + if parent != nil { - rt.lastGasChargeTime = parent.lastGasChargeTime - rt.lastGasCharge = parent.lastGasCharge defer func() { - parent.gasUsed = rt.gasUsed - parent.lastGasChargeTime = rt.lastGasChargeTime - parent.lastGasCharge = rt.lastGasCharge + parent.gasUsed += rt.gasUsed }() } if gasCharge != nil { diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index b535ed96c..c3554f939 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -119,33 +119,37 @@ var importBenchCmd = &cli.Command{ tdir = tmp } - bdgOpt := badger.DefaultOptions - bdgOpt.GcInterval = 0 - bdgOpt.Options = bdg.DefaultOptions("") - bdgOpt.Options.SyncWrites = false - bdgOpt.Options.Truncate = true - bdgOpt.Options.DetectConflicts = false - - cache := 512 - bds, err := pebbleds.NewDatastore(tdir, &pebble.Options{ - // Pebble has a single combined cache area and the write - // buffers are taken from this too. Assign all available - // memory allowance for cache. - Cache: pebble.NewCache(int64(cache * 1024 * 1024)), - // The size of memory table(as well as the write buffer). - // Note, there may have more than two memory tables in the system. - // MemTableStopWritesThreshold can be configured to avoid the memory abuse. - MemTableSize: cache * 1024 * 1024 / 4, - // The default compaction concurrency(1 thread), - // Here use all available CPUs for faster compaction. - MaxConcurrentCompactions: runtime.NumCPU(), - // Per-level options. Options for at least one level must be specified. The - // options for the last level are used for all subsequent levels. - Levels: []pebble.LevelOptions{ - {TargetFileSize: 2 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)}, - }, - Logger: log, - }) + var bds datastore.Batching + if false { + cache := 512 + bds, err = pebbleds.NewDatastore(tdir, &pebble.Options{ + // Pebble has a single combined cache area and the write + // buffers are taken from this too. Assign all available + // memory allowance for cache. + Cache: pebble.NewCache(int64(cache * 1024 * 1024)), + // The size of memory table(as well as the write buffer). + // Note, there may have more than two memory tables in the system. + // MemTableStopWritesThreshold can be configured to avoid the memory abuse. + MemTableSize: cache * 1024 * 1024 / 4, + // The default compaction concurrency(1 thread), + // Here use all available CPUs for faster compaction. + MaxConcurrentCompactions: runtime.NumCPU(), + // Per-level options. Options for at least one level must be specified. The + // options for the last level are used for all subsequent levels. + Levels: []pebble.LevelOptions{ + {TargetFileSize: 2 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)}, + }, + Logger: log, + }) + } else { + bdgOpt := badger.DefaultOptions + bdgOpt.GcInterval = 0 + bdgOpt.Options = bdg.DefaultOptions("") + bdgOpt.Options.SyncWrites = false + bdgOpt.Options.Truncate = true + bdgOpt.Options.DetectConflicts = false + bds, err = badger.NewDatastore(tdir, &bdgOpt) + } if err != nil { return err } From ff8c0af8c82c022b86fd7f1043460711c31a6c43 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Tue, 22 Sep 2020 00:54:11 +0200 Subject: [PATCH 434/885] Add only-import option Signed-off-by: Jakub Sztandera --- cmd/lotus-bench/import.go | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index c3554f939..6af1bfc02 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -90,6 +90,9 @@ var importBenchCmd = &cli.Command{ &cli.Int64Flag{ Name: "start-at", }, + &cli.BoolFlag{ + Name: "only-import", + }, }, Action: func(cctx *cli.Context) error { vm.BatchSealVerifyParallelism = cctx.Int("batch-seal-verify-threads") @@ -119,6 +122,13 @@ var importBenchCmd = &cli.Command{ tdir = tmp } + bdgOpt := badger.DefaultOptions + bdgOpt.GcInterval = 0 + bdgOpt.Options = bdg.DefaultOptions("") + bdgOpt.Options.SyncWrites = false + bdgOpt.Options.Truncate = true + bdgOpt.Options.DetectConflicts = false + var bds datastore.Batching if false { cache := 512 @@ -142,17 +152,12 @@ var importBenchCmd = &cli.Command{ Logger: log, }) } else { - bdgOpt := badger.DefaultOptions - bdgOpt.GcInterval = 0 - bdgOpt.Options = bdg.DefaultOptions("") - bdgOpt.Options.SyncWrites = false - bdgOpt.Options.Truncate = true - bdgOpt.Options.DetectConflicts = false bds, err = badger.NewDatastore(tdir, &bdgOpt) } if err != nil { return err } + defer bds.Close() if cctx.Bool("only-gc") { log.Info("calling CollectGarbage on main ds") @@ -176,6 +181,7 @@ var importBenchCmd = &cli.Command{ if err != nil { return xerrors.Errorf("opening syscall-cache datastore: %w", err) } + defer scds.Close() if cctx.Bool("only-gc") { log.Info("calling CollectGarbage on syscall ds") @@ -223,6 +229,10 @@ var importBenchCmd = &cli.Command{ } } + if cctx.Bool("only-import") { + return nil + } + gb, err := cs.GetTipsetByHeight(context.TODO(), 0, head, true) if err != nil { return err From 76db65b1afcfa3f752ff51afcdc9021bacd38415 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Tue, 22 Sep 2020 00:57:37 +0200 Subject: [PATCH 435/885] Update pebble Signed-off-by: Jakub Sztandera --- chain/vm/vm.go | 2 +- cmd/lotus-bench/import.go | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/chain/vm/vm.go b/chain/vm/vm.go index c566ec1eb..44979454f 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -241,7 +241,7 @@ func (vm *VM) send(ctx context.Context, msg *types.Message, parent *Runtime, if parent != nil { defer func() { - parent.gasUsed += rt.gasUsed + parent.gasUsed = rt.gasUsed }() } if gasCharge != nil { diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index 6af1bfc02..94dae0d98 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -130,7 +130,7 @@ var importBenchCmd = &cli.Command{ bdgOpt.Options.DetectConflicts = false var bds datastore.Batching - if false { + if true { cache := 512 bds, err = pebbleds.NewDatastore(tdir, &pebble.Options{ // Pebble has a single combined cache area and the write @@ -147,7 +147,7 @@ var importBenchCmd = &cli.Command{ // Per-level options. Options for at least one level must be specified. The // options for the last level are used for all subsequent levels. Levels: []pebble.LevelOptions{ - {TargetFileSize: 2 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)}, + {TargetFileSize: 16 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10), Compression: pebble.NoCompression}, }, Logger: log, }) diff --git a/go.mod b/go.mod index 3b3b4fed7..17cb2fd36 100644 --- a/go.mod +++ b/go.mod @@ -58,7 +58,7 @@ require ( github.com/ipfs/go-ds-badger2 v0.1.1-0.20200708190120-187fc06f714e github.com/ipfs/go-ds-leveldb v0.4.2 github.com/ipfs/go-ds-measure v0.1.0 - github.com/ipfs/go-ds-pebble v0.0.2-0.20200921211847-f1ffb3128b61 + github.com/ipfs/go-ds-pebble v0.0.2-0.20200921225637-ce220f8ac459 github.com/ipfs/go-filestore v1.0.0 github.com/ipfs/go-fs-lock v0.0.6 github.com/ipfs/go-graphsync v0.2.1 diff --git a/go.sum b/go.sum index 19e33d294..6929c9a23 100644 --- a/go.sum +++ b/go.sum @@ -524,8 +524,8 @@ github.com/ipfs/go-ds-leveldb v0.4.2 h1:QmQoAJ9WkPMUfBLnu1sBVy0xWWlJPg0m4kRAiJL9 github.com/ipfs/go-ds-leveldb v0.4.2/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s= github.com/ipfs/go-ds-measure v0.1.0 h1:vE4TyY4aeLeVgnnPBC5QzKIjKrqzha0NCujTfgvVbVQ= github.com/ipfs/go-ds-measure v0.1.0/go.mod h1:1nDiFrhLlwArTME1Ees2XaBOl49OoCgd2A3f8EchMSY= -github.com/ipfs/go-ds-pebble v0.0.2-0.20200921211847-f1ffb3128b61 h1:2wNNdpETSZgnsgy7wx7O6ueu+LCSZRedWrAsIPiOeFE= -github.com/ipfs/go-ds-pebble v0.0.2-0.20200921211847-f1ffb3128b61/go.mod h1:oh4liWHulKcDKVhCska5NLelE3MatWl+1FwSz3tY91g= +github.com/ipfs/go-ds-pebble v0.0.2-0.20200921225637-ce220f8ac459 h1:W3YMLEvOXqdW+sYMiguhWP6txJwQvIQqhvpU8yAMGQs= +github.com/ipfs/go-ds-pebble v0.0.2-0.20200921225637-ce220f8ac459/go.mod h1:oh4liWHulKcDKVhCska5NLelE3MatWl+1FwSz3tY91g= github.com/ipfs/go-filestore v1.0.0 h1:QR7ekKH+q2AGiWDc7W2Q0qHuYSRZGUJqUn0GsegEPb0= github.com/ipfs/go-filestore v1.0.0/go.mod h1:/XOCuNtIe2f1YPbiXdYvD0BKLA0JR1MgPiFOdcuu9SM= github.com/ipfs/go-fs-lock v0.0.6 h1:sn3TWwNVQqSeNjlWy6zQ1uUGAZrV3hPOyEA6y1/N2a0= From 0d914ac1d4c4d0e8955c7074e9dd03eb49f44900 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Tue, 22 Sep 2020 22:01:41 +0200 Subject: [PATCH 436/885] Switch to badger Signed-off-by: Jakub Sztandera --- cmd/lotus-bench/import.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index 94dae0d98..c8328684e 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -130,7 +130,7 @@ var importBenchCmd = &cli.Command{ bdgOpt.Options.DetectConflicts = false var bds datastore.Batching - if true { + if false { cache := 512 bds, err = pebbleds.NewDatastore(tdir, &pebble.Options{ // Pebble has a single combined cache area and the write From 7e8c6e507055f1f75607facbc0d0220608540343 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 23 Sep 2020 18:44:41 +0200 Subject: [PATCH 437/885] Remove statediff, fix lint, go mod tidy Signed-off-by: Jakub Sztandera --- cmd/lotus-bench/import.go | 20 +++----------------- go.mod | 7 ++++--- go.sum | 23 ----------------------- 3 files changed, 7 insertions(+), 43 deletions(-) diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index c8328684e..3d93b0e5e 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -30,7 +30,6 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" - "github.com/filecoin-project/statediff" bdg "github.com/dgraph-io/badger/v2" "github.com/ipfs/go-datastore" @@ -80,9 +79,6 @@ var importBenchCmd = &cli.Command{ Name: "no-import", Usage: "should we import the chain? if set to true chain has to be previously imported", }, - &cli.BoolFlag{ - Name: "only-gc", - }, &cli.BoolFlag{ Name: "global-profile", Value: true, @@ -157,13 +153,8 @@ var importBenchCmd = &cli.Command{ if err != nil { return err } - defer bds.Close() + defer bds.Close() //nolint:errcheck - if cctx.Bool("only-gc") { - log.Info("calling CollectGarbage on main ds") - //bds.CollectGarbage() - log.Info("done calling CollectGarbage on main ds") - } bs := blockstore.NewBlockstore(bds) cacheOpts := blockstore.DefaultCacheOpts() cacheOpts.HasBloomFilterSize = 0 @@ -181,13 +172,8 @@ var importBenchCmd = &cli.Command{ if err != nil { return xerrors.Errorf("opening syscall-cache datastore: %w", err) } - defer scds.Close() + defer scds.Close() //nolint:errcheck - if cctx.Bool("only-gc") { - log.Info("calling CollectGarbage on syscall ds") - scds.CollectGarbage() - log.Info("done calling CollectGarbage on syscall ds") - } verifier = &cachingVerifier{ ds: scds, backend: verifier, @@ -317,7 +303,7 @@ var importBenchCmd = &cli.Command{ } fmt.Println("TRACE") fmt.Println(string(d)) - fmt.Println(statediff.Diff(context.Background(), bs, tschain[i-1].ParentState(), st, statediff.ExpandActors)) + //fmt.Println(statediff.Diff(context.Background(), bs, tschain[i-1].ParentState(), st, statediff.ExpandActors)) return xerrors.Errorf("tipset chain had state mismatch at height %d (%s != %s)", cur.Height(), cur.ParentState(), st) } } diff --git a/go.mod b/go.mod index 17cb2fd36..b8896d78f 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/BurntSushi/toml v0.3.1 github.com/GeertJohan/go.rice v1.0.0 github.com/Gurpartap/async v0.0.0-20180927173644-4f7f499dd9ee + github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d github.com/buger/goterm v0.0.0-20200322175922-2f3e71b85129 github.com/cockroachdb/pebble v0.0.0-20200916222308-4e219a90ba5b @@ -38,10 +39,10 @@ require ( github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b github.com/filecoin-project/specs-actors v0.9.11 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 - github.com/filecoin-project/statediff v0.0.6-0.20200918150628-da86dd0d264c github.com/filecoin-project/test-vectors/schema v0.0.1 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 github.com/go-kit/kit v0.10.0 + github.com/go-ole/go-ole v1.2.4 // indirect github.com/google/uuid v1.1.1 github.com/gorilla/mux v1.7.4 github.com/gorilla/websocket v1.4.2 @@ -118,6 +119,7 @@ require ( github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542 go.opencensus.io v0.22.4 + go.uber.org/dig v1.10.0 // indirect go.uber.org/fx v1.9.0 go.uber.org/multierr v1.5.0 go.uber.org/zap v1.15.0 @@ -127,12 +129,11 @@ require ( golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 gopkg.in/cheggaaa/pb.v1 v1.0.28 gotest.tools v2.2.0+incompatible + launchpad.net/gocheck v0.0.0-20140225173054-000000000087 // indirect ) replace github.com/filecoin-project/lotus => ./ -replace github.com/filecoin-project/statediff => ./../statediff - replace github.com/golangci/golangci-lint => github.com/golangci/golangci-lint v1.18.0 replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi diff --git a/go.sum b/go.sum index 6929c9a23..cbceb52c6 100644 --- a/go.sum +++ b/go.sum @@ -169,8 +169,6 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018/go.mod h1:rQYf4tfk5sSwFsnDg3qYaBxSjsD9S8+59vW0dKUgme4= github.com/davidlazar/go-crypto v0.0.0-20190912175916-7055855a373f h1:BOaYiTvg8p9vBUXpklC22XSK/mifLF7lG9jtmYYi3Tc= github.com/davidlazar/go-crypto v0.0.0-20190912175916-7055855a373f/go.mod h1:rQYf4tfk5sSwFsnDg3qYaBxSjsD9S8+59vW0dKUgme4= -github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c h1:pFUpOrbxDR6AkioZ1ySsx5yxlDQZ8stG2b88gTPxgJU= -github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c/go.mod h1:6UhI8N9EjYm1c2odKpFpAYeR8dsBeM7PtzQhRgxRr9U= github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e h1:lj77EKYUpYXTd8CD/+QMIf8b6OIOTsfEBSXiAzuEHTU= github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e/go.mod h1:3ZQK6DMPSz/QZ73jlWxBtUhNA8xZx7LzUFSq/OfP8vk= github.com/dgraph-io/badger v1.5.5-0.20190226225317-8115aed38f8f/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ= @@ -218,7 +216,6 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanw/esbuild v0.6.28/go.mod h1:mptxmSXIzBIKKCe4jo9A5SToEd1G+AKZ9JmY85dYRJ0= github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 h1:BBso6MBKW8ncyZLv37o+KNyy0HrrHgfnOaGQC2qvN+A= github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5/go.mod h1:JpoxHjuQauoxiFMl1ie8Xc/7TfLuMZ5eOCONd1sUBHg= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -231,8 +228,6 @@ github.com/filecoin-project/go-address v0.0.4 h1:gSNMv0qWwH16fGQs7ycOUrDjY6YCSsg github.com/filecoin-project/go-address v0.0.4/go.mod h1:jr8JxKsYx+lQlQZmF5i2U0Z+cGQ59wMIps/8YW/lDj8= github.com/filecoin-project/go-amt-ipld/v2 v2.1.0 h1:t6qDiuGYYngDqaLc2ZUvdtAg4UNxPeOYaXhBWSNsVaM= github.com/filecoin-project/go-amt-ipld/v2 v2.1.0/go.mod h1:nfFPoGyX0CU9SkXX8EoCcSuHN1XcbN0c6KBh7yvP5fs= -github.com/filecoin-project/go-amt-ipld/v2 v2.1.1-0.20200731171407-e559a0579161 h1:K6t4Hrs+rwUxBz2xg88Bdqeh4k5/rycQFdPseZhRyfE= -github.com/filecoin-project/go-amt-ipld/v2 v2.1.1-0.20200731171407-e559a0579161/go.mod h1:vgmwKBkx+ca5OIeEvstiQgzAZnb7R6QaqE1oEDSqa6g= github.com/filecoin-project/go-bitfield v0.2.0 h1:gCtLcjskIPtdg4NfN7gQZSQF9yrBQ7mkT0qCJxzGI2Q= github.com/filecoin-project/go-bitfield v0.2.0/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM= github.com/filecoin-project/go-bitfield v0.2.1-0.20200920172649-837cbe6a1ed3 h1:HQa4+yCYsLq1TLM0kopeAhSCLbtZ541cWEi5N5rO+9g= @@ -720,7 +715,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lib/pq v1.7.0 h1:h93mCPfUSkaul3Ka/VG8uZdmW1uMHDGxzu0NWHuJmHY= github.com/lib/pq v1.7.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ= @@ -1048,8 +1042,6 @@ github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= @@ -1352,8 +1344,6 @@ github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5J github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -1406,7 +1396,6 @@ github.com/whyrusleeping/cbor-gen v0.0.0-20200414195334-429a0b5e922e/go.mod h1:X github.com/whyrusleeping/cbor-gen v0.0.0-20200504204219-64967432584d/go.mod h1:W5MvapuoHRP8rz4vxjwCK1pDqF1aQcWsV5PZ+AHbqdg= github.com/whyrusleeping/cbor-gen v0.0.0-20200710004633-5379fc63235d/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200715143311-227fab5a2377/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= -github.com/whyrusleeping/cbor-gen v0.0.0-20200723185710-6a3894a6352b/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200810223238-211df3b9e24c/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200812213548-958ddffe352c/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200814224545-656e08ce49ee h1:U7zWWvvAjT76EiuWPSOiZlQDnaQYPxPoxugTtTAcJK0= @@ -1433,8 +1422,6 @@ github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d/go.mod h1:g7c github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee h1:lYbXeSvJi5zk5GLKVuid9TVjS9a0OmLIDKTfoZBL6Ow= github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee/go.mod h1:m2aV4LZI4Aez7dP5PMyVKEHhUyEJ/RjmPEDOpDvudHg= github.com/whyrusleeping/yamux v1.1.5/go.mod h1:E8LnQQ8HKx5KD29HZFUwM1PxCOdPRzGwur1mcYhXcD8= -github.com/willscott/go-cmp v0.5.2-0.20200812183318-8affb9542345 h1:IJVAwIctqDFOrO0C2qzksXmANviyHJzrklU27e1ltzE= -github.com/willscott/go-cmp v0.5.2-0.20200812183318-8affb9542345/go.mod h1:D7hA8H5pyQx7Y5Em7IWx1R4vNJzfon3gpG9nxjkITjQ= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/c-for-go v0.0.0-20200718154222-87b0065af829 h1:wb7xrDzfkLgPHsSEBm+VSx6aDdi64VtV0xvP0E6j8bk= @@ -1523,7 +1510,6 @@ golang.org/x/crypto v0.0.0-20200128174031-69ecbb4d6d5d/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -1536,8 +1522,6 @@ golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd h1:zkO/Lhoka23X63N9OSzpSeROEUQ5ODw47tM3YWjygbs= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 h1:QE6XYQK6naiK1EPAe1g/ILLxN5RBoH5xkJk3CqlMI/Y= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20200513190911-00229845015e h1:rMqLP+9XLy+LdbCXHjJHAmTfXCr93W7oruWA6Hq1Alc= golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= @@ -1553,8 +1537,6 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367 h1:0IiAsCRByjO2QjX7ZPkw5oU9x+n1YqRL802rjC0c3Aw= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -1672,7 +1654,6 @@ golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1681,7 +1662,6 @@ golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200509044756-6aff5f38e54f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 h1:OjiUf46hAmXblsZdnoSXsEUSKU8r1UEzcL5RVZ4gO9Y= @@ -1725,7 +1705,6 @@ golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -1854,8 +1833,6 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqpnJdvzuiHsl/dnxl11M= howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= launchpad.net/gocheck v0.0.0-20140225173054-000000000087 h1:Izowp2XBH6Ya6rv+hqbceQyw/gSGoXfH/UPoTGduL54= From 247a5e2c496eaae153cab85d88335db922e36ed0 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 24 Sep 2020 17:49:58 +0200 Subject: [PATCH 438/885] Go mod tidy Signed-off-by: Jakub Sztandera --- go.sum | 4 ---- 1 file changed, 4 deletions(-) diff --git a/go.sum b/go.sum index cbceb52c6..0f309ce1a 100644 --- a/go.sum +++ b/go.sum @@ -528,10 +528,6 @@ github.com/ipfs/go-fs-lock v0.0.6/go.mod h1:OTR+Rj9sHiRubJh3dRhD15Juhd/+w6VPOY28 github.com/ipfs/go-graphsync v0.1.0/go.mod h1:jMXfqIEDFukLPZHqDPp8tJMbHO9Rmeb9CEGevngQbmE= github.com/ipfs/go-graphsync v0.2.1 h1:MdehhqBSuTI2LARfKLkpYnt0mUrqHs/mtuDnESXHBfU= github.com/ipfs/go-graphsync v0.2.1/go.mod h1:gEBvJUNelzMkaRPJTpg/jaKN4AQW/7wDWu0K92D8o10= -github.com/ipfs/go-graphsync v0.2.0 h1:x94MvHLNuRwBlZzVal7tR1RYK7T7H6bqQLPopxDbIF0= -github.com/ipfs/go-graphsync v0.2.0/go.mod h1:gEBvJUNelzMkaRPJTpg/jaKN4AQW/7wDWu0K92D8o10= -github.com/ipfs/go-graphsync v0.1.2 h1:25Ll9kIXCE+DY0dicvfS3KMw+U5sd01b/FJbA7KAbhg= -github.com/ipfs/go-graphsync v0.1.2/go.mod h1:sLXVXm1OxtE2XYPw62MuXCdAuNwkAdsbnfrmos5odbA= github.com/ipfs/go-hamt-ipld v0.1.1 h1:0IQdvwnAAUKmDE+PMJa5y1QiwOPHpI9+eAbQEEEYthk= github.com/ipfs/go-hamt-ipld v0.1.1/go.mod h1:1EZCr2v0jlCnhpa+aZ0JZYp8Tt2w16+JJOAVz17YcDk= github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= From baef3c8dd26318fb435c894bfd030e7daf68834a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 29 Sep 2020 15:22:46 +0200 Subject: [PATCH 439/885] sectorstorage: Fix potential panic in FinalizeSector --- .../sector-storage/ffiwrapper/sealer_cgo.go | 47 ++++++++++--------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/extern/sector-storage/ffiwrapper/sealer_cgo.go b/extern/sector-storage/ffiwrapper/sealer_cgo.go index 9bc2680ed..d75501838 100644 --- a/extern/sector-storage/ffiwrapper/sealer_cgo.go +++ b/extern/sector-storage/ffiwrapper/sealer_cgo.go @@ -546,34 +546,37 @@ func (sb *Sealer) FinalizeSector(ctx context.Context, sector abi.SectorID, keepU defer done() pf, err := openPartialFile(maxPieceSize, paths.Unsealed) - if xerrors.Is(err, os.ErrNotExist) { - return xerrors.Errorf("opening partial file: %w", err) - } + if err == nil { + var at uint64 + for sr.HasNext() { + r, err := sr.NextRun() + if err != nil { + _ = pf.Close() + return err + } - var at uint64 - for sr.HasNext() { - r, err := sr.NextRun() - if err != nil { - _ = pf.Close() + offset := at + at += r.Len + if !r.Val { + continue + } + + err = pf.Free(storiface.PaddedByteIndex(abi.UnpaddedPieceSize(offset).Padded()), abi.UnpaddedPieceSize(r.Len).Padded()) + if err != nil { + _ = pf.Close() + return xerrors.Errorf("free partial file range: %w", err) + } + } + + if err := pf.Close(); err != nil { return err } - - offset := at - at += r.Len - if !r.Val { - continue - } - - err = pf.Free(storiface.PaddedByteIndex(abi.UnpaddedPieceSize(offset).Padded()), abi.UnpaddedPieceSize(r.Len).Padded()) - if err != nil { - _ = pf.Close() - return xerrors.Errorf("free partial file range: %w", err) + } else { + if !xerrors.Is(err, os.ErrNotExist) { + return xerrors.Errorf("opening partial file: %w", err) } } - if err := pf.Close(); err != nil { - return err - } } paths, done, err := sb.sectors.AcquireSector(ctx, sector, stores.FTCache, 0, stores.PathStorage) From 9fe32b7777af8037e1572ccb84feb639970b049c Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Wed, 30 Sep 2020 01:10:03 -0400 Subject: [PATCH 440/885] Fix wallet list --- cli/wallet.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cli/wallet.go b/cli/wallet.go index 0d69673f9..aa5b9bed3 100644 --- a/cli/wallet.go +++ b/cli/wallet.go @@ -89,10 +89,8 @@ var walletList = &cli.Command{ return err } - def, err := api.WalletDefaultAddress(ctx) - if err != nil { - return err - } + // Assume an error means no default key is set + def, _ := api.WalletDefaultAddress(ctx) tw := tablewriter.New( tablewriter.Col("Address"), From a4e71174297138fdb4d25240fafe4b90abfc656d Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Sun, 27 Sep 2020 17:52:26 -0400 Subject: [PATCH 441/885] Add lotus shed util to validate a tipset --- api/api_full.go | 3 ++ api/apistruct/struct.go | 5 ++++ chain/store/store.go | 10 +++++++ cmd/lotus-shed/main.go | 1 + cmd/lotus-shed/sync.go | 64 +++++++++++++++++++++++++++++++++++++++++ node/impl/full/sync.go | 26 +++++++++++++++++ 6 files changed, 109 insertions(+) create mode 100644 cmd/lotus-shed/sync.go diff --git a/api/api_full.go b/api/api_full.go index 6d2d0c7b5..0e5622f4c 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -176,6 +176,9 @@ type FullNode interface { // the reason. SyncCheckBad(ctx context.Context, bcid cid.Cid) (string, error) + // SyncValidateTipset indicates whether the provided tipset is valid or not + SyncValidateTipset(ctx context.Context, tsk types.TipSetKey) (bool, error) + // MethodGroup: Mpool // The Mpool methods are for interacting with the message pool. The message pool // manages all incoming and outgoing 'messages' going over the network. diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index d5b6950ad..cc2b8b5b5 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -112,6 +112,7 @@ type FullNodeStruct struct { SyncMarkBad func(ctx context.Context, bcid cid.Cid) error `perm:"admin"` SyncUnmarkBad func(ctx context.Context, bcid cid.Cid) error `perm:"admin"` SyncCheckBad func(ctx context.Context, bcid cid.Cid) (string, error) `perm:"read"` + SyncValidateTipset func(ctx context.Context, tsk types.TipSetKey) (bool, error) `perm:"read"` MpoolGetConfig func(context.Context) (*types.MpoolConfig, error) `perm:"read"` MpoolSetConfig func(context.Context, *types.MpoolConfig) error `perm:"write"` @@ -735,6 +736,10 @@ func (c *FullNodeStruct) SyncCheckBad(ctx context.Context, bcid cid.Cid) (string return c.Internal.SyncCheckBad(ctx, bcid) } +func (c *FullNodeStruct) SyncValidateTipset(ctx context.Context, tsk types.TipSetKey) (bool, error) { + return c.Internal.SyncValidateTipset(ctx, tsk) +} + func (c *FullNodeStruct) StateNetworkName(ctx context.Context) (dtypes.NetworkName, error) { return c.Internal.StateNetworkName(ctx) } diff --git a/chain/store/store.go b/chain/store/store.go index 6c93db7a0..0806fb921 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -286,6 +286,16 @@ func (cs *ChainStore) MarkBlockAsValidated(ctx context.Context, blkid cid.Cid) e return nil } +func (cs *ChainStore) UnmarkBlockAsValidated(ctx context.Context, blkid cid.Cid) error { + key := blockValidationCacheKeyPrefix.Instance(blkid.String()) + + if err := cs.ds.Delete(key); err != nil { + return xerrors.Errorf("removing from valid block cache: %w", err) + } + + return nil +} + func (cs *ChainStore) SetGenesis(b *types.BlockHeader) error { ts, err := types.NewTipSet([]*types.BlockHeader{b}) if err != nil { diff --git a/cmd/lotus-shed/main.go b/cmd/lotus-shed/main.go index c7ded7a25..3864d3014 100644 --- a/cmd/lotus-shed/main.go +++ b/cmd/lotus-shed/main.go @@ -38,6 +38,7 @@ func main() { exportChainCmd, consensusCmd, serveDealStatsCmd, + syncCmd, } app := &cli.App{ diff --git a/cmd/lotus-shed/sync.go b/cmd/lotus-shed/sync.go new file mode 100644 index 000000000..bfe7cc8b7 --- /dev/null +++ b/cmd/lotus-shed/sync.go @@ -0,0 +1,64 @@ +package main + +import ( + "fmt" + + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lotus/chain/types" + lcli "github.com/filecoin-project/lotus/cli" + "github.com/urfave/cli/v2" +) + +var syncCmd = &cli.Command{ + Name: "sync", + Usage: "tools for diagnosing sync issues", + Flags: []cli.Flag{}, + Subcommands: []*cli.Command{ + syncValidateCmd, + }, +} + +var syncValidateCmd = &cli.Command{ + Name: "validate", + Usage: "checks whether a provided tipset is valid", + Action: func(cctx *cli.Context) error { + api, closer, err := lcli.GetFullNodeAPI(cctx) + if err != nil { + return err + } + + defer closer() + ctx := lcli.ReqContext(cctx) + + if cctx.Args().Len() < 1 { + fmt.Println("usage: ...") + fmt.Println("At least one block cid must be provided") + return nil + } + + args := cctx.Args().Slice() + + var tscids []cid.Cid + for _, s := range args { + c, err := cid.Decode(s) + if err != nil { + return fmt.Errorf("block cid was invalid: %s", err) + } + tscids = append(tscids, c) + } + + tsk := types.NewTipSetKey(tscids...) + + valid, err := api.SyncValidateTipset(ctx, tsk) + if err != nil { + fmt.Println("Tipset is invalid: ", err) + } + + if valid { + fmt.Println("Tipset is valid") + } + + return nil + }, +} diff --git a/node/impl/full/sync.go b/node/impl/full/sync.go index dc3bfe230..221942673 100644 --- a/node/impl/full/sync.go +++ b/node/impl/full/sync.go @@ -126,3 +126,29 @@ func (a *SyncAPI) SyncCheckBad(ctx context.Context, bcid cid.Cid) (string, error return reason, nil } + +func (a *SyncAPI) SyncValidateTipset(ctx context.Context, tsk types.TipSetKey) (bool, error) { + ts, err := a.Syncer.ChainStore().LoadTipSet(tsk) + if err != nil { + return false, err + } + + fts, err := a.Syncer.ChainStore().TryFillTipSet(ts) + if err != nil { + return false, err + } + + for _, blk := range tsk.Cids() { + err = a.Syncer.ChainStore().UnmarkBlockAsValidated(ctx, blk) + if err != nil { + return false, err + } + } + + err = a.Syncer.ValidateTipSet(ctx, fts) + if err != nil { + return false, err + } + + return true, nil +} From 73d193bd9cd1b4d5e9355058febed293087a9364 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Sun, 27 Sep 2020 18:08:58 -0400 Subject: [PATCH 442/885] Update docs --- documentation/en/api-methods.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index ed082ccbf..29271bdd5 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -169,6 +169,7 @@ * [SyncState](#SyncState) * [SyncSubmitBlock](#SyncSubmitBlock) * [SyncUnmarkBad](#SyncUnmarkBad) + * [SyncValidateTipset](#SyncValidateTipset) * [Wallet](#Wallet) * [WalletBalance](#WalletBalance) * [WalletDefaultAddress](#WalletDefaultAddress) @@ -4379,6 +4380,28 @@ Inputs: Response: `{}` +### SyncValidateTipset +SyncValidateTipset indicates whether the provided tipset is valid or not + + +Perms: read + +Inputs: +```json +[ + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `true` + ## Wallet From c45c8f34a16605a087bd4c6b5b022f8828784156 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Wed, 30 Sep 2020 01:39:06 -0400 Subject: [PATCH 443/885] Parametrise whether sync validators should use cache --- chain/sync.go | 28 ++++++++++++++++------------ chain/sync_test.go | 4 ++-- node/impl/full/sync.go | 9 +-------- 3 files changed, 19 insertions(+), 22 deletions(-) diff --git a/chain/sync.go b/chain/sync.go index 78e5178d1..b2e3bb7f1 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -597,7 +597,7 @@ func isPermanent(err error) bool { return !errors.Is(err, ErrTemporal) } -func (syncer *Syncer) ValidateTipSet(ctx context.Context, fts *store.FullTipSet) error { +func (syncer *Syncer) ValidateTipSet(ctx context.Context, fts *store.FullTipSet, useCache bool) error { ctx, span := trace.StartSpan(ctx, "validateTipSet") defer span.End() @@ -613,7 +613,7 @@ func (syncer *Syncer) ValidateTipSet(ctx context.Context, fts *store.FullTipSet) b := b // rebind to a scoped variable futures = append(futures, async.Err(func() error { - if err := syncer.ValidateBlock(ctx, b); err != nil { + if err := syncer.ValidateBlock(ctx, b, useCache); err != nil { if isPermanent(err) { syncer.bad.Add(b.Cid(), NewBadBlockReason([]cid.Cid{b.Cid()}, err.Error())) } @@ -680,7 +680,7 @@ func blockSanityChecks(h *types.BlockHeader) error { } // ValidateBlock should match up with 'Semantical Validation' in validation.md in the spec -func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) (err error) { +func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock, useCache bool) (err error) { defer func() { // b.Cid() could panic for empty blocks that are used in tests. if rerr := recover(); rerr != nil { @@ -689,13 +689,15 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) (er } }() - isValidated, err := syncer.store.IsBlockValidated(ctx, b.Cid()) - if err != nil { - return xerrors.Errorf("check block validation cache %s: %w", b.Cid(), err) - } + if useCache { + isValidated, err := syncer.store.IsBlockValidated(ctx, b.Cid()) + if err != nil { + return xerrors.Errorf("check block validation cache %s: %w", b.Cid(), err) + } - if isValidated { - return nil + if isValidated { + return nil + } } validationStart := build.Clock.Now() @@ -959,8 +961,10 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) (er return mulErr } - if err := syncer.store.MarkBlockAsValidated(ctx, b.Cid()); err != nil { - return xerrors.Errorf("caching block validation %s: %w", b.Cid(), err) + if useCache { + if err := syncer.store.MarkBlockAsValidated(ctx, b.Cid()); err != nil { + return xerrors.Errorf("caching block validation %s: %w", b.Cid(), err) + } } return nil @@ -1462,7 +1466,7 @@ func (syncer *Syncer) syncMessagesAndCheckState(ctx context.Context, headers []* return syncer.iterFullTipsets(ctx, headers, func(ctx context.Context, fts *store.FullTipSet) error { log.Debugw("validating tipset", "height", fts.TipSet().Height(), "size", len(fts.TipSet().Cids())) - if err := syncer.ValidateTipSet(ctx, fts); err != nil { + if err := syncer.ValidateTipSet(ctx, fts, true); err != nil { log.Errorf("failed to validate tipset: %+v", err) return xerrors.Errorf("message processing failed: %w", err) } diff --git a/chain/sync_test.go b/chain/sync_test.go index 7a839be2b..1b06f604b 100644 --- a/chain/sync_test.go +++ b/chain/sync_test.go @@ -732,7 +732,7 @@ func TestSyncInputs(t *testing.T) { err := s.ValidateBlock(context.TODO(), &types.FullBlock{ Header: &types.BlockHeader{}, - }) + }, false) if err == nil { t.Fatal("should error on empty block") } @@ -741,7 +741,7 @@ func TestSyncInputs(t *testing.T) { h.ElectionProof = nil - err = s.ValidateBlock(context.TODO(), &types.FullBlock{Header: h}) + err = s.ValidateBlock(context.TODO(), &types.FullBlock{Header: h}, false) if err == nil { t.Fatal("should error on block with nil election proof") } diff --git a/node/impl/full/sync.go b/node/impl/full/sync.go index 221942673..1bd3af415 100644 --- a/node/impl/full/sync.go +++ b/node/impl/full/sync.go @@ -138,14 +138,7 @@ func (a *SyncAPI) SyncValidateTipset(ctx context.Context, tsk types.TipSetKey) ( return false, err } - for _, blk := range tsk.Cids() { - err = a.Syncer.ChainStore().UnmarkBlockAsValidated(ctx, blk) - if err != nil { - return false, err - } - } - - err = a.Syncer.ValidateTipSet(ctx, fts) + err = a.Syncer.ValidateTipSet(ctx, fts, false) if err != nil { return false, err } From a388bcfad61d90dc5718982095c1707f6d858cdf Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Wed, 30 Sep 2020 01:43:10 -0400 Subject: [PATCH 444/885] Add an endpoint to validate whether a string is a well-formed address --- api/api_full.go | 2 ++ api/apistruct/struct.go | 29 +++++++++++++++++------------ node/impl/full/wallet.go | 4 ++++ 3 files changed, 23 insertions(+), 12 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 6d2d0c7b5..7b7574c26 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -244,6 +244,8 @@ type FullNode interface { WalletImport(context.Context, *types.KeyInfo) (address.Address, error) // WalletDelete deletes an address from the wallet. WalletDelete(context.Context, address.Address) error + // WalletValidateAddress validates whether a given string can be decoded as a well-formed address + WalletValidateAddress(context.Context, string) (address.Address, error) // Other diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index d5b6950ad..73c4d8ed5 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -129,18 +129,19 @@ type FullNodeStruct struct { MinerGetBaseInfo func(context.Context, address.Address, abi.ChainEpoch, types.TipSetKey) (*api.MiningBaseInfo, error) `perm:"read"` MinerCreateBlock func(context.Context, *api.BlockTemplate) (*types.BlockMsg, error) `perm:"write"` - WalletNew func(context.Context, crypto.SigType) (address.Address, error) `perm:"write"` - WalletHas func(context.Context, address.Address) (bool, error) `perm:"write"` - WalletList func(context.Context) ([]address.Address, error) `perm:"write"` - WalletBalance func(context.Context, address.Address) (types.BigInt, error) `perm:"read"` - WalletSign func(context.Context, address.Address, []byte) (*crypto.Signature, error) `perm:"sign"` - WalletSignMessage func(context.Context, address.Address, *types.Message) (*types.SignedMessage, error) `perm:"sign"` - WalletVerify func(context.Context, address.Address, []byte, *crypto.Signature) (bool, error) `perm:"read"` - WalletDefaultAddress func(context.Context) (address.Address, error) `perm:"write"` - WalletSetDefault func(context.Context, address.Address) error `perm:"admin"` - WalletExport func(context.Context, address.Address) (*types.KeyInfo, error) `perm:"admin"` - WalletImport func(context.Context, *types.KeyInfo) (address.Address, error) `perm:"admin"` - WalletDelete func(context.Context, address.Address) error `perm:"write"` + WalletNew func(context.Context, crypto.SigType) (address.Address, error) `perm:"write"` + WalletHas func(context.Context, address.Address) (bool, error) `perm:"write"` + WalletList func(context.Context) ([]address.Address, error) `perm:"write"` + WalletBalance func(context.Context, address.Address) (types.BigInt, error) `perm:"read"` + WalletSign func(context.Context, address.Address, []byte) (*crypto.Signature, error) `perm:"sign"` + WalletSignMessage func(context.Context, address.Address, *types.Message) (*types.SignedMessage, error) `perm:"sign"` + WalletVerify func(context.Context, address.Address, []byte, *crypto.Signature) (bool, error) `perm:"read"` + WalletDefaultAddress func(context.Context) (address.Address, error) `perm:"write"` + WalletSetDefault func(context.Context, address.Address) error `perm:"admin"` + WalletExport func(context.Context, address.Address) (*types.KeyInfo, error) `perm:"admin"` + WalletImport func(context.Context, *types.KeyInfo) (address.Address, error) `perm:"admin"` + WalletDelete func(context.Context, address.Address) error `perm:"write"` + WalletValidateAddress func(context.Context, string) (address.Address, error) `perm:"read"` ClientImport func(ctx context.Context, ref api.FileRef) (*api.ImportRes, error) `perm:"admin"` ClientListImports func(ctx context.Context) ([]api.Import, error) `perm:"write"` @@ -631,6 +632,10 @@ func (c *FullNodeStruct) WalletDelete(ctx context.Context, addr address.Address) return c.Internal.WalletDelete(ctx, addr) } +func (c *FullNodeStruct) WalletValidateAddress(ctx context.Context, str string) (address.Address, error) { + return c.Internal.WalletValidateAddress(ctx, str) +} + func (c *FullNodeStruct) MpoolGetNonce(ctx context.Context, addr address.Address) (uint64, error) { return c.Internal.MpoolGetNonce(ctx, addr) } diff --git a/node/impl/full/wallet.go b/node/impl/full/wallet.go index 64231b74e..b2ecdebbd 100644 --- a/node/impl/full/wallet.go +++ b/node/impl/full/wallet.go @@ -90,3 +90,7 @@ func (a *WalletAPI) WalletImport(ctx context.Context, ki *types.KeyInfo) (addres func (a *WalletAPI) WalletDelete(ctx context.Context, addr address.Address) error { return a.Wallet.DeleteKey(addr) } + +func (a *WalletAPI) WalletValidateAddress(ctx context.Context, str string) (address.Address, error) { + return address.NewFromString(str) +} From bc4cbdc8957e45daca3b24ed5f434ebf72ced58b Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Wed, 30 Sep 2020 01:45:03 -0400 Subject: [PATCH 445/885] Update docs --- documentation/en/api-methods.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index ed082ccbf..beaf7f3b6 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -181,6 +181,7 @@ * [WalletSetDefault](#WalletSetDefault) * [WalletSign](#WalletSign) * [WalletSignMessage](#WalletSignMessage) + * [WalletValidateAddress](#WalletValidateAddress) * [WalletVerify](#WalletVerify) ## @@ -4585,6 +4586,21 @@ Response: } ``` +### WalletValidateAddress +WalletValidateAddress validates whether a given string can be decoded as a well-formed address + + +Perms: read + +Inputs: +```json +[ + "string value" +] +``` + +Response: `"t01234"` + ### WalletVerify WalletVerify takes an address, a signature, and some bytes, and indicates whether the signature is valid. The address does not have to be in the wallet. From 1affd498c172ac7b29192b09f67be76b83fabe2e Mon Sep 17 00:00:00 2001 From: Dan Shao Date: Wed, 30 Sep 2020 14:23:35 +0800 Subject: [PATCH 446/885] Add --no-swap flag for worker --- cmd/lotus-seal-worker/main.go | 7 +++++++ extern/sector-storage/localworker.go | 10 +++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/cmd/lotus-seal-worker/main.go b/cmd/lotus-seal-worker/main.go index e36514bb8..d2c57e680 100644 --- a/cmd/lotus-seal-worker/main.go +++ b/cmd/lotus-seal-worker/main.go @@ -109,6 +109,11 @@ var runCmd = &cli.Command{ Name: "no-local-storage", Usage: "don't use storageminer repo for sector storage", }, + &cli.BoolFlag{ + Name: "no-swap", + Usage: "don't use swap", + Value: false, + }, &cli.BoolFlag{ Name: "addpiece", Usage: "enable addpiece", @@ -346,6 +351,7 @@ var runCmd = &cli.Command{ LocalWorker: sectorstorage.NewLocalWorker(sectorstorage.WorkerConfig{ SealProof: spt, TaskTypes: taskTypes, + NoSwap: cctx.Bool("no-swap"), }, remote, localStore, nodeApi), localStore: localStore, ls: lr, @@ -465,6 +471,7 @@ func watchMinerConn(ctx context.Context, cctx *cli.Context, nodeApi api.StorageM "run", fmt.Sprintf("--listen=%s", cctx.String("listen")), fmt.Sprintf("--no-local-storage=%t", cctx.Bool("no-local-storage")), + fmt.Sprintf("--no-swap=%t", cctx.Bool("no-swap")), fmt.Sprintf("--addpiece=%t", cctx.Bool("addpiece")), fmt.Sprintf("--precommit1=%t", cctx.Bool("precommit1")), fmt.Sprintf("--unseal=%t", cctx.Bool("unseal")), diff --git a/extern/sector-storage/localworker.go b/extern/sector-storage/localworker.go index 2c3c350f7..b1193a2e2 100644 --- a/extern/sector-storage/localworker.go +++ b/extern/sector-storage/localworker.go @@ -26,6 +26,7 @@ var pathTypes = []stores.SectorFileType{stores.FTUnsealed, stores.FTSealed, stor type WorkerConfig struct { SealProof abi.RegisteredSealProof TaskTypes []sealtasks.TaskType + NoSwap bool } type LocalWorker struct { @@ -33,6 +34,7 @@ type LocalWorker struct { storage stores.Store localStore *stores.Local sindex stores.SectorIndex + noSwap bool acceptTasks map[sealtasks.TaskType]struct{} } @@ -50,6 +52,7 @@ func NewLocalWorker(wcfg WorkerConfig, store stores.Store, local *stores.Local, storage: store, localStore: local, sindex: sindex, + noSwap: wcfg.NoSwap, acceptTasks: acceptTasks, } @@ -275,11 +278,16 @@ func (l *LocalWorker) Info(context.Context) (storiface.WorkerInfo, error) { return storiface.WorkerInfo{}, xerrors.Errorf("getting memory info: %w", err) } + memSwap := mem.VirtualTotal + if l.noSwap { + memSwap = 0 + } + return storiface.WorkerInfo{ Hostname: hostname, Resources: storiface.WorkerResources{ MemPhysical: mem.Total, - MemSwap: mem.VirtualTotal, + MemSwap: memSwap, MemReserved: mem.VirtualUsed + mem.Total - mem.Available, // TODO: sub this process CPUs: uint64(runtime.NumCPU()), GPUs: gpus, From 6abccc4d5ed9c977916a812b3aa3e4407fd124ae Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Wed, 30 Sep 2020 02:56:38 -0400 Subject: [PATCH 447/885] Add an option to set config --- cmd/lotus/daemon.go | 8 ++++++++ node/repo/fsrepo.go | 34 ++++++++++++++++++++-------------- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index b976fde79..a0f754a60 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -127,6 +127,10 @@ var DaemonCmd = &cli.Command{ Usage: "manage open file limit", Value: true, }, + &cli.StringFlag{ + Name: "config", + Usage: "specify path of config file to use", + }, }, Action: func(cctx *cli.Context) error { err := runmetrics.Enable(runmetrics.RunMetricOptions{ @@ -180,6 +184,10 @@ var DaemonCmd = &cli.Command{ return xerrors.Errorf("opening fs repo: %w", err) } + if cctx.String("config") != "" { + r.SetConfigPath(cctx.String("config")) + } + if err := r.Init(repo.FullNode); err != nil && err != repo.ErrRepoExists { return xerrors.Errorf("repo init error: %w", err) } diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index 709d78d3a..a69cdd55d 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -65,7 +65,8 @@ var ErrRepoExists = xerrors.New("repo exists") // FsRepo is struct for repo, use NewFS to create type FsRepo struct { - path string + path string + configPath string } var _ Repo = &FsRepo{} @@ -78,10 +79,15 @@ func NewFS(path string) (*FsRepo, error) { } return &FsRepo{ - path: path, + path: path, + configPath: filepath.Join(path, fsConfig), }, nil } +func (fsr *FsRepo) SetConfigPath(cfgPath string) { + fsr.configPath = cfgPath +} + func (fsr *FsRepo) Exists() (bool, error) { _, err := os.Stat(filepath.Join(fsr.path, fsDatastore)) notexist := os.IsNotExist(err) @@ -115,9 +121,7 @@ func (fsr *FsRepo) Init(t RepoType) error { } func (fsr *FsRepo) initConfig(t RepoType) error { - cfgP := filepath.Join(fsr.path, fsConfig) - - _, err := os.Stat(cfgP) + _, err := os.Stat(fsr.configPath) if err == nil { // exists return nil @@ -125,7 +129,7 @@ func (fsr *FsRepo) initConfig(t RepoType) error { return err } - c, err := os.Create(cfgP) + c, err := os.Create(fsr.configPath) if err != nil { return err } @@ -215,16 +219,18 @@ func (fsr *FsRepo) Lock(repoType RepoType) (LockedRepo, error) { return nil, xerrors.Errorf("could not lock the repo: %w", err) } return &fsLockedRepo{ - path: fsr.path, - repoType: repoType, - closer: closer, + path: fsr.path, + configPath: fsr.configPath, + repoType: repoType, + closer: closer, }, nil } type fsLockedRepo struct { - path string - repoType RepoType - closer io.Closer + path string + configPath string + repoType RepoType + closer io.Closer ds map[string]datastore.Batching dsErr error @@ -277,7 +283,7 @@ func (fsr *fsLockedRepo) Config() (interface{}, error) { } func (fsr *fsLockedRepo) loadConfigFromDisk() (interface{}, error) { - return config.FromFile(fsr.join(fsConfig), defConfForType(fsr.repoType)) + return config.FromFile(fsr.configPath, defConfForType(fsr.repoType)) } func (fsr *fsLockedRepo) SetConfig(c func(interface{})) error { @@ -306,7 +312,7 @@ func (fsr *fsLockedRepo) SetConfig(c func(interface{})) error { } // write buffer of TOML bytes to config file - err = ioutil.WriteFile(fsr.join(fsConfig), buf.Bytes(), 0644) + err = ioutil.WriteFile(fsr.configPath, buf.Bytes(), 0644) if err != nil { return err } From eb6191d0ffd01a7cf7f8544a31acf307b1799fb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Wed, 30 Sep 2020 11:02:10 +0100 Subject: [PATCH 448/885] tvx: precursor selection modes; canonical message fetching; basefee. --- cmd/tvx/extract.go | 199 +++++++++++++++++++++++----------------- cmd/tvx/extract_many.go | 43 ++++++--- conformance/driver.go | 58 ++++++++---- conformance/runner.go | 19 +++- go.mod | 2 +- go.sum | 4 +- 6 files changed, 204 insertions(+), 121 deletions(-) diff --git a/cmd/tvx/extract.go b/cmd/tvx/extract.go index 0dc7f6aa0..fef245858 100644 --- a/cmd/tvx/extract.go +++ b/cmd/tvx/extract.go @@ -21,21 +21,27 @@ import ( lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/conformance" - "github.com/filecoin-project/go-address" "github.com/filecoin-project/specs-actors/actors/builtin" + "github.com/filecoin-project/test-vectors/schema" "github.com/ipfs/go-cid" "github.com/urfave/cli/v2" ) +const ( + PrecursorSelectAll = "all" + PrecursorSelectSender = "sender" +) + type extractOpts struct { - id string - block string - class string - cid string - file string - retain string + id string + block string + class string + cid string + file string + retain string + precursor string } var extractFlags extractOpts @@ -81,6 +87,16 @@ var extractCmd = &cli.Command{ Value: "accessed-cids", Destination: &extractFlags.retain, }, + &cli.StringFlag{ + Name: "precursor-select", + Usage: "precursors to apply; values: 'all', 'sender'; 'all' selects all preceding" + + "messages in the canonicalised tipset, 'sender' selects only preceding messages from the same" + + "sender. Usually, 'sender' is a good tradeoff and gives you sufficient accuracy. If the receipt sanity" + + "check fails due to gas reasons, switch to 'all', as previous messages in the tipset may have" + + "affected state in a disruptive way", + Value: "sender", + Destination: &extractFlags.precursor, + }, }, } @@ -124,46 +140,38 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { return fmt.Errorf("failed while fetching circulating supply: %w", err) } - circSupply := circSupplyDetail.FilCirculating.Int64() + circSupply := circSupplyDetail.FilCirculating log.Printf("message was executed in tipset: %s", execTs.Key()) log.Printf("message was included in tipset: %s", incTs.Key()) log.Printf("circulating supply at inclusion tipset: %d", circSupply) - log.Printf("finding precursor messages") + log.Printf("finding precursor messages using mode: %s", opts.precursor) - // Iterate through blocks, finding the one that contains the message and its - // precursors, if any. - var allmsgs []*types.Message - for _, b := range incTs.Blocks() { - messages, err := fapi.ChainGetBlockMessages(ctx, b.Cid()) - if err != nil { - return err - } - - related, found, err := findMsgAndPrecursors(messages, msg) - if err != nil { - return fmt.Errorf("invariant failed while scanning messages in block %s: %w", b.Cid(), err) - } - - if found { - var mcids []cid.Cid - for _, m := range related { - mcids = append(mcids, m.Cid()) - } - log.Printf("found message in block %s; precursors: %v", b.Cid(), mcids[:len(mcids)-1]) - allmsgs = related - break - } - - log.Printf("message not found in block %s; number of precursors found: %d; ignoring block", b.Cid(), len(related)) + // Fetch messages in canonical order from inclusion tipset. + msgs, err := fapi.ChainGetParentMessages(ctx, execTs.Blocks()[0].Cid()) + if err != nil { + return fmt.Errorf("failed to fetch messages in canonical order from inclusion tipset: %w", err) } - if allmsgs == nil { - // Message was not found; abort. - return fmt.Errorf("did not find a block containing the message") + related, found, err := findMsgAndPrecursors(opts.precursor, msg, msgs) + if err != nil { + return fmt.Errorf("failed while finding message and precursors: %w", err) } - precursors := allmsgs[:len(allmsgs)-1] + if !found { + return fmt.Errorf("message not found; precursors found: %d", len(related)) + } + + var ( + precursors = related[:len(related)-1] + precursorsCids []cid.Cid + ) + + for _, p := range precursors { + precursorsCids = append(precursorsCids, p.Cid()) + } + + log.Println(color.GreenString("found message; precursors (count: %d): %v", len(precursors), precursorsCids)) var ( // create a read-through store that uses ChainGetObject to fetch unknown CIDs. @@ -179,11 +187,20 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { root := incTs.ParentState() log.Printf("base state tree root CID: %s", root) + basefee := incTs.Blocks()[0].ParentBaseFee + log.Printf("basefee: %s", basefee) + // on top of that state tree, we apply all precursors. log.Printf("number of precursors to apply: %d", len(precursors)) for i, m := range precursors { log.Printf("applying precursor %d, cid: %s", i, m.Cid()) - _, root, err = driver.ExecuteMessage(pst.Blockstore, root, execTs.Height(), m, &circSupplyDetail.FilCirculating) + _, root, err = driver.ExecuteMessage(pst.Blockstore, conformance.ExecuteMessageParams{ + Preroot: root, + Epoch: execTs.Height(), + Message: m, + CircSupply: &circSupplyDetail.FilCirculating, + BaseFee: &basefee, + }) if err != nil { return fmt.Errorf("failed to execute precursor message: %w", err) } @@ -208,7 +225,13 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { tbs.StartTracing() preroot = root - applyret, postroot, err = driver.ExecuteMessage(pst.Blockstore, preroot, execTs.Height(), msg, &circSupplyDetail.FilCirculating) + applyret, postroot, err = driver.ExecuteMessage(pst.Blockstore, conformance.ExecuteMessageParams{ + Preroot: preroot, + Epoch: execTs.Height(), + Message: msg, + CircSupply: &circSupplyDetail.FilCirculating, + BaseFee: &basefee, + }) if err != nil { return fmt.Errorf("failed to execute message: %w", err) } @@ -233,7 +256,13 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { if err != nil { return err } - applyret, postroot, err = driver.ExecuteMessage(pst.Blockstore, preroot, execTs.Height(), msg, &circSupplyDetail.FilCirculating) + applyret, postroot, err = driver.ExecuteMessage(pst.Blockstore, conformance.ExecuteMessageParams{ + Preroot: preroot, + Epoch: execTs.Height(), + Message: msg, + CircSupply: &circSupplyDetail.FilCirculating, + BaseFee: &basefee, + }) if err != nil { return fmt.Errorf("failed to execute message: %w", err) } @@ -248,21 +277,39 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { log.Printf("message applied; preroot: %s, postroot: %s", preroot, postroot) log.Println("performing sanity check on receipt") - receipt := &schema.Receipt{ - ExitCode: int64(applyret.ExitCode), - ReturnValue: applyret.Return, - GasUsed: applyret.GasUsed, + // TODO sometimes this returns a nil receipt and no error ¯\_(ツ)_/¯ + // ex: https://filfox.info/en/message/bafy2bzacebpxw3yiaxzy2bako62akig46x3imji7fewszen6fryiz6nymu2b2 + // This code is lenient and skips receipt comparison in case of a nil receipt. + rec, err := fapi.StateGetReceipt(ctx, mcid, execTs.Key()) + if err != nil { + return fmt.Errorf("failed to find receipt on chain: %w", err) } + log.Printf("found receipt: %+v", rec) - reporter := new(conformance.LogReporter) - conformance.AssertMsgResult(reporter, receipt, applyret, "as locally executed") - if reporter.Failed() { - log.Println(color.RedString("receipt sanity check failed; aborting")) - return fmt.Errorf("vector generation aborted") + // generate the schema receipt; if we got + var receipt *schema.Receipt + if rec != nil { + receipt = &schema.Receipt{ + ExitCode: int64(rec.ExitCode), + ReturnValue: rec.Return, + GasUsed: rec.GasUsed, + } + reporter := new(conformance.LogReporter) + conformance.AssertMsgResult(reporter, receipt, applyret, "as locally executed") + if reporter.Failed() { + log.Println(color.RedString("receipt sanity check failed; aborting")) + return fmt.Errorf("vector generation aborted") + } + log.Println(color.GreenString("receipt sanity check succeeded")) + } else { + receipt = &schema.Receipt{ + ExitCode: int64(applyret.ExitCode), + ReturnValue: applyret.Return, + GasUsed: applyret.GasUsed, + } + log.Println(color.YellowString("skipping receipts comparison; we got back a nil receipt from lotus")) } - log.Println(color.GreenString("receipt sanity check succeeded")) - log.Println("generating vector") msgBytes, err := msg.Serialize() if err != nil { @@ -312,7 +359,8 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { CAR: out.Bytes(), Pre: &schema.Preconditions{ Epoch: int64(execTs.Height()), - CircSupply: &circSupply, + CircSupply: circSupply.Int, + BaseFee: basefee.Int, StateTree: &schema.StateTree{ RootCID: preroot, }, @@ -428,41 +476,22 @@ func fetchThisAndPrevTipset(ctx context.Context, api api.FullNode, target types. return targetTs, prevTs, nil } -// findMsgAndPrecursors scans the messages in a block to locate the supplied -// message, looking into the BLS or SECP section depending on the sender's -// address type. -// -// It returns any precursors (if they exist), and the found message (if found), -// in a slice. -// -// It also returns a boolean indicating whether the message was actually found. -// -// This function also asserts invariants, and if those fail, it returns an error. -func findMsgAndPrecursors(messages *api.BlockMessages, target *types.Message) (related []*types.Message, found bool, err error) { - // Decide which block of messages to process, depending on whether the - // sender is a BLS or a SECP account. - input := messages.BlsMessages - if senderKind := target.From.Protocol(); senderKind == address.SECP256K1 { - input = make([]*types.Message, 0, len(messages.SecpkMessages)) - for _, sm := range messages.SecpkMessages { - input = append(input, &sm.Message) - } - } - - for _, other := range input { - if other.From != target.From { - continue - } - - // this message is from the same sender, so it's related. - related = append(related, other) - - if other.Nonce > target.Nonce { - return nil, false, fmt.Errorf("a message with nonce higher than the target was found before the target; offending mcid: %s", other.Cid()) +// findMsgAndPrecursors ranges through the canonical messages slice, locating +// the target message and returning precursors in accordance to the supplied +// mode. +func findMsgAndPrecursors(mode string, target *types.Message, msgs []api.Message) (related []*types.Message, found bool, err error) { + // Range through canonicalised messages, selecting only the precursors based + // on selection mode. + for _, other := range msgs { + switch { + case mode == PrecursorSelectAll: + fallthrough + case mode == PrecursorSelectSender && other.Message.From == target.From: + related = append(related, other.Message) } // this message is the target; we're done. - if other.Cid() == target.Cid() { + if other.Cid == target.Cid() { return related, true, nil } } diff --git a/cmd/tvx/extract_many.go b/cmd/tvx/extract_many.go index 83ec72b21..9de7aeab5 100644 --- a/cmd/tvx/extract_many.go +++ b/cmd/tvx/extract_many.go @@ -111,8 +111,12 @@ func runExtractMany(c *cli.Context) error { log.Println(color.GreenString("csv sanity check succeeded; header contains fields: %v", header)) } - var generated []string - merr := new(multierror.Error) + var ( + generated []string + merr = new(multierror.Error) + retry []extractOpts // to retry with 'canonical' precursor selection mode + ) + // Read each row and extract the requested message. for { row, err := reader.Read() @@ -164,19 +168,21 @@ func runExtractMany(c *cli.Context) error { // Vector filename, using a base of outdir. file := filepath.Join(outdir, actorcodename, methodname, exitcodename, id) + ".json" - log.Println(color.YellowString("processing message id: %s", id)) + log.Println(color.YellowString("processing message cid with 'sender' precursor mode: %s", id)) opts := extractOpts{ - id: id, - block: block, - class: "message", - cid: cid, - file: file, - retain: "accessed-cids", + id: id, + block: block, + class: "message", + cid: cid, + file: file, + retain: "accessed-cids", + precursor: PrecursorSelectSender, } if err := doExtract(ctx, fapi, opts); err != nil { - merr = multierror.Append(err, fmt.Errorf("failed to extract vector for message %s: %w", cid, err)) + log.Println(color.RedString("failed to extract vector for message %s: %s; queuing for 'canonical' precursor selection", cid, err)) + retry = append(retry, opts) continue } @@ -185,6 +191,21 @@ func runExtractMany(c *cli.Context) error { generated = append(generated, file) } + log.Printf("extractions to try with canonical precursor selection mode: %d", len(retry)) + + for _, r := range retry { + log.Printf("retrying %s: %s", r.cid, r.id) + + r.precursor = PrecursorSelectAll + if err := doExtract(ctx, fapi, r); err != nil { + merr = multierror.Append(merr, fmt.Errorf("failed to extract vector for message %s: %w", r.cid, err)) + continue + } + + log.Println(color.MagentaString("generated file: %s", r.file)) + generated = append(generated, r.file) + } + if len(generated) == 0 { log.Println("no files generated") } else { @@ -195,7 +216,7 @@ func runExtractMany(c *cli.Context) error { } if merr.ErrorOrNil() != nil { - log.Println(color.YellowString("done processing with errors: %s")) + log.Println(color.YellowString("done processing with errors: %s", err)) } else { log.Println(color.GreenString("done processing with no errors")) } diff --git a/conformance/driver.go b/conformance/driver.go index 3f50b67a9..9ced12d74 100644 --- a/conformance/driver.go +++ b/conformance/driver.go @@ -2,6 +2,7 @@ package conformance import ( "context" + "os" "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/stmgr" @@ -23,15 +24,14 @@ import ( ds "github.com/ipfs/go-datastore" ) -// DefaultCirculatingSupply is the fallback circulating supply returned by -// the driver's CircSupplyCalculator function, used if the vector specifies -// no circulating supply. -var DefaultCirculatingSupply = types.TotalFilecoinInt - var ( - // BaseFee to use in the VM. - // TODO make parametrisable through vector. - BaseFee = abi.NewTokenAmount(100) + // DefaultCirculatingSupply is the fallback circulating supply returned by + // the driver's CircSupplyCalculator function, used if the vector specifies + // no circulating supply. + DefaultCirculatingSupply = types.TotalFilecoinInt + + // DefaultBaseFee to use in the VM, if one is not supplied in the vector. + DefaultBaseFee = abi.NewTokenAmount(100) ) type Driver struct { @@ -139,24 +139,46 @@ func (d *Driver) ExecuteTipset(bs blockstore.Blockstore, ds ds.Batching, preroot return ret, nil } +type ExecuteMessageParams struct { + Preroot cid.Cid + Epoch abi.ChainEpoch + Message *types.Message + CircSupply *abi.TokenAmount + BaseFee *abi.TokenAmount +} + // ExecuteMessage executes a conformance test vector message in a temporary VM. -func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, preroot cid.Cid, epoch abi.ChainEpoch, msg *types.Message, circSupply *abi.TokenAmount) (*vm.ApplyRet, cid.Cid, error) { +func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, params ExecuteMessageParams) (*vm.ApplyRet, cid.Cid, error) { + if !d.vmFlush { + // do not flush the VM, just the state tree; this should be used with + // LOTUS_DISABLE_VM_BUF enabled, so writes will anyway be visible. + _ = os.Setenv("LOTUS_DISABLE_VM_BUF", "iknowitsabadidea") + } + + basefee := DefaultBaseFee + if params.BaseFee != nil { + basefee = *params.BaseFee + } + + circSupply := DefaultCirculatingSupply + if params.CircSupply != nil { + circSupply = *params.CircSupply + } + // dummy state manager; only to reference the GetNetworkVersion method, // which does not depend on state. sm := new(stmgr.StateManager) + vmOpts := &vm.VMOpts{ - StateBase: preroot, - Epoch: epoch, + StateBase: params.Preroot, + Epoch: params.Epoch, Rand: &testRand{}, // TODO always succeeds; need more flexibility. Bstore: bs, Syscalls: mkFakedSigSyscalls(vm.Syscalls(ffiwrapper.ProofVerifier)), // TODO always succeeds; need more flexibility. CircSupplyCalc: func(_ context.Context, _ abi.ChainEpoch, _ *state.StateTree) (abi.TokenAmount, error) { - if circSupply != nil { - return *circSupply, nil - } - return DefaultCirculatingSupply, nil + return circSupply, nil }, - BaseFee: BaseFee, + BaseFee: basefee, NtwkVersion: sm.GetNtwkVersion, } @@ -174,7 +196,7 @@ func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, preroot cid.Cid, epoch lvm.SetInvoker(invoker) - ret, err := lvm.ApplyMessage(d.ctx, toChainMsg(msg)) + ret, err := lvm.ApplyMessage(d.ctx, toChainMsg(params.Message)) if err != nil { return nil, cid.Undef, err } @@ -185,8 +207,6 @@ func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, preroot cid.Cid, epoch // recursive copoy from the temporary blcokstore to the real blockstore. root, err = lvm.Flush(d.ctx) } else { - // do not flush the VM, just the state tree; this should be used with - // LOTUS_DISABLE_VM_BUF enabled, so writes will anyway be visible. root, err = lvm.StateTree().(*state.StateTree).Flush(d.ctx) } diff --git a/conformance/runner.go b/conformance/runner.go index 812f3cc08..2db53b3e4 100644 --- a/conformance/runner.go +++ b/conformance/runner.go @@ -13,6 +13,7 @@ import ( "github.com/fatih/color" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/exitcode" "github.com/filecoin-project/test-vectors/schema" "github.com/ipfs/go-blockservice" @@ -43,14 +44,20 @@ func ExecuteMessageVector(r Reporter, vector *schema.TestVector) { } // Create a new Driver. - driver := NewDriver(ctx, vector.Selector, DriverOpts{}) + driver := NewDriver(ctx, vector.Selector, DriverOpts{DisableVMFlush: true}) var circSupply *abi.TokenAmount if cs := vector.Pre.CircSupply; cs != nil { - ta := abi.NewTokenAmount(*cs) + ta := big.NewFromGo(cs) circSupply = &ta } + var basefee *abi.TokenAmount + if bf := vector.Pre.BaseFee; bf != nil { + ta := big.NewFromGo(bf) + basefee = &ta + } + // Apply every message. for i, m := range vector.ApplyMessages { msg, err := types.DecodeMessage(m.Bytes) @@ -65,7 +72,13 @@ func ExecuteMessageVector(r Reporter, vector *schema.TestVector) { // Execute the message. var ret *vm.ApplyRet - ret, root, err = driver.ExecuteMessage(bs, root, abi.ChainEpoch(epoch), msg, circSupply) + ret, root, err = driver.ExecuteMessage(bs, ExecuteMessageParams{ + Preroot: root, + Epoch: abi.ChainEpoch(epoch), + Message: msg, + CircSupply: circSupply, + BaseFee: basefee, + }) if err != nil { r.Fatalf("fatal failure when executing message: %s", err) } diff --git a/go.mod b/go.mod index bf3748749..1ef228ee7 100644 --- a/go.mod +++ b/go.mod @@ -38,7 +38,7 @@ require ( github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b github.com/filecoin-project/specs-actors v0.9.11 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 - github.com/filecoin-project/test-vectors/schema v0.0.2 + github.com/filecoin-project/test-vectors/schema v0.0.3 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 github.com/go-kit/kit v0.10.0 github.com/go-ole/go-ole v1.2.4 // indirect diff --git a/go.sum b/go.sum index 6a310ed3d..924e8f5c2 100644 --- a/go.sum +++ b/go.sum @@ -258,8 +258,8 @@ github.com/filecoin-project/specs-actors v0.9.11 h1:TnpG7HAeiUrfj0mJM7UaPW0P2137 github.com/filecoin-project/specs-actors v0.9.11/go.mod h1:czlvLQGEX0fjLLfdNHD7xLymy6L3n7aQzRWzsYGf+ys= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= -github.com/filecoin-project/test-vectors/schema v0.0.2 h1:/Pp//88WBXe0h+ksntdL2HpEgAmbwXrftAfeVG39zdY= -github.com/filecoin-project/test-vectors/schema v0.0.2/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E= +github.com/filecoin-project/test-vectors/schema v0.0.3 h1:1zuBo25B3016inbygYLgYFdpJ2m1BDTbAOCgABRleiU= +github.com/filecoin-project/test-vectors/schema v0.0.3/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6 h1:u/UEqS66A5ckRmS4yNpjmVH56sVtS/RfclBAYocb4as= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ= From 4d4bab12eba36261dca9c422a83f2640c7d7af2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 30 Sep 2020 13:33:42 +0200 Subject: [PATCH 449/885] Improve miner sectors list UX --- cli/util.go | 5 +- cmd/lotus-storage-miner/sectors.go | 111 ++++++++++++++++++++++++----- go.mod | 1 + go.sum | 2 + lib/tablewriter/tablewriter.go | 15 +++- 5 files changed, 111 insertions(+), 23 deletions(-) diff --git a/cli/util.go b/cli/util.go index 762cdb565..fb555e320 100644 --- a/cli/util.go +++ b/cli/util.go @@ -5,6 +5,7 @@ import ( "fmt" "time" + "github.com/hako/durafmt" "github.com/ipfs/go-cid" "github.com/filecoin-project/go-state-types/abi" @@ -36,11 +37,11 @@ func parseTipSet(ctx context.Context, api api.FullNode, vals []string) (*types.T func EpochTime(curr, e abi.ChainEpoch) string { switch { case curr > e: - return fmt.Sprintf("%d (%s ago)", e, time.Second*time.Duration(int64(build.BlockDelaySecs)*int64(curr-e))) + return fmt.Sprintf("%d (%s ago)", e, durafmt.Parse(time.Second*time.Duration(int64(build.BlockDelaySecs)*int64(curr-e))).LimitFirstN(2)) case curr == e: return fmt.Sprintf("%d (now)", e) case curr < e: - return fmt.Sprintf("%d (in %s)", e, time.Second*time.Duration(int64(build.BlockDelaySecs)*int64(e-curr))) + return fmt.Sprintf("%d (in %s)", e, durafmt.Parse(time.Second*time.Duration(int64(build.BlockDelaySecs)*int64(e-curr))).LimitFirstN(2)) } panic("math broke") diff --git a/cmd/lotus-storage-miner/sectors.go b/cmd/lotus-storage-miner/sectors.go index 370962bdc..b50f4a86d 100644 --- a/cmd/lotus-storage-miner/sectors.go +++ b/cmd/lotus-storage-miner/sectors.go @@ -5,19 +5,22 @@ import ( "os" "sort" "strconv" - "text/tabwriter" "time" + "github.com/docker/go-units" + "github.com/fatih/color" "github.com/urfave/cli/v2" "golang.org/x/xerrors" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/tablewriter" lcli "github.com/filecoin-project/lotus/cli" sealing "github.com/filecoin-project/lotus/extern/storage-sealing" @@ -144,8 +147,19 @@ var sectorsListCmd = &cli.Command{ Name: "show-removed", Usage: "show removed sectors", }, + &cli.BoolFlag{ + Name: "color", + Aliases: []string{"c"}, + Value: true, + }, + &cli.BoolFlag{ + Name: "fast", + Usage: "don't show on-chain info for better performance", + }, }, Action: func(cctx *cli.Context) error { + color.NoColor = !cctx.Bool("color") + nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx) if err != nil { return err @@ -170,7 +184,12 @@ var sectorsListCmd = &cli.Command{ return err } - activeSet, err := fullApi.StateMinerActiveSectors(ctx, maddr, types.EmptyTSK) + head, err := fullApi.ChainHead(ctx) + if err != nil { + return err + } + + activeSet, err := fullApi.StateMinerActiveSectors(ctx, maddr, head.Key()) if err != nil { return err } @@ -179,7 +198,7 @@ var sectorsListCmd = &cli.Command{ activeIDs[info.SectorNumber] = struct{}{} } - sset, err := fullApi.StateMinerSectors(ctx, maddr, nil, types.EmptyTSK) + sset, err := fullApi.StateMinerSectors(ctx, maddr, nil, head.Key()) if err != nil { return err } @@ -192,12 +211,26 @@ var sectorsListCmd = &cli.Command{ return list[i] < list[j] }) - w := tabwriter.NewWriter(os.Stdout, 8, 4, 1, ' ', 0) + tw := tablewriter.New( + tablewriter.Col("ID"), + tablewriter.Col("State"), + tablewriter.Col("OnChain"), + tablewriter.Col("Active"), + tablewriter.Col("Expiration"), + tablewriter.Col("Deals"), + tablewriter.Col("DealWeight"), + tablewriter.NewLineCol("Error"), + tablewriter.NewLineCol("EarlyExpiration")) + + fast := cctx.Bool("fast") for _, s := range list { - st, err := nodeApi.SectorsStatus(ctx, s, false) + st, err := nodeApi.SectorsStatus(ctx, s, !fast) if err != nil { - fmt.Fprintf(w, "%d:\tError: %s\n", s, err) + tw.Write(map[string]interface{}{ + "ID": s, + "Error": err, + }) continue } @@ -205,20 +238,60 @@ var sectorsListCmd = &cli.Command{ _, inSSet := commitedIDs[s] _, inASet := activeIDs[s] - _, _ = fmt.Fprintf(w, "%d: %s\tsSet: %s\tactive: %s\ttktH: %d\tseedH: %d\tdeals: %v\t toUpgrade:%t\n", - s, - st.State, - yesno(inSSet), - yesno(inASet), - st.Ticket.Epoch, - st.Seed.Epoch, - st.Deals, - st.ToUpgrade, - ) + dw := .0 + if st.Expiration-st.Activation > 0 { + dw = float64(big.Div(st.DealWeight, big.NewInt(int64(st.Expiration-st.Activation))).Uint64()) + } + + var deals int + for _, deal := range st.Deals { + if deal != 0 { + deals++ + } + } + + exp := st.Expiration + if st.OnTime > 0 && st.OnTime < exp { + exp = st.OnTime // Can be different when the sector was CC upgraded + } + + m := map[string]interface{}{ + "ID": s, + "State": color.New(stateOrder[sealing.SectorState(st.State)].col).Sprint(st.State), + "OnChain": yesno(inSSet), + "Active": yesno(inASet), + } + + if deals > 0 { + m["Deals"] = color.GreenString("%d", deals) + } else { + m["Deals"] = color.BlueString("CC") + if st.ToUpgrade { + m["Deals"] = color.CyanString("CC(upgrade)") + } + } + + if !fast { + if !inSSet { + m["Expiration"] = "n/a" + } else { + m["Expiration"] = lcli.EpochTime(head.Height(), exp) + + if !fast && deals > 0 { + m["DealWeight"] = units.BytesSize(dw) + } + + if st.Early > 0 { + m["EarlyExpiration"] = color.YellowString(lcli.EpochTime(head.Height(), st.Early)) + } + } + } + + tw.Write(m) } } - return w.Flush() + return tw.Flush(os.Stdout) }, } @@ -447,7 +520,7 @@ var sectorsUpdateCmd = &cli.Command{ func yesno(b bool) string { if b { - return "YES" + return color.GreenString("YES") } - return "NO" + return color.RedString("NO") } diff --git a/go.mod b/go.mod index b8896d78f..b9b133f51 100644 --- a/go.mod +++ b/go.mod @@ -46,6 +46,7 @@ require ( github.com/google/uuid v1.1.1 github.com/gorilla/mux v1.7.4 github.com/gorilla/websocket v1.4.2 + github.com/hako/durafmt v0.0.0-20200710122514-c0fb7b4da026 github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e github.com/hashicorp/go-multierror v1.1.0 github.com/hashicorp/golang-lru v0.5.4 diff --git a/go.sum b/go.sum index 0f309ce1a..2d9929705 100644 --- a/go.sum +++ b/go.sum @@ -415,6 +415,8 @@ github.com/gxed/go-shellwords v1.0.3/go.mod h1:N7paucT91ByIjmVJHhvoarjoQnmsi3Jd3 github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= github.com/gxed/pubsub v0.0.0-20180201040156-26ebdf44f824/go.mod h1:OiEWyHgK+CWrmOlVquHaIK1vhpUJydC9m0Je6mhaiNE= +github.com/hako/durafmt v0.0.0-20200710122514-c0fb7b4da026 h1:BpJ2o0OR5FV7vrkDYfXYVJQeMNWa8RhklZOpW2ITAIQ= +github.com/hako/durafmt v0.0.0-20200710122514-c0fb7b4da026/go.mod h1:5Scbynm8dF1XAPwIwkGPqzkM/shndPm79Jd1003hTjE= github.com/hannahhoward/cbor-gen-for v0.0.0-20200817222906-ea96cece81f1 h1:F9k+7wv5OIk1zcq23QpdiL0hfDuXPjuOmMNaC6fgQ0Q= github.com/hannahhoward/cbor-gen-for v0.0.0-20200817222906-ea96cece81f1/go.mod h1:jvfsLIxk0fY/2BKSQ1xf2406AKA5dwMmKKv0ADcOfN8= github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e h1:3YKHER4nmd7b5qy5t0GWDTwSn4OyRgfAXSmo6VnryBY= diff --git a/lib/tablewriter/tablewriter.go b/lib/tablewriter/tablewriter.go index d77611390..cd045710e 100644 --- a/lib/tablewriter/tablewriter.go +++ b/lib/tablewriter/tablewriter.go @@ -12,6 +12,7 @@ import ( type Column struct { Name string SeparateLine bool + Lines int } type TableWriter struct { @@ -50,6 +51,7 @@ cloop: for i, column := range w.cols { if column.Name == col { byColID[i] = fmt.Sprint(val) + w.cols[i].Lines++ continue cloop } } @@ -58,6 +60,7 @@ cloop: w.cols = append(w.cols, Column{ Name: col, SeparateLine: false, + Lines: 1, }) } @@ -77,7 +80,11 @@ func (w *TableWriter) Flush(out io.Writer) error { w.rows = append([]map[int]string{header}, w.rows...) - for col := range w.cols { + for col, c := range w.cols { + if c.Lines == 0 { + continue + } + for _, row := range w.rows { val, found := row[col] if !found { @@ -94,9 +101,13 @@ func (w *TableWriter) Flush(out io.Writer) error { cols := make([]string, len(w.cols)) for ci, col := range w.cols { + if col.Lines == 0 { + continue + } + e, _ := row[ci] pad := colLengths[ci] - cliStringLength(e) + 2 - if !col.SeparateLine { + if !col.SeparateLine && col.Lines > 0 { e = e + strings.Repeat(" ", pad) if _, err := fmt.Fprint(out, e); err != nil { return err From d2f279c3de755b267562cb0fdf8bb6b471ad0356 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 30 Sep 2020 13:51:52 +0200 Subject: [PATCH 450/885] Print heman-readable transferred bytes in client list-transfers --- cli/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/client.go b/cli/client.go index 5d40dce0d..272ab0bdd 100644 --- a/cli/client.go +++ b/cli/client.go @@ -1423,7 +1423,7 @@ func toChannelOutput(useColor bool, otherPartyColumn string, channel lapi.DataTr otherPartyColumn: otherParty, "Root Cid": rootCid, "Initiated?": initiated, - "Transferred": channel.Transferred, + "Transferred": units.BytesSize(float64(channel.Transferred)), "Voucher": voucher, "Message": channel.Message, } From f58881e966dbd087172ecd5e1e1958850a3d2455 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Wed, 30 Sep 2020 14:57:24 +0100 Subject: [PATCH 451/885] minor fixes. --- cmd/tvx/extract_many.go | 4 ++-- cmd/tvx/stores.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/tvx/extract_many.go b/cmd/tvx/extract_many.go index 9de7aeab5..d7a25c10c 100644 --- a/cmd/tvx/extract_many.go +++ b/cmd/tvx/extract_many.go @@ -48,7 +48,7 @@ var extractManyCmd = &cli.Command{ Destination: &extractManyFlags.in, }, &cli.StringFlag{ - Name: "out-dir", + Name: "outdir", Usage: "output directory", Destination: &extractManyFlags.outdir, }, @@ -216,7 +216,7 @@ func runExtractMany(c *cli.Context) error { } if merr.ErrorOrNil() != nil { - log.Println(color.YellowString("done processing with errors: %s", err)) + log.Println(color.YellowString("done processing with errors: %v", merr)) } else { log.Println(color.GreenString("done processing with no errors")) } diff --git a/cmd/tvx/stores.go b/cmd/tvx/stores.go index 6e50e0839..93e0d215f 100644 --- a/cmd/tvx/stores.go +++ b/cmd/tvx/stores.go @@ -11,7 +11,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/lib/blockstore" - "github.com/filecoin-project/specs-actors/actors/util/adt" + "github.com/filecoin-project/lotus/chain/actors/adt" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-blockservice" From ff8663faa0b3a7d1871f2c4c681e243c2cbc229d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Wed, 30 Sep 2020 14:58:22 +0100 Subject: [PATCH 452/885] update test-vectors submodule. --- extern/test-vectors | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/test-vectors b/extern/test-vectors index 6bea015ed..3a6e0b5e0 160000 --- a/extern/test-vectors +++ b/extern/test-vectors @@ -1 +1 @@ -Subproject commit 6bea015edddde116001a4251dce3c4a9966c25d9 +Subproject commit 3a6e0b5e069b1452ce1a032aa315354d645f3ec4 From 1c4f8e83d79d052d1e4408d635daf80b3e81d62c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Wed, 30 Sep 2020 16:11:44 +0100 Subject: [PATCH 453/885] tvx/extract-many: add batch id, change generated filename. --- cmd/tvx/extract_many.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/cmd/tvx/extract_many.go b/cmd/tvx/extract_many.go index d7a25c10c..9679a1dbd 100644 --- a/cmd/tvx/extract_many.go +++ b/cmd/tvx/extract_many.go @@ -20,8 +20,9 @@ import ( ) var extractManyFlags struct { - in string - outdir string + in string + outdir string + batchId string } var extractManyCmd = &cli.Command{ @@ -37,11 +38,17 @@ var extractManyCmd = &cli.Command{ The first row MUST be a header row. At the bare minimum, those seven fields must appear, in the order specified. Extra fields are accepted, but always - after these compulsory six. + after these compulsory seven. `, Action: runExtractMany, Flags: []cli.Flag{ &repoFlag, + &cli.StringFlag{ + Name: "batch-id", + Usage: "batch id; a four-digit left-zero-padded sequential number (e.g. 0041)", + Required: true, + Destination: &extractManyFlags.batchId, + }, &cli.StringFlag{ Name: "in", Usage: "path to input file (csv)", @@ -164,7 +171,7 @@ func runExtractMany(c *cli.Context) error { actorcodename := strings.ReplaceAll(actorcode, "/", "_") // Compute the ID of the vector. - id := fmt.Sprintf("extracted-msg-%s-%s-%s-%s", actorcodename, methodname, exitcodename, seq) + id := fmt.Sprintf("ext-%s-%s-%s-%s-%s", extractManyFlags.batchId, actorcodename, methodname, exitcodename, seq) // Vector filename, using a base of outdir. file := filepath.Join(outdir, actorcodename, methodname, exitcodename, id) + ".json" From 724306c1104ccbb0ffa49d16ddbaf4705e393c17 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 30 Sep 2020 09:23:04 -0700 Subject: [PATCH 454/885] update specs-actors --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index a2a2ab2ba..2cf9a81fe 100644 --- a/go.mod +++ b/go.mod @@ -37,8 +37,8 @@ require ( github.com/filecoin-project/go-statemachine v0.0.0-20200813232949-df9b130df370 github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b - github.com/filecoin-project/specs-actors v0.9.12-0.20200928180918-488a087c5add - github.com/filecoin-project/specs-actors/v2 v2.0.0-20200929004132-ea2d270c9f47 + github.com/filecoin-project/specs-actors v0.9.12-0.20200930015636-a6848b2dd741 + github.com/filecoin-project/specs-actors/v2 v2.0.0-20200930035834-7115a78cb9a1 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 github.com/filecoin-project/test-vectors/schema v0.0.3 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 diff --git a/go.sum b/go.sum index 7ec9936e9..05960060e 100644 --- a/go.sum +++ b/go.sum @@ -273,10 +273,10 @@ github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/ github.com/filecoin-project/specs-actors v0.9.4/go.mod h1:BStZQzx5x7TmCkLv0Bpa07U6cPKol6fd3w9KjMPZ6Z4= github.com/filecoin-project/specs-actors v0.9.7/go.mod h1:wM2z+kwqYgXn5Z7scV1YHLyd1Q1cy0R8HfTIWQ0BFGU= github.com/filecoin-project/specs-actors v0.9.9/go.mod h1:czlvLQGEX0fjLLfdNHD7xLymy6L3n7aQzRWzsYGf+ys= -github.com/filecoin-project/specs-actors v0.9.12-0.20200928180918-488a087c5add h1:iXQOxr8uSyZ/qnTlOZf7ALp0io+jwLxmuWsNAk/YdoQ= -github.com/filecoin-project/specs-actors v0.9.12-0.20200928180918-488a087c5add/go.mod h1:TS1AW/7LbG+615j4NsjMK1qlpAwaFsG9w0V2tg2gSao= -github.com/filecoin-project/specs-actors/v2 v2.0.0-20200929004132-ea2d270c9f47 h1:bt/CdGbF1Rq5rDVaMV+XWfHBRCxN7IxZfw6lC1v6KQM= -github.com/filecoin-project/specs-actors/v2 v2.0.0-20200929004132-ea2d270c9f47/go.mod h1:/2Zra1BhLtpRywUhm++QP+3I5Ir+hBk/W24TpYjj43E= +github.com/filecoin-project/specs-actors v0.9.12-0.20200930015636-a6848b2dd741 h1:pqAfjHE+yLFj1mvtx68S3HKdd9LSy6Byziz2qRNh3fA= +github.com/filecoin-project/specs-actors v0.9.12-0.20200930015636-a6848b2dd741/go.mod h1:TS1AW/7LbG+615j4NsjMK1qlpAwaFsG9w0V2tg2gSao= +github.com/filecoin-project/specs-actors/v2 v2.0.0-20200930035834-7115a78cb9a1 h1:lNMXodbyskP3ZfTWL/t6M4NYI0hhcZQ2GAJCIVK8Y/E= +github.com/filecoin-project/specs-actors/v2 v2.0.0-20200930035834-7115a78cb9a1/go.mod h1:/2Zra1BhLtpRywUhm++QP+3I5Ir+hBk/W24TpYjj43E= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= github.com/filecoin-project/test-vectors/schema v0.0.3 h1:1zuBo25B3016inbygYLgYFdpJ2m1BDTbAOCgABRleiU= From 23b729a0569a2b14d4de799d00fc2ca4b17789e5 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 30 Sep 2020 09:35:42 -0700 Subject: [PATCH 455/885] remove proof parameter from payment channels It never worked properly, and will be removed in actors v2. --- paychmgr/manager.go | 23 +++++++-- paychmgr/paych.go | 74 ++++----------------------- paychmgr/paych_test.go | 111 ++++------------------------------------- paychmgr/store.go | 2 +- 4 files changed, 42 insertions(+), 168 deletions(-) diff --git a/paychmgr/manager.go b/paychmgr/manager.go index 7ef1264c5..1b2acfd2f 100644 --- a/paychmgr/manager.go +++ b/paychmgr/manager.go @@ -2,6 +2,7 @@ package paychmgr import ( "context" + "errors" "sync" "github.com/ipfs/go-cid" @@ -24,6 +25,8 @@ import ( var log = logging.Logger("paych") +var errProofNotSupported = errors.New("payment channel proof parameter is not supported") + // PaychAPI is used by dependency injection to pass the consituent APIs to NewManager() type PaychAPI struct { fx.In @@ -245,34 +248,43 @@ func (pm *Manager) CheckVoucherValid(ctx context.Context, ch address.Address, sv // CheckVoucherSpendable checks if the given voucher is currently spendable func (pm *Manager) CheckVoucherSpendable(ctx context.Context, ch address.Address, sv *paych.SignedVoucher, secret []byte, proof []byte) (bool, error) { + if len(proof) > 0 { + return false, errProofNotSupported + } ca, err := pm.accessorByAddress(ch) if err != nil { return false, err } - return ca.checkVoucherSpendable(ctx, ch, sv, secret, proof) + return ca.checkVoucherSpendable(ctx, ch, sv, secret) } // AddVoucherOutbound adds a voucher for an outbound channel. // Returns an error if the channel is not already in the store. func (pm *Manager) AddVoucherOutbound(ctx context.Context, ch address.Address, sv *paych.SignedVoucher, proof []byte, minDelta types.BigInt) (types.BigInt, error) { + if len(proof) > 0 { + return types.NewInt(0), errProofNotSupported + } ca, err := pm.accessorByAddress(ch) if err != nil { return types.NewInt(0), err } - return ca.addVoucher(ctx, ch, sv, proof, minDelta) + return ca.addVoucher(ctx, ch, sv, minDelta) } // AddVoucherInbound adds a voucher for an inbound channel. // If the channel is not in the store, fetches the channel from state (and checks that // the channel To address is owned by the wallet). func (pm *Manager) AddVoucherInbound(ctx context.Context, ch address.Address, sv *paych.SignedVoucher, proof []byte, minDelta types.BigInt) (types.BigInt, error) { + if len(proof) > 0 { + return types.NewInt(0), errProofNotSupported + } // Get an accessor for the channel, creating it from state if necessary ca, err := pm.inboundChannelAccessor(ctx, ch) if err != nil { return types.BigInt{}, err } - return ca.addVoucher(ctx, ch, sv, proof, minDelta) + return ca.addVoucher(ctx, ch, sv, minDelta) } // inboundChannelAccessor gets an accessor for the given channel. The channel @@ -336,11 +348,14 @@ func (pm *Manager) trackInboundChannel(ctx context.Context, ch address.Address) } func (pm *Manager) SubmitVoucher(ctx context.Context, ch address.Address, sv *paych.SignedVoucher, secret []byte, proof []byte) (cid.Cid, error) { + if len(proof) > 0 { + return cid.Undef, errProofNotSupported + } ca, err := pm.accessorByAddress(ch) if err != nil { return cid.Undef, err } - return ca.submitVoucher(ctx, ch, sv, secret, proof) + return ca.submitVoucher(ctx, ch, sv, secret) } func (pm *Manager) AllocateLane(ch address.Address) (uint64, error) { diff --git a/paychmgr/paych.go b/paychmgr/paych.go index ba96c5e40..056140653 100644 --- a/paychmgr/paych.go +++ b/paychmgr/paych.go @@ -1,7 +1,6 @@ package paychmgr import ( - "bytes" "context" "fmt" @@ -133,7 +132,7 @@ func (ca *channelAccessor) createVoucher(ctx context.Context, ch address.Address sv.Signature = sig // Store the voucher - if _, err := ca.addVoucherUnlocked(ctx, ch, sv, nil, types.NewInt(0)); err != nil { + if _, err := ca.addVoucherUnlocked(ctx, ch, sv, types.NewInt(0)); err != nil { // If there are not enough funds in the channel to cover the voucher, // return a voucher create result with the shortfall var ife insufficientFundsErr @@ -272,7 +271,7 @@ func (ca *channelAccessor) checkVoucherValidUnlocked(ctx context.Context, ch add return laneStates, nil } -func (ca *channelAccessor) checkVoucherSpendable(ctx context.Context, ch address.Address, sv *paych.SignedVoucher, secret []byte, proof []byte) (bool, error) { +func (ca *channelAccessor) checkVoucherSpendable(ctx context.Context, ch address.Address, sv *paych.SignedVoucher, secret []byte) (bool, error) { ca.lk.Lock() defer ca.lk.Unlock() @@ -295,26 +294,9 @@ func (ca *channelAccessor) checkVoucherSpendable(ctx context.Context, ch address return false, nil } - // If proof is needed and wasn't supplied as a parameter, get it from the - // datastore - if sv.Extra != nil && proof == nil { - vi, err := ci.infoForVoucher(sv) - if err != nil { - return false, err - } - - if vi.Proof != nil { - log.Info("CheckVoucherSpendable: using stored proof") - proof = vi.Proof - } else { - log.Warn("CheckVoucherSpendable: nil proof for voucher with validation") - } - } - enc, err := actors.SerializeParams(&paych0.UpdateChannelStateParams{ Sv: *sv, Secret: secret, - Proof: proof, }) if err != nil { return false, err @@ -346,44 +328,31 @@ func (ca *channelAccessor) getPaychRecipient(ctx context.Context, ch address.Add return state.To() } -func (ca *channelAccessor) addVoucher(ctx context.Context, ch address.Address, sv *paych.SignedVoucher, proof []byte, minDelta types.BigInt) (types.BigInt, error) { +func (ca *channelAccessor) addVoucher(ctx context.Context, ch address.Address, sv *paych.SignedVoucher, minDelta types.BigInt) (types.BigInt, error) { ca.lk.Lock() defer ca.lk.Unlock() - return ca.addVoucherUnlocked(ctx, ch, sv, proof, minDelta) + return ca.addVoucherUnlocked(ctx, ch, sv, minDelta) } -func (ca *channelAccessor) addVoucherUnlocked(ctx context.Context, ch address.Address, sv *paych.SignedVoucher, proof []byte, minDelta types.BigInt) (types.BigInt, error) { +func (ca *channelAccessor) addVoucherUnlocked(ctx context.Context, ch address.Address, sv *paych.SignedVoucher, minDelta types.BigInt) (types.BigInt, error) { ci, err := ca.store.ByAddress(ch) if err != nil { return types.BigInt{}, err } // Check if the voucher has already been added - for i, v := range ci.Vouchers { + for _, v := range ci.Vouchers { eq, err := cborutil.Equals(sv, v.Voucher) if err != nil { return types.BigInt{}, err } - if !eq { - continue + if eq { + // Ignore the duplicate voucher. + log.Warnf("AddVoucher: voucher re-added") + return types.NewInt(0), nil } - // This is a duplicate voucher. - // Update the proof on the existing voucher - if len(proof) > 0 && !bytes.Equal(v.Proof, proof) { - log.Warnf("AddVoucher: adding proof to stored voucher") - ci.Vouchers[i] = &VoucherInfo{ - Voucher: v.Voucher, - Proof: proof, - } - - return types.NewInt(0), ca.store.putChannelInfo(ci) - } - - // Otherwise just ignore the duplicate voucher - log.Warnf("AddVoucher: voucher re-added with matching proof") - return types.NewInt(0), nil } // Check voucher validity @@ -410,7 +379,6 @@ func (ca *channelAccessor) addVoucherUnlocked(ctx context.Context, ch address.Ad ci.Vouchers = append(ci.Vouchers, &VoucherInfo{ Voucher: sv, - Proof: proof, }) if ci.NextLane <= sv.Lane { @@ -420,7 +388,7 @@ func (ca *channelAccessor) addVoucherUnlocked(ctx context.Context, ch address.Ad return delta, ca.store.putChannelInfo(ci) } -func (ca *channelAccessor) submitVoucher(ctx context.Context, ch address.Address, sv *paych.SignedVoucher, secret []byte, proof []byte) (cid.Cid, error) { +func (ca *channelAccessor) submitVoucher(ctx context.Context, ch address.Address, sv *paych.SignedVoucher, secret []byte) (cid.Cid, error) { ca.lk.Lock() defer ca.lk.Unlock() @@ -429,21 +397,6 @@ func (ca *channelAccessor) submitVoucher(ctx context.Context, ch address.Address return cid.Undef, err } - // If voucher needs proof, and none was supplied, check datastore for proof - if sv.Extra != nil && proof == nil { - vi, err := ci.infoForVoucher(sv) - if err != nil { - return cid.Undef, err - } - - if vi.Proof != nil { - log.Info("SubmitVoucher: using stored proof") - proof = vi.Proof - } else { - log.Warn("SubmitVoucher: nil proof for voucher with validation") - } - } - has, err := ci.hasVoucher(sv) if err != nil { return cid.Undef, err @@ -462,13 +415,9 @@ func (ca *channelAccessor) submitVoucher(ctx context.Context, ch address.Address } // TODO: ActorUpgrade - // The "proof" field is going away. We will need to abstract over the - // network version here. - // Alternatively, we'd need to support the "old" method on-chain. enc, err := actors.SerializeParams(&paych0.UpdateChannelStateParams{ Sv: *sv, Secret: secret, - Proof: proof, }) if err != nil { return cid.Undef, err @@ -492,7 +441,6 @@ func (ca *channelAccessor) submitVoucher(ctx context.Context, ch address.Address // Add the voucher to the channel ci.Vouchers = append(ci.Vouchers, &VoucherInfo{ Voucher: sv, - Proof: proof, }) } diff --git a/paychmgr/paych_test.go b/paychmgr/paych_test.go index b27b1e540..ff77f5512 100644 --- a/paychmgr/paych_test.go +++ b/paychmgr/paych_test.go @@ -555,53 +555,6 @@ func TestAllocateLaneWithExistingLaneState(t *testing.T) { require.EqualValues(t, 3, lane) } -func TestAddVoucherProof(t *testing.T) { - ctx := context.Background() - - // Set up a manager with a single payment channel - s := testSetupMgrWithChannel(ctx, t) - - nonce := uint64(1) - voucherAmount := big.NewInt(1) - minDelta := big.NewInt(0) - voucherAmount = big.NewInt(2) - voucherLane := uint64(1) - - // Add a voucher with no proof - var proof []byte - sv := createTestVoucher(t, s.ch, voucherLane, nonce, voucherAmount, s.fromKeyPrivate) - _, err := s.mgr.AddVoucherOutbound(ctx, s.ch, sv, nil, minDelta) - require.NoError(t, err) - - // Expect one voucher with no proof - ci, err := s.mgr.GetChannelInfo(s.ch) - require.NoError(t, err) - require.Len(t, ci.Vouchers, 1) - require.Len(t, ci.Vouchers[0].Proof, 0) - - // Add same voucher with no proof - voucherLane = uint64(1) - _, err = s.mgr.AddVoucherOutbound(ctx, s.ch, sv, proof, minDelta) - require.NoError(t, err) - - // Expect one voucher with no proof - ci, err = s.mgr.GetChannelInfo(s.ch) - require.NoError(t, err) - require.Len(t, ci.Vouchers, 1) - require.Len(t, ci.Vouchers[0].Proof, 0) - - // Add same voucher with proof - proof = []byte{1} - _, err = s.mgr.AddVoucherOutbound(ctx, s.ch, sv, proof, minDelta) - require.NoError(t, err) - - // Should add proof to existing voucher - ci, err = s.mgr.GetChannelInfo(s.ch) - require.NoError(t, err) - require.Len(t, ci.Vouchers, 1) - require.Len(t, ci.Vouchers[0].Proof, 1) -} - func TestAddVoucherInboundWalletKey(t *testing.T) { ctx := context.Background() @@ -748,10 +701,9 @@ func TestCheckSpendable(t *testing.T) { voucherAmount := big.NewInt(1) voucher := createTestVoucherWithExtra(t, s.ch, voucherLane, nonce, voucherAmount, s.fromKeyPrivate) - // Add voucher with proof + // Add voucher minDelta := big.NewInt(0) - proof := []byte("proof") - _, err := s.mgr.AddVoucherInbound(ctx, s.ch, voucher, proof, minDelta) + _, err := s.mgr.AddVoucherInbound(ctx, s.ch, voucher, nil, minDelta) require.NoError(t, err) // Return success exit code from VM call, which indicates that voucher is @@ -765,33 +717,17 @@ func TestCheckSpendable(t *testing.T) { // Check that spendable is true secret := []byte("secret") - otherProof := []byte("other proof") - spendable, err := s.mgr.CheckVoucherSpendable(ctx, s.ch, voucher, secret, otherProof) + spendable, err := s.mgr.CheckVoucherSpendable(ctx, s.ch, voucher, secret, nil) require.NoError(t, err) require.True(t, spendable) - // Check that the secret and proof were passed through correctly + // Check that the secret was passed through correctly lastCall := s.mock.getLastCall() var p paych0.UpdateChannelStateParams err = p.UnmarshalCBOR(bytes.NewReader(lastCall.Params)) require.NoError(t, err) - require.Equal(t, otherProof, p.Proof) require.Equal(t, secret, p.Secret) - // Check that if no proof is supplied, the proof supplied to add voucher - // above is used - secret2 := []byte("secret2") - spendable, err = s.mgr.CheckVoucherSpendable(ctx, s.ch, voucher, secret2, nil) - require.NoError(t, err) - require.True(t, spendable) - - lastCall = s.mock.getLastCall() - var p2 paych0.UpdateChannelStateParams - err = p2.UnmarshalCBOR(bytes.NewReader(lastCall.Params)) - require.NoError(t, err) - require.Equal(t, proof, p2.Proof) - require.Equal(t, secret2, p2.Secret) - // Check that if VM call returns non-success exit code, spendable is false s.mock.setCallResponse(&api.InvocResult{ MsgRct: &types.MessageReceipt{ @@ -829,73 +765,48 @@ func TestSubmitVoucher(t *testing.T) { voucherAmount := big.NewInt(1) voucher := createTestVoucherWithExtra(t, s.ch, voucherLane, nonce, voucherAmount, s.fromKeyPrivate) - // Add voucher with proof + // Add voucher minDelta := big.NewInt(0) - addVoucherProof := []byte("proof") - _, err := s.mgr.AddVoucherInbound(ctx, s.ch, voucher, addVoucherProof, minDelta) + _, err := s.mgr.AddVoucherInbound(ctx, s.ch, voucher, nil, minDelta) require.NoError(t, err) // Submit voucher secret := []byte("secret") - submitProof := []byte("submit proof") - submitCid, err := s.mgr.SubmitVoucher(ctx, s.ch, voucher, secret, submitProof) + submitCid, err := s.mgr.SubmitVoucher(ctx, s.ch, voucher, secret, nil) require.NoError(t, err) - // Check that the secret and proof were passed through correctly + // Check that the secret was passed through correctly msg := s.mock.pushedMessages(submitCid) var p paych0.UpdateChannelStateParams err = p.UnmarshalCBOR(bytes.NewReader(msg.Message.Params)) require.NoError(t, err) - require.Equal(t, submitProof, p.Proof) require.Equal(t, secret, p.Secret) - // Check that if no proof is supplied to submit voucher, the proof supplied - // to add voucher is used - nonce++ - voucherAmount = big.NewInt(2) - addVoucherProof2 := []byte("proof2") - secret2 := []byte("secret2") - voucher = createTestVoucherWithExtra(t, s.ch, voucherLane, nonce, voucherAmount, s.fromKeyPrivate) - _, err = s.mgr.AddVoucherInbound(ctx, s.ch, voucher, addVoucherProof2, minDelta) - require.NoError(t, err) - - submitCid, err = s.mgr.SubmitVoucher(ctx, s.ch, voucher, secret2, nil) - require.NoError(t, err) - - msg = s.mock.pushedMessages(submitCid) - var p2 paych0.UpdateChannelStateParams - err = p2.UnmarshalCBOR(bytes.NewReader(msg.Message.Params)) - require.NoError(t, err) - require.Equal(t, addVoucherProof2, p2.Proof) - require.Equal(t, secret2, p2.Secret) - // Submit a voucher without first adding it nonce++ voucherAmount = big.NewInt(3) secret3 := []byte("secret2") - proof3 := []byte("proof3") voucher = createTestVoucherWithExtra(t, s.ch, voucherLane, nonce, voucherAmount, s.fromKeyPrivate) - submitCid, err = s.mgr.SubmitVoucher(ctx, s.ch, voucher, secret3, proof3) + submitCid, err = s.mgr.SubmitVoucher(ctx, s.ch, voucher, secret3, nil) require.NoError(t, err) msg = s.mock.pushedMessages(submitCid) var p3 paych0.UpdateChannelStateParams err = p3.UnmarshalCBOR(bytes.NewReader(msg.Message.Params)) require.NoError(t, err) - require.Equal(t, proof3, p3.Proof) require.Equal(t, secret3, p3.Secret) // Verify that vouchers are marked as submitted vis, err := s.mgr.ListVouchers(ctx, s.ch) require.NoError(t, err) - require.Len(t, vis, 3) + require.Len(t, vis, 2) for _, vi := range vis { require.True(t, vi.Submitted) } // Attempting to submit the same voucher again should fail - _, err = s.mgr.SubmitVoucher(ctx, s.ch, voucher, secret2, nil) + _, err = s.mgr.SubmitVoucher(ctx, s.ch, voucher, secret3, nil) require.Error(t, err) } diff --git a/paychmgr/store.go b/paychmgr/store.go index dbd663832..a17ad1fcd 100644 --- a/paychmgr/store.go +++ b/paychmgr/store.go @@ -49,7 +49,7 @@ const ( type VoucherInfo struct { Voucher *paych.SignedVoucher - Proof []byte + Proof []byte // ignored Submitted bool } From 4c60d7b5ceb341feb3d66371bd34310b12f49769 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 30 Sep 2020 10:04:10 -0700 Subject: [PATCH 456/885] abstract over paych messages --- chain/actors/builtin/paych/message.go | 28 +++++++ chain/actors/builtin/paych/message0.go | 74 +++++++++++++++++++ chain/actors/builtin/paych/message2.go | 74 +++++++++++++++++++ .../builtin/paych/{paych.go => state.go} | 0 .../actors/builtin/paych/{v0.go => state0.go} | 0 .../actors/builtin/paych/{v2.go => state2.go} | 0 paychmgr/manager.go | 2 + paychmgr/mock_test.go | 6 ++ paychmgr/paych.go | 68 ++++++++--------- paychmgr/paychget_test.go | 6 +- paychmgr/simple.go | 36 +++------ 11 files changed, 234 insertions(+), 60 deletions(-) create mode 100644 chain/actors/builtin/paych/message.go create mode 100644 chain/actors/builtin/paych/message0.go create mode 100644 chain/actors/builtin/paych/message2.go rename chain/actors/builtin/paych/{paych.go => state.go} (100%) rename chain/actors/builtin/paych/{v0.go => state0.go} (100%) rename chain/actors/builtin/paych/{v2.go => state2.go} (100%) diff --git a/chain/actors/builtin/paych/message.go b/chain/actors/builtin/paych/message.go new file mode 100644 index 000000000..d19f0c38f --- /dev/null +++ b/chain/actors/builtin/paych/message.go @@ -0,0 +1,28 @@ +package paych + +import ( + "fmt" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/types" +) + +func Message(version actors.Version) MessageBuilder { + switch version { + case actors.Version0: + return message0{} + case actors.Version2: + return message2{} + default: + panic(fmt.Sprintf("unsupported actors version: %d", version)) + } +} + +type MessageBuilder interface { + Create(from, to address.Address, initialAmount abi.TokenAmount) (*types.Message, error) + Update(from, paych address.Address, voucher *SignedVoucher, secret []byte) (*types.Message, error) + Settle(from, paych address.Address) (*types.Message, error) + Collect(from, paych address.Address) (*types.Message, error) +} diff --git a/chain/actors/builtin/paych/message0.go b/chain/actors/builtin/paych/message0.go new file mode 100644 index 000000000..0e7bbff92 --- /dev/null +++ b/chain/actors/builtin/paych/message0.go @@ -0,0 +1,74 @@ +package paych + +import ( + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" + init0 "github.com/filecoin-project/specs-actors/actors/builtin/init" + paych0 "github.com/filecoin-project/specs-actors/actors/builtin/paych" + + "github.com/filecoin-project/lotus/chain/actors" + init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init" + "github.com/filecoin-project/lotus/chain/types" +) + +type message0 struct{} + +func (message0) Create(from, to address.Address, initialAmount abi.TokenAmount) (*types.Message, error) { + params, aerr := actors.SerializeParams(&paych0.ConstructorParams{From: from, To: to}) + if aerr != nil { + return nil, aerr + } + enc, aerr := actors.SerializeParams(&init0.ExecParams{ + CodeCID: builtin0.PaymentChannelActorCodeID, + ConstructorParams: params, + }) + if aerr != nil { + return nil, aerr + } + + return &types.Message{ + To: init_.Address, + From: from, + Value: initialAmount, + Method: builtin0.MethodsInit.Exec, + Params: enc, + }, nil +} + +func (message0) Update(from, paych address.Address, sv *SignedVoucher, secret []byte) (*types.Message, error) { + params, aerr := actors.SerializeParams(&paych0.UpdateChannelStateParams{ + Sv: *sv, + Secret: secret, + }) + if aerr != nil { + return nil, aerr + } + + return &types.Message{ + To: paych, + From: from, + Value: abi.NewTokenAmount(0), + Method: builtin0.MethodsPaych.UpdateChannelState, + Params: params, + }, nil +} + +func (message0) Settle(from, paych address.Address) (*types.Message, error) { + return &types.Message{ + To: paych, + From: from, + Value: abi.NewTokenAmount(0), + Method: builtin0.MethodsPaych.Settle, + }, nil +} + +func (message0) Collect(from, paych address.Address) (*types.Message, error) { + return &types.Message{ + To: paych, + From: from, + Value: abi.NewTokenAmount(0), + Method: builtin0.MethodsPaych.Collect, + }, nil +} diff --git a/chain/actors/builtin/paych/message2.go b/chain/actors/builtin/paych/message2.go new file mode 100644 index 000000000..94538bc49 --- /dev/null +++ b/chain/actors/builtin/paych/message2.go @@ -0,0 +1,74 @@ +package paych + +import ( + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + + builtin2 "github.com/filecoin-project/specs-actors/actors/builtin" + init2 "github.com/filecoin-project/specs-actors/actors/builtin/init" + paych2 "github.com/filecoin-project/specs-actors/actors/builtin/paych" + + "github.com/filecoin-project/lotus/chain/actors" + init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init" + "github.com/filecoin-project/lotus/chain/types" +) + +type message2 struct{} + +func (message2) Create(from, to address.Address, initialAmount abi.TokenAmount) (*types.Message, error) { + params, aerr := actors.SerializeParams(&paych2.ConstructorParams{From: from, To: to}) + if aerr != nil { + return nil, aerr + } + enc, aerr := actors.SerializeParams(&init2.ExecParams{ + CodeCID: builtin2.PaymentChannelActorCodeID, + ConstructorParams: params, + }) + if aerr != nil { + return nil, aerr + } + + return &types.Message{ + To: init_.Address, + From: from, + Value: initialAmount, + Method: builtin2.MethodsInit.Exec, + Params: enc, + }, nil +} + +func (message2) Update(from, paych address.Address, sv *SignedVoucher, secret []byte) (*types.Message, error) { + params, aerr := actors.SerializeParams(&paych2.UpdateChannelStateParams{ + Sv: *sv, + Secret: secret, + }) + if aerr != nil { + return nil, aerr + } + + return &types.Message{ + To: paych, + From: from, + Value: abi.NewTokenAmount(0), + Method: builtin2.MethodsPaych.UpdateChannelState, + Params: params, + }, nil +} + +func (message2) Settle(from, paych address.Address) (*types.Message, error) { + return &types.Message{ + To: paych, + From: from, + Value: abi.NewTokenAmount(0), + Method: builtin2.MethodsPaych.Settle, + }, nil +} + +func (message2) Collect(from, paych address.Address) (*types.Message, error) { + return &types.Message{ + To: paych, + From: from, + Value: abi.NewTokenAmount(0), + Method: builtin2.MethodsPaych.Collect, + }, nil +} diff --git a/chain/actors/builtin/paych/paych.go b/chain/actors/builtin/paych/state.go similarity index 100% rename from chain/actors/builtin/paych/paych.go rename to chain/actors/builtin/paych/state.go diff --git a/chain/actors/builtin/paych/v0.go b/chain/actors/builtin/paych/state0.go similarity index 100% rename from chain/actors/builtin/paych/v0.go rename to chain/actors/builtin/paych/state0.go diff --git a/chain/actors/builtin/paych/v2.go b/chain/actors/builtin/paych/state2.go similarity index 100% rename from chain/actors/builtin/paych/v2.go rename to chain/actors/builtin/paych/state2.go diff --git a/paychmgr/manager.go b/paychmgr/manager.go index 1b2acfd2f..f2fc190c7 100644 --- a/paychmgr/manager.go +++ b/paychmgr/manager.go @@ -13,6 +13,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/crypto" + "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors/adt" @@ -49,6 +50,7 @@ type paychAPI interface { MpoolPushMessage(ctx context.Context, msg *types.Message, maxFee *api.MessageSendSpec) (*types.SignedMessage, error) WalletHas(ctx context.Context, addr address.Address) (bool, error) WalletSign(ctx context.Context, k address.Address, msg []byte) (*crypto.Signature, error) + StateNetworkVersion(context.Context, types.TipSetKey) (network.Version, error) } // managerAPI defines all methods needed by the manager diff --git a/paychmgr/mock_test.go b/paychmgr/mock_test.go index c761221d2..3393a3072 100644 --- a/paychmgr/mock_test.go +++ b/paychmgr/mock_test.go @@ -9,8 +9,10 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/crypto" + "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors/builtin/paych" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/sigs" @@ -241,3 +243,7 @@ func (pchapi *mockPaychAPI) addSigningKey(key []byte) { pchapi.signingKey = key } + +func (pchapi *mockPaychAPI) StateNetworkVersion(ctx context.Context, tsk types.TipSetKey) (network.Version, error) { + return build.NewestNetworkVersion, nil +} diff --git a/paychmgr/paych.go b/paychmgr/paych.go index 056140653..96ba43543 100644 --- a/paychmgr/paych.go +++ b/paychmgr/paych.go @@ -10,8 +10,6 @@ import ( "github.com/filecoin-project/go-address" cborutil "github.com/filecoin-project/go-cbor-util" "github.com/filecoin-project/go-state-types/big" - "github.com/filecoin-project/specs-actors/actors/builtin" - paych0 "github.com/filecoin-project/specs-actors/actors/builtin/paych" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" @@ -84,6 +82,15 @@ func newChannelAccessor(pm *Manager, from address.Address, to address.Address) * } } +func (ca *channelAccessor) messageBuilder(ctx context.Context) (paych.MessageBuilder, error) { + nwVersion, err := ca.api.StateNetworkVersion(ctx, types.EmptyTSK) + if err != nil { + return nil, err + } + + return paych.Message(actors.VersionForNetwork(nwVersion)), nil +} + func (ca *channelAccessor) getChannelInfo(addr address.Address) (*ChannelInfo, error) { ca.lk.Lock() defer ca.lk.Unlock() @@ -294,20 +301,17 @@ func (ca *channelAccessor) checkVoucherSpendable(ctx context.Context, ch address return false, nil } - enc, err := actors.SerializeParams(&paych0.UpdateChannelStateParams{ - Sv: *sv, - Secret: secret, - }) + mb, err := ca.messageBuilder(ctx) if err != nil { return false, err } - ret, err := ca.api.Call(ctx, &types.Message{ - From: recipient, - To: ch, - Method: builtin.MethodsPaych.UpdateChannelState, - Params: enc, - }, nil) + mes, err := mb.Update(recipient, ch, sv, secret) + if err != nil { + return false, err + } + + ret, err := ca.api.Call(ctx, mes, nil) if err != nil { return false, err } @@ -414,21 +418,14 @@ func (ca *channelAccessor) submitVoucher(ctx context.Context, ch address.Address } } - // TODO: ActorUpgrade - enc, err := actors.SerializeParams(&paych0.UpdateChannelStateParams{ - Sv: *sv, - Secret: secret, - }) + mb, err := ca.messageBuilder(ctx) if err != nil { return cid.Undef, err } - msg := &types.Message{ - From: ci.Control, - To: ch, - Value: types.NewInt(0), - Method: builtin.MethodsPaych.UpdateChannelState, - Params: enc, + msg, err := mb.Update(ci.Control, ch, sv, secret) + if err != nil { + return cid.Undef, err } smsg, err := ca.api.MpoolPushMessage(ctx, msg, nil) @@ -577,11 +574,13 @@ func (ca *channelAccessor) settle(ctx context.Context, ch address.Address) (cid. return cid.Undef, err } - msg := &types.Message{ - To: ch, - From: ci.Control, - Value: types.NewInt(0), - Method: builtin.MethodsPaych.Settle, + mb, err := ca.messageBuilder(ctx) + if err != nil { + return cid.Undef, err + } + msg, err := mb.Settle(ci.Control, ch) + if err != nil { + return cid.Undef, err } smgs, err := ca.api.MpoolPushMessage(ctx, msg, nil) if err != nil { @@ -606,11 +605,14 @@ func (ca *channelAccessor) collect(ctx context.Context, ch address.Address) (cid return cid.Undef, err } - msg := &types.Message{ - To: ch, - From: ci.Control, - Value: types.NewInt(0), - Method: builtin.MethodsPaych.Collect, + mb, err := ca.messageBuilder(ctx) + if err != nil { + return cid.Undef, err + } + + msg, err := mb.Collect(ci.Control, ch) + if err != nil { + return cid.Undef, err } smsg, err := ca.api.MpoolPushMessage(ctx, msg, nil) diff --git a/paychmgr/paychget_test.go b/paychmgr/paychget_test.go index 430e66c67..28e2ac7e3 100644 --- a/paychmgr/paychget_test.go +++ b/paychmgr/paychget_test.go @@ -15,9 +15,9 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" - "github.com/filecoin-project/specs-actors/actors/builtin" - init_ "github.com/filecoin-project/specs-actors/actors/builtin/init" - tutils "github.com/filecoin-project/specs-actors/support/testing" + "github.com/filecoin-project/specs-actors/v2/actors/builtin" + init_ "github.com/filecoin-project/specs-actors/v2/actors/builtin/init" + tutils "github.com/filecoin-project/specs-actors/v2/support/testing" lotusinit "github.com/filecoin-project/lotus/chain/actors/builtin/init" "github.com/filecoin-project/lotus/chain/actors/builtin/paych" diff --git a/paychmgr/simple.go b/paychmgr/simple.go index d49ccafe6..38804b7ea 100644 --- a/paychmgr/simple.go +++ b/paychmgr/simple.go @@ -12,14 +12,11 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/big" - "github.com/filecoin-project/specs-actors/actors/builtin" - init0 "github.com/filecoin-project/specs-actors/actors/builtin/init" - paych0 "github.com/filecoin-project/specs-actors/actors/builtin/paych" + + init2 "github.com/filecoin-project/specs-actors/actors/builtin/init" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/actors" - lotusinit "github.com/filecoin-project/lotus/chain/actors/builtin/init" "github.com/filecoin-project/lotus/chain/types" ) @@ -387,25 +384,13 @@ func (ca *channelAccessor) processTask(ctx context.Context, amt types.BigInt) *p // createPaych sends a message to create the channel and returns the message cid func (ca *channelAccessor) createPaych(ctx context.Context, amt types.BigInt) (cid.Cid, error) { - params, aerr := actors.SerializeParams(&paych0.ConstructorParams{From: ca.from, To: ca.to}) - if aerr != nil { - return cid.Undef, aerr + mb, err := ca.messageBuilder(ctx) + if err != nil { + return cid.Undef, err } - - enc, aerr := actors.SerializeParams(&init0.ExecParams{ - CodeCID: builtin.PaymentChannelActorCodeID, - ConstructorParams: params, - }) - if aerr != nil { - return cid.Undef, aerr - } - - msg := &types.Message{ - To: lotusinit.Address, - From: ca.from, - Value: amt, - Method: builtin.MethodsInit.Exec, - Params: enc, + msg, err := mb.Create(ca.from, ca.to, amt) + if err != nil { + return cid.Undef, err } smsg, err := ca.api.MpoolPushMessage(ctx, msg, nil) @@ -457,7 +442,10 @@ func (ca *channelAccessor) waitPaychCreateMsg(channelID string, mcid cid.Cid) er return err } - var decodedReturn init0.ExecReturn + // TODO: ActorUpgrade abstract over this. + // This "works" because it hasn't changed from v0 to v2, but we still + // need an abstraction here. + var decodedReturn init2.ExecReturn err = decodedReturn.UnmarshalCBOR(bytes.NewReader(mwait.Receipt.Return)) if err != nil { log.Error(err) From 5253c0d959ab16b4275f7f5a0ad702090c5430d2 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 30 Sep 2020 10:11:48 -0700 Subject: [PATCH 457/885] fix: import the correct v2 methods --- chain/stmgr/utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index 6ed94775b..98f6bc5ac 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -21,8 +21,8 @@ import ( builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" exported0 "github.com/filecoin-project/specs-actors/actors/builtin/exported" - exported2 "github.com/filecoin-project/specs-actors/actors/builtin/exported" proof0 "github.com/filecoin-project/specs-actors/actors/runtime/proof" + exported2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/exported" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" From be884e27bef3da5e0696c80363379cf8c75fc4a2 Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Tue, 29 Sep 2020 04:53:30 -0700 Subject: [PATCH 458/885] feat(markets): update markets 0.7.0 --- api/api_full.go | 2 +- api/apistruct/struct.go | 4 ++-- cli/client.go | 10 +++++----- documentation/en/api-methods.md | 22 ++++++++------------- go.mod | 10 +++++----- go.sum | 20 +++++++++++-------- markets/loggers/loggers.go | 11 +++++++++++ markets/storageadapter/client.go | 31 ------------------------------ node/builder.go | 7 ++++--- node/impl/client/client.go | 9 +++++---- node/modules/client.go | 11 +++++++---- node/modules/services.go | 24 +++++++++++++++++------ node/modules/storageminer.go | 33 +++++++++++++++++++++++++------- 13 files changed, 104 insertions(+), 90 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 6d2d0c7b5..24f17d987 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -275,7 +275,7 @@ type FullNode interface { // of status updates. ClientRetrieveWithEvents(ctx context.Context, order RetrievalOrder, ref *FileRef) (<-chan marketevents.RetrievalEvent, error) // ClientQueryAsk returns a signed StorageAsk from the specified miner. - ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.SignedStorageAsk, error) + ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.StorageAsk, error) // ClientCalcCommP calculates the CommP for a specified file ClientCalcCommP(ctx context.Context, inpath string) (*CommPRet, error) // ClientGenCar generates a CAR file for the specified file. diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index d5b6950ad..a5a3651f4 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -154,7 +154,7 @@ type FullNodeStruct struct { ClientGetDealUpdates func(ctx context.Context) (<-chan api.DealInfo, error) `perm:"read"` ClientRetrieve func(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) error `perm:"admin"` ClientRetrieveWithEvents func(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) (<-chan marketevents.RetrievalEvent, error) `perm:"admin"` - ClientQueryAsk func(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.SignedStorageAsk, error) `perm:"read"` + ClientQueryAsk func(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.StorageAsk, error) `perm:"read"` ClientCalcCommP func(ctx context.Context, inpath string) (*api.CommPRet, error) `perm:"read"` ClientGenCar func(ctx context.Context, ref api.FileRef, outpath string) error `perm:"write"` ClientDealSize func(ctx context.Context, root cid.Cid) (api.DataSize, error) `perm:"read"` @@ -484,7 +484,7 @@ func (c *FullNodeStruct) ClientRetrieveWithEvents(ctx context.Context, order api return c.Internal.ClientRetrieveWithEvents(ctx, order, ref) } -func (c *FullNodeStruct) ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.SignedStorageAsk, error) { +func (c *FullNodeStruct) ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.StorageAsk, error) { return c.Internal.ClientQueryAsk(ctx, p, miner) } func (c *FullNodeStruct) ClientCalcCommP(ctx context.Context, inpath string) (*api.CommPRet, error) { diff --git a/cli/client.go b/cli/client.go index 5d40dce0d..7c3ac9946 100644 --- a/cli/client.go +++ b/cli/client.go @@ -569,7 +569,7 @@ func interactiveDeal(cctx *cli.Context) error { continue } - ask = *a.Ask + ask = *a // TODO: run more validation state = "confirm" @@ -951,15 +951,15 @@ var clientQueryAskCmd = &cli.Command{ } fmt.Printf("Ask: %s\n", maddr) - fmt.Printf("Price per GiB: %s\n", types.FIL(ask.Ask.Price)) - fmt.Printf("Verified Price per GiB: %s\n", types.FIL(ask.Ask.VerifiedPrice)) - fmt.Printf("Max Piece size: %s\n", types.SizeStr(types.NewInt(uint64(ask.Ask.MaxPieceSize)))) + fmt.Printf("Price per GiB: %s\n", types.FIL(ask.Price)) + fmt.Printf("Verified Price per GiB: %s\n", types.FIL(ask.VerifiedPrice)) + fmt.Printf("Max Piece size: %s\n", types.SizeStr(types.NewInt(uint64(ask.MaxPieceSize)))) size := cctx.Int64("size") if size == 0 { return nil } - perEpoch := types.BigDiv(types.BigMul(ask.Ask.Price, types.NewInt(uint64(size))), types.NewInt(1<<30)) + perEpoch := types.BigDiv(types.BigMul(ask.Price, types.NewInt(uint64(size))), types.NewInt(1<<30)) fmt.Printf("Price per Block: %s\n", types.FIL(perEpoch)) duration := cctx.Int64("duration") diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index ed082ccbf..f54aa866a 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -1120,20 +1120,14 @@ Inputs: Response: ```json { - "Ask": { - "Price": "0", - "VerifiedPrice": "0", - "MinPieceSize": 1032, - "MaxPieceSize": 1032, - "Miner": "t01234", - "Timestamp": 10101, - "Expiry": 10101, - "SeqNo": 42 - }, - "Signature": { - "Type": 2, - "Data": "Ynl0ZSBhcnJheQ==" - } + "Price": "0", + "VerifiedPrice": "0", + "MinPieceSize": 1032, + "MaxPieceSize": 1032, + "Miner": "t01234", + "Timestamp": 10101, + "Expiry": 10101, + "SeqNo": 42 } ``` diff --git a/go.mod b/go.mod index 2c0322ecc..db00ba2a6 100644 --- a/go.mod +++ b/go.mod @@ -25,15 +25,15 @@ require ( github.com/filecoin-project/go-bitfield v0.2.0 github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 - github.com/filecoin-project/go-data-transfer v0.6.6 + github.com/filecoin-project/go-data-transfer v0.6.7 github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f - github.com/filecoin-project/go-fil-markets v0.6.3 + github.com/filecoin-project/go-fil-markets v0.7.0 github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200822201400-474f4fdccc52 github.com/filecoin-project/go-multistore v0.0.3 github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 github.com/filecoin-project/go-paramfetch v0.0.2-0.20200701152213-3e0f0afdc261 github.com/filecoin-project/go-state-types v0.0.0-20200911004822-964d6c679cfc - github.com/filecoin-project/go-statemachine v0.0.0-20200813232949-df9b130df370 + github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b github.com/filecoin-project/specs-actors v0.9.11 @@ -54,7 +54,7 @@ require ( github.com/ipfs/go-blockservice v0.1.4-0.20200624145336-a978cec6e834 github.com/ipfs/go-cid v0.0.7 github.com/ipfs/go-cidutil v0.0.2 - github.com/ipfs/go-datastore v0.4.4 + github.com/ipfs/go-datastore v0.4.5 github.com/ipfs/go-ds-badger2 v0.1.1-0.20200708190120-187fc06f714e github.com/ipfs/go-ds-leveldb v0.4.2 github.com/ipfs/go-ds-measure v0.1.0 @@ -112,7 +112,7 @@ require ( github.com/syndtr/goleveldb v1.0.0 github.com/urfave/cli/v2 v2.2.0 github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba - github.com/whyrusleeping/cbor-gen v0.0.0-20200814224545-656e08ce49ee + github.com/whyrusleeping/cbor-gen v0.0.0-20200826160007-0b9f6c5fb163 github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542 diff --git a/go.sum b/go.sum index 05e643708..9d3ccc7e8 100644 --- a/go.sum +++ b/go.sum @@ -224,12 +224,14 @@ github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:a github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= -github.com/filecoin-project/go-data-transfer v0.6.6 h1:2TccLSxPYJENcYRdov2WvpTvQ1qUMrPkWe8sBrfj36g= -github.com/filecoin-project/go-data-transfer v0.6.6/go.mod h1:C++k1U6+jMQODOaen5OPDo9XQbth9Yq3ie94vNjBJbk= +github.com/filecoin-project/go-data-transfer v0.6.7 h1:Kacr5qz2YWtd3sensU6aXFtES7joeapVDeXApeUD35I= +github.com/filecoin-project/go-data-transfer v0.6.7/go.mod h1:C++k1U6+jMQODOaen5OPDo9XQbth9Yq3ie94vNjBJbk= +github.com/filecoin-project/go-ds-versioning v0.1.0 h1:y/X6UksYTsK8TLCI7rttCKEvl8btmWxyFMEeeWGUxIQ= +github.com/filecoin-project/go-ds-versioning v0.1.0/go.mod h1:mp16rb4i2QPmxBnmanUx8i/XANp+PFCCJWiAb+VW4/s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f h1:GxJzR3oRIMTPtpZ0b7QF8FKPK6/iPAc7trhlL5k/g+s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= -github.com/filecoin-project/go-fil-markets v0.6.3 h1:3kTxfquGvk3zQY+hJH1kEA28tRQ47phqSRqOI4+YcQM= -github.com/filecoin-project/go-fil-markets v0.6.3/go.mod h1:Ug1yhGhzTYC6qrpKsR2QpU8QRCeBpwkTA9RICVKuOMM= +github.com/filecoin-project/go-fil-markets v0.7.0 h1:tcEZiUNIYQJ4PBzgVpLwfdJ4ZdC4WCv9LsgvsoCXIls= +github.com/filecoin-project/go-fil-markets v0.7.0/go.mod h1:5Pt4DXQqUoUrp9QzlSdlYTpItXxwAtqKrxRWQ6hAOqk= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24= @@ -248,8 +250,8 @@ github.com/filecoin-project/go-state-types v0.0.0-20200905071437-95828685f9df/go github.com/filecoin-project/go-state-types v0.0.0-20200911004822-964d6c679cfc h1:1vr/LoqGq5m5g37Q3sNSAjfwF1uJY0zmiHcvnxY6hik= github.com/filecoin-project/go-state-types v0.0.0-20200911004822-964d6c679cfc/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-statemachine v0.0.0-20200714194326-a77c3ae20989/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= -github.com/filecoin-project/go-statemachine v0.0.0-20200813232949-df9b130df370 h1:Jbburj7Ih2iaJ/o5Q9A+EAeTabME6YII7FLi9SKUf5c= -github.com/filecoin-project/go-statemachine v0.0.0-20200813232949-df9b130df370/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= +github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe h1:dF8u+LEWeIcTcfUcCf3WFVlc81Fr2JKg8zPzIbBDKDw= +github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIiWBRilQjQ+5IiwdQ= github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b h1:fkRZSPrYpk42PV3/lIXiL0LHetxde7vyYYvSsttQtfg= @@ -481,6 +483,8 @@ github.com/ipfs/go-datastore v0.4.1/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13X github.com/ipfs/go-datastore v0.4.2/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= github.com/ipfs/go-datastore v0.4.4 h1:rjvQ9+muFaJ+QZ7dN5B1MSDNQ0JVZKkkES/rMZmA8X8= github.com/ipfs/go-datastore v0.4.4/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= +github.com/ipfs/go-datastore v0.4.5 h1:cwOUcGMLdLPWgu3SlrCckCMznaGADbPqE0r8h768/Dg= +github.com/ipfs/go-datastore v0.4.5/go.mod h1:eXTcaaiN6uOlVCLS9GjJUJtlvJfM3xk23w3fyfrmmJs= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= github.com/ipfs/go-ds-badger v0.0.2/go.mod h1:Y3QpeSFWQf6MopLTiZD+VT6IC1yZqaGmjvRcKeSGij8= @@ -1371,8 +1375,8 @@ github.com/whyrusleeping/cbor-gen v0.0.0-20200710004633-5379fc63235d/go.mod h1:f github.com/whyrusleeping/cbor-gen v0.0.0-20200715143311-227fab5a2377/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200810223238-211df3b9e24c/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200812213548-958ddffe352c/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= -github.com/whyrusleeping/cbor-gen v0.0.0-20200814224545-656e08ce49ee h1:U7zWWvvAjT76EiuWPSOiZlQDnaQYPxPoxugTtTAcJK0= -github.com/whyrusleeping/cbor-gen v0.0.0-20200814224545-656e08ce49ee/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= +github.com/whyrusleeping/cbor-gen v0.0.0-20200826160007-0b9f6c5fb163 h1:TtcUeY2XZSriVWR1pXyfCBWIf/NGC2iUdNw1lofUjUU= +github.com/whyrusleeping/cbor-gen v0.0.0-20200826160007-0b9f6c5fb163/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8= github.com/whyrusleeping/go-ctrlnet v0.0.0-20180313164037-f564fbbdaa95/go.mod h1:SJqKCCPXRfBFCwXjfNT/skfsceF7+MBFLI2OrvuRA7g= diff --git a/markets/loggers/loggers.go b/markets/loggers/loggers.go index 27a6b2c63..a8e1c20aa 100644 --- a/markets/loggers/loggers.go +++ b/markets/loggers/loggers.go @@ -29,6 +29,17 @@ func RetrievalProviderLogger(event retrievalmarket.ProviderEvent, deal retrieval log.Infow("retrieval event", "name", retrievalmarket.ProviderEvents[event], "deal ID", deal.ID, "receiver", deal.Receiver, "state", retrievalmarket.DealStatuses[deal.Status], "message", deal.Message) } +// ReadyLogger returns a function to log the results of module initialization +func ReadyLogger(module string) func(error) { + return func(err error) { + if err != nil { + log.Errorw("module initialization error", "module", module, "err", err) + } else { + log.Infow("module ready", "module", module) + } + } +} + type RetrievalEvent struct { Event retrievalmarket.ClientEvent Status retrievalmarket.DealStatus diff --git a/markets/storageadapter/client.go b/markets/storageadapter/client.go index fabc5b197..411c86ec9 100644 --- a/markets/storageadapter/client.go +++ b/markets/storageadapter/client.go @@ -438,37 +438,6 @@ func (c *ClientNodeAdapter) GetDefaultWalletAddress(ctx context.Context) (addres return addr, err } -func (c *ClientNodeAdapter) ValidateAskSignature(ctx context.Context, ask *storagemarket.SignedStorageAsk, encodedTs shared.TipSetToken) (bool, error) { - tsk, err := types.TipSetKeyFromBytes(encodedTs) - if err != nil { - return false, err - } - - mi, err := c.StateMinerInfo(ctx, ask.Ask.Miner, tsk) - if err != nil { - return false, xerrors.Errorf("failed to get worker for miner in ask: %w", err) - } - - sigb, err := cborutil.Dump(ask.Ask) - if err != nil { - return false, xerrors.Errorf("failed to re-serialize ask") - } - - ts, err := c.ChainGetTipSet(ctx, tsk) - if err != nil { - return false, xerrors.Errorf("failed to load tipset") - } - - m, err := c.StateManager.ResolveToKeyAddress(ctx, mi.Worker, ts) - - if err != nil { - return false, xerrors.Errorf("failed to resolve miner to key address") - } - - err = sigs.Verify(ask.Signature, m, sigb) - return err == nil, err -} - func (c *ClientNodeAdapter) GetChainHead(ctx context.Context) (shared.TipSetToken, abi.ChainEpoch, error) { head, err := c.ChainHead(ctx) if err != nil { diff --git a/node/builder.go b/node/builder.go index c37a5db58..734726116 100644 --- a/node/builder.go +++ b/node/builder.go @@ -20,8 +20,9 @@ import ( "go.uber.org/fx" "golang.org/x/xerrors" + "github.com/filecoin-project/go-fil-markets/discovery" + discoveryimpl "github.com/filecoin-project/go-fil-markets/discovery/impl" "github.com/filecoin-project/go-fil-markets/retrievalmarket" - "github.com/filecoin-project/go-fil-markets/retrievalmarket/discovery" "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-fil-markets/storagemarket/impl/storedask" @@ -290,8 +291,8 @@ func Online() Option { Override(RunPeerMgrKey, modules.RunPeerMgr), Override(HandleIncomingBlocksKey, modules.HandleIncomingBlocks), - Override(new(*discovery.Local), modules.NewLocalDiscovery), - Override(new(retrievalmarket.PeerResolver), modules.RetrievalResolver), + Override(new(*discoveryimpl.Local), modules.NewLocalDiscovery), + Override(new(discovery.PeerResolver), modules.RetrievalResolver), Override(new(retrievalmarket.RetrievalClient), modules.RetrievalClient), Override(new(dtypes.ClientDatastore), modules.NewClientDatastore), diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 81978af16..7cb087ec7 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -33,6 +33,7 @@ import ( "go.uber.org/fx" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-fil-markets/discovery" "github.com/filecoin-project/go-fil-markets/pieceio" "github.com/filecoin-project/go-fil-markets/retrievalmarket" rm "github.com/filecoin-project/go-fil-markets/retrievalmarket" @@ -70,7 +71,7 @@ type API struct { paych.PaychAPI SMDealClient storagemarket.StorageClient - RetDiscovery rm.PeerResolver + RetDiscovery discovery.PeerResolver Retrieval rm.RetrievalClient Chain *store.ChainStore @@ -614,18 +615,18 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref return } -func (a *API) ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.SignedStorageAsk, error) { +func (a *API) ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.StorageAsk, error) { mi, err := a.StateMinerInfo(ctx, miner, types.EmptyTSK) if err != nil { return nil, xerrors.Errorf("failed getting miner info: %w", err) } info := utils.NewStorageProviderInfo(miner, mi.Worker, mi.SectorSize, p, mi.Multiaddrs) - signedAsk, err := a.SMDealClient.GetAsk(ctx, info) + ask, err := a.SMDealClient.GetAsk(ctx, info) if err != nil { return nil, err } - return signedAsk, nil + return ask, nil } func (a *API) ClientCalcCommP(ctx context.Context, inpath string) (*api.CommPRet, error) { diff --git a/node/modules/client.go b/node/modules/client.go index 6aa3cbb0e..6972ca36e 100644 --- a/node/modules/client.go +++ b/node/modules/client.go @@ -12,8 +12,9 @@ import ( dtimpl "github.com/filecoin-project/go-data-transfer/impl" dtnet "github.com/filecoin-project/go-data-transfer/network" dtgstransport "github.com/filecoin-project/go-data-transfer/transport/graphsync" + "github.com/filecoin-project/go-fil-markets/discovery" + discoveryimpl "github.com/filecoin-project/go-fil-markets/discovery/impl" "github.com/filecoin-project/go-fil-markets/retrievalmarket" - "github.com/filecoin-project/go-fil-markets/retrievalmarket/discovery" retrievalimpl "github.com/filecoin-project/go-fil-markets/retrievalmarket/impl" rmnet "github.com/filecoin-project/go-fil-markets/retrievalmarket/network" "github.com/filecoin-project/go-fil-markets/storagemarket" @@ -112,12 +113,13 @@ func NewClientDealFunds(ds dtypes.MetadataDS) (ClientDealFunds, error) { return funds.NewDealFunds(ds, datastore.NewKey("/marketfunds/client")) } -func StorageClient(lc fx.Lifecycle, h host.Host, ibs dtypes.ClientBlockstore, mds dtypes.ClientMultiDstore, r repo.LockedRepo, dataTransfer dtypes.ClientDataTransfer, discovery *discovery.Local, deals dtypes.ClientDatastore, scn storagemarket.StorageClientNode, dealFunds ClientDealFunds) (storagemarket.StorageClient, error) { +func StorageClient(lc fx.Lifecycle, h host.Host, ibs dtypes.ClientBlockstore, mds dtypes.ClientMultiDstore, r repo.LockedRepo, dataTransfer dtypes.ClientDataTransfer, discovery *discoveryimpl.Local, deals dtypes.ClientDatastore, scn storagemarket.StorageClientNode, dealFunds ClientDealFunds) (storagemarket.StorageClient, error) { net := smnet.NewFromLibp2pHost(h) c, err := storageimpl.NewClient(net, ibs, mds, dataTransfer, discovery, deals, scn, dealFunds, storageimpl.DealPollingInterval(time.Second)) if err != nil { return nil, err } + c.OnReady(marketevents.ReadyLogger("storage client")) lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { c.SubscribeToEvents(marketevents.StorageClientLogger) @@ -135,7 +137,7 @@ func StorageClient(lc fx.Lifecycle, h host.Host, ibs dtypes.ClientBlockstore, md } // RetrievalClient creates a new retrieval client attached to the client blockstore -func RetrievalClient(lc fx.Lifecycle, h host.Host, mds dtypes.ClientMultiDstore, dt dtypes.ClientDataTransfer, payAPI payapi.PaychAPI, resolver retrievalmarket.PeerResolver, ds dtypes.MetadataDS, chainAPI full.ChainAPI, stateAPI full.StateAPI) (retrievalmarket.RetrievalClient, error) { +func RetrievalClient(lc fx.Lifecycle, h host.Host, mds dtypes.ClientMultiDstore, dt dtypes.ClientDataTransfer, payAPI payapi.PaychAPI, resolver discovery.PeerResolver, ds dtypes.MetadataDS, chainAPI full.ChainAPI, stateAPI full.StateAPI) (retrievalmarket.RetrievalClient, error) { adapter := retrievaladapter.NewRetrievalClientNode(payAPI, chainAPI, stateAPI) network := rmnet.NewFromLibp2pHost(h) sc := storedcounter.New(ds, datastore.NewKey("/retr")) @@ -143,6 +145,7 @@ func RetrievalClient(lc fx.Lifecycle, h host.Host, mds dtypes.ClientMultiDstore, if err != nil { return nil, err } + client.OnReady(marketevents.ReadyLogger("retrieval client")) lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { client.SubscribeToEvents(marketevents.RetrievalClientLogger) @@ -150,7 +153,7 @@ func RetrievalClient(lc fx.Lifecycle, h host.Host, mds dtypes.ClientMultiDstore, evtType := journal.J.RegisterEventType("markets/retrieval/client", "state_change") client.SubscribeToEvents(markets.RetrievalClientJournaler(evtType)) - return nil + return client.Start(ctx) }, }) return client, nil diff --git a/node/modules/services.go b/node/modules/services.go index 7bef434be..4ee0abacc 100644 --- a/node/modules/services.go +++ b/node/modules/services.go @@ -13,8 +13,9 @@ import ( "go.uber.org/fx" "golang.org/x/xerrors" - "github.com/filecoin-project/go-fil-markets/retrievalmarket" - "github.com/filecoin-project/go-fil-markets/retrievalmarket/discovery" + "github.com/filecoin-project/go-fil-markets/discovery" + discoveryimpl "github.com/filecoin-project/go-fil-markets/discovery/impl" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain" "github.com/filecoin-project/lotus/chain/beacon" @@ -26,6 +27,7 @@ import ( "github.com/filecoin-project/lotus/chain/sub" "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/lib/peermgr" + marketevents "github.com/filecoin-project/lotus/markets/loggers" "github.com/filecoin-project/lotus/node/hello" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/helpers" @@ -117,12 +119,22 @@ func HandleIncomingMessages(mctx helpers.MetricsCtx, lc fx.Lifecycle, ps *pubsub go sub.HandleIncomingMessages(ctx, mpool, msgsub) } -func NewLocalDiscovery(ds dtypes.MetadataDS) *discovery.Local { - return discovery.NewLocal(namespace.Wrap(ds, datastore.NewKey("/deals/local"))) +func NewLocalDiscovery(lc fx.Lifecycle, ds dtypes.MetadataDS) (*discoveryimpl.Local, error) { + local, err := discoveryimpl.NewLocal(namespace.Wrap(ds, datastore.NewKey("/deals/local"))) + if err != nil { + return nil, err + } + local.OnReady(marketevents.ReadyLogger("discovery")) + lc.Append(fx.Hook{ + OnStart: func(ctx context.Context) error { + return local.Start(ctx) + }, + }) + return local, nil } -func RetrievalResolver(l *discovery.Local) retrievalmarket.PeerResolver { - return discovery.Multi(l) +func RetrievalResolver(l *discoveryimpl.Local) discovery.PeerResolver { + return discoveryimpl.Multi(l) } type RandomBeaconParams struct { diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index de466b004..5eab2ec62 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -29,10 +29,11 @@ import ( dtnet "github.com/filecoin-project/go-data-transfer/network" dtgstransport "github.com/filecoin-project/go-data-transfer/transport/graphsync" piecefilestore "github.com/filecoin-project/go-fil-markets/filestore" - "github.com/filecoin-project/go-fil-markets/piecestore" + piecestoreimpl "github.com/filecoin-project/go-fil-markets/piecestore/impl" "github.com/filecoin-project/go-fil-markets/retrievalmarket" retrievalimpl "github.com/filecoin-project/go-fil-markets/retrievalmarket/impl" rmnet "github.com/filecoin-project/go-fil-markets/retrievalmarket/network" + "github.com/filecoin-project/go-fil-markets/shared" "github.com/filecoin-project/go-fil-markets/storagemarket" storageimpl "github.com/filecoin-project/go-fil-markets/storagemarket/impl" "github.com/filecoin-project/go-fil-markets/storagemarket/impl/funds" @@ -215,14 +216,16 @@ func StorageMiner(fc config.MinerFeeConfig) func(params StorageMinerParams) (*st } func HandleRetrieval(host host.Host, lc fx.Lifecycle, m retrievalmarket.RetrievalProvider) { + m.OnReady(marketevents.ReadyLogger("retrieval provider")) lc.Append(fx.Hook{ - OnStart: func(context.Context) error { + + OnStart: func(ctx context.Context) error { m.SubscribeToEvents(marketevents.RetrievalProviderLogger) evtType := journal.J.RegisterEventType("markets/retrieval/provider", "state_change") m.SubscribeToEvents(markets.RetrievalProviderJournaler(evtType)) - return m.Start() + return m.Start(ctx) }, OnStop: func(context.Context) error { return m.Stop() @@ -232,7 +235,7 @@ func HandleRetrieval(host host.Host, lc fx.Lifecycle, m retrievalmarket.Retrieva func HandleDeals(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, h storagemarket.StorageProvider) { ctx := helpers.LifecycleCtx(mctx, lc) - + h.OnReady(marketevents.ReadyLogger("storage provider")) lc.Append(fx.Hook{ OnStart: func(context.Context) error { h.SubscribeToEvents(marketevents.StorageProviderLogger) @@ -274,8 +277,18 @@ func NewProviderDAGServiceDataTransfer(lc fx.Lifecycle, h host.Host, gs dtypes.S // NewProviderPieceStore creates a statestore for storing metadata about pieces // shared by the storage and retrieval providers -func NewProviderPieceStore(ds dtypes.MetadataDS) dtypes.ProviderPieceStore { - return piecestore.NewPieceStore(namespace.Wrap(ds, datastore.NewKey("/storagemarket"))) +func NewProviderPieceStore(lc fx.Lifecycle, ds dtypes.MetadataDS) (dtypes.ProviderPieceStore, error) { + ps, err := piecestoreimpl.NewPieceStore(namespace.Wrap(ds, datastore.NewKey("/storagemarket"))) + if err != nil { + return nil, err + } + ps.OnReady(marketevents.ReadyLogger("piecestore")) + lc.Append(fx.Hook{ + OnStart: func(ctx context.Context) error { + return ps.Start(ctx) + }, + }) + return ps, nil } func StagingMultiDatastore(lc fx.Lifecycle, r repo.LockedRepo) (dtypes.StagingMultiDstore, error) { @@ -370,7 +383,13 @@ func NewStorageAsk(ctx helpers.MetricsCtx, fapi lapi.FullNode, ds dtypes.Metadat return nil, err } - storedAsk, err := storedask.NewStoredAsk(namespace.Wrap(ds, datastore.NewKey("/deals/provider")), datastore.NewKey("latest-ask"), spn, address.Address(minerAddress)) + providerDs := namespace.Wrap(ds, datastore.NewKey("/deals/provider")) + // legacy this was mistake where this key was place -- so we move the legacy key if need be + err = shared.MoveKey(providerDs, "/latest-ask", "/storage-ask/latest") + if err != nil { + return nil, err + } + storedAsk, err := storedask.NewStoredAsk(namespace.Wrap(providerDs, datastore.NewKey("/storage-ask")), datastore.NewKey("latest"), spn, address.Address(minerAddress)) if err != nil { return nil, err } From 2e9915ac28e803763462dc2d9aef1bf905a05b3a Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 30 Sep 2020 10:58:34 -0700 Subject: [PATCH 459/885] move common from message param into message builder constructor --- chain/actors/builtin/paych/message.go | 14 +++++++------- chain/actors/builtin/paych/message0.go | 20 ++++++++++---------- chain/actors/builtin/paych/message2.go | 20 ++++++++++---------- paychmgr/paych.go | 20 ++++++++++---------- paychmgr/simple.go | 4 ++-- 5 files changed, 39 insertions(+), 39 deletions(-) diff --git a/chain/actors/builtin/paych/message.go b/chain/actors/builtin/paych/message.go index d19f0c38f..23b360394 100644 --- a/chain/actors/builtin/paych/message.go +++ b/chain/actors/builtin/paych/message.go @@ -9,20 +9,20 @@ import ( "github.com/filecoin-project/lotus/chain/types" ) -func Message(version actors.Version) MessageBuilder { +func Message(version actors.Version, from address.Address) MessageBuilder { switch version { case actors.Version0: - return message0{} + return message0{from} case actors.Version2: - return message2{} + return message2{from} default: panic(fmt.Sprintf("unsupported actors version: %d", version)) } } type MessageBuilder interface { - Create(from, to address.Address, initialAmount abi.TokenAmount) (*types.Message, error) - Update(from, paych address.Address, voucher *SignedVoucher, secret []byte) (*types.Message, error) - Settle(from, paych address.Address) (*types.Message, error) - Collect(from, paych address.Address) (*types.Message, error) + Create(to address.Address, initialAmount abi.TokenAmount) (*types.Message, error) + Update(paych address.Address, voucher *SignedVoucher, secret []byte) (*types.Message, error) + Settle(paych address.Address) (*types.Message, error) + Collect(paych address.Address) (*types.Message, error) } diff --git a/chain/actors/builtin/paych/message0.go b/chain/actors/builtin/paych/message0.go index 0e7bbff92..bfeb2731e 100644 --- a/chain/actors/builtin/paych/message0.go +++ b/chain/actors/builtin/paych/message0.go @@ -13,10 +13,10 @@ import ( "github.com/filecoin-project/lotus/chain/types" ) -type message0 struct{} +type message0 struct{ from address.Address } -func (message0) Create(from, to address.Address, initialAmount abi.TokenAmount) (*types.Message, error) { - params, aerr := actors.SerializeParams(&paych0.ConstructorParams{From: from, To: to}) +func (m message0) Create(to address.Address, initialAmount abi.TokenAmount) (*types.Message, error) { + params, aerr := actors.SerializeParams(&paych0.ConstructorParams{From: m.from, To: to}) if aerr != nil { return nil, aerr } @@ -30,14 +30,14 @@ func (message0) Create(from, to address.Address, initialAmount abi.TokenAmount) return &types.Message{ To: init_.Address, - From: from, + From: m.from, Value: initialAmount, Method: builtin0.MethodsInit.Exec, Params: enc, }, nil } -func (message0) Update(from, paych address.Address, sv *SignedVoucher, secret []byte) (*types.Message, error) { +func (m message0) Update(paych address.Address, sv *SignedVoucher, secret []byte) (*types.Message, error) { params, aerr := actors.SerializeParams(&paych0.UpdateChannelStateParams{ Sv: *sv, Secret: secret, @@ -48,26 +48,26 @@ func (message0) Update(from, paych address.Address, sv *SignedVoucher, secret [] return &types.Message{ To: paych, - From: from, + From: m.from, Value: abi.NewTokenAmount(0), Method: builtin0.MethodsPaych.UpdateChannelState, Params: params, }, nil } -func (message0) Settle(from, paych address.Address) (*types.Message, error) { +func (m message0) Settle(paych address.Address) (*types.Message, error) { return &types.Message{ To: paych, - From: from, + From: m.from, Value: abi.NewTokenAmount(0), Method: builtin0.MethodsPaych.Settle, }, nil } -func (message0) Collect(from, paych address.Address) (*types.Message, error) { +func (m message0) Collect(paych address.Address) (*types.Message, error) { return &types.Message{ To: paych, - From: from, + From: m.from, Value: abi.NewTokenAmount(0), Method: builtin0.MethodsPaych.Collect, }, nil diff --git a/chain/actors/builtin/paych/message2.go b/chain/actors/builtin/paych/message2.go index 94538bc49..7d9d8d07c 100644 --- a/chain/actors/builtin/paych/message2.go +++ b/chain/actors/builtin/paych/message2.go @@ -13,10 +13,10 @@ import ( "github.com/filecoin-project/lotus/chain/types" ) -type message2 struct{} +type message2 struct{ from address.Address } -func (message2) Create(from, to address.Address, initialAmount abi.TokenAmount) (*types.Message, error) { - params, aerr := actors.SerializeParams(&paych2.ConstructorParams{From: from, To: to}) +func (m message2) Create(to address.Address, initialAmount abi.TokenAmount) (*types.Message, error) { + params, aerr := actors.SerializeParams(&paych2.ConstructorParams{From: m.from, To: to}) if aerr != nil { return nil, aerr } @@ -30,14 +30,14 @@ func (message2) Create(from, to address.Address, initialAmount abi.TokenAmount) return &types.Message{ To: init_.Address, - From: from, + From: m.from, Value: initialAmount, Method: builtin2.MethodsInit.Exec, Params: enc, }, nil } -func (message2) Update(from, paych address.Address, sv *SignedVoucher, secret []byte) (*types.Message, error) { +func (m message2) Update(paych address.Address, sv *SignedVoucher, secret []byte) (*types.Message, error) { params, aerr := actors.SerializeParams(&paych2.UpdateChannelStateParams{ Sv: *sv, Secret: secret, @@ -48,26 +48,26 @@ func (message2) Update(from, paych address.Address, sv *SignedVoucher, secret [] return &types.Message{ To: paych, - From: from, + From: m.from, Value: abi.NewTokenAmount(0), Method: builtin2.MethodsPaych.UpdateChannelState, Params: params, }, nil } -func (message2) Settle(from, paych address.Address) (*types.Message, error) { +func (m message2) Settle(paych address.Address) (*types.Message, error) { return &types.Message{ To: paych, - From: from, + From: m.from, Value: abi.NewTokenAmount(0), Method: builtin2.MethodsPaych.Settle, }, nil } -func (message2) Collect(from, paych address.Address) (*types.Message, error) { +func (m message2) Collect(paych address.Address) (*types.Message, error) { return &types.Message{ To: paych, - From: from, + From: m.from, Value: abi.NewTokenAmount(0), Method: builtin2.MethodsPaych.Collect, }, nil diff --git a/paychmgr/paych.go b/paychmgr/paych.go index 96ba43543..28563f17d 100644 --- a/paychmgr/paych.go +++ b/paychmgr/paych.go @@ -82,13 +82,13 @@ func newChannelAccessor(pm *Manager, from address.Address, to address.Address) * } } -func (ca *channelAccessor) messageBuilder(ctx context.Context) (paych.MessageBuilder, error) { +func (ca *channelAccessor) messageBuilder(ctx context.Context, from address.Address) (paych.MessageBuilder, error) { nwVersion, err := ca.api.StateNetworkVersion(ctx, types.EmptyTSK) if err != nil { return nil, err } - return paych.Message(actors.VersionForNetwork(nwVersion)), nil + return paych.Message(actors.VersionForNetwork(nwVersion), from), nil } func (ca *channelAccessor) getChannelInfo(addr address.Address) (*ChannelInfo, error) { @@ -301,12 +301,12 @@ func (ca *channelAccessor) checkVoucherSpendable(ctx context.Context, ch address return false, nil } - mb, err := ca.messageBuilder(ctx) + mb, err := ca.messageBuilder(ctx, recipient) if err != nil { return false, err } - mes, err := mb.Update(recipient, ch, sv, secret) + mes, err := mb.Update(ch, sv, secret) if err != nil { return false, err } @@ -418,12 +418,12 @@ func (ca *channelAccessor) submitVoucher(ctx context.Context, ch address.Address } } - mb, err := ca.messageBuilder(ctx) + mb, err := ca.messageBuilder(ctx, ci.Control) if err != nil { return cid.Undef, err } - msg, err := mb.Update(ci.Control, ch, sv, secret) + msg, err := mb.Update(ch, sv, secret) if err != nil { return cid.Undef, err } @@ -574,11 +574,11 @@ func (ca *channelAccessor) settle(ctx context.Context, ch address.Address) (cid. return cid.Undef, err } - mb, err := ca.messageBuilder(ctx) + mb, err := ca.messageBuilder(ctx, ci.Control) if err != nil { return cid.Undef, err } - msg, err := mb.Settle(ci.Control, ch) + msg, err := mb.Settle(ch) if err != nil { return cid.Undef, err } @@ -605,12 +605,12 @@ func (ca *channelAccessor) collect(ctx context.Context, ch address.Address) (cid return cid.Undef, err } - mb, err := ca.messageBuilder(ctx) + mb, err := ca.messageBuilder(ctx, ci.Control) if err != nil { return cid.Undef, err } - msg, err := mb.Collect(ci.Control, ch) + msg, err := mb.Collect(ch) if err != nil { return cid.Undef, err } diff --git a/paychmgr/simple.go b/paychmgr/simple.go index 38804b7ea..9f315da09 100644 --- a/paychmgr/simple.go +++ b/paychmgr/simple.go @@ -384,11 +384,11 @@ func (ca *channelAccessor) processTask(ctx context.Context, amt types.BigInt) *p // createPaych sends a message to create the channel and returns the message cid func (ca *channelAccessor) createPaych(ctx context.Context, amt types.BigInt) (cid.Cid, error) { - mb, err := ca.messageBuilder(ctx) + mb, err := ca.messageBuilder(ctx, ca.from) if err != nil { return cid.Undef, err } - msg, err := mb.Create(ca.from, ca.to, amt) + msg, err := mb.Create(ca.to, amt) if err != nil { return cid.Undef, err } From b01a1d457aced6ca0ac53dbf955b6ac8f492e310 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 30 Sep 2020 12:30:02 -0700 Subject: [PATCH 460/885] abstract multisig calls --- chain/actors/builtin/multisig/message.go | 70 +++++++++ chain/actors/builtin/multisig/message0.go | 142 +++++++++++++++++ chain/actors/builtin/multisig/message2.go | 71 +++++++++ .../multisig/{multisig.go => state.go} | 0 .../builtin/multisig/{v0.go => state0.go} | 0 .../builtin/multisig/{v2.go => state2.go} | 0 node/impl/full/multisig.go | 143 ++++-------------- 7 files changed, 314 insertions(+), 112 deletions(-) create mode 100644 chain/actors/builtin/multisig/message.go create mode 100644 chain/actors/builtin/multisig/message0.go create mode 100644 chain/actors/builtin/multisig/message2.go rename chain/actors/builtin/multisig/{multisig.go => state.go} (100%) rename chain/actors/builtin/multisig/{v0.go => state0.go} (100%) rename chain/actors/builtin/multisig/{v2.go => state2.go} (100%) diff --git a/chain/actors/builtin/multisig/message.go b/chain/actors/builtin/multisig/message.go new file mode 100644 index 000000000..b19287432 --- /dev/null +++ b/chain/actors/builtin/multisig/message.go @@ -0,0 +1,70 @@ +package multisig + +import ( + "fmt" + + "github.com/minio/blake2b-simd" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + + multisig2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/multisig" + + "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/types" +) + +func Message(version actors.Version, from address.Address) MessageBuilder { + switch version { + case actors.Version0: + return message0{from} + case actors.Version2: + return message2{message0{from}} + default: + panic(fmt.Sprintf("unsupported actors version: %d", version)) + } +} + +type MessageBuilder interface { + // Create a new multisig with the specified parameters. + Create(signers []address.Address, threshold uint64, + vestingStart, vestingDuration abi.ChainEpoch, + initialAmount abi.TokenAmount) (*types.Message, error) + + // Propose a transaction to the given multisig. + Propose(msig, target address.Address, amt abi.TokenAmount, + method abi.MethodNum, params []byte) (*types.Message, error) + + // Approve a multisig transaction. The "hash" is optional. + Approve(msig address.Address, txID uint64, hash *ProposalHashData) (*types.Message, error) + + // Cancel a multisig transaction. The "hash" is optional. + Cancel(msig address.Address, txID uint64, hash *ProposalHashData) (*types.Message, error) +} + +// this type is the same between v0 and v2 +type ProposalHashData = multisig2.ProposalHashData + +func txnParams(id uint64, data *ProposalHashData) ([]byte, error) { + params := multisig2.TxnIDParams{ID: multisig2.TxnID(id)} + if data != nil { + if data.Requester.Protocol() != address.ID { + return nil, xerrors.Errorf("proposer address must be an ID address, was %s", data.Requester) + } + if data.Value.Sign() == -1 { + return nil, xerrors.Errorf("proposal value must be non-negative, was %s", data.Value) + } + if data.To == address.Undef { + return nil, xerrors.Errorf("proposed destination address must be set") + } + pser, err := data.Serialize() + if err != nil { + return nil, err + } + hash := blake2b.Sum256(pser) + params.ProposalHash = hash[:] + } + + return actors.SerializeParams(¶ms) +} diff --git a/chain/actors/builtin/multisig/message0.go b/chain/actors/builtin/multisig/message0.go new file mode 100644 index 000000000..dc43a9d5d --- /dev/null +++ b/chain/actors/builtin/multisig/message0.go @@ -0,0 +1,142 @@ +package multisig + +import ( + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" + init0 "github.com/filecoin-project/specs-actors/actors/builtin/init" + multisig0 "github.com/filecoin-project/specs-actors/actors/builtin/multisig" + + "github.com/filecoin-project/lotus/chain/actors" + init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init" + "github.com/filecoin-project/lotus/chain/types" +) + +type message0 struct{ from address.Address } + +func (m message0) Create( + signers []address.Address, threshold uint64, + unlockStart, unlockDuration abi.ChainEpoch, + initialAmount abi.TokenAmount, +) (*types.Message, error) { + + lenAddrs := uint64(len(signers)) + + if lenAddrs < threshold { + return nil, xerrors.Errorf("cannot require signing of more addresses than provided for multisig") + } + + if threshold == 0 { + threshold = lenAddrs + } + + if m.from == address.Undef { + return nil, xerrors.Errorf("must provide source address") + } + + if unlockStart != 0 { + return nil, xerrors.Errorf("actors v0 does not support a non-zero vesting start time") + } + + // Set up constructor parameters for multisig + msigParams := &multisig0.ConstructorParams{ + Signers: signers, + NumApprovalsThreshold: threshold, + UnlockDuration: unlockDuration, + } + + enc, actErr := actors.SerializeParams(msigParams) + if actErr != nil { + return nil, actErr + } + + // new actors are created by invoking 'exec' on the init actor with the constructor params + execParams := &init0.ExecParams{ + CodeCID: builtin0.MultisigActorCodeID, + ConstructorParams: enc, + } + + enc, actErr = actors.SerializeParams(execParams) + if actErr != nil { + return nil, actErr + } + + return &types.Message{ + To: init_.Address, + From: m.from, + Method: builtin0.MethodsInit.Exec, + Params: enc, + Value: initialAmount, + }, nil +} + +func (m message0) Propose(msig, to address.Address, amt abi.TokenAmount, + method abi.MethodNum, params []byte) (*types.Message, error) { + + if msig == address.Undef { + return nil, xerrors.Errorf("must provide a multisig address for proposal") + } + + if to == address.Undef { + return nil, xerrors.Errorf("must provide a target address for proposal") + } + + if amt.Sign() == -1 { + return nil, xerrors.Errorf("must provide a non-negative amount for proposed send") + } + + if m.from == address.Undef { + return nil, xerrors.Errorf("must provide source address") + } + + enc, actErr := actors.SerializeParams(&multisig0.ProposeParams{ + To: to, + Value: amt, + Method: method, + Params: params, + }) + if actErr != nil { + return nil, xerrors.Errorf("failed to serialize parameters: %w", actErr) + } + + return &types.Message{ + To: msig, + From: m.from, + Value: abi.NewTokenAmount(0), + Method: builtin0.MethodsMultisig.Propose, + Params: enc, + }, nil +} + +func (m message0) Approve(msig address.Address, txID uint64, hashData *ProposalHashData) (*types.Message, error) { + enc, err := txnParams(txID, hashData) + if err != nil { + return nil, err + } + + return &types.Message{ + To: msig, + From: m.from, + Value: types.NewInt(0), + Method: builtin0.MethodsMultisig.Approve, + Params: enc, + }, nil +} + +func (m message0) Cancel(msig address.Address, txID uint64, hashData *ProposalHashData) (*types.Message, error) { + enc, err := txnParams(txID, hashData) + if err != nil { + return nil, err + } + + return &types.Message{ + To: msig, + From: m.from, + Value: types.NewInt(0), + Method: builtin0.MethodsMultisig.Cancel, + Params: enc, + }, nil +} diff --git a/chain/actors/builtin/multisig/message2.go b/chain/actors/builtin/multisig/message2.go new file mode 100644 index 000000000..da2700d06 --- /dev/null +++ b/chain/actors/builtin/multisig/message2.go @@ -0,0 +1,71 @@ +package multisig + +import ( + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + init2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/init" + multisig2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/multisig" + + "github.com/filecoin-project/lotus/chain/actors" + init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init" + "github.com/filecoin-project/lotus/chain/types" +) + +type message2 struct{ message0 } + +func (m message2) Create( + signers []address.Address, threshold uint64, + unlockStart, unlockDuration abi.ChainEpoch, + initialAmount abi.TokenAmount, +) (*types.Message, error) { + + lenAddrs := uint64(len(signers)) + + if lenAddrs < threshold { + return nil, xerrors.Errorf("cannot require signing of more addresses than provided for multisig") + } + + if threshold == 0 { + threshold = lenAddrs + } + + if m.from == address.Undef { + return nil, xerrors.Errorf("must provide source address") + } + + // Set up constructor parameters for multisig + msigParams := &multisig2.ConstructorParams{ + Signers: signers, + NumApprovalsThreshold: threshold, + UnlockDuration: unlockDuration, + StartEpoch: unlockStart, + } + + enc, actErr := actors.SerializeParams(msigParams) + if actErr != nil { + return nil, actErr + } + + // new actors are created by invoking 'exec' on the init actor with the constructor params + execParams := &init2.ExecParams{ + CodeCID: builtin2.MultisigActorCodeID, + ConstructorParams: enc, + } + + enc, actErr = actors.SerializeParams(execParams) + if actErr != nil { + return nil, actErr + } + + return &types.Message{ + To: init_.Address, + From: m.from, + Method: builtin2.MethodsInit.Exec, + Params: enc, + Value: initialAmount, + }, nil +} diff --git a/chain/actors/builtin/multisig/multisig.go b/chain/actors/builtin/multisig/state.go similarity index 100% rename from chain/actors/builtin/multisig/multisig.go rename to chain/actors/builtin/multisig/state.go diff --git a/chain/actors/builtin/multisig/v0.go b/chain/actors/builtin/multisig/state0.go similarity index 100% rename from chain/actors/builtin/multisig/v0.go rename to chain/actors/builtin/multisig/state0.go diff --git a/chain/actors/builtin/multisig/v2.go b/chain/actors/builtin/multisig/state2.go similarity index 100% rename from chain/actors/builtin/multisig/v2.go rename to chain/actors/builtin/multisig/state2.go diff --git a/node/impl/full/multisig.go b/node/impl/full/multisig.go index 8c15a27be..715689edc 100644 --- a/node/impl/full/multisig.go +++ b/node/impl/full/multisig.go @@ -9,15 +9,13 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" - init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init" + "github.com/filecoin-project/lotus/chain/actors/builtin/multisig" "github.com/filecoin-project/lotus/chain/types" builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" - init0 "github.com/filecoin-project/specs-actors/actors/builtin/init" multisig0 "github.com/filecoin-project/specs-actors/actors/builtin/multisig" "github.com/ipfs/go-cid" - "github.com/minio/blake2b-simd" "go.uber.org/fx" "golang.org/x/xerrors" ) @@ -30,58 +28,31 @@ type MsigAPI struct { MpoolAPI MpoolAPI } +func (a *MsigAPI) messageBuilder(ctx context.Context, from address.Address) (multisig.MessageBuilder, error) { + nver, err := a.StateAPI.StateNetworkVersion(ctx, types.EmptyTSK) + if err != nil { + return nil, err + } + + return multisig.Message(actors.VersionForNetwork(nver), from), nil +} + // TODO: remove gp (gasPrice) from arguments +// TODO: Add "vesting start" to arguments. func (a *MsigAPI) MsigCreate(ctx context.Context, req uint64, addrs []address.Address, duration abi.ChainEpoch, val types.BigInt, src address.Address, gp types.BigInt) (cid.Cid, error) { - lenAddrs := uint64(len(addrs)) - - if lenAddrs < req { - return cid.Undef, xerrors.Errorf("cannot require signing of more addresses than provided for multisig") + mb, err := a.messageBuilder(ctx, src) + if err != nil { + return cid.Undef, err } - if req == 0 { - req = lenAddrs - } - - if src == address.Undef { - return cid.Undef, xerrors.Errorf("must provide source address") - } - - // Set up constructor parameters for multisig - msigParams := &multisig0.ConstructorParams{ - Signers: addrs, - NumApprovalsThreshold: req, - UnlockDuration: duration, - } - - enc, actErr := actors.SerializeParams(msigParams) - if actErr != nil { - return cid.Undef, actErr - } - - // new actors are created by invoking 'exec' on the init actor with the constructor params - // TODO: network upgrade? - execParams := &init0.ExecParams{ - CodeCID: builtin0.MultisigActorCodeID, - ConstructorParams: enc, - } - - enc, actErr = actors.SerializeParams(execParams) - if actErr != nil { - return cid.Undef, actErr - } - - // now we create the message to send this with - msg := types.Message{ - To: init_.Address, - From: src, - Method: builtin0.MethodsInit.Exec, - Params: enc, - Value: val, + msg, err := mb.Create(addrs, req, 0, duration, val) + if err != nil { + return cid.Undef, err } // send the message out to the network - smsg, err := a.MpoolAPI.MpoolPushMessage(ctx, &msg, nil) + smsg, err := a.MpoolAPI.MpoolPushMessage(ctx, msg, nil) if err != nil { return cid.Undef, err } @@ -91,38 +62,14 @@ func (a *MsigAPI) MsigCreate(ctx context.Context, req uint64, addrs []address.Ad func (a *MsigAPI) MsigPropose(ctx context.Context, msig address.Address, to address.Address, amt types.BigInt, src address.Address, method uint64, params []byte) (cid.Cid, error) { - if msig == address.Undef { - return cid.Undef, xerrors.Errorf("must provide a multisig address for proposal") + mb, err := a.messageBuilder(ctx, src) + if err != nil { + return cid.Undef, err } - if to == address.Undef { - return cid.Undef, xerrors.Errorf("must provide a target address for proposal") - } - - if amt.Sign() == -1 { - return cid.Undef, xerrors.Errorf("must provide a positive amount for proposed send") - } - - if src == address.Undef { - return cid.Undef, xerrors.Errorf("must provide source address") - } - - enc, actErr := actors.SerializeParams(&multisig0.ProposeParams{ - To: to, - Value: amt, - Method: abi.MethodNum(method), - Params: params, - }) - if actErr != nil { - return cid.Undef, xerrors.Errorf("failed to serialize parameters: %w", actErr) - } - - msg := &types.Message{ - To: msig, - From: src, - Value: types.NewInt(0), - Method: builtin0.MethodsMultisig.Propose, - Params: enc, + msg, err := mb.Propose(msig, to, amt, abi.MethodNum(method), params) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to create proposal: %w", err) } smsg, err := a.MpoolAPI.MpoolPushMessage(ctx, msg, nil) @@ -200,14 +147,6 @@ func (a *MsigAPI) msigApproveOrCancel(ctx context.Context, operation api.MsigPro return cid.Undef, xerrors.Errorf("must provide multisig address") } - if to == address.Undef { - return cid.Undef, xerrors.Errorf("must provide proposed target address") - } - - if amt.Sign() == -1 { - return cid.Undef, xerrors.Errorf("must provide the positive amount that was proposed") - } - if src == address.Undef { return cid.Undef, xerrors.Errorf("must provide source address") } @@ -220,7 +159,7 @@ func (a *MsigAPI) msigApproveOrCancel(ctx context.Context, operation api.MsigPro proposer = proposerID } - p := multisig0.ProposalHashData{ + p := multisig.ProposalHashData{ Requester: proposer, To: to, Value: amt, @@ -228,42 +167,22 @@ func (a *MsigAPI) msigApproveOrCancel(ctx context.Context, operation api.MsigPro Params: params, } - pser, err := p.Serialize() - if err != nil { - return cid.Undef, err - } - phash := blake2b.Sum256(pser) - - enc, err := actors.SerializeParams(&multisig0.TxnIDParams{ - ID: multisig0.TxnID(txID), - ProposalHash: phash[:], - }) - + mb, err := a.messageBuilder(ctx, src) if err != nil { return cid.Undef, err } - var msigResponseMethod abi.MethodNum - - /* - We pass in a MsigProposeResponse instead of MethodNum to - tighten the possible inputs to just Approve and Cancel. - */ + var msg *types.Message switch operation { case api.MsigApprove: - msigResponseMethod = builtin0.MethodsMultisig.Approve + msg, err = mb.Approve(msig, txID, &p) case api.MsigCancel: - msigResponseMethod = builtin0.MethodsMultisig.Cancel + msg, err = mb.Cancel(msig, txID, &p) default: return cid.Undef, xerrors.Errorf("Invalid operation for msigApproveOrCancel") } - - msg := &types.Message{ - To: msig, - From: src, - Value: types.NewInt(0), - Method: msigResponseMethod, - Params: enc, + if err != nil { + return cid.Undef, err } smsg, err := a.MpoolAPI.MpoolPushMessage(ctx, msg, nil) From 99fe63716bb7147f856541442cf506e2ddeacce0 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 30 Sep 2020 13:12:17 -0700 Subject: [PATCH 461/885] regen lotuspond info --- lotuspond/front/src/chain/methods.json | 106 +++++++++++++++++++++++++ 1 file changed, 106 insertions(+) diff --git a/lotuspond/front/src/chain/methods.json b/lotuspond/front/src/chain/methods.json index ce4919cc4..5e15b053b 100644 --- a/lotuspond/front/src/chain/methods.json +++ b/lotuspond/front/src/chain/methods.json @@ -87,6 +87,10 @@ "SubmitPoRepForBulkVerify", "CurrentTotalPower" ], + "fil/1/system": [ + "Send", + "Constructor" + ], "fil/1/verifiedregistry": [ "Send", "Constructor", @@ -95,5 +99,107 @@ "AddVerifiedClient", "UseBytes", "RestoreBytes" + ], + "fil/2/account": [ + "Send", + "Constructor", + "PubkeyAddress" + ], + "fil/2/cron": [ + "Send", + "Constructor", + "EpochTick" + ], + "fil/2/init": [ + "Send", + "Constructor", + "Exec" + ], + "fil/2/multisig": [ + "Send", + "Constructor", + "Propose", + "Approve", + "Cancel", + "AddSigner", + "RemoveSigner", + "SwapSigner", + "ChangeNumApprovalsThreshold", + "LockBalance" + ], + "fil/2/paymentchannel": [ + "Send", + "Constructor", + "UpdateChannelState", + "Settle", + "Collect" + ], + "fil/2/reward": [ + "Send", + "Constructor", + "AwardBlockReward", + "ThisEpochReward", + "UpdateNetworkKPI" + ], + "fil/2/storagemarket": [ + "Send", + "Constructor", + "AddBalance", + "WithdrawBalance", + "PublishStorageDeals", + "VerifyDealsForActivation", + "ActivateDeals", + "OnMinerSectorsTerminate", + "ComputeDataCommitment", + "CronTick" + ], + "fil/2/storageminer": [ + "Send", + "Constructor", + "ControlAddresses", + "ChangeWorkerAddress", + "ChangePeerID", + "SubmitWindowedPoSt", + "PreCommitSector", + "ProveCommitSector", + "ExtendSectorExpiration", + "TerminateSectors", + "DeclareFaults", + "DeclareFaultsRecovered", + "OnDeferredCronEvent", + "CheckSectorProven", + "ApplyRewards", + "ReportConsensusFault", + "WithdrawBalance", + "ConfirmSectorProofsValid", + "ChangeMultiaddrs", + "CompactPartitions", + "CompactSectorNumbers", + "ConfirmUpdateWorkerKey", + "RepayDebt" + ], + "fil/2/storagepower": [ + "Send", + "Constructor", + "CreateMiner", + "UpdateClaimedPower", + "EnrollCronEvent", + "OnEpochTickEnd", + "UpdatePledgeTotal", + "SubmitPoRepForBulkVerify", + "CurrentTotalPower" + ], + "fil/2/system": [ + "Send", + "Constructor" + ], + "fil/2/verifiedregistry": [ + "Send", + "Constructor", + "AddVerifier", + "RemoveVerifier", + "AddVerifiedClient", + "UseBytes", + "RestoreBytes" ] } \ No newline at end of file From 35e606d39775fbbfb90801ba8218e7ba441cd1cc Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 30 Sep 2020 13:17:56 -0700 Subject: [PATCH 462/885] remove direct specs-actors miner access from expiration calculation --- node/impl/client/client.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 7cb087ec7..ece29bc65 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -42,7 +42,6 @@ import ( "github.com/filecoin-project/go-multistore" "github.com/filecoin-project/go-padreader" "github.com/filecoin-project/go-state-types/abi" - miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" marketevents "github.com/filecoin-project/lotus/markets/loggers" @@ -88,7 +87,7 @@ func calcDealExpiration(minDuration uint64, md *dline.Info, startEpoch abi.Chain minExp := startEpoch + abi.ChainEpoch(minDuration) // Align on miners ProvingPeriodBoundary - return minExp + miner0.WPoStProvingPeriod - (minExp % miner0.WPoStProvingPeriod) + (md.PeriodStart % miner0.WPoStProvingPeriod) - 1 + return minExp + md.WPoStProvingPeriod - (minExp % md.WPoStProvingPeriod) + (md.PeriodStart % md.WPoStProvingPeriod) - 1 } func (a *API) imgr() *importmgr.Mgr { From 8dcbd525da9a65d713398f24d1e28bbae3a78464 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 30 Sep 2020 13:30:24 -0700 Subject: [PATCH 463/885] abstract over deal collateral --- chain/actors/policy/policy.go | 18 ++++++++++++++++++ node/impl/full/state.go | 22 ++++++++++------------ 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/chain/actors/policy/policy.go b/chain/actors/policy/policy.go index 2101cfdcf..8606119f0 100644 --- a/chain/actors/policy/policy.go +++ b/chain/actors/policy/policy.go @@ -2,12 +2,15 @@ package policy import ( "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/chain/actors" + market0 "github.com/filecoin-project/specs-actors/actors/builtin/market" miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" power0 "github.com/filecoin-project/specs-actors/actors/builtin/power" verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + market2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/market" miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" verifreg2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/verifreg" ) @@ -79,3 +82,18 @@ func GetMaxProveCommitDuration(ver actors.Version, t abi.RegisteredSealProof) ab panic("unsupported actors version") } } + +func DealProviderCollateralBounds( + size abi.PaddedPieceSize, verified bool, + rawBytePower, qaPower, baselinePower abi.StoragePower, + circulatingFil abi.TokenAmount, nwVer network.Version, +) (min, max abi.TokenAmount) { + switch actors.VersionForNetwork(nwVer) { + case actors.Version0: + return market0.DealProviderCollateralBounds(size, verified, rawBytePower, qaPower, baselinePower, circulatingFil, nwVer) + case actors.Version2: + return market2.DealProviderCollateralBounds(size, verified, rawBytePower, qaPower, baselinePower, circulatingFil) + default: + panic("unsupported network version") + } +} diff --git a/node/impl/full/state.go b/node/impl/full/state.go index d2bf5cf25..fc30eb3bb 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -5,10 +5,8 @@ import ( "context" "strconv" - lotusbuiltin "github.com/filecoin-project/lotus/chain/actors/builtin" - - builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" - market0 "github.com/filecoin-project/specs-actors/actors/builtin/market" + "github.com/filecoin-project/lotus/chain/actors/builtin" + "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/actors/builtin/verifreg" @@ -883,10 +881,10 @@ func (a *StateAPI) StateMinerPreCommitDepositForPower(ctx context.Context, maddr } else { // NB: not exactly accurate, but should always lead us to *over* estimate, not under duration := pci.Expiration - ts.Height() - sectorWeight = lotusbuiltin.QAPowerForWeight(ssize, duration, w, vw) + sectorWeight = builtin.QAPowerForWeight(ssize, duration, w, vw) } - var powerSmoothed lotusbuiltin.FilterEstimate + var powerSmoothed builtin.FilterEstimate if act, err := state.GetActor(power.Address); err != nil { return types.EmptyInt, xerrors.Errorf("loading power actor: %w", err) } else if s, err := power.Load(store, act); err != nil { @@ -944,11 +942,11 @@ func (a *StateAPI) StateMinerInitialPledgeCollateral(ctx context.Context, maddr } else { // NB: not exactly accurate, but should always lead us to *over* estimate, not under duration := pci.Expiration - ts.Height() - sectorWeight = lotusbuiltin.QAPowerForWeight(ssize, duration, w, vw) + sectorWeight = builtin.QAPowerForWeight(ssize, duration, w, vw) } var ( - powerSmoothed lotusbuiltin.FilterEstimate + powerSmoothed builtin.FilterEstimate pledgeCollateral abi.TokenAmount ) if act, err := state.GetActor(power.Address); err != nil { @@ -1025,7 +1023,7 @@ func (a *StateAPI) StateMinerAvailableBalance(ctx context.Context, maddr address // Returns zero if there is no entry in the data cap table for the // address. func (a *StateAPI) StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) { - act, err := a.StateGetActor(ctx, builtin0.VerifiedRegistryActorAddr, tsk) + act, err := a.StateGetActor(ctx, verifreg.Address, tsk) if err != nil { return nil, err } @@ -1063,12 +1061,12 @@ func (a *StateAPI) StateDealProviderCollateralBounds(ctx context.Context, size a return api.DealCollateralBounds{}, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - pact, err := a.StateGetActor(ctx, builtin0.StoragePowerActorAddr, tsk) + pact, err := a.StateGetActor(ctx, power.Address, tsk) if err != nil { return api.DealCollateralBounds{}, xerrors.Errorf("failed to load power actor: %w", err) } - ract, err := a.StateGetActor(ctx, builtin0.RewardActorAddr, tsk) + ract, err := a.StateGetActor(ctx, reward.Address, tsk) if err != nil { return api.DealCollateralBounds{}, xerrors.Errorf("failed to load reward actor: %w", err) } @@ -1098,7 +1096,7 @@ func (a *StateAPI) StateDealProviderCollateralBounds(ctx context.Context, size a return api.DealCollateralBounds{}, xerrors.Errorf("getting reward baseline power: %w", err) } - min, max := market0.DealProviderCollateralBounds(size, + min, max := policy.DealProviderCollateralBounds(size, verified, powClaim.RawBytePower, powClaim.QualityAdjPower, From 8091bb3117366874fcd58059198fb13adf79f970 Mon Sep 17 00:00:00 2001 From: ZenGround0 Date: Wed, 30 Sep 2020 16:54:30 -0400 Subject: [PATCH 464/885] Run fork function after cron --- chain/stmgr/stmgr.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index e800ce665..b9810fab9 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -209,6 +209,18 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp } for i := parentEpoch; i < epoch; i++ { + if i > parentEpoch { + // run cron for null rounds if any + if err := runCron(); err != nil { + return cid.Cid{}, cid.Cid{}, err + } + + pstate, err = vmi.Flush(ctx) + if err != nil { + return cid.Undef, cid.Undef, xerrors.Errorf("flushing vm: %w", err) + } + } + // handle state forks // XXX: The state tree newState, err := sm.handleStateForks(ctx, pstate, i, cb, ts) @@ -223,18 +235,6 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp } } - if i > parentEpoch { - // run cron for null rounds if any - if err := runCron(); err != nil { - return cid.Cid{}, cid.Cid{}, err - } - - newState, err = vmi.Flush(ctx) - if err != nil { - return cid.Undef, cid.Undef, xerrors.Errorf("flushing vm: %w", err) - } - } - vmi.SetBlockHeight(i + 1) pstate = newState } From 6825a485eb5f13f3900734780e8a2572cc1b7120 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 30 Sep 2020 14:28:25 -0700 Subject: [PATCH 465/885] update message extraction tool for specs-actors update --- cmd/tvx/actor_mapping.go | 44 ---------------------------------------- cmd/tvx/extract_many.go | 21 ++++++++++++++----- 2 files changed, 16 insertions(+), 49 deletions(-) delete mode 100644 cmd/tvx/actor_mapping.go diff --git a/cmd/tvx/actor_mapping.go b/cmd/tvx/actor_mapping.go deleted file mode 100644 index 8c306aca0..000000000 --- a/cmd/tvx/actor_mapping.go +++ /dev/null @@ -1,44 +0,0 @@ -package main - -import ( - "reflect" - - "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/ipfs/go-cid" - "github.com/multiformats/go-multihash" -) - -var ActorMethodTable = make(map[string][]string, 64) - -var Actors = map[cid.Cid]interface{}{ - builtin.InitActorCodeID: builtin.MethodsInit, - builtin.CronActorCodeID: builtin.MethodsCron, - builtin.AccountActorCodeID: builtin.MethodsAccount, - builtin.StoragePowerActorCodeID: builtin.MethodsPower, - builtin.StorageMinerActorCodeID: builtin.MethodsMiner, - builtin.StorageMarketActorCodeID: builtin.MethodsMarket, - builtin.PaymentChannelActorCodeID: builtin.MethodsPaych, - builtin.MultisigActorCodeID: builtin.MethodsMultisig, - builtin.RewardActorCodeID: builtin.MethodsReward, - builtin.VerifiedRegistryActorCodeID: builtin.MethodsVerifiedRegistry, -} - -func init() { - for code, methods := range Actors { - cmh, err := multihash.Decode(code.Hash()) // identity hash. - if err != nil { - panic(err) - } - - var ( - aname = string(cmh.Digest) - rt = reflect.TypeOf(methods) - nf = rt.NumField() - ) - - ActorMethodTable[aname] = append(ActorMethodTable[aname], "Send") - for i := 0; i < nf; i++ { - ActorMethodTable[aname] = append(ActorMethodTable[aname], rt.Field(i).Name) - } - } -} diff --git a/cmd/tvx/extract_many.go b/cmd/tvx/extract_many.go index 9679a1dbd..fe0ce6a6c 100644 --- a/cmd/tvx/extract_many.go +++ b/cmd/tvx/extract_many.go @@ -12,10 +12,14 @@ import ( "strings" "github.com/fatih/color" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/exitcode" "github.com/hashicorp/go-multierror" + "github.com/ipfs/go-cid" + "github.com/multiformats/go-multihash" "github.com/urfave/cli/v2" + "github.com/filecoin-project/lotus/chain/stmgr" lcli "github.com/filecoin-project/lotus/cli" ) @@ -118,6 +122,8 @@ func runExtractMany(c *cli.Context) error { log.Println(color.GreenString("csv sanity check succeeded; header contains fields: %v", header)) } + codeCidBuilder := cid.V1Builder{Codec: cid.Raw, MhType: multihash.IDENTITY} + var ( generated []string merr = new(multierror.Error) @@ -133,7 +139,7 @@ func runExtractMany(c *cli.Context) error { return fmt.Errorf("failed to read row: %w", err) } var ( - cid = row[0] + mcid = row[0] actorcode = row[1] methodnumstr = row[2] exitcodestr = row[3] @@ -155,13 +161,18 @@ func runExtractMany(c *cli.Context) error { return fmt.Errorf("invalid method number: %s", methodnumstr) } + codeCid, err := codeCidBuilder.Sum([]byte(actorcode)) + if err != nil { + return fmt.Errorf("failed to compute actor code CID") + } + // Lookup the method in actor method table. - if m, ok := ActorMethodTable[actorcode]; !ok { + if m, ok := stmgr.MethodsMap[codeCid]; !ok { return fmt.Errorf("unrecognized actor: %s", actorcode) } else if methodnum >= len(m) { return fmt.Errorf("unrecognized method number for actor %s: %d", actorcode, methodnum) } else { - methodname = m[methodnum] + methodname = m[abi.MethodNum(methodnum)].Name } // exitcode string representations are of kind ErrType(0); strip out @@ -181,14 +192,14 @@ func runExtractMany(c *cli.Context) error { id: id, block: block, class: "message", - cid: cid, + cid: mcid, file: file, retain: "accessed-cids", precursor: PrecursorSelectSender, } if err := doExtract(ctx, fapi, opts); err != nil { - log.Println(color.RedString("failed to extract vector for message %s: %s; queuing for 'canonical' precursor selection", cid, err)) + log.Println(color.RedString("failed to extract vector for message %s: %s; queuing for 'canonical' precursor selection", mcid, err)) retry = append(retry, opts) continue } From 486812e082ec85693e2d4dd33c2248cae7148aea Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 30 Sep 2020 14:55:50 -0700 Subject: [PATCH 466/885] name test networks Necessary to get upgrades to run. --- node/test/builder.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/node/test/builder.go b/node/test/builder.go index c496b1e4c..fb3aa71d4 100644 --- a/node/test/builder.go +++ b/node/test/builder.go @@ -198,6 +198,7 @@ func Builder(t *testing.T, nFull int, storage []test.StorageMiner) ([]test.TestN templ := &genesis.Template{ Accounts: genaccs, Miners: genms, + NetworkName: "test", Timestamp: uint64(time.Now().Unix() - 10000), // some time sufficiently far in the past VerifregRootKey: gen.DefaultVerifregRootkeyActor, RemainderAccount: gen.DefaultRemainderAccountActor, @@ -344,6 +345,7 @@ func MockSbBuilder(t *testing.T, nFull int, storage []test.StorageMiner) ([]test templ := &genesis.Template{ Accounts: genaccs, Miners: genms, + NetworkName: "test", Timestamp: uint64(time.Now().Unix()) - (build.BlockDelaySecs * 20000), VerifregRootKey: gen.DefaultVerifregRootkeyActor, RemainderAccount: gen.DefaultRemainderAccountActor, From 42d5d8d784ed1602725dc8878088ba46c764001b Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 30 Sep 2020 14:56:16 -0700 Subject: [PATCH 467/885] run upgrade logic on stmgr.Call --- chain/stmgr/call.go | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/chain/stmgr/call.go b/chain/stmgr/call.go index f4dfc7115..2f049cae3 100644 --- a/chain/stmgr/call.go +++ b/chain/stmgr/call.go @@ -5,7 +5,6 @@ import ( "fmt" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/crypto" "github.com/ipfs/go-cid" "go.opencensus.io/trace" @@ -18,14 +17,30 @@ import ( "github.com/filecoin-project/lotus/chain/vm" ) -func (sm *StateManager) CallRaw(ctx context.Context, msg *types.Message, bstate cid.Cid, r vm.Rand, bheight abi.ChainEpoch) (*api.InvocResult, error) { - ctx, span := trace.StartSpan(ctx, "statemanager.CallRaw") +func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types.TipSet) (*api.InvocResult, error) { + ctx, span := trace.StartSpan(ctx, "statemanager.Call") defer span.End() + if ts == nil { + ts = sm.cs.GetHeaviestTipSet() + } + + bstate := ts.ParentState() + bheight := ts.Height() + + newState, err := sm.handleStateForks(ctx, bstate, bheight-1, nil, ts) + if err != nil { + return nil, fmt.Errorf("failed to handle fork") + } + if newState != bstate { + fmt.Println("IT WORKED!") + } + bstate = newState + vmopt := &vm.VMOpts{ StateBase: bstate, Epoch: bheight, - Rand: r, + Rand: store.NewChainRand(sm.cs, ts.Cids()), Bstore: sm.cs.Blockstore(), Syscalls: sm.cs.VMSys(), CircSupplyCalc: sm.GetCirculatingSupply, @@ -89,18 +104,6 @@ func (sm *StateManager) CallRaw(ctx context.Context, msg *types.Message, bstate } -func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types.TipSet) (*api.InvocResult, error) { - if ts == nil { - ts = sm.cs.GetHeaviestTipSet() - } - - state := ts.ParentState() - - r := store.NewChainRand(sm.cs, ts.Cids()) - - return sm.CallRaw(ctx, msg, state, r, ts.Height()) -} - func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, priorMsgs []types.ChainMsg, ts *types.TipSet) (*api.InvocResult, error) { ctx, span := trace.StartSpan(ctx, "statemanager.CallWithGas") defer span.End() From 9d59635b427f4a2f827b5dc48d1ca99ef5931f0a Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 30 Sep 2020 15:04:16 -0700 Subject: [PATCH 468/885] fix error messages --- chain/stmgr/stmgr.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index c5968c4a4..2ce393378 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -280,7 +280,7 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp sysAct, actErr := vmi.StateTree().GetActor(builtin0.SystemActorAddr) if actErr != nil { - return cid.Undef, cid.Undef, xerrors.Errorf("failed to get system actor: %w", err) + return cid.Undef, cid.Undef, xerrors.Errorf("failed to get system actor: %w", actErr) } rwMsg := &types.Message{ @@ -296,7 +296,7 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp } ret, actErr := vmi.ApplyImplicitMessage(ctx, rwMsg) if actErr != nil { - return cid.Undef, cid.Undef, xerrors.Errorf("failed to apply reward message for miner %s: %w", b.Miner, err) + return cid.Undef, cid.Undef, xerrors.Errorf("failed to apply reward message for miner %s: %w", b.Miner, actErr) } if cb != nil { if err := cb(rwMsg.Cid(), rwMsg, ret); err != nil { From 9705b25bbdb349b8efb502ee53cfeb3406e99381 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 30 Sep 2020 15:15:34 -0700 Subject: [PATCH 469/885] fix testground actor versions --- build/params_testground.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/params_testground.go b/build/params_testground.go index 975c26599..de554f49e 100644 --- a/build/params_testground.go +++ b/build/params_testground.go @@ -82,8 +82,8 @@ var ( 0: DrandMainnet, } - NewestNetworkVersion = network.Version2 - ActorUpgradeNetworkVersion = network.Version3 + NewestNetworkVersion = network.Version4 + ActorUpgradeNetworkVersion = network.Version4 Devnet = true ) From 5e08d56630adae797e2f94e9279bb77ab3a7f820 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 1 Oct 2020 00:27:54 +0200 Subject: [PATCH 470/885] sched: Allow some single-thread tasks to run in parallel with PC2/C2 --- extern/sector-storage/resources.go | 116 ++++++++++++++--------- extern/sector-storage/sched_resources.go | 27 +----- 2 files changed, 75 insertions(+), 68 deletions(-) diff --git a/extern/sector-storage/resources.go b/extern/sector-storage/resources.go index 28ab47e6f..320fec71f 100644 --- a/extern/sector-storage/resources.go +++ b/extern/sector-storage/resources.go @@ -10,14 +10,38 @@ type Resources struct { MinMemory uint64 // What Must be in RAM for decent perf MaxMemory uint64 // Memory required (swap + ram) - Threads int // -1 = multithread - CanGPU bool + MaxParallelism int // -1 = multithread + CanGPU bool BaseMinMemory uint64 // What Must be in RAM for decent perf (shared between threads) } -func (r Resources) MultiThread() bool { - return r.Threads == -1 +/* + + Percent of threads to allocate to parallel tasks + + 12 * 0.92 = 11 + 16 * 0.92 = 14 + 24 * 0.92 = 22 + 32 * 0.92 = 29 + 64 * 0.92 = 58 + 128 * 0.92 = 117 + +*/ +var ParallelNum uint64 = 92 +var ParallelDenom uint64 = 100 + +// TODO: Take NUMA into account +func (r Resources) Threads(wcpus uint64) uint64 { + if r.MaxParallelism == -1 { + n := (wcpus * ParallelNum) / ParallelDenom + if n == 0 { + return wcpus + } + return n + } + + return uint64(r.MaxParallelism) } var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources{ @@ -26,7 +50,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 8 << 30, MinMemory: 8 << 30, - Threads: 1, + MaxParallelism: 1, BaseMinMemory: 1 << 30, }, @@ -34,7 +58,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 4 << 30, MinMemory: 4 << 30, - Threads: 1, + MaxParallelism: 1, BaseMinMemory: 1 << 30, }, @@ -42,7 +66,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 1 << 30, MinMemory: 1 << 30, - Threads: 1, + MaxParallelism: 1, BaseMinMemory: 1 << 30, }, @@ -50,7 +74,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 2 << 10, MinMemory: 2 << 10, - Threads: 1, + MaxParallelism: 1, BaseMinMemory: 2 << 10, }, @@ -58,7 +82,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 8 << 20, MinMemory: 8 << 20, - Threads: 1, + MaxParallelism: 1, BaseMinMemory: 8 << 20, }, @@ -68,7 +92,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 128 << 30, MinMemory: 112 << 30, - Threads: 1, + MaxParallelism: 1, BaseMinMemory: 10 << 20, }, @@ -76,7 +100,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 64 << 30, MinMemory: 56 << 30, - Threads: 1, + MaxParallelism: 1, BaseMinMemory: 10 << 20, }, @@ -84,7 +108,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 1 << 30, MinMemory: 768 << 20, - Threads: 1, + MaxParallelism: 1, BaseMinMemory: 1 << 20, }, @@ -92,7 +116,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 2 << 10, MinMemory: 2 << 10, - Threads: 1, + MaxParallelism: 1, BaseMinMemory: 2 << 10, }, @@ -100,7 +124,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 8 << 20, MinMemory: 8 << 20, - Threads: 1, + MaxParallelism: 1, BaseMinMemory: 8 << 20, }, @@ -110,8 +134,8 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 64 << 30, MinMemory: 64 << 30, - Threads: -1, - CanGPU: true, + MaxParallelism: -1, + CanGPU: true, BaseMinMemory: 60 << 30, }, @@ -119,8 +143,8 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 32 << 30, MinMemory: 32 << 30, - Threads: -1, - CanGPU: true, + MaxParallelism: -1, + CanGPU: true, BaseMinMemory: 30 << 30, }, @@ -128,7 +152,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 3 << 29, // 1.5G MinMemory: 1 << 30, - Threads: -1, + MaxParallelism: -1, BaseMinMemory: 1 << 30, }, @@ -136,7 +160,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 2 << 10, MinMemory: 2 << 10, - Threads: -1, + MaxParallelism: -1, BaseMinMemory: 2 << 10, }, @@ -144,7 +168,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 8 << 20, MinMemory: 8 << 20, - Threads: -1, + MaxParallelism: -1, BaseMinMemory: 8 << 20, }, @@ -154,7 +178,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 1 << 30, MinMemory: 1 << 30, - Threads: 0, + MaxParallelism: 0, BaseMinMemory: 1 << 30, }, @@ -162,7 +186,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 1 << 30, MinMemory: 1 << 30, - Threads: 0, + MaxParallelism: 0, BaseMinMemory: 1 << 30, }, @@ -170,7 +194,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 1 << 30, MinMemory: 1 << 30, - Threads: 0, + MaxParallelism: 0, BaseMinMemory: 1 << 30, }, @@ -178,7 +202,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 2 << 10, MinMemory: 2 << 10, - Threads: 0, + MaxParallelism: 0, BaseMinMemory: 2 << 10, }, @@ -186,7 +210,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 8 << 20, MinMemory: 8 << 20, - Threads: 0, + MaxParallelism: 0, BaseMinMemory: 8 << 20, }, @@ -196,8 +220,8 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 190 << 30, // TODO: Confirm MinMemory: 60 << 30, - Threads: -1, - CanGPU: true, + MaxParallelism: -1, + CanGPU: true, BaseMinMemory: 64 << 30, // params }, @@ -205,8 +229,8 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 150 << 30, // TODO: ~30G of this should really be BaseMaxMemory MinMemory: 30 << 30, - Threads: -1, - CanGPU: true, + MaxParallelism: -1, + CanGPU: true, BaseMinMemory: 32 << 30, // params }, @@ -214,8 +238,8 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 3 << 29, // 1.5G MinMemory: 1 << 30, - Threads: 1, // This is fine - CanGPU: true, + MaxParallelism: 1, // This is fine + CanGPU: true, BaseMinMemory: 10 << 30, }, @@ -223,8 +247,8 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 2 << 10, MinMemory: 2 << 10, - Threads: 1, - CanGPU: true, + MaxParallelism: 1, + CanGPU: true, BaseMinMemory: 2 << 10, }, @@ -232,8 +256,8 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 8 << 20, MinMemory: 8 << 20, - Threads: 1, - CanGPU: true, + MaxParallelism: 1, + CanGPU: true, BaseMinMemory: 8 << 20, }, @@ -243,8 +267,8 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 1 << 20, MinMemory: 1 << 20, - Threads: 0, - CanGPU: false, + MaxParallelism: 0, + CanGPU: false, BaseMinMemory: 0, }, @@ -252,8 +276,8 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 1 << 20, MinMemory: 1 << 20, - Threads: 0, - CanGPU: false, + MaxParallelism: 0, + CanGPU: false, BaseMinMemory: 0, }, @@ -261,8 +285,8 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 1 << 20, MinMemory: 1 << 20, - Threads: 0, - CanGPU: false, + MaxParallelism: 0, + CanGPU: false, BaseMinMemory: 0, }, @@ -270,8 +294,8 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 1 << 20, MinMemory: 1 << 20, - Threads: 0, - CanGPU: false, + MaxParallelism: 0, + CanGPU: false, BaseMinMemory: 0, }, @@ -279,8 +303,8 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 1 << 20, MinMemory: 1 << 20, - Threads: 0, - CanGPU: false, + MaxParallelism: 0, + CanGPU: false, BaseMinMemory: 0, }, diff --git a/extern/sector-storage/sched_resources.go b/extern/sector-storage/sched_resources.go index 623472a20..d6dae577b 100644 --- a/extern/sector-storage/sched_resources.go +++ b/extern/sector-storage/sched_resources.go @@ -28,12 +28,7 @@ func (a *activeResources) withResources(id WorkerID, wr storiface.WorkerResource func (a *activeResources) add(wr storiface.WorkerResources, r Resources) { a.gpuUsed = r.CanGPU - if r.MultiThread() { - a.cpuUse += wr.CPUs - } else { - a.cpuUse += uint64(r.Threads) - } - + a.cpuUse += r.Threads(wr.CPUs) a.memUsedMin += r.MinMemory a.memUsedMax += r.MaxMemory } @@ -42,12 +37,7 @@ func (a *activeResources) free(wr storiface.WorkerResources, r Resources) { if r.CanGPU { a.gpuUsed = false } - if r.MultiThread() { - a.cpuUse -= wr.CPUs - } else { - a.cpuUse -= uint64(r.Threads) - } - + a.cpuUse -= r.Threads(wr.CPUs) a.memUsedMin -= r.MinMemory a.memUsedMax -= r.MaxMemory } @@ -68,16 +58,9 @@ func (a *activeResources) canHandleRequest(needRes Resources, wid WorkerID, call return false } - if needRes.MultiThread() { - if a.cpuUse > 0 { - log.Debugf("sched: not scheduling on worker %d for %s; multicore process needs %d threads, %d in use, target %d", wid, caller, res.CPUs, a.cpuUse, res.CPUs) - return false - } - } else { - if a.cpuUse+uint64(needRes.Threads) > res.CPUs { - log.Debugf("sched: not scheduling on worker %d for %s; not enough threads, need %d, %d in use, target %d", wid, caller, needRes.Threads, a.cpuUse, res.CPUs) - return false - } + if a.cpuUse+needRes.Threads(res.CPUs) > res.CPUs { + log.Debugf("sched: not scheduling on worker %d for %s; not enough threads, need %d, %d in use, target %d", wid, caller, needRes.Threads(res.CPUs), a.cpuUse, res.CPUs) + return false } if len(res.GPUs) > 0 && needRes.CanGPU { From 47a83b422e3500714dea2445123a05133a57322b Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 30 Sep 2020 15:51:11 -0700 Subject: [PATCH 471/885] handle state forks in gas estimation --- chain/stmgr/call.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/chain/stmgr/call.go b/chain/stmgr/call.go index 2f049cae3..15cbac53e 100644 --- a/chain/stmgr/call.go +++ b/chain/stmgr/call.go @@ -28,14 +28,10 @@ func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types. bstate := ts.ParentState() bheight := ts.Height() - newState, err := sm.handleStateForks(ctx, bstate, bheight-1, nil, ts) + bstate, err := sm.handleStateForks(ctx, bstate, bheight-1, nil, ts) if err != nil { - return nil, fmt.Errorf("failed to handle fork") + return nil, fmt.Errorf("failed to handle fork: %w", err) } - if newState != bstate { - fmt.Println("IT WORKED!") - } - bstate = newState vmopt := &vm.VMOpts{ StateBase: bstate, @@ -117,6 +113,11 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri return nil, xerrors.Errorf("computing tipset state: %w", err) } + state, err = sm.handleStateForks(ctx, state, ts.Height(), nil, ts) + if err != nil { + return nil, fmt.Errorf("failed to handle fork: %w", err) + } + r := store.NewChainRand(sm.cs, ts.Cids()) if span.IsRecordingEvents() { From 1b7cdb9341c4cf6dce0c23790bb5b2beb68493dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 1 Oct 2020 00:54:34 +0200 Subject: [PATCH 472/885] Fix storage manager tests --- extern/sector-storage/sched_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/extern/sector-storage/sched_test.go b/extern/sector-storage/sched_test.go index c560a58f6..579a6d913 100644 --- a/extern/sector-storage/sched_test.go +++ b/extern/sector-storage/sched_test.go @@ -290,6 +290,9 @@ func TestSched(t *testing.T) { } testFunc := func(workers []workerSpec, tasks []task) func(t *testing.T) { + ParallelNum = 1 + ParallelDenom = 1 + return func(t *testing.T) { index := stores.NewIndex() From 6981f776f4b10fa7c255f86e2467e95d99f6aec9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 1 Oct 2020 00:54:53 +0200 Subject: [PATCH 473/885] Lower PC2 memory requirements --- extern/sector-storage/resources.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/extern/sector-storage/resources.go b/extern/sector-storage/resources.go index 320fec71f..6b531e82b 100644 --- a/extern/sector-storage/resources.go +++ b/extern/sector-storage/resources.go @@ -131,22 +131,22 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources }, sealtasks.TTPreCommit2: { abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{ - MaxMemory: 64 << 30, - MinMemory: 64 << 30, + MaxMemory: 30 << 30, + MinMemory: 30 << 30, MaxParallelism: -1, CanGPU: true, - BaseMinMemory: 60 << 30, + BaseMinMemory: 1 << 30, }, abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{ - MaxMemory: 32 << 30, - MinMemory: 32 << 30, + MaxMemory: 15 << 30, + MinMemory: 15 << 30, MaxParallelism: -1, CanGPU: true, - BaseMinMemory: 30 << 30, + BaseMinMemory: 1 << 30, }, abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{ MaxMemory: 3 << 29, // 1.5G From 196bf3957462987767911eb6942a2b66d7b3828b Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 30 Sep 2020 17:33:20 -0700 Subject: [PATCH 474/885] use non-faulty sectors, instead of active, in post proof In actors v2, active doesn't include "new" (unproven) sectors. --- storage/wdpost_run.go | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 65dbd6e59..6aa3d5188 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -327,17 +327,22 @@ func (s *WindowPoStScheduler) checkNextFaults(ctx context.Context, dlIdx uint64, } for partIdx, partition := range partitions { - good, err := s.checkSectors(ctx, partition.ActiveSectors) + nonFaulty, err := bitfield.SubtractBitField(partition.LiveSectors, partition.FaultySectors) + if err != nil { + return nil, nil, xerrors.Errorf("determining non faulty sectors: %w", err) + } + + good, err := s.checkSectors(ctx, nonFaulty) if err != nil { return nil, nil, xerrors.Errorf("checking sectors: %w", err) } - faulty, err := bitfield.SubtractBitField(partition.ActiveSectors, good) + newFaulty, err := bitfield.SubtractBitField(nonFaulty, good) if err != nil { return nil, nil, xerrors.Errorf("calculating faulty sector set: %w", err) } - c, err := faulty.Count() + c, err := newFaulty.Count() if err != nil { return nil, nil, xerrors.Errorf("counting faulty sectors: %w", err) } @@ -351,7 +356,7 @@ func (s *WindowPoStScheduler) checkNextFaults(ctx context.Context, dlIdx uint64, params.Faults = append(params.Faults, miner.FaultDeclaration{ Deadline: dlIdx, Partition: uint64(partIdx), - Sectors: faulty, + Sectors: newFaulty, }) } @@ -509,7 +514,11 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty var sinfos []proof.SectorInfo for partIdx, partition := range batch { // TODO: Can do this in parallel - toProve, err := bitfield.MergeBitFields(partition.ActiveSectors, partition.RecoveringSectors) + toProve, err := bitfield.SubtractBitField(partition.LiveSectors, partition.FaultySectors) + if err != nil { + return nil, xerrors.Errorf("removing faults from set of sectors to prove: %w", err) + } + toProve, err = bitfield.MergeBitFields(toProve, partition.RecoveringSectors) if err != nil { return nil, xerrors.Errorf("adding recoveries to set of sectors to prove: %w", err) } From 636810daa5e63a6ec132d78993d028a41f179276 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Wed, 30 Sep 2020 21:30:52 -0400 Subject: [PATCH 475/885] Lotus version 0.8.1 --- CHANGELOG.md | 66 ++++++++++++++++++++++++++++++++++++++++++++++++ build/version.go | 2 +- 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac687675e..6de6ddc2c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,71 @@ # Lotus changelog +# 0.8.1 / 2020-09-30 + +This optional release of Lotus introduces a new version of markets which switches to CBOR-map encodings, and allows datastore migrations. The release also introduces several improvements to the mining process, a few performance optimizations, and a battery of UX additions and enhancements. + +## Changes + +#### Dependencies + +- Markets 0.7.0 with updated data stores (https://github.com/filecoin-project/lotus/pull/4089) +- Update ffi to code with blst fixes (https://github.com/filecoin-project/lotus/pull/3998) + +#### Core Lotus + +- Fix GetPower with no miner address (https://github.com/filecoin-project/lotus/pull/4049) +- Refactor: Move nonce generation out of mpool (https://github.com/filecoin-project/lotus/pull/3970) + +#### Performance + +- Implement caching syscalls for import-bench (https://github.com/filecoin-project/lotus/pull/3888) +- Fetch tipset blocks in parallel (https://github.com/filecoin-project/lotus/pull/4074) +- Optimize Tipset equals() (https://github.com/filecoin-project/lotus/pull/4056) +- Make state transition in validation async (https://github.com/filecoin-project/lotus/pull/3868) + +#### Mining + +- Add trace window post (https://github.com/filecoin-project/lotus/pull/4020) +- Use abstract types for Dont recompute post on revert (https://github.com/filecoin-project/lotus/pull/4022) +- Fix injectNulls logic in test miner (https://github.com/filecoin-project/lotus/pull/4058) +- Fix potential panic in FinalizeSector (https://github.com/filecoin-project/lotus/pull/4092) +- Don't recompute post on revert (https://github.com/filecoin-project/lotus/pull/3924) +- Fix some failed precommit handling (https://github.com/filecoin-project/lotus/pull/3445) +- Add --no-swap flag for worker (https://github.com/filecoin-project/lotus/pull/4107) +- Allow some single-thread tasks to run in parallel with PC2/C2 (https://github.com/filecoin-project/lotus/pull/4116) + +#### UX + +- Add an envvar to set address network version (https://github.com/filecoin-project/lotus/pull/4028) +- Add logging to chain export (https://github.com/filecoin-project/lotus/pull/4030) +- Add JSON output to state compute (https://github.com/filecoin-project/lotus/pull/4038) +- Wallet list CLI: Print balances/nonces (https://github.com/filecoin-project/lotus/pull/4088) +- Added an option to show or not show sector info for `lotus-miner info` (https://github.com/filecoin-project/lotus/pull/4003) +- Add a command to import an ipld object into the chainstore (https://github.com/filecoin-project/lotus/pull/3434) +- Improve the lotus-shed dealtracker (https://github.com/filecoin-project/lotus/pull/4051) +- Docs review and re-organization (https://github.com/filecoin-project/lotus/pull/3431) +- Fix wallet list (https://github.com/filecoin-project/lotus/pull/4104) +- Add an endpoint to validate whether a string is a well-formed address (https://github.com/filecoin-project/lotus/pull/4106) +- Add an option to set config path (https://github.com/filecoin-project/lotus/pull/4103) +- Add printf in TestWindowPost (https://github.com/filecoin-project/lotus/pull/4043) +- Improve miner sectors list UX (https://github.com/filecoin-project/lotus/pull/4108) + +#### Tooling + +- Move policy change to seal bench (https://github.com/filecoin-project/lotus/pull/4032) +- Add back network power to stats (https://github.com/filecoin-project/lotus/pull/4050) +- Conformance: Record and feed circulating supply (https://github.com/filecoin-project/lotus/pull/4078) +- Snapshot import progress bar, add HTTP support (https://github.com/filecoin-project/lotus/pull/4070) +- Add lotus shed util to validate a tipset (https://github.com/filecoin-project/lotus/pull/4065) +- tvx: a test vector extraction and execution tool (https://github.com/filecoin-project/lotus/pull/4064) + +#### Bootstrap + +- Add new bootstrappers (https://github.com/filecoin-project/lotus/pull/4007) +- Add Glif node to bootstrap peers (https://github.com/filecoin-project/lotus/pull/4004) +- Add one more node located in China (https://github.com/filecoin-project/lotus/pull/4041) +- Add ipfsmain bootstrapper (https://github.com/filecoin-project/lotus/pull/4067) + # 0.8.0 / 2020-09-26 This consensus-breaking release of Lotus introduces an upgrade to the network. The changes that break consensus are: diff --git a/build/version.go b/build/version.go index 77b98f008..0b317aa17 100644 --- a/build/version.go +++ b/build/version.go @@ -29,7 +29,7 @@ func buildType() string { } // BuildVersion is the local build version, set by build system -const BuildVersion = "0.8.0" +const BuildVersion = "0.8.1" func UserVersion() string { return BuildVersion + buildType() + CurrentCommit From 60d442e36a6cc6b32da3fa7541537cfee3b7b050 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Wed, 30 Sep 2020 22:05:16 -0400 Subject: [PATCH 476/885] Slash filter shouldn't be trigerred if the same block is submitted multiple times --- chain/gen/slashfilter/slashfilter.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/chain/gen/slashfilter/slashfilter.go b/chain/gen/slashfilter/slashfilter.go index fadd3dd27..ee0435156 100644 --- a/chain/gen/slashfilter/slashfilter.go +++ b/chain/gen/slashfilter/slashfilter.go @@ -105,6 +105,10 @@ func checkFault(t ds.Datastore, key ds.Key, bh *types.BlockHeader, faultType str return err } + if other == bh.Cid() { + return nil + } + return xerrors.Errorf("produced block would trigger '%s' consensus fault; miner: %s; bh: %s, other: %s", faultType, bh.Miner, bh.Cid(), other) } From 6f0453c581205988db108c01f30d5b431662edca Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Thu, 1 Oct 2020 04:30:34 -0400 Subject: [PATCH 477/885] Convert ID addresses to key addresses before checking wallet --- node/impl/client/client.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 7cb087ec7..e7cfd0277 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -117,7 +117,13 @@ func (a *API) ClientStartDeal(ctx context.Context, params *api.StartDealParams) } } } - exist, err := a.WalletHas(ctx, params.Wallet) + + walletKey, err := a.StateAPI.StateManager.ResolveToKeyAddress(ctx, params.Wallet, nil) + if err != nil { + return nil, xerrors.Errorf("failed resolving params.Wallet addr: %w", params.Wallet) + } + + exist, err := a.WalletHas(ctx, walletKey) if err != nil { return nil, xerrors.Errorf("failed getting addr from wallet: %w", params.Wallet) } From 93e4eae94cfd5cffed88a0ad10358b57f6a2cf11 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Thu, 1 Oct 2020 03:14:59 -0400 Subject: [PATCH 478/885] Some helpers for verifreg work --- api/api_full.go | 6 +++ api/apistruct/struct.go | 10 +++++ chain/actors/builtin/verifreg/v0.go | 4 ++ chain/actors/builtin/verifreg/verifreg.go | 1 + cmd/lotus-shed/verifreg.go | 43 +++++++++++--------- documentation/en/api-methods.md | 49 +++++++++++++++++++++++ node/impl/full/state.go | 47 +++++++++++++++++++++- 7 files changed, 139 insertions(+), 21 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 767739582..b443aef3f 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -390,10 +390,16 @@ type FullNode interface { // StateCompute is a flexible command that applies the given messages on the given tipset. // The messages are run as though the VM were at the provided height. StateCompute(context.Context, abi.ChainEpoch, []*types.Message, types.TipSetKey) (*ComputeStateOutput, error) + // StateVerifierStatus returns the data cap for the given address. + // Returns nil if there is no entry in the data cap table for the + // address. + StateVerifierStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) // StateVerifiedClientStatus returns the data cap for the given address. // Returns nil if there is no entry in the data cap table for the // address. StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) + // StateVerifiedClientStatus returns the address of the Verified Registry's root key + StateVerifiedRegistryRootKey(ctx context.Context, tsk types.TipSetKey) (address.Address, error) // StateDealProviderCollateralBounds returns the min and max collateral a storage provider // can issue. It takes the deal size and verified status as parameters. StateDealProviderCollateralBounds(context.Context, abi.PaddedPieceSize, bool, types.TipSetKey) (DealCollateralBounds, error) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index c79d0841c..7bb0ac866 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -202,7 +202,9 @@ type FullNodeStruct struct { StateMinerSectorCount func(context.Context, address.Address, types.TipSetKey) (api.MinerSectors, error) `perm:"read"` StateListMessages func(ctx context.Context, match *types.Message, tsk types.TipSetKey, toht abi.ChainEpoch) ([]cid.Cid, error) `perm:"read"` StateCompute func(context.Context, abi.ChainEpoch, []*types.Message, types.TipSetKey) (*api.ComputeStateOutput, error) `perm:"read"` + StateVerifierStatus func(context.Context, address.Address, types.TipSetKey) (*abi.StoragePower, error) `perm:"read"` StateVerifiedClientStatus func(context.Context, address.Address, types.TipSetKey) (*abi.StoragePower, error) `perm:"read"` + StateVerifiedRegistryRootKey func(ctx context.Context, tsk types.TipSetKey) (address.Address, error) `perm:"read"` StateDealProviderCollateralBounds func(context.Context, abi.PaddedPieceSize, bool, types.TipSetKey) (api.DealCollateralBounds, error) `perm:"read"` StateCirculatingSupply func(context.Context, types.TipSetKey) (api.CirculatingSupply, error) `perm:"read"` StateNetworkVersion func(context.Context, types.TipSetKey) (stnetwork.Version, error) `perm:"read"` @@ -893,10 +895,18 @@ func (c *FullNodeStruct) StateCompute(ctx context.Context, height abi.ChainEpoch return c.Internal.StateCompute(ctx, height, msgs, tsk) } +func (c *FullNodeStruct) StateVerifierStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) { + return c.Internal.StateVerifierStatus(ctx, addr, tsk) +} + func (c *FullNodeStruct) StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) { return c.Internal.StateVerifiedClientStatus(ctx, addr, tsk) } +func (c *FullNodeStruct) StateVerifiedRegistryRootKey(ctx context.Context, tsk types.TipSetKey) (address.Address, error) { + return c.Internal.StateVerifiedRegistryRootKey(ctx, tsk) +} + func (c *FullNodeStruct) StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (api.DealCollateralBounds, error) { return c.Internal.StateDealProviderCollateralBounds(ctx, size, verified, tsk) } diff --git a/chain/actors/builtin/verifreg/v0.go b/chain/actors/builtin/verifreg/v0.go index c59a58811..51ed3b456 100644 --- a/chain/actors/builtin/verifreg/v0.go +++ b/chain/actors/builtin/verifreg/v0.go @@ -39,6 +39,10 @@ func getDataCap(store adt.Store, root cid.Cid, addr address.Address) (bool, abi. return true, dcap, nil } +func (s *state0) RootKey() (address.Address, error) { + return s.State.RootKey, nil +} + func (s *state0) VerifiedClientDataCap(addr address.Address) (bool, abi.StoragePower, error) { return getDataCap(s.store, s.State.VerifiedClients, addr) } diff --git a/chain/actors/builtin/verifreg/verifreg.go b/chain/actors/builtin/verifreg/verifreg.go index c861f862f..95a60d1d9 100644 --- a/chain/actors/builtin/verifreg/verifreg.go +++ b/chain/actors/builtin/verifreg/verifreg.go @@ -30,6 +30,7 @@ func Load(store adt.Store, act *types.Actor) (State, error) { type State interface { cbor.Marshaler + RootKey() (address.Address, error) VerifiedClientDataCap(address.Address) (bool, abi.StoragePower, error) VerifierDataCap(address.Address) (bool, abi.StoragePower, error) ForEachVerifier(func(addr address.Address, dcap abi.StoragePower) error) error diff --git a/cmd/lotus-shed/verifreg.go b/cmd/lotus-shed/verifreg.go index 3e2f34f4b..860498302 100644 --- a/cmd/lotus-shed/verifreg.go +++ b/cmd/lotus-shed/verifreg.go @@ -3,6 +3,8 @@ package main import ( "fmt" + "github.com/filecoin-project/go-state-types/big" + "github.com/urfave/cli/v2" "golang.org/x/xerrors" @@ -37,29 +39,31 @@ var verifRegCmd = &cli.Command{ } var verifRegAddVerifierCmd = &cli.Command{ - Name: "add-verifier", - Usage: "make a given account a verifier", + Name: "add-verifier", + Usage: "make a given account a verifier", + ArgsUsage: " ", Action: func(cctx *cli.Context) error { - fromk, err := address.NewFromString("t3qfoulel6fy6gn3hjmbhpdpf6fs5aqjb5fkurhtwvgssizq4jey5nw4ptq5up6h7jk7frdvvobv52qzmgjinq") + if cctx.Args().Len() != 3 { + return fmt.Errorf("must specify three arguments: sender, verifier, and allowance") + } + + sender, err := address.NewFromString(cctx.Args().Get(0)) if err != nil { return err } - if cctx.Args().Len() != 2 { - return fmt.Errorf("must specify two arguments: address and allowance") - } - - target, err := address.NewFromString(cctx.Args().Get(0)) + verifier, err := address.NewFromString(cctx.Args().Get(1)) if err != nil { return err } - allowance, err := types.BigFromString(cctx.Args().Get(1)) + allowance, err := types.BigFromString(cctx.Args().Get(2)) if err != nil { return err } - params, err := actors.SerializeParams(&verifreg0.AddVerifierParams{Address: target, Allowance: allowance}) + // TODO: ActorUpgrade: Abstract + params, err := actors.SerializeParams(&verifreg0.AddVerifierParams{Address: verifier, Allowance: allowance}) if err != nil { return err } @@ -71,21 +75,19 @@ var verifRegAddVerifierCmd = &cli.Command{ defer closer() ctx := lcli.ReqContext(cctx) - msg := &types.Message{ - To: verifreg.Address, - From: fromk, - Method: builtin0.MethodsVerifiedRegistry.AddVerifier, - Params: params, - } - - smsg, err := api.MpoolPushMessage(ctx, msg, nil) + vrk, err := api.StateVerifiedRegistryRootKey(ctx, types.EmptyTSK) if err != nil { return err } - fmt.Printf("message sent, now waiting on cid: %s\n", smsg.Cid()) + smsg, err := api.MsigPropose(ctx, vrk, verifreg.Address, big.Zero(), sender, uint64(builtin0.MethodsVerifiedRegistry.AddVerifier), params) + if err != nil { + return err + } - mwait, err := api.StateWaitMsg(ctx, smsg.Cid(), build.MessageConfidence) + fmt.Printf("message sent, now waiting on cid: %s\n", smsg) + + mwait, err := api.StateWaitMsg(ctx, smsg, build.MessageConfidence) if err != nil { return err } @@ -94,6 +96,7 @@ var verifRegAddVerifierCmd = &cli.Command{ return fmt.Errorf("failed to add verifier: %d", mwait.Receipt.ExitCode) } + //TODO: Internal msg might still have failed return nil }, diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index 2b28816f7..ff01f648e 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -160,6 +160,8 @@ * [StateSectorPartition](#StateSectorPartition) * [StateSectorPreCommitInfo](#StateSectorPreCommitInfo) * [StateVerifiedClientStatus](#StateVerifiedClientStatus) + * [StateVerifiedRegistryRootKey](#StateVerifiedRegistryRootKey) + * [StateVerifierStatus](#StateVerifierStatus) * [StateWaitMsg](#StateWaitMsg) * [Sync](#Sync) * [SyncCheckBad](#SyncCheckBad) @@ -4117,6 +4119,53 @@ Returns nil if there is no entry in the data cap table for the address. +Perms: read + +Inputs: +```json +[ + "t01234", + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `"0"` + +### StateVerifiedRegistryRootKey +StateVerifiedClientStatus returns the address of the Verified Registry's root key + + +Perms: read + +Inputs: +```json +[ + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `"t01234"` + +### StateVerifierStatus +StateVerifierStatus returns the data cap for the given address. +Returns nil if there is no entry in the data cap table for the +address. + + Perms: read Inputs: diff --git a/node/impl/full/state.go b/node/impl/full/state.go index f8bf92a92..e5bd9f9d8 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -1021,11 +1021,42 @@ func (a *StateAPI) StateMinerAvailableBalance(ctx context.Context, maddr address return types.BigAdd(abal, vested), nil } +// StateVerifiedClientStatus returns the data cap for the given address. +// Returns zero if there is no entry in the data cap table for the +// address. +func (a *StateAPI) StateVerifierStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) { + act, err := a.StateGetActor(ctx, verifreg.Address, tsk) + if err != nil { + return nil, err + } + + aid, err := a.StateLookupID(ctx, addr, tsk) + if err != nil { + log.Warnf("lookup failure %v", err) + return nil, err + } + + vrs, err := verifreg.Load(a.StateManager.ChainStore().Store(ctx), act) + if err != nil { + return nil, xerrors.Errorf("failed to load verified registry state: %w", err) + } + + verified, dcap, err := vrs.VerifierDataCap(aid) + if err != nil { + return nil, xerrors.Errorf("looking up verifier: %w", err) + } + if !verified { + return nil, nil + } + + return &dcap, nil +} + // StateVerifiedClientStatus returns the data cap for the given address. // Returns zero if there is no entry in the data cap table for the // address. func (a *StateAPI) StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) { - act, err := a.StateGetActor(ctx, builtin0.VerifiedRegistryActorAddr, tsk) + act, err := a.StateGetActor(ctx, verifreg.Address, tsk) if err != nil { return nil, err } @@ -1052,6 +1083,20 @@ func (a *StateAPI) StateVerifiedClientStatus(ctx context.Context, addr address.A return &dcap, nil } +func (a *StateAPI) StateVerifiedRegistryRootKey(ctx context.Context, tsk types.TipSetKey) (address.Address, error) { + vact, err := a.StateGetActor(ctx, verifreg.Address, tsk) + if err != nil { + return address.Undef, err + } + + vst, err := verifreg.Load(a.StateManager.ChainStore().Store(ctx), vact) + if err != nil { + return address.Undef, err + } + + return vst.RootKey() +} + var dealProviderCollateralNum = types.NewInt(110) var dealProviderCollateralDen = types.NewInt(100) From 3d91633699634c720978069bfce4ae9fa5206c71 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Thu, 1 Oct 2020 04:58:13 -0400 Subject: [PATCH 479/885] Update to v2/actors master --- build/params_2k.go | 2 +- build/params_shared_funcs.go | 2 +- build/params_testground.go | 2 +- build/params_testnet.go | 2 +- chain/stmgr/forks.go | 4 ++-- chain/stmgr/stmgr.go | 2 +- go.mod | 4 ++-- go.sum | 11 ++++------- 8 files changed, 13 insertions(+), 16 deletions(-) diff --git a/build/params_2k.go b/build/params_2k.go index 4428da748..f4a17f724 100644 --- a/build/params_2k.go +++ b/build/params_2k.go @@ -14,7 +14,7 @@ const BreezeGasTampingDuration = 0 const UpgradeSmokeHeight = -1 const UpgradeIgnitionHeight = -2 const UpgradeLiftoffHeight = -3 -const UpgradeActorsV2 = 10 +const UpgradeActorsV2Height = 10 var DrandSchedule = map[abi.ChainEpoch]DrandEnum{ 0: DrandMainnet, diff --git a/build/params_shared_funcs.go b/build/params_shared_funcs.go index 95daa45e7..40ccca50b 100644 --- a/build/params_shared_funcs.go +++ b/build/params_shared_funcs.go @@ -41,7 +41,7 @@ func DhtProtocolName(netName dtypes.NetworkName) protocol.ID { func UseNewestNetwork() bool { // TODO: Put these in a container we can iterate over - if UpgradeBreezeHeight <= 0 && UpgradeSmokeHeight <= 0 && UpgradeActorsV2 <= 0 { + if UpgradeBreezeHeight <= 0 && UpgradeSmokeHeight <= 0 && UpgradeActorsV2Height <= 0 { return true } return false diff --git a/build/params_testground.go b/build/params_testground.go index de554f49e..6d51100e4 100644 --- a/build/params_testground.go +++ b/build/params_testground.go @@ -76,7 +76,7 @@ var ( UpgradeSmokeHeight abi.ChainEpoch = -1 UpgradeIgnitionHeight abi.ChainEpoch = -2 UpgradeLiftoffHeight abi.ChainEpoch = -3 - UpgradeActorsV2 abi.ChainEpoch = 10 + UpgradeActorsV2Height abi.ChainEpoch = 10 DrandSchedule = map[abi.ChainEpoch]DrandEnum{ 0: DrandMainnet, diff --git a/build/params_testnet.go b/build/params_testnet.go index 66f30f869..ab579fd10 100644 --- a/build/params_testnet.go +++ b/build/params_testnet.go @@ -23,7 +23,7 @@ const UpgradeSmokeHeight = 51000 const UpgradeIgnitionHeight = 94000 -const UpgradeActorsV2 = 128888 +const UpgradeActorsV2Height = 128888 // This signals our tentative epoch for mainnet launch. Can make it later, but not earlier. // Miners, clients, developers, custodians all need time to prepare. diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index 96873b675..7d1555114 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -36,7 +36,7 @@ import ( var ForksAtHeight = map[abi.ChainEpoch]func(context.Context, *StateManager, ExecCallback, cid.Cid, *types.TipSet) (cid.Cid, error){ build.UpgradeBreezeHeight: UpgradeFaucetBurnRecovery, build.UpgradeIgnitionHeight: UpgradeIgnition, - build.UpgradeActorsV2: UpgradeActorsV2, + build.UpgradeActorsV2Height: UpgradeActorsV2, build.UpgradeLiftoffHeight: UpgradeLiftoff, } @@ -427,7 +427,7 @@ func UpgradeActorsV2(ctx context.Context, sm *StateManager, cb ExecCallback, roo return cid.Undef, xerrors.Errorf("failed to create new state info for actors v2: %w", err) } - newHamtRoot, err := m2.MigrateStateTree(ctx, store, root) + newHamtRoot, err := m2.MigrateStateTree(ctx, store, root, build.UpgradeActorsV2Height, m2.DefaultConfig()) if err != nil { return cid.Undef, xerrors.Errorf("upgrading to actors v2: %w", err) } diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index 2ce393378..e7947cac3 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -1277,7 +1277,7 @@ func (sm *StateManager) GetNtwkVersion(ctx context.Context, height abi.ChainEpoc return network.Version2 } - if height <= build.UpgradeActorsV2 { + if height <= build.UpgradeActorsV2Height { return network.Version3 } diff --git a/go.mod b/go.mod index 1f8d043fa..ea33b810e 100644 --- a/go.mod +++ b/go.mod @@ -37,8 +37,8 @@ require ( github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b - github.com/filecoin-project/specs-actors v0.9.12-0.20200930015636-a6848b2dd741 - github.com/filecoin-project/specs-actors/v2 v2.0.0-20200930035834-7115a78cb9a1 + github.com/filecoin-project/specs-actors v0.9.12 + github.com/filecoin-project/specs-actors/v2 v2.0.0-20201001041506-c7ff44a3ce9b github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 github.com/filecoin-project/test-vectors/schema v0.0.3 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 diff --git a/go.sum b/go.sum index 3e16f9b74..6b3b00907 100644 --- a/go.sum +++ b/go.sum @@ -261,8 +261,6 @@ github.com/filecoin-project/go-paramfetch v0.0.2-0.20200701152213-3e0f0afdc261/g github.com/filecoin-project/go-state-types v0.0.0-20200903145444-247639ffa6ad/go.mod h1:IQ0MBPnonv35CJHtWSN3YY1Hz2gkPru1Q9qoaYLxx9I= github.com/filecoin-project/go-state-types v0.0.0-20200904021452-1883f36ca2f4/go.mod h1:IQ0MBPnonv35CJHtWSN3YY1Hz2gkPru1Q9qoaYLxx9I= github.com/filecoin-project/go-state-types v0.0.0-20200905071437-95828685f9df/go.mod h1:IQ0MBPnonv35CJHtWSN3YY1Hz2gkPru1Q9qoaYLxx9I= -github.com/filecoin-project/go-state-types v0.0.0-20200911004822-964d6c679cfc h1:1vr/LoqGq5m5g37Q3sNSAjfwF1uJY0zmiHcvnxY6hik= -github.com/filecoin-project/go-state-types v0.0.0-20200911004822-964d6c679cfc/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-state-types v0.0.0-20200928172055-2df22083d8ab h1:cEDC5Ei8UuT99hPWhCjA72SM9AuRtnpvdSTIYbnzN8I= github.com/filecoin-project/go-state-types v0.0.0-20200928172055-2df22083d8ab/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-statemachine v0.0.0-20200714194326-a77c3ae20989/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= @@ -274,11 +272,10 @@ github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/go.mod h1:Q0GQOBtKf1oE10eSXSlhN45kDBdGvEcVOqMiffqX+N8= github.com/filecoin-project/specs-actors v0.9.4/go.mod h1:BStZQzx5x7TmCkLv0Bpa07U6cPKol6fd3w9KjMPZ6Z4= github.com/filecoin-project/specs-actors v0.9.7/go.mod h1:wM2z+kwqYgXn5Z7scV1YHLyd1Q1cy0R8HfTIWQ0BFGU= -github.com/filecoin-project/specs-actors v0.9.9/go.mod h1:czlvLQGEX0fjLLfdNHD7xLymy6L3n7aQzRWzsYGf+ys= -github.com/filecoin-project/specs-actors v0.9.12-0.20200930015636-a6848b2dd741 h1:pqAfjHE+yLFj1mvtx68S3HKdd9LSy6Byziz2qRNh3fA= -github.com/filecoin-project/specs-actors v0.9.12-0.20200930015636-a6848b2dd741/go.mod h1:TS1AW/7LbG+615j4NsjMK1qlpAwaFsG9w0V2tg2gSao= -github.com/filecoin-project/specs-actors/v2 v2.0.0-20200930035834-7115a78cb9a1 h1:lNMXodbyskP3ZfTWL/t6M4NYI0hhcZQ2GAJCIVK8Y/E= -github.com/filecoin-project/specs-actors/v2 v2.0.0-20200930035834-7115a78cb9a1/go.mod h1:/2Zra1BhLtpRywUhm++QP+3I5Ir+hBk/W24TpYjj43E= +github.com/filecoin-project/specs-actors v0.9.12 h1:iIvk58tuMtmloFNHhAOQHG+4Gci6Lui0n7DYQGi3cJk= +github.com/filecoin-project/specs-actors v0.9.12/go.mod h1:TS1AW/7LbG+615j4NsjMK1qlpAwaFsG9w0V2tg2gSao= +github.com/filecoin-project/specs-actors/v2 v2.0.0-20201001041506-c7ff44a3ce9b h1:8JRxR0rKdiuCHIDKi7Zbs+/jiygr6eXEX47QXhjX+kA= +github.com/filecoin-project/specs-actors/v2 v2.0.0-20201001041506-c7ff44a3ce9b/go.mod h1:52FuQUNDXq2WDg+6+UOhkqBuNc2e62h9BCIB67Bluxg= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= github.com/filecoin-project/test-vectors/schema v0.0.3 h1:1zuBo25B3016inbygYLgYFdpJ2m1BDTbAOCgABRleiU= From 9dc75a7bc46a013bb388eec9c12896a4050eb5fe Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Mon, 14 Sep 2020 10:11:11 +0200 Subject: [PATCH 480/885] fix: verify voucher amount check --- api/test/paych.go | 47 ++++- chain/actors/builtin/paych/mock/mock.go | 3 +- paychmgr/mock_test.go | 2 +- paychmgr/paych.go | 29 ++- paychmgr/paych_test.go | 234 +++++++----------------- paychmgr/paychget_test.go | 2 +- paychmgr/simple.go | 2 +- 7 files changed, 130 insertions(+), 189 deletions(-) diff --git a/api/test/paych.go b/api/test/paych.go index 36eb2c256..15ce352bd 100644 --- a/api/test/paych.go +++ b/api/test/paych.go @@ -67,7 +67,7 @@ func TestPaymentChannels(t *testing.T, b APIBuilder, blocktime time.Duration) { t.Fatal(err) } - channelAmt := int64(100000) + channelAmt := int64(7000) channelInfo, err := paymentCreator.PaychGet(ctx, createrAddr, receiverAddr, abi.NewTokenAmount(channelAmt)) if err != nil { t.Fatal(err) @@ -169,6 +169,51 @@ func TestPaymentChannels(t *testing.T, b APIBuilder, blocktime time.Duration) { t.Fatal("Timed out waiting for receiver to submit vouchers") } + // Create a new voucher now that some vouchers have already been submitted + vouchRes, err := paymentCreator.PaychVoucherCreate(ctx, channel, abi.NewTokenAmount(1000), 3) + if err != nil { + t.Fatal(err) + } + if vouchRes.Voucher == nil { + t.Fatal(fmt.Errorf("Not enough funds to create voucher: missing %d", vouchRes.Shortfall)) + } + vdelta, err := paymentReceiver.PaychVoucherAdd(ctx, channel, vouchRes.Voucher, nil, abi.NewTokenAmount(1000)) + if err != nil { + t.Fatal(err) + } + if !vdelta.Equals(abi.NewTokenAmount(1000)) { + t.Fatal("voucher didn't have the right amount") + } + + // Create a new voucher whose value would exceed the channel balance + excessAmt := abi.NewTokenAmount(1000) + vouchRes, err = paymentCreator.PaychVoucherCreate(ctx, channel, excessAmt, 4) + if err != nil { + t.Fatal(err) + } + if vouchRes.Voucher != nil { + t.Fatal("Expected not to be able to create voucher whose value would exceed channel balance") + } + if !vouchRes.Shortfall.Equals(excessAmt) { + t.Fatal(fmt.Errorf("Expected voucher shortfall of %d, got %d", excessAmt, vouchRes.Shortfall)) + } + + // Add a voucher whose value would exceed the channel balance + vouch := &paych.SignedVoucher{ChannelAddr: channel, Amount: excessAmt, Lane: 4, Nonce: 1} + vb, err := vouch.SigningBytes() + if err != nil { + t.Fatal(err) + } + sig, err := paymentCreator.WalletSign(ctx, createrAddr, vb) + if err != nil { + t.Fatal(err) + } + vouch.Signature = sig + _, err = paymentReceiver.PaychVoucherAdd(ctx, channel, vouch, nil, abi.NewTokenAmount(1000)) + if err == nil { + t.Fatal(fmt.Errorf("Expected shortfall error of %d", excessAmt)) + } + // wait for the settlement period to pass before collecting waitForBlocks(ctx, t, bm, paymentReceiver, receiverAddr, paych.SettleDelay) diff --git a/chain/actors/builtin/paych/mock/mock.go b/chain/actors/builtin/paych/mock/mock.go index c4903f3ac..3b82511ff 100644 --- a/chain/actors/builtin/paych/mock/mock.go +++ b/chain/actors/builtin/paych/mock/mock.go @@ -27,10 +27,9 @@ type mockLaneState struct { func NewMockPayChState(from address.Address, to address.Address, settlingAt abi.ChainEpoch, - toSend abi.TokenAmount, lanes map[uint64]paych.LaneState, ) paych.State { - return &mockState{from, to, settlingAt, toSend, lanes} + return &mockState{from: from, to: to, settlingAt: settlingAt, toSend: big.NewInt(0), lanes: lanes} } // NewMockLaneState constructs a state for a payment channel lane with the set fixed values diff --git a/paychmgr/mock_test.go b/paychmgr/mock_test.go index c761221d2..3232e0893 100644 --- a/paychmgr/mock_test.go +++ b/paychmgr/mock_test.go @@ -9,7 +9,6 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/crypto" - "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors/builtin/paych" "github.com/filecoin-project/lotus/chain/types" @@ -57,6 +56,7 @@ func (sm *mockStateManager) setAccountAddress(a address.Address, lookup address. func (sm *mockStateManager) setPaychState(a address.Address, actor *types.Actor, state paych.State) { sm.lk.Lock() defer sm.lk.Unlock() + sm.paychState[a] = mockPchState{actor, state} } diff --git a/paychmgr/paych.go b/paychmgr/paych.go index f856b9890..2cf200986 100644 --- a/paychmgr/paych.go +++ b/paychmgr/paych.go @@ -205,7 +205,7 @@ func (ca *channelAccessor) checkVoucherValidUnlocked(ctx context.Context, ch add } // Check the voucher against the highest known voucher nonce / value - laneStates, err := ca.laneState(ctx, pchState, ch) + laneStates, err := ca.laneState(pchState, ch) if err != nil { return nil, err } @@ -253,16 +253,9 @@ func (ca *channelAccessor) checkVoucherValidUnlocked(ctx context.Context, ch add return nil, err } - // Total required balance = total redeemed + toSend - // Must not exceed actor balance - ts, err := pchState.ToSend() - if err != nil { - return nil, err - } - - newTotal := types.BigAdd(totalRedeemed, ts) - if act.Balance.LessThan(newTotal) { - return nil, newErrInsufficientFunds(types.BigSub(newTotal, act.Balance)) + // Total required balance must not exceed actor balance + if act.Balance.LessThan(totalRedeemed) { + return nil, newErrInsufficientFunds(types.BigSub(totalRedeemed, act.Balance)) } if len(sv.Merges) != 0 { @@ -505,7 +498,6 @@ func (ca *channelAccessor) allocateLane(ch address.Address) (uint64, error) { ca.lk.Lock() defer ca.lk.Unlock() - // TODO: should this take into account lane state? return ca.store.AllocateLane(ch) } @@ -520,7 +512,7 @@ func (ca *channelAccessor) listVouchers(ctx context.Context, ch address.Address) // laneState gets the LaneStates from chain, then applies all vouchers in // the data store over the chain state -func (ca *channelAccessor) laneState(ctx context.Context, state paych.State, ch address.Address) (map[uint64]paych.LaneState, error) { +func (ca *channelAccessor) laneState(state paych.State, ch address.Address) (map[uint64]paych.LaneState, error) { // TODO: we probably want to call UpdateChannelState with all vouchers to be fully correct // (but technically dont't need to) @@ -547,14 +539,20 @@ func (ca *channelAccessor) laneState(ctx context.Context, state paych.State, ch return nil, err } + // Note: we use a map instead of an array to store laneStates because the + // client sets the lane ID (the index) and potentially they could use a + // very large index. for _, v := range vouchers { for range v.Voucher.Merges { return nil, xerrors.Errorf("paych merges not handled yet") } - // If there's a voucher for a lane that isn't in chain state just - // create it + // Check if there is an existing laneState in the payment channel + // for this voucher's lane ls, ok := laneStates[v.Voucher.Lane] + + // If the voucher does not have a higher nonce than the existing + // laneState for this lane, ignore it if ok { n, err := ls.Nonce() if err != nil { @@ -565,6 +563,7 @@ func (ca *channelAccessor) laneState(ctx context.Context, state paych.State, ch } } + // Voucher has a higher nonce, so replace laneState with this voucher laneStates[v.Voucher.Lane] = laneState{v.Voucher.Amount, v.Voucher.Nonce} } diff --git a/paychmgr/paych_test.go b/paychmgr/paych_test.go index b27b1e540..71c2bd854 100644 --- a/paychmgr/paych_test.go +++ b/paychmgr/paych_test.go @@ -47,7 +47,6 @@ func TestCheckVoucherValid(t *testing.T) { expectError bool key []byte actorBalance big.Int - toSend big.Int voucherAmount big.Int voucherLane uint64 voucherNonce uint64 @@ -56,35 +55,30 @@ func TestCheckVoucherValid(t *testing.T) { name: "passes when voucher amount < balance", key: fromKeyPrivate, actorBalance: big.NewInt(10), - toSend: big.NewInt(0), voucherAmount: big.NewInt(5), }, { name: "fails when funds too low", expectError: true, key: fromKeyPrivate, actorBalance: big.NewInt(5), - toSend: big.NewInt(0), voucherAmount: big.NewInt(10), }, { name: "fails when invalid signature", expectError: true, key: randKeyPrivate, actorBalance: big.NewInt(10), - toSend: big.NewInt(0), voucherAmount: big.NewInt(5), }, { name: "fails when signed by channel To account (instead of From account)", expectError: true, key: toKeyPrivate, actorBalance: big.NewInt(10), - toSend: big.NewInt(0), voucherAmount: big.NewInt(5), }, { name: "fails when nonce too low", expectError: true, key: fromKeyPrivate, actorBalance: big.NewInt(10), - toSend: big.NewInt(0), voucherAmount: big.NewInt(5), voucherLane: 1, voucherNonce: 2, @@ -95,7 +89,6 @@ func TestCheckVoucherValid(t *testing.T) { name: "passes when nonce higher", key: fromKeyPrivate, actorBalance: big.NewInt(10), - toSend: big.NewInt(0), voucherAmount: big.NewInt(5), voucherLane: 1, voucherNonce: 3, @@ -106,7 +99,6 @@ func TestCheckVoucherValid(t *testing.T) { name: "passes when nonce for different lane", key: fromKeyPrivate, actorBalance: big.NewInt(10), - toSend: big.NewInt(0), voucherAmount: big.NewInt(5), voucherLane: 2, voucherNonce: 2, @@ -118,32 +110,22 @@ func TestCheckVoucherValid(t *testing.T) { expectError: true, key: fromKeyPrivate, actorBalance: big.NewInt(10), - toSend: big.NewInt(0), voucherAmount: big.NewInt(5), voucherLane: 1, voucherNonce: 3, laneStates: map[uint64]paych.LaneState{ 1: paychmock.NewMockLaneState(big.NewInt(6), 2), }, - }, { - name: "fails when voucher + ToSend > balance", - expectError: true, - key: fromKeyPrivate, - actorBalance: big.NewInt(10), - toSend: big.NewInt(9), - voucherAmount: big.NewInt(2), }, { // voucher supersedes lane 1 redeemed so // lane 1 effective redeemed = voucher amount // - // required balance = toSend + total redeemed - // = 1 + 6 (lane1) + // required balance = voucher amt // = 7 // So required balance: 7 < actor balance: 10 - name: "passes when voucher + total redeemed <= balance", + name: "passes when voucher total redeemed <= balance", key: fromKeyPrivate, actorBalance: big.NewInt(10), - toSend: big.NewInt(1), voucherAmount: big.NewInt(6), voucherLane: 1, voucherNonce: 2, @@ -152,29 +134,68 @@ func TestCheckVoucherValid(t *testing.T) { 1: paychmock.NewMockLaneState(big.NewInt(4), 1), }, }, { - // required balance = toSend + total redeemed - // = 1 + 4 (lane 2) + 6 (voucher lane 1) + // required balance = total redeemed + // = 6 (voucher lane 1) + 5 (lane 2) // = 11 // So required balance: 11 > actor balance: 10 - name: "fails when voucher + total redeemed > balance", + name: "fails when voucher total redeemed > balance", expectError: true, key: fromKeyPrivate, actorBalance: big.NewInt(10), - toSend: big.NewInt(1), voucherAmount: big.NewInt(6), voucherLane: 1, voucherNonce: 1, laneStates: map[uint64]paych.LaneState{ // Lane 2 (different from voucher lane 1) - 2: paychmock.NewMockLaneState(big.NewInt(4), 1), + 2: paychmock.NewMockLaneState(big.NewInt(5), 1), + }, + }, { + // voucher supersedes lane 1 redeemed so + // lane 1 effective redeemed = voucher amount + // + // required balance = total redeemed + // = 6 (new voucher lane 1) + 5 (lane 2) + // = 11 + // So required balance: 11 > actor balance: 10 + name: "fails when voucher total redeemed > balance", + expectError: true, + key: fromKeyPrivate, + actorBalance: big.NewInt(10), + voucherAmount: big.NewInt(6), + voucherLane: 1, + voucherNonce: 2, + laneStates: map[uint64]paych.LaneState{ + // Lane 1 (superseded by new voucher in voucher lane 1) + 1: paychmock.NewMockLaneState(big.NewInt(5), 1), + // Lane 2 (different from voucher lane 1) + 2: paychmock.NewMockLaneState(big.NewInt(5), 1), + }, + }, { + // voucher supersedes lane 1 redeemed so + // lane 1 effective redeemed = voucher amount + // + // required balance = total redeemed + // = 5 (new voucher lane 1) + 5 (lane 2) + // = 10 + // So required balance: 10 <= actor balance: 10 + name: "passes when voucher total redeemed <= balance", + expectError: false, + key: fromKeyPrivate, + actorBalance: big.NewInt(10), + voucherAmount: big.NewInt(5), + voucherLane: 1, + voucherNonce: 2, + laneStates: map[uint64]paych.LaneState{ + // Lane 1 (superseded by new voucher in voucher lane 1) + 1: paychmock.NewMockLaneState(big.NewInt(4), 1), + // Lane 2 (different from voucher lane 1) + 2: paychmock.NewMockLaneState(big.NewInt(5), 1), }, }} for _, tcase := range tcases { tcase := tcase t.Run(tcase.name, func(t *testing.T) { - store := NewStore(ds_sync.MutexWrap(ds.NewMapDatastore())) - // Create an actor for the channel with the test case balance act := &types.Actor{ Code: builtin.AccountActorCodeID, @@ -184,16 +205,17 @@ func TestCheckVoucherValid(t *testing.T) { } mock.setPaychState(ch, act, paychmock.NewMockPayChState( - fromAcct, toAcct, abi.ChainEpoch(0), tcase.toSend, tcase.laneStates)) + fromAcct, toAcct, abi.ChainEpoch(0), tcase.laneStates)) // Create a manager + store := NewStore(ds_sync.MutexWrap(ds.NewMapDatastore())) mgr, err := newManager(store, mock) require.NoError(t, err) // Add channel To address to wallet mock.addWalletAddress(to) - // Create a signed voucher + // Create the test case signed voucher sv := createTestVoucher(t, ch, tcase.voucherLane, tcase.voucherNonce, tcase.voucherAmount, tcase.key) // Check the voucher's validity @@ -207,135 +229,11 @@ func TestCheckVoucherValid(t *testing.T) { } } -func TestCheckVoucherValidCountingAllLanes(t *testing.T) { - ctx := context.Background() - - fromKeyPrivate, fromKeyPublic := testGenerateKeyPair(t) - - ch := tutils.NewIDAddr(t, 100) - from := tutils.NewSECP256K1Addr(t, string(fromKeyPublic)) - to := tutils.NewSECP256K1Addr(t, "secpTo") - fromAcct := tutils.NewActorAddr(t, "fromAct") - toAcct := tutils.NewActorAddr(t, "toAct") - minDelta := big.NewInt(0) - - mock := newMockManagerAPI() - mock.setAccountAddress(fromAcct, from) - mock.setAccountAddress(toAcct, to) - - store := NewStore(ds_sync.MutexWrap(ds.NewMapDatastore())) - - actorBalance := big.NewInt(10) - toSend := big.NewInt(1) - laneStates := map[uint64]paych.LaneState{ - 1: paychmock.NewMockLaneState(big.NewInt(3), 1), - 2: paychmock.NewMockLaneState(big.NewInt(4), 1), - } - - act := &types.Actor{ - Code: builtin.AccountActorCodeID, - Head: cid.Cid{}, - Nonce: 0, - Balance: actorBalance, - } - - mock.setPaychState(ch, act, paychmock.NewMockPayChState(fromAcct, toAcct, abi.ChainEpoch(0), toSend, laneStates)) - - mgr, err := newManager(store, mock) - require.NoError(t, err) - - // Add channel To address to wallet - mock.addWalletAddress(to) - - // - // Should not be possible to add a voucher with a value such that - // + toSend > - // - // lane 1 redeemed: 3 - // voucher amount (lane 1): 6 - // lane 1 redeemed (with voucher): 6 - // - // Lane 1: 6 - // Lane 2: 4 - // toSend: 1 - // -- - // total: 11 - // - // actor balance is 10 so total is too high. - // - voucherLane := uint64(1) - voucherNonce := uint64(2) - voucherAmount := big.NewInt(6) - sv := createTestVoucher(t, ch, voucherLane, voucherNonce, voucherAmount, fromKeyPrivate) - err = mgr.CheckVoucherValid(ctx, ch, sv) - require.Error(t, err) - - // - // lane 1 redeemed: 3 - // voucher amount (lane 1): 4 - // lane 1 redeemed (with voucher): 4 - // - // Lane 1: 4 - // Lane 2: 4 - // toSend: 1 - // -- - // total: 9 - // - // actor balance is 10 so total is ok. - // - voucherAmount = big.NewInt(4) - sv = createTestVoucher(t, ch, voucherLane, voucherNonce, voucherAmount, fromKeyPrivate) - err = mgr.CheckVoucherValid(ctx, ch, sv) - require.NoError(t, err) - - // Add voucher to lane 1, so Lane 1 effective redeemed - // (with first voucher) is now 4 - _, err = mgr.AddVoucherOutbound(ctx, ch, sv, nil, minDelta) - require.NoError(t, err) - - // - // lane 1 redeemed: 4 - // voucher amount (lane 1): 6 - // lane 1 redeemed (with voucher): 6 - // - // Lane 1: 6 - // Lane 2: 4 - // toSend: 1 - // -- - // total: 11 - // - // actor balance is 10 so total is too high. - // - voucherNonce++ - voucherAmount = big.NewInt(6) - sv = createTestVoucher(t, ch, voucherLane, voucherNonce, voucherAmount, fromKeyPrivate) - err = mgr.CheckVoucherValid(ctx, ch, sv) - require.Error(t, err) - - // - // lane 1 redeemed: 4 - // voucher amount (lane 1): 5 - // lane 1 redeemed (with voucher): 5 - // - // Lane 1: 5 - // Lane 2: 4 - // toSend: 1 - // -- - // total: 10 - // - // actor balance is 10 so total is ok. - // - voucherAmount = big.NewInt(5) - sv = createTestVoucher(t, ch, voucherLane, voucherNonce, voucherAmount, fromKeyPrivate) - err = mgr.CheckVoucherValid(ctx, ch, sv) - require.NoError(t, err) -} - func TestCreateVoucher(t *testing.T) { ctx := context.Background() // Set up a manager with a single payment channel - s := testSetupMgrWithChannel(ctx, t) + s := testSetupMgrWithChannel(t) // Create a voucher in lane 1 voucherLane1Amt := big.NewInt(5) @@ -400,7 +298,7 @@ func TestAddVoucherDelta(t *testing.T) { ctx := context.Background() // Set up a manager with a single payment channel - s := testSetupMgrWithChannel(ctx, t) + s := testSetupMgrWithChannel(t) voucherLane := uint64(1) @@ -442,7 +340,7 @@ func TestAddVoucherNextLane(t *testing.T) { ctx := context.Background() // Set up a manager with a single payment channel - s := testSetupMgrWithChannel(ctx, t) + s := testSetupMgrWithChannel(t) minDelta := big.NewInt(0) voucherAmount := big.NewInt(2) @@ -489,10 +387,8 @@ func TestAddVoucherNextLane(t *testing.T) { } func TestAllocateLane(t *testing.T) { - ctx := context.Background() - // Set up a manager with a single payment channel - s := testSetupMgrWithChannel(ctx, t) + s := testSetupMgrWithChannel(t) // First lane should be 0 lane, err := s.mgr.AllocateLane(s.ch) @@ -525,7 +421,6 @@ func TestAllocateLaneWithExistingLaneState(t *testing.T) { // Create a channel that will be retrieved from state actorBalance := big.NewInt(10) - toSend := big.NewInt(1) act := &types.Actor{ Code: builtin.AccountActorCodeID, @@ -534,7 +429,7 @@ func TestAllocateLaneWithExistingLaneState(t *testing.T) { Balance: actorBalance, } - mock.setPaychState(ch, act, paychmock.NewMockPayChState(fromAcct, toAcct, abi.ChainEpoch(0), toSend, make(map[uint64]paych.LaneState))) + mock.setPaychState(ch, act, paychmock.NewMockPayChState(fromAcct, toAcct, abi.ChainEpoch(0), make(map[uint64]paych.LaneState))) mgr, err := newManager(store, mock) require.NoError(t, err) @@ -559,7 +454,7 @@ func TestAddVoucherProof(t *testing.T) { ctx := context.Background() // Set up a manager with a single payment channel - s := testSetupMgrWithChannel(ctx, t) + s := testSetupMgrWithChannel(t) nonce := uint64(1) voucherAmount := big.NewInt(1) @@ -622,10 +517,11 @@ func TestAddVoucherInboundWalletKey(t *testing.T) { } mock := newMockManagerAPI() + mock.setAccountAddress(fromAcct, from) mock.setAccountAddress(toAcct, to) - mock.setPaychState(ch, act, paychmock.NewMockPayChState(fromAcct, toAcct, abi.ChainEpoch(0), types.NewInt(0), make(map[uint64]paych.LaneState))) + mock.setPaychState(ch, act, paychmock.NewMockPayChState(fromAcct, toAcct, abi.ChainEpoch(0), make(map[uint64]paych.LaneState))) // Create a manager store := NewStore(ds_sync.MutexWrap(ds.NewMapDatastore())) @@ -660,7 +556,7 @@ func TestBestSpendable(t *testing.T) { ctx := context.Background() // Set up a manager with a single payment channel - s := testSetupMgrWithChannel(ctx, t) + s := testSetupMgrWithChannel(t) // Add vouchers to lane 1 with amounts: [1, 2, 3] voucherLane := uint64(1) @@ -740,7 +636,7 @@ func TestCheckSpendable(t *testing.T) { ctx := context.Background() // Set up a manager with a single payment channel - s := testSetupMgrWithChannel(ctx, t) + s := testSetupMgrWithChannel(t) // Create voucher with Extra voucherLane := uint64(1) @@ -821,7 +717,7 @@ func TestSubmitVoucher(t *testing.T) { ctx := context.Background() // Set up a manager with a single payment channel - s := testSetupMgrWithChannel(ctx, t) + s := testSetupMgrWithChannel(t) // Create voucher with Extra voucherLane := uint64(1) @@ -908,7 +804,7 @@ type testScaffold struct { fromKeyPrivate []byte } -func testSetupMgrWithChannel(ctx context.Context, t *testing.T) *testScaffold { +func testSetupMgrWithChannel(t *testing.T) *testScaffold { fromKeyPrivate, fromKeyPublic := testGenerateKeyPair(t) ch := tutils.NewIDAddr(t, 100) @@ -920,6 +816,8 @@ func testSetupMgrWithChannel(ctx context.Context, t *testing.T) *testScaffold { mock := newMockManagerAPI() mock.setAccountAddress(fromAcct, from) mock.setAccountAddress(toAcct, to) + //mock.setAccountState(fromAcct, account.State{Address: from}) + //mock.setAccountState(toAcct, account.State{Address: to}) // Create channel in state balance := big.NewInt(20) @@ -929,7 +827,7 @@ func testSetupMgrWithChannel(ctx context.Context, t *testing.T) *testScaffold { Nonce: 0, Balance: balance, } - mock.setPaychState(ch, act, paychmock.NewMockPayChState(fromAcct, toAcct, abi.ChainEpoch(0), big.NewInt(0), make(map[uint64]paych.LaneState))) + mock.setPaychState(ch, act, paychmock.NewMockPayChState(fromAcct, toAcct, abi.ChainEpoch(0), make(map[uint64]paych.LaneState))) store := NewStore(ds_sync.MutexWrap(ds.NewMapDatastore())) mgr, err := newManager(store, mock) diff --git a/paychmgr/paychget_test.go b/paychmgr/paychget_test.go index 430e66c67..e35ae8371 100644 --- a/paychmgr/paychget_test.go +++ b/paychmgr/paychget_test.go @@ -978,7 +978,7 @@ func TestPaychAvailableFunds(t *testing.T) { Nonce: 0, Balance: createAmt, } - mock.setPaychState(ch, act, paychmock.NewMockPayChState(fromAcct, toAcct, abi.ChainEpoch(0), big.NewInt(0), make(map[uint64]paych.LaneState))) + mock.setPaychState(ch, act, paychmock.NewMockPayChState(fromAcct, toAcct, abi.ChainEpoch(0), make(map[uint64]paych.LaneState))) // Send create channel response response := testChannelResponse(t, ch) mock.receiveMsgResponse(createMsgCid, response) diff --git a/paychmgr/simple.go b/paychmgr/simple.go index d49ccafe6..ca778829f 100644 --- a/paychmgr/simple.go +++ b/paychmgr/simple.go @@ -312,7 +312,7 @@ func (ca *channelAccessor) currentAvailableFunds(channelID string, queuedAmt typ return nil, err } - laneStates, err := ca.laneState(ca.chctx, pchState, ch) + laneStates, err := ca.laneState(pchState, ch) if err != nil { return nil, err } From e2ecc35dffdc290df95554b5c9ccfd8d02fccd61 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Mon, 14 Sep 2020 11:49:58 +0200 Subject: [PATCH 481/885] test: add test for voucher after add-funds --- paychmgr/paychvoucherfunds_test.go | 107 +++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 paychmgr/paychvoucherfunds_test.go diff --git a/paychmgr/paychvoucherfunds_test.go b/paychmgr/paychvoucherfunds_test.go new file mode 100644 index 000000000..735ea54ac --- /dev/null +++ b/paychmgr/paychvoucherfunds_test.go @@ -0,0 +1,107 @@ +package paychmgr + +import ( + "context" + "testing" + + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/specs-actors/actors/builtin" + "github.com/filecoin-project/specs-actors/actors/builtin/account" + paych "github.com/filecoin-project/specs-actors/actors/builtin/paych" + tutils "github.com/filecoin-project/specs-actors/support/testing" + "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + ds_sync "github.com/ipfs/go-datastore/sync" + "github.com/stretchr/testify/require" +) + +// TestPaychAddVoucherAfterAddFunds tests adding a voucher to a channel with +// insufficient funds, then adding funds to the channel, then adding the +// voucher again +func TestPaychAddVoucherAfterAddFunds(t *testing.T) { + ctx := context.Background() + store := NewStore(ds_sync.MutexWrap(ds.NewMapDatastore())) + + fromKeyPrivate, fromKeyPublic := testGenerateKeyPair(t) + ch := tutils.NewIDAddr(t, 100) + from := tutils.NewSECP256K1Addr(t, string(fromKeyPublic)) + to := tutils.NewSECP256K1Addr(t, "secpTo") + fromAcct := tutils.NewActorAddr(t, "fromAct") + toAcct := tutils.NewActorAddr(t, "toAct") + + mock := newMockManagerAPI() + defer mock.close() + + // Add the from signing key to the wallet + mock.setAccountState(fromAcct, account.State{Address: from}) + mock.setAccountState(toAcct, account.State{Address: to}) + mock.addSigningKey(fromKeyPrivate) + + mgr, err := newManager(store, mock) + require.NoError(t, err) + + // Send create message for a channel with value 10 + createAmt := big.NewInt(10) + _, createMsgCid, err := mgr.GetPaych(ctx, from, to, createAmt) + require.NoError(t, err) + + // Send create channel response + response := testChannelResponse(t, ch) + mock.receiveMsgResponse(createMsgCid, response) + + // Create an actor in state for the channel with the initial channel balance + act := &types.Actor{ + Code: builtin.AccountActorCodeID, + Head: cid.Cid{}, + Nonce: 0, + Balance: createAmt, + } + mock.setPaychState(ch, act, paych.State{ + From: fromAcct, + To: toAcct, + SettlingAt: abi.ChainEpoch(0), + MinSettleHeight: abi.ChainEpoch(0), + }) + + // Wait for create response to be processed by manager + _, err = mgr.GetPaychWaitReady(ctx, createMsgCid) + require.NoError(t, err) + + // Create a voucher with a value equal to the channel balance + voucher := paych.SignedVoucher{Amount: createAmt, Lane: 1} + res, err := mgr.CreateVoucher(ctx, ch, voucher) + require.NoError(t, err) + require.NotNil(t, res.Voucher) + + // Create a voucher in a different lane with an amount that exceeds the + // channel balance + excessAmt := types.NewInt(5) + voucher = paych.SignedVoucher{Amount: excessAmt, Lane: 2} + res, err = mgr.CreateVoucher(ctx, ch, voucher) + require.NoError(t, err) + require.Nil(t, res.Voucher) + require.Equal(t, res.Shortfall, excessAmt) + + // Add funds so as to cover the voucher shortfall + _, addFundsMsgCid, err := mgr.GetPaych(ctx, from, to, excessAmt) + require.NoError(t, err) + + // Trigger add funds confirmation + mock.receiveMsgResponse(addFundsMsgCid, types.MessageReceipt{ExitCode: 0}) + + // Update actor test case balance to reflect added funds + act.Balance = types.BigAdd(createAmt, excessAmt) + + // Wait for add funds confirmation to be processed by manager + _, err = mgr.GetPaychWaitReady(ctx, addFundsMsgCid) + require.NoError(t, err) + + // Adding same voucher that previously exceeded channel balance + // should succeed now that the channel balance has been increased + res, err = mgr.CreateVoucher(ctx, ch, voucher) + require.NoError(t, err) + require.NotNil(t, res.Voucher) +} From 594ec8855c4cc9a4932a22d4463c2c970b6cc6e1 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Tue, 15 Sep 2020 20:06:33 +0200 Subject: [PATCH 482/885] fix: read lane state from chain as well as datastore --- paychmgr/mock_test.go | 2 +- paychmgr/paych.go | 3 --- paychmgr/paych_test.go | 2 -- paychmgr/paychvoucherfunds_test.go | 16 ++++++---------- 4 files changed, 7 insertions(+), 16 deletions(-) diff --git a/paychmgr/mock_test.go b/paychmgr/mock_test.go index 3232e0893..c761221d2 100644 --- a/paychmgr/mock_test.go +++ b/paychmgr/mock_test.go @@ -9,6 +9,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/crypto" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors/builtin/paych" "github.com/filecoin-project/lotus/chain/types" @@ -56,7 +57,6 @@ func (sm *mockStateManager) setAccountAddress(a address.Address, lookup address. func (sm *mockStateManager) setPaychState(a address.Address, actor *types.Actor, state paych.State) { sm.lk.Lock() defer sm.lk.Unlock() - sm.paychState[a] = mockPchState{actor, state} } diff --git a/paychmgr/paych.go b/paychmgr/paych.go index 2cf200986..e67991911 100644 --- a/paychmgr/paych.go +++ b/paychmgr/paych.go @@ -539,9 +539,6 @@ func (ca *channelAccessor) laneState(state paych.State, ch address.Address) (map return nil, err } - // Note: we use a map instead of an array to store laneStates because the - // client sets the lane ID (the index) and potentially they could use a - // very large index. for _, v := range vouchers { for range v.Voucher.Merges { return nil, xerrors.Errorf("paych merges not handled yet") diff --git a/paychmgr/paych_test.go b/paychmgr/paych_test.go index 71c2bd854..56e7e9089 100644 --- a/paychmgr/paych_test.go +++ b/paychmgr/paych_test.go @@ -816,8 +816,6 @@ func testSetupMgrWithChannel(t *testing.T) *testScaffold { mock := newMockManagerAPI() mock.setAccountAddress(fromAcct, from) mock.setAccountAddress(toAcct, to) - //mock.setAccountState(fromAcct, account.State{Address: from}) - //mock.setAccountState(toAcct, account.State{Address: to}) // Create channel in state balance := big.NewInt(20) diff --git a/paychmgr/paychvoucherfunds_test.go b/paychmgr/paychvoucherfunds_test.go index 735ea54ac..dcbb4acc9 100644 --- a/paychmgr/paychvoucherfunds_test.go +++ b/paychmgr/paychvoucherfunds_test.go @@ -4,13 +4,14 @@ import ( "context" "testing" + "github.com/filecoin-project/lotus/chain/actors/builtin/paych" + paychmock "github.com/filecoin-project/lotus/chain/actors/builtin/paych/mock" + "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/builtin/account" - paych "github.com/filecoin-project/specs-actors/actors/builtin/paych" tutils "github.com/filecoin-project/specs-actors/support/testing" "github.com/ipfs/go-cid" ds "github.com/ipfs/go-datastore" @@ -36,8 +37,8 @@ func TestPaychAddVoucherAfterAddFunds(t *testing.T) { defer mock.close() // Add the from signing key to the wallet - mock.setAccountState(fromAcct, account.State{Address: from}) - mock.setAccountState(toAcct, account.State{Address: to}) + mock.setAccountAddress(fromAcct, from) + mock.setAccountAddress(toAcct, to) mock.addSigningKey(fromKeyPrivate) mgr, err := newManager(store, mock) @@ -59,12 +60,7 @@ func TestPaychAddVoucherAfterAddFunds(t *testing.T) { Nonce: 0, Balance: createAmt, } - mock.setPaychState(ch, act, paych.State{ - From: fromAcct, - To: toAcct, - SettlingAt: abi.ChainEpoch(0), - MinSettleHeight: abi.ChainEpoch(0), - }) + mock.setPaychState(ch, act, paychmock.NewMockPayChState(fromAcct, toAcct, abi.ChainEpoch(0), make(map[uint64]paych.LaneState))) // Wait for create response to be processed by manager _, err = mgr.GetPaychWaitReady(ctx, createMsgCid) From 889dd3cb3fa6e44c2e590fd1908df90db9360010 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 1 Oct 2020 08:51:01 -0700 Subject: [PATCH 483/885] improve the UX for replacing messages --- cli/mpool.go | 65 +++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 47 insertions(+), 18 deletions(-) diff --git a/cli/mpool.go b/cli/mpool.go index 65f4ef942..a8c73b656 100644 --- a/cli/mpool.go +++ b/cli/mpool.go @@ -7,6 +7,7 @@ import ( "sort" "strconv" + cid "github.com/ipfs/go-cid" "github.com/urfave/cli/v2" "golang.org/x/xerrors" @@ -43,6 +44,10 @@ var mpoolPending = &cli.Command{ Name: "local", Usage: "print pending messages for addresses in local wallet only", }, + &cli.BoolFlag{ + Name: "cids", + Usage: "only print cids of messages in output", + }, }, Action: func(cctx *cli.Context) error { api, closer, err := GetFullNodeAPI(cctx) @@ -79,11 +84,15 @@ var mpoolPending = &cli.Command{ } } - out, err := json.MarshalIndent(msg, "", " ") - if err != nil { - return err + if cctx.Bool("cids") { + fmt.Println(msg.Cid()) + } else { + out, err := json.MarshalIndent(msg, "", " ") + if err != nil { + return err + } + fmt.Println(string(out)) } - fmt.Println(string(out)) } return nil @@ -308,21 +317,8 @@ var mpoolReplaceCmd = &cli.Command{ Usage: "Spend up to X FIL for this message (applicable for auto mode)", }, }, - ArgsUsage: "[from] [nonce]", + ArgsUsage: " | ", Action: func(cctx *cli.Context) error { - if cctx.Args().Len() < 2 { - return cli.ShowCommandHelp(cctx, cctx.Command.Name) - } - - from, err := address.NewFromString(cctx.Args().Get(0)) - if err != nil { - return err - } - - nonce, err := strconv.ParseUint(cctx.Args().Get(1), 10, 64) - if err != nil { - return err - } api, closer, err := GetFullNodeAPI(cctx) if err != nil { @@ -332,6 +328,39 @@ var mpoolReplaceCmd = &cli.Command{ ctx := ReqContext(cctx) + var from address.Address + var nonce uint64 + switch cctx.Args().Len() { + case 1: + mcid, err := cid.Decode(cctx.Args().First()) + if err != nil { + return err + } + + msg, err := api.ChainGetMessage(ctx, mcid) + if err != nil { + return fmt.Errorf("could not find referenced message: %w", err) + } + + from = msg.From + nonce = msg.Nonce + case 2: + f, err := address.NewFromString(cctx.Args().Get(0)) + if err != nil { + return err + } + + n, err := strconv.ParseUint(cctx.Args().Get(1), 10, 64) + if err != nil { + return err + } + + from = f + nonce = n + default: + return cli.ShowCommandHelp(cctx, cctx.Command.Name) + } + ts, err := api.ChainHead(ctx) if err != nil { return xerrors.Errorf("getting chain head: %w", err) From 6875527224e4840f4df0032bac4e365d33290b64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 30 Sep 2020 15:17:29 +0200 Subject: [PATCH 484/885] shed: Datastore utils --- cmd/lotus-shed/datastore.go | 193 ++++++++++++++++++++++++++++++++++++ cmd/lotus-shed/main.go | 1 + go.mod | 1 + 3 files changed, 195 insertions(+) create mode 100644 cmd/lotus-shed/datastore.go diff --git a/cmd/lotus-shed/datastore.go b/cmd/lotus-shed/datastore.go new file mode 100644 index 000000000..dcd774a90 --- /dev/null +++ b/cmd/lotus-shed/datastore.go @@ -0,0 +1,193 @@ +package main + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/ipfs/go-datastore" + dsq "github.com/ipfs/go-datastore/query" + logging "github.com/ipfs/go-log" + "github.com/polydawn/refmt/cbor" + "github.com/urfave/cli/v2" + "golang.org/x/xerrors" + + "github.com/filecoin-project/lotus/node/repo" +) + +var datastoreCmd = &cli.Command{ + Name: "datastore", + Description: "access node datastores directly", + Subcommands: []*cli.Command{ + datastoreListCmd, + datastoreGetCmd, + }, +} + +var datastoreListCmd = &cli.Command{ + Name: "list", + Description: "list datastore keys", + Flags: []cli.Flag{ + &cli.IntFlag{ + Name: "repo-type", + Value: 1, + }, + &cli.BoolFlag{ + Name: "top-level", + Usage: "only print top-level keys", + }, + &cli.StringFlag{ + Name: "get-enc", + Usage: "print values [esc/hex/cbor]", + }, + }, + ArgsUsage: "[namespace prefix]", + Action: func(cctx *cli.Context) error { + logging.SetLogLevel("badger", "ERROR") // nolint:errchec + + r, err := repo.NewFS(cctx.String("repo")) + if err != nil { + return xerrors.Errorf("opening fs repo: %w", err) + } + + exists, err := r.Exists() + if err != nil { + return err + } + if !exists { + return xerrors.Errorf("lotus repo doesn't exist") + } + + lr, err := r.Lock(repo.RepoType(cctx.Int("repo-type"))) + if err != nil { + return err + } + defer lr.Close() //nolint:errcheck + + ds, err := lr.Datastore(datastore.NewKey(cctx.Args().First()).String()) + if err != nil { + return err + } + + genc := cctx.String("get-enc") + + q, err := ds.Query(dsq.Query{ + Prefix: datastore.NewKey(cctx.Args().Get(1)).String(), + KeysOnly: genc == "", + }) + if err != nil { + return xerrors.Errorf("datastore query: %w", err) + } + defer q.Close() //nolint:errcheck + + seen := map[string]struct{}{} + for res := range q.Next() { + s := res.Key + if cctx.Bool("top-level") { + k := datastore.NewKey(datastore.NewKey(s).List()[0]) + if k.Type() != "" { + s = k.Type() + } else { + s = k.String() + } + + _, has := seen[s] + if has { + continue + } + seen[s] = struct{}{} + } + + + s = fmt.Sprintf("%q", s) + s = strings.Trim(s, "\"") + fmt.Println(s) + + if genc != "" { + fmt.Print("\t") + if err := printVal(genc, res.Value); err != nil { + return err + } + } + } + + return nil + }, +} + +var datastoreGetCmd = &cli.Command{ + Name: "get", + Description: "list datastore keys", + Flags: []cli.Flag{ + &cli.IntFlag{ + Name: "repo-type", + Value: 1, + }, + &cli.StringFlag{ + Name: "enc", + Usage: "encoding (esc/hex/cbor)", + Value: "esc", + }, + }, + ArgsUsage: "[namespace key]", + Action: func(cctx *cli.Context) error { + logging.SetLogLevel("badger", "ERROR") // nolint:errchec + + r, err := repo.NewFS(cctx.String("repo")) + if err != nil { + return xerrors.Errorf("opening fs repo: %w", err) + } + + exists, err := r.Exists() + if err != nil { + return err + } + if !exists { + return xerrors.Errorf("lotus repo doesn't exist") + } + + lr, err := r.Lock(repo.RepoType(cctx.Int("repo-type"))) + if err != nil { + return err + } + defer lr.Close() //nolint:errcheck + + ds, err := lr.Datastore(datastore.NewKey(cctx.Args().First()).String()) + if err != nil { + return err + } + + val, err := ds.Get(datastore.NewKey(cctx.Args().Get(1))) + if err != nil { + return xerrors.Errorf("get: %w", err) + } + + return printVal(cctx.String("enc"), val) + }, +} + +func printVal(enc string, val []byte) error { + switch enc { + case "esc": + s := fmt.Sprintf("%q", string(val)) + s = strings.Trim(s, "\"") + fmt.Println(s) + case "hex": + fmt.Printf("%x\n", val) + case "cbor": + var out interface{} + if err := cbor.Unmarshal(cbor.DecodeOptions{}, val, &out); err != nil { + return xerrors.Errorf("unmarshaling cbor: %w", err) + } + s, err := json.Marshal(&out) + if err != nil { + return xerrors.Errorf("remarshaling as json: %w", err) + } + + fmt.Println(string(s)) + default: + return xerrors.New("unknown encoding") + } + + return nil +} diff --git a/cmd/lotus-shed/main.go b/cmd/lotus-shed/main.go index 3864d3014..409d8928b 100644 --- a/cmd/lotus-shed/main.go +++ b/cmd/lotus-shed/main.go @@ -39,6 +39,7 @@ func main() { consensusCmd, serveDealStatsCmd, syncCmd, + datastoreCmd, } app := &cli.App{ diff --git a/go.mod b/go.mod index f745ccf67..3f2661d0e 100644 --- a/go.mod +++ b/go.mod @@ -109,6 +109,7 @@ require ( github.com/multiformats/go-multibase v0.0.3 github.com/multiformats/go-multihash v0.0.14 github.com/opentracing/opentracing-go v1.2.0 + github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a github.com/raulk/clock v1.1.0 github.com/stretchr/testify v1.6.1 github.com/supranational/blst v0.1.1 From d20ebe93b9f46bd2f138e1a3172599c1a9434d00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 30 Sep 2020 17:26:24 +0200 Subject: [PATCH 485/885] shed: gofmt --- cmd/lotus-shed/datastore.go | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/cmd/lotus-shed/datastore.go b/cmd/lotus-shed/datastore.go index dcd774a90..5c668e081 100644 --- a/cmd/lotus-shed/datastore.go +++ b/cmd/lotus-shed/datastore.go @@ -16,7 +16,7 @@ import ( ) var datastoreCmd = &cli.Command{ - Name: "datastore", + Name: "datastore", Description: "access node datastores directly", Subcommands: []*cli.Command{ datastoreListCmd, @@ -25,19 +25,19 @@ var datastoreCmd = &cli.Command{ } var datastoreListCmd = &cli.Command{ - Name: "list", + Name: "list", Description: "list datastore keys", Flags: []cli.Flag{ &cli.IntFlag{ - Name: "repo-type", + Name: "repo-type", Value: 1, }, &cli.BoolFlag{ - Name: "top-level", + Name: "top-level", Usage: "only print top-level keys", }, &cli.StringFlag{ - Name: "get-enc", + Name: "get-enc", Usage: "print values [esc/hex/cbor]", }, }, @@ -72,8 +72,8 @@ var datastoreListCmd = &cli.Command{ genc := cctx.String("get-enc") q, err := ds.Query(dsq.Query{ - Prefix: datastore.NewKey(cctx.Args().Get(1)).String(), - KeysOnly: genc == "", + Prefix: datastore.NewKey(cctx.Args().Get(1)).String(), + KeysOnly: genc == "", }) if err != nil { return xerrors.Errorf("datastore query: %w", err) @@ -98,7 +98,6 @@ var datastoreListCmd = &cli.Command{ seen[s] = struct{}{} } - s = fmt.Sprintf("%q", s) s = strings.Trim(s, "\"") fmt.Println(s) @@ -116,15 +115,15 @@ var datastoreListCmd = &cli.Command{ } var datastoreGetCmd = &cli.Command{ - Name: "get", + Name: "get", Description: "list datastore keys", Flags: []cli.Flag{ &cli.IntFlag{ - Name: "repo-type", + Name: "repo-type", Value: 1, }, &cli.StringFlag{ - Name: "enc", + Name: "enc", Usage: "encoding (esc/hex/cbor)", Value: "esc", }, From 2dc9a1ee4ed042471784982c4459ffd52dc524f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 1 Oct 2020 13:58:26 +0200 Subject: [PATCH 486/885] lotus-miner backup command --- api/api_storage.go | 6 ++ api/apistruct/struct.go | 6 ++ cmd/lotus-storage-miner/backup.go | 42 ++++++++ cmd/lotus-storage-miner/main.go | 1 + lib/backupds/backupds.go | 165 ++++++++++++++++++++++++++++++ node/impl/storminer.go | 67 +++++++++++- node/modules/storage.go | 8 +- 7 files changed, 292 insertions(+), 3 deletions(-) create mode 100644 cmd/lotus-storage-miner/backup.go create mode 100644 lib/backupds/backupds.go diff --git a/api/api_storage.go b/api/api_storage.go index 824772181..529224f6e 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -101,6 +101,12 @@ type StorageMiner interface { PiecesListCidInfos(ctx context.Context) ([]cid.Cid, error) PiecesGetPieceInfo(ctx context.Context, pieceCid cid.Cid) (*piecestore.PieceInfo, error) PiecesGetCIDInfo(ctx context.Context, payloadCid cid.Cid) (*piecestore.CIDInfo, error) + + // CreateBackup creates node backup onder the specified file name. The + // method requires that the lotus-miner is running with the + // LOTUS_BACKUP_BASE_PATH environment variable set to some path, and that + // the path specified when calling CreateBackup is within the base path + CreateBackup(ctx context.Context, fpath string) error } type SealRes struct { diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index c79d0841c..6a3d709a7 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -319,6 +319,8 @@ type StorageMinerStruct struct { PiecesListCidInfos func(ctx context.Context) ([]cid.Cid, error) `perm:"read"` PiecesGetPieceInfo func(ctx context.Context, pieceCid cid.Cid) (*piecestore.PieceInfo, error) `perm:"read"` PiecesGetCIDInfo func(ctx context.Context, payloadCid cid.Cid) (*piecestore.CIDInfo, error) `perm:"read"` + + CreateBackup func(ctx context.Context, fpath string) error `perm:"admin"` } } @@ -1265,6 +1267,10 @@ func (c *StorageMinerStruct) PiecesGetCIDInfo(ctx context.Context, payloadCid ci return c.Internal.PiecesGetCIDInfo(ctx, payloadCid) } +func (c *StorageMinerStruct) CreateBackup(ctx context.Context, fpath string) error { + return c.Internal.CreateBackup(ctx, fpath) +} + // WorkerStruct func (w *WorkerStruct) Version(ctx context.Context) (build.Version, error) { diff --git a/cmd/lotus-storage-miner/backup.go b/cmd/lotus-storage-miner/backup.go new file mode 100644 index 000000000..779fd1fb4 --- /dev/null +++ b/cmd/lotus-storage-miner/backup.go @@ -0,0 +1,42 @@ +package main + +import ( + "fmt" + + "github.com/urfave/cli/v2" + "golang.org/x/xerrors" + + lcli "github.com/filecoin-project/lotus/cli" +) + +var backupCmd = &cli.Command{ + Name: "backup", + Usage: "Create node metadata backup", + Description: `The backup command writes a copy of node metadata under the specified path + +For security reasons, the daemon must be have LOTUS_BACKUP_BASE_PATH env var set +to a path where backup files are supposed to be saved, and the path specified in +this command must be within this base path`, + ArgsUsage: "[backup file path]", + Flags: []cli.Flag{}, + Action: func(cctx *cli.Context) error { + api, closer, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return err + } + defer closer() + + if cctx.Args().Len() != 1 { + return xerrors.Errorf("expected 1 argument") + } + + err = api.CreateBackup(lcli.ReqContext(cctx), cctx.Args().First()) + if err != nil { + return err + } + + fmt.Println("Success") + + return nil + }, +} diff --git a/cmd/lotus-storage-miner/main.go b/cmd/lotus-storage-miner/main.go index cee64f077..671f75cf0 100644 --- a/cmd/lotus-storage-miner/main.go +++ b/cmd/lotus-storage-miner/main.go @@ -35,6 +35,7 @@ func main() { runCmd, stopCmd, configCmd, + backupCmd, lcli.WithCategory("chain", actorCmd), lcli.WithCategory("chain", infoCmd), lcli.WithCategory("market", storageDealsCmd), diff --git a/lib/backupds/backupds.go b/lib/backupds/backupds.go new file mode 100644 index 000000000..8845eacc7 --- /dev/null +++ b/lib/backupds/backupds.go @@ -0,0 +1,165 @@ +package backupds + +import ( + "io" + "sync" + + logging "github.com/ipfs/go-log/v2" + cbg "github.com/whyrusleeping/cbor-gen" + "golang.org/x/xerrors" + + "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/query" +) + +var log = logging.Logger("backupds") + +type Datastore struct { + child datastore.Batching + + backupLk sync.RWMutex +} + +func Wrap(child datastore.Batching) *Datastore { + return &Datastore{ + child: child, + } +} + +// Writes a datastore dump into the provided writer as indefinite length cbor +// array of [key, value] tuples +func (d *Datastore) Backup(out io.Writer) error { + // write indefinite length array header + if _, err := out.Write([]byte{0x9f}); err != nil { + return xerrors.Errorf("writing header: %w", err) + } + + d.backupLk.Lock() + defer d.backupLk.Unlock() + + log.Info("Starting datastore backup") + defer log.Info("Datastore backup done") + + qr, err := d.child.Query(query.Query{}) + if err != nil { + return xerrors.Errorf("query: %w", err) + } + defer func() { + if err := qr.Close(); err != nil { + log.Errorf("query close error: %+v", err) + return + } + }() + + scratch := make([]byte, 9) + + for result := range qr.Next() { + if err := cbg.WriteMajorTypeHeaderBuf(scratch, out, cbg.MajArray, 2); err != nil { + return xerrors.Errorf("writing tuple header: %w", err) + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, out, cbg.MajByteString, uint64(len([]byte(result.Key)))); err != nil { + return xerrors.Errorf("writing key header: %w", err) + } + + if _, err := out.Write([]byte(result.Key)[:]); err != nil { + return xerrors.Errorf("writing key: %w", err) + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, out, cbg.MajByteString, uint64(len(result.Value))); err != nil { + return xerrors.Errorf("writing value header: %w", err) + } + + if _, err := out.Write(result.Value[:]); err != nil { + return xerrors.Errorf("writing value: %w", err) + } + } + + // array break + if _, err := out.Write([]byte{0xff}); err != nil { + return xerrors.Errorf("writing array 'break': %w", err) + } + + return nil +} + +// proxy + +func (d *Datastore) Get(key datastore.Key) (value []byte, err error) { + return d.child.Get(key) +} + +func (d *Datastore) Has(key datastore.Key) (exists bool, err error) { + return d.child.Has(key) +} + +func (d *Datastore) GetSize(key datastore.Key) (size int, err error) { + return d.child.GetSize(key) +} + +func (d *Datastore) Query(q query.Query) (query.Results, error) { + return d.child.Query(q) +} + +func (d *Datastore) Put(key datastore.Key, value []byte) error { + d.backupLk.RLock() + defer d.backupLk.RUnlock() + + return d.child.Put(key, value) +} + +func (d *Datastore) Delete(key datastore.Key) error { + d.backupLk.RLock() + defer d.backupLk.RUnlock() + + return d.child.Delete(key) +} + +func (d *Datastore) Sync(prefix datastore.Key) error { + d.backupLk.RLock() + defer d.backupLk.RUnlock() + + return d.child.Sync(prefix) +} + +func (d *Datastore) Close() error { + d.backupLk.RLock() + defer d.backupLk.RUnlock() + + return d.child.Close() +} + +func (d *Datastore) Batch() (datastore.Batch, error) { + b, err := d.child.Batch() + if err != nil { + return nil, err + } + + return &bbatch{ + b: b, + rlk: d.backupLk.RLocker(), + }, nil +} + +type bbatch struct { + b datastore.Batch + rlk sync.Locker +} + +func (b *bbatch) Put(key datastore.Key, value []byte) error { + return b.b.Put(key, value) +} + +func (b *bbatch) Delete(key datastore.Key) error { + return b.b.Delete(key) +} + +func (b *bbatch) Commit() error { + b.rlk.Lock() + defer b.rlk.Unlock() + + return b.b.Commit() +} + +var _ datastore.Batch = &bbatch{} +var _ datastore.Batching = &Datastore{} diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 5634c140b..c0cbe0081 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -5,21 +5,24 @@ import ( "encoding/json" "net/http" "os" + "path/filepath" "strconv" + "strings" "time" - datatransfer "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-state-types/big" "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p-core/host" + "github.com/mitchellh/go-homedir" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" + datatransfer "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/go-fil-markets/piecestore" retrievalmarket "github.com/filecoin-project/go-fil-markets/retrievalmarket" storagemarket "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-jsonrpc/auth" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" @@ -31,9 +34,11 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/apistruct" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/backupds" "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node/impl/common" "github.com/filecoin-project/lotus/node/modules/dtypes" + "github.com/filecoin-project/lotus/node/repo" "github.com/filecoin-project/lotus/storage" "github.com/filecoin-project/lotus/storage/sectorblocks" ) @@ -56,6 +61,9 @@ type StorageMinerAPI struct { DataTransfer dtypes.ProviderDataTransfer Host host.Host + DS dtypes.MetadataDS + Repo repo.LockedRepo + ConsiderOnlineStorageDealsConfigFunc dtypes.ConsiderOnlineStorageDealsConfigFunc SetConsiderOnlineStorageDealsConfigFunc dtypes.SetConsiderOnlineStorageDealsConfigFunc ConsiderOnlineRetrievalDealsConfigFunc dtypes.ConsiderOnlineRetrievalDealsConfigFunc @@ -516,4 +524,59 @@ func (sm *StorageMinerAPI) PiecesGetCIDInfo(ctx context.Context, payloadCid cid. return &ci, nil } +func (sm *StorageMinerAPI) CreateBackup(ctx context.Context, fpath string) error { + // TODO: Config + bb, ok := os.LookupEnv("LOTUS_BACKUP_BASE_PATH") + if !ok { + return xerrors.Errorf("LOTUS_BACKUP_BASE_PATH env var not set") + } + + bds, ok := sm.DS.(*backupds.Datastore) + if !ok { + return xerrors.Errorf("expected a backup datastore") + } + + bb, err := homedir.Expand(bb) + if err != nil { + return xerrors.Errorf("expanding base path: %w", err) + } + + bb, err = filepath.Abs(bb) + if err != nil { + return xerrors.Errorf("getting absolute base path: %w", err) + } + + fpath, err = homedir.Expand(fpath) + if err != nil { + return xerrors.Errorf("expanding file path: %w", err) + } + + fpath, err = filepath.Abs(fpath) + if err != nil { + return xerrors.Errorf("getting absolute file path: %w", err) + } + + if !strings.HasPrefix(fpath, bb) { + return xerrors.Errorf("backup file name (%s) must be inside base path (%s)", fpath, bb) + } + + out, err := os.OpenFile(fpath, os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return xerrors.Errorf("open %s: %w", fpath, err) + } + + if err := bds.Backup(out); err != nil { + if cerr := out.Close(); cerr != nil { + log.Errorw("error closing backup file while handling backup error", "closeErr", cerr, "backupErr", err) + } + return xerrors.Errorf("backup error: %w", err) + } + + if err := out.Close(); err != nil { + return xerrors.Errorf("closing backup file: %w", err) + } + + return nil +} + var _ api.StorageMiner = &StorageMinerAPI{} diff --git a/node/modules/storage.go b/node/modules/storage.go index 1bdce1d2f..9c1a18368 100644 --- a/node/modules/storage.go +++ b/node/modules/storage.go @@ -6,6 +6,7 @@ import ( "go.uber.org/fx" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/backupds" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/repo" ) @@ -27,5 +28,10 @@ func KeyStore(lr repo.LockedRepo) (types.KeyStore, error) { } func Datastore(r repo.LockedRepo) (dtypes.MetadataDS, error) { - return r.Datastore("/metadata") + mds, err := r.Datastore("/metadata") + if err != nil { + return nil, err + } + + return backupds.Wrap(mds), nil } From c8a3269c4b472133da5d61e3f5b9a17808f7a8c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 1 Oct 2020 14:36:19 +0200 Subject: [PATCH 487/885] shed: utils to read datastore backups --- cmd/lotus-shed/datastore.go | 148 +++++++++++++++++---- lib/backupds/{backupds.go => datastore.go} | 0 lib/backupds/read.go | 52 ++++++++ 3 files changed, 174 insertions(+), 26 deletions(-) rename lib/backupds/{backupds.go => datastore.go} (100%) create mode 100644 lib/backupds/read.go diff --git a/cmd/lotus-shed/datastore.go b/cmd/lotus-shed/datastore.go index 5c668e081..eb5db2aa9 100644 --- a/cmd/lotus-shed/datastore.go +++ b/cmd/lotus-shed/datastore.go @@ -3,8 +3,10 @@ package main import ( "encoding/json" "fmt" + "os" "strings" + "github.com/docker/go-units" "github.com/ipfs/go-datastore" dsq "github.com/ipfs/go-datastore/query" logging "github.com/ipfs/go-log" @@ -12,6 +14,7 @@ import ( "github.com/urfave/cli/v2" "golang.org/x/xerrors" + "github.com/filecoin-project/lotus/lib/backupds" "github.com/filecoin-project/lotus/node/repo" ) @@ -19,6 +22,7 @@ var datastoreCmd = &cli.Command{ Name: "datastore", Description: "access node datastores directly", Subcommands: []*cli.Command{ + datastoreBackupCmd, datastoreListCmd, datastoreGetCmd, }, @@ -80,33 +84,11 @@ var datastoreListCmd = &cli.Command{ } defer q.Close() //nolint:errcheck - seen := map[string]struct{}{} + printKv := kvPrinter(cctx.Bool("top-level"), genc) + for res := range q.Next() { - s := res.Key - if cctx.Bool("top-level") { - k := datastore.NewKey(datastore.NewKey(s).List()[0]) - if k.Type() != "" { - s = k.Type() - } else { - s = k.String() - } - - _, has := seen[s] - if has { - continue - } - seen[s] = struct{}{} - } - - s = fmt.Sprintf("%q", s) - s = strings.Trim(s, "\"") - fmt.Println(s) - - if genc != "" { - fmt.Print("\t") - if err := printVal(genc, res.Value); err != nil { - return err - } + if err := printKv(res.Key, res.Value); err != nil { + return err } } @@ -165,6 +147,120 @@ var datastoreGetCmd = &cli.Command{ }, } +var datastoreBackupCmd = &cli.Command{ + Name: "backup", + Description: "manage datastore backups", + Subcommands: []*cli.Command{ + datastoreBackupStatCmd, + datastoreBackupListCmd, + }, +} + +var datastoreBackupStatCmd = &cli.Command{ + Name: "stat", + Description: "validate and print info about datastore backup", + ArgsUsage: "[file]", + Action: func(cctx *cli.Context) error { + if cctx.Args().Len() != 1 { + return xerrors.Errorf("expected 1 argument") + } + + f, err := os.Open(cctx.Args().First()) + if err != nil { + return xerrors.Errorf("opening backup file: %w", err) + } + defer f.Close() // nolint:errcheck + + var keys, kbytes, vbytes uint64 + err = backupds.ReadBackup(f, func(key datastore.Key, value []byte) error { + keys++ + kbytes += uint64(len(key.String())) + vbytes += uint64(len(value)) + return nil + }) + if err != nil { + return err + } + + fmt.Println("Keys: ", keys) + fmt.Println("Key bytes: ", units.BytesSize(float64(kbytes))) + fmt.Println("Value bytes: ", units.BytesSize(float64(vbytes))) + + return err + }, +} + +var datastoreBackupListCmd = &cli.Command{ + Name: "list", + Description: "list data in a backup", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "top-level", + Usage: "only print top-level keys", + }, + &cli.StringFlag{ + Name: "get-enc", + Usage: "print values [esc/hex/cbor]", + }, + }, + ArgsUsage: "[file]", + Action: func(cctx *cli.Context) error { + if cctx.Args().Len() != 1 { + return xerrors.Errorf("expected 1 argument") + } + + f, err := os.Open(cctx.Args().First()) + if err != nil { + return xerrors.Errorf("opening backup file: %w", err) + } + defer f.Close() // nolint:errcheck + + printKv := kvPrinter(cctx.Bool("top-level"), cctx.String("get-enc")) + err = backupds.ReadBackup(f, func(key datastore.Key, value []byte) error { + return printKv(key.String(), value) + }) + if err != nil { + return err + } + + return err + }, +} + +func kvPrinter(toplevel bool, genc string) func(sk string, value []byte) error { + seen := map[string]struct{}{} + + return func(s string, value []byte) error { + if toplevel { + k := datastore.NewKey(datastore.NewKey(s).List()[0]) + if k.Type() != "" { + s = k.Type() + } else { + s = k.String() + } + + _, has := seen[s] + if has { + return nil + } + seen[s] = struct{}{} + } + + s = fmt.Sprintf("%q", s) + s = strings.Trim(s, "\"") + fmt.Println(s) + + if genc != "" { + fmt.Print("\t") + if err := printVal(genc, value); err != nil { + return err + } + } + + return nil + } +} + func printVal(enc string, val []byte) error { switch enc { case "esc": diff --git a/lib/backupds/backupds.go b/lib/backupds/datastore.go similarity index 100% rename from lib/backupds/backupds.go rename to lib/backupds/datastore.go diff --git a/lib/backupds/read.go b/lib/backupds/read.go new file mode 100644 index 000000000..e1ae5837e --- /dev/null +++ b/lib/backupds/read.go @@ -0,0 +1,52 @@ +package backupds + +import ( + "io" + + "github.com/ipfs/go-datastore" + cbg "github.com/whyrusleeping/cbor-gen" + "golang.org/x/xerrors" +) + +func ReadBackup(r io.Reader, cb func(key datastore.Key, value []byte) error) error { + scratch := make([]byte, 9) + + if _, err := r.Read(scratch[:1]); err != nil { + return xerrors.Errorf("reading array header: %w", err) + } + + if scratch[0] != 0x9f { + return xerrors.Errorf("expected indefinite length array header byte 0x9f, got %x", scratch[0]) + } + + for { + if _, err := r.Read(scratch[:1]); err != nil { + return xerrors.Errorf("reading tuple header: %w", err) + } + + if scratch[0] == 0xff { + break + } + + if scratch[0] != 0x82 { + return xerrors.Errorf("expected array(2) header 0x82, got %x", scratch[0]) + } + + keyb, err := cbg.ReadByteArray(r, 1<<40) + if err != nil { + return xerrors.Errorf("reading key: %w", err) + } + key := datastore.NewKey(string(keyb)) + + value, err := cbg.ReadByteArray(r, 1<<40) + if err != nil { + return xerrors.Errorf("reading value: %w", err) + } + + if err := cb(key, value); err != nil { + return err + } + } + + return nil +} From deac7166b5251f7c503cce3a3b20f1f6fa55efeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 1 Oct 2020 14:46:07 +0200 Subject: [PATCH 488/885] Support offline backups --- cmd/lotus-shed/datastore.go | 2 +- cmd/lotus-storage-miner/backup.go | 99 +++++++++++++++++++++++++++---- 2 files changed, 87 insertions(+), 14 deletions(-) diff --git a/cmd/lotus-shed/datastore.go b/cmd/lotus-shed/datastore.go index eb5db2aa9..62ee6bc6e 100644 --- a/cmd/lotus-shed/datastore.go +++ b/cmd/lotus-shed/datastore.go @@ -47,7 +47,7 @@ var datastoreListCmd = &cli.Command{ }, ArgsUsage: "[namespace prefix]", Action: func(cctx *cli.Context) error { - logging.SetLogLevel("badger", "ERROR") // nolint:errchec + logging.SetLogLevel("badger", "ERROR") // nolint:errcheck r, err := repo.NewFS(cctx.String("repo")) if err != nil { diff --git a/cmd/lotus-storage-miner/backup.go b/cmd/lotus-storage-miner/backup.go index 779fd1fb4..a1ad360c9 100644 --- a/cmd/lotus-storage-miner/backup.go +++ b/cmd/lotus-storage-miner/backup.go @@ -2,11 +2,16 @@ package main import ( "fmt" + "os" + logging "github.com/ipfs/go-log/v2" + "github.com/mitchellh/go-homedir" "github.com/urfave/cli/v2" "golang.org/x/xerrors" lcli "github.com/filecoin-project/lotus/cli" + "github.com/filecoin-project/lotus/lib/backupds" + "github.com/filecoin-project/lotus/node/repo" ) var backupCmd = &cli.Command{ @@ -14,29 +19,97 @@ var backupCmd = &cli.Command{ Usage: "Create node metadata backup", Description: `The backup command writes a copy of node metadata under the specified path +Online backups: For security reasons, the daemon must be have LOTUS_BACKUP_BASE_PATH env var set to a path where backup files are supposed to be saved, and the path specified in this command must be within this base path`, + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "offline", + Usage: "create backup without the node running", + }, + }, ArgsUsage: "[backup file path]", - Flags: []cli.Flag{}, Action: func(cctx *cli.Context) error { - api, closer, err := lcli.GetStorageMinerAPI(cctx) - if err != nil { - return err - } - defer closer() - if cctx.Args().Len() != 1 { return xerrors.Errorf("expected 1 argument") } - err = api.CreateBackup(lcli.ReqContext(cctx), cctx.Args().First()) - if err != nil { - return err + if cctx.Bool("offline") { + return offlineBackup(cctx) } - fmt.Println("Success") - - return nil + return onlineBackup(cctx) }, } + +func offlineBackup(cctx *cli.Context) error { + logging.SetLogLevel("badger", "ERROR") // nolint:errcheck + + repoPath := cctx.String(FlagMinerRepo) + r, err := repo.NewFS(repoPath) + if err != nil { + return err + } + + ok, err := r.Exists() + if err != nil { + return err + } + if !ok { + return xerrors.Errorf("repo at '%s' is not initialized", cctx.String(FlagMinerRepo)) + } + + lr, err := r.Lock(repo.StorageMiner) + if err != nil { + return xerrors.Errorf("locking repo: %w", err) + } + defer lr.Close() // nolint:errcheck + + mds, err := lr.Datastore("/metadata") + if err != nil { + return xerrors.Errorf("getting metadata datastore: %w", err) + } + + bds := backupds.Wrap(mds) + + fpath, err := homedir.Expand(cctx.Args().First()) + if err != nil { + return xerrors.Errorf("expanding file path: %w", err) + } + + out, err := os.OpenFile(fpath, os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return xerrors.Errorf("opening backup file %s: %w", fpath, err) + } + + if err := bds.Backup(out); err != nil { + if cerr := out.Close(); cerr != nil { + log.Errorw("error closing backup file while handling backup error", "closeErr", cerr, "backupErr", err) + } + return xerrors.Errorf("backup error: %w", err) + } + + if err := out.Close(); err != nil { + return xerrors.Errorf("closing backup file: %w", err) + } + + return nil +} + +func onlineBackup(cctx *cli.Context) error { + api, closer, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return xerrors.Errorf("getting api: %w (if the node isn't running you can use the --offline flag)", err) + } + defer closer() + + err = api.CreateBackup(lcli.ReqContext(cctx), cctx.Args().First()) + if err != nil { + return err + } + + fmt.Println("Success") + + return nil +} From 9b5a0815fc557f41fac8ebccc40721b86292481c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 1 Oct 2020 14:51:37 +0200 Subject: [PATCH 489/885] backup: open datastores in readonly in offline mode --- cmd/lotus-storage-miner/backup.go | 2 +- node/repo/fsrepo.go | 12 ++++++++++++ node/repo/fsrepo_ds.go | 18 +++++++++++------- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/cmd/lotus-storage-miner/backup.go b/cmd/lotus-storage-miner/backup.go index a1ad360c9..55604b6fb 100644 --- a/cmd/lotus-storage-miner/backup.go +++ b/cmd/lotus-storage-miner/backup.go @@ -60,7 +60,7 @@ func offlineBackup(cctx *cli.Context) error { return xerrors.Errorf("repo at '%s' is not initialized", cctx.String(FlagMinerRepo)) } - lr, err := r.Lock(repo.StorageMiner) + lr, err := r.LockRO(repo.StorageMiner) if err != nil { return xerrors.Errorf("locking repo: %w", err) } diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index a69cdd55d..c4571c478 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -226,11 +226,23 @@ func (fsr *FsRepo) Lock(repoType RepoType) (LockedRepo, error) { }, nil } +// Like Lock, except datastores will work in read-only mode +func (fsr *FsRepo) LockRO(repoType RepoType) (LockedRepo, error) { + lr, err := fsr.Lock(repoType) + if err != nil { + return nil, err + } + + lr.(*fsLockedRepo).readonly = true + return lr, nil +} + type fsLockedRepo struct { path string configPath string repoType RepoType closer io.Closer + readonly bool ds map[string]datastore.Batching dsErr error diff --git a/node/repo/fsrepo_ds.go b/node/repo/fsrepo_ds.go index f4afe0dee..aa91d2514 100644 --- a/node/repo/fsrepo_ds.go +++ b/node/repo/fsrepo_ds.go @@ -14,7 +14,7 @@ import ( ldbopts "github.com/syndtr/goleveldb/leveldb/opt" ) -type dsCtor func(path string) (datastore.Batching, error) +type dsCtor func(path string, readonly bool) (datastore.Batching, error) var fsDatastores = map[string]dsCtor{ "chain": chainBadgerDs, @@ -26,9 +26,10 @@ var fsDatastores = map[string]dsCtor{ "client": badgerDs, // client specific } -func chainBadgerDs(path string) (datastore.Batching, error) { +func chainBadgerDs(path string, readonly bool) (datastore.Batching, error) { opts := badger.DefaultOptions opts.GcInterval = 0 // disable GC for chain datastore + opts.ReadOnly = readonly opts.Options = dgbadger.DefaultOptions("").WithTruncate(true). WithValueThreshold(1 << 10) @@ -36,23 +37,26 @@ func chainBadgerDs(path string) (datastore.Batching, error) { return badger.NewDatastore(path, &opts) } -func badgerDs(path string) (datastore.Batching, error) { +func badgerDs(path string, readonly bool) (datastore.Batching, error) { opts := badger.DefaultOptions + opts.ReadOnly = readonly + opts.Options = dgbadger.DefaultOptions("").WithTruncate(true). WithValueThreshold(1 << 10) return badger.NewDatastore(path, &opts) } -func levelDs(path string) (datastore.Batching, error) { +func levelDs(path string, readonly bool) (datastore.Batching, error) { return levelds.NewDatastore(path, &levelds.Options{ Compression: ldbopts.NoCompression, NoSync: false, Strict: ldbopts.StrictAll, + ReadOnly: readonly, }) } -func (fsr *fsLockedRepo) openDatastores() (map[string]datastore.Batching, error) { +func (fsr *fsLockedRepo) openDatastores(readonly bool) (map[string]datastore.Batching, error) { if err := os.MkdirAll(fsr.join(fsDatastore), 0755); err != nil { return nil, xerrors.Errorf("mkdir %s: %w", fsr.join(fsDatastore), err) } @@ -63,7 +67,7 @@ func (fsr *fsLockedRepo) openDatastores() (map[string]datastore.Batching, error) prefix := datastore.NewKey(p) // TODO: optimization: don't init datastores we don't need - ds, err := ctor(fsr.join(filepath.Join(fsDatastore, p))) + ds, err := ctor(fsr.join(filepath.Join(fsDatastore, p)), readonly) if err != nil { return nil, xerrors.Errorf("opening datastore %s: %w", prefix, err) } @@ -78,7 +82,7 @@ func (fsr *fsLockedRepo) openDatastores() (map[string]datastore.Batching, error) func (fsr *fsLockedRepo) Datastore(ns string) (datastore.Batching, error) { fsr.dsOnce.Do(func() { - fsr.ds, fsr.dsErr = fsr.openDatastores() + fsr.ds, fsr.dsErr = fsr.openDatastores(fsr.readonly) }) if fsr.dsErr != nil { From d7ec5e36184cbfc3a152d06c26ae2531f1524aee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 1 Oct 2020 16:22:54 +0200 Subject: [PATCH 490/885] lotus-miner init restore --- cmd/lotus-storage-miner/init.go | 3 + cmd/lotus-storage-miner/init_restore.go | 169 ++++++++++++++++++++++++ lib/backupds/read.go | 24 ++++ 3 files changed, 196 insertions(+) create mode 100644 cmd/lotus-storage-miner/init_restore.go diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index 4faf6a25b..9218faa77 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -115,6 +115,9 @@ var initCmd = &cli.Command{ Usage: "select which address to send actor creation message from", }, }, + Subcommands: []*cli.Command{ + initRestoreCmd, + }, Action: func(cctx *cli.Context) error { log.Info("Initializing lotus miner") diff --git a/cmd/lotus-storage-miner/init_restore.go b/cmd/lotus-storage-miner/init_restore.go new file mode 100644 index 000000000..8c8c6213e --- /dev/null +++ b/cmd/lotus-storage-miner/init_restore.go @@ -0,0 +1,169 @@ +package main + +import ( + "os" + + "github.com/docker/go-units" + "github.com/ipfs/go-datastore" + "github.com/libp2p/go-libp2p-core/peer" + "github.com/urfave/cli/v2" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + paramfetch "github.com/filecoin-project/go-paramfetch" + "github.com/filecoin-project/go-state-types/big" + + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/types" + lcli "github.com/filecoin-project/lotus/cli" + "github.com/filecoin-project/lotus/lib/backupds" + "github.com/filecoin-project/lotus/node/repo" +) + +var initRestoreCmd = &cli.Command{ + Name: "restore", + Usage: "Initialize a lotus miner repo from a backup", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "nosync", + Usage: "don't check full-node sync status", + }, + // TODO: Config + // TODO: Storage paths + }, + ArgsUsage: "[backupFile]", + Action: func(cctx *cli.Context) error { + log.Info("Initializing lotus miner using a backup") + if cctx.Args().Len() != 1 { + return xerrors.Errorf("expected 1 argument") + } + + log.Info("Trying to connect to full node RPC") + + api, closer, err := lcli.GetFullNodeAPI(cctx) // TODO: consider storing full node address in config + if err != nil { + return err + } + defer closer() + + log.Info("Checking full node version") + + ctx := lcli.ReqContext(cctx) + + v, err := api.Version(ctx) + if err != nil { + return err + } + + if !v.APIVersion.EqMajorMinor(build.FullAPIVersion) { + return xerrors.Errorf("Remote API version didn't match (expected %s, remote %s)", build.FullAPIVersion, v.APIVersion) + } + + if !cctx.Bool("nosync") { + if err := lcli.SyncWait(ctx, api); err != nil { + return xerrors.Errorf("sync wait: %w", err) + } + } + + f, err := os.Open(cctx.Args().First()) + if err != nil { + return xerrors.Errorf("opening backup file: %w", err) + } + defer f.Close() // nolint:errcheck + + log.Info("Checking if repo exists") + + repoPath := cctx.String(FlagMinerRepo) + r, err := repo.NewFS(repoPath) + if err != nil { + return err + } + + ok, err := r.Exists() + if err != nil { + return err + } + if ok { + return xerrors.Errorf("repo at '%s' is already initialized", cctx.String(FlagMinerRepo)) + } + + log.Info("Initializing repo") + + if err := r.Init(repo.StorageMiner); err != nil { + return err + } + + lr, err := r.Lock(repo.StorageMiner) + if err != nil { + return err + } + defer lr.Close() //nolint:errcheck + + log.Info("Restoring metadata backup") + + mds, err := lr.Datastore("/metadata") + if err != nil { + return err + } + + if err := backupds.RestoreInto(f, mds); err != nil { + return xerrors.Errorf("restoring metadata: %w", err) + } + + log.Info("Checking actor metadata") + + abytes, err := mds.Get(datastore.NewKey("miner-address")) + if err != nil { + return xerrors.Errorf("getting actor address from metadata datastore: %w", err) + } + + maddr, err := address.NewFromBytes(abytes) + if err != nil { + return xerrors.Errorf("parsing actor address: %w", err) + } + + log.Info("ACTOR ADDRESS: ", maddr.String()) + + mi, err := api.StateMinerInfo(ctx, maddr, types.EmptyTSK) + if err != nil { + return xerrors.Errorf("getting miner info: %w", err) + } + + log.Info("SECTOR SIZE: ", units.BytesSize(float64(mi.SectorSize))) + + has, err := api.WalletHas(ctx, mi.Worker) + if err != nil { + return xerrors.Errorf("checking worker address: %w", err) + } + + if !has { + return xerrors.Errorf("worker address %s for miner actor %s not present in full node wallet", mi.Worker, maddr) + } + + log.Info("Checking proof parameters") + + if err := paramfetch.GetParams(ctx, build.ParametersJSON(), uint64(mi.SectorSize)); err != nil { + return xerrors.Errorf("fetching proof parameters: %w", err) + } + + log.Info("Initializing libp2p identity") + + p2pSk, err := makeHostKey(lr) + if err != nil { + return xerrors.Errorf("make host key: %w", err) + } + + peerid, err := peer.IDFromPrivateKey(p2pSk) + if err != nil { + return xerrors.Errorf("peer ID from private key: %w", err) + } + + log.Info("Configuring miner actor") + + if err := configureStorageMiner(ctx, api, maddr, peerid, big.Zero()); err != nil { + return err + } + + return nil + }, +} diff --git a/lib/backupds/read.go b/lib/backupds/read.go index e1ae5837e..d368d1d09 100644 --- a/lib/backupds/read.go +++ b/lib/backupds/read.go @@ -50,3 +50,27 @@ func ReadBackup(r io.Reader, cb func(key datastore.Key, value []byte) error) err return nil } + +func RestoreInto(r io.Reader, dest datastore.Batching) error { + batch, err := dest.Batch() + if err != nil { + return xerrors.Errorf("creating batch: %w", err) + } + + err = ReadBackup(r, func(key datastore.Key, value []byte) error { + if err := batch.Put(key, value); err != nil { + return xerrors.Errorf("put key: %w", err) + } + + return nil + }) + if err != nil { + return xerrors.Errorf("reading backup: %w", err) + } + + if err := batch.Commit(); err != nil { + return xerrors.Errorf("committing batch: %w", err) + } + + return nil +} From 44747446a95e5e4e32a473f3291ff0842f787c82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 1 Oct 2020 16:45:02 +0200 Subject: [PATCH 491/885] miner restore: config flag --- cmd/lotus-storage-miner/init_restore.go | 47 ++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/cmd/lotus-storage-miner/init_restore.go b/cmd/lotus-storage-miner/init_restore.go index 8c8c6213e..e1219c039 100644 --- a/cmd/lotus-storage-miner/init_restore.go +++ b/cmd/lotus-storage-miner/init_restore.go @@ -1,6 +1,8 @@ package main import ( + "github.com/filecoin-project/lotus/node/config" + "github.com/mitchellh/go-homedir" "os" "github.com/docker/go-units" @@ -28,7 +30,10 @@ var initRestoreCmd = &cli.Command{ Name: "nosync", Usage: "don't check full-node sync status", }, - // TODO: Config + &cli.StringFlag{ + Name: "config", + Usage: "config file", + }, // TODO: Storage paths }, ArgsUsage: "[backupFile]", @@ -99,6 +104,46 @@ var initRestoreCmd = &cli.Command{ } defer lr.Close() //nolint:errcheck + if cctx.IsSet("config") { + log.Info("Restoring config") + + cf, err := homedir.Expand(cctx.String("config")) + if err != nil { + return xerrors.Errorf("expanding config path: %w", err) + } + + _, err = os.Stat(cf) + if err != nil { + return xerrors.Errorf("stat config file (%s): %w", cf, err) + } + + var cerr error + err = lr.SetConfig(func(raw interface{}) { + rcfg, ok := raw.(*config.StorageMiner) + if !ok { + cerr = xerrors.New("expected miner config") + return + } + + ff, err := config.FromFile(cf, rcfg) + if err != nil { + cerr = xerrors.Errorf("loading config: %w", err) + return + } + + *rcfg = *ff.(*config.StorageMiner) + }) + if cerr != nil { + return cerr + } + if err != nil { + return xerrors.Errorf("setting config: %w", err) + } + + } else { + log.Warn("--config NOT SET, WILL USE DEFAULT VALUES") + } + log.Info("Restoring metadata backup") mds, err := lr.Datastore("/metadata") From 5c33982f72bece681bd0f3465cd6ff66eae58263 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 1 Oct 2020 16:51:30 +0200 Subject: [PATCH 492/885] miner restore: storage config flag --- cmd/lotus-storage-miner/init_restore.go | 37 +++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/cmd/lotus-storage-miner/init_restore.go b/cmd/lotus-storage-miner/init_restore.go index e1219c039..560eae33b 100644 --- a/cmd/lotus-storage-miner/init_restore.go +++ b/cmd/lotus-storage-miner/init_restore.go @@ -1,8 +1,11 @@ package main import ( + "encoding/json" + "github.com/filecoin-project/lotus/extern/sector-storage/stores" "github.com/filecoin-project/lotus/node/config" "github.com/mitchellh/go-homedir" + "io/ioutil" "os" "github.com/docker/go-units" @@ -32,9 +35,12 @@ var initRestoreCmd = &cli.Command{ }, &cli.StringFlag{ Name: "config", - Usage: "config file", + Usage: "config file (config.toml)", + }, + &cli.StringFlag{ + Name: "storage-config", + Usage: "storage paths config (storage.json)", }, - // TODO: Storage paths }, ArgsUsage: "[backupFile]", Action: func(cctx *cli.Context) error { @@ -144,6 +150,33 @@ var initRestoreCmd = &cli.Command{ log.Warn("--config NOT SET, WILL USE DEFAULT VALUES") } + if cctx.IsSet("storage-config") { + log.Info("Restoring storage path config") + + cf, err := homedir.Expand(cctx.String("storage-config")) + if err != nil { + return xerrors.Errorf("expanding storage config path: %w", err) + } + + cfb, err := ioutil.ReadFile(cf) + if err != nil { + return xerrors.Errorf("reading storage config: %w", err) + } + + var cerr error + err = lr.SetStorage(func(scfg *stores.StorageConfig) { + cerr = json.Unmarshal(cfb, scfg) + }) + if cerr != nil { + return xerrors.Errorf("unmarshalling storage config: %w", cerr) + } + if err != nil { + return xerrors.Errorf("setting storage config: %w", err) + } + } else { + log.Warn("--storage-config NOT SET. NO SECTOR PATHS WILL BE CONFIGURED") + } + log.Info("Restoring metadata backup") mds, err := lr.Datastore("/metadata") From e444977891bd17e8809c3ed72105537046bb6a36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 1 Oct 2020 17:14:08 +0200 Subject: [PATCH 493/885] Full node metadata backup --- api/api_full.go | 6 ++ api/apistruct/struct.go | 6 ++ cli/backup.go | 125 ++++++++++++++++++++++++++++++ cmd/lotus-storage-miner/backup.go | 111 ++------------------------ cmd/lotus/backup.go | 14 ++++ cmd/lotus/main.go | 1 + node/impl/backup.go | 67 ++++++++++++++++ node/impl/full.go | 9 +++ node/impl/storminer.go | 61 +-------------- 9 files changed, 235 insertions(+), 165 deletions(-) create mode 100644 cli/backup.go create mode 100644 cmd/lotus/backup.go create mode 100644 node/impl/backup.go diff --git a/api/api_full.go b/api/api_full.go index 767739582..acbae19c4 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -475,6 +475,12 @@ type FullNode interface { PaychVoucherAdd(context.Context, address.Address, *paych.SignedVoucher, []byte, types.BigInt) (types.BigInt, error) PaychVoucherList(context.Context, address.Address) ([]*paych.SignedVoucher, error) PaychVoucherSubmit(context.Context, address.Address, *paych.SignedVoucher, []byte, []byte) (cid.Cid, error) + + // CreateBackup creates node backup onder the specified file name. The + // method requires that the lotus daemon is running with the + // LOTUS_BACKUP_BASE_PATH environment variable set to some path, and that + // the path specified when calling CreateBackup is within the base path + CreateBackup(ctx context.Context, fpath string) error } type FileRef struct { diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 6a3d709a7..2ae4ac08d 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -239,6 +239,8 @@ type FullNodeStruct struct { PaychVoucherCreate func(context.Context, address.Address, big.Int, uint64) (*api.VoucherCreateResult, error) `perm:"sign"` PaychVoucherList func(context.Context, address.Address) ([]*paych.SignedVoucher, error) `perm:"write"` PaychVoucherSubmit func(context.Context, address.Address, *paych.SignedVoucher, []byte, []byte) (cid.Cid, error) `perm:"sign"` + + CreateBackup func(ctx context.Context, fpath string) error `perm:"admin"` } } @@ -1027,6 +1029,10 @@ func (c *FullNodeStruct) PaychVoucherSubmit(ctx context.Context, ch address.Addr return c.Internal.PaychVoucherSubmit(ctx, ch, sv, secret, proof) } +func (c *FullNodeStruct) CreateBackup(ctx context.Context, fpath string) error { + return c.Internal.CreateBackup(ctx, fpath) +} + // StorageMinerStruct func (c *StorageMinerStruct) ActorAddress(ctx context.Context) (address.Address, error) { diff --git a/cli/backup.go b/cli/backup.go new file mode 100644 index 000000000..c748e47c4 --- /dev/null +++ b/cli/backup.go @@ -0,0 +1,125 @@ +package cli + +import ( + "context" + "fmt" + "os" + + logging "github.com/ipfs/go-log/v2" + "github.com/mitchellh/go-homedir" + "github.com/urfave/cli/v2" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-jsonrpc" + + "github.com/filecoin-project/lotus/lib/backupds" + "github.com/filecoin-project/lotus/node/repo" +) + +type BackupAPI interface { + CreateBackup(ctx context.Context, fpath string) error +} + +type BackupApiFn func(ctx *cli.Context) (BackupAPI, jsonrpc.ClientCloser, error) + +func BackupCmd(repoFlag string, rt repo.RepoType, getApi BackupApiFn) *cli.Command { + var offlineBackup = func(cctx *cli.Context) error { + logging.SetLogLevel("badger", "ERROR") // nolint:errcheck + + repoPath := cctx.String(repoFlag) + r, err := repo.NewFS(repoPath) + if err != nil { + return err + } + + ok, err := r.Exists() + if err != nil { + return err + } + if !ok { + return xerrors.Errorf("repo at '%s' is not initialized", cctx.String(repoFlag)) + } + + lr, err := r.LockRO(rt) + if err != nil { + return xerrors.Errorf("locking repo: %w", err) + } + defer lr.Close() // nolint:errcheck + + mds, err := lr.Datastore("/metadata") + if err != nil { + return xerrors.Errorf("getting metadata datastore: %w", err) + } + + bds := backupds.Wrap(mds) + + fpath, err := homedir.Expand(cctx.Args().First()) + if err != nil { + return xerrors.Errorf("expanding file path: %w", err) + } + + out, err := os.OpenFile(fpath, os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return xerrors.Errorf("opening backup file %s: %w", fpath, err) + } + + if err := bds.Backup(out); err != nil { + if cerr := out.Close(); cerr != nil { + log.Errorw("error closing backup file while handling backup error", "closeErr", cerr, "backupErr", err) + } + return xerrors.Errorf("backup error: %w", err) + } + + if err := out.Close(); err != nil { + return xerrors.Errorf("closing backup file: %w", err) + } + + return nil + } + + var onlineBackup = func(cctx *cli.Context) error { + api, closer, err := getApi(cctx) + if err != nil { + return xerrors.Errorf("getting api: %w (if the node isn't running you can use the --offline flag)", err) + } + defer closer() + + err = api.CreateBackup(ReqContext(cctx), cctx.Args().First()) + if err != nil { + return err + } + + fmt.Println("Success") + + return nil + } + + return &cli.Command{ + Name: "backup", + Usage: "Create node metadata backup", + Description: `The backup command writes a copy of node metadata under the specified path + +Online backups: +For security reasons, the daemon must be have LOTUS_BACKUP_BASE_PATH env var set +to a path where backup files are supposed to be saved, and the path specified in +this command must be within this base path`, + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "offline", + Usage: "create backup without the node running", + }, + }, + ArgsUsage: "[backup file path]", + Action: func(cctx *cli.Context) error { + if cctx.Args().Len() != 1 { + return xerrors.Errorf("expected 1 argument") + } + + if cctx.Bool("offline") { + return offlineBackup(cctx) + } + + return onlineBackup(cctx) + }, + } +} diff --git a/cmd/lotus-storage-miner/backup.go b/cmd/lotus-storage-miner/backup.go index 55604b6fb..cf8c9f912 100644 --- a/cmd/lotus-storage-miner/backup.go +++ b/cmd/lotus-storage-miner/backup.go @@ -1,115 +1,14 @@ package main import ( - "fmt" - "os" - - logging "github.com/ipfs/go-log/v2" - "github.com/mitchellh/go-homedir" "github.com/urfave/cli/v2" - "golang.org/x/xerrors" + + "github.com/filecoin-project/go-jsonrpc" lcli "github.com/filecoin-project/lotus/cli" - "github.com/filecoin-project/lotus/lib/backupds" "github.com/filecoin-project/lotus/node/repo" ) -var backupCmd = &cli.Command{ - Name: "backup", - Usage: "Create node metadata backup", - Description: `The backup command writes a copy of node metadata under the specified path - -Online backups: -For security reasons, the daemon must be have LOTUS_BACKUP_BASE_PATH env var set -to a path where backup files are supposed to be saved, and the path specified in -this command must be within this base path`, - Flags: []cli.Flag{ - &cli.BoolFlag{ - Name: "offline", - Usage: "create backup without the node running", - }, - }, - ArgsUsage: "[backup file path]", - Action: func(cctx *cli.Context) error { - if cctx.Args().Len() != 1 { - return xerrors.Errorf("expected 1 argument") - } - - if cctx.Bool("offline") { - return offlineBackup(cctx) - } - - return onlineBackup(cctx) - }, -} - -func offlineBackup(cctx *cli.Context) error { - logging.SetLogLevel("badger", "ERROR") // nolint:errcheck - - repoPath := cctx.String(FlagMinerRepo) - r, err := repo.NewFS(repoPath) - if err != nil { - return err - } - - ok, err := r.Exists() - if err != nil { - return err - } - if !ok { - return xerrors.Errorf("repo at '%s' is not initialized", cctx.String(FlagMinerRepo)) - } - - lr, err := r.LockRO(repo.StorageMiner) - if err != nil { - return xerrors.Errorf("locking repo: %w", err) - } - defer lr.Close() // nolint:errcheck - - mds, err := lr.Datastore("/metadata") - if err != nil { - return xerrors.Errorf("getting metadata datastore: %w", err) - } - - bds := backupds.Wrap(mds) - - fpath, err := homedir.Expand(cctx.Args().First()) - if err != nil { - return xerrors.Errorf("expanding file path: %w", err) - } - - out, err := os.OpenFile(fpath, os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - return xerrors.Errorf("opening backup file %s: %w", fpath, err) - } - - if err := bds.Backup(out); err != nil { - if cerr := out.Close(); cerr != nil { - log.Errorw("error closing backup file while handling backup error", "closeErr", cerr, "backupErr", err) - } - return xerrors.Errorf("backup error: %w", err) - } - - if err := out.Close(); err != nil { - return xerrors.Errorf("closing backup file: %w", err) - } - - return nil -} - -func onlineBackup(cctx *cli.Context) error { - api, closer, err := lcli.GetStorageMinerAPI(cctx) - if err != nil { - return xerrors.Errorf("getting api: %w (if the node isn't running you can use the --offline flag)", err) - } - defer closer() - - err = api.CreateBackup(lcli.ReqContext(cctx), cctx.Args().First()) - if err != nil { - return err - } - - fmt.Println("Success") - - return nil -} +var backupCmd = lcli.BackupCmd(FlagMinerRepo, repo.StorageMiner, func(cctx *cli.Context) (lcli.BackupAPI, jsonrpc.ClientCloser, error) { + return lcli.GetStorageMinerAPI(cctx) +}) diff --git a/cmd/lotus/backup.go b/cmd/lotus/backup.go new file mode 100644 index 000000000..aec0000c9 --- /dev/null +++ b/cmd/lotus/backup.go @@ -0,0 +1,14 @@ +package main + +import ( + "github.com/urfave/cli/v2" + + "github.com/filecoin-project/go-jsonrpc" + + lcli "github.com/filecoin-project/lotus/cli" + "github.com/filecoin-project/lotus/node/repo" +) + +var backupCmd = lcli.BackupCmd("repo", repo.FullNode, func(cctx *cli.Context) (lcli.BackupAPI, jsonrpc.ClientCloser, error) { + return lcli.GetFullNodeAPI(cctx) +}) diff --git a/cmd/lotus/main.go b/cmd/lotus/main.go index ee95ad64b..eb97045ee 100644 --- a/cmd/lotus/main.go +++ b/cmd/lotus/main.go @@ -22,6 +22,7 @@ func main() { local := []*cli.Command{ DaemonCmd, + backupCmd, } if AdvanceBlockCmd != nil { local = append(local, AdvanceBlockCmd) diff --git a/node/impl/backup.go b/node/impl/backup.go new file mode 100644 index 000000000..10f673a4b --- /dev/null +++ b/node/impl/backup.go @@ -0,0 +1,67 @@ +package impl + +import ( + "os" + "path/filepath" + "strings" + + "github.com/mitchellh/go-homedir" + "golang.org/x/xerrors" + + "github.com/filecoin-project/lotus/lib/backupds" + "github.com/filecoin-project/lotus/node/modules/dtypes" +) + +func backup(mds dtypes.MetadataDS, fpath string) error { + bb, ok := os.LookupEnv("LOTUS_BACKUP_BASE_PATH") + if !ok { + return xerrors.Errorf("LOTUS_BACKUP_BASE_PATH env var not set") + } + + bds, ok := mds.(*backupds.Datastore) + if !ok { + return xerrors.Errorf("expected a backup datastore") + } + + bb, err := homedir.Expand(bb) + if err != nil { + return xerrors.Errorf("expanding base path: %w", err) + } + + bb, err = filepath.Abs(bb) + if err != nil { + return xerrors.Errorf("getting absolute base path: %w", err) + } + + fpath, err = homedir.Expand(fpath) + if err != nil { + return xerrors.Errorf("expanding file path: %w", err) + } + + fpath, err = filepath.Abs(fpath) + if err != nil { + return xerrors.Errorf("getting absolute file path: %w", err) + } + + if !strings.HasPrefix(fpath, bb) { + return xerrors.Errorf("backup file name (%s) must be inside base path (%s)", fpath, bb) + } + + out, err := os.OpenFile(fpath, os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return xerrors.Errorf("open %s: %w", fpath, err) + } + + if err := bds.Backup(out); err != nil { + if cerr := out.Close(); cerr != nil { + log.Errorw("error closing backup file while handling backup error", "closeErr", cerr, "backupErr", err) + } + return xerrors.Errorf("backup error: %w", err) + } + + if err := out.Close(); err != nil { + return xerrors.Errorf("closing backup file: %w", err) + } + + return nil +} diff --git a/node/impl/full.go b/node/impl/full.go index feeb29745..add40917c 100644 --- a/node/impl/full.go +++ b/node/impl/full.go @@ -1,6 +1,8 @@ package impl import ( + "context" + logging "github.com/ipfs/go-log/v2" "github.com/filecoin-project/lotus/api" @@ -9,6 +11,7 @@ import ( "github.com/filecoin-project/lotus/node/impl/full" "github.com/filecoin-project/lotus/node/impl/market" "github.com/filecoin-project/lotus/node/impl/paych" + "github.com/filecoin-project/lotus/node/modules/dtypes" ) var log = logging.Logger("node") @@ -26,6 +29,12 @@ type FullNodeAPI struct { full.WalletAPI full.SyncAPI full.BeaconAPI + + DS dtypes.MetadataDS +} + +func (n *FullNodeAPI) CreateBackup(ctx context.Context, fpath string) error { + return backup(n.DS, fpath) } var _ api.FullNode = &FullNodeAPI{} diff --git a/node/impl/storminer.go b/node/impl/storminer.go index c0cbe0081..6090e8a58 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -5,14 +5,11 @@ import ( "encoding/json" "net/http" "os" - "path/filepath" "strconv" - "strings" "time" "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p-core/host" - "github.com/mitchellh/go-homedir" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" @@ -34,11 +31,9 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/apistruct" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/backupds" "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node/impl/common" "github.com/filecoin-project/lotus/node/modules/dtypes" - "github.com/filecoin-project/lotus/node/repo" "github.com/filecoin-project/lotus/storage" "github.com/filecoin-project/lotus/storage/sectorblocks" ) @@ -61,8 +56,7 @@ type StorageMinerAPI struct { DataTransfer dtypes.ProviderDataTransfer Host host.Host - DS dtypes.MetadataDS - Repo repo.LockedRepo + DS dtypes.MetadataDS ConsiderOnlineStorageDealsConfigFunc dtypes.ConsiderOnlineStorageDealsConfigFunc SetConsiderOnlineStorageDealsConfigFunc dtypes.SetConsiderOnlineStorageDealsConfigFunc @@ -525,58 +519,7 @@ func (sm *StorageMinerAPI) PiecesGetCIDInfo(ctx context.Context, payloadCid cid. } func (sm *StorageMinerAPI) CreateBackup(ctx context.Context, fpath string) error { - // TODO: Config - bb, ok := os.LookupEnv("LOTUS_BACKUP_BASE_PATH") - if !ok { - return xerrors.Errorf("LOTUS_BACKUP_BASE_PATH env var not set") - } - - bds, ok := sm.DS.(*backupds.Datastore) - if !ok { - return xerrors.Errorf("expected a backup datastore") - } - - bb, err := homedir.Expand(bb) - if err != nil { - return xerrors.Errorf("expanding base path: %w", err) - } - - bb, err = filepath.Abs(bb) - if err != nil { - return xerrors.Errorf("getting absolute base path: %w", err) - } - - fpath, err = homedir.Expand(fpath) - if err != nil { - return xerrors.Errorf("expanding file path: %w", err) - } - - fpath, err = filepath.Abs(fpath) - if err != nil { - return xerrors.Errorf("getting absolute file path: %w", err) - } - - if !strings.HasPrefix(fpath, bb) { - return xerrors.Errorf("backup file name (%s) must be inside base path (%s)", fpath, bb) - } - - out, err := os.OpenFile(fpath, os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - return xerrors.Errorf("open %s: %w", fpath, err) - } - - if err := bds.Backup(out); err != nil { - if cerr := out.Close(); cerr != nil { - log.Errorw("error closing backup file while handling backup error", "closeErr", cerr, "backupErr", err) - } - return xerrors.Errorf("backup error: %w", err) - } - - if err := out.Close(); err != nil { - return xerrors.Errorf("closing backup file: %w", err) - } - - return nil + return backup(sm.DS, fpath) } var _ api.StorageMiner = &StorageMinerAPI{} From 738efe3f9a53f6231fb36f0c09adf53deb7c4a4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 1 Oct 2020 17:18:34 +0200 Subject: [PATCH 494/885] miner restore: Add progerss bar for metadata import --- cmd/lotus-storage-miner/init_restore.go | 32 +++++++++++++++++++++---- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/cmd/lotus-storage-miner/init_restore.go b/cmd/lotus-storage-miner/init_restore.go index 560eae33b..cfb13be31 100644 --- a/cmd/lotus-storage-miner/init_restore.go +++ b/cmd/lotus-storage-miner/init_restore.go @@ -2,17 +2,16 @@ package main import ( "encoding/json" - "github.com/filecoin-project/lotus/extern/sector-storage/stores" - "github.com/filecoin-project/lotus/node/config" - "github.com/mitchellh/go-homedir" "io/ioutil" "os" "github.com/docker/go-units" "github.com/ipfs/go-datastore" "github.com/libp2p/go-libp2p-core/peer" + "github.com/mitchellh/go-homedir" "github.com/urfave/cli/v2" "golang.org/x/xerrors" + "gopkg.in/cheggaaa/pb.v1" "github.com/filecoin-project/go-address" paramfetch "github.com/filecoin-project/go-paramfetch" @@ -21,7 +20,9 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" + "github.com/filecoin-project/lotus/extern/sector-storage/stores" "github.com/filecoin-project/lotus/lib/backupds" + "github.com/filecoin-project/lotus/node/config" "github.com/filecoin-project/lotus/node/repo" ) @@ -76,7 +77,17 @@ var initRestoreCmd = &cli.Command{ } } - f, err := os.Open(cctx.Args().First()) + bf, err := homedir.Expand(cctx.Args().First()) + if err != nil { + return xerrors.Errorf("expand backup file path: %w", err) + } + + st, err := os.Stat(bf) + if err != nil { + return xerrors.Errorf("stat backup file (%s): %w", bf, err) + } + + f, err := os.Open(bf) if err != nil { return xerrors.Errorf("opening backup file: %w", err) } @@ -184,7 +195,18 @@ var initRestoreCmd = &cli.Command{ return err } - if err := backupds.RestoreInto(f, mds); err != nil { + bar := pb.New64(st.Size()) + br := bar.NewProxyReader(f) + bar.ShowTimeLeft = true + bar.ShowPercent = true + bar.ShowSpeed = true + bar.Units = pb.U_BYTES + + bar.Start() + err = backupds.RestoreInto(br, mds) + bar.Finish() + + if err != nil { return xerrors.Errorf("restoring metadata: %w", err) } From ec7282ff090107f105285e99aec78e8532d25f83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 1 Oct 2020 17:51:56 +0200 Subject: [PATCH 495/885] miner restore: Resolve worker key for wallet check --- cmd/lotus-storage-miner/init_restore.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cmd/lotus-storage-miner/init_restore.go b/cmd/lotus-storage-miner/init_restore.go index cfb13be31..bdbb99fe0 100644 --- a/cmd/lotus-storage-miner/init_restore.go +++ b/cmd/lotus-storage-miner/init_restore.go @@ -231,7 +231,12 @@ var initRestoreCmd = &cli.Command{ log.Info("SECTOR SIZE: ", units.BytesSize(float64(mi.SectorSize))) - has, err := api.WalletHas(ctx, mi.Worker) + wk, err := api.StateAccountKey(ctx, mi.Worker, types.EmptyTSK) + if err != nil { + return xerrors.Errorf("resolving worker key: %w", err) + } + + has, err := api.WalletHas(ctx, wk) if err != nil { return xerrors.Errorf("checking worker address: %w", err) } From 15885ad535e7330fd8ad6ae8def9a5a8b5c8480f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 1 Oct 2020 18:00:24 +0200 Subject: [PATCH 496/885] docsgen --- documentation/en/api-methods.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index 2b28816f7..f74a975b0 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -50,6 +50,8 @@ * [ClientRetrieveTryRestartInsufficientFunds](#ClientRetrieveTryRestartInsufficientFunds) * [ClientRetrieveWithEvents](#ClientRetrieveWithEvents) * [ClientStartDeal](#ClientStartDeal) +* [Create](#Create) + * [CreateBackup](#CreateBackup) * [Gas](#Gas) * [GasEstimateFeeCap](#GasEstimateFeeCap) * [GasEstimateGasLimit](#GasEstimateGasLimit) @@ -1278,6 +1280,27 @@ Inputs: Response: `null` +## Create + + +### CreateBackup +CreateBackup creates node backup onder the specified file name. The +method requires that the lotus daemon is running with the +LOTUS_BACKUP_BASE_PATH environment variable set to some path, and that +the path specified when calling CreateBackup is within the base path + + +Perms: admin + +Inputs: +```json +[ + "string value" +] +``` + +Response: `{}` + ## Gas From 9b4274638c55ee80a69520332325d91dceed5785 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 1 Oct 2020 20:51:36 +0200 Subject: [PATCH 497/885] pcr: fix build --- cmd/lotus-pcr/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/lotus-pcr/main.go b/cmd/lotus-pcr/main.go index 1545ec224..70568a17a 100644 --- a/cmd/lotus-pcr/main.go +++ b/cmd/lotus-pcr/main.go @@ -564,10 +564,10 @@ type refunderNodeApi interface { ChainGetTipSetByHeight(ctx context.Context, epoch abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) ChainReadObj(context.Context, cid.Cid) ([]byte, error) StateMinerInitialPledgeCollateral(ctx context.Context, addr address.Address, precommitInfo miner.SectorPreCommitInfo, tsk types.TipSetKey) (types.BigInt, error) - StateMinerInfo(context.Context, address.Address, types.TipSetKey) (api.MinerInfo, error) + StateMinerInfo(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error) StateSectorPreCommitInfo(ctx context.Context, addr address.Address, sector abi.SectorNumber, tsk types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) StateMinerAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) - StateMinerSectors(ctx context.Context, addr address.Address, filter *bitfield.BitField, filterOut bool, tsk types.TipSetKey) ([]*api.ChainSectorInfo, error) + StateMinerSectors(ctx context.Context, addr address.Address, filter *bitfield.BitField, tsk types.TipSetKey) ([]*miner.SectorOnChainInfo, error) StateMinerFaults(ctx context.Context, addr address.Address, tsk types.TipSetKey) (bitfield.BitField, error) StateListMiners(context.Context, types.TipSetKey) ([]address.Address, error) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) From 078044153f598e486b37c9044f52fc376e15e2cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 1 Oct 2020 21:07:39 +0200 Subject: [PATCH 498/885] pcr: Appease the linter --- cmd/lotus-pcr/main.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cmd/lotus-pcr/main.go b/cmd/lotus-pcr/main.go index 70568a17a..4ce5bbb9f 100644 --- a/cmd/lotus-pcr/main.go +++ b/cmd/lotus-pcr/main.go @@ -615,6 +615,10 @@ func (r *refunder) FindMiners(ctx context.Context, tipset *types.TipSet, refunds // Look up and find all addresses associated with the miner minerInfo, err := r.api.StateMinerInfo(ctx, maddr, tipset.Key()) + if err != nil { + log.Errorw("failed", "err", err, "height", tipset.Height(), "key", tipset.Key(), "miner", maddr) + continue + } allAddresses := []address.Address{} @@ -673,7 +677,7 @@ func (r *refunder) EnsureMinerMinimums(ctx context.Context, tipset *types.TipSet return nil, err } - defer f.Close() + defer f.Close() // nolint:errcheck w = bufio.NewWriter(f) } @@ -703,6 +707,10 @@ func (r *refunder) EnsureMinerMinimums(ctx context.Context, tipset *types.TipSet // Look up and find all addresses associated with the miner minerInfo, err := r.api.StateMinerInfo(ctx, maddr, tipset.Key()) + if err != nil { + log.Errorw("failed", "err", err, "height", tipset.Height(), "key", tipset.Key(), "miner", maddr) + continue + } allAddresses := []address.Address{minerInfo.Worker, minerInfo.Owner} allAddresses = append(allAddresses, minerInfo.ControlAddresses...) From 1cc0f74744fd256dfc972d3f7518e95d55082d15 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 1 Oct 2020 15:02:40 -0700 Subject: [PATCH 499/885] make the upgrade schedule fully configurable This should help with testing. --- chain/stmgr/forks.go | 23 ++++++++++---- chain/stmgr/forks_test.go | 62 ++++++++++++++++++------------------ chain/stmgr/options.go | 23 ++++++++++++++ chain/stmgr/stmgr.go | 67 +++++++++++++++++++++++---------------- go.mod | 2 +- go.sum | 2 ++ 6 files changed, 115 insertions(+), 64 deletions(-) create mode 100644 chain/stmgr/options.go diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index 7d1555114..807f5666c 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -9,6 +9,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/go-state-types/network" "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" "golang.org/x/xerrors" @@ -33,17 +34,27 @@ import ( "github.com/filecoin-project/lotus/chain/vm" ) -var ForksAtHeight = map[abi.ChainEpoch]func(context.Context, *StateManager, ExecCallback, cid.Cid, *types.TipSet) (cid.Cid, error){ - build.UpgradeBreezeHeight: UpgradeFaucetBurnRecovery, - build.UpgradeIgnitionHeight: UpgradeIgnition, - build.UpgradeActorsV2Height: UpgradeActorsV2, - build.UpgradeLiftoffHeight: UpgradeLiftoff, +type UpgradeFunc func(context.Context, *StateManager, ExecCallback, cid.Cid, *types.TipSet) (cid.Cid, error) + +type Upgrade struct { + Height abi.ChainEpoch + Migration UpgradeFunc +} + +type UpgradeSchedule map[network.Version]Upgrade + +var DefaultUpgradeSchedule = UpgradeSchedule{ + network.Version1: {build.UpgradeBreezeHeight, UpgradeFaucetBurnRecovery}, + network.Version2: {build.UpgradeSmokeHeight, nil}, + network.Version3: {build.UpgradeIgnitionHeight, UpgradeIgnition}, + network.Version4: {build.UpgradeActorsV2Height, UpgradeActorsV2}, + network.Version5: {build.UpgradeLiftoffHeight, UpgradeLiftoff}, } func (sm *StateManager) handleStateForks(ctx context.Context, root cid.Cid, height abi.ChainEpoch, cb ExecCallback, ts *types.TipSet) (cid.Cid, error) { retCid := root var err error - f, ok := ForksAtHeight[height] + f, ok := sm.stateMigrations[height] if ok { retCid, err = f(ctx, sm, cb, root, ts) if err != nil { diff --git a/chain/stmgr/forks_test.go b/chain/stmgr/forks_test.go index 9db6a491a..1b794f31c 100644 --- a/chain/stmgr/forks_test.go +++ b/chain/stmgr/forks_test.go @@ -19,7 +19,6 @@ import ( lotusinit "github.com/filecoin-project/lotus/chain/actors/builtin/init" "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/gen" - "github.com/filecoin-project/lotus/chain/stmgr" . "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" @@ -110,51 +109,54 @@ func TestForkHeightTriggers(t *testing.T) { t.Fatal(err) } - sm := NewStateManager(cg.ChainStore()) - - inv := vm.NewActorRegistry() - // predicting the address here... may break if other assumptions change taddr, err := address.NewIDAddress(1002) if err != nil { t.Fatal(err) } - stmgr.ForksAtHeight[testForkHeight] = func(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, ts *types.TipSet) (cid.Cid, error) { - cst := ipldcbor.NewCborStore(sm.ChainStore().Blockstore()) + sm := NewStateManager(cg.ChainStore(), + WithUpgradeSchedule(UpgradeSchedule{ + 1: {Height: testForkHeight, Migration: func(ctx context.Context, sm *StateManager, cb ExecCallback, + root cid.Cid, ts *types.TipSet) (cid.Cid, error) { + cst := ipldcbor.NewCborStore(sm.ChainStore().Blockstore()) - st, err := sm.StateTree(root) - if err != nil { - return cid.Undef, xerrors.Errorf("getting state tree: %w", err) - } + st, err := sm.StateTree(root) + if err != nil { + return cid.Undef, xerrors.Errorf("getting state tree: %w", err) + } - act, err := st.GetActor(taddr) - if err != nil { - return cid.Undef, err - } + act, err := st.GetActor(taddr) + if err != nil { + return cid.Undef, err + } - var tas testActorState - if err := cst.Get(ctx, act.Head, &tas); err != nil { - return cid.Undef, xerrors.Errorf("in fork handler, failed to run get: %w", err) - } + var tas testActorState + if err := cst.Get(ctx, act.Head, &tas); err != nil { + return cid.Undef, xerrors.Errorf("in fork handler, failed to run get: %w", err) + } - tas.HasUpgraded = 55 + tas.HasUpgraded = 55 - ns, err := cst.Put(ctx, &tas) - if err != nil { - return cid.Undef, err - } + ns, err := cst.Put(ctx, &tas) + if err != nil { + return cid.Undef, err + } - act.Head = ns + act.Head = ns - if err := st.SetActor(taddr, act); err != nil { - return cid.Undef, err - } + if err := st.SetActor(taddr, act); err != nil { + return cid.Undef, err + } - return st.Flush(ctx) - } + return st.Flush(ctx) + }}, + }), + ) + inv := vm.NewActorRegistry() inv.Register(nil, testActor{}) + sm.SetVMConstructor(func(ctx context.Context, vmopt *vm.VMOpts) (*vm.VM, error) { nvm, err := vm.NewVM(ctx, vmopt) if err != nil { diff --git a/chain/stmgr/options.go b/chain/stmgr/options.go new file mode 100644 index 000000000..8c63149a1 --- /dev/null +++ b/chain/stmgr/options.go @@ -0,0 +1,23 @@ +package stmgr + +type Option func(*config) + +type config struct { + upgradeSchedule UpgradeSchedule +} + +func parseOptions(opts ...Option) *config { + cfg := &config{ + upgradeSchedule: DefaultUpgradeSchedule, + } + for _, opt := range opts { + opt(cfg) + } + return cfg +} + +func WithUpgradeSchedule(schedule UpgradeSchedule) Option { + return func(cfg *config) { + cfg.upgradeSchedule = schedule + } +} diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index e7947cac3..8560690ea 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -42,6 +42,12 @@ var log = logging.Logger("statemgr") type StateManager struct { cs *store.ChainStore + // Sorted network upgrade epochs (starting at version 1). + // -1 means the version is skipped. + networkVersions []abi.ChainEpoch + // Maps chain epochs to upgrade functions. + stateMigrations map[abi.ChainEpoch]UpgradeFunc + stCache map[string][]cid.Cid compWait map[string]chan struct{} stlk sync.Mutex @@ -51,12 +57,36 @@ type StateManager struct { postIgnitionGenInfos *genesisInfo } -func NewStateManager(cs *store.ChainStore) *StateManager { +func NewStateManager(cs *store.ChainStore, options ...Option) *StateManager { + cfg := parseOptions(options...) + stateMigrations := make(map[abi.ChainEpoch]UpgradeFunc, len(cfg.upgradeSchedule)) + networkVersions := make([]abi.ChainEpoch, 0, len(stateMigrations)) + + // Iterate version by version, to make sure we handle skipped version numbers. + // Always skip version 0. + for i, version := 0, network.Version(1); i < len(cfg.upgradeSchedule); version++ { + upgrade, ok := cfg.upgradeSchedule[version] + if ok { + // We've processed an upgrade. + i++ + } + + epoch := abi.ChainEpoch(-1) + if ok && upgrade.Height >= 0 { + epoch = upgrade.Height + stateMigrations[epoch] = upgrade.Migration + } + + networkVersions = append(networkVersions, epoch) + } + return &StateManager{ - newVM: vm.NewVM, - cs: cs, - stCache: make(map[string][]cid.Cid), - compWait: make(map[string]chan struct{}), + networkVersions: networkVersions, + stateMigrations: stateMigrations, + newVM: vm.NewVM, + cs: cs, + stCache: make(map[string][]cid.Cid), + compWait: make(map[string]chan struct{}), } } @@ -1259,29 +1289,12 @@ func (sm *StateManager) GetCirculatingSupply(ctx context.Context, height abi.Cha } func (sm *StateManager) GetNtwkVersion(ctx context.Context, height abi.ChainEpoch) network.Version { - // TODO: move hard fork epoch checks to a schedule defined in build/ - - if build.UseNewestNetwork() { - return build.NewestNetworkVersion + for v, epoch := range sm.networkVersions { + if epoch >= 0 && height <= epoch { + return network.Version(v + 1) // we've skipped version 0 + } } - - if height <= build.UpgradeBreezeHeight { - return network.Version0 - } - - if height <= build.UpgradeSmokeHeight { - return network.Version1 - } - - if height <= build.UpgradeIgnitionHeight { - return network.Version2 - } - - if height <= build.UpgradeActorsV2Height { - return network.Version3 - } - - return build.NewestNetworkVersion + return network.Version(len(sm.networkVersions)) } func (sm *StateManager) GetPaychState(ctx context.Context, addr address.Address, ts *types.TipSet) (*types.Actor, paych.State, error) { diff --git a/go.mod b/go.mod index ea33b810e..50e5aecc1 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( github.com/filecoin-project/go-multistore v0.0.3 github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 github.com/filecoin-project/go-paramfetch v0.0.2-0.20200701152213-3e0f0afdc261 - github.com/filecoin-project/go-state-types v0.0.0-20200928172055-2df22083d8ab + github.com/filecoin-project/go-state-types v0.0.0-20201001162932-93a412d8e9fa github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b diff --git a/go.sum b/go.sum index 6b3b00907..aa8e3f633 100644 --- a/go.sum +++ b/go.sum @@ -263,6 +263,8 @@ github.com/filecoin-project/go-state-types v0.0.0-20200904021452-1883f36ca2f4/go github.com/filecoin-project/go-state-types v0.0.0-20200905071437-95828685f9df/go.mod h1:IQ0MBPnonv35CJHtWSN3YY1Hz2gkPru1Q9qoaYLxx9I= github.com/filecoin-project/go-state-types v0.0.0-20200928172055-2df22083d8ab h1:cEDC5Ei8UuT99hPWhCjA72SM9AuRtnpvdSTIYbnzN8I= github.com/filecoin-project/go-state-types v0.0.0-20200928172055-2df22083d8ab/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= +github.com/filecoin-project/go-state-types v0.0.0-20201001162932-93a412d8e9fa h1:WVxLrmc6w5naeX3QikfRvc1G7EFfBZbhXuzM8Lj2J+o= +github.com/filecoin-project/go-state-types v0.0.0-20201001162932-93a412d8e9fa/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-statemachine v0.0.0-20200714194326-a77c3ae20989/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe h1:dF8u+LEWeIcTcfUcCf3WFVlc81Fr2JKg8zPzIbBDKDw= github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= From e865ba0c645ada6b026cdc2263b24461befc32bd Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 1 Oct 2020 15:18:59 -0700 Subject: [PATCH 500/885] improve upgrade schedule configurable Unfortunately, it looks like functional ops cannot be passed in via lotus's di system. --- chain/stmgr/forks_test.go | 8 ++++---- chain/stmgr/options.go | 23 ----------------------- chain/stmgr/stmgr.go | 13 ++++++++----- node/builder.go | 3 ++- 4 files changed, 14 insertions(+), 33 deletions(-) delete mode 100644 chain/stmgr/options.go diff --git a/chain/stmgr/forks_test.go b/chain/stmgr/forks_test.go index 1b794f31c..db65d2485 100644 --- a/chain/stmgr/forks_test.go +++ b/chain/stmgr/forks_test.go @@ -115,8 +115,9 @@ func TestForkHeightTriggers(t *testing.T) { t.Fatal(err) } - sm := NewStateManager(cg.ChainStore(), - WithUpgradeSchedule(UpgradeSchedule{ + sm := NewStateManagerWithUpgradeSchedule( + cg.ChainStore(), + UpgradeSchedule{ 1: {Height: testForkHeight, Migration: func(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, ts *types.TipSet) (cid.Cid, error) { cst := ipldcbor.NewCborStore(sm.ChainStore().Blockstore()) @@ -151,8 +152,7 @@ func TestForkHeightTriggers(t *testing.T) { return st.Flush(ctx) }}, - }), - ) + }) inv := vm.NewActorRegistry() inv.Register(nil, testActor{}) diff --git a/chain/stmgr/options.go b/chain/stmgr/options.go deleted file mode 100644 index 8c63149a1..000000000 --- a/chain/stmgr/options.go +++ /dev/null @@ -1,23 +0,0 @@ -package stmgr - -type Option func(*config) - -type config struct { - upgradeSchedule UpgradeSchedule -} - -func parseOptions(opts ...Option) *config { - cfg := &config{ - upgradeSchedule: DefaultUpgradeSchedule, - } - for _, opt := range opts { - opt(cfg) - } - return cfg -} - -func WithUpgradeSchedule(schedule UpgradeSchedule) Option { - return func(cfg *config) { - cfg.upgradeSchedule = schedule - } -} diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index 8560690ea..966049693 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -57,15 +57,18 @@ type StateManager struct { postIgnitionGenInfos *genesisInfo } -func NewStateManager(cs *store.ChainStore, options ...Option) *StateManager { - cfg := parseOptions(options...) - stateMigrations := make(map[abi.ChainEpoch]UpgradeFunc, len(cfg.upgradeSchedule)) +func NewStateManager(cs *store.ChainStore) *StateManager { + return NewStateManagerWithUpgradeSchedule(cs, DefaultUpgradeSchedule) +} + +func NewStateManagerWithUpgradeSchedule(cs *store.ChainStore, us UpgradeSchedule) *StateManager { + stateMigrations := make(map[abi.ChainEpoch]UpgradeFunc, len(us)) networkVersions := make([]abi.ChainEpoch, 0, len(stateMigrations)) // Iterate version by version, to make sure we handle skipped version numbers. // Always skip version 0. - for i, version := 0, network.Version(1); i < len(cfg.upgradeSchedule); version++ { - upgrade, ok := cfg.upgradeSchedule[version] + for i, version := 0, network.Version(1); i < len(us); version++ { + upgrade, ok := us[version] if ok { // We've processed an upgrade. i++ diff --git a/node/builder.go b/node/builder.go index 8512d0a0c..c738740ec 100644 --- a/node/builder.go +++ b/node/builder.go @@ -259,7 +259,8 @@ func Online() Option { Override(new(ffiwrapper.Verifier), ffiwrapper.ProofVerifier), Override(new(vm.SyscallBuilder), vm.Syscalls), Override(new(*store.ChainStore), modules.ChainStore), - Override(new(*stmgr.StateManager), stmgr.NewStateManager), + Override(new(stmgr.UpgradeSchedule), stmgr.DefaultUpgradeSchedule), + Override(new(*stmgr.StateManager), stmgr.NewStateManagerWithUpgradeSchedule), Override(new(*wallet.Wallet), wallet.NewWallet), Override(new(*messagesigner.MessageSigner), messagesigner.NewMessageSigner), From 8ad8a945fb688c0968af8fcb3c1e3d94b1615a76 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 1 Oct 2020 15:26:00 -0700 Subject: [PATCH 501/885] make it possible to override options when testing --- api/test/test.go | 3 ++- node/test/builder.go | 18 ++++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/api/test/test.go b/api/test/test.go index 409274ff1..853267eff 100644 --- a/api/test/test.go +++ b/api/test/test.go @@ -12,6 +12,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/miner" + "github.com/filecoin-project/lotus/node" ) type TestNode struct { @@ -44,7 +45,7 @@ type StorageMiner struct { // // storage array defines storage nodes, numbers in the array specify full node // index the storage node 'belongs' to -type APIBuilder func(t *testing.T, nFull int, storage []StorageMiner) ([]TestNode, []TestStorageNode) +type APIBuilder func(t *testing.T, nFull int, storage []StorageMiner, opts ...node.Option) ([]TestNode, []TestStorageNode) type testSuite struct { makeNodes APIBuilder } diff --git a/node/test/builder.go b/node/test/builder.go index fb3aa71d4..a3455f376 100644 --- a/node/test/builder.go +++ b/node/test/builder.go @@ -137,7 +137,7 @@ func CreateTestStorageNode(ctx context.Context, t *testing.T, waddr address.Addr return test.TestStorageNode{StorageMiner: minerapi, MineOne: mineOne} } -func Builder(t *testing.T, nFull int, storage []test.StorageMiner) ([]test.TestNode, []test.TestStorageNode) { +func Builder(t *testing.T, nFull int, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { ctx := context.Background() mn := mocknet.New(ctx) @@ -224,6 +224,7 @@ func Builder(t *testing.T, nFull int, storage []test.StorageMiner) ([]test.TestN node.Test(), genesis, + node.Options(opts...), ) if err != nil { t.Fatal(err) @@ -285,7 +286,7 @@ func Builder(t *testing.T, nFull int, storage []test.StorageMiner) ([]test.TestN return fulls, storers } -func MockSbBuilder(t *testing.T, nFull int, storage []test.StorageMiner) ([]test.TestNode, []test.TestStorageNode) { +func MockSbBuilder(t *testing.T, nFull int, storage []test.StorageMiner, options ...node.Option) ([]test.TestNode, []test.TestStorageNode) { ctx := context.Background() mn := mocknet.New(ctx) @@ -373,6 +374,7 @@ func MockSbBuilder(t *testing.T, nFull int, storage []test.StorageMiner) ([]test node.Override(new(ffiwrapper.Verifier), mock.MockVerifier), genesis, + node.Options(options...), ) if err != nil { t.Fatalf("%+v", err) @@ -435,16 +437,16 @@ func MockSbBuilder(t *testing.T, nFull int, storage []test.StorageMiner) ([]test return fulls, storers } -func RPCBuilder(t *testing.T, nFull int, storage []test.StorageMiner) ([]test.TestNode, []test.TestStorageNode) { - return rpcWithBuilder(t, Builder, nFull, storage) +func RPCBuilder(t *testing.T, nFull int, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { + return rpcWithBuilder(t, Builder, nFull, storage, opts...) } -func RPCMockSbBuilder(t *testing.T, nFull int, storage []test.StorageMiner) ([]test.TestNode, []test.TestStorageNode) { - return rpcWithBuilder(t, MockSbBuilder, nFull, storage) +func RPCMockSbBuilder(t *testing.T, nFull int, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { + return rpcWithBuilder(t, MockSbBuilder, nFull, storage, opts...) } -func rpcWithBuilder(t *testing.T, b test.APIBuilder, nFull int, storage []test.StorageMiner) ([]test.TestNode, []test.TestStorageNode) { - fullApis, storaApis := b(t, nFull, storage) +func rpcWithBuilder(t *testing.T, b test.APIBuilder, nFull int, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { + fullApis, storaApis := b(t, nFull, storage, opts...) fulls := make([]test.TestNode, nFull) storers := make([]test.TestStorageNode, len(storage)) From 101d07f704c3f120829477219b9a7159ad546c55 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 1 Oct 2020 15:33:32 -0700 Subject: [PATCH 502/885] fix version off by one error --- chain/stmgr/stmgr.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index 966049693..3efe26f53 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -1292,9 +1292,12 @@ func (sm *StateManager) GetCirculatingSupply(ctx context.Context, height abi.Cha } func (sm *StateManager) GetNtwkVersion(ctx context.Context, height abi.ChainEpoch) network.Version { + // The epochs here are the _last_ epoch for every version, or -1 if the + // version is disabled. for v, epoch := range sm.networkVersions { if epoch >= 0 && height <= epoch { - return network.Version(v + 1) // we've skipped version 0 + // Use the _previous_ version + return network.Version(v) } } return network.Version(len(sm.networkVersions)) From ed86ac14a17f8d09d5a35e8f2311e2bef322ac79 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 1 Oct 2020 20:37:14 -0700 Subject: [PATCH 503/885] use upgrade epochs from upgrade schedule --- chain/stmgr/forks.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index 807f5666c..ccb6221d7 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -382,7 +382,9 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCal func UpgradeIgnition(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, ts *types.TipSet) (cid.Cid, error) { store := sm.cs.Store(ctx) - nst, err := nv3.MigrateStateTree(ctx, store, root, build.UpgradeIgnitionHeight) + epoch := ts.Height() - 1 + + nst, err := nv3.MigrateStateTree(ctx, store, root, epoch) if err != nil { return cid.Undef, xerrors.Errorf("migrating actors state: %w", err) } @@ -407,7 +409,7 @@ func UpgradeIgnition(ctx context.Context, sm *StateManager, cb ExecCallback, roo return cid.Undef, xerrors.Errorf("second split address: %w", err) } - err = resetGenesisMsigs(ctx, sm, store, tree) + err = resetGenesisMsigs(ctx, sm, store, tree, epoch) if err != nil { return cid.Undef, xerrors.Errorf("resetting genesis msig start epochs: %w", err) } @@ -422,7 +424,7 @@ func UpgradeIgnition(ctx context.Context, sm *StateManager, cb ExecCallback, roo return cid.Undef, xerrors.Errorf("splitting second msig: %w", err) } - err = nv3.CheckStateTree(ctx, store, nst, build.UpgradeIgnitionHeight, builtin0.TotalFilecoin) + err = nv3.CheckStateTree(ctx, store, nst, epoch, builtin0.TotalFilecoin) if err != nil { return cid.Undef, xerrors.Errorf("sanity check after ignition upgrade failed: %w", err) } @@ -433,12 +435,14 @@ func UpgradeIgnition(ctx context.Context, sm *StateManager, cb ExecCallback, roo func UpgradeActorsV2(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, ts *types.TipSet) (cid.Cid, error) { store := sm.cs.Store(ctx) + epoch := ts.Height() - 1 + info, err := store.Put(ctx, new(types.StateInfo)) if err != nil { return cid.Undef, xerrors.Errorf("failed to create new state info for actors v2: %w", err) } - newHamtRoot, err := m2.MigrateStateTree(ctx, store, root, build.UpgradeActorsV2Height, m2.DefaultConfig()) + newHamtRoot, err := m2.MigrateStateTree(ctx, store, root, epoch, m2.DefaultConfig()) if err != nil { return cid.Undef, xerrors.Errorf("upgrading to actors v2: %w", err) } @@ -638,7 +642,7 @@ func makeKeyAddr(splitAddr address.Address, count uint64) (address.Address, erro return addr, nil } -func resetGenesisMsigs(ctx context.Context, sm *StateManager, store adt0.Store, tree *state.StateTree) error { +func resetGenesisMsigs(ctx context.Context, sm *StateManager, store adt0.Store, tree *state.StateTree, epoch abi.ChainEpoch) error { gb, err := sm.cs.GetGenesis() if err != nil { return xerrors.Errorf("getting genesis block: %w", err) @@ -667,7 +671,7 @@ func resetGenesisMsigs(ctx context.Context, sm *StateManager, store adt0.Store, return xerrors.Errorf("reading multisig state: %w", err) } - currState.StartEpoch = build.UpgradeLiftoffHeight + currState.StartEpoch = epoch currActor.Head, err = store.Put(ctx, &currState) if err != nil { From 8fe8a5df454a3dff2ce06ce542760724dde21477 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 2 Oct 2020 18:34:50 +0200 Subject: [PATCH 504/885] Cap market provider messages --- markets/storageadapter/provider.go | 30 ++++++++++++++++++++---------- node/builder.go | 4 +++- node/config/def.go | 16 ++++++++++------ 3 files changed, 33 insertions(+), 17 deletions(-) diff --git a/markets/storageadapter/provider.go b/markets/storageadapter/provider.go index 9f610d76a..b1071adcd 100644 --- a/markets/storageadapter/provider.go +++ b/markets/storageadapter/provider.go @@ -25,15 +25,15 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/events" - "github.com/filecoin-project/lotus/chain/actors/builtin/market" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + "github.com/filecoin-project/lotus/chain/events" "github.com/filecoin-project/lotus/chain/events/state" "github.com/filecoin-project/lotus/chain/types" sealing "github.com/filecoin-project/lotus/extern/storage-sealing" "github.com/filecoin-project/lotus/lib/sigs" "github.com/filecoin-project/lotus/markets/utils" + "github.com/filecoin-project/lotus/node/config" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/storage/sectorblocks" ) @@ -50,14 +50,24 @@ type ProviderNodeAdapter struct { secb *sectorblocks.SectorBlocks ev *events.Events + + publishSpec, addBalanceSpec *api.MessageSendSpec } -func NewProviderNodeAdapter(dag dtypes.StagingDAG, secb *sectorblocks.SectorBlocks, full api.FullNode) storagemarket.StorageProviderNode { - return &ProviderNodeAdapter{ - FullNode: full, - dag: dag, - secb: secb, - ev: events.NewEvents(context.TODO(), full), +func NewProviderNodeAdapter(fc *config.MinerFeeConfig) func(dag dtypes.StagingDAG, secb *sectorblocks.SectorBlocks, full api.FullNode) storagemarket.StorageProviderNode { + return func(dag dtypes.StagingDAG, secb *sectorblocks.SectorBlocks, full api.FullNode) storagemarket.StorageProviderNode { + na := &ProviderNodeAdapter{ + FullNode: full, + + dag: dag, + secb: secb, + ev: events.NewEvents(context.TODO(), full), + } + if fc != nil { + na.publishSpec = &api.MessageSendSpec{MaxFee: abi.TokenAmount(fc.MaxPublishDealsFee)} + na.addBalanceSpec = &api.MessageSendSpec{MaxFee: abi.TokenAmount(fc.MaxMarketBalanceAddFee)} + } + return na } } @@ -84,7 +94,7 @@ func (n *ProviderNodeAdapter) PublishDeals(ctx context.Context, deal storagemark Value: types.NewInt(0), Method: builtin0.MethodsMarket.PublishStorageDeals, Params: params, - }, nil) + }, n.publishSpec) if err != nil { return cid.Undef, err } @@ -183,7 +193,7 @@ func (n *ProviderNodeAdapter) AddFunds(ctx context.Context, addr address.Address From: addr, Value: amount, Method: builtin0.MethodsMarket.AddBalance, - }, nil) + }, n.addBalanceSpec) if err != nil { return cid.Undef, err } diff --git a/node/builder.go b/node/builder.go index 8512d0a0c..41fb01ef7 100644 --- a/node/builder.go +++ b/node/builder.go @@ -344,7 +344,7 @@ func Online() Option { Override(new(dtypes.DealFilter), modules.BasicDealFilter(nil)), Override(new(modules.ProviderDealFunds), modules.NewProviderDealFunds), Override(new(storagemarket.StorageProvider), modules.StorageProvider), - Override(new(storagemarket.StorageProviderNode), storageadapter.NewProviderNodeAdapter), + Override(new(storagemarket.StorageProviderNode), storageadapter.NewProviderNodeAdapter(nil)), Override(HandleRetrievalKey, modules.HandleRetrieval), Override(GetParamsKey, modules.GetParams), Override(HandleDealsKey, modules.HandleDeals), @@ -463,6 +463,8 @@ func ConfigStorageMiner(c interface{}) Option { Override(new(dtypes.DealFilter), modules.BasicDealFilter(dealfilter.CliDealFilter(cfg.Dealmaking.Filter))), ), + Override(new(storagemarket.StorageProviderNode), storageadapter.NewProviderNodeAdapter(&cfg.Fees)), + Override(new(sectorstorage.SealerConfig), cfg.Storage), Override(new(*storage.Miner), modules.StorageMiner(cfg.Fees)), ) diff --git a/node/config/def.go b/node/config/def.go index 9fee8895a..420f8b990 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -61,9 +61,11 @@ type SealingConfig struct { } type MinerFeeConfig struct { - MaxPreCommitGasFee types.FIL - MaxCommitGasFee types.FIL - MaxWindowPoStGasFee types.FIL + MaxPreCommitGasFee types.FIL + MaxCommitGasFee types.FIL + MaxWindowPoStGasFee types.FIL + MaxPublishDealsFee types.FIL + MaxMarketBalanceAddFee types.FIL } // API contains configs for API endpoint @@ -173,9 +175,11 @@ func DefaultStorageMiner() *StorageMiner { }, Fees: MinerFeeConfig{ - MaxPreCommitGasFee: types.FIL(types.BigDiv(types.FromFil(1), types.NewInt(20))), // 0.05 - MaxCommitGasFee: types.FIL(types.BigDiv(types.FromFil(1), types.NewInt(20))), - MaxWindowPoStGasFee: types.FIL(types.FromFil(50)), + MaxPreCommitGasFee: types.FIL(types.BigDiv(types.FromFil(1), types.NewInt(20))), // 0.05 + MaxCommitGasFee: types.FIL(types.BigDiv(types.FromFil(1), types.NewInt(20))), + MaxWindowPoStGasFee: types.FIL(types.FromFil(50)), + MaxPublishDealsFee: types.FIL(types.BigDiv(types.FromFil(1), types.NewInt(33))), // 0.03ish + MaxMarketBalanceAddFee: types.FIL(types.BigDiv(types.FromFil(1), types.NewInt(100))), // 0.01 }, } cfg.Common.API.ListenAddress = "/ip4/127.0.0.1/tcp/2345/http" From e848c13ff1f25cc0bb610bfbae8c7c206b9ce570 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 2 Oct 2020 19:31:21 +0200 Subject: [PATCH 505/885] client: bump default deal start buffer --- node/impl/client/client.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 7cb087ec7..39f0ab19a 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -60,7 +60,7 @@ import ( var DefaultHashFunction = uint64(mh.BLAKE2B_MIN + 31) -const dealStartBufferHours uint64 = 24 +const dealStartBufferHours uint64 = 49 type API struct { fx.In @@ -154,7 +154,7 @@ func (a *API) ClientStartDeal(ctx context.Context, params *api.StartDealParams) } blocksPerHour := 60 * 60 / build.BlockDelaySecs - dealStart = ts.Height() + abi.ChainEpoch(dealStartBufferHours*blocksPerHour) + dealStart = ts.Height() + abi.ChainEpoch(dealStartBufferHours*blocksPerHour) // TODO: Get this from storage ask } result, err := a.SMDealClient.ProposeStorageDeal(ctx, storagemarket.ProposeStorageDealParams{ From 53cc1f18d020089fb079a8b3083f66daee3710df Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 2 Oct 2020 10:19:26 -0700 Subject: [PATCH 506/885] use bitswap sessions when fetching messages, and cancel them 1. Explicitly use a session when fetching messages for a block. Technically, GetBlocks would use an internal session so we'd only end up with two, but one per block is even better. 2. Actually cancel sessions after a timeout (threading through the context). NOTE: We should seriously consider having a single session for all blocks, but we can do that in a followup as it may have unintended consequences (e.g., leaks). --- chain/sub/incoming.go | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/chain/sub/incoming.go b/chain/sub/incoming.go index c6e0c8b80..0c72b4d80 100644 --- a/chain/sub/incoming.go +++ b/chain/sub/incoming.go @@ -43,7 +43,11 @@ var log = logging.Logger("sub") var ErrSoftFailure = errors.New("soft validation failure") var ErrInsufficientPower = errors.New("incoming block's miner does not have minimum power") -func HandleIncomingBlocks(ctx context.Context, bsub *pubsub.Subscription, s *chain.Syncer, bserv bserv.BlockService, cmgr connmgr.ConnManager) { +func HandleIncomingBlocks(ctx context.Context, bsub *pubsub.Subscription, s *chain.Syncer, bs bserv.BlockService, cmgr connmgr.ConnManager) { + // Timeout after (block time + propagation delay). This is useless at + // this point. + timeout := time.Duration(build.BlockDelaySecs+build.PropagationDelaySecs) * time.Second + for { msg, err := bsub.Next(ctx) if err != nil { @@ -64,15 +68,22 @@ func HandleIncomingBlocks(ctx context.Context, bsub *pubsub.Subscription, s *cha src := msg.GetFrom() go func() { + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + // NOTE: we could also share a single session between + // all requests but that may have other consequences. + ses := bserv.NewSession(ctx, bs) + start := build.Clock.Now() log.Debug("about to fetch messages for block from pubsub") - bmsgs, err := FetchMessagesByCids(context.TODO(), bserv, blk.BlsMessages) + bmsgs, err := FetchMessagesByCids(ctx, ses, blk.BlsMessages) if err != nil { log.Errorf("failed to fetch all bls messages for block received over pubusb: %s; source: %s", err, src) return } - smsgs, err := FetchSignedMessagesByCids(context.TODO(), bserv, blk.SecpkMessages) + smsgs, err := FetchSignedMessagesByCids(ctx, ses, blk.SecpkMessages) if err != nil { log.Errorf("failed to fetch all secpk messages for block received over pubusb: %s; source: %s", err, src) return @@ -97,7 +108,7 @@ func HandleIncomingBlocks(ctx context.Context, bsub *pubsub.Subscription, s *cha func FetchMessagesByCids( ctx context.Context, - bserv bserv.BlockService, + bserv bserv.BlockGetter, cids []cid.Cid, ) ([]*types.Message, error) { out := make([]*types.Message, len(cids)) @@ -126,7 +137,7 @@ func FetchMessagesByCids( // FIXME: Duplicate of above. func FetchSignedMessagesByCids( ctx context.Context, - bserv bserv.BlockService, + bserv bserv.BlockGetter, cids []cid.Cid, ) ([]*types.SignedMessage, error) { out := make([]*types.SignedMessage, len(cids)) @@ -156,12 +167,11 @@ func FetchSignedMessagesByCids( // blocks we did not request. func fetchCids( ctx context.Context, - bserv bserv.BlockService, + bserv bserv.BlockGetter, cids []cid.Cid, cb func(int, blocks.Block) error, ) error { - // FIXME: Why don't we use the context here? - fetchedBlocks := bserv.GetBlocks(context.TODO(), cids) + fetchedBlocks := bserv.GetBlocks(ctx, cids) cidIndex := make(map[cid.Cid]int) for i, c := range cids { From 2c1ef3708eeccf67dd0361f9d48d4aa27f8d10c2 Mon Sep 17 00:00:00 2001 From: Ingar Shu Date: Fri, 2 Oct 2020 12:16:59 -0700 Subject: [PATCH 507/885] Add verified flag to interactive deal creation --- cli/client.go | 54 ++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 3 deletions(-) diff --git a/cli/client.go b/cli/client.go index 255f5a70c..6ba143753 100644 --- a/cli/client.go +++ b/cli/client.go @@ -12,13 +12,12 @@ import ( "text/tabwriter" "time" - "github.com/filecoin-project/specs-actors/actors/builtin" - tm "github.com/buger/goterm" "github.com/docker/go-units" "github.com/fatih/color" datatransfer "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/go-fil-markets/retrievalmarket" + "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/ipfs/go-cid" "github.com/ipfs/go-cidutil/cidenc" "github.com/libp2p/go-libp2p-core/peer" @@ -476,6 +475,7 @@ func interactiveDeal(cctx *cli.Context) error { var ask storagemarket.StorageAsk var epochPrice big.Int var epochs abi.ChainEpoch + var verified bool var a address.Address if from := cctx.String("from"); from != "" { @@ -572,6 +572,53 @@ func interactiveDeal(cctx *cli.Context) error { ask = *a // TODO: run more validation + state = "verified" + case "verified": + ts, err := api.ChainHead(ctx) + if err != nil { + return err + } + + dcap, err := api.StateVerifiedClientStatus(ctx, a, ts.Key()) + if err != nil { + return err + } + + if dcap == nil { + state = "confirm" + continue + } + + color.Blue(".. checking verified deal eligibility\n") + ds, err := api.ClientDealSize(ctx, data) + if err != nil { + return err + } + + if dcap.Uint64() < uint64(ds.PieceSize) { + color.Yellow(".. not enough DataCap available for a verified deal\n") + state = "confirm" + continue + } + + fmt.Print("\nMake this a verified deal? (yes/no): ") + + var yn string + _, err = fmt.Scan(&yn) + if err != nil { + return err + } + + switch yn { + case "yes": + verified = true + case "no": + verified = false + default: + fmt.Println("Type in full 'yes' or 'no'") + continue + } + state = "confirm" case "confirm": fromBal, err := api.WalletBalance(ctx, a) @@ -603,6 +650,7 @@ func interactiveDeal(cctx *cli.Context) error { fmt.Printf("Piece size: %s (Payload size: %s)\n", units.BytesSize(float64(ds.PieceSize)), units.BytesSize(float64(ds.PayloadSize))) fmt.Printf("Duration: %s\n", dur) fmt.Printf("Total price: ~%s (%s per epoch)\n", types.FIL(totalPrice), types.FIL(epochPrice)) + fmt.Printf("Verified: %v\n", verified) state = "accept" case "accept": @@ -637,7 +685,7 @@ func interactiveDeal(cctx *cli.Context) error { MinBlocksDuration: uint64(epochs), DealStartEpoch: abi.ChainEpoch(cctx.Int64("start-epoch")), FastRetrieval: cctx.Bool("fast-retrieval"), - VerifiedDeal: false, // TODO: Allow setting + VerifiedDeal: verified, }) if err != nil { return err From 73b9fe36e3e24cedf027de51796529fdd1cf7aae Mon Sep 17 00:00:00 2001 From: Ingar Shu Date: Fri, 2 Oct 2020 13:07:40 -0700 Subject: [PATCH 508/885] Use verified price for verified deals --- cli/client.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cli/client.go b/cli/client.go index 6ba143753..7494815bf 100644 --- a/cli/client.go +++ b/cli/client.go @@ -637,10 +637,15 @@ func interactiveDeal(cctx *cli.Context) error { epochs = abi.ChainEpoch(dur / (time.Duration(build.BlockDelaySecs) * time.Second)) // TODO: do some more or epochs math (round to miner PP, deal start buffer) + pricePerGib := ask.Price + if verified { + pricePerGib = ask.VerifiedPrice + } + gib := types.NewInt(1 << 30) // TODO: price is based on PaddedPieceSize, right? - epochPrice = types.BigDiv(types.BigMul(ask.Price, types.NewInt(uint64(ds.PieceSize))), gib) + epochPrice = types.BigDiv(types.BigMul(pricePerGib, types.NewInt(uint64(ds.PieceSize))), gib) totalPrice := types.BigMul(epochPrice, types.NewInt(uint64(epochs))) fmt.Printf("-----\n") From fecc1ee0ef135352c3079c2ef1fe5f7ec355079f Mon Sep 17 00:00:00 2001 From: ZenGround0 Date: Fri, 2 Oct 2020 17:12:12 -0400 Subject: [PATCH 509/885] Review cleanup --- chain/stmgr/stmgr.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index b9810fab9..90e43ebbc 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -212,7 +212,7 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp if i > parentEpoch { // run cron for null rounds if any if err := runCron(); err != nil { - return cid.Cid{}, cid.Cid{}, err + return cid.Undef, cid.Undef, err } pstate, err = vmi.Flush(ctx) From db91d22a8a53c5b2cc8d976ec270f0f553c9799b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 2 Oct 2020 23:14:47 +0200 Subject: [PATCH 510/885] docsgen --- documentation/en/api-methods.md | 38 +++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index 2b28816f7..19774802e 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -72,6 +72,7 @@ * [MpoolPending](#MpoolPending) * [MpoolPush](#MpoolPush) * [MpoolPushMessage](#MpoolPushMessage) + * [MpoolPushUntrusted](#MpoolPushUntrusted) * [MpoolSelect](#MpoolSelect) * [MpoolSetConfig](#MpoolSetConfig) * [MpoolSub](#MpoolSub) @@ -1779,6 +1780,43 @@ Response: } ``` +### MpoolPushUntrusted +MpoolPushUntrusted pushes a signed message to mempool from untrusted sources. + + +Perms: write + +Inputs: +```json +[ + { + "Message": { + "Version": 42, + "To": "t01234", + "From": "t01234", + "Nonce": 42, + "Value": "0", + "GasLimit": 9, + "GasFeeCap": "0", + "GasPremium": "0", + "Method": 1, + "Params": "Ynl0ZSBhcnJheQ==" + }, + "Signature": { + "Type": 2, + "Data": "Ynl0ZSBhcnJheQ==" + } + } +] +``` + +Response: +```json +{ + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" +} +``` + ### MpoolSelect MpoolSelect returns a list of pending messages for inclusion in the next block From 0591bac768ca241506cfd139147ebdd14da2e263 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 2 Oct 2020 23:18:37 +0200 Subject: [PATCH 511/885] Compare more accurately Co-authored-by: dirkmc --- chain/messagepool/messagepool.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chain/messagepool/messagepool.go b/chain/messagepool/messagepool.go index d9a2b75f4..83aa5c6b7 100644 --- a/chain/messagepool/messagepool.go +++ b/chain/messagepool/messagepool.go @@ -251,7 +251,7 @@ func (ms *msgSet) add(m *types.SignedMessage, mp *MessagePool, strict, untrusted //ms.requiredFunds.Sub(ms.requiredFunds, exms.Message.Value.Int) } - if !has && strict && len(ms.msgs) > maxActorPendingMessages { + if !has && strict && len(ms.msgs) >= maxActorPendingMessages { log.Errorf("too many pending messages from actor %s", m.Message.From) return false, ErrTooManyPendingMessages } From ef3987e2abbc38346974f9eba8b3d46a0c8b7dae Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Fri, 2 Oct 2020 14:51:32 -0700 Subject: [PATCH 512/885] decode parameters for multisig transactions in inspect --- cli/multisig.go | 45 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/cli/multisig.go b/cli/multisig.go index 5b0977382..ade938f3d 100644 --- a/cli/multisig.go +++ b/cli/multisig.go @@ -3,13 +3,17 @@ package cli import ( "bytes" "encoding/hex" + "encoding/json" "fmt" "os" + "reflect" "sort" "strconv" "text/tabwriter" "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/stmgr" + cbg "github.com/whyrusleeping/cbor-gen" "github.com/filecoin-project/go-state-types/big" @@ -171,6 +175,10 @@ var msigInspectCmd = &cli.Command{ Name: "vesting", Usage: "Include vesting details", }, + &cli.BoolFlag{ + Name: "decode-params", + Usage: "Decode parameters of transaction proposals", + }, }, Action: func(cctx *cli.Context) error { if !cctx.Args().Present() { @@ -201,6 +209,11 @@ var msigInspectCmd = &cli.Command{ return err } + ownId, err := api.StateLookupID(ctx, maddr, types.EmptyTSK) + if err != nil { + return err + } + mstate, err := multisig.Load(store, act) if err != nil { return err @@ -253,6 +266,7 @@ var msigInspectCmd = &cli.Command{ return xerrors.Errorf("reading pending transactions: %w", err) } + decParams := cctx.Bool("decode-params") fmt.Println("Transactions: ", len(pending)) if len(pending) > 0 { var txids []int64 @@ -263,11 +277,36 @@ var msigInspectCmd = &cli.Command{ return txids[i] < txids[j] }) - w := tabwriter.NewWriter(os.Stdout, 8, 4, 0, ' ', 0) + w := tabwriter.NewWriter(os.Stdout, 8, 4, 2, ' ', 0) fmt.Fprintf(w, "ID\tState\tApprovals\tTo\tValue\tMethod\tParams\n") for _, txid := range txids { tx := pending[txid] - fmt.Fprintf(w, "%d\t%s\t%d\t%s\t%s\t%d\t%x\n", txid, "pending", len(tx.Approved), tx.To, types.FIL(tx.Value), tx.Method, tx.Params) + target := tx.To.String() + if tx.To == ownId { + target += " (self)" + } + targAct, err := api.StateGetActor(ctx, tx.To, types.EmptyTSK) + if err != nil { + return xerrors.Errorf("failed to resolve 'To' address of multisig transaction %d: %w", txid, err) + } + method := stmgr.MethodsMap[targAct.Code][tx.Method] + + paramStr := fmt.Sprintf("%x", tx.Params) + if decParams && tx.Method != 0 { + ptyp := reflect.New(method.Params.Elem()).Interface().(cbg.CBORUnmarshaler) + if err := ptyp.UnmarshalCBOR(bytes.NewReader(tx.Params)); err != nil { + return xerrors.Errorf("failed to decode parameters of transaction %d: %w", txid, err) + } + + b, err := json.Marshal(ptyp) + if err != nil { + return xerrors.Errorf("could not json marshal parameter type: %w", err) + } + + paramStr = string(b) + } + + fmt.Fprintf(w, "%d\t%s\t%d\t%s\t%s\t%s(%d)\t%s\n", txid, "pending", len(tx.Approved), target, types.FIL(tx.Value), method.Name, tx.Method, paramStr) } if err := w.Flush(); err != nil { return xerrors.Errorf("flushing output: %+v", err) @@ -395,7 +434,7 @@ var msigProposeCmd = &cli.Command{ var msigApproveCmd = &cli.Command{ Name: "approve", Usage: "Approve a multisig message", - ArgsUsage: "[multisigAddress messageId proposerAddress destination value (optional)]", + ArgsUsage: " [proposerAddress destination value [methodId methodParams]]", Flags: []cli.Flag{ &cli.StringFlag{ Name: "from", From 1319d7072ecf9094e6515e2de4bb8e41743a5948 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 2 Oct 2020 23:38:54 +0200 Subject: [PATCH 513/885] shed: Fix lint --- cmd/lotus-shed/pruning.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/cmd/lotus-shed/pruning.go b/cmd/lotus-shed/pruning.go index 6f0c20541..79158c3a3 100644 --- a/cmd/lotus-shed/pruning.go +++ b/cmd/lotus-shed/pruning.go @@ -137,13 +137,13 @@ var stateTreePruneCmd = &cli.Command{ return err } - defer ds.Close() + defer ds.Close() //nolint:errcheck mds, err := lkrepo.Datastore("/metadata") if err != nil { return err } - defer mds.Close() + defer mds.Close() //nolint:errcheck if cctx.Bool("only-ds-gc") { gcds, ok := ds.(datastore.GCDatastore) @@ -156,9 +156,8 @@ var stateTreePruneCmd = &cli.Command{ } fmt.Println("gc complete!") return nil - } else { - return fmt.Errorf("datastore doesnt support gc") } + return fmt.Errorf("datastore doesnt support gc") } bs := blockstore.NewBlockstore(ds) @@ -194,7 +193,7 @@ var stateTreePruneCmd = &cli.Command{ } fmt.Println() - fmt.Printf("Succesfully marked keep set! (%d objects)\n", goodSet.Len()) + fmt.Printf("Successfully marked keep set! (%d objects)\n", goodSet.Len()) if cctx.Bool("dry-run") { return nil From c5de617af61e54639c008fd006c8259ff019e264 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 2 Oct 2020 17:09:56 -0700 Subject: [PATCH 514/885] decouple network version and upgrades Not all upgrades require network version bumps. --- chain/stmgr/forks.go | 67 ++++++++++++++++++++++++++++++++---- chain/stmgr/forks_test.go | 15 +++++---- chain/stmgr/stmgr.go | 71 ++++++++++++++++++++++++--------------- go.mod | 4 +-- go.sum | 6 ++-- node/builder.go | 2 +- 6 files changed, 117 insertions(+), 48 deletions(-) diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index ccb6221d7..96cdd1f08 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -38,17 +38,70 @@ type UpgradeFunc func(context.Context, *StateManager, ExecCallback, cid.Cid, *ty type Upgrade struct { Height abi.ChainEpoch + Network network.Version Migration UpgradeFunc } -type UpgradeSchedule map[network.Version]Upgrade +type UpgradeSchedule []Upgrade -var DefaultUpgradeSchedule = UpgradeSchedule{ - network.Version1: {build.UpgradeBreezeHeight, UpgradeFaucetBurnRecovery}, - network.Version2: {build.UpgradeSmokeHeight, nil}, - network.Version3: {build.UpgradeIgnitionHeight, UpgradeIgnition}, - network.Version4: {build.UpgradeActorsV2Height, UpgradeActorsV2}, - network.Version5: {build.UpgradeLiftoffHeight, UpgradeLiftoff}, +func DefaultUpgradeSchedule() UpgradeSchedule { + var us UpgradeSchedule + + for _, u := range []Upgrade{{ + Height: build.UpgradeBreezeHeight, + Network: network.Version1, + Migration: UpgradeFaucetBurnRecovery, + }, { + Height: build.UpgradeSmokeHeight, + Network: network.Version2, + Migration: nil, + }, { + Height: build.UpgradeIgnitionHeight, + Network: network.Version3, + Migration: UpgradeIgnition, + }, { + Height: build.UpgradeActorsV2Height, + Network: network.Version4, + Migration: UpgradeActorsV2, + }, { + Height: build.UpgradeLiftoffHeight, + Network: network.Version4, + Migration: UpgradeLiftoff, + }} { + if u.Height < 0 { + // upgrade disabled + continue + } + us = append(us, u) + } + return us +} + +func (us UpgradeSchedule) Validate() error { + // Make sure we're not trying to upgrade to version 0. + for _, u := range us { + if u.Network <= 0 { + return xerrors.Errorf("cannot upgrade to version <= 0: %d", u.Network) + } + } + + // Make sure all the upgrades make sense. + for i := 1; i < len(us); i++ { + prev := &us[i-1] + curr := &us[i] + if !(prev.Network <= curr.Network) { + return xerrors.Errorf("cannot downgrade from version %d to version %d", prev.Network, curr.Network) + } + // Make sure the heights make sense. + if prev.Height < 0 { + // Previous upgrade was disabled. + continue + } + if !(prev.Height < curr.Height) { + return xerrors.Errorf("upgrade heights must be strictly increasing: upgrade %d was at height %d, followed by upgrade %d at height %d", i-1, prev.Height, i, curr.Height) + } + } + return nil } func (sm *StateManager) handleStateForks(ctx context.Context, root cid.Cid, height abi.ChainEpoch, cb ExecCallback, ts *types.TipSet) (cid.Cid, error) { diff --git a/chain/stmgr/forks_test.go b/chain/stmgr/forks_test.go index db65d2485..bb03f13b9 100644 --- a/chain/stmgr/forks_test.go +++ b/chain/stmgr/forks_test.go @@ -115,10 +115,11 @@ func TestForkHeightTriggers(t *testing.T) { t.Fatal(err) } - sm := NewStateManagerWithUpgradeSchedule( - cg.ChainStore(), - UpgradeSchedule{ - 1: {Height: testForkHeight, Migration: func(ctx context.Context, sm *StateManager, cb ExecCallback, + sm, err := NewStateManagerWithUpgradeSchedule( + cg.ChainStore(), UpgradeSchedule{{ + Network: 1, + Height: testForkHeight, + Migration: func(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, ts *types.TipSet) (cid.Cid, error) { cst := ipldcbor.NewCborStore(sm.ChainStore().Blockstore()) @@ -151,8 +152,10 @@ func TestForkHeightTriggers(t *testing.T) { } return st.Flush(ctx) - }}, - }) + }}}) + if err != nil { + t.Fatal(err) + } inv := vm.NewActorRegistry() inv.Register(nil, testActor{}) diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index 3efe26f53..91109d9bd 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -39,12 +39,18 @@ import ( var log = logging.Logger("statemgr") +type versionSpec struct { + networkVersion network.Version + atOrBelow abi.ChainEpoch +} + type StateManager struct { cs *store.ChainStore - // Sorted network upgrade epochs (starting at version 1). - // -1 means the version is skipped. - networkVersions []abi.ChainEpoch + // Determines the network version at any given epoch. + networkVersions []versionSpec + latestVersion network.Version + // Maps chain epochs to upgrade functions. stateMigrations map[abi.ChainEpoch]UpgradeFunc @@ -58,39 +64,49 @@ type StateManager struct { } func NewStateManager(cs *store.ChainStore) *StateManager { - return NewStateManagerWithUpgradeSchedule(cs, DefaultUpgradeSchedule) + sm, err := NewStateManagerWithUpgradeSchedule(cs, DefaultUpgradeSchedule()) + if err != nil { + panic(fmt.Sprintf("default upgrade schedule is invalid: %s", err)) + } + return sm } -func NewStateManagerWithUpgradeSchedule(cs *store.ChainStore, us UpgradeSchedule) *StateManager { +func NewStateManagerWithUpgradeSchedule(cs *store.ChainStore, us UpgradeSchedule) (*StateManager, error) { + // If we have upgrades, make sure they're in-order and make sense. + if err := us.Validate(); err != nil { + return nil, err + } + stateMigrations := make(map[abi.ChainEpoch]UpgradeFunc, len(us)) - networkVersions := make([]abi.ChainEpoch, 0, len(stateMigrations)) - - // Iterate version by version, to make sure we handle skipped version numbers. - // Always skip version 0. - for i, version := 0, network.Version(1); i < len(us); version++ { - upgrade, ok := us[version] - if ok { - // We've processed an upgrade. - i++ + var networkVersions []versionSpec + lastVersion := network.Version0 + if len(us) > 0 { + // If we have any upgrades, process them and create a version + // schedule. + for _, upgrade := range us { + if upgrade.Migration != nil { + stateMigrations[upgrade.Height] = upgrade.Migration + } + networkVersions = append(networkVersions, versionSpec{ + networkVersion: lastVersion, + atOrBelow: upgrade.Height, + }) + lastVersion = upgrade.Network } - - epoch := abi.ChainEpoch(-1) - if ok && upgrade.Height >= 0 { - epoch = upgrade.Height - stateMigrations[epoch] = upgrade.Migration - } - - networkVersions = append(networkVersions, epoch) + } else { + // Otherwise, go directly to the latest version. + lastVersion = build.NewestNetworkVersion } return &StateManager{ networkVersions: networkVersions, + latestVersion: lastVersion, stateMigrations: stateMigrations, newVM: vm.NewVM, cs: cs, stCache: make(map[string][]cid.Cid), compWait: make(map[string]chan struct{}), - } + }, nil } func cidsToKey(cids []cid.Cid) string { @@ -1294,13 +1310,12 @@ func (sm *StateManager) GetCirculatingSupply(ctx context.Context, height abi.Cha func (sm *StateManager) GetNtwkVersion(ctx context.Context, height abi.ChainEpoch) network.Version { // The epochs here are the _last_ epoch for every version, or -1 if the // version is disabled. - for v, epoch := range sm.networkVersions { - if epoch >= 0 && height <= epoch { - // Use the _previous_ version - return network.Version(v) + for _, spec := range sm.networkVersions { + if height <= spec.atOrBelow { + return spec.networkVersion } } - return network.Version(len(sm.networkVersions)) + return sm.latestVersion } func (sm *StateManager) GetPaychState(ctx context.Context, addr address.Address, ts *types.TipSet) (*types.Actor, paych.State, error) { diff --git a/go.mod b/go.mod index 50e5aecc1..44dd4acfa 100644 --- a/go.mod +++ b/go.mod @@ -33,12 +33,12 @@ require ( github.com/filecoin-project/go-multistore v0.0.3 github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 github.com/filecoin-project/go-paramfetch v0.0.2-0.20200701152213-3e0f0afdc261 - github.com/filecoin-project/go-state-types v0.0.0-20201001162932-93a412d8e9fa + github.com/filecoin-project/go-state-types v0.0.0-20200928172055-2df22083d8ab github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b github.com/filecoin-project/specs-actors v0.9.12 - github.com/filecoin-project/specs-actors/v2 v2.0.0-20201001041506-c7ff44a3ce9b + github.com/filecoin-project/specs-actors/v2 v2.0.0-20201002200957-bdd876b3bbe9 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 github.com/filecoin-project/test-vectors/schema v0.0.3 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 diff --git a/go.sum b/go.sum index aa8e3f633..78daf3af2 100644 --- a/go.sum +++ b/go.sum @@ -263,8 +263,6 @@ github.com/filecoin-project/go-state-types v0.0.0-20200904021452-1883f36ca2f4/go github.com/filecoin-project/go-state-types v0.0.0-20200905071437-95828685f9df/go.mod h1:IQ0MBPnonv35CJHtWSN3YY1Hz2gkPru1Q9qoaYLxx9I= github.com/filecoin-project/go-state-types v0.0.0-20200928172055-2df22083d8ab h1:cEDC5Ei8UuT99hPWhCjA72SM9AuRtnpvdSTIYbnzN8I= github.com/filecoin-project/go-state-types v0.0.0-20200928172055-2df22083d8ab/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= -github.com/filecoin-project/go-state-types v0.0.0-20201001162932-93a412d8e9fa h1:WVxLrmc6w5naeX3QikfRvc1G7EFfBZbhXuzM8Lj2J+o= -github.com/filecoin-project/go-state-types v0.0.0-20201001162932-93a412d8e9fa/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-statemachine v0.0.0-20200714194326-a77c3ae20989/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe h1:dF8u+LEWeIcTcfUcCf3WFVlc81Fr2JKg8zPzIbBDKDw= github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= @@ -276,8 +274,8 @@ github.com/filecoin-project/specs-actors v0.9.4/go.mod h1:BStZQzx5x7TmCkLv0Bpa07 github.com/filecoin-project/specs-actors v0.9.7/go.mod h1:wM2z+kwqYgXn5Z7scV1YHLyd1Q1cy0R8HfTIWQ0BFGU= github.com/filecoin-project/specs-actors v0.9.12 h1:iIvk58tuMtmloFNHhAOQHG+4Gci6Lui0n7DYQGi3cJk= github.com/filecoin-project/specs-actors v0.9.12/go.mod h1:TS1AW/7LbG+615j4NsjMK1qlpAwaFsG9w0V2tg2gSao= -github.com/filecoin-project/specs-actors/v2 v2.0.0-20201001041506-c7ff44a3ce9b h1:8JRxR0rKdiuCHIDKi7Zbs+/jiygr6eXEX47QXhjX+kA= -github.com/filecoin-project/specs-actors/v2 v2.0.0-20201001041506-c7ff44a3ce9b/go.mod h1:52FuQUNDXq2WDg+6+UOhkqBuNc2e62h9BCIB67Bluxg= +github.com/filecoin-project/specs-actors/v2 v2.0.0-20201002200957-bdd876b3bbe9 h1:UXVcGwUD9GJYNcgfGbf7ok790wrunvUhl5ZlgApGhGs= +github.com/filecoin-project/specs-actors/v2 v2.0.0-20201002200957-bdd876b3bbe9/go.mod h1:52FuQUNDXq2WDg+6+UOhkqBuNc2e62h9BCIB67Bluxg= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= github.com/filecoin-project/test-vectors/schema v0.0.3 h1:1zuBo25B3016inbygYLgYFdpJ2m1BDTbAOCgABRleiU= diff --git a/node/builder.go b/node/builder.go index c738740ec..d267ac229 100644 --- a/node/builder.go +++ b/node/builder.go @@ -259,7 +259,7 @@ func Online() Option { Override(new(ffiwrapper.Verifier), ffiwrapper.ProofVerifier), Override(new(vm.SyscallBuilder), vm.Syscalls), Override(new(*store.ChainStore), modules.ChainStore), - Override(new(stmgr.UpgradeSchedule), stmgr.DefaultUpgradeSchedule), + Override(new(stmgr.UpgradeSchedule), stmgr.DefaultUpgradeSchedule()), Override(new(*stmgr.StateManager), stmgr.NewStateManagerWithUpgradeSchedule), Override(new(*wallet.Wallet), wallet.NewWallet), Override(new(*messagesigner.MessageSigner), messagesigner.NewMessageSigner), From a57288a4b919020d1ae82940e30c27db00127cc7 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 2 Oct 2020 17:15:31 -0700 Subject: [PATCH 515/885] run post before, while, and after upgrading --- api/test/window_post.go | 46 ++++++++++++++++++++++++++++++----------- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/api/test/window_post.go b/api/test/window_post.go index 958c91816..e77f25fe0 100644 --- a/api/test/window_post.go +++ b/api/test/window_post.go @@ -15,9 +15,11 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/extern/sector-storage/mock" sealing "github.com/filecoin-project/lotus/extern/storage-sealing" + "github.com/filecoin-project/lotus/node" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/types" bminer "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node/impl" @@ -114,8 +116,29 @@ func pledgeSectors(t *testing.T, ctx context.Context, miner TestStorageNode, n, } func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSectors int) { - ctx := context.Background() - n, sn := b(t, 1, OneMiner) + for _, height := range []abi.ChainEpoch{ + 1, // before + 162, // while sealing + 3000, // while proving + 10_000, // after + } { + t.Run(fmt.Sprintf("upgrade-%d", height), func(t *testing.T) { + testWindowPostUpgrade(t, b, blocktime, nSectors, height) + }) + } + +} +func testWindowPostUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration, nSectors int, + upgradeHeight abi.ChainEpoch) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + n, sn := b(t, 1, OneMiner, node.Override(new(stmgr.UpgradeSchedule), stmgr.UpgradeSchedule{{ + Network: build.ActorUpgradeNetworkVersion, + Height: upgradeHeight, + Migration: stmgr.UpgradeActorsV2, + }})) + client := n[0].FullNode.(*impl.FullNodeAPI) miner := sn[0] @@ -129,17 +152,20 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector } build.Clock.Sleep(time.Second) - mine := true done := make(chan struct{}) go func() { defer close(done) - for mine { + for ctx.Err() == nil { build.Clock.Sleep(blocktime) if err := sn[0].MineOne(ctx, MineNext); err != nil { t.Error(err) } } }() + defer func() { + cancel() + <-done + }() pledgeSectors(t, ctx, miner, nSectors, 0, nil) @@ -159,7 +185,7 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector head, err := client.ChainHead(ctx) require.NoError(t, err) - if head.Height() > di.PeriodStart+(di.WPoStProvingPeriod)+2 { + if head.Height() > di.PeriodStart+di.WPoStProvingPeriod+2 { fmt.Printf("Now head.Height = %d\n", head.Height()) break } @@ -289,12 +315,11 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector pledgeSectors(t, ctx, miner, 1, nSectors, nil) { - // wait a bit more - - head, err := client.ChainHead(ctx) + // Wait until proven. + di, err = client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK) require.NoError(t, err) - waitUntil := head.Height() + 10 + waitUntil := di.PeriodStart + di.WPoStProvingPeriod + 2 fmt.Printf("End for head.Height > %d\n", waitUntil) for { @@ -315,7 +340,4 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector sectors = p.MinerPower.RawBytePower.Uint64() / uint64(ssz) require.Equal(t, nSectors+GenesisPreseals-2+1, int(sectors)) // -2 not recovered sectors + 1 just pledged - - mine = false - <-done } From f1e5e9fe72540bc52d131d2ff747df6883cb2aa8 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 2 Oct 2020 17:31:26 -0700 Subject: [PATCH 516/885] better upgrade test times --- api/test/window_post.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/api/test/window_post.go b/api/test/window_post.go index e77f25fe0..8f5c4db75 100644 --- a/api/test/window_post.go +++ b/api/test/window_post.go @@ -117,10 +117,9 @@ func pledgeSectors(t *testing.T, ctx context.Context, miner TestStorageNode, n, func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSectors int) { for _, height := range []abi.ChainEpoch{ - 1, // before - 162, // while sealing - 3000, // while proving - 10_000, // after + 1, // before + 162, // while sealing + 5000, // while proving } { t.Run(fmt.Sprintf("upgrade-%d", height), func(t *testing.T) { testWindowPostUpgrade(t, b, blocktime, nSectors, height) From 863a6ed38c526561cb8823106dacb97bd91e39d4 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 2 Oct 2020 17:45:01 -0700 Subject: [PATCH 517/885] fix lints --- api/test/window_post.go | 1 + 1 file changed, 1 insertion(+) diff --git a/api/test/window_post.go b/api/test/window_post.go index 8f5c4db75..a9a465b74 100644 --- a/api/test/window_post.go +++ b/api/test/window_post.go @@ -121,6 +121,7 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector 162, // while sealing 5000, // while proving } { + height := height // copy to satisfy lints t.Run(fmt.Sprintf("upgrade-%d", height), func(t *testing.T) { testWindowPostUpgrade(t, b, blocktime, nSectors, height) }) From 6aba3e3b94b6ad7df7146fa2505f535bde105565 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 2 Oct 2020 17:45:15 -0700 Subject: [PATCH 518/885] fix deal error message logging --- api/test/deals.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/api/test/deals.go b/api/test/deals.go index 12cd0607a..aa5bfa716 100644 --- a/api/test/deals.go +++ b/api/test/deals.go @@ -402,9 +402,12 @@ func testRetrieval(t *testing.T, ctx context.Context, client *impl.FullNodeAPI, IsCAR: carExport, } updates, err := client.ClientRetrieveWithEvents(ctx, offers[0].Order(caddr), ref) + if err != nil { + t.Fatal(err) + } for update := range updates { if update.Err != "" { - t.Fatalf("%v", err) + t.Fatalf("retrieval failed: %s", update.Err) } } From f6ccab1c6cc14bf685151e5b9130f2fff8b5a2a9 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 2 Oct 2020 17:47:01 -0700 Subject: [PATCH 519/885] ignore context canceled error in window post test --- api/test/window_post.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/api/test/window_post.go b/api/test/window_post.go index a9a465b74..eadcdbb05 100644 --- a/api/test/window_post.go +++ b/api/test/window_post.go @@ -158,6 +158,10 @@ func testWindowPostUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration, for ctx.Err() == nil { build.Clock.Sleep(blocktime) if err := sn[0].MineOne(ctx, MineNext); err != nil { + if ctx.Err() != nil { + // context was canceled, ignore the error. + return + } t.Error(err) } } From 8fe8da6f4c787e26e180721a98b14758a175adee Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Fri, 2 Oct 2020 18:41:08 -0400 Subject: [PATCH 520/885] Add verified status to api.DealInfo --- api/api_full.go | 1 + node/impl/client/client.go | 3 +++ 2 files changed, 4 insertions(+) diff --git a/api/api_full.go b/api/api_full.go index ceb0518b8..5cbdde8e3 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -530,6 +530,7 @@ type DealInfo struct { DealID abi.DealID CreationTime time.Time + Verified bool } type MsgLookup struct { diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 39f0ab19a..5997fc286 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -200,6 +200,7 @@ func (a *API) ClientListDeals(ctx context.Context) ([]api.DealInfo, error) { Duration: uint64(v.Proposal.Duration()), DealID: v.DealID, CreationTime: v.CreationTime.Time(), + Verified: v.Proposal.VerifiedDeal, } } @@ -223,6 +224,7 @@ func (a *API) ClientGetDealInfo(ctx context.Context, d cid.Cid) (*api.DealInfo, Duration: uint64(v.Proposal.Duration()), DealID: v.DealID, CreationTime: v.CreationTime.Time(), + Verified: v.Proposal.VerifiedDeal, }, nil } @@ -848,6 +850,7 @@ func newDealInfo(v storagemarket.ClientDeal) api.DealInfo { Duration: uint64(v.Proposal.Duration()), DealID: v.DealID, CreationTime: v.CreationTime.Time(), + Verified: v.Proposal.VerifiedDeal, } } From 306dfaa11f7e687c3cc001ce86cefaf9766166f1 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Sat, 3 Oct 2020 03:41:01 -0400 Subject: [PATCH 521/885] Update docs --- documentation/en/api-methods.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index a80f0b3e5..d03bb94ed 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -956,7 +956,8 @@ Response: "PricePerEpoch": "0", "Duration": 42, "DealID": 5432, - "CreationTime": "0001-01-01T00:00:00Z" + "CreationTime": "0001-01-01T00:00:00Z", + "Verified": true } ``` @@ -992,7 +993,8 @@ Response: "PricePerEpoch": "0", "Duration": 42, "DealID": 5432, - "CreationTime": "0001-01-01T00:00:00Z" + "CreationTime": "0001-01-01T00:00:00Z", + "Verified": true } ``` From 1fc23fb466e61327d953a909411f3c954194b912 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 3 Oct 2020 00:34:09 +0200 Subject: [PATCH 522/885] lotus-miner: Cleanup storage attach helptext a bit --- cmd/lotus-storage-miner/storage.go | 32 +++++++++++++++------------ extern/sector-storage/stores/local.go | 6 ++--- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/cmd/lotus-storage-miner/storage.go b/cmd/lotus-storage-miner/storage.go index 0c366cb01..77792f32a 100644 --- a/cmd/lotus-storage-miner/storage.go +++ b/cmd/lotus-storage-miner/storage.go @@ -31,9 +31,10 @@ const metaFile = "sectorstore.json" var storageCmd = &cli.Command{ Name: "storage", Usage: "manage sector storage", - Description: `Sectors can be stored across many filesystem paths. These commands provide ways to - manage the storage the miner will used to store sectors long term for proving (refernces as 'store') - as well as how sectors will be stored while moving through the sealing pipeline (references as 'seal').`, + Description: `Sectors can be stored across many filesystem paths. These +commands provide ways to manage the storage the miner will used to store sectors +long term for proving (references as 'store') as well as how sectors will be +stored while moving through the sealing pipeline (references as 'seal').`, Subcommands: []*cli.Command{ storageAttachCmd, storageListCmd, @@ -44,21 +45,24 @@ var storageCmd = &cli.Command{ var storageAttachCmd = &cli.Command{ Name: "attach", Usage: "attach local storage path", - Description: `Storage can be attach to the miner using this command. The storage volume list is stored local - to the miner in $LOTUS_MINER_PATH/storage.json. We do not recommend modifying this value without further - understanding of the storage system. + Description: `Storage can be attached to the miner using this command. The storage volume +list is stored local to the miner in $LOTUS_MINER_PATH/storage.json. We do not +recommend manually modifying this value without further understanding of the +storage system. - Each storage volume contains a configuration file which descripbes the capabilities of the volume. When the - '--init' flag is provided, this file will be created using the additional flags. +Each storage volume contains a configuration file which describes the +capabilities of the volume. When the '--init' flag is provided, this file will +be created using the additional flags. - Weight - A high weight value means data will be more likely to be stored +Weight +A high weight value means data will be more likely to be stored in this path - Seal - Intermittment data for the sealing process will be stored here +Seal +Data for the sealing process will be stored here - Store - Finalized sectors that will be proved over will be stored here +Store +Finalized sectors that will be moved here for long term storage and be proven +over time `, Flags: []cli.Flag{ &cli.BoolFlag{ diff --git a/extern/sector-storage/stores/local.go b/extern/sector-storage/stores/local.go index 70b37f10e..50968e7bd 100644 --- a/extern/sector-storage/stores/local.go +++ b/extern/sector-storage/stores/local.go @@ -32,13 +32,13 @@ type StoragePath struct { type LocalStorageMeta struct { ID ID - // A height wait means data is more likely to be stored here + // A high weight means data is more likely to be stored in this path Weight uint64 // 0 = readonly - // Intermittment data for the sealing process will be stored here + // Intermediate data for the sealing process will be stored here CanSeal bool - // Finalized sectors that will be proved over will be stored here + // Finalized sectors that will be proved over time will be stored here CanStore bool } From ed03202c1d1ed96c3e68d9d446b52223e708f0bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 5 Oct 2020 14:14:43 +0200 Subject: [PATCH 523/885] Test lotus-miner info all --- cli/cmd.go | 15 +++++ cmd/lotus-storage-miner/allinfo_test.go | 76 +++++++++++++++++++++++++ cmd/lotus-storage-miner/info_all.go | 10 +++- node/impl/common/common.go | 6 ++ 4 files changed, 104 insertions(+), 3 deletions(-) create mode 100644 cmd/lotus-storage-miner/allinfo_test.go diff --git a/cli/cmd.go b/cli/cmd.go index c6617dcfd..92a366eb5 100644 --- a/cli/cmd.go +++ b/cli/cmd.go @@ -216,6 +216,13 @@ func GetAPI(ctx *cli.Context) (api.Common, jsonrpc.ClientCloser, error) { log.Errorf("repoType type does not match the type of repo.RepoType") } + if tn, ok := ctx.App.Metadata["testnode-storage"]; ok { + return tn.(api.StorageMiner), func() {}, nil + } + if tn, ok := ctx.App.Metadata["testnode-full"]; ok { + return tn.(api.FullNode), func() {}, nil + } + addr, headers, err := GetRawAPI(ctx, t) if err != nil { return nil, nil, err @@ -225,6 +232,10 @@ func GetAPI(ctx *cli.Context) (api.Common, jsonrpc.ClientCloser, error) { } func GetFullNodeAPI(ctx *cli.Context) (api.FullNode, jsonrpc.ClientCloser, error) { + if tn, ok := ctx.App.Metadata["testnode-full"]; ok { + return tn.(api.FullNode), func() {}, nil + } + addr, headers, err := GetRawAPI(ctx, repo.FullNode) if err != nil { return nil, nil, err @@ -234,6 +245,10 @@ func GetFullNodeAPI(ctx *cli.Context) (api.FullNode, jsonrpc.ClientCloser, error } func GetStorageMinerAPI(ctx *cli.Context, opts ...jsonrpc.Option) (api.StorageMiner, jsonrpc.ClientCloser, error) { + if tn, ok := ctx.App.Metadata["testnode-storage"]; ok { + return tn.(api.StorageMiner), func() {}, nil + } + addr, headers, err := GetRawAPI(ctx, repo.StorageMiner) if err != nil { return nil, nil, err diff --git a/cmd/lotus-storage-miner/allinfo_test.go b/cmd/lotus-storage-miner/allinfo_test.go new file mode 100644 index 000000000..562714fa0 --- /dev/null +++ b/cmd/lotus-storage-miner/allinfo_test.go @@ -0,0 +1,76 @@ +package main + +import ( + "flag" + "testing" + "time" + + logging "github.com/ipfs/go-log/v2" + "github.com/stretchr/testify/require" + "github.com/urfave/cli/v2" + + "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/lotus/api/test" + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors/policy" + "github.com/filecoin-project/lotus/lib/lotuslog" + "github.com/filecoin-project/lotus/node/repo" + builder "github.com/filecoin-project/lotus/node/test" +) + +func TestMinerAllInfo(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode") + } + + _ = logging.SetLogLevel("*", "INFO") + + policy.SetConsensusMinerMinPower(abi.NewStoragePower(2048)) + policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg2KiBV1) + policy.SetMinVerifiedDealSize(abi.NewStoragePower(256)) + + _test = true + + lotuslog.SetupLogLevels() + logging.SetLogLevel("miner", "ERROR") + logging.SetLogLevel("chainstore", "ERROR") + logging.SetLogLevel("chain", "ERROR") + logging.SetLogLevel("sub", "ERROR") + logging.SetLogLevel("storageminer", "ERROR") + + oldDelay := policy.GetPreCommitChallengeDelay() + policy.SetPreCommitChallengeDelay(5) + t.Cleanup(func() { + policy.SetPreCommitChallengeDelay(oldDelay) + }) + + var n []test.TestNode + var sn []test.TestStorageNode + + run := func(t *testing.T) { + app := cli.NewApp() + app.Metadata = map[string]interface{}{ + "repoType": repo.StorageMiner, + "testnode-full": n[0], + "testnode-storage": sn[0], + } + build.RunningNodeType = build.NodeMiner + + cctx := cli.NewContext(app, flag.NewFlagSet("", flag.ContinueOnError), nil) + + require.NoError(t, infoAllCmd.Action(cctx)) + } + + bp := func(t *testing.T, nFull int, storage []test.StorageMiner) ([]test.TestNode, []test.TestStorageNode) { + n, sn = builder.Builder(t, nFull, storage) + + t.Run("pre-info-all", run) + + return n, sn + } + + test.TestDealFlow(t, bp, time.Second, false, false) + + t.Run("post-info-all", run) +} diff --git a/cmd/lotus-storage-miner/info_all.go b/cmd/lotus-storage-miner/info_all.go index 265ba78a4..517553028 100644 --- a/cmd/lotus-storage-miner/info_all.go +++ b/cmd/lotus-storage-miner/info_all.go @@ -10,6 +10,8 @@ import ( lcli "github.com/filecoin-project/lotus/cli" ) +var _test = false + var infoAllCmd = &cli.Command{ Name: "all", Usage: "dump all related miner info", @@ -150,9 +152,11 @@ var infoAllCmd = &cli.Command{ } } - fmt.Println("\n#: Goroutines") - if err := lcli.PprofGoroutines.Action(cctx); err != nil { - return err + if !_test { + fmt.Println("\n#: Goroutines") + if err := lcli.PprofGoroutines.Action(cctx); err != nil { + return err + } } return nil diff --git a/node/impl/common/common.go b/node/impl/common/common.go index b07c18305..da7cfff25 100644 --- a/node/impl/common/common.go +++ b/node/impl/common/common.go @@ -121,6 +121,12 @@ func (a *CommonAPI) NetFindPeer(ctx context.Context, p peer.ID) (peer.AddrInfo, func (a *CommonAPI) NetAutoNatStatus(ctx context.Context) (i api.NatInfo, err error) { autonat := a.RawHost.(*basichost.BasicHost).AutoNat + if autonat == nil { + return api.NatInfo{ + Reachability: network.ReachabilityUnknown, + }, nil + } + var maddr string if autonat.Status() == network.ReachabilityPublic { pa, err := autonat.PublicAddr() From 140671599c03c89516d95b253889e25b6731ab28 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Mon, 5 Oct 2020 14:17:40 +0200 Subject: [PATCH 524/885] fix: make message signer nonce generation transactional --- chain/messagesigner/messagesigner.go | 45 ++++++++++++++++----- chain/messagesigner/messagesigner_test.go | 49 +++++++++++++++++++++-- node/impl/full/mpool.go | 18 ++++----- 3 files changed, 89 insertions(+), 23 deletions(-) diff --git a/chain/messagesigner/messagesigner.go b/chain/messagesigner/messagesigner.go index 1ad83543b..6346c36ab 100644 --- a/chain/messagesigner/messagesigner.go +++ b/chain/messagesigner/messagesigner.go @@ -3,6 +3,7 @@ package messagesigner import ( "bytes" "context" + "sync" "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/messagepool" @@ -28,6 +29,7 @@ type mpoolAPI interface { // when signing a message type MessageSigner struct { wallet *wallet.Wallet + lk sync.Mutex mpool mpoolAPI ds datastore.Batching } @@ -47,22 +49,39 @@ func newMessageSigner(wallet *wallet.Wallet, mpool mpoolAPI, ds dtypes.MetadataD // SignMessage increments the nonce for the message From address, and signs // the message -func (ms *MessageSigner) SignMessage(ctx context.Context, msg *types.Message) (*types.SignedMessage, error) { +func (ms *MessageSigner) SignMessage(ctx context.Context, msg *types.Message, cb func(*types.SignedMessage) error) (*types.SignedMessage, error) { + ms.lk.Lock() + defer ms.lk.Unlock() + + // Get the next message nonce nonce, err := ms.nextNonce(msg.From) if err != nil { return nil, xerrors.Errorf("failed to create nonce: %w", err) } + // Sign the message with the nonce msg.Nonce = nonce sig, err := ms.wallet.Sign(ctx, msg.From, msg.Cid().Bytes()) if err != nil { return nil, xerrors.Errorf("failed to sign message: %w", err) } - return &types.SignedMessage{ + // Callback with the signed message + smsg := &types.SignedMessage{ Message: *msg, Signature: *sig, - }, nil + } + err = cb(smsg) + if err != nil { + return nil, err + } + + // If the callback executed successfully, write the nonce to the datastore + if err := ms.saveNonce(msg.From, nonce); err != nil { + return nil, xerrors.Errorf("failed to save nonce: %w", err) + } + + return smsg, nil } // nextNonce increments the nonce. @@ -78,7 +97,7 @@ func (ms *MessageSigner) nextNonce(addr address.Address) (uint64, error) { } // Get the nonce for this address from the datastore - addrNonceKey := datastore.KeyWithNamespaces([]string{dsKeyActorNonce, addr.String()}) + addrNonceKey := ms.dstoreKey(addr) dsNonceBytes, err := ms.ds.Get(addrNonceKey) switch { @@ -109,16 +128,24 @@ func (ms *MessageSigner) nextNonce(addr address.Address) (uint64, error) { } } - // Write the nonce for this address to the datastore + return nonce, nil +} + +// saveNonce writes the nonce for this address to the datastore +func (ms *MessageSigner) saveNonce(addr address.Address, nonce uint64) error { + addrNonceKey := ms.dstoreKey(addr) buf := bytes.Buffer{} - _, err = buf.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, nonce)) + _, err := buf.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, nonce)) if err != nil { - return 0, xerrors.Errorf("failed to marshall nonce: %w", err) + return xerrors.Errorf("failed to marshall nonce: %w", err) } err = ms.ds.Put(addrNonceKey, buf.Bytes()) if err != nil { - return 0, xerrors.Errorf("failed to write nonce to datastore: %w", err) + return xerrors.Errorf("failed to write nonce to datastore: %w", err) } + return nil +} - return nonce, nil +func (ms *MessageSigner) dstoreKey(addr address.Address) datastore.Key { + return datastore.KeyWithNamespaces([]string{dsKeyActorNonce, addr.String()}) } diff --git a/chain/messagesigner/messagesigner_test.go b/chain/messagesigner/messagesigner_test.go index 55676b258..04869ff6d 100644 --- a/chain/messagesigner/messagesigner_test.go +++ b/chain/messagesigner/messagesigner_test.go @@ -5,6 +5,8 @@ import ( "sync" "testing" + "golang.org/x/xerrors" + "github.com/filecoin-project/lotus/chain/wallet" "github.com/filecoin-project/go-state-types/crypto" @@ -58,6 +60,7 @@ func TestMessageSignerSignMessage(t *testing.T) { msg *types.Message mpoolNonce [1]uint64 expNonce uint64 + cbErr error } tests := []struct { name string @@ -137,6 +140,37 @@ func TestMessageSignerSignMessage(t *testing.T) { }, expNonce: 2, }}, + }, { + name: "recover from callback error", + msgs: []msgSpec{{ + // No nonce yet in datastore + msg: &types.Message{ + To: to1, + From: from1, + }, + expNonce: 0, + }, { + // Increment nonce + msg: &types.Message{ + To: to1, + From: from1, + }, + expNonce: 1, + }, { + // Callback returns error + msg: &types.Message{ + To: to1, + From: from1, + }, + cbErr: xerrors.Errorf("err"), + }, { + // Callback successful, should increment nonce in datastore + msg: &types.Message{ + To: to1, + From: from1, + }, + expNonce: 2, + }}, }} for _, tt := range tests { tt := tt @@ -149,9 +183,18 @@ func TestMessageSignerSignMessage(t *testing.T) { if len(m.mpoolNonce) == 1 { mpool.setNonce(m.msg.From, m.mpoolNonce[0]) } - smsg, err := ms.SignMessage(ctx, m.msg) - require.NoError(t, err) - require.Equal(t, m.expNonce, smsg.Message.Nonce) + merr := m.cbErr + smsg, err := ms.SignMessage(ctx, m.msg, func(message *types.SignedMessage) error { + return merr + }) + + if m.cbErr != nil { + require.Error(t, err) + require.Nil(t, smsg) + } else { + require.NoError(t, err) + require.Equal(t, m.expNonce, smsg.Message.Nonce) + } } }) } diff --git a/node/impl/full/mpool.go b/node/impl/full/mpool.go index e0dd3ecef..1f093606c 100644 --- a/node/impl/full/mpool.go +++ b/node/impl/full/mpool.go @@ -160,17 +160,13 @@ func (a *MpoolAPI) MpoolPushMessage(ctx context.Context, msg *types.Message, spe return nil, xerrors.Errorf("mpool push: not enough funds: %s < %s", b, msg.Value) } - smsg, err := a.MessageSigner.SignMessage(ctx, msg) - if err != nil { - return nil, xerrors.Errorf("mpool push: failed to sign message: %w", err) - } - - _, err = a.Mpool.Push(smsg) - if err != nil { - return nil, xerrors.Errorf("mpool push: failed to push message: %w", err) - } - - return smsg, err + // Sign and push the message + return a.MessageSigner.SignMessage(ctx, msg, func(smsg *types.SignedMessage) error { + if _, err := a.Mpool.Push(smsg); err != nil { + return xerrors.Errorf("mpool push: failed to push message: %w", err) + } + return nil + }) } func (a *MpoolAPI) MpoolGetNonce(ctx context.Context, addr address.Address) (uint64, error) { From 7c3f622fae69500a77c97764b3a9e57164a96adc Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Mon, 5 Oct 2020 14:54:22 +0200 Subject: [PATCH 525/885] refactor: message signer - always store *next* nonce in datastore --- chain/messagesigner/messagesigner.go | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/chain/messagesigner/messagesigner.go b/chain/messagesigner/messagesigner.go index 6346c36ab..ac94d6a3e 100644 --- a/chain/messagesigner/messagesigner.go +++ b/chain/messagesigner/messagesigner.go @@ -17,7 +17,7 @@ import ( "golang.org/x/xerrors" ) -const dsKeyActorNonce = "ActorNonce" +const dsKeyActorNonce = "ActorNextNonce" var log = logging.Logger("messagesigner") @@ -84,7 +84,7 @@ func (ms *MessageSigner) SignMessage(ctx context.Context, msg *types.Message, cb return smsg, nil } -// nextNonce increments the nonce. +// nextNonce gets the next nonce for the given address. // If there is no nonce in the datastore, gets the nonce from the message pool. func (ms *MessageSigner) nextNonce(addr address.Address) (uint64, error) { // Nonces used to be created by the mempool and we need to support nodes @@ -96,7 +96,7 @@ func (ms *MessageSigner) nextNonce(addr address.Address) (uint64, error) { return 0, xerrors.Errorf("failed to get nonce from mempool: %w", err) } - // Get the nonce for this address from the datastore + // Get the next nonce for this address from the datastore addrNonceKey := ms.dstoreKey(addr) dsNonceBytes, err := ms.ds.Get(addrNonceKey) @@ -104,13 +104,14 @@ func (ms *MessageSigner) nextNonce(addr address.Address) (uint64, error) { case xerrors.Is(err, datastore.ErrNotFound): // If a nonce for this address hasn't yet been created in the // datastore, just use the nonce from the mempool + return nonce, nil case err != nil: return 0, xerrors.Errorf("failed to get nonce from datastore: %w", err) default: - // There is a nonce in the datastore, so unmarshall and increment it - maj, val, err := cbg.CborReadHeader(bytes.NewReader(dsNonceBytes)) + // There is a nonce in the datastore, so unmarshall it + maj, dsNonce, err := cbg.CborReadHeader(bytes.NewReader(dsNonceBytes)) if err != nil { return 0, xerrors.Errorf("failed to parse nonce from datastore: %w", err) } @@ -118,21 +119,24 @@ func (ms *MessageSigner) nextNonce(addr address.Address) (uint64, error) { return 0, xerrors.Errorf("bad cbor type parsing nonce from datastore") } - dsNonce := val + 1 - // The message pool nonce should be <= than the datastore nonce if nonce <= dsNonce { nonce = dsNonce } else { log.Warnf("mempool nonce was larger than datastore nonce (%d > %d)", nonce, dsNonce) } - } - return nonce, nil + return nonce, nil + } } -// saveNonce writes the nonce for this address to the datastore +// saveNonce increments the nonce for this address and writes it to the +// datastore func (ms *MessageSigner) saveNonce(addr address.Address, nonce uint64) error { + // Increment the nonce + nonce++ + + // Write the nonce to the datastore addrNonceKey := ms.dstoreKey(addr) buf := bytes.Buffer{} _, err := buf.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, nonce)) From b9f8831aaffc138ecb6b1ee62cb79a3c9033b813 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 5 Oct 2020 09:51:04 -0700 Subject: [PATCH 526/885] fix specs-actors import in payment channel abstraction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Łukasz Magiera --- chain/actors/builtin/paych/message2.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/chain/actors/builtin/paych/message2.go b/chain/actors/builtin/paych/message2.go index 7d9d8d07c..2cf3ef22e 100644 --- a/chain/actors/builtin/paych/message2.go +++ b/chain/actors/builtin/paych/message2.go @@ -4,9 +4,9 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" - builtin2 "github.com/filecoin-project/specs-actors/actors/builtin" - init2 "github.com/filecoin-project/specs-actors/actors/builtin/init" - paych2 "github.com/filecoin-project/specs-actors/actors/builtin/paych" + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + init2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/init" + paych2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/paych" "github.com/filecoin-project/lotus/chain/actors" init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init" From feb85a272cc2e9a42f818f34e7c31420d1c6f7ad Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Mon, 5 Oct 2020 19:06:47 +0200 Subject: [PATCH 527/885] Update go-bitfield Signed-off-by: Jakub Sztandera --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 3dabb5443..024a6a204 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/fatih/color v1.8.0 github.com/filecoin-project/filecoin-ffi v0.30.4-0.20200716204036-cddc56607e1d github.com/filecoin-project/go-address v0.0.4 - github.com/filecoin-project/go-bitfield v0.2.1-0.20200920172649-837cbe6a1ed3 + github.com/filecoin-project/go-bitfield v0.2.1 github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 github.com/filecoin-project/go-data-transfer v0.6.7 diff --git a/go.sum b/go.sum index c02c6daea..e820ede3a 100644 --- a/go.sum +++ b/go.sum @@ -230,8 +230,8 @@ github.com/filecoin-project/go-amt-ipld/v2 v2.1.0 h1:t6qDiuGYYngDqaLc2ZUvdtAg4UN github.com/filecoin-project/go-amt-ipld/v2 v2.1.0/go.mod h1:nfFPoGyX0CU9SkXX8EoCcSuHN1XcbN0c6KBh7yvP5fs= github.com/filecoin-project/go-bitfield v0.2.0 h1:gCtLcjskIPtdg4NfN7gQZSQF9yrBQ7mkT0qCJxzGI2Q= github.com/filecoin-project/go-bitfield v0.2.0/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM= -github.com/filecoin-project/go-bitfield v0.2.1-0.20200920172649-837cbe6a1ed3 h1:HQa4+yCYsLq1TLM0kopeAhSCLbtZ541cWEi5N5rO+9g= -github.com/filecoin-project/go-bitfield v0.2.1-0.20200920172649-837cbe6a1ed3/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM= +github.com/filecoin-project/go-bitfield v0.2.1 h1:S6Uuqcspqu81sWJ0He4OAfFLm1tSwPdVjtKTkl5m/xQ= +github.com/filecoin-project/go-bitfield v0.2.1/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM= github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:av5fw6wmm58FYMgJeoB/lK9XXrgdugYiTqkdxjTy9k8= github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= From 4be89640620147b50624e9623b8a71a41e1540ad Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Mon, 5 Oct 2020 18:35:18 +0200 Subject: [PATCH 528/885] Add stats for base fee Signed-off-by: Jakub Sztandera --- cli/mpool.go | 69 +++++++++++++++++++++++++++++++++++----------------- 1 file changed, 47 insertions(+), 22 deletions(-) diff --git a/cli/mpool.go b/cli/mpool.go index a8c73b656..8f3e937b6 100644 --- a/cli/mpool.go +++ b/cli/mpool.go @@ -164,14 +164,6 @@ var mpoolSub = &cli.Command{ }, } -type statBucket struct { - msgs map[uint64]*types.SignedMessage -} -type mpStat struct { - addr string - past, cur, future uint64 -} - var mpoolStat = &cli.Command{ Name: "stat", Usage: "print mempool stats", @@ -180,6 +172,11 @@ var mpoolStat = &cli.Command{ Name: "local", Usage: "print stats for addresses in local wallet only", }, + &cli.IntFlag{ + Name: "basefee-lookback", + Usage: "number of blocks to look back for minimum basefee", + Value: 60, + }, }, Action: func(cctx *cli.Context) error { api, closer, err := GetFullNodeAPI(cctx) @@ -194,6 +191,20 @@ var mpoolStat = &cli.Command{ if err != nil { return xerrors.Errorf("getting chain head: %w", err) } + currBF := ts.Blocks()[0].ParentBaseFee + minBF := currBF + { + currTs := ts + for i := 0; i < cctx.Int("basefee-lookback"); i++ { + currTs, err = api.ChainGetTipSet(ctx, currTs.Parents()) + if err != nil { + return xerrors.Errorf("walking chain: %w", err) + } + if newBF := currTs.Blocks()[0].ParentBaseFee; newBF.LessThan(minBF) { + minBF = newBF + } + } + } var filter map[address.Address]struct{} if cctx.Bool("local") { @@ -214,8 +225,16 @@ var mpoolStat = &cli.Command{ return err } - buckets := map[address.Address]*statBucket{} + type statBucket struct { + msgs map[uint64]*types.SignedMessage + } + type mpStat struct { + addr string + past, cur, future uint64 + belowCurr, belowPast uint64 + } + buckets := map[address.Address]*statBucket{} for _, v := range msgs { if filter != nil { if _, has := filter[v.Message.From]; !has { @@ -252,23 +271,27 @@ var mpoolStat = &cli.Command{ cur++ } - past := uint64(0) - future := uint64(0) + var s mpStat + s.addr = a.String() + for _, m := range bkt.msgs { if m.Message.Nonce < act.Nonce { - past++ + s.past++ + } else if m.Message.Nonce > cur { + s.future++ + } else { + s.cur++ } - if m.Message.Nonce > cur { - future++ + + if m.Message.GasFeeCap.LessThan(currBF) { + s.belowCurr++ + } + if m.Message.GasFeeCap.LessThan(minBF) { + s.belowPast++ } } - out = append(out, mpStat{ - addr: a.String(), - past: past, - cur: cur - act.Nonce, - future: future, - }) + out = append(out, s) } sort.Slice(out, func(i, j int) bool { @@ -281,12 +304,14 @@ var mpoolStat = &cli.Command{ total.past += stat.past total.cur += stat.cur total.future += stat.future + total.belowCurr += stat.belowCurr + total.belowPast += stat.belowPast - fmt.Printf("%s: past: %d, cur: %d, future: %d\n", stat.addr, stat.past, stat.cur, stat.future) + fmt.Printf("%s: Nonce past: %d, cur: %d, future: %d; FeeCap cur: %d, min-%d: %d \n", stat.addr, stat.past, stat.cur, stat.future, stat.belowCurr, cctx.Int("basefee-lookback"), stat.belowPast) } fmt.Println("-----") - fmt.Printf("total: past: %d, cur: %d, future: %d\n", total.past, total.cur, total.future) + fmt.Printf("total: Nonce past: %d, cur: %d, future: %d; FeeCap cur: %d, min-%d: %d \n", total.past, total.cur, total.future, total.belowCurr, cctx.Int("basefee-lookback"), total.belowPast) return nil }, From 420c44ed2cb9fc19c4aedb79f123e9f3e3f3b55d Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 5 Oct 2020 10:35:21 -0700 Subject: [PATCH 529/885] fix msig start height --- chain/stmgr/forks.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index 96cdd1f08..d5d0dbf7e 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -437,6 +437,10 @@ func UpgradeIgnition(ctx context.Context, sm *StateManager, cb ExecCallback, roo epoch := ts.Height() - 1 + if build.UpgradeLiftoffHeight <= epoch { + return cid.Undef, xerrors.Errorf("liftoff height must be beyond ignition height") + } + nst, err := nv3.MigrateStateTree(ctx, store, root, epoch) if err != nil { return cid.Undef, xerrors.Errorf("migrating actors state: %w", err) @@ -462,7 +466,7 @@ func UpgradeIgnition(ctx context.Context, sm *StateManager, cb ExecCallback, roo return cid.Undef, xerrors.Errorf("second split address: %w", err) } - err = resetGenesisMsigs(ctx, sm, store, tree, epoch) + err = resetGenesisMsigs(ctx, sm, store, tree, build.UpgradeLiftoffHeight) if err != nil { return cid.Undef, xerrors.Errorf("resetting genesis msig start epochs: %w", err) } @@ -695,7 +699,7 @@ func makeKeyAddr(splitAddr address.Address, count uint64) (address.Address, erro return addr, nil } -func resetGenesisMsigs(ctx context.Context, sm *StateManager, store adt0.Store, tree *state.StateTree, epoch abi.ChainEpoch) error { +func resetGenesisMsigs(ctx context.Context, sm *StateManager, store adt0.Store, tree *state.StateTree, startEpoch abi.ChainEpoch) error { gb, err := sm.cs.GetGenesis() if err != nil { return xerrors.Errorf("getting genesis block: %w", err) @@ -724,7 +728,7 @@ func resetGenesisMsigs(ctx context.Context, sm *StateManager, store adt0.Store, return xerrors.Errorf("reading multisig state: %w", err) } - currState.StartEpoch = epoch + currState.StartEpoch = startEpoch currActor.Head, err = store.Put(ctx, &currState) if err != nil { From 8bcc201fc048271b0185321afd24fa4203eb3840 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 5 Oct 2020 10:52:32 -0700 Subject: [PATCH 530/885] fix v2 GetSectorExpiration --- chain/actors/builtin/miner/v2.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/chain/actors/builtin/miner/v2.go b/chain/actors/builtin/miner/v2.go index 6f496a51e..934554d23 100644 --- a/chain/actors/builtin/miner/v2.go +++ b/chain/actors/builtin/miner/v2.go @@ -11,6 +11,7 @@ import ( "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p-core/peer" cbg "github.com/whyrusleeping/cbor-gen" + "golang.org/x/xerrors" "github.com/filecoin-project/lotus/chain/actors/adt" @@ -106,9 +107,7 @@ func (s *state2) NumLiveSectors() (uint64, error) { // GetSectorExpiration returns the effective expiration of the given sector. // -// If the sector isn't found or has already been terminated, this method returns -// nil and no error. If the sector does not expire early, the Early expiration -// field is 0. +// If the sector does not expire early, the Early expiration field is 0. func (s *state2) GetSectorExpiration(num abi.SectorNumber) (*SectorExpiration, error) { dls, err := s.State.LoadDeadlines(s.store) if err != nil { @@ -171,7 +170,7 @@ func (s *state2) GetSectorExpiration(num abi.SectorNumber) (*SectorExpiration, e return nil, err } if out.Early == 0 && out.OnTime == 0 { - return nil, nil + return nil, xerrors.Errorf("failed to find sector %d", num) } return &out, nil } From 5a08b881d6fa2de17580c0b8220c4dba890e6575 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 5 Oct 2020 10:56:00 -0700 Subject: [PATCH 531/885] Test upgrade before/during/after cc upgrade --- api/test/ccupgrade.go | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/api/test/ccupgrade.go b/api/test/ccupgrade.go index f58f1ff6e..958d2b5aa 100644 --- a/api/test/ccupgrade.go +++ b/api/test/ccupgrade.go @@ -12,15 +12,35 @@ import ( "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/node" "github.com/filecoin-project/lotus/node/impl" ) func TestCCUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration) { _ = os.Setenv("BELLMAN_NO_GPU", "1") + for _, height := range []abi.ChainEpoch{ + 1, // before + 162, // while sealing + 520, // after upgrade deal + 5000, // after + } { + t.Run(fmt.Sprintf("upgrade-%d", height), func(t *testing.T) { + testCCUpgrade(t, b, blocktime, height) + }) + } +} + +func testCCUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration, upgradeHeight abi.ChainEpoch) { ctx := context.Background() - n, sn := b(t, 1, OneMiner) + n, sn := b(t, 1, OneMiner, node.Override(new(stmgr.UpgradeSchedule), stmgr.UpgradeSchedule{{ + Network: build.ActorUpgradeNetworkVersion, + Height: upgradeHeight, + Migration: stmgr.UpgradeActorsV2, + }})) client := n[0].FullNode.(*impl.FullNodeAPI) miner := sn[0] From f8b2022f82a51e7d899bd0b7eaaeed973f0115d4 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 5 Oct 2020 11:08:46 -0700 Subject: [PATCH 532/885] fix linters --- api/test/ccupgrade.go | 1 + 1 file changed, 1 insertion(+) diff --git a/api/test/ccupgrade.go b/api/test/ccupgrade.go index 958d2b5aa..97fb665ed 100644 --- a/api/test/ccupgrade.go +++ b/api/test/ccupgrade.go @@ -28,6 +28,7 @@ func TestCCUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration) { 520, // after upgrade deal 5000, // after } { + height := height // make linters happy by copying t.Run(fmt.Sprintf("upgrade-%d", height), func(t *testing.T) { testCCUpgrade(t, b, blocktime, height) }) From 550f3771ea1efd9d5c4e494856e30a852e35ba60 Mon Sep 17 00:00:00 2001 From: Ingar Shu Date: Mon, 5 Oct 2020 10:32:49 -0700 Subject: [PATCH 533/885] Reject deals that are > 90 days in the future in the BasicDealFilter --- node/modules/storageminer.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 5eab2ec62..494603ba2 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -44,6 +44,7 @@ import ( paramfetch "github.com/filecoin-project/go-paramfetch" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-storedcounter" + "github.com/filecoin-project/specs-actors/actors/builtin" sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" @@ -468,6 +469,13 @@ func BasicDealFilter(user dtypes.DealFilter) func(onlineOk dtypes.ConsiderOnline return false, fmt.Sprintf("cannot seal a sector before %s", deal.Proposal.StartEpoch), nil } + // Reject if it's more than 90 days in the future + // TODO: read from cfg + maxStartEpoch := ht + abi.ChainEpoch(90*builtin.EpochsInDay) + if deal.Proposal.StartEpoch > maxStartEpoch { + return false, fmt.Sprintf("deal start epoch is too far in the future: %s > %s", deal.Proposal.StartEpoch, maxStartEpoch), nil + } + if user != nil { return user(ctx, deal) } From 1227bf28936b988f2cc638e9140a208618ff490b Mon Sep 17 00:00:00 2001 From: Ingar Shu Date: Mon, 5 Oct 2020 12:35:58 -0700 Subject: [PATCH 534/885] Change max time in future for StartEpoch to 7 days --- node/modules/storageminer.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 494603ba2..95bfb6c11 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -469,9 +469,9 @@ func BasicDealFilter(user dtypes.DealFilter) func(onlineOk dtypes.ConsiderOnline return false, fmt.Sprintf("cannot seal a sector before %s", deal.Proposal.StartEpoch), nil } - // Reject if it's more than 90 days in the future + // Reject if it's more than 7 days in the future // TODO: read from cfg - maxStartEpoch := ht + abi.ChainEpoch(90*builtin.EpochsInDay) + maxStartEpoch := ht + abi.ChainEpoch(7*builtin.EpochsInDay) if deal.Proposal.StartEpoch > maxStartEpoch { return false, fmt.Sprintf("deal start epoch is too far in the future: %s > %s", deal.Proposal.StartEpoch, maxStartEpoch), nil } From 9621dbb4f1d6a465ade8545a8939e334685bacdd Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 5 Oct 2020 13:27:34 -0700 Subject: [PATCH 535/885] fix tests to use v2 types --- paychmgr/paych_test.go | 32 ++++++++++++++++---------------- paychmgr/paychget_test.go | 4 ++-- paychmgr/simple.go | 2 +- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/paychmgr/paych_test.go b/paychmgr/paych_test.go index ea8a775b5..fcd3d50a8 100644 --- a/paychmgr/paych_test.go +++ b/paychmgr/paych_test.go @@ -15,8 +15,8 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/specs-actors/actors/builtin" - paych0 "github.com/filecoin-project/specs-actors/actors/builtin/paych" tutils "github.com/filecoin-project/specs-actors/support/testing" + paych2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/paych" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors/builtin/paych" @@ -237,7 +237,7 @@ func TestCreateVoucher(t *testing.T) { // Create a voucher in lane 1 voucherLane1Amt := big.NewInt(5) - voucher := paych0.SignedVoucher{ + voucher := paych2.SignedVoucher{ Lane: 1, Amount: voucherLane1Amt, } @@ -252,7 +252,7 @@ func TestCreateVoucher(t *testing.T) { // Create a voucher in lane 1 again, with a higher amount voucherLane1Amt = big.NewInt(8) - voucher = paych0.SignedVoucher{ + voucher = paych2.SignedVoucher{ Lane: 1, Amount: voucherLane1Amt, } @@ -267,7 +267,7 @@ func TestCreateVoucher(t *testing.T) { // Create a voucher in lane 2 that covers all the remaining funds // in the channel voucherLane2Amt := big.Sub(s.amt, voucherLane1Amt) - voucher = paych0.SignedVoucher{ + voucher = paych2.SignedVoucher{ Lane: 2, Amount: voucherLane2Amt, } @@ -281,7 +281,7 @@ func TestCreateVoucher(t *testing.T) { // Create a voucher in lane 2 that exceeds the remaining funds in the // channel voucherLane2Amt = big.Add(voucherLane2Amt, big.NewInt(1)) - voucher = paych0.SignedVoucher{ + voucher = paych2.SignedVoucher{ Lane: 2, Amount: voucherLane2Amt, } @@ -619,7 +619,7 @@ func TestCheckSpendable(t *testing.T) { // Check that the secret was passed through correctly lastCall := s.mock.getLastCall() - var p paych0.UpdateChannelStateParams + var p paych2.UpdateChannelStateParams err = p.UnmarshalCBOR(bytes.NewReader(lastCall.Params)) require.NoError(t, err) require.Equal(t, secret, p.Secret) @@ -673,7 +673,7 @@ func TestSubmitVoucher(t *testing.T) { // Check that the secret was passed through correctly msg := s.mock.pushedMessages(submitCid) - var p paych0.UpdateChannelStateParams + var p paych2.UpdateChannelStateParams err = p.UnmarshalCBOR(bytes.NewReader(msg.Message.Params)) require.NoError(t, err) require.Equal(t, secret, p.Secret) @@ -687,7 +687,7 @@ func TestSubmitVoucher(t *testing.T) { require.NoError(t, err) msg = s.mock.pushedMessages(submitCid) - var p3 paych0.UpdateChannelStateParams + var p3 paych2.UpdateChannelStateParams err = p3.UnmarshalCBOR(bytes.NewReader(msg.Message.Params)) require.NoError(t, err) require.Equal(t, secret3, p3.Secret) @@ -773,8 +773,8 @@ func testGenerateKeyPair(t *testing.T) ([]byte, []byte) { return priv, pub } -func createTestVoucher(t *testing.T, ch address.Address, voucherLane uint64, nonce uint64, voucherAmount big.Int, key []byte) *paych0.SignedVoucher { - sv := &paych0.SignedVoucher{ +func createTestVoucher(t *testing.T, ch address.Address, voucherLane uint64, nonce uint64, voucherAmount big.Int, key []byte) *paych2.SignedVoucher { + sv := &paych2.SignedVoucher{ ChannelAddr: ch, Lane: voucherLane, Nonce: nonce, @@ -789,13 +789,13 @@ func createTestVoucher(t *testing.T, ch address.Address, voucherLane uint64, non return sv } -func createTestVoucherWithExtra(t *testing.T, ch address.Address, voucherLane uint64, nonce uint64, voucherAmount big.Int, key []byte) *paych0.SignedVoucher { - sv := &paych0.SignedVoucher{ +func createTestVoucherWithExtra(t *testing.T, ch address.Address, voucherLane uint64, nonce uint64, voucherAmount big.Int, key []byte) *paych2.SignedVoucher { + sv := &paych2.SignedVoucher{ ChannelAddr: ch, Lane: voucherLane, Nonce: nonce, Amount: voucherAmount, - Extra: &paych0.ModVerifyParams{ + Extra: &paych2.ModVerifyParams{ Actor: tutils.NewActorAddr(t, "act"), }, } @@ -813,13 +813,13 @@ type mockBestSpendableAPI struct { mgr *Manager } -func (m *mockBestSpendableAPI) PaychVoucherList(ctx context.Context, ch address.Address) ([]*paych0.SignedVoucher, error) { +func (m *mockBestSpendableAPI) PaychVoucherList(ctx context.Context, ch address.Address) ([]*paych2.SignedVoucher, error) { vi, err := m.mgr.ListVouchers(ctx, ch) if err != nil { return nil, err } - out := make([]*paych0.SignedVoucher, len(vi)) + out := make([]*paych2.SignedVoucher, len(vi)) for k, v := range vi { out[k] = v.Voucher } @@ -827,7 +827,7 @@ func (m *mockBestSpendableAPI) PaychVoucherList(ctx context.Context, ch address. return out, nil } -func (m *mockBestSpendableAPI) PaychVoucherCheckSpendable(ctx context.Context, ch address.Address, voucher *paych0.SignedVoucher, secret []byte, proof []byte) (bool, error) { +func (m *mockBestSpendableAPI) PaychVoucherCheckSpendable(ctx context.Context, ch address.Address, voucher *paych2.SignedVoucher, secret []byte, proof []byte) (bool, error) { return m.mgr.CheckVoucherSpendable(ctx, ch, voucher, secret, proof) } diff --git a/paychmgr/paychget_test.go b/paychmgr/paychget_test.go index dca660029..9f19dd13d 100644 --- a/paychmgr/paychget_test.go +++ b/paychmgr/paychget_test.go @@ -16,7 +16,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/specs-actors/v2/actors/builtin" - init_ "github.com/filecoin-project/specs-actors/v2/actors/builtin/init" + init2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/init" tutils "github.com/filecoin-project/specs-actors/v2/support/testing" lotusinit "github.com/filecoin-project/lotus/chain/actors/builtin/init" @@ -26,7 +26,7 @@ import ( ) func testChannelResponse(t *testing.T, ch address.Address) types.MessageReceipt { - createChannelRet := init_.ExecReturn{ + createChannelRet := init2.ExecReturn{ IDAddress: ch, RobustAddress: ch, } diff --git a/paychmgr/simple.go b/paychmgr/simple.go index cb3e3a102..253075604 100644 --- a/paychmgr/simple.go +++ b/paychmgr/simple.go @@ -13,7 +13,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/big" - init2 "github.com/filecoin-project/specs-actors/actors/builtin/init" + init2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/init" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" From bd0633ff90a0698d4d1c39dc3973025c74f2d397 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Mon, 5 Oct 2020 14:06:03 -0700 Subject: [PATCH 536/885] add a command to view block space utilization --- chain/exchange/client.go | 2 +- cli/chain.go | 115 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+), 1 deletion(-) diff --git a/chain/exchange/client.go b/chain/exchange/client.go index 371c50aed..cb030bcf7 100644 --- a/chain/exchange/client.go +++ b/chain/exchange/client.go @@ -117,7 +117,7 @@ func (c *client) doRequest( res, err := c.sendRequestToPeer(ctx, peer, req) if err != nil { if !xerrors.Is(err, network.ErrNoConn) { - log.Warnf("could not connect to peer %s: %s", + log.Warnf("could not send request to peer %s: %s", peer.String(), err) } continue diff --git a/cli/chain.go b/cli/chain.go index 73b25324b..288793ee4 100644 --- a/cli/chain.go +++ b/cli/chain.go @@ -8,6 +8,7 @@ import ( "os" "os/exec" "path" + "sort" "strconv" "strings" "time" @@ -30,6 +31,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/stmgr" types "github.com/filecoin-project/lotus/chain/types" ) @@ -50,6 +52,7 @@ var chainCmd = &cli.Command{ chainExportCmd, slashConsensusFault, chainGasPriceCmd, + chainInspectUsage, }, } @@ -375,6 +378,118 @@ var chainSetHeadCmd = &cli.Command{ }, } +var chainInspectUsage = &cli.Command{ + Name: "inspect-usage", + Usage: "Inspect block space usage of a given tipset", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "tipset", + Usage: "specify tipset to view block space usage of", + Value: "@head", + }, + }, + Action: func(cctx *cli.Context) error { + api, closer, err := GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + ctx := ReqContext(cctx) + + ts, err := LoadTipSet(ctx, cctx, api) + if err != nil { + return err + } + + pmsgs, err := api.ChainGetParentMessages(ctx, ts.Blocks()[0].Cid()) + if err != nil { + return err + } + + codeCache := make(map[address.Address]cid.Cid) + + lookupActorCode := func(a address.Address) (cid.Cid, error) { + c, ok := codeCache[a] + if ok { + return c, nil + } + + act, err := api.StateGetActor(ctx, a, ts.Key()) + if err != nil { + return cid.Undef, err + } + + codeCache[a] = act.Code + return act.Code, nil + } + + bySender := make(map[string]int64) + byDest := make(map[string]int64) + byMethod := make(map[string]int64) + + var sum int64 + for _, m := range pmsgs { + bySender[m.Message.From.String()] += m.Message.GasLimit + byDest[m.Message.To.String()] += m.Message.GasLimit + sum += m.Message.GasLimit + + code, err := lookupActorCode(m.Message.To) + if err != nil { + return err + } + + mm := stmgr.MethodsMap[code][m.Message.Method] + + byMethod[mm.Name] += m.Message.GasLimit + + } + + type keyGasPair struct { + Key string + Gas int64 + } + + mapToSortedKvs := func(m map[string]int64) []keyGasPair { + var vals []keyGasPair + for k, v := range m { + vals = append(vals, keyGasPair{ + Key: k, + Gas: v, + }) + } + sort.Slice(vals, func(i, j int) bool { + return vals[i].Gas > vals[j].Gas + }) + return vals + } + + senderVals := mapToSortedKvs(bySender) + destVals := mapToSortedKvs(byDest) + methodVals := mapToSortedKvs(byMethod) + + fmt.Printf("Total Gas Limit: %d\n", sum) + fmt.Printf("By Sender:\n") + for i := 0; i < 10 && i < len(senderVals); i++ { + sv := senderVals[i] + fmt.Printf("%s\t%0.2f\t(%d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas) + } + fmt.Println() + fmt.Printf("By Receiver:\n") + for i := 0; i < 10 && i < len(destVals); i++ { + sv := destVals[i] + fmt.Printf("%s\t%0.2f\t(%d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas) + } + fmt.Println() + fmt.Printf("By Method:\n") + for i := 0; i < 10 && i < len(methodVals); i++ { + sv := methodVals[i] + fmt.Printf("%s\t%0.2f\t(%d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas) + } + + return nil + }, +} + var chainListCmd = &cli.Command{ Name: "list", Usage: "View a segment of the chain", From 17846aad2f6f2d5828c9cb35baf8556d2317e1e0 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 5 Oct 2020 14:28:47 -0700 Subject: [PATCH 537/885] add some more policy toggles for testground --- chain/actors/policy/policy.go | 10 ++++++++++ chain/actors/policy/policy_test.go | 3 +++ extern/oni | 2 +- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/chain/actors/policy/policy.go b/chain/actors/policy/policy.go index 8606119f0..f29da9802 100644 --- a/chain/actors/policy/policy.go +++ b/chain/actors/policy/policy.go @@ -97,3 +97,13 @@ func DealProviderCollateralBounds( panic("unsupported network version") } } + +// Sets the challenge window and scales the proving period to match (such that +// there are always 48 challenge windows in a proving period). +func SetWPoStChallengeWindow(period abi.ChainEpoch) { + miner0.WPoStChallengeWindow = period + miner0.WPoStProvingPeriod = period * abi.ChainEpoch(miner0.WPoStPeriodDeadlines) + + miner2.WPoStChallengeWindow = period + miner2.WPoStProvingPeriod = period * abi.ChainEpoch(miner2.WPoStPeriodDeadlines) +} diff --git a/chain/actors/policy/policy_test.go b/chain/actors/policy/policy_test.go index 8d6aac09f..62e7f8964 100644 --- a/chain/actors/policy/policy_test.go +++ b/chain/actors/policy/policy_test.go @@ -43,5 +43,8 @@ func TestAssumptions(t *testing.T) { require.EqualValues(t, miner0.SupportedProofTypes, miner2.SupportedProofTypes) require.Equal(t, miner0.PreCommitChallengeDelay, miner2.PreCommitChallengeDelay) require.Equal(t, miner0.ChainFinality, miner2.ChainFinality) + require.Equal(t, miner0.WPoStChallengeWindow, miner2.WPoStChallengeWindow) + require.Equal(t, miner0.WPoStProvingPeriod, miner2.WPoStProvingPeriod) + require.Equal(t, miner0.WPoStPeriodDeadlines, miner2.WPoStPeriodDeadlines) require.True(t, verifreg0.MinVerifiedDealSize.Equals(verifreg2.MinVerifiedDealSize)) } diff --git a/extern/oni b/extern/oni index 9a0d5cd73..a5b57068d 160000 --- a/extern/oni +++ b/extern/oni @@ -1 +1 @@ -Subproject commit 9a0d5cd739de77b357589ac1fc8b756ed27299be +Subproject commit a5b57068d4c2ee0fda76f053edf9825613d1fe9b From 1e78cd21ec21d4ada6b732ec8be317b2fd287793 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 5 Oct 2020 14:34:13 -0700 Subject: [PATCH 538/885] fix lotus soup build --- extern/oni | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/oni b/extern/oni index a5b57068d..dbee44e4f 160000 --- a/extern/oni +++ b/extern/oni @@ -1 +1 @@ -Subproject commit a5b57068d4c2ee0fda76f053edf9825613d1fe9b +Subproject commit dbee44e4f940a502971f17116ccbba61ceaf2674 From 6d25f3d2ae190e8b162c6e21ca89dcbbe8b23f97 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Mon, 5 Oct 2020 18:39:39 -0400 Subject: [PATCH 539/885] Add some comments --- build/params_testnet.go | 1 + chain/messagepool/gasguess/guessgas.go | 1 + 2 files changed, 2 insertions(+) diff --git a/build/params_testnet.go b/build/params_testnet.go index ab579fd10..0aa77788b 100644 --- a/build/params_testnet.go +++ b/build/params_testnet.go @@ -23,6 +23,7 @@ const UpgradeSmokeHeight = 51000 const UpgradeIgnitionHeight = 94000 +// TODO: Actual epoch needs to be filled in const UpgradeActorsV2Height = 128888 // This signals our tentative epoch for mainnet launch. Can make it later, but not earlier. diff --git a/chain/messagepool/gasguess/guessgas.go b/chain/messagepool/gasguess/guessgas.go index 1a6a21755..607c7824a 100644 --- a/chain/messagepool/gasguess/guessgas.go +++ b/chain/messagepool/gasguess/guessgas.go @@ -42,6 +42,7 @@ var Costs = map[CostKey]int64{ {builtin0.StorageMinerActorCodeID, 16}: 5325185, {builtin0.StorageMinerActorCodeID, 18}: 2328637, {builtin0.StoragePowerActorCodeID, 2}: 23600956, + // TODO: Just reuse v0 values for now, this isn't actually used {builtin2.InitActorCodeID, 2}: 8916753, {builtin2.StorageMarketActorCodeID, 2}: 6955002, {builtin2.StorageMarketActorCodeID, 4}: 245436108, From e9417562e05114946e08a98bddae738b514f4497 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Mon, 5 Oct 2020 15:53:43 -0700 Subject: [PATCH 540/885] allow usage inspection on a chain segment --- cli/chain.go | 57 +++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 41 insertions(+), 16 deletions(-) diff --git a/cli/chain.go b/cli/chain.go index 288793ee4..e47bc5d81 100644 --- a/cli/chain.go +++ b/cli/chain.go @@ -28,7 +28,7 @@ import ( cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" - "github.com/filecoin-project/lotus/api" + lapi "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/stmgr" @@ -162,7 +162,7 @@ var chainGetBlock = &cli.Command{ }, } -func apiMsgCids(in []api.Message) []cid.Cid { +func apiMsgCids(in []lapi.Message) []cid.Cid { out := make([]cid.Cid, len(in)) for k, v := range in { out[k] = v.Cid @@ -387,6 +387,16 @@ var chainInspectUsage = &cli.Command{ Usage: "specify tipset to view block space usage of", Value: "@head", }, + &cli.IntFlag{ + Name: "length", + Usage: "length of chain to inspect block space usage for", + Value: 1, + }, + &cli.IntFlag{ + Name: "num-results", + Usage: "number of results to print per category", + Value: 10, + }, }, Action: func(cctx *cli.Context) error { api, closer, err := GetFullNodeAPI(cctx) @@ -401,9 +411,22 @@ var chainInspectUsage = &cli.Command{ return err } - pmsgs, err := api.ChainGetParentMessages(ctx, ts.Blocks()[0].Cid()) - if err != nil { - return err + cur := ts + var msgs []lapi.Message + for i := 0; i < cctx.Int("length"); i++ { + pmsgs, err := api.ChainGetParentMessages(ctx, cur.Blocks()[0].Cid()) + if err != nil { + return err + } + + msgs = append(msgs, pmsgs...) + + next, err := api.ChainGetTipSet(ctx, cur.Parents()) + if err != nil { + return err + } + + cur = next } codeCache := make(map[address.Address]cid.Cid) @@ -428,7 +451,7 @@ var chainInspectUsage = &cli.Command{ byMethod := make(map[string]int64) var sum int64 - for _, m := range pmsgs { + for _, m := range msgs { bySender[m.Message.From.String()] += m.Message.GasLimit byDest[m.Message.To.String()] += m.Message.GasLimit sum += m.Message.GasLimit @@ -467,23 +490,25 @@ var chainInspectUsage = &cli.Command{ destVals := mapToSortedKvs(byDest) methodVals := mapToSortedKvs(byMethod) + numRes := cctx.Int("num-results") + fmt.Printf("Total Gas Limit: %d\n", sum) fmt.Printf("By Sender:\n") - for i := 0; i < 10 && i < len(senderVals); i++ { + for i := 0; i < numRes && i < len(senderVals); i++ { sv := senderVals[i] - fmt.Printf("%s\t%0.2f\t(%d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas) + fmt.Printf("%s\t%0.2f%%\t(%d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas) } fmt.Println() fmt.Printf("By Receiver:\n") - for i := 0; i < 10 && i < len(destVals); i++ { + for i := 0; i < numRes && i < len(destVals); i++ { sv := destVals[i] - fmt.Printf("%s\t%0.2f\t(%d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas) + fmt.Printf("%s\t%0.2f%%\t(%d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas) } fmt.Println() fmt.Printf("By Method:\n") - for i := 0; i < 10 && i < len(methodVals); i++ { + for i := 0; i < numRes && i < len(methodVals); i++ { sv := methodVals[i] - fmt.Printf("%s\t%0.2f\t(%d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas) + fmt.Printf("%s\t%0.2f%%\t(%d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas) } return nil @@ -763,7 +788,7 @@ var chainGetCmd = &cli.Command{ type apiIpldStore struct { ctx context.Context - api api.FullNode + api lapi.FullNode } func (ht *apiIpldStore) Context() context.Context { @@ -791,7 +816,7 @@ func (ht *apiIpldStore) Put(ctx context.Context, v interface{}) (cid.Cid, error) panic("No mutations allowed") } -func handleAmt(ctx context.Context, api api.FullNode, r cid.Cid) error { +func handleAmt(ctx context.Context, api lapi.FullNode, r cid.Cid) error { s := &apiIpldStore{ctx, api} mp, err := adt.AsArray(s, r) if err != nil { @@ -804,7 +829,7 @@ func handleAmt(ctx context.Context, api api.FullNode, r cid.Cid) error { }) } -func handleHamtEpoch(ctx context.Context, api api.FullNode, r cid.Cid) error { +func handleHamtEpoch(ctx context.Context, api lapi.FullNode, r cid.Cid) error { s := &apiIpldStore{ctx, api} mp, err := adt.AsMap(s, r) if err != nil { @@ -822,7 +847,7 @@ func handleHamtEpoch(ctx context.Context, api api.FullNode, r cid.Cid) error { }) } -func handleHamtAddress(ctx context.Context, api api.FullNode, r cid.Cid) error { +func handleHamtAddress(ctx context.Context, api lapi.FullNode, r cid.Cid) error { s := &apiIpldStore{ctx, api} mp, err := adt.AsMap(s, r) if err != nil { From 70d88f226e16e925b02bfa945720684f215e6341 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 6 Oct 2020 01:33:51 +0200 Subject: [PATCH 541/885] Add usage to repo-type in shed datastore --- cmd/lotus-shed/datastore.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/lotus-shed/datastore.go b/cmd/lotus-shed/datastore.go index 62ee6bc6e..c6bac6815 100644 --- a/cmd/lotus-shed/datastore.go +++ b/cmd/lotus-shed/datastore.go @@ -34,6 +34,7 @@ var datastoreListCmd = &cli.Command{ Flags: []cli.Flag{ &cli.IntFlag{ Name: "repo-type", + Usage: "node type (1 - full, 2 - storage, 3 - worker)", Value: 1, }, &cli.BoolFlag{ @@ -102,6 +103,7 @@ var datastoreGetCmd = &cli.Command{ Flags: []cli.Flag{ &cli.IntFlag{ Name: "repo-type", + Usage: "node type (1 - full, 2 - storage, 3 - worker)", Value: 1, }, &cli.StringFlag{ From 8cdf07899953021b4237e847248b8d39517d0867 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 6 Oct 2020 01:50:43 +0200 Subject: [PATCH 542/885] backup: append checksum --- lib/backupds/datastore.go | 108 +++++++++++++++++++++++--------------- lib/backupds/read.go | 30 +++++++++-- 2 files changed, 93 insertions(+), 45 deletions(-) diff --git a/lib/backupds/datastore.go b/lib/backupds/datastore.go index 8845eacc7..1555577f3 100644 --- a/lib/backupds/datastore.go +++ b/lib/backupds/datastore.go @@ -1,6 +1,7 @@ package backupds import ( + "crypto/sha256" "io" "sync" @@ -26,58 +27,81 @@ func Wrap(child datastore.Batching) *Datastore { } } -// Writes a datastore dump into the provided writer as indefinite length cbor -// array of [key, value] tuples +// Writes a datastore dump into the provided writer as +// [array(*) of [key, value] tuples, checksum] func (d *Datastore) Backup(out io.Writer) error { - // write indefinite length array header - if _, err := out.Write([]byte{0x9f}); err != nil { - return xerrors.Errorf("writing header: %w", err) - } - - d.backupLk.Lock() - defer d.backupLk.Unlock() - - log.Info("Starting datastore backup") - defer log.Info("Datastore backup done") - - qr, err := d.child.Query(query.Query{}) - if err != nil { - return xerrors.Errorf("query: %w", err) - } - defer func() { - if err := qr.Close(); err != nil { - log.Errorf("query close error: %+v", err) - return - } - }() - scratch := make([]byte, 9) - for result := range qr.Next() { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, out, cbg.MajArray, 2); err != nil { - return xerrors.Errorf("writing tuple header: %w", err) + if err := cbg.WriteMajorTypeHeaderBuf(scratch, out, cbg.MajArray, 2); err != nil { + return xerrors.Errorf("writing tuple header: %w", err) + } + + hasher := sha256.New() + hout := io.MultiWriter(hasher, out) + + // write KVs + { + // write indefinite length array header + if _, err := hout.Write([]byte{0x9f}); err != nil { + return xerrors.Errorf("writing header: %w", err) } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, out, cbg.MajByteString, uint64(len([]byte(result.Key)))); err != nil { - return xerrors.Errorf("writing key header: %w", err) + d.backupLk.Lock() + defer d.backupLk.Unlock() + + log.Info("Starting datastore backup") + defer log.Info("Datastore backup done") + + qr, err := d.child.Query(query.Query{}) + if err != nil { + return xerrors.Errorf("query: %w", err) + } + defer func() { + if err := qr.Close(); err != nil { + log.Errorf("query close error: %+v", err) + return + } + }() + + for result := range qr.Next() { + if err := cbg.WriteMajorTypeHeaderBuf(scratch, hout, cbg.MajArray, 2); err != nil { + return xerrors.Errorf("writing tuple header: %w", err) + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, hout, cbg.MajByteString, uint64(len([]byte(result.Key)))); err != nil { + return xerrors.Errorf("writing key header: %w", err) + } + + if _, err := hout.Write([]byte(result.Key)[:]); err != nil { + return xerrors.Errorf("writing key: %w", err) + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, hout, cbg.MajByteString, uint64(len(result.Value))); err != nil { + return xerrors.Errorf("writing value header: %w", err) + } + + if _, err := hout.Write(result.Value[:]); err != nil { + return xerrors.Errorf("writing value: %w", err) + } } - if _, err := out.Write([]byte(result.Key)[:]); err != nil { - return xerrors.Errorf("writing key: %w", err) - } - - if err := cbg.WriteMajorTypeHeaderBuf(scratch, out, cbg.MajByteString, uint64(len(result.Value))); err != nil { - return xerrors.Errorf("writing value header: %w", err) - } - - if _, err := out.Write(result.Value[:]); err != nil { - return xerrors.Errorf("writing value: %w", err) + // array break + if _, err := hout.Write([]byte{0xff}); err != nil { + return xerrors.Errorf("writing array 'break': %w", err) } } - // array break - if _, err := out.Write([]byte{0xff}); err != nil { - return xerrors.Errorf("writing array 'break': %w", err) + // Write the checksum + { + sum := hasher.Sum(nil) + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, hout, cbg.MajByteString, uint64(len(sum))); err != nil { + return xerrors.Errorf("writing checksum header: %w", err) + } + + if _, err := hout.Write(sum[:]); err != nil { + return xerrors.Errorf("writing checksum: %w", err) + } } return nil diff --git a/lib/backupds/read.go b/lib/backupds/read.go index d368d1d09..f9a433637 100644 --- a/lib/backupds/read.go +++ b/lib/backupds/read.go @@ -1,6 +1,8 @@ package backupds import ( + "bytes" + "crypto/sha256" "io" "github.com/ipfs/go-datastore" @@ -15,12 +17,23 @@ func ReadBackup(r io.Reader, cb func(key datastore.Key, value []byte) error) err return xerrors.Errorf("reading array header: %w", err) } + if scratch[0] != 0x82 { + return xerrors.Errorf("expected array(2) header byte 0x82, got %x", scratch[0]) + } + + hasher := sha256.New() + hr := io.TeeReader(r, hasher) + + if _, err := hr.Read(scratch[:1]); err != nil { + return xerrors.Errorf("reading array header: %w", err) + } + if scratch[0] != 0x9f { return xerrors.Errorf("expected indefinite length array header byte 0x9f, got %x", scratch[0]) } for { - if _, err := r.Read(scratch[:1]); err != nil { + if _, err := hr.Read(scratch[:1]); err != nil { return xerrors.Errorf("reading tuple header: %w", err) } @@ -32,13 +45,13 @@ func ReadBackup(r io.Reader, cb func(key datastore.Key, value []byte) error) err return xerrors.Errorf("expected array(2) header 0x82, got %x", scratch[0]) } - keyb, err := cbg.ReadByteArray(r, 1<<40) + keyb, err := cbg.ReadByteArray(hr, 1<<40) if err != nil { return xerrors.Errorf("reading key: %w", err) } key := datastore.NewKey(string(keyb)) - value, err := cbg.ReadByteArray(r, 1<<40) + value, err := cbg.ReadByteArray(hr, 1<<40) if err != nil { return xerrors.Errorf("reading value: %w", err) } @@ -48,6 +61,17 @@ func ReadBackup(r io.Reader, cb func(key datastore.Key, value []byte) error) err } } + sum := hasher.Sum(nil) + + expSum, err := cbg.ReadByteArray(r, 32) + if err != nil { + return xerrors.Errorf("reading expected checksum: %w", err) + } + + if !bytes.Equal(sum, expSum) { + return xerrors.Errorf("checksum didn't match; expected %x, got %x", expSum, sum) + } + return nil } From 89b88c1e0d21272249ed9a45d844176cca60f94f Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 5 Oct 2020 17:02:11 -0700 Subject: [PATCH 543/885] fix conformance tests --- conformance/driver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conformance/driver.go b/conformance/driver.go index 2e8d5a203..5648084ce 100644 --- a/conformance/driver.go +++ b/conformance/driver.go @@ -167,7 +167,7 @@ func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, params ExecuteMessageP // dummy state manager; only to reference the GetNetworkVersion method, // which does not depend on state. - sm := new(stmgr.StateManager) + sm := stmgr.NewStateManager(nil) vmOpts := &vm.VMOpts{ StateBase: params.Preroot, From 16243b97aa8479e28d23c47c69d5fa5427f07bd1 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 5 Oct 2020 17:02:43 -0700 Subject: [PATCH 544/885] make state diffing work with partial state trees --- conformance/runner.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/conformance/runner.go b/conformance/runner.go index 2db53b3e4..626545b2b 100644 --- a/conformance/runner.go +++ b/conformance/runner.go @@ -233,7 +233,12 @@ func writeStateToTempCAR(bs blockstore.Blockstore, roots ...cid.Cid) (string, er if link.Cid.Prefix().Codec == cid.FilCommitmentSealed || link.Cid.Prefix().Codec == cid.FilCommitmentUnsealed { continue } - out = append(out, link) + // ignore things we don't have, the state tree is incomplete. + if has, err := bs.Has(link.Cid); err != nil { + return nil, err + } else if has { + out = append(out, link) + } } return out, nil } From c920257a337f2bbe234e0146152b568aca5541f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 6 Oct 2020 02:59:01 +0200 Subject: [PATCH 545/885] chain export: Error with unfinished exports --- cli/chain.go | 7 +++++++ node/impl/full/chain.go | 30 ++++++++++++++++++------------ 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/cli/chain.go b/cli/chain.go index 73b25324b..866469f78 100644 --- a/cli/chain.go +++ b/cli/chain.go @@ -930,13 +930,20 @@ var chainExportCmd = &cli.Command{ return err } + var last bool for b := range stream { + last = len(b) == 0 + _, err := fi.Write(b) if err != nil { return err } } + if !last { + return xerrors.Errorf("incomplete export (remote connection lost?)") + } + return nil }, } diff --git a/node/impl/full/chain.go b/node/impl/full/chain.go index 9606a023a..15caee752 100644 --- a/node/impl/full/chain.go +++ b/node/impl/full/chain.go @@ -507,15 +507,11 @@ func (a *ChainAPI) ChainExport(ctx context.Context, nroots abi.ChainEpoch, skipo r, w := io.Pipe() out := make(chan []byte) go func() { - defer w.Close() //nolint:errcheck // it is a pipe - bw := bufio.NewWriterSize(w, 1<<20) - defer bw.Flush() //nolint:errcheck // it is a write to a pipe - if err := a.Chain.Export(ctx, ts, nroots, skipoldmsgs, bw); err != nil { - log.Errorf("chain export call failed: %s", err) - return - } + err := a.Chain.Export(ctx, ts, nroots, skipoldmsgs, bw) + bw.Flush() //nolint:errcheck // it is a write to a pipe + w.CloseWithError(err) //nolint:errcheck // it is a pipe }() go func() { @@ -527,13 +523,23 @@ func (a *ChainAPI) ChainExport(ctx context.Context, nroots abi.ChainEpoch, skipo log.Errorf("chain export pipe read failed: %s", err) return } - select { - case out <- buf[:n]: - case <-ctx.Done(): - log.Warnf("export writer failed: %s", ctx.Err()) - return + if n > 0 { + select { + case out <- buf[:n]: + case <-ctx.Done(): + log.Warnf("export writer failed: %s", ctx.Err()) + return + } } if err == io.EOF { + // send empty slice to indicate correct eof + select { + case out <- []byte{}: + case <-ctx.Done(): + log.Warnf("export writer failed: %s", ctx.Err()) + return + } + return } } From f55b18eabeebc480ac3808dcc62443ed49bbce12 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Sun, 4 Oct 2020 21:38:52 -0400 Subject: [PATCH 546/885] Add funds that have left FilReserve to circ supply --- build/params_shared_vals.go | 5 +++++ build/params_testground.go | 8 ++++++++ chain/actors/builtin/builtin.go | 11 +++++++++++ chain/gen/genesis/genesis.go | 9 +++------ chain/stmgr/stmgr.go | 27 +++++++++++++++++++++++---- 5 files changed, 50 insertions(+), 10 deletions(-) diff --git a/build/params_shared_vals.go b/build/params_shared_vals.go index 4734a15ed..cc711b7bc 100644 --- a/build/params_shared_vals.go +++ b/build/params_shared_vals.go @@ -72,8 +72,10 @@ const FilBase = uint64(2_000_000_000) const FilAllocStorageMining = uint64(1_100_000_000) const FilecoinPrecision = uint64(1_000_000_000_000_000_000) +const FilReserved = uint64(300_000_000) var InitialRewardBalance *big.Int +var InitialFilReserved *big.Int // TODO: Move other important consts here @@ -81,6 +83,9 @@ func init() { InitialRewardBalance = big.NewInt(int64(FilAllocStorageMining)) InitialRewardBalance = InitialRewardBalance.Mul(InitialRewardBalance, big.NewInt(int64(FilecoinPrecision))) + InitialFilReserved = big.NewInt(int64(FilReserved)) + InitialFilReserved = InitialFilReserved.Mul(InitialFilReserved, big.NewInt(int64(FilecoinPrecision))) + if os.Getenv("LOTUS_ADDRESS_TYPE") == AddressMainnetEnvVar { SetAddressNetwork(address.Mainnet) } diff --git a/build/params_testground.go b/build/params_testground.go index 6d51100e4..162b3477c 100644 --- a/build/params_testground.go +++ b/build/params_testground.go @@ -55,6 +55,7 @@ var ( FilBase uint64 = 2_000_000_000 FilAllocStorageMining uint64 = 1_400_000_000 + FilReserved uint64 = 300_000_000 FilecoinPrecision uint64 = 1_000_000_000_000_000_000 @@ -63,6 +64,13 @@ var ( v = v.Mul(v, big.NewInt(int64(FilecoinPrecision))) return v }() + + InitialFilReserved = func() *big.Int { + v := big.NewInt(int64(FilReserved)) + v = v.Mul(v, big.NewInt(int64(FilecoinPrecision))) + return v + }() + // Actor consts // TODO: Pull from actors when its made not private MinDealDuration = abi.ChainEpoch(180 * builtin.EpochsInDay) diff --git a/chain/actors/builtin/builtin.go b/chain/actors/builtin/builtin.go index d49164486..7def78dcf 100644 --- a/chain/actors/builtin/builtin.go +++ b/chain/actors/builtin/builtin.go @@ -1,6 +1,7 @@ package builtin import ( + "github.com/filecoin-project/go-address" "github.com/ipfs/go-cid" "golang.org/x/xerrors" @@ -20,6 +21,7 @@ import ( var SystemActorAddr = builtin0.SystemActorAddr var BurntFundsActorAddr = builtin0.BurntFundsActorAddr +var ReserveAddress = makeAddress("t090") // TODO: Why does actors have 2 different versions of this? type SectorInfo = proof0.SectorInfo @@ -86,3 +88,12 @@ func IsMultisigActor(c cid.Cid) bool { func IsPaymentChannelActor(c cid.Cid) bool { return c == builtin0.PaymentChannelActorCodeID || c == builtin2.PaymentChannelActorCodeID } + +func makeAddress(addr string) address.Address { + ret, err := address.NewFromString(addr) + if err != nil { + panic(err) + } + + return ret +} diff --git a/chain/gen/genesis/genesis.go b/chain/gen/genesis/genesis.go index bb1056e2e..f532b9f5e 100644 --- a/chain/gen/genesis/genesis.go +++ b/chain/gen/genesis/genesis.go @@ -6,6 +6,8 @@ import ( "encoding/json" "fmt" + "github.com/filecoin-project/lotus/chain/actors/builtin" + "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" cbor "github.com/ipfs/go-ipld-cbor" @@ -296,14 +298,9 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge return nil, nil, xerrors.Errorf("somehow overallocated filecoin (allocated = %s)", types.FIL(totalFilAllocated)) } - remAccKey, err := address.NewIDAddress(90) - if err != nil { - return nil, nil, err - } - template.RemainderAccount.Balance = remainingFil - if err := createMultisigAccount(ctx, bs, cst, state, remAccKey, template.RemainderAccount, keyIDs); err != nil { + if err := createMultisigAccount(ctx, bs, cst, state, builtin.ReserveAddress, template.RemainderAccount, keyIDs); err != nil { return nil, nil, xerrors.Errorf("failed to set up remainder account: %w", err) } diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index e371992e8..b27bed587 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -1171,14 +1171,27 @@ func (sm *StateManager) GetFilVested(ctx context.Context, height abi.ChainEpoch, } } - // continue to use preIgnitionGenInfos, nothing changed at the Ignition epoch - vf = big.Add(vf, sm.preIgnitionGenInfos.genesisPledge) - // continue to use preIgnitionGenInfos, nothing changed at the Ignition epoch - vf = big.Add(vf, sm.preIgnitionGenInfos.genesisMarketFunds) + // After UpgradeActorsV2Height these funds are accounted for in GetFilReserveDisbursed + if height <= build.UpgradeActorsV2Height { + // continue to use preIgnitionGenInfos, nothing changed at the Ignition epoch + vf = big.Add(vf, sm.preIgnitionGenInfos.genesisPledge) + // continue to use preIgnitionGenInfos, nothing changed at the Ignition epoch + vf = big.Add(vf, sm.preIgnitionGenInfos.genesisMarketFunds) + } return vf, nil } +func GetFilReserveDisbursed(ctx context.Context, st *state.StateTree) (abi.TokenAmount, error) { + ract, err := st.GetActor(builtin.ReserveAddress) + if err != nil { + return big.Zero(), xerrors.Errorf("failed to get reserve actor: %w", err) + } + + // If money enters the reserve actor, this could lead to a negative term + return big.Sub(big.NewFromGo(build.InitialFilReserved), ract.Balance), nil +} + func GetFilMined(ctx context.Context, st *state.StateTree) (abi.TokenAmount, error) { ractor, err := st.GetActor(reward.Address) if err != nil { @@ -1266,6 +1279,11 @@ func (sm *StateManager) GetCirculatingSupplyDetailed(ctx context.Context, height return api.CirculatingSupply{}, xerrors.Errorf("failed to calculate filVested: %w", err) } + filReserveDisbursed, err := GetFilReserveDisbursed(ctx, st) + if err != nil { + return api.CirculatingSupply{}, xerrors.Errorf("failed to calculate filReserveDisbursed: %w", err) + } + filMined, err := GetFilMined(ctx, st) if err != nil { return api.CirculatingSupply{}, xerrors.Errorf("failed to calculate filMined: %w", err) @@ -1282,6 +1300,7 @@ func (sm *StateManager) GetCirculatingSupplyDetailed(ctx context.Context, height } ret := types.BigAdd(filVested, filMined) + ret = types.BigAdd(ret, filReserveDisbursed) ret = types.BigSub(ret, filBurnt) ret = types.BigSub(ret, filLocked) From 14ad91c53ff0d159a94fd8f6ad4c3cd0cf565a54 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Tue, 6 Oct 2020 01:27:45 -0400 Subject: [PATCH 547/885] Add type assertion for v2 runtime --- chain/vm/runtime.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/chain/vm/runtime.go b/chain/vm/runtime.go index 20d79e378..472017a00 100644 --- a/chain/vm/runtime.go +++ b/chain/vm/runtime.go @@ -15,6 +15,7 @@ import ( "github.com/filecoin-project/go-state-types/network" rtt "github.com/filecoin-project/go-state-types/rt" rt0 "github.com/filecoin-project/specs-actors/actors/runtime" + rt2 "github.com/filecoin-project/specs-actors/v2/actors/runtime" "github.com/ipfs/go-cid" ipldcbor "github.com/ipfs/go-ipld-cbor" "go.opencensus.io/trace" @@ -108,6 +109,7 @@ func (rt *Runtime) StorePut(x cbor.Marshaler) cid.Cid { } var _ rt0.Runtime = (*Runtime)(nil) +var _ rt2.Runtime = (*Runtime)(nil) func (rt *Runtime) shimCall(f func() interface{}) (rval []byte, aerr aerrors.ActorError) { defer func() { From 3672f2cc4418961d72e4158afc0a317e65dd327d Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Tue, 6 Oct 2020 01:59:15 -0400 Subject: [PATCH 548/885] Set default address type to mainnet --- build/params_testnet.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build/params_testnet.go b/build/params_testnet.go index 960f3a9b6..49abd30c0 100644 --- a/build/params_testnet.go +++ b/build/params_testnet.go @@ -5,6 +5,7 @@ package build import ( + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/actors/policy" @@ -34,6 +35,9 @@ func init() { abi.RegisteredSealProof_StackedDrg32GiBV1, abi.RegisteredSealProof_StackedDrg64GiBV1, ) + + SetAddressNetwork(address.Mainnet) + Devnet = false } From de390749fcb98912b03fd7c8ff538fb1f4bdfc0d Mon Sep 17 00:00:00 2001 From: vyzo Date: Tue, 6 Oct 2020 09:11:44 +0300 Subject: [PATCH 549/885] set pubsub IPColocationFactor to 5 --- node/modules/lp2p/pubsub.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go index aefb06d89..7dca3608f 100644 --- a/node/modules/lp2p/pubsub.go +++ b/node/modules/lp2p/pubsub.go @@ -223,8 +223,8 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) { }, AppSpecificWeight: 1, - // This sets the IP colocation threshold to 1 peer per - IPColocationFactorThreshold: 1, + // This sets the IP colocation threshold to 5 peers before we apply penalties + IPColocationFactorThreshold: 5, IPColocationFactorWeight: -100, // TODO we want to whitelist IPv6 /64s that belong to datacenters etc // IPColocationFactorWhitelist: map[string]struct{}{}, From d02fc08ca4cc5c6cdf7d559f7991e3701dcd8f42 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Tue, 6 Oct 2020 03:06:12 -0400 Subject: [PATCH 550/885] Support both testnet and mainnet addresses in the internal wallet --- chain/wallet/wallet.go | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/chain/wallet/wallet.go b/chain/wallet/wallet.go index 7cdb1929e..bc9cff6e4 100644 --- a/chain/wallet/wallet.go +++ b/chain/wallet/wallet.go @@ -81,7 +81,7 @@ func (w *Wallet) findKey(addr address.Address) (*Key, error) { return nil, nil } - ki, err := w.keystore.Get(KNamePrefix + addr.String()) + ki, err := w.tryFind(addr) if err != nil { if xerrors.Is(err, types.ErrKeyInfoNotFound) { return nil, nil @@ -96,6 +96,42 @@ func (w *Wallet) findKey(addr address.Address) (*Key, error) { return k, nil } +func (w *Wallet) tryFind(addr address.Address) (types.KeyInfo, error) { + + ki, err := w.keystore.Get(KNamePrefix + addr.String()) + if err == nil { + return ki, err + } + + if !xerrors.Is(err, types.ErrKeyInfoNotFound) { + return types.KeyInfo{}, err + } + + // We got an ErrKeyInfoNotFound error + // Try again, this time with the testnet prefix + + aChars := []rune(addr.String()) + prefixRunes := []rune(address.TestnetPrefix) + if len(prefixRunes) != 1 { + return types.KeyInfo{}, xerrors.Errorf("unexpected prefix length: %d", len(prefixRunes)) + } + + aChars[0] = prefixRunes[0] + ki, err = w.keystore.Get(KNamePrefix + string(aChars)) + if err != nil { + return types.KeyInfo{}, err + } + + // We found it with the testnet prefix + // Add this KeyInfo with the mainnet prefix address string + err = w.keystore.Put(KNamePrefix+addr.String(), ki) + if err != nil { + return types.KeyInfo{}, err + } + + return ki, nil +} + func (w *Wallet) Export(addr address.Address) (*types.KeyInfo, error) { k, err := w.findKey(addr) if err != nil { From ba25806fbd350a02c4aa84bcf0ed9eb6fa2e247f Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Tue, 6 Oct 2020 03:30:40 -0400 Subject: [PATCH 551/885] Update docs --- documentation/en/api-methods.md | 360 ++++++++++++++++---------------- 1 file changed, 180 insertions(+), 180 deletions(-) diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index 810e6ffa4..630eae32e 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -352,7 +352,7 @@ Inputs: Response: ```json { - "Miner": "t01234", + "Miner": "f01234", "Ticket": { "VRFProof": "Ynl0ZSBhcnJheQ==" }, @@ -449,8 +449,8 @@ Response: ```json { "Version": 42, - "To": "t01234", - "From": "t01234", + "To": "f01234", + "From": "f01234", "Nonce": 42, "Value": "0", "GasLimit": 9, @@ -942,7 +942,7 @@ Response: }, "State": 42, "Message": "string value", - "Provider": "t01234", + "Provider": "f01234", "DataRef": { "TransferType": "string value", "Root": { @@ -979,7 +979,7 @@ Response: }, "State": 42, "Message": "string value", - "Provider": "t01234", + "Provider": "f01234", "DataRef": { "TransferType": "string value", "Root": { @@ -1082,7 +1082,7 @@ Perms: read Inputs: ```json [ - "t01234", + "f01234", { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" }, @@ -1103,9 +1103,9 @@ Response: "UnsealPrice": "0", "PaymentInterval": 42, "PaymentIntervalIncrease": 42, - "Miner": "t01234", + "Miner": "f01234", "MinerPeer": { - "Address": "t01234", + "Address": "f01234", "ID": "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf", "PieceCID": null } @@ -1122,7 +1122,7 @@ Inputs: ```json [ "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf", - "t01234" + "f01234" ] ``` @@ -1133,7 +1133,7 @@ Response: "VerifiedPrice": "0", "MinPieceSize": 1032, "MaxPieceSize": 1032, - "Miner": "t01234", + "Miner": "f01234", "Timestamp": 10101, "Expiry": 10101, "SeqNo": 42 @@ -1174,10 +1174,10 @@ Inputs: "UnsealPrice": "0", "PaymentInterval": 42, "PaymentIntervalIncrease": 42, - "Client": "t01234", - "Miner": "t01234", + "Client": "f01234", + "Miner": "f01234", "MinerPeer": { - "Address": "t01234", + "Address": "f01234", "ID": "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf", "PieceCID": null } @@ -1201,7 +1201,7 @@ Perms: write Inputs: ```json [ - "t01234" + "f01234" ] ``` @@ -1227,10 +1227,10 @@ Inputs: "UnsealPrice": "0", "PaymentInterval": 42, "PaymentIntervalIncrease": 42, - "Client": "t01234", - "Miner": "t01234", + "Client": "f01234", + "Miner": "f01234", "MinerPeer": { - "Address": "t01234", + "Address": "f01234", "ID": "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf", "PieceCID": null } @@ -1271,8 +1271,8 @@ Inputs: "PieceCid": null, "PieceSize": 1024 }, - "Wallet": "t01234", - "Miner": "t01234", + "Wallet": "f01234", + "Miner": "f01234", "EpochPrice": "0", "MinBlocksDuration": 42, "ProviderCollateral": "0", @@ -1320,8 +1320,8 @@ Inputs: [ { "Version": 42, - "To": "t01234", - "From": "t01234", + "To": "f01234", + "From": "f01234", "Nonce": 42, "Value": "0", "GasLimit": 9, @@ -1356,8 +1356,8 @@ Inputs: [ { "Version": 42, - "To": "t01234", - "From": "t01234", + "To": "f01234", + "From": "f01234", "Nonce": 42, "Value": "0", "GasLimit": 9, @@ -1390,7 +1390,7 @@ Inputs: ```json [ 42, - "t01234", + "f01234", 9, [ { @@ -1416,8 +1416,8 @@ Inputs: [ { "Version": 42, - "To": "t01234", - "From": "t01234", + "To": "f01234", + "From": "f01234", "Nonce": 42, "Value": "0", "GasLimit": 9, @@ -1444,8 +1444,8 @@ Response: ```json { "Version": 42, - "To": "t01234", - "From": "t01234", + "To": "f01234", + "From": "f01234", "Nonce": 42, "Value": "0", "GasLimit": 9, @@ -1507,8 +1507,8 @@ Perms: sign Inputs: ```json [ - "t01234", - "t01234", + "f01234", + "f01234", "0" ] ``` @@ -1532,7 +1532,7 @@ Inputs: ```json [ { - "Miner": "t01234", + "Miner": "f01234", "Parents": [ { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" @@ -1561,7 +1561,7 @@ Response: ```json { "Header": { - "Miner": "t01234", + "Miner": "f01234", "Ticket": { "VRFProof": "Ynl0ZSBhcnJheQ==" }, @@ -1608,7 +1608,7 @@ Perms: read Inputs: ```json [ - "t01234", + "f01234", 10101, [ { @@ -1627,7 +1627,7 @@ Response: "MinerPower": "0", "NetworkPower": "0", "Sectors": null, - "WorkerKey": "t01234", + "WorkerKey": "f01234", "SectorSize": 34359738368, "PrevBeaconEntry": { "Round": 42, @@ -1688,7 +1688,7 @@ Perms: read Inputs: ```json [ - "t01234" + "f01234" ] ``` @@ -1728,8 +1728,8 @@ Inputs: { "Message": { "Version": 42, - "To": "t01234", - "From": "t01234", + "To": "f01234", + "From": "f01234", "Nonce": 42, "Value": "0", "GasLimit": 9, @@ -1769,8 +1769,8 @@ Inputs: [ { "Version": 42, - "To": "t01234", - "From": "t01234", + "To": "f01234", + "From": "f01234", "Nonce": 42, "Value": "0", "GasLimit": 9, @@ -1790,8 +1790,8 @@ Response: { "Message": { "Version": 42, - "To": "t01234", - "From": "t01234", + "To": "f01234", + "From": "f01234", "Nonce": 42, "Value": "0", "GasLimit": 9, @@ -1819,8 +1819,8 @@ Inputs: { "Message": { "Version": 42, - "To": "t01234", - "From": "t01234", + "To": "f01234", + "From": "f01234", "Nonce": 42, "Value": "0", "GasLimit": 9, @@ -1903,8 +1903,8 @@ Response: "Message": { "Message": { "Version": 42, - "To": "t01234", - "From": "t01234", + "To": "f01234", + "From": "f01234", "Nonce": 42, "Value": "0", "GasLimit": 9, @@ -1937,11 +1937,11 @@ Perms: sign Inputs: ```json [ - "t01234", - "t01234", + "f01234", + "f01234", 42, - "t01234", - "t01234", + "f01234", + "f01234", true ] ``` @@ -1964,10 +1964,10 @@ Perms: sign Inputs: ```json [ - "t01234", - "t01234", + "f01234", + "f01234", 42, - "t01234", + "f01234", true ] ``` @@ -1990,9 +1990,9 @@ Perms: sign Inputs: ```json [ - "t01234", - "t01234", - "t01234", + "f01234", + "f01234", + "f01234", true ] ``` @@ -2015,12 +2015,12 @@ Perms: sign Inputs: ```json [ - "t01234", + "f01234", 42, - "t01234", - "t01234", + "f01234", + "f01234", "0", - "t01234", + "f01234", 42, "Ynl0ZSBhcnJheQ==" ] @@ -2044,11 +2044,11 @@ Perms: sign Inputs: ```json [ - "t01234", + "f01234", 42, - "t01234", + "f01234", "0", - "t01234", + "f01234", 42, "Ynl0ZSBhcnJheQ==" ] @@ -2076,7 +2076,7 @@ Inputs: null, 10101, "0", - "t01234", + "f01234", "0" ] ``` @@ -2097,7 +2097,7 @@ Perms: read Inputs: ```json [ - "t01234", + "f01234", [ { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" @@ -2121,7 +2121,7 @@ Perms: read Inputs: ```json [ - "t01234", + "f01234", [ { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" @@ -2154,10 +2154,10 @@ Perms: sign Inputs: ```json [ - "t01234", - "t01234", + "f01234", + "f01234", "0", - "t01234", + "f01234", 42, "Ynl0ZSBhcnJheQ==" ] @@ -2181,12 +2181,12 @@ Perms: sign Inputs: ```json [ - "t01234", - "t01234", + "f01234", + "f01234", 42, - "t01234", - "t01234", - "t01234" + "f01234", + "f01234", + "f01234" ] ``` @@ -2208,11 +2208,11 @@ Perms: sign Inputs: ```json [ - "t01234", - "t01234", + "f01234", + "f01234", 42, - "t01234", - "t01234" + "f01234", + "f01234" ] ``` @@ -2234,10 +2234,10 @@ Perms: sign Inputs: ```json [ - "t01234", - "t01234", - "t01234", - "t01234" + "f01234", + "f01234", + "f01234", + "f01234" ] ``` @@ -2445,7 +2445,7 @@ Perms: sign Inputs: ```json [ - "t01234" + "f01234" ] ``` @@ -2459,7 +2459,7 @@ Perms: sign Inputs: ```json [ - "t01234" + "f01234" ] ``` @@ -2467,8 +2467,8 @@ Response: ```json { "Channel": "\u003cempty\u003e", - "From": "t01234", - "To": "t01234", + "From": "f01234", + "To": "f01234", "ConfirmedAmt": "0", "PendingAmt": "0", "PendingWaitSentinel": null, @@ -2485,8 +2485,8 @@ Perms: sign Inputs: ```json [ - "t01234", - "t01234" + "f01234", + "f01234" ] ``` @@ -2494,8 +2494,8 @@ Response: ```json { "Channel": "\u003cempty\u003e", - "From": "t01234", - "To": "t01234", + "From": "f01234", + "To": "f01234", "ConfirmedAmt": "0", "PendingAmt": "0", "PendingWaitSentinel": null, @@ -2512,7 +2512,7 @@ Perms: sign Inputs: ```json [ - "t01234" + "f01234" ] ``` @@ -2531,8 +2531,8 @@ Perms: sign Inputs: ```json [ - "t01234", - "t01234", + "f01234", + "f01234", "0" ] ``` @@ -2540,7 +2540,7 @@ Inputs: Response: ```json { - "Channel": "t01234", + "Channel": "f01234", "WaitSentinel": { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" } @@ -2561,7 +2561,7 @@ Inputs: ] ``` -Response: `"t01234"` +Response: `"f01234"` ### PaychList There are not yet any comments for this method. @@ -2580,8 +2580,8 @@ Perms: sign Inputs: ```json [ - "t01234", - "t01234", + "f01234", + "f01234", null ] ``` @@ -2589,7 +2589,7 @@ Inputs: Response: ```json { - "Channel": "t01234", + "Channel": "f01234", "WaitSentinel": { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" }, @@ -2605,7 +2605,7 @@ Perms: sign Inputs: ```json [ - "t01234" + "f01234" ] ``` @@ -2624,14 +2624,14 @@ Perms: read Inputs: ```json [ - "t01234" + "f01234" ] ``` Response: ```json { - "ControlAddr": "t01234", + "ControlAddr": "f01234", "Direction": 1 } ``` @@ -2644,14 +2644,14 @@ Perms: write Inputs: ```json [ - "t01234", + "f01234", { - "ChannelAddr": "t01234", + "ChannelAddr": "f01234", "TimeLockMin": 10101, "TimeLockMax": 10101, "SecretPreimage": "Ynl0ZSBhcnJheQ==", "Extra": { - "Actor": "t01234", + "Actor": "f01234", "Method": 1, "Data": "Ynl0ZSBhcnJheQ==" }, @@ -2680,14 +2680,14 @@ Perms: read Inputs: ```json [ - "t01234", + "f01234", { - "ChannelAddr": "t01234", + "ChannelAddr": "f01234", "TimeLockMin": 10101, "TimeLockMax": 10101, "SecretPreimage": "Ynl0ZSBhcnJheQ==", "Extra": { - "Actor": "t01234", + "Actor": "f01234", "Method": 1, "Data": "Ynl0ZSBhcnJheQ==" }, @@ -2716,14 +2716,14 @@ Perms: read Inputs: ```json [ - "t01234", + "f01234", { - "ChannelAddr": "t01234", + "ChannelAddr": "f01234", "TimeLockMin": 10101, "TimeLockMax": 10101, "SecretPreimage": "Ynl0ZSBhcnJheQ==", "Extra": { - "Actor": "t01234", + "Actor": "f01234", "Method": 1, "Data": "Ynl0ZSBhcnJheQ==" }, @@ -2750,7 +2750,7 @@ Perms: sign Inputs: ```json [ - "t01234", + "f01234", "0", 42 ] @@ -2760,12 +2760,12 @@ Response: ```json { "Voucher": { - "ChannelAddr": "t01234", + "ChannelAddr": "f01234", "TimeLockMin": 10101, "TimeLockMax": 10101, "SecretPreimage": "Ynl0ZSBhcnJheQ==", "Extra": { - "Actor": "t01234", + "Actor": "f01234", "Method": 1, "Data": "Ynl0ZSBhcnJheQ==" }, @@ -2791,7 +2791,7 @@ Perms: write Inputs: ```json [ - "t01234" + "f01234" ] ``` @@ -2805,14 +2805,14 @@ Perms: sign Inputs: ```json [ - "t01234", + "f01234", { - "ChannelAddr": "t01234", + "ChannelAddr": "f01234", "TimeLockMin": 10101, "TimeLockMax": 10101, "SecretPreimage": "Ynl0ZSBhcnJheQ==", "Extra": { - "Actor": "t01234", + "Actor": "f01234", "Method": 1, "Data": "Ynl0ZSBhcnJheQ==" }, @@ -2853,7 +2853,7 @@ Perms: read Inputs: ```json [ - "t01234", + "f01234", [ { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" @@ -2865,7 +2865,7 @@ Inputs: ] ``` -Response: `"t01234"` +Response: `"f01234"` ### StateAllMinerFaults StateAllMinerFaults returns all non-expired Faults that occur within lookback epochs of the given tipset @@ -2901,8 +2901,8 @@ Inputs: [ { "Version": 42, - "To": "t01234", - "From": "t01234", + "To": "f01234", + "From": "f01234", "Nonce": 42, "Value": "0", "GasLimit": 9, @@ -2927,8 +2927,8 @@ Response: { "Msg": { "Version": 42, - "To": "t01234", - "From": "t01234", + "To": "f01234", + "From": "f01234", "Nonce": 42, "Value": "0", "GasLimit": 9, @@ -2945,8 +2945,8 @@ Response: "ExecutionTrace": { "Msg": { "Version": 42, - "To": "t01234", - "From": "t01234", + "To": "f01234", + "From": "f01234", "Nonce": 42, "Value": "0", "GasLimit": 9, @@ -3109,7 +3109,7 @@ Perms: read Inputs: ```json [ - "t01234", + "f01234", [ { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" @@ -3200,8 +3200,8 @@ Inputs: [ { "Version": 42, - "To": "t01234", - "From": "t01234", + "To": "f01234", + "From": "f01234", "Nonce": 42, "Value": "0", "GasLimit": 9, @@ -3255,7 +3255,7 @@ Perms: read Inputs: ```json [ - "t01234", + "f01234", [ { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" @@ -3267,7 +3267,7 @@ Inputs: ] ``` -Response: `"t01234"` +Response: `"f01234"` ### StateMarketBalance StateMarketBalance looks up the Escrow and Locked balances of the given address in the Storage Market @@ -3278,7 +3278,7 @@ Perms: read Inputs: ```json [ - "t01234", + "f01234", [ { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" @@ -3328,8 +3328,8 @@ Response: }, "PieceSize": 1032, "VerifiedDeal": true, - "Client": "t01234", - "Provider": "t01234", + "Client": "f01234", + "Provider": "f01234", "Label": "string value", "StartEpoch": 10101, "EndEpoch": 10101, @@ -3406,8 +3406,8 @@ Response: }, "PieceSize": 1032, "VerifiedDeal": true, - "Client": "t01234", - "Provider": "t01234", + "Client": "f01234", + "Provider": "f01234", "Label": "string value", "StartEpoch": 10101, "EndEpoch": 10101, @@ -3432,7 +3432,7 @@ Perms: read Inputs: ```json [ - "t01234", + "f01234", [ { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" @@ -3455,7 +3455,7 @@ Perms: read Inputs: ```json [ - "t01234", + "f01234", [ { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" @@ -3478,7 +3478,7 @@ Perms: read Inputs: ```json [ - "t01234", + "f01234", [ { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" @@ -3501,7 +3501,7 @@ Perms: read Inputs: ```json [ - "t01234", + "f01234", [ { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" @@ -3530,7 +3530,7 @@ Perms: read Inputs: ```json [ - "t01234", + "f01234", [ { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" @@ -3545,9 +3545,9 @@ Inputs: Response: ```json { - "Owner": "t01234", - "Worker": "t01234", - "NewWorker": "t01234", + "Owner": "f01234", + "Worker": "f01234", + "NewWorker": "f01234", "ControlAddresses": null, "WorkerChangeEpoch": 10101, "PeerId": "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf", @@ -3567,7 +3567,7 @@ Perms: read Inputs: ```json [ - "t01234", + "f01234", { "SealProof": 3, "SectorNumber": 9, @@ -3604,7 +3604,7 @@ Perms: read Inputs: ```json [ - "t01234", + "f01234", 42, [ { @@ -3628,7 +3628,7 @@ Perms: read Inputs: ```json [ - "t01234", + "f01234", [ { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" @@ -3664,7 +3664,7 @@ Perms: read Inputs: ```json [ - "t01234", + "f01234", { "SealProof": 3, "SectorNumber": 9, @@ -3702,7 +3702,7 @@ Perms: read Inputs: ```json [ - "t01234", + "f01234", [ { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" @@ -3741,7 +3741,7 @@ Perms: read Inputs: ```json [ - "t01234", + "f01234", [ { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" @@ -3770,7 +3770,7 @@ Perms: read Inputs: ```json [ - "t01234", + "f01234", [ { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" @@ -3800,7 +3800,7 @@ Perms: read Inputs: ```json [ - "t01234", + "f01234", [ 0 ], @@ -3897,7 +3897,7 @@ Perms: read Inputs: ```json [ - "t01234", + "f01234", [ { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" @@ -3945,8 +3945,8 @@ Response: { "Msg": { "Version": 42, - "To": "t01234", - "From": "t01234", + "To": "f01234", + "From": "f01234", "Nonce": 42, "Value": "0", "GasLimit": 9, @@ -3963,8 +3963,8 @@ Response: "ExecutionTrace": { "Msg": { "Version": 42, - "To": "t01234", - "From": "t01234", + "To": "f01234", + "From": "f01234", "Nonce": 42, "Value": "0", "GasLimit": 9, @@ -4036,7 +4036,7 @@ Perms: read Inputs: ```json [ - "t01234", + "f01234", 9, [ { @@ -4068,7 +4068,7 @@ Perms: read Inputs: ```json [ - "t01234", + "f01234", 9, [ { @@ -4109,7 +4109,7 @@ Perms: read Inputs: ```json [ - "t01234", + "f01234", 9, [ { @@ -4139,7 +4139,7 @@ Perms: read Inputs: ```json [ - "t01234", + "f01234", 9, [ { @@ -4187,7 +4187,7 @@ Perms: read Inputs: ```json [ - "t01234", + "f01234", [ { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" @@ -4221,7 +4221,7 @@ Inputs: ] ``` -Response: `"t01234"` +Response: `"f01234"` ### StateVerifierStatus StateVerifierStatus returns the data cap for the given address. @@ -4234,7 +4234,7 @@ Perms: read Inputs: ```json [ - "t01234", + "f01234", [ { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" @@ -4346,7 +4346,7 @@ Inputs: `null` Response: ```json { - "Miner": "t01234", + "Miner": "f01234", "Ticket": { "VRFProof": "Ynl0ZSBhcnJheQ==" }, @@ -4428,7 +4428,7 @@ Inputs: [ { "Header": { - "Miner": "t01234", + "Miner": "f01234", "Ticket": { "VRFProof": "Ynl0ZSBhcnJheQ==" }, @@ -4521,7 +4521,7 @@ Perms: read Inputs: ```json [ - "t01234" + "f01234" ] ``` @@ -4535,7 +4535,7 @@ Perms: write Inputs: `null` -Response: `"t01234"` +Response: `"f01234"` ### WalletDelete WalletDelete deletes an address from the wallet. @@ -4546,7 +4546,7 @@ Perms: write Inputs: ```json [ - "t01234" + "f01234" ] ``` @@ -4561,7 +4561,7 @@ Perms: admin Inputs: ```json [ - "t01234" + "f01234" ] ``` @@ -4582,7 +4582,7 @@ Perms: write Inputs: ```json [ - "t01234" + "f01234" ] ``` @@ -4604,7 +4604,7 @@ Inputs: ] ``` -Response: `"t01234"` +Response: `"f01234"` ### WalletList WalletList lists all the addresses in the wallet. @@ -4629,7 +4629,7 @@ Inputs: ] ``` -Response: `"t01234"` +Response: `"f01234"` ### WalletSetDefault WalletSetDefault marks the given address as as the default one. @@ -4640,7 +4640,7 @@ Perms: admin Inputs: ```json [ - "t01234" + "f01234" ] ``` @@ -4655,7 +4655,7 @@ Perms: sign Inputs: ```json [ - "t01234", + "f01234", "Ynl0ZSBhcnJheQ==" ] ``` @@ -4677,11 +4677,11 @@ Perms: sign Inputs: ```json [ - "t01234", + "f01234", { "Version": 42, - "To": "t01234", - "From": "t01234", + "To": "f01234", + "From": "f01234", "Nonce": 42, "Value": "0", "GasLimit": 9, @@ -4698,8 +4698,8 @@ Response: { "Message": { "Version": 42, - "To": "t01234", - "From": "t01234", + "To": "f01234", + "From": "f01234", "Nonce": 42, "Value": "0", "GasLimit": 9, @@ -4728,7 +4728,7 @@ Inputs: ] ``` -Response: `"t01234"` +Response: `"f01234"` ### WalletVerify WalletVerify takes an address, a signature, and some bytes, and indicates whether the signature is valid. @@ -4740,7 +4740,7 @@ Perms: read Inputs: ```json [ - "t01234", + "f01234", "Ynl0ZSBhcnJheQ==", { "Type": 2, From b5907c665bfb8f7b2e9adc984cdc7944fba50e69 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Tue, 6 Oct 2020 03:43:20 -0400 Subject: [PATCH 552/885] Fix test --- paychmgr/store_test.go | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/paychmgr/store_test.go b/paychmgr/store_test.go index 36ff7a5b0..1ec8895fa 100644 --- a/paychmgr/store_test.go +++ b/paychmgr/store_test.go @@ -53,8 +53,12 @@ func TestStore(t *testing.T) { addrs, err = store.ListChannels() require.NoError(t, err) require.Len(t, addrs, 2) - require.Contains(t, addrsStrings(addrs), "t0100") - require.Contains(t, addrsStrings(addrs), "t0200") + t0100, err := address.NewIDAddress(100) + require.NoError(t, err) + t0200, err := address.NewIDAddress(200) + require.NoError(t, err) + require.Contains(t, addrs, t0100) + require.Contains(t, addrs, t0200) // Request vouchers for channel vouchers, err := store.VouchersForPaych(*ci.Channel) @@ -79,11 +83,3 @@ func TestStore(t *testing.T) { _, err = store.AllocateLane(tutils.NewIDAddr(t, 300)) require.Equal(t, err, ErrChannelNotTracked) } - -func addrsStrings(addrs []address.Address) []string { - str := make([]string, len(addrs)) - for i, a := range addrs { - str[i] = a.String() - } - return str -} From f8c886a611b83091f6d4b699493fb351ff27b6c3 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Tue, 6 Oct 2020 03:49:11 -0400 Subject: [PATCH 553/885] Rename MiningBaseInfo.HasMinPower to EligibleForMining --- api/api_full.go | 16 ++++++++-------- chain/stmgr/utils.go | 16 ++++++++-------- miner/miner.go | 2 +- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 5cbdde8e3..07b48a7eb 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -806,14 +806,14 @@ type CirculatingSupply struct { } type MiningBaseInfo struct { - MinerPower types.BigInt - NetworkPower types.BigInt - Sectors []builtin.SectorInfo - WorkerKey address.Address - SectorSize abi.SectorSize - PrevBeaconEntry types.BeaconEntry - BeaconEntries []types.BeaconEntry - HasMinPower bool + MinerPower types.BigInt + NetworkPower types.BigInt + Sectors []builtin.SectorInfo + WorkerKey address.Address + SectorSize abi.SectorSize + PrevBeaconEntry types.BeaconEntry + BeaconEntries []types.BeaconEntry + EligibleForMining bool } type BlockTemplate struct { diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index 98f6bc5ac..6b43303f0 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -506,14 +506,14 @@ func MinerGetBaseInfo(ctx context.Context, sm *StateManager, bcs beacon.Schedule } return &api.MiningBaseInfo{ - MinerPower: mpow.QualityAdjPower, - NetworkPower: tpow.QualityAdjPower, - Sectors: sectors, - WorkerKey: worker, - SectorSize: info.SectorSize, - PrevBeaconEntry: *prev, - BeaconEntries: entries, - HasMinPower: hmp, + MinerPower: mpow.QualityAdjPower, + NetworkPower: tpow.QualityAdjPower, + Sectors: sectors, + WorkerKey: worker, + SectorSize: info.SectorSize, + PrevBeaconEntry: *prev, + BeaconEntries: entries, + EligibleForMining: hmp, }, nil } diff --git a/miner/miner.go b/miner/miner.go index d4e7b2317..a256e8b3a 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -362,7 +362,7 @@ func (m *Miner) mineOne(ctx context.Context, base *MiningBase) (*types.BlockMsg, if mbi == nil { return nil, nil } - if !mbi.HasMinPower { + if !mbi.EligibleForMining { // slashed or just have no power yet return nil, nil } From 4a9155ba854d05f8acccde58ddb059543be7fdc5 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Tue, 6 Oct 2020 04:33:55 -0400 Subject: [PATCH 554/885] Use new MinerEligibleToMine method post-v2 actors --- chain/actors/builtin/miner/miner.go | 2 + chain/actors/builtin/miner/v0.go | 7 +++ chain/actors/builtin/miner/v2.go | 5 ++ chain/actors/policy/policy.go | 1 - chain/stmgr/utils.go | 80 +++++++++++++++++++++++++++-- chain/sub/incoming.go | 6 +-- chain/sync.go | 6 +-- 7 files changed, 97 insertions(+), 10 deletions(-) diff --git a/chain/actors/builtin/miner/miner.go b/chain/actors/builtin/miner/miner.go index 125d142d3..8649d4351 100644 --- a/chain/actors/builtin/miner/miner.go +++ b/chain/actors/builtin/miner/miner.go @@ -58,6 +58,7 @@ type State interface { VestedFunds(abi.ChainEpoch) (abi.TokenAmount, error) // Funds locked for various reasons. LockedFunds() (LockedFunds, error) + FeeDebt() (abi.TokenAmount, error) GetSector(abi.SectorNumber) (*SectorOnChainInfo, error) FindSector(abi.SectorNumber) (*SectorLocation, error) @@ -144,6 +145,7 @@ type MinerInfo struct { SealProofType abi.RegisteredSealProof SectorSize abi.SectorSize WindowPoStPartitionSectors uint64 + ConsensusFaultElapsed abi.ChainEpoch } type SectorExpiration struct { diff --git a/chain/actors/builtin/miner/v0.go b/chain/actors/builtin/miner/v0.go index 902ed20f2..7e71c7611 100644 --- a/chain/actors/builtin/miner/v0.go +++ b/chain/actors/builtin/miner/v0.go @@ -4,6 +4,8 @@ import ( "bytes" "errors" + "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-bitfield" "github.com/filecoin-project/go-state-types/abi" @@ -61,6 +63,10 @@ func (s *state0) LockedFunds() (LockedFunds, error) { }, nil } +func (s *state0) FeeDebt() (abi.TokenAmount, error) { + return big.Zero(), nil +} + func (s *state0) InitialPledge() (abi.TokenAmount, error) { return s.State.InitialPledgeRequirement, nil } @@ -287,6 +293,7 @@ func (s *state0) Info() (MinerInfo, error) { SealProofType: info.SealProofType, SectorSize: info.SectorSize, WindowPoStPartitionSectors: info.WindowPoStPartitionSectors, + ConsensusFaultElapsed: -1, } if info.PendingWorkerKey != nil { diff --git a/chain/actors/builtin/miner/v2.go b/chain/actors/builtin/miner/v2.go index 934554d23..e2b53e9a6 100644 --- a/chain/actors/builtin/miner/v2.go +++ b/chain/actors/builtin/miner/v2.go @@ -61,6 +61,10 @@ func (s *state2) LockedFunds() (LockedFunds, error) { }, nil } +func (s *state2) FeeDebt() (abi.TokenAmount, error) { + return s.State.FeeDebt, nil +} + func (s *state2) InitialPledge() (abi.TokenAmount, error) { return s.State.InitialPledge, nil } @@ -288,6 +292,7 @@ func (s *state2) Info() (MinerInfo, error) { SealProofType: info.SealProofType, SectorSize: info.SectorSize, WindowPoStPartitionSectors: info.WindowPoStPartitionSectors, + ConsensusFaultElapsed: info.ConsensusFaultElapsed, } if info.PendingWorkerKey != nil { diff --git a/chain/actors/policy/policy.go b/chain/actors/policy/policy.go index f29da9802..e7f213687 100644 --- a/chain/actors/policy/policy.go +++ b/chain/actors/policy/policy.go @@ -4,7 +4,6 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/chain/actors" - market0 "github.com/filecoin-project/specs-actors/actors/builtin/market" miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" power0 "github.com/filecoin-project/specs-actors/actors/builtin/power" diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index 6b43303f0..f2e2bfd19 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -9,6 +9,10 @@ import ( "runtime" "strings" + "github.com/filecoin-project/go-state-types/big" + + "github.com/filecoin-project/go-state-types/network" + cid "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" @@ -490,7 +494,7 @@ func MinerGetBaseInfo(ctx context.Context, sm *StateManager, bcs beacon.Schedule return nil, nil } - mpow, tpow, hmp, err := GetPowerRaw(ctx, sm, lbst, maddr) + mpow, tpow, _, err := GetPowerRaw(ctx, sm, lbst, maddr) if err != nil { return nil, xerrors.Errorf("failed to get power: %w", err) } @@ -505,6 +509,12 @@ func MinerGetBaseInfo(ctx context.Context, sm *StateManager, bcs beacon.Schedule return nil, xerrors.Errorf("resolving worker address: %w", err) } + // TODO: Not ideal performance...This method reloads miner and power state (already looked up here and in GetPowerRaw) + eligible, err := MinerEligibleToMine(ctx, sm, maddr, ts, lbts) + if err != nil { + return nil, xerrors.Errorf("determining miner eligibility: %w", err) + } + return &api.MiningBaseInfo{ MinerPower: mpow.QualityAdjPower, NetworkPower: tpow.QualityAdjPower, @@ -513,7 +523,7 @@ func MinerGetBaseInfo(ctx context.Context, sm *StateManager, bcs beacon.Schedule SectorSize: info.SectorSize, PrevBeaconEntry: *prev, BeaconEntries: entries, - EligibleForMining: hmp, + EligibleForMining: eligible, }, nil } @@ -595,7 +605,7 @@ func GetReturnType(ctx context.Context, sm *StateManager, to address.Address, me return reflect.New(m.Ret.Elem()).Interface().(cbg.CBORUnmarshaler), nil } -func MinerHasMinPower(ctx context.Context, sm *StateManager, addr address.Address, ts *types.TipSet) (bool, error) { +func minerHasMinPower(ctx context.Context, sm *StateManager, addr address.Address, ts *types.TipSet) (bool, error) { pact, err := sm.LoadActor(ctx, power.Address, ts) if err != nil { return false, xerrors.Errorf("loading power actor state: %w", err) @@ -609,6 +619,70 @@ func MinerHasMinPower(ctx context.Context, sm *StateManager, addr address.Addres return ps.MinerNominalPowerMeetsConsensusMinimum(addr) } +func MinerEligibleToMine(ctx context.Context, sm *StateManager, addr address.Address, baseTs *types.TipSet, lookbackTs *types.TipSet) (bool, error) { + hmp, err := minerHasMinPower(ctx, sm, addr, lookbackTs) + + // TODO: We're blurring the lines between a "runtime network version" and a "Lotus upgrade epoch", is that unavoidable? + if sm.GetNtwkVersion(ctx, baseTs.Height()) <= network.Version3 { + return hmp, err + } + + if err != nil { + return false, err + } + + if !hmp { + return false, nil + } + + // Post actors v2, also check MinerEligibleForElection with base ts + + pact, err := sm.LoadActor(ctx, power.Address, baseTs) + if err != nil { + return false, xerrors.Errorf("loading power actor state: %w", err) + } + + pstate, err := power.Load(sm.cs.Store(ctx), pact) + if err != nil { + return false, err + } + + mact, err := sm.LoadActor(ctx, addr, baseTs) + if err != nil { + return false, xerrors.Errorf("loading miner actor state: %w", err) + } + + mstate, err := miner.Load(sm.cs.Store(ctx), mact) + if err != nil { + return false, err + } + + // Non-empty power claim. + if claim, found, err := pstate.MinerPower(addr); err != nil { + return false, err + } else if !found { + return false, err + } else if claim.QualityAdjPower.LessThanEqual(big.Zero()) { + return false, err + } + + // No fee debt. + if debt, err := mstate.FeeDebt(); err != nil { + return false, err + } else if !debt.IsZero() { + return false, err + } + + // No active consensus faults. + if mInfo, err := mstate.Info(); err != nil { + return false, err + } else if baseTs.Height() <= mInfo.ConsensusFaultElapsed { + return false, nil + } + + return true, nil +} + func CheckTotalFIL(ctx context.Context, sm *StateManager, ts *types.TipSet) (abi.TokenAmount, error) { str, err := state.LoadStateTree(sm.ChainStore().Store(ctx), ts.ParentState()) if err != nil { diff --git a/chain/sub/incoming.go b/chain/sub/incoming.go index 53bc3d68e..07b3343d2 100644 --- a/chain/sub/incoming.go +++ b/chain/sub/incoming.go @@ -485,14 +485,14 @@ func (bv *BlockValidator) checkPowerAndGetWorkerKey(ctx context.Context, bh *typ return address.Undef, ErrSoftFailure } - hmp, err := stmgr.MinerHasMinPower(ctx, bv.stmgr, bh.Miner, lbts) + eligible, err := stmgr.MinerEligibleToMine(ctx, bv.stmgr, bh.Miner, baseTs, lbts) if err != nil { log.Warnf("failed to determine if incoming block's miner has minimum power: %s", err) return address.Undef, ErrSoftFailure } - if !hmp { - log.Warnf("incoming block's miner does not have minimum power") + if !eligible { + log.Warnf("incoming block's miner is ineligible") return address.Undef, ErrInsufficientPower } diff --git a/chain/sync.go b/chain/sync.go index 46d3934b1..655bb4a1d 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -827,13 +827,13 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock, use return xerrors.Errorf("block is not claiming to be a winner") } - hp, err := stmgr.MinerHasMinPower(ctx, syncer.sm, h.Miner, lbts) + eligible, err := stmgr.MinerEligibleToMine(ctx, syncer.sm, h.Miner, baseTs, lbts) if err != nil { return xerrors.Errorf("determining if miner has min power failed: %w", err) } - if !hp { - return xerrors.New("block's miner does not meet minimum power threshold") + if !eligible { + return xerrors.New("block's miner is ineligible to mine") } rBeacon := *prevBeacon From c53cef15267f8124587e313561aec12b1361af00 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Tue, 6 Oct 2020 04:47:00 -0400 Subject: [PATCH 555/885] Update docs --- documentation/en/api-methods.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index 1bae2c1e6..518d38089 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -1611,7 +1611,7 @@ Response: "Data": "Ynl0ZSBhcnJheQ==" }, "BeaconEntries": null, - "HasMinPower": true + "EligibleForMining": true } ``` @@ -3531,7 +3531,8 @@ Response: "Multiaddrs": null, "SealProofType": 3, "SectorSize": 34359738368, - "WindowPoStPartitionSectors": 42 + "WindowPoStPartitionSectors": 42, + "ConsensusFaultElapsed": 10101 } ``` From 7a80e356decf0bd36ee976eacf00b43e984fd493 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Tue, 6 Oct 2020 05:05:39 -0400 Subject: [PATCH 556/885] Add a CLI command to set a miner's owner address --- cmd/lotus-storage-miner/actor.go | 118 +++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) diff --git a/cmd/lotus-storage-miner/actor.go b/cmd/lotus-storage-miner/actor.go index ea21aa90d..fa320289e 100644 --- a/cmd/lotus-storage-miner/actor.go +++ b/cmd/lotus-storage-miner/actor.go @@ -5,6 +5,9 @@ import ( "os" "strings" + "github.com/filecoin-project/lotus/build" + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + "github.com/fatih/color" "github.com/libp2p/go-libp2p-core/peer" ma "github.com/multiformats/go-multiaddr" @@ -32,6 +35,7 @@ var actorCmd = &cli.Command{ actorSetAddrsCmd, actorWithdrawCmd, actorSetPeeridCmd, + actorSetOwnerCmd, actorControl, }, } @@ -478,3 +482,117 @@ var actorControlSet = &cli.Command{ return nil }, } + +var actorSetOwnerCmd = &cli.Command{ + Name: "set-owner", + Usage: "Set owner address", + ArgsUsage: "[address]", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "really-do-it", + Usage: "Actually send transaction performing the action", + Value: false, + }, + }, + Action: func(cctx *cli.Context) error { + if !cctx.Bool("really-do-it") { + fmt.Println("Pass --really-do-it to actually execute this action") + return nil + } + + if !cctx.Args().Present() { + return fmt.Errorf("must pass address of new owner address") + } + + nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return err + } + defer closer() + + api, acloser, err := lcli.GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer acloser() + + ctx := lcli.ReqContext(cctx) + + na, err := address.NewFromString(cctx.Args().First()) + if err != nil { + return err + } + + newAddr, err := api.StateLookupID(ctx, na, types.EmptyTSK) + if err != nil { + return err + } + + maddr, err := nodeApi.ActorAddress(ctx) + if err != nil { + return err + } + + mi, err := api.StateMinerInfo(ctx, maddr, types.EmptyTSK) + if err != nil { + return err + } + + sp, err := actors.SerializeParams(&newAddr) + if err != nil { + return xerrors.Errorf("serializing params: %w", err) + } + + smsg, err := api.MpoolPushMessage(ctx, &types.Message{ + From: mi.Owner, + To: maddr, + Method: builtin2.MethodsMiner.ChangeOwnerAddress, + Value: big.Zero(), + Params: sp, + }, nil) + if err != nil { + return xerrors.Errorf("mpool push: %w", err) + } + + fmt.Println("Propose Message CID:", smsg.Cid()) + + // wait for it to get mined into a block + wait, err := api.StateWaitMsg(ctx, smsg.Cid(), build.MessageConfidence) + if err != nil { + return err + } + + // check it executed successfully + if wait.Receipt.ExitCode != 0 { + fmt.Println("Propose owner change failed!") + return err + } + + smsg, err = api.MpoolPushMessage(ctx, &types.Message{ + From: newAddr, + To: maddr, + Method: builtin2.MethodsMiner.ChangeOwnerAddress, + Value: big.Zero(), + Params: sp, + }, nil) + if err != nil { + return xerrors.Errorf("mpool push: %w", err) + } + + fmt.Println("Approve Message CID:", smsg.Cid()) + + // wait for it to get mined into a block + wait, err = api.StateWaitMsg(ctx, smsg.Cid(), build.MessageConfidence) + if err != nil { + return err + } + + // check it executed successfully + if wait.Receipt.ExitCode != 0 { + fmt.Println("Approve owner change failed!") + return err + } + + return nil + }, +} From 7b556252db0597a3737a24216d495addc47b842d Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Tue, 6 Oct 2020 05:34:53 -0400 Subject: [PATCH 557/885] Use SysErrReserved1 in the event of an actors panic --- chain/vm/runtime.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/chain/vm/runtime.go b/chain/vm/runtime.go index 472017a00..52b2dcd2c 100644 --- a/chain/vm/runtime.go +++ b/chain/vm/runtime.go @@ -122,7 +122,11 @@ func (rt *Runtime) shimCall(f func() interface{}) (rval []byte, aerr aerrors.Act //log.Desugar().WithOptions(zap.AddStacktrace(zapcore.ErrorLevel)). //Sugar().Errorf("spec actors failure: %s", r) log.Errorf("spec actors failure: %s", r) - aerr = aerrors.Newf(1, "spec actors failure: %s", r) + if rt.NetworkVersion() <= network.Version3 { + aerr = aerrors.Newf(1, "spec actors failure: %s", r) + } else { + aerr = aerrors.Newf(exitcode.SysErrReserved1, "spec actors failure: %s", r) + } } }() From a179cadf0ef34e7d0638b53c05c0d8fc178839df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 6 Oct 2020 13:35:31 +0200 Subject: [PATCH 558/885] wallet: Don't output duplicate keys from WalletList --- chain/wallet/wallet.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/chain/wallet/wallet.go b/chain/wallet/wallet.go index bc9cff6e4..9438d2051 100644 --- a/chain/wallet/wallet.go +++ b/chain/wallet/wallet.go @@ -165,6 +165,7 @@ func (w *Wallet) ListAddrs() ([]address.Address, error) { sort.Strings(all) + seen := map[address.Address]struct{}{} out := make([]address.Address, 0, len(all)) for _, a := range all { if strings.HasPrefix(a, KNamePrefix) { @@ -173,10 +174,19 @@ func (w *Wallet) ListAddrs() ([]address.Address, error) { if err != nil { return nil, xerrors.Errorf("converting name to address: %w", err) } + if _, ok := seen[addr]; ok { + continue // got duplicate with a different prefix + } + seen[addr] = struct{}{} + out = append(out, addr) } } + sort.Slice(out, func(i, j int) bool { + return out[i].String() < out[j].String() + }) + return out, nil } From 15330396bba66dd18d94516a09807d3eed604298 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 6 Oct 2020 16:53:49 +0200 Subject: [PATCH 559/885] Fix circ supply before v2 upgrade epoch --- chain/stmgr/stmgr.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index b27bed587..ac01ebb61 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -1279,9 +1279,12 @@ func (sm *StateManager) GetCirculatingSupplyDetailed(ctx context.Context, height return api.CirculatingSupply{}, xerrors.Errorf("failed to calculate filVested: %w", err) } - filReserveDisbursed, err := GetFilReserveDisbursed(ctx, st) - if err != nil { - return api.CirculatingSupply{}, xerrors.Errorf("failed to calculate filReserveDisbursed: %w", err) + filReserveDisbursed := big.Zero() + if height > build.UpgradeActorsV2Height { + filReserveDisbursed, err = GetFilReserveDisbursed(ctx, st) + if err != nil { + return api.CirculatingSupply{}, xerrors.Errorf("failed to calculate filReserveDisbursed: %w", err) + } } filMined, err := GetFilMined(ctx, st) From d4cbd4fb55077ccfd96cb5619b4bca2a29fef069 Mon Sep 17 00:00:00 2001 From: frrist Date: Mon, 5 Oct 2020 16:48:29 -0700 Subject: [PATCH 560/885] feat(miner): add miner deadline diffing - logic gathering all removed, faulted, recovered, and recovering sectors for a miner. --- chain/actors/builtin/miner/diff_deadlines.go | 180 +++++++++++++++++++ 1 file changed, 180 insertions(+) create mode 100644 chain/actors/builtin/miner/diff_deadlines.go diff --git a/chain/actors/builtin/miner/diff_deadlines.go b/chain/actors/builtin/miner/diff_deadlines.go new file mode 100644 index 000000000..e1e839960 --- /dev/null +++ b/chain/actors/builtin/miner/diff_deadlines.go @@ -0,0 +1,180 @@ +package miner + +import ( + "errors" + + "github.com/filecoin-project/go-bitfield" + "github.com/filecoin-project/go-state-types/exitcode" +) + +type DeadlinesDiff map[uint64]*DeadlineDiff + +func DiffDeadlines(pre, cur State) (*DeadlinesDiff, error) { + changed, err := pre.DeadlinesChanged(cur) + if err != nil { + return nil, err + } + if !changed { + return nil, nil + } + + numDl, err := pre.NumDeadlines() + if err != nil { + return nil, err + } + dlDiff := make(DeadlinesDiff, numDl) + if err := pre.ForEachDeadline(func(idx uint64, preDl Deadline) error { + curDl, err := cur.LoadDeadline(idx) + if err != nil { + return err + } + + diff, err := DiffDeadline(preDl, curDl) + if err != nil { + return err + } + + dlDiff[idx] = diff + return nil + }); err != nil { + return nil, err + } + return &dlDiff, nil +} + +type DeadlineDiff map[uint64]*PartitionDiff + +func DiffDeadline(pre, cur Deadline) (*DeadlineDiff, error) { + changed, err := pre.PartitionsChanged(cur) + if err != nil { + return nil, err + } + if !changed { + return nil, nil + } + + partDiff := make(DeadlineDiff) + if err := pre.ForEachPartition(func(idx uint64, prePart Partition) error { + // try loading current partition at this index + curPart, err := cur.LoadPartition(idx) + if err != nil { + if errors.Is(err, exitcode.ErrNotFound) { + // TODO correctness? + return nil // the partition was removed. + } + return err + } + + // compare it with the previous partition + diff, err := DiffPartition(prePart, curPart) + if err != nil { + return err + } + + partDiff[idx] = diff + return nil + }); err != nil { + return nil, err + } + + // all previous partitions have been walked. + // all partitions in cur and not in prev are new... can they be faulty already? + // TODO is this correct? + if err := cur.ForEachPartition(func(idx uint64, curPart Partition) error { + if _, found := partDiff[idx]; found { + return nil + } + faults, err := curPart.FaultySectors() + if err != nil { + return err + } + recovering, err := curPart.RecoveringSectors() + if err != nil { + return err + } + partDiff[idx] = &PartitionDiff{ + Removed: bitfield.New(), + Recovered: bitfield.New(), + Faulted: faults, + Recovering: recovering, + } + + return nil + }); err != nil { + return nil, err + } + + return &partDiff, nil +} + +type PartitionDiff struct { + Removed bitfield.BitField + Recovered bitfield.BitField + Faulted bitfield.BitField + Recovering bitfield.BitField +} + +func DiffPartition(pre, cur Partition) (*PartitionDiff, error) { + prevLiveSectors, err := pre.LiveSectors() + if err != nil { + return nil, err + } + curLiveSectors, err := cur.LiveSectors() + if err != nil { + return nil, err + } + + removed, err := bitfield.SubtractBitField(prevLiveSectors, curLiveSectors) + if err != nil { + return nil, err + } + + prevRecoveries, err := pre.RecoveringSectors() + if err != nil { + return nil, err + } + + curRecoveries, err := cur.RecoveringSectors() + if err != nil { + return nil, err + } + + recovering, err := bitfield.SubtractBitField(curRecoveries, prevRecoveries) + if err != nil { + return nil, err + } + + prevFaults, err := pre.FaultySectors() + if err != nil { + return nil, err + } + + curFaults, err := cur.FaultySectors() + if err != nil { + return nil, err + } + + faulted, err := bitfield.SubtractBitField(curFaults, prevFaults) + if err != nil { + return nil, err + } + + // all current good sectors + curActiveSectors, err := cur.ActiveSectors() + if err != nil { + return nil, err + } + + // sectors that were previously fault and are now currently active are considered recovered. + recovered, err := bitfield.IntersectBitField(prevFaults, curActiveSectors) + if err != nil { + return nil, err + } + + return &PartitionDiff{ + Removed: removed, + Recovered: recovered, + Faulted: faulted, + Recovering: recovering, + }, nil +} From 4a6e9c920dd566f6170beb70d6c30f1891de2137 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 6 Oct 2020 11:42:04 -0700 Subject: [PATCH 561/885] set genesis name in test --- chain/gen/gen.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/chain/gen/gen.go b/chain/gen/gen.go index d05165ab1..3ebd127f3 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -9,6 +9,7 @@ import ( "time" "github.com/filecoin-project/specs-actors/actors/runtime/proof" + "github.com/google/uuid" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" @@ -224,7 +225,7 @@ func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) { }, VerifregRootKey: DefaultVerifregRootkeyActor, RemainderAccount: DefaultRemainderAccountActor, - NetworkName: "", + NetworkName: uuid.New().String(), Timestamp: uint64(build.Clock.Now().Add(-500 * time.Duration(build.BlockDelaySecs) * time.Second).Unix()), } From 256be285ba7910dd2b59c3cf828622463d971d65 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 6 Oct 2020 13:00:06 -0700 Subject: [PATCH 562/885] only enable gas tampering if the breeze upgrade is enabled --- chain/store/basefee.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chain/store/basefee.go b/chain/store/basefee.go index 45785240e..33367abcc 100644 --- a/chain/store/basefee.go +++ b/chain/store/basefee.go @@ -46,7 +46,7 @@ func ComputeNextBaseFee(baseFee types.BigInt, gasLimitUsed int64, noOfBlocks int } func (cs *ChainStore) ComputeBaseFee(ctx context.Context, ts *types.TipSet) (abi.TokenAmount, error) { - if ts.Height() > build.UpgradeBreezeHeight && ts.Height() < build.UpgradeBreezeHeight+build.BreezeGasTampingDuration { + if build.UpgradeBreezeHeight >= 0 && ts.Height() > build.UpgradeBreezeHeight && ts.Height() < build.UpgradeBreezeHeight+build.BreezeGasTampingDuration { return abi.NewTokenAmount(100), nil } From b1818a10555c2962bd241eee2af5dde55b874d10 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 6 Oct 2020 13:12:16 -0700 Subject: [PATCH 563/885] fix base fee tests for different smoke heights --- chain/store/basefee_test.go | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/chain/store/basefee_test.go b/chain/store/basefee_test.go index b4757f70e..b3d414cf5 100644 --- a/chain/store/basefee_test.go +++ b/chain/store/basefee_test.go @@ -11,24 +11,27 @@ import ( func TestBaseFee(t *testing.T) { tests := []struct { - basefee uint64 - limitUsed int64 - noOfBlocks int - output uint64 + basefee uint64 + limitUsed int64 + noOfBlocks int + preSmoke, postSmoke uint64 }{ - {100e6, 0, 1, 87.5e6}, - {100e6, 0, 5, 87.5e6}, - {100e6, build.BlockGasTarget, 1, 103.125e6}, - {100e6, build.BlockGasTarget * 2, 2, 103.125e6}, - {100e6, build.BlockGasLimit * 2, 2, 112.5e6}, - {100e6, build.BlockGasLimit * 1.5, 2, 110937500}, + {100e6, 0, 1, 87.5e6, 87.5e6}, + {100e6, 0, 5, 87.5e6, 87.5e6}, + {100e6, build.BlockGasTarget, 1, 103.125e6, 100e6}, + {100e6, build.BlockGasTarget * 2, 2, 103.125e6, 100e6}, + {100e6, build.BlockGasLimit * 2, 2, 112.5e6, 112.5e6}, + {100e6, build.BlockGasLimit * 1.5, 2, 110937500, 106.250e6}, } for _, test := range tests { test := test t.Run(fmt.Sprintf("%v", test), func(t *testing.T) { - output := ComputeNextBaseFee(types.NewInt(test.basefee), test.limitUsed, test.noOfBlocks, 0) - assert.Equal(t, fmt.Sprintf("%d", test.output), output.String()) + preSmoke := ComputeNextBaseFee(types.NewInt(test.basefee), test.limitUsed, test.noOfBlocks, build.UpgradeSmokeHeight-1) + assert.Equal(t, fmt.Sprintf("%d", test.preSmoke), preSmoke.String()) + + postSmoke := ComputeNextBaseFee(types.NewInt(test.basefee), test.limitUsed, test.noOfBlocks, build.UpgradeSmokeHeight+1) + assert.Equal(t, fmt.Sprintf("%d", test.postSmoke), postSmoke.String()) }) } } From 4a3081c77a99fc1d6ec989545ad4d55b532ab465 Mon Sep 17 00:00:00 2001 From: Travis Person Date: Tue, 6 Oct 2020 20:13:48 +0000 Subject: [PATCH 564/885] lotus-pcr: limit refunds to properly priced messages --- cmd/lotus-pcr/main.go | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/cmd/lotus-pcr/main.go b/cmd/lotus-pcr/main.go index 4ce5bbb9f..f446b2c9d 100644 --- a/cmd/lotus-pcr/main.go +++ b/cmd/lotus-pcr/main.go @@ -376,6 +376,18 @@ var runCmd = &cli.Command{ Usage: "percent of refund to issue", Value: 110, }, + &cli.StringFlag{ + Name: "pre-fee-cap-max", + EnvVars: []string{"LOTUS_PCR_PRE_FEE_CAP_MAX"}, + Usage: "messages with a fee cap larger than this will be skipped when processing pre commit messages", + Value: "0.0000000001", + }, + &cli.StringFlag{ + Name: "prove-fee-cap-max", + EnvVars: []string{"LOTUS_PCR_PROVE_FEE_CAP_MAX"}, + Usage: "messages with a prove cap larger than this will be skipped when processing pre commit messages", + Value: "0.0000000001", + }, }, Action: func(cctx *cli.Context) error { go func() { @@ -426,6 +438,16 @@ var runCmd = &cli.Command{ minerRecoveryCutoff := uint64(cctx.Int("miner-recovery-cutoff")) minerRecoveryBonus := uint64(cctx.Int("miner-recovery-bonus")) + preFeeCapMax, err := types.ParseFIL(cctx.String("pre-fee-cap-max")) + if err != nil { + return err + } + + proveFeeCapMax, err := types.ParseFIL(cctx.String("prove-fee-cap-max")) + if err != nil { + return err + } + rf := &refunder{ api: api, wallet: from, @@ -436,6 +458,8 @@ var runCmd = &cli.Command{ dryRun: dryRun, preCommitEnabled: preCommitEnabled, proveCommitEnabled: proveCommitEnabled, + preFeeCapMax: types.BigInt(preFeeCapMax), + proveFeeCapMax: types.BigInt(proveFeeCapMax), } var refunds *MinersRefund = NewMinersRefund() @@ -588,6 +612,9 @@ type refunder struct { preCommitEnabled bool proveCommitEnabled bool threshold big.Int + + preFeeCapMax big.Int + proveFeeCapMax big.Int } func (r *refunder) FindMiners(ctx context.Context, tipset *types.TipSet, refunds *MinersRefund, owner, worker, control bool) (*MinersRefund, error) { @@ -869,6 +896,11 @@ func (r *refunder) ProcessTipset(ctx context.Context, tipset *types.TipSet, refu continue } + if m.GasFeeCap.GreaterThan(r.proveFeeCapMax) { + log.Debugw("skipping high fee cap message", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "gas_fee_cap", m.GasFeeCap, "fee_cap_max", r.proveFeeCapMax) + continue + } + var sn abi.SectorNumber var proveCommitSector miner0.ProveCommitSectorParams @@ -916,6 +948,11 @@ func (r *refunder) ProcessTipset(ctx context.Context, tipset *types.TipSet, refu continue } + if m.GasFeeCap.GreaterThan(r.preFeeCapMax) { + log.Debugw("skipping high fee cap message", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "gas_fee_cap", m.GasFeeCap, "fee_cap_max", r.preFeeCapMax) + continue + } + var precommitInfo miner.SectorPreCommitInfo if err := precommitInfo.UnmarshalCBOR(bytes.NewBuffer(m.Params)); err != nil { log.Warnw("failed to decode precommit params", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To) From b71b7434200c56f40ace3ce22314171f3e1a0f18 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Sun, 6 Sep 2020 03:48:45 -0400 Subject: [PATCH 565/885] Bugfix: Runtime's Receiver() should only return ID addresses --- chain/vm/mkactor.go | 18 +++++++++--------- chain/vm/vm.go | 12 +++++++++++- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/chain/vm/mkactor.go b/chain/vm/mkactor.go index 33884368f..22a2acb8b 100644 --- a/chain/vm/mkactor.go +++ b/chain/vm/mkactor.go @@ -31,41 +31,41 @@ func init() { var EmptyObjectCid cid.Cid // TryCreateAccountActor creates account actors from only BLS/SECP256K1 addresses. -func TryCreateAccountActor(rt *Runtime, addr address.Address) (*types.Actor, aerrors.ActorError) { +func TryCreateAccountActor(rt *Runtime, addr address.Address) (*types.Actor, address.Address, aerrors.ActorError) { if err := rt.chargeGasSafe(PricelistByEpoch(rt.height).OnCreateActor()); err != nil { - return nil, err + return nil, address.Undef, err } addrID, err := rt.state.RegisterNewAddress(addr) if err != nil { - return nil, aerrors.Escalate(err, "registering actor address") + return nil, address.Undef, aerrors.Escalate(err, "registering actor address") } act, aerr := makeActor(actors.VersionForNetwork(rt.NetworkVersion()), addr) if aerr != nil { - return nil, aerr + return nil, address.Undef, aerr } if err := rt.state.SetActor(addrID, act); err != nil { - return nil, aerrors.Escalate(err, "creating new actor failed") + return nil, address.Undef, aerrors.Escalate(err, "creating new actor failed") } p, err := actors.SerializeParams(&addr) if err != nil { - return nil, aerrors.Escalate(err, "couldn't serialize params for actor construction") + return nil, address.Undef, aerrors.Escalate(err, "couldn't serialize params for actor construction") } // call constructor on account _, aerr = rt.internalSend(builtin0.SystemActorAddr, addrID, builtin0.MethodsAccount.Constructor, big.Zero(), p) if aerr != nil { - return nil, aerrors.Wrap(aerr, "failed to invoke account constructor") + return nil, address.Undef, aerrors.Wrap(aerr, "failed to invoke account constructor") } act, err = rt.state.GetActor(addrID) if err != nil { - return nil, aerrors.Escalate(err, "loading newly created actor failed") + return nil, address.Undef, aerrors.Escalate(err, "loading newly created actor failed") } - return act, nil + return act, addrID, nil } func makeActor(ver actors.Version, addr address.Address) (*types.Actor, aerrors.ActorError) { diff --git a/chain/vm/vm.go b/chain/vm/vm.go index 1361c6fbe..aec356fb4 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -131,6 +131,9 @@ func (vm *VM) makeRuntime(ctx context.Context, msg *types.Message, origin addres rt.Abortf(exitcode.SysErrInvalidReceiver, "resolve msg.From address failed") } vmm.From = resF + resT, _ := rt.ResolveAddress(msg.To) + // may be set to undef if recipient doesn't exist yet + vmm.To = resT rt.Message = vmm return rt @@ -257,11 +260,18 @@ func (vm *VM) send(ctx context.Context, msg *types.Message, parent *Runtime, toActor, err := st.GetActor(msg.To) if err != nil { if xerrors.Is(err, types.ErrActorNotFound) { - a, err := TryCreateAccountActor(rt, msg.To) + a, aid, err := TryCreateAccountActor(rt, msg.To) if err != nil { return nil, aerrors.Wrapf(err, "could not create account") } toActor = a + nmsg := types.Message{ + To: aid, + From: rt.vmsg.Caller(), + Value: rt.vmsg.ValueReceived(), + } + + rt.vmsg = &nmsg } else { return nil, aerrors.Escalate(err, "getting actor") } From 757aa1039fe65202cfedd6c3c6819b41bc2d0473 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Wed, 23 Sep 2020 22:45:53 -0400 Subject: [PATCH 566/885] Panic if runtime's Message has non-ID caller or receiver --- chain/vm/runtime.go | 24 +++++++++++++++++++++++- chain/vm/vm.go | 14 ++++++++------ 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/chain/vm/runtime.go b/chain/vm/runtime.go index 472017a00..4c6bf1848 100644 --- a/chain/vm/runtime.go +++ b/chain/vm/runtime.go @@ -27,8 +27,30 @@ import ( "github.com/filecoin-project/lotus/chain/types" ) +type Message struct { + msg types.Message +} + +func (m *Message) Caller() address.Address { + if m.msg.From.Protocol() != address.ID { + panic("runtime message has a non-ID caller") + } + return m.msg.From +} + +func (m *Message) Receiver() address.Address { + if m.msg.To != address.Undef && m.msg.To.Protocol() != address.ID { + panic("runtime message has a non-ID receiver") + } + return m.msg.To +} + +func (m *Message) ValueReceived() abi.TokenAmount { + return m.msg.Value +} + type Runtime struct { - types.Message + Message rt0.Syscalls ctx context.Context diff --git a/chain/vm/vm.go b/chain/vm/vm.go index aec356fb4..a5072f8c5 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -134,7 +134,7 @@ func (vm *VM) makeRuntime(ctx context.Context, msg *types.Message, origin addres resT, _ := rt.ResolveAddress(msg.To) // may be set to undef if recipient doesn't exist yet vmm.To = resT - rt.Message = vmm + rt.Message = Message{msg: vmm} return rt } @@ -265,13 +265,15 @@ func (vm *VM) send(ctx context.Context, msg *types.Message, parent *Runtime, return nil, aerrors.Wrapf(err, "could not create account") } toActor = a - nmsg := types.Message{ - To: aid, - From: rt.vmsg.Caller(), - Value: rt.vmsg.ValueReceived(), + nmsg := Message{ + msg: types.Message{ + To: aid, + From: rt.Message.Caller(), + Value: rt.Message.ValueReceived(), + }, } - rt.vmsg = &nmsg + rt.Message = nmsg } else { return nil, aerrors.Escalate(err, "getting actor") } From d9fad5fad09764504320e6aa82a7030b42b26a18 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Tue, 6 Oct 2020 01:25:55 -0400 Subject: [PATCH 567/885] Move rt.Message fixes behind fork logic --- chain/vm/runtime.go | 2 +- chain/vm/vm.go | 33 +++++++++++++++++++++------------ 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/chain/vm/runtime.go b/chain/vm/runtime.go index 4c6bf1848..530adfe95 100644 --- a/chain/vm/runtime.go +++ b/chain/vm/runtime.go @@ -50,7 +50,7 @@ func (m *Message) ValueReceived() abi.TokenAmount { } type Runtime struct { - Message + rt0.Message rt0.Syscalls ctx context.Context diff --git a/chain/vm/vm.go b/chain/vm/vm.go index a5072f8c5..bb20c14e9 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -131,10 +131,15 @@ func (vm *VM) makeRuntime(ctx context.Context, msg *types.Message, origin addres rt.Abortf(exitcode.SysErrInvalidReceiver, "resolve msg.From address failed") } vmm.From = resF - resT, _ := rt.ResolveAddress(msg.To) - // may be set to undef if recipient doesn't exist yet - vmm.To = resT - rt.Message = Message{msg: vmm} + + if vm.ntwkVersion(ctx, vm.blockHeight) <= network.Version3 { + rt.Message = &vmm + } else { + resT, _ := rt.ResolveAddress(msg.To) + // may be set to undef if recipient doesn't exist yet + vmm.To = resT + rt.Message = &Message{msg: vmm} + } return rt } @@ -265,15 +270,19 @@ func (vm *VM) send(ctx context.Context, msg *types.Message, parent *Runtime, return nil, aerrors.Wrapf(err, "could not create account") } toActor = a - nmsg := Message{ - msg: types.Message{ - To: aid, - From: rt.Message.Caller(), - Value: rt.Message.ValueReceived(), - }, - } + if vm.ntwkVersion(ctx, vm.blockHeight) <= network.Version3 { + // Leave the rt.Message as is + } else { + nmsg := Message{ + msg: types.Message{ + To: aid, + From: rt.Message.Caller(), + Value: rt.Message.ValueReceived(), + }, + } - rt.Message = nmsg + rt.Message = &nmsg + } } else { return nil, aerrors.Escalate(err, "getting actor") } From 4d6a0f7b8c510e83b862be4639804e8bec9c54bb Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Tue, 6 Oct 2020 05:26:25 -0400 Subject: [PATCH 568/885] Set WinningPoStSectorSetLookback to finality post-v2 actors --- build/params_shared_vals.go | 2 -- build/params_testground.go | 3 +-- chain/actors/policy/policy.go | 8 ++++++++ chain/stmgr/utils.go | 7 ++++--- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/build/params_shared_vals.go b/build/params_shared_vals.go index cc711b7bc..722590575 100644 --- a/build/params_shared_vals.go +++ b/build/params_shared_vals.go @@ -56,8 +56,6 @@ const SealRandomnessLookback = policy.SealRandomnessLookback // Epochs const TicketRandomnessLookback = abi.ChainEpoch(1) -const WinningPoStSectorSetLookback = abi.ChainEpoch(10) - // ///// // Address diff --git a/build/params_testground.go b/build/params_testground.go index 162b3477c..bd064b3d6 100644 --- a/build/params_testground.go +++ b/build/params_testground.go @@ -50,8 +50,7 @@ var ( SealRandomnessLookback = policy.SealRandomnessLookback - TicketRandomnessLookback = abi.ChainEpoch(1) - WinningPoStSectorSetLookback = abi.ChainEpoch(10) + TicketRandomnessLookback = abi.ChainEpoch(1) FilBase uint64 = 2_000_000_000 FilAllocStorageMining uint64 = 1_400_000_000 diff --git a/chain/actors/policy/policy.go b/chain/actors/policy/policy.go index e7f213687..ba09e4424 100644 --- a/chain/actors/policy/policy.go +++ b/chain/actors/policy/policy.go @@ -106,3 +106,11 @@ func SetWPoStChallengeWindow(period abi.ChainEpoch) { miner2.WPoStChallengeWindow = period miner2.WPoStProvingPeriod = period * abi.ChainEpoch(miner2.WPoStPeriodDeadlines) } + +func GetWinningPoStSectorSetLookback(nwVer network.Version) abi.ChainEpoch { + if nwVer <= network.Version3 { + return 10 + } + + return ChainFinality +} diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index f2e2bfd19..2c9c5ad94 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -12,6 +12,7 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/network" + "github.com/filecoin-project/lotus/chain/actors/policy" cid "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" @@ -29,7 +30,6 @@ import ( exported2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/exported" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/build" init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init" "github.com/filecoin-project/lotus/chain/actors/builtin/market" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" @@ -413,8 +413,9 @@ func ComputeState(ctx context.Context, sm *StateManager, height abi.ChainEpoch, func GetLookbackTipSetForRound(ctx context.Context, sm *StateManager, ts *types.TipSet, round abi.ChainEpoch) (*types.TipSet, error) { var lbr abi.ChainEpoch - if round > build.WinningPoStSectorSetLookback { - lbr = round - build.WinningPoStSectorSetLookback + lb := policy.GetWinningPoStSectorSetLookback(sm.GetNtwkVersion(ctx, round)) + if round > lb { + lbr = round - lb } // more null blocks than our lookback From 00187d4aa8c66ae56ff77274698b773730a3a304 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 6 Oct 2020 11:51:32 -0700 Subject: [PATCH 569/885] update the AMT implementation This change ensures the bitmap is reset on flush. --- go.mod | 1 + go.sum | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index a160389b0..acafdc7c8 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/fatih/color v1.8.0 github.com/filecoin-project/filecoin-ffi v0.30.4-0.20200716204036-cddc56607e1d github.com/filecoin-project/go-address v0.0.4 + github.com/filecoin-project/go-amt-ipld/v2 v2.1.1-0.20201006184820-924ee87a1349 // indirect github.com/filecoin-project/go-bitfield v0.2.1 github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 diff --git a/go.sum b/go.sum index 5a00fff77..708ad72c3 100644 --- a/go.sum +++ b/go.sum @@ -228,6 +228,8 @@ github.com/filecoin-project/go-address v0.0.4 h1:gSNMv0qWwH16fGQs7ycOUrDjY6YCSsg github.com/filecoin-project/go-address v0.0.4/go.mod h1:jr8JxKsYx+lQlQZmF5i2U0Z+cGQ59wMIps/8YW/lDj8= github.com/filecoin-project/go-amt-ipld/v2 v2.1.0 h1:t6qDiuGYYngDqaLc2ZUvdtAg4UNxPeOYaXhBWSNsVaM= github.com/filecoin-project/go-amt-ipld/v2 v2.1.0/go.mod h1:nfFPoGyX0CU9SkXX8EoCcSuHN1XcbN0c6KBh7yvP5fs= +github.com/filecoin-project/go-amt-ipld/v2 v2.1.1-0.20201006184820-924ee87a1349 h1:pIuR0dnMD0i+as8wNnjjHyQrnhP5O5bmba/lmgQeRgU= +github.com/filecoin-project/go-amt-ipld/v2 v2.1.1-0.20201006184820-924ee87a1349/go.mod h1:vgmwKBkx+ca5OIeEvstiQgzAZnb7R6QaqE1oEDSqa6g= github.com/filecoin-project/go-bitfield v0.2.0 h1:gCtLcjskIPtdg4NfN7gQZSQF9yrBQ7mkT0qCJxzGI2Q= github.com/filecoin-project/go-bitfield v0.2.0/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM= github.com/filecoin-project/go-bitfield v0.2.1 h1:S6Uuqcspqu81sWJ0He4OAfFLm1tSwPdVjtKTkl5m/xQ= @@ -1400,6 +1402,7 @@ github.com/whyrusleeping/cbor-gen v0.0.0-20200414195334-429a0b5e922e/go.mod h1:X github.com/whyrusleeping/cbor-gen v0.0.0-20200504204219-64967432584d/go.mod h1:W5MvapuoHRP8rz4vxjwCK1pDqF1aQcWsV5PZ+AHbqdg= github.com/whyrusleeping/cbor-gen v0.0.0-20200710004633-5379fc63235d/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200715143311-227fab5a2377/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= +github.com/whyrusleeping/cbor-gen v0.0.0-20200723185710-6a3894a6352b/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200806213330-63aa96ca5488/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200810223238-211df3b9e24c/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200812213548-958ddffe352c/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= @@ -1525,7 +1528,6 @@ golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm0 golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd h1:zkO/Lhoka23X63N9OSzpSeROEUQ5ODw47tM3YWjygbs= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200513190911-00229845015e h1:rMqLP+9XLy+LdbCXHjJHAmTfXCr93W7oruWA6Hq1Alc= golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= From 087030fe371476cfad576796a2060f81ec2b7487 Mon Sep 17 00:00:00 2001 From: Travis Person Date: Tue, 6 Oct 2020 22:35:04 +0000 Subject: [PATCH 570/885] lotus-pcr: refund windowed post and storage deal gas fees --- cmd/lotus-pcr/main.go | 260 ++++++++++++++++++++++++++++-------------- 1 file changed, 174 insertions(+), 86 deletions(-) diff --git a/cmd/lotus-pcr/main.go b/cmd/lotus-pcr/main.go index f446b2c9d..14f4778c2 100644 --- a/cmd/lotus-pcr/main.go +++ b/cmd/lotus-pcr/main.go @@ -340,6 +340,18 @@ var runCmd = &cli.Command{ Usage: "process ProveCommitSector messages", Value: true, }, + &cli.BoolFlag{ + Name: "windowed-post", + EnvVars: []string{"LOTUS_PCR_WINDOWED_POST"}, + Usage: "process SubmitWindowedPoSt messages and refund gas fees", + Value: false, + }, + &cli.BoolFlag{ + Name: "storage-deals", + EnvVars: []string{"LOTUS_PCR_STORAGE_DEALS"}, + Usage: "process PublishStorageDeals messages and refund gas fees", + Value: false, + }, &cli.IntFlag{ Name: "head-delay", EnvVars: []string{"LOTUS_PCR_HEAD_DELAY"}, @@ -431,6 +443,8 @@ var runCmd = &cli.Command{ dryRun := cctx.Bool("dry-run") preCommitEnabled := cctx.Bool("pre-commit") proveCommitEnabled := cctx.Bool("prove-commit") + windowedPoStEnabled := cctx.Bool("windowed-post") + publishStorageDealsEnabled := cctx.Bool("storage-deals") aggregateTipsets := cctx.Int("aggregate-tipsets") minerRecoveryEnabled := cctx.Bool("miner-recovery") minerRecoveryPeriod := abi.ChainEpoch(int64(cctx.Int("miner-recovery-period"))) @@ -458,6 +472,8 @@ var runCmd = &cli.Command{ dryRun: dryRun, preCommitEnabled: preCommitEnabled, proveCommitEnabled: proveCommitEnabled, + windowedPoStEnabled: windowedPoStEnabled, + publishStorageDealsEnabled: publishStorageDealsEnabled, preFeeCapMax: types.BigInt(preFeeCapMax), proveFeeCapMax: types.BigInt(proveFeeCapMax), } @@ -611,6 +627,8 @@ type refunder struct { dryRun bool preCommitEnabled bool proveCommitEnabled bool + windowedPoStEnabled bool + publishStorageDealsEnabled bool threshold big.Int preFeeCapMax big.Int @@ -842,6 +860,147 @@ func (r *refunder) EnsureMinerMinimums(ctx context.Context, tipset *types.TipSet return refunds, nil } +func (r *refunder) processTipsetStorageMarketActor(ctx context.Context, tipset *types.TipSet, msg api.Message, recp *types.MessageReceipt) (bool, string, types.BigInt, error) { + + m := msg.Message + refundValue := types.NewInt(0) + var messageMethod string + + switch m.Method { + case builtin.MethodsMarket.PublishStorageDeals: + if !r.publishStorageDealsEnabled { + return false, messageMethod, types.NewInt(0), nil + } + + messageMethod = "PublishStorageDeals" + + if recp.ExitCode != exitcode.Ok { + log.Debugw("skipping non-ok exitcode message", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "exitcode", recp.ExitCode) + return false, messageMethod, types.NewInt(0), nil + } + + refundValue = types.BigMul(types.NewInt(uint64(recp.GasUsed)), tipset.Blocks()[0].ParentBaseFee) + } + + return true, messageMethod, refundValue, nil +} + +func (r *refunder) processTipsetStorageMinerActor(ctx context.Context, tipset *types.TipSet, msg api.Message, recp *types.MessageReceipt) (bool, string, types.BigInt, error) { + + m := msg.Message + refundValue := types.NewInt(0) + var messageMethod string + + switch m.Method { + case builtin.MethodsMiner.SubmitWindowedPoSt: + if !r.windowedPoStEnabled { + return false, messageMethod, types.NewInt(0), nil + } + + messageMethod = "SubmitWindowedPoSt" + + if recp.ExitCode != exitcode.Ok { + log.Debugw("skipping non-ok exitcode message", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "exitcode", recp.ExitCode) + return false, messageMethod, types.NewInt(0), nil + } + + refundValue = types.BigMul(types.NewInt(uint64(recp.GasUsed)), tipset.Blocks()[0].ParentBaseFee) + case builtin.MethodsMiner.ProveCommitSector: + if !r.proveCommitEnabled { + return false, messageMethod, types.NewInt(0), nil + } + + messageMethod = "ProveCommitSector" + + if recp.ExitCode != exitcode.Ok { + log.Debugw("skipping non-ok exitcode message", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "exitcode", recp.ExitCode) + return false, messageMethod, types.NewInt(0), nil + } + + if m.GasFeeCap.GreaterThan(r.proveFeeCapMax) { + log.Debugw("skipping high fee cap message", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "gas_fee_cap", m.GasFeeCap, "fee_cap_max", r.proveFeeCapMax) + return false, messageMethod, types.NewInt(0), nil + } + + var sn abi.SectorNumber + + var proveCommitSector miner0.ProveCommitSectorParams + if err := proveCommitSector.UnmarshalCBOR(bytes.NewBuffer(m.Params)); err != nil { + log.Warnw("failed to decode provecommit params", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To) + return false, messageMethod, types.NewInt(0), nil + } + + sn = proveCommitSector.SectorNumber + + // We use the parent tipset key because precommit information is removed when ProveCommitSector is executed + precommitChainInfo, err := r.api.StateSectorPreCommitInfo(ctx, m.To, sn, tipset.Parents()) + if err != nil { + log.Warnw("failed to get precommit info for sector", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", sn) + return false, messageMethod, types.NewInt(0), nil + } + + precommitTipset, err := r.api.ChainGetTipSetByHeight(ctx, precommitChainInfo.PreCommitEpoch, tipset.Key()) + if err != nil { + log.Warnf("failed to lookup precommit epoch", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", sn) + return false, messageMethod, types.NewInt(0), nil + } + + collateral, err := r.api.StateMinerInitialPledgeCollateral(ctx, m.To, precommitChainInfo.Info, precommitTipset.Key()) + if err != nil { + log.Warnw("failed to get initial pledge collateral", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", sn) + return false, messageMethod, types.NewInt(0), nil + } + + collateral = big.Sub(collateral, precommitChainInfo.PreCommitDeposit) + if collateral.LessThan(big.Zero()) { + log.Debugw("skipping zero pledge collateral difference", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", sn) + return false, messageMethod, types.NewInt(0), nil + } + + refundValue = collateral + if r.refundPercent > 0 { + refundValue = types.BigMul(types.BigDiv(refundValue, types.NewInt(100)), types.NewInt(uint64(r.refundPercent))) + } + case builtin.MethodsMiner.PreCommitSector: + if !r.preCommitEnabled { + return false, messageMethod, types.NewInt(0), nil + } + + messageMethod = "PreCommitSector" + + if recp.ExitCode != exitcode.Ok { + log.Debugw("skipping non-ok exitcode message", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "exitcode", recp.ExitCode) + return false, messageMethod, types.NewInt(0), nil + } + + if m.GasFeeCap.GreaterThan(r.preFeeCapMax) { + log.Debugw("skipping high fee cap message", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "gas_fee_cap", m.GasFeeCap, "fee_cap_max", r.preFeeCapMax) + return false, messageMethod, types.NewInt(0), nil + } + + var precommitInfo miner.SectorPreCommitInfo + if err := precommitInfo.UnmarshalCBOR(bytes.NewBuffer(m.Params)); err != nil { + log.Warnw("failed to decode precommit params", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To) + return false, messageMethod, types.NewInt(0), nil + } + + collateral, err := r.api.StateMinerInitialPledgeCollateral(ctx, m.To, precommitInfo, tipset.Key()) + if err != nil { + log.Warnw("failed to calculate initial pledge collateral", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", precommitInfo.SectorNumber) + return false, messageMethod, types.NewInt(0), nil + } + + refundValue = collateral + if r.refundPercent > 0 { + refundValue = types.BigMul(types.BigDiv(refundValue, types.NewInt(100)), types.NewInt(uint64(r.refundPercent))) + } + default: + return false, messageMethod, types.NewInt(0), nil + } + + return true, messageMethod, refundValue, nil +} + func (r *refunder) ProcessTipset(ctx context.Context, tipset *types.TipSet, refunds *MinersRefund) (*MinersRefund, error) { cids := tipset.Cids() if len(cids) == 0 { @@ -877,101 +1036,30 @@ func (r *refunder) ProcessTipset(ctx context.Context, tipset *types.TipSet, refu continue } - if !a.IsStorageMinerActor() { - continue - } - var messageMethod string - switch m.Method { - case builtin.MethodsMiner.ProveCommitSector: - if !r.proveCommitEnabled { - continue - } - - messageMethod = "ProveCommitSector" - - if recps[i].ExitCode != exitcode.Ok { - log.Debugw("skipping non-ok exitcode message", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "exitcode", recps[i].ExitCode) - continue - } - - if m.GasFeeCap.GreaterThan(r.proveFeeCapMax) { - log.Debugw("skipping high fee cap message", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "gas_fee_cap", m.GasFeeCap, "fee_cap_max", r.proveFeeCapMax) - continue - } - - var sn abi.SectorNumber - - var proveCommitSector miner0.ProveCommitSectorParams - if err := proveCommitSector.UnmarshalCBOR(bytes.NewBuffer(m.Params)); err != nil { - log.Warnw("failed to decode provecommit params", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To) - continue - } - - sn = proveCommitSector.SectorNumber - - // We use the parent tipset key because precommit information is removed when ProveCommitSector is executed - precommitChainInfo, err := r.api.StateSectorPreCommitInfo(ctx, m.To, sn, tipset.Parents()) + if m.To == builtin.StorageMarketActorAddr { + var err error + var processed bool + processed, messageMethod, refundValue, err = r.processTipsetStorageMarketActor(ctx, tipset, msg, recps[i]) if err != nil { - log.Warnw("failed to get precommit info for sector", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", sn) continue } - - precommitTipset, err := r.api.ChainGetTipSetByHeight(ctx, precommitChainInfo.PreCommitEpoch, tipset.Key()) - if err != nil { - log.Warnf("failed to lookup precommit epoch", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", sn) + if !processed { continue } - - collateral, err := r.api.StateMinerInitialPledgeCollateral(ctx, m.To, precommitChainInfo.Info, precommitTipset.Key()) - if err != nil { - log.Warnw("failed to get initial pledge collateral", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", sn) - } - - collateral = big.Sub(collateral, precommitChainInfo.PreCommitDeposit) - if collateral.LessThan(big.Zero()) { - log.Debugw("skipping zero pledge collateral difference", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", sn) - continue - } - - refundValue = collateral - case builtin.MethodsMiner.PreCommitSector: - if !r.preCommitEnabled { - continue - } - - messageMethod = "PreCommitSector" - - if recps[i].ExitCode != exitcode.Ok { - log.Debugw("skipping non-ok exitcode message", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "exitcode", recps[i].ExitCode) - continue - } - - if m.GasFeeCap.GreaterThan(r.preFeeCapMax) { - log.Debugw("skipping high fee cap message", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "gas_fee_cap", m.GasFeeCap, "fee_cap_max", r.preFeeCapMax) - continue - } - - var precommitInfo miner.SectorPreCommitInfo - if err := precommitInfo.UnmarshalCBOR(bytes.NewBuffer(m.Params)); err != nil { - log.Warnw("failed to decode precommit params", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To) - continue - } - - collateral, err := r.api.StateMinerInitialPledgeCollateral(ctx, m.To, precommitInfo, tipset.Key()) - if err != nil { - log.Warnw("failed to calculate initial pledge collateral", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", precommitInfo.SectorNumber) - continue - } - - refundValue = collateral - default: - continue } - if r.refundPercent > 0 { - refundValue = types.BigMul(types.BigDiv(refundValue, types.NewInt(100)), types.NewInt(uint64(r.refundPercent))) + if a.IsStorageMinerActor() { + var err error + var processed bool + processed, messageMethod, refundValue, err = r.processTipsetStorageMinerActor(ctx, tipset, msg, recps[i]) + if err != nil { + continue + } + if !processed { + continue + } } log.Debugw( From bd474617ed6f51726f645119f1da4bf2ece092bb Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 6 Oct 2020 16:06:24 -0700 Subject: [PATCH 571/885] implement command to get execution traces of any message --- cli/state.go | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/cli/state.go b/cli/state.go index 7baf57df2..371ef4af0 100644 --- a/cli/state.go +++ b/cli/state.go @@ -72,6 +72,7 @@ var stateCmd = &cli.Command{ stateMsgCostCmd, stateMinerInfo, stateMarketCmd, + stateExecTraceCmd, }, } @@ -315,6 +316,74 @@ var stateActiveSectorsCmd = &cli.Command{ }, } +var stateExecTraceCmd = &cli.Command{ + Name: "exec-trace", + Usage: "Get the execution trace of a given message", + ArgsUsage: "", + Action: func(cctx *cli.Context) error { + if !cctx.Args().Present() { + return ShowHelp(cctx, fmt.Errorf("must pass message cid")) + } + + mcid, err := cid.Decode(cctx.Args().First()) + if err != nil { + return fmt.Errorf("message cid was invalid: %s", err) + } + + capi, closer, err := GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + + ctx := ReqContext(cctx) + + msg, err := capi.ChainGetMessage(ctx, mcid) + if err != nil { + return err + } + + lookup, err := capi.StateSearchMsg(ctx, mcid) + if err != nil { + return err + } + + ts, err := capi.ChainGetTipSet(ctx, lookup.TipSet) + if err != nil { + return err + } + + pts, err := capi.ChainGetTipSet(ctx, ts.Parents()) + if err != nil { + return err + } + + cso, err := capi.StateCompute(ctx, pts.Height(), nil, pts.Key()) + if err != nil { + return err + } + + var trace *api.InvocResult + for _, t := range cso.Trace { + if t.Msg.From == msg.From && t.Msg.Nonce == msg.Nonce { + trace = t + break + } + } + if trace == nil { + return fmt.Errorf("failed to find message in tipset trace output") + } + + out, err := json.MarshalIndent(trace, "", " ") + if err != nil { + return err + } + + fmt.Println(string(out)) + return nil + }, +} + var stateReplaySetCmd = &cli.Command{ Name: "replay", Usage: "Replay a particular message within a tipset", From 74016ab1cce5fc842898b9ac793b457b48098e19 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 6 Oct 2020 16:46:47 -0700 Subject: [PATCH 572/885] fix noncefix --- cmd/lotus-shed/nonce-fix.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/cmd/lotus-shed/nonce-fix.go b/cmd/lotus-shed/nonce-fix.go index 4824a7bce..3cd9726f4 100644 --- a/cmd/lotus-shed/nonce-fix.go +++ b/cmd/lotus-shed/nonce-fix.go @@ -91,12 +91,8 @@ var noncefix = &cli.Command{ Value: types.NewInt(1), Nonce: i, } - smsg, err := api.WalletSignMessage(ctx, addr, msg) - if err != nil { - return err - } - _, err = api.MpoolPush(ctx, smsg) + _, err = api.MpoolPushMessage(ctx, msg, nil) if err != nil { return err } From 15670942c36dd6ffc185ae80152c2cd3f0651eae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 6 Oct 2020 16:29:16 +0200 Subject: [PATCH 573/885] Fix pond --- cmd/lotus-storage-miner/run.go | 16 ++++++++-------- lotuspond/spawn.go | 12 +++++++----- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/cmd/lotus-storage-miner/run.go b/cmd/lotus-storage-miner/run.go index a5d996f78..98a9cfaba 100644 --- a/cmd/lotus-storage-miner/run.go +++ b/cmd/lotus-storage-miner/run.go @@ -33,7 +33,7 @@ var runCmd = &cli.Command{ Usage: "Start a lotus miner process", Flags: []cli.Flag{ &cli.StringFlag{ - Name: "api", + Name: "miner-api", Usage: "2345", }, &cli.BoolFlag{ @@ -61,7 +61,7 @@ var runCmd = &cli.Command{ nodeApi, ncloser, err := lcli.GetFullNodeAPI(cctx) if err != nil { - return err + return xerrors.Errorf("getting full node api: %w", err) } defer ncloser() ctx := lcli.DaemonContext(cctx) @@ -112,29 +112,29 @@ var runCmd = &cli.Command{ node.Online(), node.Repo(r), - node.ApplyIf(func(s *node.Settings) bool { return cctx.IsSet("api") }, + node.ApplyIf(func(s *node.Settings) bool { return cctx.IsSet("miner-api") }, node.Override(new(dtypes.APIEndpoint), func() (dtypes.APIEndpoint, error) { - return multiaddr.NewMultiaddr("/ip4/127.0.0.1/tcp/" + cctx.String("api")) + return multiaddr.NewMultiaddr("/ip4/127.0.0.1/tcp/" + cctx.String("miner-api")) })), node.Override(new(api.FullNode), nodeApi), ) if err != nil { - return err + return xerrors.Errorf("creating node: %w", err) } endpoint, err := r.APIEndpoint() if err != nil { - return err + return xerrors.Errorf("getting API endpoint: %w", err) } // Bootstrap with full node remoteAddrs, err := nodeApi.NetAddrsListen(ctx) if err != nil { - return err + return xerrors.Errorf("getting full node libp2p address: %w", err) } if err := minerapi.NetConnect(ctx, remoteAddrs); err != nil { - return err + return xerrors.Errorf("connecting to full node (libp2p): %w", err) } log.Infof("Remote version %s", v) diff --git a/lotuspond/spawn.go b/lotuspond/spawn.go index 8b2e8661d..ce01b115e 100644 --- a/lotuspond/spawn.go +++ b/lotuspond/spawn.go @@ -11,16 +11,16 @@ import ( "sync/atomic" "time" - "github.com/filecoin-project/lotus/chain/actors/policy" - "github.com/filecoin-project/lotus/chain/types" - + "github.com/google/uuid" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" genesis2 "github.com/filecoin-project/lotus/chain/gen/genesis" + "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/gen" + "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/cmd/lotus-seed/seed" "github.com/filecoin-project/lotus/genesis" ) @@ -69,6 +69,8 @@ func (api *api) Spawn() (nodeInfo, error) { Meta: (&genesis.AccountMeta{Owner: genm.Owner}).ActorMeta(), }) template.VerifregRootKey = gen.DefaultVerifregRootkeyActor + template.RemainderAccount = gen.DefaultRemainderAccountActor + template.NetworkName = "pond-" + uuid.New().String() tb, err := json.Marshal(&template) if err != nil { @@ -188,7 +190,7 @@ func (api *api) SpawnStorage(fullNodeRepo string) (nodeInfo, error) { mux := newWsMux() - cmd = exec.Command("./lotus-miner", "run", "--api", fmt.Sprintf("%d", 2500+id), "--nosync") + cmd = exec.Command("./lotus-miner", "run", "--miner-api", fmt.Sprintf("%d", 2500+id), "--nosync") cmd.Stderr = io.MultiWriter(os.Stderr, errlogfile, mux.errpw) cmd.Stdout = io.MultiWriter(os.Stdout, logfile, mux.outpw) cmd.Env = append(os.Environ(), "LOTUS_MINER_PATH="+dir, "LOTUS_PATH="+fullNodeRepo) @@ -248,7 +250,7 @@ func (api *api) RestartNode(id int32) (nodeInfo, error) { var cmd *exec.Cmd if nd.meta.Storage { - cmd = exec.Command("./lotus-miner", "run", "--api", fmt.Sprintf("%d", 2500+id), "--nosync") + cmd = exec.Command("./lotus-miner", "run", "--miner-api", fmt.Sprintf("%d", 2500+id), "--nosync") } else { cmd = exec.Command("./lotus", "daemon", "--api", fmt.Sprintf("%d", 2500+id)) } From 839d1121ce5845d0ba302bd1d1543376793ca10f Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Tue, 6 Oct 2020 18:56:42 -0400 Subject: [PATCH 574/885] Update to actors v2.0.1 --- chain/actors/builtin/miner/v2.go | 2 +- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/chain/actors/builtin/miner/v2.go b/chain/actors/builtin/miner/v2.go index e2b53e9a6..eed82257f 100644 --- a/chain/actors/builtin/miner/v2.go +++ b/chain/actors/builtin/miner/v2.go @@ -46,7 +46,7 @@ type partition2 struct { } func (s *state2) AvailableBalance(bal abi.TokenAmount) (abi.TokenAmount, error) { - return s.GetAvailableBalance(bal), nil + return s.GetAvailableBalance(bal) } func (s *state2) VestedFunds(epoch abi.ChainEpoch) (abi.TokenAmount, error) { diff --git a/go.mod b/go.mod index 6f8a23a9d..8a25b3232 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,7 @@ require ( github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b github.com/filecoin-project/specs-actors v0.9.12 - github.com/filecoin-project/specs-actors/v2 v2.0.0-20201002200957-bdd876b3bbe9 + github.com/filecoin-project/specs-actors/v2 v2.0.1 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 github.com/filecoin-project/test-vectors/schema v0.0.3 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 diff --git a/go.sum b/go.sum index 708ad72c3..6062ee4ee 100644 --- a/go.sum +++ b/go.sum @@ -276,8 +276,8 @@ github.com/filecoin-project/specs-actors v0.9.4/go.mod h1:BStZQzx5x7TmCkLv0Bpa07 github.com/filecoin-project/specs-actors v0.9.7/go.mod h1:wM2z+kwqYgXn5Z7scV1YHLyd1Q1cy0R8HfTIWQ0BFGU= github.com/filecoin-project/specs-actors v0.9.12 h1:iIvk58tuMtmloFNHhAOQHG+4Gci6Lui0n7DYQGi3cJk= github.com/filecoin-project/specs-actors v0.9.12/go.mod h1:TS1AW/7LbG+615j4NsjMK1qlpAwaFsG9w0V2tg2gSao= -github.com/filecoin-project/specs-actors/v2 v2.0.0-20201002200957-bdd876b3bbe9 h1:UXVcGwUD9GJYNcgfGbf7ok790wrunvUhl5ZlgApGhGs= -github.com/filecoin-project/specs-actors/v2 v2.0.0-20201002200957-bdd876b3bbe9/go.mod h1:52FuQUNDXq2WDg+6+UOhkqBuNc2e62h9BCIB67Bluxg= +github.com/filecoin-project/specs-actors/v2 v2.0.1 h1:bf08x6tqCDfClzrv2q/rmt/A/UbBOy1KgaoogQEcLhU= +github.com/filecoin-project/specs-actors/v2 v2.0.1/go.mod h1:v2NZVYinNIKA9acEMBm5wWXxqv5+frFEbekBFemYghY= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= github.com/filecoin-project/test-vectors/schema v0.0.3 h1:1zuBo25B3016inbygYLgYFdpJ2m1BDTbAOCgABRleiU= From c23f1623a1a3ef52b09291b6ec40e2560120355d Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Tue, 6 Oct 2020 20:48:00 -0400 Subject: [PATCH 575/885] Fix test --- cmd/lotus-storage-miner/allinfo_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/lotus-storage-miner/allinfo_test.go b/cmd/lotus-storage-miner/allinfo_test.go index 562714fa0..8f744c4b3 100644 --- a/cmd/lotus-storage-miner/allinfo_test.go +++ b/cmd/lotus-storage-miner/allinfo_test.go @@ -5,6 +5,8 @@ import ( "testing" "time" + "github.com/filecoin-project/lotus/node" + logging "github.com/ipfs/go-log/v2" "github.com/stretchr/testify/require" "github.com/urfave/cli/v2" @@ -62,8 +64,8 @@ func TestMinerAllInfo(t *testing.T) { require.NoError(t, infoAllCmd.Action(cctx)) } - bp := func(t *testing.T, nFull int, storage []test.StorageMiner) ([]test.TestNode, []test.TestStorageNode) { - n, sn = builder.Builder(t, nFull, storage) + bp := func(t *testing.T, nFull int, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { + n, sn = builder.Builder(t, nFull, storage, opts...) t.Run("pre-info-all", run) From be92dd9e63a840324a757d55bbdba7ff8f7c781e Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 6 Oct 2020 18:07:34 -0700 Subject: [PATCH 576/885] allow manual setting of noncefix fee cap --- cmd/lotus-shed/nonce-fix.go | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/cmd/lotus-shed/nonce-fix.go b/cmd/lotus-shed/nonce-fix.go index 3cd9726f4..8102fd8a9 100644 --- a/cmd/lotus-shed/nonce-fix.go +++ b/cmd/lotus-shed/nonce-fix.go @@ -5,6 +5,8 @@ import ( "math" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" "github.com/urfave/cli/v2" "github.com/filecoin-project/lotus/chain/types" @@ -32,6 +34,10 @@ var noncefix = &cli.Command{ &cli.BoolFlag{ Name: "auto", }, + &cli.Int64Flag{ + Name: "gas-fee-cap", + Usage: "specify gas fee cap for nonce filling messages", + }, }, Action: func(cctx *cli.Context) error { api, closer, err := lcli.GetFullNodeAPI(cctx) @@ -84,15 +90,32 @@ var noncefix = &cli.Command{ } fmt.Printf("Creating %d filler messages (%d ~ %d)\n", end-start, start, end) + ts, err := api.ChainHead(ctx) + if err != nil { + return err + } + + feeCap := big.Mul(ts.Blocks()[0].ParentBaseFee, big.NewInt(2)) // default fee cap to 2 * parent base fee + if fcf := cctx.Int64("gas-fee-cap"); fcf != 0 { + feeCap = abi.NewTokenAmount(fcf) + } + for i := start; i < end; i++ { msg := &types.Message{ - From: addr, - To: addr, - Value: types.NewInt(1), - Nonce: i, + From: addr, + To: addr, + Value: types.NewInt(0), + Nonce: i, + GasLimit: 1000000, + GasFeeCap: feeCap, + GasPremium: abi.NewTokenAmount(5), + } + smsg, err := api.WalletSignMessage(ctx, addr, msg) + if err != nil { + return err } - _, err = api.MpoolPushMessage(ctx, msg, nil) + _, err = api.MpoolPush(ctx, smsg) if err != nil { return err } From ce247bcab3edc758b9f6903e4e84224068d28c98 Mon Sep 17 00:00:00 2001 From: Animesh Date: Wed, 7 Oct 2020 10:27:51 +0530 Subject: [PATCH 577/885] Add api for getting status given a code --- api/api_full.go | 2 ++ api/apistruct/struct.go | 21 +++++++++++++-------- node/impl/client/client.go | 9 +++++++++ 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 88f18943c..072d1cea6 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -271,6 +271,8 @@ type FullNode interface { ClientListDeals(ctx context.Context) ([]DealInfo, error) // ClientGetDealUpdates returns the status of updated deals ClientGetDealUpdates(ctx context.Context) (<-chan DealInfo, error) + // ClientGetDealStatus returns status given a code + ClientGetDealStatus(ctx context.Context, statusCode uint64) (string, error) // ClientHasLocal indicates whether a certain CID is locally stored. ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error) // ClientFindData identifies peers that have a certain file, and returns QueryOffers (one per peer). diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index a09700eb9..dd47c79a4 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -146,14 +146,15 @@ type FullNodeStruct struct { WalletDelete func(context.Context, address.Address) error `perm:"write"` WalletValidateAddress func(context.Context, string) (address.Address, error) `perm:"read"` - ClientImport func(ctx context.Context, ref api.FileRef) (*api.ImportRes, error) `perm:"admin"` - ClientListImports func(ctx context.Context) ([]api.Import, error) `perm:"write"` - ClientRemoveImport func(ctx context.Context, importID multistore.StoreID) error `perm:"admin"` - ClientHasLocal func(ctx context.Context, root cid.Cid) (bool, error) `perm:"write"` - ClientFindData func(ctx context.Context, root cid.Cid, piece *cid.Cid) ([]api.QueryOffer, error) `perm:"read"` - ClientMinerQueryOffer func(ctx context.Context, miner address.Address, root cid.Cid, piece *cid.Cid) (api.QueryOffer, error) `perm:"read"` - ClientStartDeal func(ctx context.Context, params *api.StartDealParams) (*cid.Cid, error) `perm:"admin"` - ClientGetDealInfo func(context.Context, cid.Cid) (*api.DealInfo, error) `perm:"read"` + ClientImport func(ctx context.Context, ref api.FileRef) (*api.ImportRes, error) `perm:"admin"` + ClientListImports func(ctx context.Context) ([]api.Import, error) `perm:"write"` + ClientRemoveImport func(ctx context.Context, importID multistore.StoreID) error `perm:"admin"` + ClientHasLocal func(ctx context.Context, root cid.Cid) (bool, error) `perm:"write"` + ClientFindData func(ctx context.Context, root cid.Cid, piece *cid.Cid) ([]api.QueryOffer, error) `perm:"read"` + ClientMinerQueryOffer func(ctx context.Context, miner address.Address, root cid.Cid, piece *cid.Cid) (api.QueryOffer, error) `perm:"read"` + ClientStartDeal func(ctx context.Context, params *api.StartDealParams) (*cid.Cid, error) `perm:"admin"` + ClientGetDealInfo func(context.Context, cid.Cid) (*api.DealInfo, error) `perm:"read"` + ClientGetDealStatus func(context.Context, uint64) (string, error) ClientListDeals func(ctx context.Context) ([]api.DealInfo, error) `perm:"write"` ClientGetDealUpdates func(ctx context.Context) (<-chan api.DealInfo, error) `perm:"read"` ClientRetrieve func(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) error `perm:"admin"` @@ -478,6 +479,10 @@ func (c *FullNodeStruct) ClientGetDealInfo(ctx context.Context, deal cid.Cid) (* return c.Internal.ClientGetDealInfo(ctx, deal) } +func (c *FullNodeStruct) ClientGetDealStatus(ctx context.Context, statusCode uint64) (string, error) { + return c.Internal.ClientGetDealStatus(ctx, statusCode) +} + func (c *FullNodeStruct) ClientListDeals(ctx context.Context) ([]api.DealInfo, error) { return c.Internal.ClientListDeals(ctx) } diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 5c1752154..6af266baa 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -863,3 +863,12 @@ func newDealInfo(v storagemarket.ClientDeal) api.DealInfo { func (a *API) ClientRetrieveTryRestartInsufficientFunds(ctx context.Context, paymentChannel address.Address) error { return a.Retrieval.TryRestartInsufficientFunds(paymentChannel) } + +func (a *API) ClientGetDealStatus(ctx context.Context, statusCode uint64) (string, error) { + ststr, ok := storagemarket.DealStates[statusCode] + if !ok { + return "", fmt.Errorf("no such deal state %d", statusCode) + } + + return ststr, nil +} From b7c8f65af5c474310d988b59da0d036b9d252a26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 7 Oct 2020 01:15:19 +0200 Subject: [PATCH 578/885] stmgr: Make v2 upgrade faster --- chain/stmgr/forks.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index d5d0dbf7e..8143524c0 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -4,6 +4,9 @@ import ( "bytes" "context" "encoding/binary" + "github.com/filecoin-project/lotus/chain/store" + bstore "github.com/filecoin-project/lotus/lib/blockstore" + "github.com/filecoin-project/lotus/lib/bufbstore" "math" "github.com/filecoin-project/go-address" @@ -490,7 +493,8 @@ func UpgradeIgnition(ctx context.Context, sm *StateManager, cb ExecCallback, roo } func UpgradeActorsV2(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, ts *types.TipSet) (cid.Cid, error) { - store := sm.cs.Store(ctx) + buf := bufbstore.NewTieredBstore(sm.cs.Blockstore(), bstore.NewTemporarySync()) + store := store.ActorStore(ctx, buf) epoch := ts.Height() - 1 @@ -538,6 +542,15 @@ func UpgradeActorsV2(ctx context.Context, sm *StateManager, cb ExecCallback, roo return cid.Undef, xerrors.Errorf("failed to load init actor after upgrade: %w", err) } + { + from := buf + to := buf.Read() + + if err := vm.Copy(ctx, from, to, newRoot); err != nil { + return cid.Undef, xerrors.Errorf("copying migrated tree: %w", err) + } + } + return newRoot, nil } From 3b12a9f425c81715bf9fe0ad8087b6b880aee437 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 6 Oct 2020 16:38:31 -0700 Subject: [PATCH 579/885] Add flag to accept previous output of compute-state as input --- cli/state.go | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/cli/state.go b/cli/state.go index cb1c974a7..b2beb8ffb 100644 --- a/cli/state.go +++ b/cli/state.go @@ -7,6 +7,7 @@ import ( "fmt" "html/template" "io" + "io/ioutil" "os" "reflect" "sort" @@ -32,6 +33,7 @@ import ( "github.com/filecoin-project/go-state-types/exitcode" "github.com/filecoin-project/lotus/api" + lapi "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/apibstore" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/state" @@ -823,6 +825,10 @@ var stateComputeStateCmd = &cli.Command{ Name: "json", Usage: "generate json output", }, + &cli.StringFlag{ + Name: "compute-state-output", + Usage: "a json file containing pre-existing compute-state output, to generate html reports without rerunning state changes", + }, }, Action: func(cctx *cli.Context) error { api, closer, err := GetFullNodeAPI(cctx) @@ -862,9 +868,26 @@ var stateComputeStateCmd = &cli.Command{ } } - stout, err := api.StateCompute(ctx, h, msgs, ts.Key()) - if err != nil { - return err + var stout *lapi.ComputeStateOutput + if csofile := cctx.String("compute-state-output"); csofile != "" { + data, err := ioutil.ReadFile(csofile) + if err != nil { + return err + } + + var o lapi.ComputeStateOutput + if err := json.Unmarshal(data, &o); err != nil { + return err + } + + stout = &o + } else { + o, err := api.StateCompute(ctx, h, msgs, ts.Key()) + if err != nil { + return err + } + + stout = o } if cctx.Bool("json") { From bf959e45fb5f1c5f9a406ffba407583af5517a21 Mon Sep 17 00:00:00 2001 From: Animesh Date: Wed, 7 Oct 2020 11:43:30 +0530 Subject: [PATCH 580/885] Add api for getting status given a code --- api/api_full.go | 2 ++ api/apistruct/struct.go | 21 +++++++++++++-------- node/impl/client/client.go | 9 +++++++++ 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 88f18943c..072d1cea6 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -271,6 +271,8 @@ type FullNode interface { ClientListDeals(ctx context.Context) ([]DealInfo, error) // ClientGetDealUpdates returns the status of updated deals ClientGetDealUpdates(ctx context.Context) (<-chan DealInfo, error) + // ClientGetDealStatus returns status given a code + ClientGetDealStatus(ctx context.Context, statusCode uint64) (string, error) // ClientHasLocal indicates whether a certain CID is locally stored. ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error) // ClientFindData identifies peers that have a certain file, and returns QueryOffers (one per peer). diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index a09700eb9..dd47c79a4 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -146,14 +146,15 @@ type FullNodeStruct struct { WalletDelete func(context.Context, address.Address) error `perm:"write"` WalletValidateAddress func(context.Context, string) (address.Address, error) `perm:"read"` - ClientImport func(ctx context.Context, ref api.FileRef) (*api.ImportRes, error) `perm:"admin"` - ClientListImports func(ctx context.Context) ([]api.Import, error) `perm:"write"` - ClientRemoveImport func(ctx context.Context, importID multistore.StoreID) error `perm:"admin"` - ClientHasLocal func(ctx context.Context, root cid.Cid) (bool, error) `perm:"write"` - ClientFindData func(ctx context.Context, root cid.Cid, piece *cid.Cid) ([]api.QueryOffer, error) `perm:"read"` - ClientMinerQueryOffer func(ctx context.Context, miner address.Address, root cid.Cid, piece *cid.Cid) (api.QueryOffer, error) `perm:"read"` - ClientStartDeal func(ctx context.Context, params *api.StartDealParams) (*cid.Cid, error) `perm:"admin"` - ClientGetDealInfo func(context.Context, cid.Cid) (*api.DealInfo, error) `perm:"read"` + ClientImport func(ctx context.Context, ref api.FileRef) (*api.ImportRes, error) `perm:"admin"` + ClientListImports func(ctx context.Context) ([]api.Import, error) `perm:"write"` + ClientRemoveImport func(ctx context.Context, importID multistore.StoreID) error `perm:"admin"` + ClientHasLocal func(ctx context.Context, root cid.Cid) (bool, error) `perm:"write"` + ClientFindData func(ctx context.Context, root cid.Cid, piece *cid.Cid) ([]api.QueryOffer, error) `perm:"read"` + ClientMinerQueryOffer func(ctx context.Context, miner address.Address, root cid.Cid, piece *cid.Cid) (api.QueryOffer, error) `perm:"read"` + ClientStartDeal func(ctx context.Context, params *api.StartDealParams) (*cid.Cid, error) `perm:"admin"` + ClientGetDealInfo func(context.Context, cid.Cid) (*api.DealInfo, error) `perm:"read"` + ClientGetDealStatus func(context.Context, uint64) (string, error) ClientListDeals func(ctx context.Context) ([]api.DealInfo, error) `perm:"write"` ClientGetDealUpdates func(ctx context.Context) (<-chan api.DealInfo, error) `perm:"read"` ClientRetrieve func(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) error `perm:"admin"` @@ -478,6 +479,10 @@ func (c *FullNodeStruct) ClientGetDealInfo(ctx context.Context, deal cid.Cid) (* return c.Internal.ClientGetDealInfo(ctx, deal) } +func (c *FullNodeStruct) ClientGetDealStatus(ctx context.Context, statusCode uint64) (string, error) { + return c.Internal.ClientGetDealStatus(ctx, statusCode) +} + func (c *FullNodeStruct) ClientListDeals(ctx context.Context) ([]api.DealInfo, error) { return c.Internal.ClientListDeals(ctx) } diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 5c1752154..6af266baa 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -863,3 +863,12 @@ func newDealInfo(v storagemarket.ClientDeal) api.DealInfo { func (a *API) ClientRetrieveTryRestartInsufficientFunds(ctx context.Context, paymentChannel address.Address) error { return a.Retrieval.TryRestartInsufficientFunds(paymentChannel) } + +func (a *API) ClientGetDealStatus(ctx context.Context, statusCode uint64) (string, error) { + ststr, ok := storagemarket.DealStates[statusCode] + if !ok { + return "", fmt.Errorf("no such deal state %d", statusCode) + } + + return ststr, nil +} From 673de6aeb67eac8b0536b86cfe3c105a96ee8213 Mon Sep 17 00:00:00 2001 From: Animesh Date: Wed, 7 Oct 2020 12:35:42 +0530 Subject: [PATCH 581/885] Add perm tag to ClientGetDealStatus --- api/apistruct/struct.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index dd47c79a4..b27e9082c 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -146,15 +146,15 @@ type FullNodeStruct struct { WalletDelete func(context.Context, address.Address) error `perm:"write"` WalletValidateAddress func(context.Context, string) (address.Address, error) `perm:"read"` - ClientImport func(ctx context.Context, ref api.FileRef) (*api.ImportRes, error) `perm:"admin"` - ClientListImports func(ctx context.Context) ([]api.Import, error) `perm:"write"` - ClientRemoveImport func(ctx context.Context, importID multistore.StoreID) error `perm:"admin"` - ClientHasLocal func(ctx context.Context, root cid.Cid) (bool, error) `perm:"write"` - ClientFindData func(ctx context.Context, root cid.Cid, piece *cid.Cid) ([]api.QueryOffer, error) `perm:"read"` - ClientMinerQueryOffer func(ctx context.Context, miner address.Address, root cid.Cid, piece *cid.Cid) (api.QueryOffer, error) `perm:"read"` - ClientStartDeal func(ctx context.Context, params *api.StartDealParams) (*cid.Cid, error) `perm:"admin"` - ClientGetDealInfo func(context.Context, cid.Cid) (*api.DealInfo, error) `perm:"read"` - ClientGetDealStatus func(context.Context, uint64) (string, error) + ClientImport func(ctx context.Context, ref api.FileRef) (*api.ImportRes, error) `perm:"admin"` + ClientListImports func(ctx context.Context) ([]api.Import, error) `perm:"write"` + ClientRemoveImport func(ctx context.Context, importID multistore.StoreID) error `perm:"admin"` + ClientHasLocal func(ctx context.Context, root cid.Cid) (bool, error) `perm:"write"` + ClientFindData func(ctx context.Context, root cid.Cid, piece *cid.Cid) ([]api.QueryOffer, error) `perm:"read"` + ClientMinerQueryOffer func(ctx context.Context, miner address.Address, root cid.Cid, piece *cid.Cid) (api.QueryOffer, error) `perm:"read"` + ClientStartDeal func(ctx context.Context, params *api.StartDealParams) (*cid.Cid, error) `perm:"admin"` + ClientGetDealInfo func(context.Context, cid.Cid) (*api.DealInfo, error) `perm:"read"` + ClientGetDealStatus func(context.Context, uint64) (string, error) `perm:"read"` ClientListDeals func(ctx context.Context) ([]api.DealInfo, error) `perm:"write"` ClientGetDealUpdates func(ctx context.Context) (<-chan api.DealInfo, error) `perm:"read"` ClientRetrieve func(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) error `perm:"admin"` From d3dc560f53aee4eb843a3c081b44282e36c6dca0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Wed, 7 Oct 2020 12:45:11 +0100 Subject: [PATCH 582/885] conformance: minor refactors. --- cmd/tvx/extract.go | 12 +++++----- conformance/driver.go | 47 +++++++++++++++++++++++++++------------ conformance/rand_fixed.go | 28 +++++++++++++++++++++++ conformance/runner.go | 20 ++++------------- conformance/stubs.go | 14 +----------- 5 files changed, 72 insertions(+), 49 deletions(-) create mode 100644 conformance/rand_fixed.go diff --git a/cmd/tvx/extract.go b/cmd/tvx/extract.go index fef245858..afdeb9540 100644 --- a/cmd/tvx/extract.go +++ b/cmd/tvx/extract.go @@ -198,8 +198,8 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { Preroot: root, Epoch: execTs.Height(), Message: m, - CircSupply: &circSupplyDetail.FilCirculating, - BaseFee: &basefee, + CircSupply: circSupplyDetail.FilCirculating, + BaseFee: basefee, }) if err != nil { return fmt.Errorf("failed to execute precursor message: %w", err) @@ -229,8 +229,8 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { Preroot: preroot, Epoch: execTs.Height(), Message: msg, - CircSupply: &circSupplyDetail.FilCirculating, - BaseFee: &basefee, + CircSupply: circSupplyDetail.FilCirculating, + BaseFee: basefee, }) if err != nil { return fmt.Errorf("failed to execute message: %w", err) @@ -260,8 +260,8 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { Preroot: preroot, Epoch: execTs.Height(), Message: msg, - CircSupply: &circSupplyDetail.FilCirculating, - BaseFee: &basefee, + CircSupply: circSupplyDetail.FilCirculating, + BaseFee: basefee, }) if err != nil { return fmt.Errorf("failed to execute message: %w", err) diff --git a/conformance/driver.go b/conformance/driver.go index 9ced12d74..d51798a52 100644 --- a/conformance/driver.go +++ b/conformance/driver.go @@ -2,6 +2,7 @@ package conformance import ( "context" + gobig "math/big" "os" "github.com/filecoin-project/lotus/chain/state" @@ -14,6 +15,7 @@ import ( "github.com/filecoin-project/lotus/lib/blockstore" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/test-vectors/schema" @@ -80,7 +82,7 @@ type ExecuteTipsetResult struct { func (d *Driver) ExecuteTipset(bs blockstore.Blockstore, ds ds.Batching, preroot cid.Cid, parentEpoch abi.ChainEpoch, tipset *schema.Tipset) (*ExecuteTipsetResult, error) { var ( syscalls = mkFakedSigSyscalls(vm.Syscalls(ffiwrapper.ProofVerifier)) - vmRand = new(testRand) + vmRand = NewFixedRand() cs = store.NewChainStore(bs, ds, syscalls) sm = stmgr.NewStateManager(cs) @@ -143,8 +145,12 @@ type ExecuteMessageParams struct { Preroot cid.Cid Epoch abi.ChainEpoch Message *types.Message - CircSupply *abi.TokenAmount - BaseFee *abi.TokenAmount + CircSupply abi.TokenAmount + BaseFee abi.TokenAmount + + // Rand is an optional vm.Rand implementation to use. If nil, the driver + // will use a vm.Rand that returns a fixed value for all calls. + Rand vm.Rand } // ExecuteMessage executes a conformance test vector message in a temporary VM. @@ -155,14 +161,8 @@ func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, params ExecuteMessageP _ = os.Setenv("LOTUS_DISABLE_VM_BUF", "iknowitsabadidea") } - basefee := DefaultBaseFee - if params.BaseFee != nil { - basefee = *params.BaseFee - } - - circSupply := DefaultCirculatingSupply - if params.CircSupply != nil { - circSupply = *params.CircSupply + if params.Rand == nil { + params.Rand = NewFixedRand() } // dummy state manager; only to reference the GetNetworkVersion method, @@ -172,13 +172,13 @@ func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, params ExecuteMessageP vmOpts := &vm.VMOpts{ StateBase: params.Preroot, Epoch: params.Epoch, - Rand: &testRand{}, // TODO always succeeds; need more flexibility. Bstore: bs, Syscalls: mkFakedSigSyscalls(vm.Syscalls(ffiwrapper.ProofVerifier)), // TODO always succeeds; need more flexibility. CircSupplyCalc: func(_ context.Context, _ abi.ChainEpoch, _ *state.StateTree) (abi.TokenAmount, error) { - return circSupply, nil + return params.CircSupply, nil }, - BaseFee: basefee, + Rand: params.Rand, + BaseFee: params.BaseFee, NtwkVersion: sm.GetNtwkVersion, } @@ -231,3 +231,22 @@ func toChainMsg(msg *types.Message) (ret types.ChainMsg) { } return ret } + +// BaseFeeOrDefault converts a basefee as passed in a test vector (go *big.Int +// type) to an abi.TokenAmount, or if nil it returns the DefaultBaseFee. +func BaseFeeOrDefault(basefee *gobig.Int) abi.TokenAmount { + if basefee == nil { + return DefaultBaseFee + } + return big.NewFromGo(basefee) +} + +// CircSupplyOrDefault converts a circulating supply as passed in a test vector +// (go *big.Int type) to an abi.TokenAmount, or if nil it returns the +// DefaultCirculatingSupply. +func CircSupplyOrDefault(circSupply *gobig.Int) abi.TokenAmount { + if circSupply == nil { + return DefaultBaseFee + } + return big.NewFromGo(circSupply) +} diff --git a/conformance/rand_fixed.go b/conformance/rand_fixed.go new file mode 100644 index 000000000..d356b53d0 --- /dev/null +++ b/conformance/rand_fixed.go @@ -0,0 +1,28 @@ +package conformance + +import ( + "context" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/crypto" + + "github.com/filecoin-project/lotus/chain/vm" +) + +type fixedRand struct{} + +var _ vm.Rand = (*fixedRand)(nil) + +// NewFixedRand creates a test vm.Rand that always returns fixed bytes value +// of utf-8 string 'i_am_random_____i_am_random_____'. +func NewFixedRand() vm.Rand { + return &fixedRand{} +} + +func (r *fixedRand) GetChainRandomness(_ context.Context, _ crypto.DomainSeparationTag, _ abi.ChainEpoch, _ []byte) ([]byte, error) { + return []byte("i_am_random_____i_am_random_____"), nil // 32 bytes. +} + +func (r *fixedRand) GetBeaconRandomness(_ context.Context, _ crypto.DomainSeparationTag, _ abi.ChainEpoch, _ []byte) ([]byte, error) { + return []byte("i_am_random_____i_am_random_____"), nil // 32 bytes. +} diff --git a/conformance/runner.go b/conformance/runner.go index 2db53b3e4..a240c9067 100644 --- a/conformance/runner.go +++ b/conformance/runner.go @@ -13,9 +13,7 @@ import ( "github.com/fatih/color" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/exitcode" - "github.com/filecoin-project/test-vectors/schema" "github.com/ipfs/go-blockservice" "github.com/ipfs/go-cid" ds "github.com/ipfs/go-datastore" @@ -24,6 +22,8 @@ import ( "github.com/ipfs/go-merkledag" "github.com/ipld/go-car" + "github.com/filecoin-project/test-vectors/schema" + "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" "github.com/filecoin-project/lotus/lib/blockstore" @@ -46,18 +46,6 @@ func ExecuteMessageVector(r Reporter, vector *schema.TestVector) { // Create a new Driver. driver := NewDriver(ctx, vector.Selector, DriverOpts{DisableVMFlush: true}) - var circSupply *abi.TokenAmount - if cs := vector.Pre.CircSupply; cs != nil { - ta := big.NewFromGo(cs) - circSupply = &ta - } - - var basefee *abi.TokenAmount - if bf := vector.Pre.BaseFee; bf != nil { - ta := big.NewFromGo(bf) - basefee = &ta - } - // Apply every message. for i, m := range vector.ApplyMessages { msg, err := types.DecodeMessage(m.Bytes) @@ -76,8 +64,8 @@ func ExecuteMessageVector(r Reporter, vector *schema.TestVector) { Preroot: root, Epoch: abi.ChainEpoch(epoch), Message: msg, - CircSupply: circSupply, - BaseFee: basefee, + BaseFee: BaseFeeOrDefault(vector.Pre.BaseFee), + CircSupply: CircSupplyOrDefault(vector.Pre.CircSupply), }) if err != nil { r.Fatalf("fatal failure when executing message: %s", err) diff --git a/conformance/stubs.go b/conformance/stubs.go index a7100892f..9307fdd65 100644 --- a/conformance/stubs.go +++ b/conformance/stubs.go @@ -6,28 +6,16 @@ import ( "github.com/filecoin-project/specs-actors/actors/runtime/proof" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/vm" - "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/specs-actors/actors/runtime" cbor "github.com/ipfs/go-ipld-cbor" ) -type testRand struct{} - -var _ vm.Rand = (*testRand)(nil) - -func (r *testRand) GetChainRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) { - return []byte("i_am_random_____i_am_random_____"), nil // 32 bytes. -} - -func (r *testRand) GetBeaconRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) { - return []byte("i_am_random_____i_am_random_____"), nil // 32 bytes. -} - type testSyscalls struct { runtime.Syscalls } From d6fa64497446d8f8337b16411e840bcdaa46ded4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 7 Oct 2020 14:10:23 +0200 Subject: [PATCH 583/885] Fix lint --- chain/stmgr/forks.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index 8143524c0..2c2304a1f 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -4,9 +4,6 @@ import ( "bytes" "context" "encoding/binary" - "github.com/filecoin-project/lotus/chain/store" - bstore "github.com/filecoin-project/lotus/lib/blockstore" - "github.com/filecoin-project/lotus/lib/bufbstore" "math" "github.com/filecoin-project/go-address" @@ -33,8 +30,11 @@ import ( init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init" "github.com/filecoin-project/lotus/chain/actors/builtin/multisig" "github.com/filecoin-project/lotus/chain/state" + "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" + bstore "github.com/filecoin-project/lotus/lib/blockstore" + "github.com/filecoin-project/lotus/lib/bufbstore" ) type UpgradeFunc func(context.Context, *StateManager, ExecCallback, cid.Cid, *types.TipSet) (cid.Cid, error) @@ -547,7 +547,7 @@ func UpgradeActorsV2(ctx context.Context, sm *StateManager, cb ExecCallback, roo to := buf.Read() if err := vm.Copy(ctx, from, to, newRoot); err != nil { - return cid.Undef, xerrors.Errorf("copying migrated tree: %w", err) + return cid.Undef, xerrors.Errorf("copying migrated tree: %w", err) } } From 39bc816a79c76d22e401310f9cae3fc3d3725992 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 7 Oct 2020 15:01:30 +0200 Subject: [PATCH 584/885] build: Env var to keep test address output --- build/params_testnet.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/build/params_testnet.go b/build/params_testnet.go index 49abd30c0..1369b2c70 100644 --- a/build/params_testnet.go +++ b/build/params_testnet.go @@ -5,6 +5,8 @@ package build import ( + "os" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/actors/policy" @@ -36,7 +38,9 @@ func init() { abi.RegisteredSealProof_StackedDrg64GiBV1, ) - SetAddressNetwork(address.Mainnet) + if os.Getenv("LOTUS_USE_TEST_ADDRESSES") != "1" { + SetAddressNetwork(address.Mainnet) + } Devnet = false } From 7cd1330acd9802ee5709b324ba584f3882d8d2af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Wed, 7 Oct 2020 15:52:03 +0100 Subject: [PATCH 585/885] make vm.EnableGasTracing public. --- chain/vm/runtime.go | 9 +++++---- chain/vm/runtime_test.go | 20 ++++++++++++++++++++ chain/vm/vm.go | 2 +- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/chain/vm/runtime.go b/chain/vm/runtime.go index 72dd413ed..c612e374f 100644 --- a/chain/vm/runtime.go +++ b/chain/vm/runtime.go @@ -28,6 +28,9 @@ import ( "github.com/filecoin-project/lotus/chain/types" ) +// EnableGasTracing, if true, outputs gas tracing in execution traces. +var EnableGasTracing = false + type Runtime struct { types.Message rt0.Syscalls @@ -459,7 +462,7 @@ func (rt *Runtime) stateCommit(oldh, newh cid.Cid) aerrors.ActorError { } func (rt *Runtime) finilizeGasTracing() { - if enableTracing { + if EnableGasTracing { if rt.lastGasCharge != nil { rt.lastGasCharge.TimeTaken = time.Since(rt.lastGasChargeTime) } @@ -491,11 +494,9 @@ func (rt *Runtime) chargeGasFunc(skip int) func(GasCharge) { } -var enableTracing = false - func (rt *Runtime) chargeGasInternal(gas GasCharge, skip int) aerrors.ActorError { toUse := gas.Total() - if enableTracing { + if EnableGasTracing { var callers [10]uintptr cout := 0 //gruntime.Callers(2+skip, callers[:]) diff --git a/chain/vm/runtime_test.go b/chain/vm/runtime_test.go index c22a8b615..9fc87f7c5 100644 --- a/chain/vm/runtime_test.go +++ b/chain/vm/runtime_test.go @@ -45,3 +45,23 @@ func TestRuntimePutErrors(t *testing.T) { rt.StorePut(&NotAVeryGoodMarshaler{}) t.Error("expected panic") } + +func BenchmarkRuntime_CreateRuntimeChargeGas_TracingDisabled(b *testing.B) { + var ( + cst = cbor.NewCborStore(nil) + gch = newGasCharge("foo", 1000, 1000) + ) + + b.ResetTimer() + + EnableGasTracing = false + noop := func() bool { return EnableGasTracing } + for n := 0; n < b.N; n++ { + // flip the value and access it to make sure + // the compiler doesn't optimize away + EnableGasTracing = true + _ = noop() + EnableGasTracing = false + _ = (&Runtime{cst: cst}).chargeGasInternal(gch, 0) + } +} diff --git a/chain/vm/vm.go b/chain/vm/vm.go index 44979454f..e2ccd33c7 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -227,7 +227,7 @@ func (vm *VM) send(ctx context.Context, msg *types.Message, parent *Runtime, } rt := vm.makeRuntime(ctx, msg, origin, on, gasUsed, nac) - if enableTracing { + if EnableGasTracing { rt.lastGasChargeTime = start if parent != nil { rt.lastGasChargeTime = parent.lastGasChargeTime From 0d34881a0ca3b3b823dfd3c16e84e66625b1c331 Mon Sep 17 00:00:00 2001 From: Travis Person Date: Wed, 7 Oct 2020 18:50:50 +0000 Subject: [PATCH 586/885] lotus-pcr: ignore all other messages --- cmd/lotus-pcr/main.go | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/cmd/lotus-pcr/main.go b/cmd/lotus-pcr/main.go index 14f4778c2..5b59ff03d 100644 --- a/cmd/lotus-pcr/main.go +++ b/cmd/lotus-pcr/main.go @@ -1037,29 +1037,22 @@ func (r *refunder) ProcessTipset(ctx context.Context, tipset *types.TipSet, refu } var messageMethod string + var processed bool if m.To == builtin.StorageMarketActorAddr { - var err error - var processed bool processed, messageMethod, refundValue, err = r.processTipsetStorageMarketActor(ctx, tipset, msg, recps[i]) - if err != nil { - continue - } - if !processed { - continue - } } if a.IsStorageMinerActor() { - var err error - var processed bool processed, messageMethod, refundValue, err = r.processTipsetStorageMinerActor(ctx, tipset, msg, recps[i]) - if err != nil { - continue - } - if !processed { - continue - } + } + + if err != nil { + log.Errorw("error while processing message", "cid", msg.Cid) + continue + } + if !processed { + continue } log.Debugw( From 5bd6a3cdadd47fc5f26027b522fd0e94724350df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Wed, 7 Oct 2020 19:52:37 +0100 Subject: [PATCH 587/885] conformance: record randomness in tvx; replay in driver. --- cmd/tvx/extract.go | 10 ++++- conformance/rand_record.go | 86 ++++++++++++++++++++++++++++++++++++++ conformance/rand_replay.go | 78 ++++++++++++++++++++++++++++++++++ conformance/runner.go | 1 + go.mod | 2 +- go.sum | 4 +- 6 files changed, 177 insertions(+), 4 deletions(-) create mode 100644 conformance/rand_record.go create mode 100644 conformance/rand_replay.go diff --git a/cmd/tvx/extract.go b/cmd/tvx/extract.go index afdeb9540..b0ed574df 100644 --- a/cmd/tvx/extract.go +++ b/cmd/tvx/extract.go @@ -200,6 +200,8 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { Message: m, CircSupply: circSupplyDetail.FilCirculating, BaseFee: basefee, + // recorded randomness will be discarded. + Rand: conformance.NewRecordingRand(new(conformance.LogReporter), fapi), }) if err != nil { return fmt.Errorf("failed to execute precursor message: %w", err) @@ -212,6 +214,9 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { applyret *vm.ApplyRet carWriter func(w io.Writer) error retention = opts.retain + + // recordingRand will record randomness so we can embed it in the test vector. + recordingRand = conformance.NewRecordingRand(new(conformance.LogReporter), fapi) ) log.Printf("using state retention strategy: %s", retention) @@ -231,6 +236,7 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { Message: msg, CircSupply: circSupplyDetail.FilCirculating, BaseFee: basefee, + Rand: recordingRand, }) if err != nil { return fmt.Errorf("failed to execute message: %w", err) @@ -262,6 +268,7 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { Message: msg, CircSupply: circSupplyDetail.FilCirculating, BaseFee: basefee, + Rand: recordingRand, }) if err != nil { return fmt.Errorf("failed to execute message: %w", err) @@ -356,7 +363,8 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { {Source: fmt.Sprintf("execution_tipset:%s", execTs.Key().String())}, {Source: "github.com/filecoin-project/lotus", Version: version.String()}}, }, - CAR: out.Bytes(), + Randomness: recordingRand.Recorded(), + CAR: out.Bytes(), Pre: &schema.Preconditions{ Epoch: int64(execTs.Height()), CircSupply: circSupply.Int, diff --git a/conformance/rand_record.go b/conformance/rand_record.go new file mode 100644 index 000000000..2d3477c94 --- /dev/null +++ b/conformance/rand_record.go @@ -0,0 +1,86 @@ +package conformance + +import ( + "context" + "sync" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/crypto" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/vm" + "github.com/filecoin-project/test-vectors/schema" +) + +type RecordingRand struct { + reporter Reporter + api api.FullNode + + lk sync.Mutex + recorded schema.Randomness +} + +var _ vm.Rand = (*RecordingRand)(nil) + +// NewRecordingRand returns a vm.Rand implementation that proxies calls to a +// full Lotus node via JSON-RPC, and records matching rules and responses so +// they can later be embedded in test vectors. +func NewRecordingRand(reporter Reporter, api api.FullNode) *RecordingRand { + return &RecordingRand{reporter: reporter, api: api} +} + +func (r *RecordingRand) GetChainRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) { + ret, err := r.api.ChainGetRandomnessFromTickets(ctx, types.EmptyTSK, pers, round, entropy) + if err != nil { + return ret, err + } + + r.reporter.Logf("fetched and recorded chain randomness for: dst=%d, epoch=%d, entropy=%x, result=%x", pers, round, entropy, ret) + + match := schema.RandomnessMatch{ + On: schema.RandomnessRule{ + Kind: schema.RandomnessChain, + DomainSeparationTag: int64(pers), + Epoch: int64(round), + Entropy: entropy, + }, + Return: []byte(ret), + } + r.lk.Lock() + r.recorded = append(r.recorded, match) + r.lk.Unlock() + + return ret, err +} + +func (r *RecordingRand) GetBeaconRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) { + ret, err := r.api.ChainGetRandomnessFromBeacon(ctx, types.EmptyTSK, pers, round, entropy) + if err != nil { + return ret, err + } + + r.reporter.Logf("fetched and recorded beacon randomness for: dst=%d, epoch=%d, entropy=%x, result=%x", pers, round, entropy, ret) + + match := schema.RandomnessMatch{ + On: schema.RandomnessRule{ + Kind: schema.RandomnessBeacon, + DomainSeparationTag: int64(pers), + Epoch: int64(round), + Entropy: entropy, + }, + Return: []byte(ret), + } + r.lk.Lock() + r.recorded = append(r.recorded, match) + r.lk.Unlock() + + return ret, err +} + +func (r *RecordingRand) Recorded() schema.Randomness { + r.lk.Lock() + defer r.lk.Unlock() + + return r.recorded +} diff --git a/conformance/rand_replay.go b/conformance/rand_replay.go new file mode 100644 index 000000000..b820b8ced --- /dev/null +++ b/conformance/rand_replay.go @@ -0,0 +1,78 @@ +package conformance + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/crypto" + + "github.com/filecoin-project/lotus/chain/vm" + "github.com/filecoin-project/test-vectors/schema" +) + +type ReplayingRand struct { + reporter Reporter + recorded schema.Randomness + fallback vm.Rand +} + +var _ vm.Rand = (*ReplayingRand)(nil) + +// NewReplayingRand replays recorded randomness when requested, falling back to +// fixed randomness if the value cannot be found; hence this is a safe +// backwards-compatible replacement for fixedRand. +func NewReplayingRand(reporter Reporter, recorded schema.Randomness) *ReplayingRand { + return &ReplayingRand{ + reporter: reporter, + recorded: recorded, + fallback: NewFixedRand(), + } +} + +func (r *ReplayingRand) match(requested schema.RandomnessRule) ([]byte, bool) { + for _, other := range r.recorded { + if other.On.Kind == requested.Kind && + other.On.Epoch == requested.Epoch && + other.On.DomainSeparationTag == requested.DomainSeparationTag && + bytes.Equal(other.On.Entropy, requested.Entropy) { + return other.Return, true + } + } + return nil, false +} + +func (r *ReplayingRand) GetChainRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) { + rule := schema.RandomnessRule{ + Kind: schema.RandomnessChain, + DomainSeparationTag: int64(pers), + Epoch: int64(round), + Entropy: entropy, + } + + if ret, ok := r.match(rule); ok { + r.reporter.Logf("returning saved chain randomness: dst=%d, epoch=%d, entropy=%x, result=%x", pers, round, entropy, ret) + return ret, nil + } + + r.reporter.Logf("returning fallback chain randomness: dst=%d, epoch=%d, entropy=%x", pers, round, entropy) + return r.fallback.GetChainRandomness(ctx, pers, round, entropy) +} + +func (r *ReplayingRand) GetBeaconRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) { + rule := schema.RandomnessRule{ + Kind: schema.RandomnessBeacon, + DomainSeparationTag: int64(pers), + Epoch: int64(round), + Entropy: entropy, + } + + if ret, ok := r.match(rule); ok { + r.reporter.Logf("returning saved beacon randomness: dst=%d, epoch=%d, entropy=%x, result=%x", pers, round, entropy, ret) + return ret, nil + } + + r.reporter.Logf("returning fallback beacon randomness: dst=%d, epoch=%d, entropy=%x", pers, round, entropy) + return r.fallback.GetBeaconRandomness(ctx, pers, round, entropy) + +} diff --git a/conformance/runner.go b/conformance/runner.go index a240c9067..3897ad853 100644 --- a/conformance/runner.go +++ b/conformance/runner.go @@ -66,6 +66,7 @@ func ExecuteMessageVector(r Reporter, vector *schema.TestVector) { Message: msg, BaseFee: BaseFeeOrDefault(vector.Pre.BaseFee), CircSupply: CircSupplyOrDefault(vector.Pre.CircSupply), + Rand: NewReplayingRand(r, vector.Randomness), }) if err != nil { r.Fatalf("fatal failure when executing message: %s", err) diff --git a/go.mod b/go.mod index 170dc3003..e401af36d 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,7 @@ require ( github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b github.com/filecoin-project/specs-actors v0.9.11 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 - github.com/filecoin-project/test-vectors/schema v0.0.3 + github.com/filecoin-project/test-vectors/schema v0.0.4-0.20201007174510-548c9399aa6a github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 github.com/go-kit/kit v0.10.0 github.com/go-ole/go-ole v1.2.4 // indirect diff --git a/go.sum b/go.sum index e820ede3a..7d6c743ae 100644 --- a/go.sum +++ b/go.sum @@ -274,8 +274,8 @@ github.com/filecoin-project/specs-actors v0.9.11 h1:TnpG7HAeiUrfj0mJM7UaPW0P2137 github.com/filecoin-project/specs-actors v0.9.11/go.mod h1:czlvLQGEX0fjLLfdNHD7xLymy6L3n7aQzRWzsYGf+ys= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= -github.com/filecoin-project/test-vectors/schema v0.0.3 h1:1zuBo25B3016inbygYLgYFdpJ2m1BDTbAOCgABRleiU= -github.com/filecoin-project/test-vectors/schema v0.0.3/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E= +github.com/filecoin-project/test-vectors/schema v0.0.4-0.20201007174510-548c9399aa6a h1:2jRLaT3/sHyAinWR2asCAkvzcnOAIi13eTlWf3YEVvY= +github.com/filecoin-project/test-vectors/schema v0.0.4-0.20201007174510-548c9399aa6a/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6 h1:u/UEqS66A5ckRmS4yNpjmVH56sVtS/RfclBAYocb4as= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ= From e803cf151f1dd6df0dd30681a44665a07e7b4ef9 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 6 Oct 2020 15:22:54 -0700 Subject: [PATCH 588/885] introduce separate state-tree versions Instead of versioning the state tree along with the actors, version it separately. This structure may not upgrade every time we update actors. --- chain/gen/genesis/genesis.go | 3 +- chain/state/statetree.go | 56 ++++++++++++++++++-------- chain/state/statetree_test.go | 13 +++--- chain/stmgr/forks.go | 2 +- chain/types/cbor_gen.go | 16 ++++---- chain/types/state.go | 17 ++++++-- gen/main.go | 2 +- lotuspond/front/src/chain/methods.json | 3 +- 8 files changed, 73 insertions(+), 39 deletions(-) diff --git a/chain/gen/genesis/genesis.go b/chain/gen/genesis/genesis.go index f532b9f5e..9f15ecaed 100644 --- a/chain/gen/genesis/genesis.go +++ b/chain/gen/genesis/genesis.go @@ -26,7 +26,6 @@ import ( adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" @@ -117,7 +116,7 @@ func MakeInitialStateTree(ctx context.Context, bs bstore.Blockstore, template ge return nil, nil, xerrors.Errorf("putting empty object: %w", err) } - state, err := state.NewStateTree(cst, actors.Version0) + state, err := state.NewStateTree(cst, types.StateTreeVersion0) if err != nil { return nil, nil, xerrors.Errorf("making new state tree: %w", err) } diff --git a/chain/state/statetree.go b/chain/state/statetree.go index 3f9597420..e9b76ea77 100644 --- a/chain/state/statetree.go +++ b/chain/state/statetree.go @@ -13,6 +13,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/chain/actors" init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init" cbg "github.com/whyrusleeping/cbor-gen" @@ -26,7 +27,7 @@ var log = logging.Logger("statetree") // StateTree stores actors state by their ID. type StateTree struct { root adt.Map - version actors.Version // TODO + version types.StateTreeVersion info cid.Cid Store cbor.IpldStore @@ -120,21 +121,41 @@ func (ss *stateSnaps) deleteActor(addr address.Address) { ss.layers[len(ss.layers)-1].actors[addr] = streeOp{Delete: true} } -func NewStateTree(cst cbor.IpldStore, version actors.Version) (*StateTree, error) { +// VersionForNetwork returns the state tree version for the given network +// version. +func VersionForNetwork(ver network.Version) types.StateTreeVersion { + if actors.VersionForNetwork(ver) == actors.Version0 { + return types.StateTreeVersion0 + } + return types.StateTreeVersion1 +} + +func adtForSTVersion(ver types.StateTreeVersion) actors.Version { + switch ver { + case types.StateTreeVersion0: + return actors.Version0 + case types.StateTreeVersion1: + return actors.Version2 + default: + panic("unhandled state tree version") + } +} + +func NewStateTree(cst cbor.IpldStore, ver types.StateTreeVersion) (*StateTree, error) { var info cid.Cid - switch version { - case actors.Version0: + switch ver { + case types.StateTreeVersion0: // info is undefined - case actors.Version2: + case types.StateTreeVersion1: var err error - info, err = cst.Put(context.TODO(), new(types.StateInfo)) + info, err = cst.Put(context.TODO(), new(types.StateInfo0)) if err != nil { return nil, err } default: - return nil, xerrors.Errorf("unsupported state tree version: %d", version) + return nil, xerrors.Errorf("unsupported state tree version: %d", ver) } - root, err := adt.NewMap(adt.WrapStore(context.TODO(), cst), version) + root, err := adt.NewMap(adt.WrapStore(context.TODO(), cst), adtForSTVersion(ver)) if err != nil { return nil, err } @@ -142,7 +163,7 @@ func NewStateTree(cst cbor.IpldStore, version actors.Version) (*StateTree, error return &StateTree{ root: root, info: info, - version: version, + version: ver, Store: cst, snaps: newStateSnaps(), }, nil @@ -154,13 +175,16 @@ func LoadStateTree(cst cbor.IpldStore, c cid.Cid) (*StateTree, error) { if err := cst.Get(context.TODO(), c, &root); err != nil { // We failed to decode as the new version, must be an old version. root.Actors = c - root.Version = actors.Version0 + root.Version = types.StateTreeVersion0 } switch root.Version { - case actors.Version0, actors.Version2: + case types.StateTreeVersion0, types.StateTreeVersion1: // Load the actual state-tree HAMT. - nd, err := adt.AsMap(adt.WrapStore(context.TODO(), cst), root.Actors, actors.Version(root.Version)) + nd, err := adt.AsMap( + adt.WrapStore(context.TODO(), cst), root.Actors, + adtForSTVersion(root.Version), + ) if err != nil { log.Errorf("loading hamt node %s failed: %s", c, err) return nil, err @@ -169,7 +193,7 @@ func LoadStateTree(cst cbor.IpldStore, c cid.Cid) (*StateTree, error) { return &StateTree{ root: nd, info: root.Info, - version: actors.Version(root.Version), + version: root.Version, Store: cst, snaps: newStateSnaps(), }, nil @@ -309,11 +333,11 @@ func (st *StateTree) Flush(ctx context.Context) (cid.Cid, error) { return cid.Undef, xerrors.Errorf("failed to flush state-tree hamt: %w", err) } // If we're version 0, return a raw tree. - if st.version == actors.Version0 { + if st.version == types.StateTreeVersion0 { return root, nil } // Otherwise, return a versioned tree. - return st.Store.Put(ctx, &types.StateRoot{Version: uint64(st.version), Actors: root, Info: st.info}) + return st.Store.Put(ctx, &types.StateRoot{Version: st.version, Actors: root, Info: st.info}) } func (st *StateTree) Snapshot(ctx context.Context) error { @@ -400,7 +424,7 @@ func (st *StateTree) ForEach(f func(address.Address, *types.Actor) error) error } // Version returns the version of the StateTree data structure in use. -func (st *StateTree) Version() actors.Version { +func (st *StateTree) Version() types.StateTreeVersion { return st.version } diff --git a/chain/state/statetree_test.go b/chain/state/statetree_test.go index 3b08a4b53..ed1fb1889 100644 --- a/chain/state/statetree_test.go +++ b/chain/state/statetree_test.go @@ -13,13 +13,12 @@ import ( "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/types" ) func BenchmarkStateTreeSet(b *testing.B) { cst := cbor.NewMemCborStore() - st, err := NewStateTree(cst, actors.VersionForNetwork(build.NewestNetworkVersion)) + st, err := NewStateTree(cst, VersionForNetwork(build.NewestNetworkVersion)) if err != nil { b.Fatal(err) } @@ -46,7 +45,7 @@ func BenchmarkStateTreeSet(b *testing.B) { func BenchmarkStateTreeSetFlush(b *testing.B) { cst := cbor.NewMemCborStore() - st, err := NewStateTree(cst, actors.VersionForNetwork(build.NewestNetworkVersion)) + st, err := NewStateTree(cst, VersionForNetwork(build.NewestNetworkVersion)) if err != nil { b.Fatal(err) } @@ -76,7 +75,7 @@ func BenchmarkStateTreeSetFlush(b *testing.B) { func BenchmarkStateTree10kGetActor(b *testing.B) { cst := cbor.NewMemCborStore() - st, err := NewStateTree(cst, actors.VersionForNetwork(build.NewestNetworkVersion)) + st, err := NewStateTree(cst, VersionForNetwork(build.NewestNetworkVersion)) if err != nil { b.Fatal(err) } @@ -118,7 +117,7 @@ func BenchmarkStateTree10kGetActor(b *testing.B) { func TestSetCache(t *testing.T) { cst := cbor.NewMemCborStore() - st, err := NewStateTree(cst, actors.VersionForNetwork(build.NewestNetworkVersion)) + st, err := NewStateTree(cst, VersionForNetwork(build.NewestNetworkVersion)) if err != nil { t.Fatal(err) } @@ -155,7 +154,7 @@ func TestSetCache(t *testing.T) { func TestSnapshots(t *testing.T) { ctx := context.Background() cst := cbor.NewMemCborStore() - st, err := NewStateTree(cst, actors.VersionForNetwork(build.NewestNetworkVersion)) + st, err := NewStateTree(cst, VersionForNetwork(build.NewestNetworkVersion)) if err != nil { t.Fatal(err) } @@ -239,7 +238,7 @@ func assertNotHas(t *testing.T, st *StateTree, addr address.Address) { func TestStateTreeConsistency(t *testing.T) { cst := cbor.NewMemCborStore() // TODO: ActorUpgrade: this test tests pre actors v2 - st, err := NewStateTree(cst, actors.VersionForNetwork(network.Version3)) + st, err := NewStateTree(cst, VersionForNetwork(network.Version3)) if err != nil { t.Fatal(err) } diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index d5d0dbf7e..fd6058127 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -494,7 +494,7 @@ func UpgradeActorsV2(ctx context.Context, sm *StateManager, cb ExecCallback, roo epoch := ts.Height() - 1 - info, err := store.Put(ctx, new(types.StateInfo)) + info, err := store.Put(ctx, new(types.StateInfo0)) if err != nil { return cid.Undef, xerrors.Errorf("failed to create new state info for actors v2: %w", err) } diff --git a/chain/types/cbor_gen.go b/chain/types/cbor_gen.go index f95df33bc..d063ce8c9 100644 --- a/chain/types/cbor_gen.go +++ b/chain/types/cbor_gen.go @@ -1648,7 +1648,7 @@ func (t *StateRoot) MarshalCBOR(w io.Writer) error { scratch := make([]byte, 9) - // t.Version (uint64) (uint64) + // t.Version (types.StateTreeVersion) (uint64) if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Version)); err != nil { return err @@ -1687,7 +1687,7 @@ func (t *StateRoot) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.Version (uint64) (uint64) + // t.Version (types.StateTreeVersion) (uint64) { @@ -1698,7 +1698,7 @@ func (t *StateRoot) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Version = uint64(extra) + t.Version = StateTreeVersion(extra) } // t.Actors (cid.Cid) (struct) @@ -1728,22 +1728,22 @@ func (t *StateRoot) UnmarshalCBOR(r io.Reader) error { return nil } -var lengthBufStateInfo = []byte{128} +var lengthBufStateInfo0 = []byte{128} -func (t *StateInfo) MarshalCBOR(w io.Writer) error { +func (t *StateInfo0) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write(lengthBufStateInfo); err != nil { + if _, err := w.Write(lengthBufStateInfo0); err != nil { return err } return nil } -func (t *StateInfo) UnmarshalCBOR(r io.Reader) error { - *t = StateInfo{} +func (t *StateInfo0) UnmarshalCBOR(r io.Reader) error { + *t = StateInfo0{} br := cbg.GetPeeker(r) scratch := make([]byte, 8) diff --git a/chain/types/state.go b/chain/types/state.go index b99eb19c2..a96883604 100644 --- a/chain/types/state.go +++ b/chain/types/state.go @@ -2,9 +2,20 @@ package types import "github.com/ipfs/go-cid" +// StateTreeVersion is the version of the state tree itself, independent of the +// network version or the actors version. +type StateTreeVersion uint64 + +const ( + // StateTreeVersion0 corresponds to actors < v2. + StateTreeVersion0 StateTreeVersion = iota + // StateTreeVersion1 corresponds to actors >= v2. + StateTreeVersion1 +) + type StateRoot struct { - // State root version. Versioned along with actors (for now). - Version uint64 + // State tree version. + Version StateTreeVersion // Actors tree. The structure depends on the state root version. Actors cid.Cid // Info. The structure depends on the state root version. @@ -12,4 +23,4 @@ type StateRoot struct { } // TODO: version this. -type StateInfo struct{} +type StateInfo0 struct{} diff --git a/gen/main.go b/gen/main.go index bcb43a8f0..d5874af2c 100644 --- a/gen/main.go +++ b/gen/main.go @@ -27,7 +27,7 @@ func main() { types.ExpTipSet{}, types.BeaconEntry{}, types.StateRoot{}, - types.StateInfo{}, + types.StateInfo0{}, ) if err != nil { fmt.Println(err) diff --git a/lotuspond/front/src/chain/methods.json b/lotuspond/front/src/chain/methods.json index 5e15b053b..b271bfae5 100644 --- a/lotuspond/front/src/chain/methods.json +++ b/lotuspond/front/src/chain/methods.json @@ -176,7 +176,8 @@ "CompactPartitions", "CompactSectorNumbers", "ConfirmUpdateWorkerKey", - "RepayDebt" + "RepayDebt", + "ChangeOwnerAddress" ], "fil/2/storagepower": [ "Send", From c17fa4bc35d3a7847fcc50a967fce468b56993f9 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 6 Oct 2020 16:58:18 -0700 Subject: [PATCH 589/885] update test-vectors for StateManager constructor change --- extern/test-vectors | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/test-vectors b/extern/test-vectors index 3a6e0b5e0..56f0676eb 160000 --- a/extern/test-vectors +++ b/extern/test-vectors @@ -1 +1 @@ -Subproject commit 3a6e0b5e069b1452ce1a032aa315354d645f3ec4 +Subproject commit 56f0676eb3be4b1e1dea892eea330614de755177 From cca17f607898615ded848d89a5de27c139413f90 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 6 Oct 2020 17:51:43 -0700 Subject: [PATCH 590/885] fix state tree version in v2 upgrade --- chain/actors/version.go | 4 ++-- chain/stmgr/forks.go | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/chain/actors/version.go b/chain/actors/version.go index 385ff592f..17af8b08b 100644 --- a/chain/actors/version.go +++ b/chain/actors/version.go @@ -9,8 +9,8 @@ import ( type Version int const ( - Version0 = 0 - Version2 = 2 + Version0 Version = 0 + Version2 Version = 2 ) // Converts a network version into an actors adt version. diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index fd6058127..52c764ba2 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -25,7 +25,6 @@ import ( states2 "github.com/filecoin-project/specs-actors/v2/actors/states" "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/adt" init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init" "github.com/filecoin-project/lotus/chain/actors/builtin/multisig" @@ -518,8 +517,7 @@ func UpgradeActorsV2(ctx context.Context, sm *StateManager, cb ExecCallback, roo } newRoot, err := store.Put(ctx, &types.StateRoot{ - // TODO: ActorUpgrade: should be state-tree specific, not just the actors version. - Version: actors.Version2, + Version: types.StateTreeVersion1, Actors: newHamtRoot, Info: info, }) From c02daca9e973354a75660c18bf1cb921e4ff096e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Wed, 7 Oct 2020 20:12:35 +0100 Subject: [PATCH 591/885] work around #4223. --- conformance/rand_record.go | 23 ++++++++++++++++++++--- conformance/rand_replay.go | 3 ++- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/conformance/rand_record.go b/conformance/rand_record.go index 2d3477c94..6f6d064dc 100644 --- a/conformance/rand_record.go +++ b/conformance/rand_record.go @@ -2,21 +2,28 @@ package conformance import ( "context" + "fmt" "sync" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/crypto" + "github.com/filecoin-project/test-vectors/schema" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" - "github.com/filecoin-project/test-vectors/schema" ) type RecordingRand struct { reporter Reporter api api.FullNode + // once guards the loading of the head tipset. + // can be removed when https://github.com/filecoin-project/lotus/issues/4223 + // is fixed. + once sync.Once + head types.TipSetKey lk sync.Mutex recorded schema.Randomness } @@ -30,8 +37,17 @@ func NewRecordingRand(reporter Reporter, api api.FullNode) *RecordingRand { return &RecordingRand{reporter: reporter, api: api} } +func (r *RecordingRand) loadHead() { + head, err := r.api.ChainHead(context.Background()) + if err != nil { + panic(fmt.Sprintf("could not fetch chain head while fetching randomness: %s", err)) + } + r.head = head.Key() +} + func (r *RecordingRand) GetChainRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) { - ret, err := r.api.ChainGetRandomnessFromTickets(ctx, types.EmptyTSK, pers, round, entropy) + r.once.Do(r.loadHead) + ret, err := r.api.ChainGetRandomnessFromTickets(ctx, r.head, pers, round, entropy) if err != nil { return ret, err } @@ -55,7 +71,8 @@ func (r *RecordingRand) GetChainRandomness(ctx context.Context, pers crypto.Doma } func (r *RecordingRand) GetBeaconRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) { - ret, err := r.api.ChainGetRandomnessFromBeacon(ctx, types.EmptyTSK, pers, round, entropy) + r.once.Do(r.loadHead) + ret, err := r.api.ChainGetRandomnessFromBeacon(ctx, r.head, pers, round, entropy) if err != nil { return ret, err } diff --git a/conformance/rand_replay.go b/conformance/rand_replay.go index b820b8ced..1b73e5a08 100644 --- a/conformance/rand_replay.go +++ b/conformance/rand_replay.go @@ -7,8 +7,9 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/crypto" - "github.com/filecoin-project/lotus/chain/vm" "github.com/filecoin-project/test-vectors/schema" + + "github.com/filecoin-project/lotus/chain/vm" ) type ReplayingRand struct { From 6f4a76266441873da7d5f19dad57a9d610c83869 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 7 Oct 2020 21:13:04 +0200 Subject: [PATCH 592/885] Fix Messages field in MessagePoolEvtMessage journal entry Signed-off-by: Jakub Sztandera --- chain/messagepool/repub.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/chain/messagepool/repub.go b/chain/messagepool/repub.go index db31e18c2..672119ba9 100644 --- a/chain/messagepool/repub.go +++ b/chain/messagepool/repub.go @@ -149,13 +149,13 @@ loop: if len(msgs) > 0 { journal.J.RecordEvent(mp.evtTypes[evtTypeMpoolRepub], func() interface{} { - msgs := make([]MessagePoolEvtMessage, 0, len(msgs)) + msgsEv := make([]MessagePoolEvtMessage, 0, len(msgs)) for _, m := range msgs { - msgs = append(msgs, MessagePoolEvtMessage{Message: m.Message, CID: m.Cid()}) + msgsEv = append(msgsEv, MessagePoolEvtMessage{Message: m.Message, CID: m.Cid()}) } return MessagePoolEvt{ Action: "repub", - Messages: msgs, + Messages: msgsEv, } }) } From 767c346cf82dd2a41d72926dd2b406707f049a0c Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 7 Oct 2020 12:17:32 -0700 Subject: [PATCH 593/885] update test-vectors --- extern/test-vectors | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/test-vectors b/extern/test-vectors index 56f0676eb..7471e2805 160000 --- a/extern/test-vectors +++ b/extern/test-vectors @@ -1 +1 @@ -Subproject commit 56f0676eb3be4b1e1dea892eea330614de755177 +Subproject commit 7471e2805fc3e459e4ee325775633e8ec76cb7c6 From e55b00f3582215e974cf6534eb3e09fbe411ee9d Mon Sep 17 00:00:00 2001 From: Jennifer <42981373+jennijuju@users.noreply.github.com> Date: Wed, 7 Oct 2020 17:07:19 -0400 Subject: [PATCH 594/885] Increase the message fee cap upper bounds in PCR. --- cmd/lotus-pcr/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/lotus-pcr/main.go b/cmd/lotus-pcr/main.go index 5b59ff03d..df329d8d3 100644 --- a/cmd/lotus-pcr/main.go +++ b/cmd/lotus-pcr/main.go @@ -392,13 +392,13 @@ var runCmd = &cli.Command{ Name: "pre-fee-cap-max", EnvVars: []string{"LOTUS_PCR_PRE_FEE_CAP_MAX"}, Usage: "messages with a fee cap larger than this will be skipped when processing pre commit messages", - Value: "0.0000000001", + Value: "0.000000001", }, &cli.StringFlag{ Name: "prove-fee-cap-max", EnvVars: []string{"LOTUS_PCR_PROVE_FEE_CAP_MAX"}, Usage: "messages with a prove cap larger than this will be skipped when processing pre commit messages", - Value: "0.0000000001", + Value: "0.000000001", }, }, Action: func(cctx *cli.Context) error { From ae3691e0a2ee49858ba2d2f7f8dda26f728462c9 Mon Sep 17 00:00:00 2001 From: Travis Person Date: Wed, 7 Oct 2020 21:53:56 +0000 Subject: [PATCH 595/885] lotus-pcr: zero refund --- cmd/lotus-pcr/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/lotus-pcr/main.go b/cmd/lotus-pcr/main.go index 5b59ff03d..a1406789e 100644 --- a/cmd/lotus-pcr/main.go +++ b/cmd/lotus-pcr/main.go @@ -1025,9 +1025,9 @@ func (r *refunder) ProcessTipset(ctx context.Context, tipset *types.TipSet, refu return nil, nil } - refundValue := types.NewInt(0) tipsetRefunds := NewMinersRefund() for i, msg := range msgs { + refundValue := types.NewInt(0) m := msg.Message a, err := r.api.StateGetActor(ctx, m.To, tipset.Key()) From 12cd478e11c5ee6e1eadab98392710c379f1f93a Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Wed, 7 Oct 2020 15:14:46 -0700 Subject: [PATCH 596/885] Add message counts to inspect chain output --- cli/chain.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/cli/chain.go b/cli/chain.go index 20a8b7797..763752f23 100644 --- a/cli/chain.go +++ b/cli/chain.go @@ -449,11 +449,16 @@ var chainInspectUsage = &cli.Command{ bySender := make(map[string]int64) byDest := make(map[string]int64) byMethod := make(map[string]int64) + bySenderC := make(map[string]int64) + byDestC := make(map[string]int64) + byMethodC := make(map[string]int64) var sum int64 for _, m := range msgs { bySender[m.Message.From.String()] += m.Message.GasLimit + bySenderC[m.Message.From.String()]++ byDest[m.Message.To.String()] += m.Message.GasLimit + byDestC[m.Message.To.String()]++ sum += m.Message.GasLimit code, err := lookupActorCode(m.Message.To) @@ -464,7 +469,7 @@ var chainInspectUsage = &cli.Command{ mm := stmgr.MethodsMap[code][m.Message.Method] byMethod[mm.Name] += m.Message.GasLimit - + byMethodC[mm.Name]++ } type keyGasPair struct { @@ -496,19 +501,19 @@ var chainInspectUsage = &cli.Command{ fmt.Printf("By Sender:\n") for i := 0; i < numRes && i < len(senderVals); i++ { sv := senderVals[i] - fmt.Printf("%s\t%0.2f%%\t(%d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas) + fmt.Printf("%s\t%0.2f%%\t(total: %d, count: %d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas, bySenderC[sv.Key]) } fmt.Println() fmt.Printf("By Receiver:\n") for i := 0; i < numRes && i < len(destVals); i++ { sv := destVals[i] - fmt.Printf("%s\t%0.2f%%\t(%d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas) + fmt.Printf("%s\t%0.2f%%\t(total: %d, count: %d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas, byDestC[sv.Key]) } fmt.Println() fmt.Printf("By Method:\n") for i := 0; i < numRes && i < len(methodVals); i++ { sv := methodVals[i] - fmt.Printf("%s\t%0.2f%%\t(%d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas) + fmt.Printf("%s\t%0.2f%%\t(total: %d, count: %d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas, byMethodC[sv.Key]) } return nil From fe912223bdfa000b008c143b21f2cca1c0ac7388 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 7 Oct 2020 13:56:38 -0700 Subject: [PATCH 597/885] pass an explicit upgrade height to migrations The tipset height may not be the correct one, given null blocks. --- chain/stmgr/forks.go | 23 +++++++++++++---------- chain/stmgr/forks_test.go | 2 +- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index 52c764ba2..b897c86eb 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -33,7 +33,14 @@ import ( "github.com/filecoin-project/lotus/chain/vm" ) -type UpgradeFunc func(context.Context, *StateManager, ExecCallback, cid.Cid, *types.TipSet) (cid.Cid, error) +// UpgradeFunc is a migration function run ate very upgrade. +// +// - The oldState is the state produced by the upgrade epoch. +// - The returned newState is the new state that will be used by the next epoch. +// - The height is the upgrade epoch height (already executed). +// - The tipset is the tipset for the last non-null block before the upgrade. Do +// not assume that ts.Height() is the upgrade height. +type UpgradeFunc func(ctx context.Context, sm *StateManager, cb ExecCallback, oldState cid.Cid, height abi.ChainEpoch, ts *types.TipSet) (newState cid.Cid, err error) type Upgrade struct { Height abi.ChainEpoch @@ -108,7 +115,7 @@ func (sm *StateManager) handleStateForks(ctx context.Context, root cid.Cid, heig var err error f, ok := sm.stateMigrations[height] if ok { - retCid, err = f(ctx, sm, cb, root, ts) + retCid, err = f(ctx, sm, cb, root, height, ts) if err != nil { return cid.Undef, err } @@ -179,7 +186,7 @@ func doTransfer(cb ExecCallback, tree types.StateTree, from, to address.Address, return nil } -func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, ts *types.TipSet) (cid.Cid, error) { +func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) { // Some initial parameters FundsForMiners := types.FromFil(1_000_000) LookbackEpoch := abi.ChainEpoch(32000) @@ -431,11 +438,9 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCal return tree.Flush(ctx) } -func UpgradeIgnition(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, ts *types.TipSet) (cid.Cid, error) { +func UpgradeIgnition(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) { store := sm.cs.Store(ctx) - epoch := ts.Height() - 1 - if build.UpgradeLiftoffHeight <= epoch { return cid.Undef, xerrors.Errorf("liftoff height must be beyond ignition height") } @@ -488,11 +493,9 @@ func UpgradeIgnition(ctx context.Context, sm *StateManager, cb ExecCallback, roo return tree.Flush(ctx) } -func UpgradeActorsV2(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, ts *types.TipSet) (cid.Cid, error) { +func UpgradeActorsV2(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) { store := sm.cs.Store(ctx) - epoch := ts.Height() - 1 - info, err := store.Put(ctx, new(types.StateInfo0)) if err != nil { return cid.Undef, xerrors.Errorf("failed to create new state info for actors v2: %w", err) @@ -539,7 +542,7 @@ func UpgradeActorsV2(ctx context.Context, sm *StateManager, cb ExecCallback, roo return newRoot, nil } -func UpgradeLiftoff(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, ts *types.TipSet) (cid.Cid, error) { +func UpgradeLiftoff(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) { tree, err := sm.StateTree(root) if err != nil { return cid.Undef, xerrors.Errorf("getting state tree: %w", err) diff --git a/chain/stmgr/forks_test.go b/chain/stmgr/forks_test.go index bb03f13b9..daa39a8d6 100644 --- a/chain/stmgr/forks_test.go +++ b/chain/stmgr/forks_test.go @@ -120,7 +120,7 @@ func TestForkHeightTriggers(t *testing.T) { Network: 1, Height: testForkHeight, Migration: func(ctx context.Context, sm *StateManager, cb ExecCallback, - root cid.Cid, ts *types.TipSet) (cid.Cid, error) { + root cid.Cid, height abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) { cst := ipldcbor.NewCborStore(sm.ChainStore().Blockstore()) st, err := sm.StateTree(root) From dab1107f5b86f6f6dbef8e630b92fc9d0b4efac7 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 7 Oct 2020 14:35:07 -0700 Subject: [PATCH 598/885] avoid estimating gas and explicitly calling blocks on fork tipsets These tipsets can be slow. --- chain/stmgr/call.go | 39 +++++++++++++++++++++++++++++++++++++++ chain/stmgr/forks.go | 5 +++++ node/impl/full/gas.go | 13 ++++++++++++- 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/chain/stmgr/call.go b/chain/stmgr/call.go index 15cbac53e..0497a20ed 100644 --- a/chain/stmgr/call.go +++ b/chain/stmgr/call.go @@ -2,6 +2,7 @@ package stmgr import ( "context" + "errors" "fmt" "github.com/filecoin-project/go-address" @@ -17,17 +18,37 @@ import ( "github.com/filecoin-project/lotus/chain/vm" ) +var ErrWouldFork = errors.New("refusing explicit call due to state fork at epoch") + func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types.TipSet) (*api.InvocResult, error) { ctx, span := trace.StartSpan(ctx, "statemanager.Call") defer span.End() + // If no tipset is provided, try to find one without a fork. if ts == nil { ts = sm.cs.GetHeaviestTipSet() + + // Search back till we find a height with no fork, or we reach the beginning. + for ts.Height() > 0 && sm.hasStateFork(ctx, ts.Height()-1) { + var err error + ts, err = sm.cs.GetTipSetFromKey(ts.Parents()) + if err != nil { + return nil, xerrors.Errorf("failed to find a non-forking epoch: %w", err) + } + } } bstate := ts.ParentState() bheight := ts.Height() + // If we have to run a migration, and we're not at genesis, return an + // error because the migration will take too long. + // + // We allow this at height 0 for at-genesis migrations (for testing). + if bheight-1 > 0 && sm.hasStateFork(ctx, bheight-1) { + return nil, ErrWouldFork + } + bstate, err := sm.handleStateForks(ctx, bstate, bheight-1, nil, ts) if err != nil { return nil, fmt.Errorf("failed to handle fork: %w", err) @@ -106,6 +127,24 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri if ts == nil { ts = sm.cs.GetHeaviestTipSet() + + // Search back till we find a height with no fork, or we reach the beginning. + // We need the _previous_ height to have no fork, because we'll + // run the fork logic in `sm.TipSetState`. We need the _current_ + // height to have no fork, because we'll run it inside this + // function before executing the given message. + for ts.Height() > 0 && (sm.hasStateFork(ctx, ts.Height()) || sm.hasStateFork(ctx, ts.Height()-1)) { + var err error + ts, err = sm.cs.GetTipSetFromKey(ts.Parents()) + if err != nil { + return nil, xerrors.Errorf("failed to find a non-forking epoch: %w", err) + } + } + } + + // When we're not at the genesis block, make sure we're at a migration height. + if ts.Height() > 0 && (sm.hasStateFork(ctx, ts.Height()) || sm.hasStateFork(ctx, ts.Height()-1)) { + return nil, ErrWouldFork } state, _, err := sm.TipSetState(ctx, ts) diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index b897c86eb..69159f0fe 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -124,6 +124,11 @@ func (sm *StateManager) handleStateForks(ctx context.Context, root cid.Cid, heig return retCid, nil } +func (sm *StateManager) hasStateFork(ctx context.Context, height abi.ChainEpoch) bool { + _, ok := sm.stateMigrations[height] + return ok +} + func doTransfer(cb ExecCallback, tree types.StateTree, from, to address.Address, amt abi.TokenAmount) error { fromAct, err := tree.GetActor(from) if err != nil { diff --git a/node/impl/full/gas.go b/node/impl/full/gas.go index c912c7a8c..1b07aa950 100644 --- a/node/impl/full/gas.go +++ b/node/impl/full/gas.go @@ -160,7 +160,18 @@ func (a *GasAPI) GasEstimateGasLimit(ctx context.Context, msgIn *types.Message, priorMsgs = append(priorMsgs, m) } - res, err := a.Stmgr.CallWithGas(ctx, &msg, priorMsgs, ts) + // Try calling until we find a height with no migration. + var res *api.InvocResult + for { + res, err = a.Stmgr.CallWithGas(ctx, &msg, priorMsgs, ts) + if err != stmgr.ErrWouldFork { + break + } + ts, err = a.Chain.GetTipSetFromKey(ts.Parents()) + if err != nil { + return -1, xerrors.Errorf("getting parent tipset: %w", err) + } + } if err != nil { return -1, xerrors.Errorf("CallWithGas failed: %w", err) } From 9b7b6146ebf6e3c18ecd421a0f148045d937325e Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 7 Oct 2020 15:26:08 -0700 Subject: [PATCH 599/885] construct the new vm with the state manager's vm constructor --- chain/stmgr/call.go | 4 ++-- chain/stmgr/utils.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/chain/stmgr/call.go b/chain/stmgr/call.go index 0497a20ed..14a4359c1 100644 --- a/chain/stmgr/call.go +++ b/chain/stmgr/call.go @@ -65,7 +65,7 @@ func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types. BaseFee: types.NewInt(0), } - vmi, err := vm.NewVM(ctx, vmopt) + vmi, err := sm.newVM(ctx, vmopt) if err != nil { return nil, xerrors.Errorf("failed to set up vm: %w", err) } @@ -177,7 +177,7 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri NtwkVersion: sm.GetNtwkVersion, BaseFee: ts.Blocks()[0].ParentBaseFee, } - vmi, err := vm.NewVM(ctx, vmopt) + vmi, err := sm.newVM(ctx, vmopt) if err != nil { return nil, xerrors.Errorf("failed to set up vm: %w", err) } diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index 2c9c5ad94..c0f0c4d2f 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -387,7 +387,7 @@ func ComputeState(ctx context.Context, sm *StateManager, height abi.ChainEpoch, NtwkVersion: sm.GetNtwkVersion, BaseFee: ts.Blocks()[0].ParentBaseFee, } - vmi, err := vm.NewVM(ctx, vmopt) + vmi, err := sm.newVM(ctx, vmopt) if err != nil { return cid.Undef, nil, err } From 1fea550ce5028997cbe7423b0e9347251dc5878f Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 7 Oct 2020 15:28:16 -0700 Subject: [PATCH 600/885] test that we refuse explicit calls at the migration epochs --- chain/stmgr/forks_test.go | 87 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 84 insertions(+), 3 deletions(-) diff --git a/chain/stmgr/forks_test.go b/chain/stmgr/forks_test.go index daa39a8d6..bd3e960d0 100644 --- a/chain/stmgr/forks_test.go +++ b/chain/stmgr/forks_test.go @@ -10,8 +10,9 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/cbor" "github.com/filecoin-project/specs-actors/actors/builtin" - init_ "github.com/filecoin-project/specs-actors/actors/builtin/init" + init0 "github.com/filecoin-project/specs-actors/actors/builtin/init" "github.com/filecoin-project/specs-actors/actors/runtime" + "github.com/stretchr/testify/require" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/chain/actors" @@ -76,7 +77,7 @@ func (ta testActor) Exports() []interface{} { func (ta *testActor) Constructor(rt runtime.Runtime, params *abi.EmptyValue) *abi.EmptyValue { rt.ValidateImmediateCallerAcceptAny() rt.StateCreate(&testActorState{11}) - fmt.Println("NEW ACTOR ADDRESS IS: ", rt.Receiver()) + //fmt.Println("NEW ACTOR ADDRESS IS: ", rt.Receiver()) return abi.Empty } @@ -173,7 +174,7 @@ func TestForkHeightTriggers(t *testing.T) { var msgs []*types.SignedMessage - enc, err := actors.SerializeParams(&init_.ExecParams{CodeCID: (testActor{}).Code()}) + enc, err := actors.SerializeParams(&init0.ExecParams{CodeCID: (testActor{}).Code()}) if err != nil { t.Fatal(err) } @@ -233,3 +234,83 @@ func TestForkHeightTriggers(t *testing.T) { } } } + +func TestForkRefuseCall(t *testing.T) { + logging.SetAllLoggers(logging.LevelInfo) + + ctx := context.TODO() + + cg, err := gen.NewGenerator() + if err != nil { + t.Fatal(err) + } + + sm, err := NewStateManagerWithUpgradeSchedule( + cg.ChainStore(), UpgradeSchedule{{ + Network: 1, + Height: testForkHeight, + Migration: func(ctx context.Context, sm *StateManager, cb ExecCallback, + root cid.Cid, height abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) { + return root, nil + }}}) + if err != nil { + t.Fatal(err) + } + + inv := vm.NewActorRegistry() + inv.Register(nil, testActor{}) + + sm.SetVMConstructor(func(ctx context.Context, vmopt *vm.VMOpts) (*vm.VM, error) { + nvm, err := vm.NewVM(ctx, vmopt) + if err != nil { + return nil, err + } + nvm.SetInvoker(inv) + return nvm, nil + }) + + cg.SetStateManager(sm) + + enc, err := actors.SerializeParams(&init0.ExecParams{CodeCID: (testActor{}).Code()}) + if err != nil { + t.Fatal(err) + } + + m := &types.Message{ + From: cg.Banker(), + To: lotusinit.Address, + Method: builtin.MethodsInit.Exec, + Params: enc, + GasLimit: types.TestGasLimit, + Value: types.NewInt(0), + GasPremium: types.NewInt(0), + GasFeeCap: types.NewInt(0), + } + + for i := 0; i < 50; i++ { + ts, err := cg.NextTipSet() + if err != nil { + t.Fatal(err) + } + + ret, err := sm.CallWithGas(ctx, m, nil, ts.TipSet.TipSet()) + switch ts.TipSet.TipSet().Height() { + case testForkHeight, testForkHeight + 1: + // If I had a fork, or I _will_ have a fork, it should fail. + require.Equal(t, ErrWouldFork, err) + default: + require.NoError(t, err) + require.True(t, ret.MsgRct.ExitCode.IsSuccess()) + } + // Call just runs on the parent state for a tipset, so we only + // expect an error at the fork height. + ret, err = sm.Call(ctx, m, ts.TipSet.TipSet()) + switch ts.TipSet.TipSet().Height() { + case testForkHeight + 1: + require.Equal(t, ErrWouldFork, err) + default: + require.NoError(t, err) + require.True(t, ret.MsgRct.ExitCode.IsSuccess()) + } + } +} From b0bea5f14558f49a676d471c5c612d97c925158b Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 7 Oct 2020 15:46:56 -0700 Subject: [PATCH 601/885] return an illegal actor error when we see an unsupported actor version As far as the chain is concerned, this actor does not exist. --- chain/vm/invoker.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/chain/vm/invoker.go b/chain/vm/invoker.go index 1e9f04081..661e31178 100644 --- a/chain/vm/invoker.go +++ b/chain/vm/invoker.go @@ -70,12 +70,12 @@ func (ar *ActorRegistry) Invoke(codeCid cid.Cid, rt vmr.Runtime, method abi.Meth log.Errorf("no code for actor %s (Addr: %s)", codeCid, rt.Receiver()) return nil, aerrors.Newf(exitcode.SysErrorIllegalActor, "no code for actor %s(%d)(%s)", codeCid, method, hex.EncodeToString(params)) } + if err := act.predicate(rt, act.vmActor); err != nil { + return nil, aerrors.Newf(exitcode.SysErrorIllegalActor, "unsupported actor: %s", err) + } if method >= abi.MethodNum(len(act.methods)) || act.methods[method] == nil { return nil, aerrors.Newf(exitcode.SysErrInvalidMethod, "no method %d on actor", method) } - if err := act.predicate(rt, act.vmActor); err != nil { - return nil, aerrors.Newf(exitcode.SysErrInvalidMethod, "unsupported actor: %s", err) - } return act.methods[method](rt, params) } From 8f4911ac0d558a978acd3981d526c07792f8e65f Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Wed, 7 Oct 2020 16:00:52 -0700 Subject: [PATCH 602/885] dont print fil suffix in lotus shed audit balances --- chain/types/fil.go | 6 +++++- cmd/lotus-shed/balances.go | 6 +++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/chain/types/fil.go b/chain/types/fil.go index 99a896e38..64f6a52b3 100644 --- a/chain/types/fil.go +++ b/chain/types/fil.go @@ -12,11 +12,15 @@ import ( type FIL BigInt func (f FIL) String() string { + return f.Unitless() + " FIL" +} + +func (f FIL) Unitless() string { r := new(big.Rat).SetFrac(f.Int, big.NewInt(int64(build.FilecoinPrecision))) if r.Sign() == 0 { return "0 FIL" } - return strings.TrimRight(strings.TrimRight(r.FloatString(18), "0"), ".") + " FIL" + return strings.TrimRight(strings.TrimRight(r.FloatString(18), "0"), ".") } func (f FIL) Format(s fmt.State, ch rune) { diff --git a/cmd/lotus-shed/balances.go b/cmd/lotus-shed/balances.go index 1c89a00cf..3aeb0c879 100644 --- a/cmd/lotus-shed/balances.go +++ b/cmd/lotus-shed/balances.go @@ -117,7 +117,7 @@ var chainBalanceCmd = &cli.Command{ fmt.Printf("Address,Balance,Type,Power,Worker,Owner\n") for _, acc := range infos { - fmt.Printf("%s,%s,%s,%s,%s,%s\n", acc.Address, acc.Balance, acc.Type, acc.Power, acc.Worker, acc.Owner) + fmt.Printf("%s,%s,%s,%s,%s,%s\n", acc.Address, acc.Balance.Unitless(), acc.Type, acc.Power, acc.Worker, acc.Owner) } return nil }, @@ -244,12 +244,12 @@ var chainBalanceStateCmd = &cli.Command{ if minerInfo { fmt.Printf("Address,Balance,Type,Sectors,Worker,Owner,InitialPledge,Locked,PreCommits\n") for _, acc := range infos { - fmt.Printf("%s,%s,%s,%d,%s,%s,%s,%s,%s\n", acc.Address, acc.Balance, acc.Type, acc.Sectors, acc.Worker, acc.Owner, acc.InitialPledge, acc.LockedFunds, acc.PreCommits) + fmt.Printf("%s,%s,%s,%d,%s,%s,%s,%s,%s\n", acc.Address, acc.Balance.Unitless(), acc.Type, acc.Sectors, acc.Worker, acc.Owner, acc.InitialPledge, acc.LockedFunds, acc.PreCommits) } } else { fmt.Printf("Address,Balance,Type\n") for _, acc := range infos { - fmt.Printf("%s,%s,%s\n", acc.Address, acc.Balance, acc.Type) + fmt.Printf("%s,%s,%s\n", acc.Address, acc.Balance.Unitless(), acc.Type) } } From a4e954197caf1c10ee67eb6487398640a96028b5 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 7 Oct 2020 16:03:42 -0700 Subject: [PATCH 603/885] retry StateCall at different height if we're at an expensive fork height --- node/impl/full/state.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 3a4390d03..0a5c856bd 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -307,12 +307,22 @@ func (a *StateAPI) StateMinerPower(ctx context.Context, addr address.Address, ts }, nil } -func (a *StateAPI) StateCall(ctx context.Context, msg *types.Message, tsk types.TipSetKey) (*api.InvocResult, error) { +func (a *StateAPI) StateCall(ctx context.Context, msg *types.Message, tsk types.TipSetKey) (res *api.InvocResult, err error) { ts, err := a.Chain.GetTipSetFromKey(tsk) if err != nil { return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - return a.StateManager.Call(ctx, msg, ts) + for { + res, err = a.StateManager.Call(ctx, msg, ts) + if err != stmgr.ErrWouldFork { + break + } + ts, err = a.Chain.GetTipSetFromKey(ts.Parents()) + if err != nil { + return nil, xerrors.Errorf("getting parent tipset: %w", err) + } + } + return res, err } func (a *StateAPI) StateReplay(ctx context.Context, tsk types.TipSetKey, mc cid.Cid) (*api.InvocResult, error) { From e8253d22c6600d8b03733bf1d32fb1b508101fa3 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 7 Oct 2020 16:14:11 -0700 Subject: [PATCH 604/885] only forbid Call* at expensive forks --- chain/stmgr/call.go | 21 +++++++++++---------- chain/stmgr/forks.go | 6 ++++-- chain/stmgr/forks_test.go | 9 +++++---- chain/stmgr/stmgr.go | 23 ++++++++++++++++------- node/impl/full/gas.go | 2 +- node/impl/full/state.go | 2 +- 6 files changed, 38 insertions(+), 25 deletions(-) diff --git a/chain/stmgr/call.go b/chain/stmgr/call.go index 14a4359c1..df3bfa357 100644 --- a/chain/stmgr/call.go +++ b/chain/stmgr/call.go @@ -18,7 +18,7 @@ import ( "github.com/filecoin-project/lotus/chain/vm" ) -var ErrWouldFork = errors.New("refusing explicit call due to state fork at epoch") +var ErrExpensiveFork = errors.New("refusing explicit call due to state fork at epoch") func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types.TipSet) (*api.InvocResult, error) { ctx, span := trace.StartSpan(ctx, "statemanager.Call") @@ -29,7 +29,7 @@ func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types. ts = sm.cs.GetHeaviestTipSet() // Search back till we find a height with no fork, or we reach the beginning. - for ts.Height() > 0 && sm.hasStateFork(ctx, ts.Height()-1) { + for ts.Height() > 0 && sm.hasExpensiveFork(ctx, ts.Height()-1) { var err error ts, err = sm.cs.GetTipSetFromKey(ts.Parents()) if err != nil { @@ -41,14 +41,15 @@ func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types. bstate := ts.ParentState() bheight := ts.Height() - // If we have to run a migration, and we're not at genesis, return an - // error because the migration will take too long. + // If we have to run an expensive migration, and we're not at genesis, + // return an error because the migration will take too long. // // We allow this at height 0 for at-genesis migrations (for testing). - if bheight-1 > 0 && sm.hasStateFork(ctx, bheight-1) { - return nil, ErrWouldFork + if bheight-1 > 0 && sm.hasExpensiveFork(ctx, bheight-1) { + return nil, ErrExpensiveFork } + // Run the (not expensive) migration. bstate, err := sm.handleStateForks(ctx, bstate, bheight-1, nil, ts) if err != nil { return nil, fmt.Errorf("failed to handle fork: %w", err) @@ -133,7 +134,7 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri // run the fork logic in `sm.TipSetState`. We need the _current_ // height to have no fork, because we'll run it inside this // function before executing the given message. - for ts.Height() > 0 && (sm.hasStateFork(ctx, ts.Height()) || sm.hasStateFork(ctx, ts.Height()-1)) { + for ts.Height() > 0 && (sm.hasExpensiveFork(ctx, ts.Height()) || sm.hasExpensiveFork(ctx, ts.Height()-1)) { var err error ts, err = sm.cs.GetTipSetFromKey(ts.Parents()) if err != nil { @@ -142,9 +143,9 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri } } - // When we're not at the genesis block, make sure we're at a migration height. - if ts.Height() > 0 && (sm.hasStateFork(ctx, ts.Height()) || sm.hasStateFork(ctx, ts.Height()-1)) { - return nil, ErrWouldFork + // When we're not at the genesis block, make sure we don't have an expensive migration. + if ts.Height() > 0 && (sm.hasExpensiveFork(ctx, ts.Height()) || sm.hasExpensiveFork(ctx, ts.Height()-1)) { + return nil, ErrExpensiveFork } state, _, err := sm.TipSetState(ctx, ts) diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index 69159f0fe..6aa5e5d87 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -45,6 +45,7 @@ type UpgradeFunc func(ctx context.Context, sm *StateManager, cb ExecCallback, ol type Upgrade struct { Height abi.ChainEpoch Network network.Version + Expensive bool Migration UpgradeFunc } @@ -68,6 +69,7 @@ func DefaultUpgradeSchedule() UpgradeSchedule { }, { Height: build.UpgradeActorsV2Height, Network: network.Version4, + Expensive: true, Migration: UpgradeActorsV2, }, { Height: build.UpgradeLiftoffHeight, @@ -124,8 +126,8 @@ func (sm *StateManager) handleStateForks(ctx context.Context, root cid.Cid, heig return retCid, nil } -func (sm *StateManager) hasStateFork(ctx context.Context, height abi.ChainEpoch) bool { - _, ok := sm.stateMigrations[height] +func (sm *StateManager) hasExpensiveFork(ctx context.Context, height abi.ChainEpoch) bool { + _, ok := sm.expensiveUpgrades[height] return ok } diff --git a/chain/stmgr/forks_test.go b/chain/stmgr/forks_test.go index bd3e960d0..bf1c711e4 100644 --- a/chain/stmgr/forks_test.go +++ b/chain/stmgr/forks_test.go @@ -247,8 +247,9 @@ func TestForkRefuseCall(t *testing.T) { sm, err := NewStateManagerWithUpgradeSchedule( cg.ChainStore(), UpgradeSchedule{{ - Network: 1, - Height: testForkHeight, + Network: 1, + Expensive: true, + Height: testForkHeight, Migration: func(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, height abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) { return root, nil @@ -297,7 +298,7 @@ func TestForkRefuseCall(t *testing.T) { switch ts.TipSet.TipSet().Height() { case testForkHeight, testForkHeight + 1: // If I had a fork, or I _will_ have a fork, it should fail. - require.Equal(t, ErrWouldFork, err) + require.Equal(t, ErrExpensiveFork, err) default: require.NoError(t, err) require.True(t, ret.MsgRct.ExitCode.IsSuccess()) @@ -307,7 +308,7 @@ func TestForkRefuseCall(t *testing.T) { ret, err = sm.Call(ctx, m, ts.TipSet.TipSet()) switch ts.TipSet.TipSet().Height() { case testForkHeight + 1: - require.Equal(t, ErrWouldFork, err) + require.Equal(t, ErrExpensiveFork, err) default: require.NoError(t, err) require.True(t, ret.MsgRct.ExitCode.IsSuccess()) diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index ac01ebb61..d81cf1c72 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -53,6 +53,10 @@ type StateManager struct { // Maps chain epochs to upgrade functions. stateMigrations map[abi.ChainEpoch]UpgradeFunc + // A set of potentially expensive/time consuming upgrades. Explicit + // calls for, e.g., gas estimation fail against this epoch with + // ErrExpensiveFork. + expensiveUpgrades map[abi.ChainEpoch]struct{} stCache map[string][]cid.Cid compWait map[string]chan struct{} @@ -78,6 +82,7 @@ func NewStateManagerWithUpgradeSchedule(cs *store.ChainStore, us UpgradeSchedule } stateMigrations := make(map[abi.ChainEpoch]UpgradeFunc, len(us)) + expensiveUpgrades := make(map[abi.ChainEpoch]struct{}, len(us)) var networkVersions []versionSpec lastVersion := network.Version0 if len(us) > 0 { @@ -87,6 +92,9 @@ func NewStateManagerWithUpgradeSchedule(cs *store.ChainStore, us UpgradeSchedule if upgrade.Migration != nil { stateMigrations[upgrade.Height] = upgrade.Migration } + if upgrade.Expensive { + expensiveUpgrades[upgrade.Height] = struct{}{} + } networkVersions = append(networkVersions, versionSpec{ networkVersion: lastVersion, atOrBelow: upgrade.Height, @@ -99,13 +107,14 @@ func NewStateManagerWithUpgradeSchedule(cs *store.ChainStore, us UpgradeSchedule } return &StateManager{ - networkVersions: networkVersions, - latestVersion: lastVersion, - stateMigrations: stateMigrations, - newVM: vm.NewVM, - cs: cs, - stCache: make(map[string][]cid.Cid), - compWait: make(map[string]chan struct{}), + networkVersions: networkVersions, + latestVersion: lastVersion, + stateMigrations: stateMigrations, + expensiveUpgrades: expensiveUpgrades, + newVM: vm.NewVM, + cs: cs, + stCache: make(map[string][]cid.Cid), + compWait: make(map[string]chan struct{}), }, nil } diff --git a/node/impl/full/gas.go b/node/impl/full/gas.go index 1b07aa950..3580ca26d 100644 --- a/node/impl/full/gas.go +++ b/node/impl/full/gas.go @@ -164,7 +164,7 @@ func (a *GasAPI) GasEstimateGasLimit(ctx context.Context, msgIn *types.Message, var res *api.InvocResult for { res, err = a.Stmgr.CallWithGas(ctx, &msg, priorMsgs, ts) - if err != stmgr.ErrWouldFork { + if err != stmgr.ErrExpensiveFork { break } ts, err = a.Chain.GetTipSetFromKey(ts.Parents()) diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 0a5c856bd..7d654985a 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -314,7 +314,7 @@ func (a *StateAPI) StateCall(ctx context.Context, msg *types.Message, tsk types. } for { res, err = a.StateManager.Call(ctx, msg, ts) - if err != stmgr.ErrWouldFork { + if err != stmgr.ErrExpensiveFork { break } ts, err = a.Chain.GetTipSetFromKey(ts.Parents()) From d97eb10349398e8fae800bf11b2bfc046d96d57e Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 7 Oct 2020 16:32:54 -0700 Subject: [PATCH 605/885] fix spelling nit --- chain/stmgr/forks.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index 6aa5e5d87..1888d1ee9 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -33,7 +33,7 @@ import ( "github.com/filecoin-project/lotus/chain/vm" ) -// UpgradeFunc is a migration function run ate very upgrade. +// UpgradeFunc is a migration function run at every upgrade. // // - The oldState is the state produced by the upgrade epoch. // - The returned newState is the new state that will be used by the next epoch. From d1555106a474bf75c2e5329df02579165ffcbff8 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Tue, 6 Oct 2020 17:47:03 -0400 Subject: [PATCH 606/885] Set actorsv2 upgrade epoch --- build/params_2k.go | 3 +- build/params_testground.go | 3 +- build/params_testnet.go | 5 +-- chain/actors/builtin/builtin.go | 1 + chain/stmgr/forks.go | 68 +++++++++++++++++++++++++++++++++ 5 files changed, 75 insertions(+), 5 deletions(-) diff --git a/build/params_2k.go b/build/params_2k.go index f4a17f724..a9277831d 100644 --- a/build/params_2k.go +++ b/build/params_2k.go @@ -13,8 +13,9 @@ const BreezeGasTampingDuration = 0 const UpgradeSmokeHeight = -1 const UpgradeIgnitionHeight = -2 -const UpgradeLiftoffHeight = -3 +const UpgradeRefuelHeight = -3 const UpgradeActorsV2Height = 10 +const UpgradeLiftoffHeight = -4 var DrandSchedule = map[abi.ChainEpoch]DrandEnum{ 0: DrandMainnet, diff --git a/build/params_testground.go b/build/params_testground.go index bd064b3d6..6109cbc04 100644 --- a/build/params_testground.go +++ b/build/params_testground.go @@ -82,8 +82,9 @@ var ( UpgradeSmokeHeight abi.ChainEpoch = -1 UpgradeIgnitionHeight abi.ChainEpoch = -2 - UpgradeLiftoffHeight abi.ChainEpoch = -3 + UpgradeRefuelHeight abi.ChainEpoch = -3 UpgradeActorsV2Height abi.ChainEpoch = 10 + UpgradeLiftoffHeight abi.ChainEpoch = -4 DrandSchedule = map[abi.ChainEpoch]DrandEnum{ 0: DrandMainnet, diff --git a/build/params_testnet.go b/build/params_testnet.go index 16de8ff75..aef3ff450 100644 --- a/build/params_testnet.go +++ b/build/params_testnet.go @@ -25,9 +25,8 @@ const BreezeGasTampingDuration = 120 const UpgradeSmokeHeight = 51000 const UpgradeIgnitionHeight = 94000 - -// TODO: Actual epoch needs to be filled in -const UpgradeActorsV2Height = 128888 +const UpgradeRefuelHeight = 130800 +const UpgradeActorsV2Height = 138720 // This signals our tentative epoch for mainnet launch. Can make it later, but not earlier. // Miners, clients, developers, custodians all need time to prepare. diff --git a/chain/actors/builtin/builtin.go b/chain/actors/builtin/builtin.go index 7def78dcf..cb24a2c33 100644 --- a/chain/actors/builtin/builtin.go +++ b/chain/actors/builtin/builtin.go @@ -22,6 +22,7 @@ import ( var SystemActorAddr = builtin0.SystemActorAddr var BurntFundsActorAddr = builtin0.BurntFundsActorAddr var ReserveAddress = makeAddress("t090") +var RootVerifierAddress = makeAddress("t080") // TODO: Why does actors have 2 different versions of this? type SectorInfo = proof0.SectorInfo diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index 1888d1ee9..989a94870 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -6,6 +6,8 @@ import ( "encoding/binary" "math" + "github.com/filecoin-project/lotus/chain/actors/builtin" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" @@ -66,6 +68,10 @@ func DefaultUpgradeSchedule() UpgradeSchedule { Height: build.UpgradeIgnitionHeight, Network: network.Version3, Migration: UpgradeIgnition, + }, { + Height: build.UpgradeRefuelHeight, + Network: network.Version3, + Migration: UpgradeRefuel, }, { Height: build.UpgradeActorsV2Height, Network: network.Version4, @@ -500,6 +506,36 @@ func UpgradeIgnition(ctx context.Context, sm *StateManager, cb ExecCallback, roo return tree.Flush(ctx) } +func UpgradeRefuel(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) { + store := sm.cs.Store(ctx) + tree, err := sm.StateTree(root) + if err != nil { + return cid.Undef, xerrors.Errorf("getting state tree: %w", err) + } + + addr, err := address.NewFromString("t0122") + if err != nil { + return cid.Undef, xerrors.Errorf("getting address: %w", err) + } + + err = resetMultisigVesting(ctx, store, tree, addr, 0, 0, big.Zero()) + if err != nil { + return cid.Undef, xerrors.Errorf("tweaking msig vesting: %w", err) + } + + err = resetMultisigVesting(ctx, store, tree, builtin.ReserveAddress, 0, 0, big.Zero()) + if err != nil { + return cid.Undef, xerrors.Errorf("tweaking msig vesting: %w", err) + } + + err = resetMultisigVesting(ctx, store, tree, builtin.RootVerifierAddress, 0, 0, big.Zero()) + if err != nil { + return cid.Undef, xerrors.Errorf("tweaking msig vesting: %w", err) + } + + return tree.Flush(ctx) +} + func UpgradeActorsV2(ctx context.Context, sm *StateManager, cb ExecCallback, root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) { store := sm.cs.Store(ctx) @@ -707,6 +743,7 @@ func makeKeyAddr(splitAddr address.Address, count uint64) (address.Address, erro return addr, nil } +// TODO: After the Liftoff epoch, refactor this to use resetMultisigVesting func resetGenesisMsigs(ctx context.Context, sm *StateManager, store adt0.Store, tree *state.StateTree, startEpoch abi.ChainEpoch) error { gb, err := sm.cs.GetGenesis() if err != nil { @@ -756,3 +793,34 @@ func resetGenesisMsigs(ctx context.Context, sm *StateManager, store adt0.Store, return nil } + +func resetMultisigVesting(ctx context.Context, store adt0.Store, tree *state.StateTree, addr address.Address, startEpoch abi.ChainEpoch, duration abi.ChainEpoch, balance abi.TokenAmount) error { + act, err := tree.GetActor(addr) + if err != nil { + return xerrors.Errorf("getting actor: %w", err) + } + + if !builtin.IsMultisigActor(act.Code) { + return xerrors.Errorf("actor wasn't msig: %w", err) + } + + var msigState multisig0.State + if err := store.Get(ctx, act.Head, &msigState); err != nil { + return xerrors.Errorf("reading multisig state: %w", err) + } + + msigState.StartEpoch = startEpoch + msigState.UnlockDuration = duration + msigState.InitialBalance = balance + + act.Head, err = store.Put(ctx, &msigState) + if err != nil { + return xerrors.Errorf("writing new multisig state: %w", err) + } + + if err := tree.SetActor(addr, act); err != nil { + return xerrors.Errorf("setting multisig actor: %w", err) + } + + return nil +} From f9f54819d41d333e021b9d2c3b4d32c4d3dbd138 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 8 Oct 2020 00:17:24 +0200 Subject: [PATCH 607/885] Env var to control v2 actor migration Env var to control v2 actor migration --- build/params_2k.go | 13 +++++++++++-- build/params_testnet.go | 8 +++++++- chain/stmgr/forks.go | 30 ++++++++++++++++++++++++++++-- 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/build/params_2k.go b/build/params_2k.go index a9277831d..c6538dc08 100644 --- a/build/params_2k.go +++ b/build/params_2k.go @@ -3,6 +3,9 @@ package build import ( + "math" + "os" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/actors/policy" @@ -14,8 +17,9 @@ const BreezeGasTampingDuration = 0 const UpgradeSmokeHeight = -1 const UpgradeIgnitionHeight = -2 const UpgradeRefuelHeight = -3 -const UpgradeActorsV2Height = 10 -const UpgradeLiftoffHeight = -4 + +var UpgradeActorsV2Height = abi.ChainEpoch(10) +var UpgradeLiftoffHeight = abi.ChainEpoch(-4) var DrandSchedule = map[abi.ChainEpoch]DrandEnum{ 0: DrandMainnet, @@ -26,6 +30,11 @@ func init() { policy.SetConsensusMinerMinPower(abi.NewStoragePower(2048)) policy.SetMinVerifiedDealSize(abi.NewStoragePower(256)) + if os.Getenv("LOTUS_DISABLE_V2_ACTOR_MIGRATION") == "1" { + UpgradeActorsV2Height = math.MaxInt64 + UpgradeLiftoffHeight = 11 + } + BuildType |= Build2k } diff --git a/build/params_testnet.go b/build/params_testnet.go index aef3ff450..fe70deaef 100644 --- a/build/params_testnet.go +++ b/build/params_testnet.go @@ -5,6 +5,7 @@ package build import ( + "math" "os" "github.com/filecoin-project/go-address" @@ -26,7 +27,8 @@ const UpgradeSmokeHeight = 51000 const UpgradeIgnitionHeight = 94000 const UpgradeRefuelHeight = 130800 -const UpgradeActorsV2Height = 138720 + +var UpgradeActorsV2Height = abi.ChainEpoch(138720) // This signals our tentative epoch for mainnet launch. Can make it later, but not earlier. // Miners, clients, developers, custodians all need time to prepare. @@ -44,6 +46,10 @@ func init() { SetAddressNetwork(address.Mainnet) } + if os.Getenv("LOTUS_DISABLE_V2_ACTOR_MIGRATION") == "1" { + UpgradeActorsV2Height = math.MaxInt64 + } + Devnet = false } diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index 989a94870..cbbc351f9 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -56,7 +56,7 @@ type UpgradeSchedule []Upgrade func DefaultUpgradeSchedule() UpgradeSchedule { var us UpgradeSchedule - for _, u := range []Upgrade{{ + updates := []Upgrade{{ Height: build.UpgradeBreezeHeight, Network: network.Version1, Migration: UpgradeFaucetBurnRecovery, @@ -81,7 +81,33 @@ func DefaultUpgradeSchedule() UpgradeSchedule { Height: build.UpgradeLiftoffHeight, Network: network.Version4, Migration: UpgradeLiftoff, - }} { + }} + + if build.UpgradeActorsV2Height == math.MaxInt64 { // disable actors upgrade + updates = []Upgrade{{ + Height: build.UpgradeBreezeHeight, + Network: network.Version1, + Migration: UpgradeFaucetBurnRecovery, + }, { + Height: build.UpgradeSmokeHeight, + Network: network.Version2, + Migration: nil, + }, { + Height: build.UpgradeIgnitionHeight, + Network: network.Version3, + Migration: UpgradeIgnition, + }, { + Height: build.UpgradeRefuelHeight, + Network: network.Version3, + Migration: UpgradeRefuel, + }, { + Height: build.UpgradeLiftoffHeight, + Network: network.Version3, + Migration: UpgradeLiftoff, + }} + } + + for _, u := range updates { if u.Height < 0 { // upgrade disabled continue From cd9af278b46b172e9f918835348f09023bb8b074 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Wed, 7 Oct 2020 17:01:01 -0700 Subject: [PATCH 608/885] more info in audit outputs --- chain/types/fil.go | 2 +- cmd/lotus-shed/balances.go | 84 ++++++++++++++++++++++++++------------ 2 files changed, 60 insertions(+), 26 deletions(-) diff --git a/chain/types/fil.go b/chain/types/fil.go index 64f6a52b3..7eac8ce93 100644 --- a/chain/types/fil.go +++ b/chain/types/fil.go @@ -18,7 +18,7 @@ func (f FIL) String() string { func (f FIL) Unitless() string { r := new(big.Rat).SetFrac(f.Int, big.NewInt(int64(build.FilecoinPrecision))) if r.Sign() == 0 { - return "0 FIL" + return "0" } return strings.TrimRight(strings.TrimRight(r.FloatString(18), "0"), ".") } diff --git a/cmd/lotus-shed/balances.go b/cmd/lotus-shed/balances.go index 3aeb0c879..8aedbedf6 100644 --- a/cmd/lotus-shed/balances.go +++ b/cmd/lotus-shed/balances.go @@ -7,6 +7,7 @@ import ( "github.com/docker/go-units" lotusbuiltin "github.com/filecoin-project/lotus/chain/actors/builtin" + "github.com/filecoin-project/lotus/chain/actors/builtin/multisig" "github.com/filecoin-project/lotus/chain/actors/builtin/power" "github.com/filecoin-project/lotus/chain/actors/builtin/reward" @@ -33,16 +34,19 @@ import ( ) type accountInfo struct { - Address address.Address - Balance types.FIL - Type string - Power abi.StoragePower - Worker address.Address - Owner address.Address - InitialPledge types.FIL - PreCommits types.FIL - LockedFunds types.FIL - Sectors uint64 + Address address.Address + Balance types.FIL + Type string + Power abi.StoragePower + Worker address.Address + Owner address.Address + InitialPledge types.FIL + PreCommits types.FIL + LockedFunds types.FIL + Sectors uint64 + VestingStart abi.ChainEpoch + VestingDuration abi.ChainEpoch + VestingAmount types.FIL } var auditsCmd = &cli.Command{ @@ -115,10 +119,8 @@ var chainBalanceCmd = &cli.Command{ infos = append(infos, ai) } - fmt.Printf("Address,Balance,Type,Power,Worker,Owner\n") - for _, acc := range infos { - fmt.Printf("%s,%s,%s,%s,%s,%s\n", acc.Address, acc.Balance.Unitless(), acc.Type, acc.Power, acc.Worker, acc.Owner) - } + printAccountInfos(infos, false) + return nil }, } @@ -196,6 +198,7 @@ var chainBalanceStateCmd = &cli.Command{ LockedFunds: types.FIL(big.NewInt(0)), InitialPledge: types.FIL(big.NewInt(0)), PreCommits: types.FIL(big.NewInt(0)), + VestingAmount: types.FIL(big.NewInt(0)), } if minerInfo && act.IsStorageMinerActor() { @@ -234,6 +237,32 @@ var chainBalanceStateCmd = &cli.Command{ ai.Worker = minfo.Worker ai.Owner = minfo.Owner } + + if act.IsMultisigActor() { + mst, err := multisig.Load(store, act) + if err != nil { + return err + } + + ai.VestingStart, err = mst.StartEpoch() + if err != nil { + return err + } + + ib, err := mst.InitialBalance() + if err != nil { + return err + } + + ai.VestingAmount = types.FIL(ib) + + ai.VestingDuration, err = mst.UnlockDuration() + if err != nil { + return err + } + + } + infos = append(infos, ai) return nil }) @@ -241,22 +270,27 @@ var chainBalanceStateCmd = &cli.Command{ return xerrors.Errorf("failed to loop over actors: %w", err) } - if minerInfo { - fmt.Printf("Address,Balance,Type,Sectors,Worker,Owner,InitialPledge,Locked,PreCommits\n") - for _, acc := range infos { - fmt.Printf("%s,%s,%s,%d,%s,%s,%s,%s,%s\n", acc.Address, acc.Balance.Unitless(), acc.Type, acc.Sectors, acc.Worker, acc.Owner, acc.InitialPledge, acc.LockedFunds, acc.PreCommits) - } - } else { - fmt.Printf("Address,Balance,Type\n") - for _, acc := range infos { - fmt.Printf("%s,%s,%s\n", acc.Address, acc.Balance.Unitless(), acc.Type) - } - } + printAccountInfos(infos, minerInfo) return nil }, } +func printAccountInfos(infos []accountInfo, minerInfo bool) { + if minerInfo { + fmt.Printf("Address,Balance,Type,Sectors,Worker,Owner,InitialPledge,Locked,PreCommits,VestingStart,VestingDuration,VestingAmount\n") + for _, acc := range infos { + fmt.Printf("%s,%s,%s,%d,%s,%s,%s,%s,%s,%d,%d,%s\n", acc.Address, acc.Balance.Unitless(), acc.Type, acc.Sectors, acc.Worker, acc.Owner, acc.InitialPledge.Unitless(), acc.LockedFunds.Unitless(), acc.PreCommits.Unitless(), acc.VestingStart, acc.VestingDuration, acc.VestingAmount.Unitless()) + } + } else { + fmt.Printf("Address,Balance,Type\n") + for _, acc := range infos { + fmt.Printf("%s,%s,%s\n", acc.Address, acc.Balance.Unitless(), acc.Type) + } + } + +} + var chainPledgeCmd = &cli.Command{ Name: "stateroot-pledge", Description: "Calculate sector pledge numbers", From da34548a05782982b7a152efc0876cd59d8bc336 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Wed, 7 Oct 2020 21:45:41 -0400 Subject: [PATCH 609/885] Set head should unmark blocks as valid --- node/impl/full/chain.go | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/node/impl/full/chain.go b/node/impl/full/chain.go index b405fb3dc..aa2ae4df1 100644 --- a/node/impl/full/chain.go +++ b/node/impl/full/chain.go @@ -254,11 +254,31 @@ func (a *ChainAPI) ChainStatObj(ctx context.Context, obj cid.Cid, base cid.Cid) } func (a *ChainAPI) ChainSetHead(ctx context.Context, tsk types.TipSetKey) error { - ts, err := a.Chain.GetTipSetFromKey(tsk) + newHeadTs, err := a.Chain.GetTipSetFromKey(tsk) if err != nil { return xerrors.Errorf("loading tipset %s: %w", tsk, err) } - return a.Chain.SetHead(ts) + + currentTs, err := a.ChainHead(ctx) + if err != nil { + return xerrors.Errorf("getting head: %w", err) + } + + for currentTs.Height() >= newHeadTs.Height() { + for _, blk := range currentTs.Key().Cids() { + err = a.Chain.UnmarkBlockAsValidated(ctx, blk) + if err != nil { + return xerrors.Errorf("unmarking block as validated %s: %w", blk, err) + } + } + + currentTs, err = a.ChainGetTipSet(ctx, currentTs.Parents()) + if err != nil { + return xerrors.Errorf("loading tipset: %w", err) + } + } + + return a.Chain.SetHead(newHeadTs) } func (a *ChainAPI) ChainGetGenesis(ctx context.Context) (*types.TipSet, error) { From 0f53eca88356acd3e087ef8ef57c41f08b64665a Mon Sep 17 00:00:00 2001 From: hunjixin <1084400399@qq.com> Date: Thu, 8 Oct 2020 10:04:43 +0800 Subject: [PATCH 610/885] add batch api for push messages --- api/api_full.go | 9 +++++++++ api/apistruct/struct.go | 16 ++++++++++++++++ node/impl/full/mpool.go | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+) diff --git a/api/api_full.go b/api/api_full.go index 88f18943c..6696a9f92 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -203,6 +203,15 @@ type FullNode interface { // based on current chain conditions MpoolPushMessage(ctx context.Context, msg *types.Message, spec *MessageSendSpec) (*types.SignedMessage, error) + // MpoolBatchPush batch pushes a signed message to mempool. + MpoolBatchPush(context.Context, []*types.SignedMessage) ([]cid.Cid, error) + + // MpoolBatchPushUntrusted batch pushes a signed message to mempool from untrusted sources. + MpoolBatchPushUntrusted(context.Context, []*types.SignedMessage) ([]cid.Cid, error) + + // MpoolBatchPushMessage batch pushes a unsigned message to mempool. + MpoolBatchPushMessage(context.Context, []*types.Message, *MessageSendSpec) ([]*types.SignedMessage, error) + // MpoolGetNonce gets next nonce for the specified sender. // Note that this method may not be atomic. Use MpoolPushMessage instead. MpoolGetNonce(context.Context, address.Address) (uint64, error) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index a09700eb9..e9e09606e 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -129,6 +129,10 @@ type FullNodeStruct struct { MpoolGetNonce func(context.Context, address.Address) (uint64, error) `perm:"read"` MpoolSub func(context.Context) (<-chan api.MpoolUpdate, error) `perm:"read"` + MpoolBatchPush func(ctx context.Context, smsgs []*types.SignedMessage) ([]cid.Cid, error) `perm:"write"` + MpoolBatchPushUntrusted func(ctx context.Context, smsgs []*types.SignedMessage) ([]cid.Cid, error) `perm:"write"` + MpoolBatchPushMessage func(ctx context.Context, msgs []*types.Message, spec *api.MessageSendSpec) ([]*types.SignedMessage, error) `perm:"sign"` + MinerGetBaseInfo func(context.Context, address.Address, abi.ChainEpoch, types.TipSetKey) (*api.MiningBaseInfo, error) `perm:"read"` MinerCreateBlock func(context.Context, *api.BlockTemplate) (*types.BlockMsg, error) `perm:"write"` @@ -569,6 +573,18 @@ func (c *FullNodeStruct) MpoolPushMessage(ctx context.Context, msg *types.Messag return c.Internal.MpoolPushMessage(ctx, msg, spec) } +func (c *FullNodeStruct) MpoolBatchPush(ctx context.Context, smsgs []*types.SignedMessage) ([]cid.Cid, error) { + return c.Internal.MpoolBatchPush(ctx, smsgs) +} + +func (c *FullNodeStruct) MpoolBatchPushUntrusted(ctx context.Context, smsgs []*types.SignedMessage) ([]cid.Cid, error) { + return c.Internal.MpoolBatchPushUntrusted(ctx, smsgs) +} + +func (c *FullNodeStruct) MpoolBatchPushMessage(ctx context.Context, msgs []*types.Message, spec *api.MessageSendSpec) ([]*types.SignedMessage, error) { + return c.Internal.MpoolBatchPushMessage(ctx, msgs, spec) +} + func (c *FullNodeStruct) MpoolSub(ctx context.Context) (<-chan api.MpoolUpdate, error) { return c.Internal.MpoolSub(ctx) } diff --git a/node/impl/full/mpool.go b/node/impl/full/mpool.go index 1f093606c..11c3fb045 100644 --- a/node/impl/full/mpool.go +++ b/node/impl/full/mpool.go @@ -169,6 +169,42 @@ func (a *MpoolAPI) MpoolPushMessage(ctx context.Context, msg *types.Message, spe }) } +func (a *MpoolAPI) MpoolBatchPush(ctx context.Context, smsgs []*types.SignedMessage) ([]cid.Cid, error) { + var messageCids []cid.Cid + for _, smsg := range smsgs { + smsgCid, err := a.Mpool.Push(smsg) + if err != nil { + return messageCids, err + } + messageCids = append(messageCids, smsgCid) + } + return messageCids, nil +} + +func (a *MpoolAPI) MpoolBatchPushUntrusted(ctx context.Context, smsgs []*types.SignedMessage) ([]cid.Cid, error) { + var messageCids []cid.Cid + for _, smsg := range smsgs { + smsgCid, err := a.Mpool.PushUntrusted(smsg) + if err != nil { + return messageCids, err + } + messageCids = append(messageCids, smsgCid) + } + return messageCids, nil +} + +func (a *MpoolAPI) MpoolBatchPushMessage(ctx context.Context, msgs []*types.Message, spec *api.MessageSendSpec) ([]*types.SignedMessage, error) { + var smsgs []*types.SignedMessage + for _, msg := range msgs { + smsg, err := a.MpoolPushMessage(ctx, msg, spec) + if err != nil { + return smsgs, err + } + smsgs = append(smsgs, smsg) + } + return smsgs, nil +} + func (a *MpoolAPI) MpoolGetNonce(ctx context.Context, addr address.Address) (uint64, error) { return a.Mpool.GetNonce(addr) } From dea70426689002cd4fbba6004012e1567328ea8d Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Wed, 7 Oct 2020 19:00:56 -0400 Subject: [PATCH 611/885] Lotus version 0.9.0 --- CHANGELOG.md | 95 +++++++++++++++++++++++++++++++++++++++++++++++- build/version.go | 2 +- 2 files changed, 95 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6de6ddc2c..4f6f0b3a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,98 @@ # Lotus changelog +# 0.9.0 / 2020-10-07 + +This consensus-breaking release of Lotus upgrades the actors version to v2.0.0. This requires migrating actor state from v0 to v2. The changes that break consensus are: + +- Introducing v2 actors and its migration (https://github.com/filecoin-project/lotus/pull/3936) +- Runtime's Receiver() should only return ID addresses (https://github.com/filecoin-project/lotus/pull/3589) +- Update miner eligibility checks for v2 actors (https://github.com/filecoin-project/lotus/pull/4188) +- Add funds that have left FilReserve to circ supply (https://github.com/filecoin-project/lotus/pull/4160) +- Set WinningPoStSectorSetLookback to finality post-v2 actors (https://github.com/filecoin-project/lotus/pull/4190) +- fix: error when actor panics directly (https://github.com/filecoin-project/lotus/pull/3697) + +## Changes + +#### Dependencies + +- Update go-bitfield (https://github.com/filecoin-project/lotus/pull/4171) +- update the AMT implementation (https://github.com/filecoin-project/lotus/pull/4194) +- Update to actors v0.2.1 (https://github.com/filecoin-project/lotus/pull/4199) + +#### Core Lotus + +- Paych: fix voucher amount verification (https://github.com/filecoin-project/lotus/pull/3821) +- Cap market provider messages (https://github.com/filecoin-project/lotus/pull/4141) +- Run fork function after cron for null block safety (https://github.com/filecoin-project/lotus/pull/4114) +- use bitswap sessions when fetching messages, and cancel them (https://github.com/filecoin-project/lotus/pull/4142) +- relax pubsub IPColocationFactorThreshold to 5 (https://github.com/filecoin-project/lotus/pull/4183) +- Support addresses with mainnet prefixes (https://github.com/filecoin-project/lotus/pull/4186) +- fix: make message signer nonce generation transactional (https://github.com/filecoin-project/lotus/pull/4165) +- build: Env var to keep test address output (https://github.com/filecoin-project/lotus/pull/4213) +- make vm.EnableGasTracing public (https://github.com/filecoin-project/lotus/pull/4214) +- introduce separate state-tree versions (https://github.com/filecoin-project/lotus/pull/4197) +- reject explicit "calls" at the upgrade height (https://github.com/filecoin-project/lotus/pull/4231) +- return an illegal actor error when we see an unsupported actor version (https://github.com/filecoin-project/lotus/pull/4232) +- Set head should unmark blocks as valid (https://gist.github.com/travisperson/3c7cddd77a33979a519ccef4e6515f20) + +#### Mining + +- Increased ExpectedSealDuration and and WaitDealsDelay (https://github.com/filecoin-project/lotus/pull/3743) +- Miner backup/restore commands (https://github.com/filecoin-project/lotus/pull/4133) +- lotus-miner: add more help text to storage / attach (https://github.com/filecoin-project/lotus/pull/3961) +- Reject deals that are > 7 days in the future in the BasicDealFilter (https://github.com/filecoin-project/lotus/pull/4173) +- feat(miner): add miner deadline diffing logic (https://github.com/filecoin-project/lotus/pull/4178) + +#### UX + +- Improve the UX for replacing messages (https://github.com/filecoin-project/lotus/pull/4134) +- Add verified flag to interactive deal creation (https://github.com/filecoin-project/lotus/pull/4145) +- Add command to (slowly) prune lotus chain datastore (https://github.com/filecoin-project/lotus/pull/3876) +- Some helpers for verifreg work (https://github.com/filecoin-project/lotus/pull/4124) +- Always use default 720h for setask duration and hide the duration param option (https://github.com/filecoin-project/lotus/pull/4077) +- Convert ID addresses to key addresses before checking wallet (https://github.com/filecoin-project/lotus/pull/4122) +- add a command to view block space utilization (https://github.com/filecoin-project/lotus/pull/4176) +- allow usage inspection on a chain segment (https://github.com/filecoin-project/lotus/pull/4177) +- Add mpool stats for base fee (https://github.com/filecoin-project/lotus/pull/4170) +- Add verified status to api.DealInfo (https://github.com/filecoin-project/lotus/pull/4153) +- Add a CLI command to set a miner's owner address (https://github.com/filecoin-project/lotus/pull/4189) + +#### Tooling and validation + +- Lotus-pcr: add recover-miners command (https://github.com/filecoin-project/lotus/pull/3714) +- MpoolPushUntrusted API for gateway (https://github.com/filecoin-project/lotus/pull/3915) +- Test lotus-miner info all (https://github.com/filecoin-project/lotus/pull/4166) +- chain export: Error with unfinished exports (https://github.com/filecoin-project/lotus/pull/4179) +- add printf in TestWindowPost (https://github.com/filecoin-project/lotus/pull/4043) +- add trace wdpost (https://github.com/filecoin-project/lotus/pull/4020) +- Fix noncefix (https://github.com/filecoin-project/lotus/pull/4202) +- Lotus-pcr: Limit the fee cap of messages we will process, refund gas fees for windowed post and storage deals (https://github.com/filecoin-project/lotus/pull/4198) +- Fix pond (https://github.com/filecoin-project/lotus/pull/4203) +- allow manual setting of noncefix fee cap (https://github.com/filecoin-project/lotus/pull/4205) +- implement command to get execution traces of any message (https://github.com/filecoin-project/lotus/pull/4200) +- conformance: minor driver refactors (https://github.com/filecoin-project/lotus/pull/4211) +- lotus-pcr: ignore all other messages (https://github.com/filecoin-project/lotus/pull/4218) +- lotus-pcr: zero refund (https://github.com/filecoin-project/lotus/pull/4229) + +## Contributors + +The following contributors had 5 or more commits go into this release. +We are grateful for every contribution! + +| Contributor | Commits | Lines ± | +|--------------------|---------|---------------| +| Stebalien | 84 | +3425/-2287 | +| magik6k | 41 | +2121/-506 | +| arajasek | 39 | +2467/-424 | +| Kubuxu | 25 | +2344/-775 | +| raulk | 21 | +287/-196 | +| whyrusleeping | 13 | +727/-71 | +| hsanjuan | 13 | +5886/-7956 | +| dirkmc | 11 | +2634/-576 | +| travisperson | 8 | +923/-202 | +| ribasushi | 6 | +188/-128 | +| zgfzgf | 5 | +21/-17 | + # 0.8.1 / 2020-09-30 This optional release of Lotus introduces a new version of markets which switches to CBOR-map encodings, and allows datastore migrations. The release also introduces several improvements to the mining process, a few performance optimizations, and a battery of UX additions and enhancements. @@ -117,7 +210,7 @@ We are grateful for every contribution! | vyzo | 22 | +287/-196 | | alanshaw | 15 | +761/-146 | | whyrusleeping | 15 | +736/-52 | -| hannahhoward | 14 | +1237/837- | +| hannahhoward | 14 | +1237/-837 | | anton | 6 | +32/-8 | | travisperson | 5 | +502/-6 | | Frank | 5 | +78/-39 | diff --git a/build/version.go b/build/version.go index 0b317aa17..1602551e4 100644 --- a/build/version.go +++ b/build/version.go @@ -29,7 +29,7 @@ func buildType() string { } // BuildVersion is the local build version, set by build system -const BuildVersion = "0.8.1" +const BuildVersion = "0.9.0" func UserVersion() string { return BuildVersion + buildType() + CurrentCommit From cb7d99fcc7075fb565abd2d41278d2b9cae08bd6 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Thu, 8 Oct 2020 14:15:09 +0200 Subject: [PATCH 612/885] fix: clash between daemon --api flag and cli tests --- cli/cmd.go | 57 ++++++++++++++++++++++++++----------- cli/paych_test.go | 10 +++---- cli/pprof.go | 3 +- cmd/lotus-shed/consensus.go | 2 +- 4 files changed, 47 insertions(+), 25 deletions(-) diff --git a/cli/cmd.go b/cli/cmd.go index 92a366eb5..edcb69adc 100644 --- a/cli/cmd.go +++ b/cli/cmd.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/http" + "net/url" "os" "os/signal" "strings" @@ -47,14 +48,44 @@ func NewCliError(s string) error { type ApiConnector func() api.FullNode type APIInfo struct { - Addr multiaddr.Multiaddr + Addr string Token []byte } func (a APIInfo) DialArgs() (string, error) { - _, addr, err := manet.DialArgs(a.Addr) + ma, err := multiaddr.NewMultiaddr(a.Addr) + if err == nil { + _, addr, err := manet.DialArgs(ma) + if err != nil { + return "", err + } - return "ws://" + addr + "/rpc/v0", err + return "ws://" + addr + "/rpc/v0", nil + } + + _, err = url.Parse(a.Addr) + if err != nil { + return "", err + } + return a.Addr + "/rpc/v0", nil +} + +func (a APIInfo) Host() (string, error) { + ma, err := multiaddr.NewMultiaddr(a.Addr) + if err == nil { + _, addr, err := manet.DialArgs(ma) + if err != nil { + return "", err + } + + return addr, nil + } + + spec, err := url.Parse(a.Addr) + if err != nil { + return "", err + } + return spec.Host, nil } func (a APIInfo) AuthHeader() http.Header { @@ -72,11 +103,11 @@ func (a APIInfo) AuthHeader() http.Header { func flagForAPI(t repo.RepoType) string { switch t { case repo.FullNode: - return "api" + return "api-url" case repo.StorageMiner: - return "miner-api" + return "miner-api-url" case repo.Worker: - return "worker-api" + return "worker-api-url" default: panic(fmt.Sprintf("Unknown repo type: %v", t)) } @@ -130,11 +161,7 @@ func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (APIInfo, error) { strma := ctx.String(apiFlag) strma = strings.TrimSpace(strma) - apima, err := multiaddr.NewMultiaddr(strma) - if err != nil { - return APIInfo{}, err - } - return APIInfo{Addr: apima}, nil + return APIInfo{Addr: strma}, nil } envKey := envForRepo(t) @@ -152,12 +179,8 @@ func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (APIInfo, error) { if len(sp) != 2 { log.Warnf("invalid env(%s) value, missing token or address", envKey) } else { - ma, err := multiaddr.NewMultiaddr(sp[1]) - if err != nil { - return APIInfo{}, xerrors.Errorf("could not parse multiaddr from env(%s): %w", envKey, err) - } return APIInfo{ - Addr: ma, + Addr: sp[1], Token: []byte(sp[0]), }, nil } @@ -186,7 +209,7 @@ func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (APIInfo, error) { } return APIInfo{ - Addr: ma, + Addr: ma.String(), Token: token, }, nil } diff --git a/cli/paych_test.go b/cli/paych_test.go index 35f56d90e..18782b4e8 100644 --- a/cli/paych_test.go +++ b/cli/paych_test.go @@ -439,12 +439,12 @@ type mockCLI struct { } func newMockCLI(t *testing.T) *mockCLI { - // Create a CLI App with an --api flag so that we can specify which node + // Create a CLI App with an --api-url flag so that we can specify which node // the command should be executed against app := cli.NewApp() app.Flags = []cli.Flag{ &cli.StringFlag{ - Name: "api", + Name: "api-url", Hidden: true, }, } @@ -476,8 +476,8 @@ func (c *mockCLIClient) runCmd(cmd *cli.Command, input []string) string { } func (c *mockCLIClient) runCmdRaw(cmd *cli.Command, input []string) (string, error) { - // prepend --api= - apiFlag := "--api=" + c.addr.String() + // prepend --api-url= + apiFlag := "--api-url=" + c.addr.String() input = append([]string{apiFlag}, input...) fs := c.flagSet(cmd) @@ -493,7 +493,7 @@ func (c *mockCLIClient) runCmdRaw(cmd *cli.Command, input []string) (string, err } func (c *mockCLIClient) flagSet(cmd *cli.Command) *flag.FlagSet { - // Apply app level flags (so we can process --api flag) + // Apply app level flags (so we can process --api-url flag) fs := &flag.FlagSet{} for _, f := range c.cctx.App.Flags { err := f.Apply(fs) diff --git a/cli/pprof.go b/cli/pprof.go index 6819b362a..dccb97f9a 100644 --- a/cli/pprof.go +++ b/cli/pprof.go @@ -9,7 +9,6 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/lotus/node/repo" - manet "github.com/multiformats/go-multiaddr/net" ) var pprofCmd = &cli.Command{ @@ -37,7 +36,7 @@ var PprofGoroutines = &cli.Command{ if err != nil { return xerrors.Errorf("could not get API info: %w", err) } - _, addr, err := manet.DialArgs(ainfo.Addr) + addr, err := ainfo.Host() if err != nil { return err } diff --git a/cmd/lotus-shed/consensus.go b/cmd/lotus-shed/consensus.go index 59d9555df..38a8cd8ef 100644 --- a/cmd/lotus-shed/consensus.go +++ b/cmd/lotus-shed/consensus.go @@ -111,7 +111,7 @@ var consensusCheckCmd = &cli.Command{ if err != nil { return err } - ainfo := lcli.APIInfo{Addr: apima} + ainfo := lcli.APIInfo{Addr: apima.String()} addr, err := ainfo.DialArgs() if err != nil { return err From 4937ca1a96cd2cb70344a54130b7f35d43c511e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Thu, 8 Oct 2020 14:55:03 +0100 Subject: [PATCH 613/885] conformance: remove usage of fake syscalls. --- conformance/driver.go | 4 ++-- conformance/stubs.go | 44 ------------------------------------------- 2 files changed, 2 insertions(+), 46 deletions(-) delete mode 100644 conformance/stubs.go diff --git a/conformance/driver.go b/conformance/driver.go index d51798a52..137653141 100644 --- a/conformance/driver.go +++ b/conformance/driver.go @@ -81,7 +81,7 @@ type ExecuteTipsetResult struct { // and reward withdrawal per miner. func (d *Driver) ExecuteTipset(bs blockstore.Blockstore, ds ds.Batching, preroot cid.Cid, parentEpoch abi.ChainEpoch, tipset *schema.Tipset) (*ExecuteTipsetResult, error) { var ( - syscalls = mkFakedSigSyscalls(vm.Syscalls(ffiwrapper.ProofVerifier)) + syscalls = vm.Syscalls(ffiwrapper.ProofVerifier) vmRand = NewFixedRand() cs = store.NewChainStore(bs, ds, syscalls) @@ -173,7 +173,7 @@ func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, params ExecuteMessageP StateBase: params.Preroot, Epoch: params.Epoch, Bstore: bs, - Syscalls: mkFakedSigSyscalls(vm.Syscalls(ffiwrapper.ProofVerifier)), // TODO always succeeds; need more flexibility. + Syscalls: vm.Syscalls(ffiwrapper.ProofVerifier), CircSupplyCalc: func(_ context.Context, _ abi.ChainEpoch, _ *state.StateTree) (abi.TokenAmount, error) { return params.CircSupply, nil }, diff --git a/conformance/stubs.go b/conformance/stubs.go deleted file mode 100644 index 9307fdd65..000000000 --- a/conformance/stubs.go +++ /dev/null @@ -1,44 +0,0 @@ -package conformance - -import ( - "context" - - "github.com/filecoin-project/specs-actors/actors/runtime/proof" - - "github.com/filecoin-project/go-address" - - "github.com/filecoin-project/lotus/chain/state" - "github.com/filecoin-project/lotus/chain/vm" - - "github.com/filecoin-project/go-state-types/crypto" - "github.com/filecoin-project/specs-actors/actors/runtime" - - cbor "github.com/ipfs/go-ipld-cbor" -) - -type testSyscalls struct { - runtime.Syscalls -} - -// TODO VerifySignature this will always succeed; but we want to be able to test failures too. -func (fss *testSyscalls) VerifySignature(_ crypto.Signature, _ address.Address, _ []byte) error { - return nil -} - -// TODO VerifySeal this will always succeed; but we want to be able to test failures too. -func (fss *testSyscalls) VerifySeal(_ proof.SealVerifyInfo) error { - return nil -} - -// TODO VerifyPoSt this will always succeed; but we want to be able to test failures too. -func (fss *testSyscalls) VerifyPoSt(_ proof.WindowPoStVerifyInfo) error { - return nil -} - -func mkFakedSigSyscalls(base vm.SyscallBuilder) vm.SyscallBuilder { - return func(ctx context.Context, cstate *state.StateTree, cst cbor.IpldStore) runtime.Syscalls { - return &testSyscalls{ - base(ctx, cstate, cst), - } - } -} From 1f9446f91fc19473f51408680585035b4dc468ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Thu, 8 Oct 2020 15:00:35 +0100 Subject: [PATCH 614/885] upgrade to test vectors schema v0.0.4. --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e401af36d..5473dbb59 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,7 @@ require ( github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b github.com/filecoin-project/specs-actors v0.9.11 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 - github.com/filecoin-project/test-vectors/schema v0.0.4-0.20201007174510-548c9399aa6a + github.com/filecoin-project/test-vectors/schema v0.0.4 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 github.com/go-kit/kit v0.10.0 github.com/go-ole/go-ole v1.2.4 // indirect diff --git a/go.sum b/go.sum index 7d6c743ae..45d20d118 100644 --- a/go.sum +++ b/go.sum @@ -274,8 +274,8 @@ github.com/filecoin-project/specs-actors v0.9.11 h1:TnpG7HAeiUrfj0mJM7UaPW0P2137 github.com/filecoin-project/specs-actors v0.9.11/go.mod h1:czlvLQGEX0fjLLfdNHD7xLymy6L3n7aQzRWzsYGf+ys= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= -github.com/filecoin-project/test-vectors/schema v0.0.4-0.20201007174510-548c9399aa6a h1:2jRLaT3/sHyAinWR2asCAkvzcnOAIi13eTlWf3YEVvY= -github.com/filecoin-project/test-vectors/schema v0.0.4-0.20201007174510-548c9399aa6a/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E= +github.com/filecoin-project/test-vectors/schema v0.0.4 h1:QTRd0gb/NP4ZOTM7Dib5U3xE1/ToGDKnYLfxkC3t/m8= +github.com/filecoin-project/test-vectors/schema v0.0.4/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6 h1:u/UEqS66A5ckRmS4yNpjmVH56sVtS/RfclBAYocb4as= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ= From 67b3a55e30ffb75efc8191d75709fbdfac06f268 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Thu, 8 Oct 2020 15:44:11 +0100 Subject: [PATCH 615/885] update test-vectors submodule. --- extern/test-vectors | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/test-vectors b/extern/test-vectors index 7471e2805..a8f968ade 160000 --- a/extern/test-vectors +++ b/extern/test-vectors @@ -1 +1 @@ -Subproject commit 7471e2805fc3e459e4ee325775633e8ec76cb7c6 +Subproject commit a8f968adeba1995f161f7be0048188affc425079 From cdc2f86f5a33d3e3c0f930287b91c4f36312e6a0 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 8 Oct 2020 09:45:07 -0500 Subject: [PATCH 616/885] add more info to chain sync lookback failure --- chain/sync.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chain/sync.go b/chain/sync.go index 655bb4a1d..240d1edef 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -731,7 +731,7 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock, use lbst, _, err := syncer.sm.TipSetState(ctx, lbts) if err != nil { - return xerrors.Errorf("failed to compute lookback tipset state: %w", err) + return xerrors.Errorf("failed to compute lookback tipset state (epoch %d): %w", lbts.Height(), err) } prevBeacon, err := syncer.store.GetLatestBeaconEntry(baseTs) From 98d702fb5a6bff295a5b216b46b9ddd8effd10c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Thu, 8 Oct 2020 16:44:20 +0100 Subject: [PATCH 617/885] add signature imports. --- conformance/driver.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/conformance/driver.go b/conformance/driver.go index 0451868e4..940c137cf 100644 --- a/conformance/driver.go +++ b/conformance/driver.go @@ -14,6 +14,9 @@ import ( "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/lib/blockstore" + _ "github.com/filecoin-project/lotus/lib/sigs/bls" // enable bls signatures + _ "github.com/filecoin-project/lotus/lib/sigs/secp" // enable secp signatures + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" From 2ad671649c150995d98525999bb3b44547c6454e Mon Sep 17 00:00:00 2001 From: Ingar Shu Date: Wed, 7 Oct 2020 10:35:57 -0700 Subject: [PATCH 618/885] Use FIL units for `miner set-ask` --- cmd/lotus-storage-miner/market.go | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/cmd/lotus-storage-miner/market.go b/cmd/lotus-storage-miner/market.go index 6e3743143..67e694c78 100644 --- a/cmd/lotus-storage-miner/market.go +++ b/cmd/lotus-storage-miner/market.go @@ -154,14 +154,14 @@ var setAskCmd = &cli.Command{ Name: "set-ask", Usage: "Configure the miner's ask", Flags: []cli.Flag{ - &cli.Uint64Flag{ + &cli.StringFlag{ Name: "price", - Usage: "Set the price of the ask for unverified deals (specified as attoFIL / GiB / Epoch) to `PRICE`", + Usage: "Set the price of the ask for unverified deals (specified as FIL / GiB / Epoch) to `PRICE`.", Required: true, }, - &cli.Uint64Flag{ + &cli.StringFlag{ Name: "verified-price", - Usage: "Set the price of the ask for verified deals (specified as attoFIL / GiB / Epoch) to `PRICE`", + Usage: "Set the price of the ask for verified deals (specified as FIL / GiB / Epoch) to `PRICE`", Required: true, }, &cli.StringFlag{ @@ -185,8 +185,15 @@ var setAskCmd = &cli.Command{ } defer closer() - pri := types.NewInt(cctx.Uint64("price")) - vpri := types.NewInt(cctx.Uint64("verified-price")) + pri, err := types.ParseFIL(cctx.String("price")) + if err != nil { + return err + } + + vpri, err := types.ParseFIL(cctx.String("verified-price")) + if err != nil { + return err + } dur, err := time.ParseDuration("720h0m0s") if err != nil { @@ -229,7 +236,7 @@ var setAskCmd = &cli.Command{ return xerrors.Errorf("max piece size (w/bit-padding) %s cannot exceed miner sector size %s", types.SizeStr(types.NewInt(uint64(max))), types.SizeStr(types.NewInt(uint64(smax)))) } - return api.MarketSetAsk(ctx, pri, vpri, abi.ChainEpoch(qty), abi.PaddedPieceSize(min), abi.PaddedPieceSize(max)) + return api.MarketSetAsk(ctx, types.BigInt(pri), types.BigInt(vpri), abi.ChainEpoch(qty), abi.PaddedPieceSize(min), abi.PaddedPieceSize(max)) }, } From 1323e07a2c4913194e45eeb9088ad4eb11840b7e Mon Sep 17 00:00:00 2001 From: Ingar Shu Date: Wed, 7 Oct 2020 10:45:59 -0700 Subject: [PATCH 619/885] Don't show failed deals by default in `client list-deals`, add option to to show failed deals. --- cli/client.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/cli/client.go b/cli/client.go index 7494815bf..caf3ed758 100644 --- a/cli/client.go +++ b/cli/client.go @@ -1039,6 +1039,10 @@ var clientListDeals = &cli.Command{ Usage: "use color in display output", Value: true, }, + &cli.BoolFlag{ + Name: "show-failed", + Usage: "show failed/failing deals", + }, &cli.BoolFlag{ Name: "watch", Usage: "watch deal updates in real-time, rather than a one time list", @@ -1055,6 +1059,7 @@ var clientListDeals = &cli.Command{ verbose := cctx.Bool("verbose") color := cctx.Bool("color") watch := cctx.Bool("watch") + showFailed := cctx.Bool("show-failed") localDeals, err := api.ClientListDeals(ctx) if err != nil { @@ -1071,7 +1076,7 @@ var clientListDeals = &cli.Command{ tm.Clear() tm.MoveCursor(1, 1) - err = outputStorageDeals(ctx, tm.Screen, api, localDeals, verbose, color) + err = outputStorageDeals(ctx, tm.Screen, api, localDeals, verbose, color, showFailed) if err != nil { return err } @@ -1097,7 +1102,7 @@ var clientListDeals = &cli.Command{ } } - return outputStorageDeals(ctx, os.Stdout, api, localDeals, cctx.Bool("verbose"), cctx.Bool("color")) + return outputStorageDeals(ctx, os.Stdout, api, localDeals, cctx.Bool("verbose"), cctx.Bool("color"), showFailed) }, } @@ -1120,7 +1125,7 @@ func dealFromDealInfo(ctx context.Context, full api.FullNode, head *types.TipSet } } -func outputStorageDeals(ctx context.Context, out io.Writer, full api.FullNode, localDeals []api.DealInfo, verbose bool, color bool) error { +func outputStorageDeals(ctx context.Context, out io.Writer, full lapi.FullNode, localDeals []lapi.DealInfo, verbose bool, color bool, showFailed bool) error { sort.Slice(localDeals, func(i, j int) bool { return localDeals[i].CreationTime.Before(localDeals[j].CreationTime) }) @@ -1132,7 +1137,9 @@ func outputStorageDeals(ctx context.Context, out io.Writer, full api.FullNode, l var deals []deal for _, localDeal := range localDeals { - deals = append(deals, dealFromDealInfo(ctx, full, head, localDeal)) + if showFailed || localDeal.State != storagemarket.StorageDealError { + deals = append(deals, dealFromDealInfo(ctx, full, head, localDeal)) + } } if verbose { From f7f229ae54e7782aa12a5ed311626c50857af6f7 Mon Sep 17 00:00:00 2001 From: Ingar Shu Date: Wed, 7 Oct 2020 10:55:07 -0700 Subject: [PATCH 620/885] Show verified status in `client list-deals` --- cli/client.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cli/client.go b/cli/client.go index caf3ed758..5411965a7 100644 --- a/cli/client.go +++ b/cli/client.go @@ -1144,7 +1144,7 @@ func outputStorageDeals(ctx context.Context, out io.Writer, full lapi.FullNode, if verbose { w := tabwriter.NewWriter(out, 2, 4, 2, ' ', 0) - fmt.Fprintf(w, "Created\tDealCid\tDealId\tProvider\tState\tOn Chain?\tSlashed?\tPieceCID\tSize\tPrice\tDuration\tMessage\n") + fmt.Fprintf(w, "Created\tDealCid\tDealId\tProvider\tState\tOn Chain?\tSlashed?\tPieceCID\tSize\tPrice\tDuration\tVerified\tMessage\n") for _, d := range deals { onChain := "N" if d.OnChainDealState.SectorStartEpoch != -1 { @@ -1157,7 +1157,7 @@ func outputStorageDeals(ctx context.Context, out io.Writer, full lapi.FullNode, } price := types.FIL(types.BigMul(d.LocalDeal.PricePerEpoch, types.NewInt(d.LocalDeal.Duration))) - fmt.Fprintf(w, "%s\t%s\t%d\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%d\t%s\n", d.LocalDeal.CreationTime.Format(time.Stamp), d.LocalDeal.ProposalCid, d.LocalDeal.DealID, d.LocalDeal.Provider, dealStateString(color, d.LocalDeal.State), onChain, slashed, d.LocalDeal.PieceCID, types.SizeStr(types.NewInt(d.LocalDeal.Size)), price, d.LocalDeal.Duration, d.LocalDeal.Message) + fmt.Fprintf(w, "%s\t%s\t%d\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%d\t%v\t%s\n", d.LocalDeal.CreationTime.Format(time.Stamp), d.LocalDeal.ProposalCid, d.LocalDeal.DealID, d.LocalDeal.Provider, dealStateString(color, d.LocalDeal.State), onChain, slashed, d.LocalDeal.PieceCID, types.SizeStr(types.NewInt(d.LocalDeal.Size)), price, d.LocalDeal.Duration, d.LocalDeal.Verified, d.LocalDeal.Message) } return w.Flush() } @@ -1172,6 +1172,7 @@ func outputStorageDeals(ctx context.Context, out io.Writer, full lapi.FullNode, tablewriter.Col("Size"), tablewriter.Col("Price"), tablewriter.Col("Duration"), + tablewriter.Col("Verified"), tablewriter.NewLineCol("Message")) for _, d := range deals { @@ -1201,6 +1202,7 @@ func outputStorageDeals(ctx context.Context, out io.Writer, full lapi.FullNode, "PieceCID": piece, "Size": types.SizeStr(types.NewInt(d.LocalDeal.Size)), "Price": price, + "Verified": d.LocalDeal.Verified, "Duration": d.LocalDeal.Duration, "Message": d.LocalDeal.Message, }) From 8a175a7465aa4115703849339b0259e0fc728e75 Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 8 Oct 2020 21:32:10 +0300 Subject: [PATCH 621/885] update go-libp2p-pubsub and go-libp2p-quic-transport --- go.mod | 4 ++-- go.sum | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index cec47e76b..f2d95d22a 100644 --- a/go.mod +++ b/go.mod @@ -96,8 +96,8 @@ require ( github.com/libp2p/go-libp2p-mplex v0.2.4 github.com/libp2p/go-libp2p-noise v0.1.1 github.com/libp2p/go-libp2p-peerstore v0.2.6 - github.com/libp2p/go-libp2p-pubsub v0.3.6-0.20200910093904-f7f33e10cc18 - github.com/libp2p/go-libp2p-quic-transport v0.8.0 + github.com/libp2p/go-libp2p-pubsub v0.3.6 + github.com/libp2p/go-libp2p-quic-transport v0.8.2 github.com/libp2p/go-libp2p-record v0.1.3 github.com/libp2p/go-libp2p-routing-helpers v0.2.3 github.com/libp2p/go-libp2p-swarm v0.2.8 diff --git a/go.sum b/go.sum index 55810050f..efb1d2dfb 100644 --- a/go.sum +++ b/go.sum @@ -884,12 +884,12 @@ github.com/libp2p/go-libp2p-protocol v0.0.1/go.mod h1:Af9n4PiruirSDjHycM1QuiMi/1 github.com/libp2p/go-libp2p-protocol v0.1.0/go.mod h1:KQPHpAabB57XQxGrXCNvbL6UEXfQqUgC/1adR2Xtflk= github.com/libp2p/go-libp2p-pubsub v0.1.1/go.mod h1:ZwlKzRSe1eGvSIdU5bD7+8RZN/Uzw0t1Bp9R1znpR/Q= github.com/libp2p/go-libp2p-pubsub v0.3.2-0.20200527132641-c0712c6e92cf/go.mod h1:TxPOBuo1FPdsTjFnv+FGZbNbWYsp74Culx+4ViQpato= -github.com/libp2p/go-libp2p-pubsub v0.3.6-0.20200910093904-f7f33e10cc18 h1:+ae7vHSv/PJ4xGXwLV6LKGj32zjyB8ttJHtyV4TXal0= -github.com/libp2p/go-libp2p-pubsub v0.3.6-0.20200910093904-f7f33e10cc18/go.mod h1:DTMSVmZZfXodB/pvdTGrY2eHPZ9W2ev7hzTH83OKHrI= +github.com/libp2p/go-libp2p-pubsub v0.3.6 h1:9oO8W7qIWCYQYyz5z8nUsPcb3rrFehBlkbqvbSVjBxY= +github.com/libp2p/go-libp2p-pubsub v0.3.6/go.mod h1:DTMSVmZZfXodB/pvdTGrY2eHPZ9W2ev7hzTH83OKHrI= github.com/libp2p/go-libp2p-quic-transport v0.1.1/go.mod h1:wqG/jzhF3Pu2NrhJEvE+IE0NTHNXslOPn9JQzyCAxzU= github.com/libp2p/go-libp2p-quic-transport v0.5.0/go.mod h1:IEcuC5MLxvZ5KuHKjRu+dr3LjCT1Be3rcD/4d8JrX8M= -github.com/libp2p/go-libp2p-quic-transport v0.8.0 h1:mHA94K2+TD0e9XtjWx/P5jGGZn0GdQ4OFYwNllagv4E= -github.com/libp2p/go-libp2p-quic-transport v0.8.0/go.mod h1:F2FG/6Bzz0U6essUVxDzE0s9CrY4XGLbl7QEmDNvU7A= +github.com/libp2p/go-libp2p-quic-transport v0.8.2 h1:FDaXBCBJ1e5hY6gnWEJ4NbYyLk8eezr4J6AY3q3KqwM= +github.com/libp2p/go-libp2p-quic-transport v0.8.2/go.mod h1:L+e0q15ZNaYm3seHgbsXjWP8kXLEqz+elLWKk9l8DhM= github.com/libp2p/go-libp2p-record v0.0.1/go.mod h1:grzqg263Rug/sRex85QrDOLntdFAymLDLm7lxMgU79Q= github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7xI4hAIl8pE6wu5Q= github.com/libp2p/go-libp2p-record v0.1.1/go.mod h1:VRgKajOyMVgP/F0L5g3kH7SVskp17vFi2xheb5uMJtg= @@ -1022,8 +1022,8 @@ github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-b github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/lucas-clemente/quic-go v0.11.2/go.mod h1:PpMmPfPKO9nKJ/psF49ESTAGQSdfXxlg1otPbEB2nOw= github.com/lucas-clemente/quic-go v0.16.0/go.mod h1:I0+fcNTdb9eS1ZcjQZbDVPGchJ86chcIxPALn9lEJqE= -github.com/lucas-clemente/quic-go v0.18.0 h1:JhQDdqxdwdmGdKsKgXi1+coHRoGhvU6z0rNzOJqZ/4o= -github.com/lucas-clemente/quic-go v0.18.0/go.mod h1:yXttHsSNxQi8AWijC/vLP+OJczXqzHSOcJrM5ITUlCg= +github.com/lucas-clemente/quic-go v0.18.1 h1:DMR7guC0NtVS8zNZR3IO7NARZvZygkSC56GGtC6cyys= +github.com/lucas-clemente/quic-go v0.18.1/go.mod h1:yXttHsSNxQi8AWijC/vLP+OJczXqzHSOcJrM5ITUlCg= github.com/lufia/iostat v1.1.0/go.mod h1:rEPNA0xXgjHQjuI5Cy05sLlS2oRcSlWHRLrvh/AQ+Pg= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= From 4cd73f156021b697ac0d5b9ab25de9cb06e13d2d Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 8 Oct 2020 21:40:36 +0300 Subject: [PATCH 622/885] use subscription filter in pubsub --- node/modules/lp2p/pubsub.go | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go index 7dca3608f..f36e436d8 100644 --- a/node/modules/lp2p/pubsub.go +++ b/node/modules/lp2p/pubsub.go @@ -187,13 +187,14 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) { build.MessagesTopic(in.Nn): 1, } + var drandTopic string for _, d := range in.Dr { - topic, err := getDrandTopic(d.Config.ChainInfoJSON) + drandTopic, err = getDrandTopic(d.Config.ChainInfoJSON) if err != nil { return nil, err } - topicParams[topic] = drandTopicParams - pgTopicWeights[topic] = 5 + topicParams[drandTopic] = drandTopicParams + pgTopicWeights[drandTopic] = 5 } options := []pubsub.Option{ @@ -308,6 +309,14 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) { } options = append(options, pubsub.WithPeerGater(pgParams)) + options = append(options, + pubsub.WithSubscriptionFilter( + pubsub.WrapLimitSubscriptionFilter( + pubsub.NewAllowlistSubscriptionFilter( + build.BlocksTopic(in.Nn), + build.MessagesTopic(in.Nn), + drandTopic), + 100))) // tracer if in.Cfg.RemoteTracer != "" { @@ -359,14 +368,9 @@ type tracerWrapper struct { topics map[string]struct{} } -func (trw *tracerWrapper) traceMessage(topics []string) bool { - for _, topic := range topics { - _, ok := trw.topics[topic] - if ok { - return true - } - } - return false +func (trw *tracerWrapper) traceMessage(topic string) bool { + _, ok := trw.topics[topic] + return ok } func (trw *tracerWrapper) Trace(evt *pubsub_pb.TraceEvent) { @@ -379,12 +383,12 @@ func (trw *tracerWrapper) Trace(evt *pubsub_pb.TraceEvent) { switch evt.GetType() { case pubsub_pb.TraceEvent_PUBLISH_MESSAGE: stats.Record(context.TODO(), metrics.PubsubPublishMessage.M(1)) - if trw.tr != nil && trw.traceMessage(evt.GetPublishMessage().Topics) { + if trw.tr != nil && trw.traceMessage(evt.GetPublishMessage().GetTopic()) { trw.tr.Trace(evt) } case pubsub_pb.TraceEvent_DELIVER_MESSAGE: stats.Record(context.TODO(), metrics.PubsubDeliverMessage.M(1)) - if trw.tr != nil && trw.traceMessage(evt.GetDeliverMessage().Topics) { + if trw.tr != nil && trw.traceMessage(evt.GetDeliverMessage().GetTopic()) { trw.tr.Trace(evt) } case pubsub_pb.TraceEvent_REJECT_MESSAGE: From 7933cf094f3d9d1f1086ef559506a4acd1dda2cb Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 8 Oct 2020 21:48:55 +0300 Subject: [PATCH 623/885] allow all drand topics --- node/modules/lp2p/pubsub.go | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go index f36e436d8..9724eb3b4 100644 --- a/node/modules/lp2p/pubsub.go +++ b/node/modules/lp2p/pubsub.go @@ -187,14 +187,15 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) { build.MessagesTopic(in.Nn): 1, } - var drandTopic string + var drandTopics []string for _, d := range in.Dr { - drandTopic, err = getDrandTopic(d.Config.ChainInfoJSON) + topic, err := getDrandTopic(d.Config.ChainInfoJSON) if err != nil { return nil, err } - topicParams[drandTopic] = drandTopicParams - pgTopicWeights[drandTopic] = 5 + topicParams[topic] = drandTopicParams + pgTopicWeights[topic] = 5 + drandTopics = append(drandTopics, topic) } options := []pubsub.Option{ @@ -309,13 +310,16 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) { } options = append(options, pubsub.WithPeerGater(pgParams)) + + allowTopics := []string{ + build.BlocksTopic(in.Nn), + build.MessagesTopic(in.Nn), + } + allowTopics = append(allowTopics, drandTopics...) options = append(options, pubsub.WithSubscriptionFilter( pubsub.WrapLimitSubscriptionFilter( - pubsub.NewAllowlistSubscriptionFilter( - build.BlocksTopic(in.Nn), - build.MessagesTopic(in.Nn), - drandTopic), + pubsub.NewAllowlistSubscriptionFilter(allowTopics...), 100))) // tracer From c3420c0ceb0023b166ed7a5386b0512ef94e1369 Mon Sep 17 00:00:00 2001 From: Ingar Shu Date: Thu, 8 Oct 2020 12:08:01 -0700 Subject: [PATCH 624/885] Display price, verified-price with units in `lotus-miner storage-deals get-ask` --- cmd/lotus-storage-miner/market.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/lotus-storage-miner/market.go b/cmd/lotus-storage-miner/market.go index 67e694c78..bb1ebd9ec 100644 --- a/cmd/lotus-storage-miner/market.go +++ b/cmd/lotus-storage-miner/market.go @@ -288,7 +288,7 @@ var getAskCmd = &cli.Command{ rem = (time.Second * time.Duration(int64(dlt)*int64(build.BlockDelaySecs))).String() } - fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%d\t%s\t%d\n", ask.Price, ask.VerifiedPrice, types.SizeStr(types.NewInt(uint64(ask.MinPieceSize))), types.SizeStr(types.NewInt(uint64(ask.MaxPieceSize))), ask.Expiry, rem, ask.SeqNo) + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%d\t%s\t%d\n", types.FIL(ask.Price), types.FIL(ask.VerifiedPrice), types.SizeStr(types.NewInt(uint64(ask.MinPieceSize))), types.SizeStr(types.NewInt(uint64(ask.MaxPieceSize))), ask.Expiry, rem, ask.SeqNo) return w.Flush() }, From fe5b32026ac725167e38208576eb23d5333475ff Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Thu, 8 Oct 2020 15:19:39 -0400 Subject: [PATCH 625/885] Add some new endpoints for querying Msig info --- api/api_full.go | 15 +++++++++++++ api/apistruct/struct.go | 5 +++++ documentation/en/api-methods.md | 32 +++++++++++++++++++++++++++ node/impl/full/state.go | 38 +++++++++++++++++++++++++++++++++ 4 files changed, 90 insertions(+) diff --git a/api/api_full.go b/api/api_full.go index b6ae77f77..601b14660 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -418,6 +418,9 @@ type FullNode interface { // MsigGetAvailableBalance returns the portion of a multisig's balance that can be withdrawn or spent MsigGetAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) + // MsigGetLockedBalance returns the locked balance of an msig at a vien epoch. + // The return may be greater than the multisig actor's actual balance. + MsigGetVestingSchedule(context.Context, address.Address, types.TipSetKey) (MsigVesting, error) // MsigGetVested returns the amount of FIL that vested in a multisig in a certain period. // It takes the following params: , , MsigGetVested(context.Context, address.Address, types.TipSetKey, types.TipSetKey) (types.BigInt, error) @@ -871,3 +874,15 @@ type Fault struct { Miner address.Address Epoch abi.ChainEpoch } + +var EmptyVesting = MsigVesting{ + InitialBalance: types.EmptyInt, + StartEpoch: -1, + UnlockDuration: -1, +} + +type MsigVesting struct { + InitialBalance abi.TokenAmount + StartEpoch abi.ChainEpoch + UnlockDuration abi.ChainEpoch +} diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index a09700eb9..1a3ddda58 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -212,6 +212,7 @@ type FullNodeStruct struct { StateNetworkVersion func(context.Context, types.TipSetKey) (stnetwork.Version, error) `perm:"read"` MsigGetAvailableBalance func(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) `perm:"read"` + MsigGetVestingSchedule func(context.Context, address.Address, types.TipSetKey) (api.MsigVesting, error) `perm:"read"` MsigGetVested func(context.Context, address.Address, types.TipSetKey, types.TipSetKey) (types.BigInt, error) `perm:"read"` MsigCreate func(context.Context, uint64, []address.Address, abi.ChainEpoch, types.BigInt, address.Address, types.BigInt) (cid.Cid, error) `perm:"sign"` MsigPropose func(context.Context, address.Address, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error) `perm:"sign"` @@ -933,6 +934,10 @@ func (c *FullNodeStruct) MsigGetAvailableBalance(ctx context.Context, a address. return c.Internal.MsigGetAvailableBalance(ctx, a, tsk) } +func (c *FullNodeStruct) MsigGetVestingSchedule(ctx context.Context, a address.Address, tsk types.TipSetKey) (api.MsigVesting, error) { + return c.Internal.MsigGetVestingSchedule(ctx, a, tsk) +} + func (c *FullNodeStruct) MsigGetVested(ctx context.Context, a address.Address, sTsk types.TipSetKey, eTsk types.TipSetKey) (types.BigInt, error) { return c.Internal.MsigGetVested(ctx, a, sTsk, eTsk) } diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index a6ad1ecb2..8a288a4bf 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -87,6 +87,7 @@ * [MsigCreate](#MsigCreate) * [MsigGetAvailableBalance](#MsigGetAvailableBalance) * [MsigGetVested](#MsigGetVested) + * [MsigGetVestingSchedule](#MsigGetVestingSchedule) * [MsigPropose](#MsigPropose) * [MsigSwapApprove](#MsigSwapApprove) * [MsigSwapCancel](#MsigSwapCancel) @@ -2143,6 +2144,37 @@ Inputs: Response: `"0"` +### MsigGetVestingSchedule +MsigGetLockedBalance returns the locked balance of an msig at a vien epoch. +The return may be greater than the multisig actor's actual balance. + + +Perms: read + +Inputs: +```json +[ + "f01234", + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "InitialBalance": "0", + "StartEpoch": 10101, + "UnlockDuration": 10101 +} +``` + ### MsigPropose MsigPropose proposes a multisig message It takes the following params: , , , diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 7d654985a..56dfc0a14 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -820,6 +820,44 @@ func (a *StateAPI) MsigGetAvailableBalance(ctx context.Context, addr address.Add return types.BigSub(act.Balance, locked), nil } +func (a *StateAPI) MsigGetVestingSchedule(ctx context.Context, addr address.Address, tsk types.TipSetKey) (api.MsigVesting, error) { + ts, err := a.Chain.GetTipSetFromKey(tsk) + if err != nil { + return api.EmptyVesting, xerrors.Errorf("loading tipset %s: %w", tsk, err) + } + + act, err := a.StateManager.LoadActor(ctx, addr, ts) + if err != nil { + return api.EmptyVesting, xerrors.Errorf("failed to load multisig actor: %w", err) + } + + msas, err := multisig.Load(a.Chain.Store(ctx), act) + if err != nil { + return api.EmptyVesting, xerrors.Errorf("failed to load multisig actor state: %w", err) + } + + ib, err := msas.InitialBalance() + if err != nil { + return api.EmptyVesting, xerrors.Errorf("failed to load multisig initial balance: %w", err) + } + + se, err := msas.StartEpoch() + if err != nil { + return api.EmptyVesting, xerrors.Errorf("failed to load multisig start epoch: %w", err) + } + + ud, err := msas.UnlockDuration() + if err != nil { + return api.EmptyVesting, xerrors.Errorf("failed to load multisig unlock duration: %w", err) + } + + return api.MsigVesting{ + InitialBalance: ib, + StartEpoch: se, + UnlockDuration: ud, + }, nil +} + func (a *StateAPI) MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error) { startTs, err := a.Chain.GetTipSetFromKey(start) if err != nil { From 21de538d09fb47b732d5a4709ae429ec1fbb4da1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Thu, 8 Oct 2020 21:18:33 +0100 Subject: [PATCH 626/885] upgrade go-libp2p-noise to v0.1.2. Improves throughput in large and/or fast transfers and reduces syscalls when data is queued in receive buffer, by using bufio.Reader. --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index f2d95d22a..24de7012e 100644 --- a/go.mod +++ b/go.mod @@ -94,7 +94,7 @@ require ( github.com/libp2p/go-libp2p-discovery v0.5.0 github.com/libp2p/go-libp2p-kad-dht v0.8.3 github.com/libp2p/go-libp2p-mplex v0.2.4 - github.com/libp2p/go-libp2p-noise v0.1.1 + github.com/libp2p/go-libp2p-noise v0.1.2 github.com/libp2p/go-libp2p-peerstore v0.2.6 github.com/libp2p/go-libp2p-pubsub v0.3.6 github.com/libp2p/go-libp2p-quic-transport v0.8.2 diff --git a/go.sum b/go.sum index efb1d2dfb..d2b122929 100644 --- a/go.sum +++ b/go.sum @@ -863,6 +863,8 @@ github.com/libp2p/go-libp2p-netutil v0.1.0 h1:zscYDNVEcGxyUpMd0JReUZTrpMfia8PmLK github.com/libp2p/go-libp2p-netutil v0.1.0/go.mod h1:3Qv/aDqtMLTUyQeundkKsA+YCThNdbQD54k3TqjpbFU= github.com/libp2p/go-libp2p-noise v0.1.1 h1:vqYQWvnIcHpIoWJKC7Al4D6Hgj0H012TuXRhPwSMGpQ= github.com/libp2p/go-libp2p-noise v0.1.1/go.mod h1:QDFLdKX7nluB7DEnlVPbz7xlLHdwHFA9HiohJRr3vwM= +github.com/libp2p/go-libp2p-noise v0.1.2 h1:IH9GRihQJTx56obm+GnpdPX4KeVIlvpXrP6xnJ0wxWk= +github.com/libp2p/go-libp2p-noise v0.1.2/go.mod h1:9B10b7ueo7TIxZHHcjcDCo5Hd6kfKT2m77by82SFRfE= github.com/libp2p/go-libp2p-peer v0.0.1/go.mod h1:nXQvOBbwVqoP+T5Y5nCjeH4sP9IX/J0AMzcDUVruVoo= github.com/libp2p/go-libp2p-peer v0.1.1/go.mod h1:jkF12jGB4Gk/IOo+yomm+7oLWxF278F7UnrYUQ1Q8es= github.com/libp2p/go-libp2p-peer v0.2.0/go.mod h1:RCffaCvUyW2CJmG2gAWVqwePwW7JMgxjsHm7+J5kjWY= From 0f8fc119f32fe312ec96ec94239e8ca2f0b71450 Mon Sep 17 00:00:00 2001 From: jennijuju Date: Thu, 8 Oct 2020 16:45:07 -0400 Subject: [PATCH 627/885] Update address prefix in client cli. --- cli/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/client.go b/cli/client.go index 7494815bf..678c7d7f4 100644 --- a/cli/client.go +++ b/cli/client.go @@ -536,7 +536,7 @@ func interactiveDeal(cctx *cli.Context) error { state = "miner" case "miner": - fmt.Print("Miner Address (t0..): ") + fmt.Print("Miner Address (f0..): ") var maddrStr string _, err := fmt.Scan(&maddrStr) From c60d13ee56342fd4be784574bf05691927f7e4cd Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Thu, 8 Oct 2020 13:52:39 -0700 Subject: [PATCH 628/885] feat(markets): update markets v0.7.1 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 24de7012e..e9687c7b8 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 github.com/filecoin-project/go-data-transfer v0.6.7 github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f - github.com/filecoin-project/go-fil-markets v0.7.0 + github.com/filecoin-project/go-fil-markets v0.7.1 github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200822201400-474f4fdccc52 github.com/filecoin-project/go-multistore v0.0.3 github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 diff --git a/go.sum b/go.sum index d2b122929..bab4e929d 100644 --- a/go.sum +++ b/go.sum @@ -244,8 +244,8 @@ github.com/filecoin-project/go-ds-versioning v0.1.0 h1:y/X6UksYTsK8TLCI7rttCKEvl github.com/filecoin-project/go-ds-versioning v0.1.0/go.mod h1:mp16rb4i2QPmxBnmanUx8i/XANp+PFCCJWiAb+VW4/s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f h1:GxJzR3oRIMTPtpZ0b7QF8FKPK6/iPAc7trhlL5k/g+s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= -github.com/filecoin-project/go-fil-markets v0.7.0 h1:tcEZiUNIYQJ4PBzgVpLwfdJ4ZdC4WCv9LsgvsoCXIls= -github.com/filecoin-project/go-fil-markets v0.7.0/go.mod h1:5Pt4DXQqUoUrp9QzlSdlYTpItXxwAtqKrxRWQ6hAOqk= +github.com/filecoin-project/go-fil-markets v0.7.1 h1:e0NlpSnaeGyDUhCOzevjcxkSA54kt9BzlXpLRgduUFI= +github.com/filecoin-project/go-fil-markets v0.7.1/go.mod h1:5Pt4DXQqUoUrp9QzlSdlYTpItXxwAtqKrxRWQ6hAOqk= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24= From 60768f48636633401f58bef5dc9e51af89ebfe94 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 8 Oct 2020 18:38:22 +0200 Subject: [PATCH 629/885] Optimize SearchForMessage and GetReceipt Signed-off-by: Jakub Sztandera --- chain/stmgr/stmgr.go | 57 +++++++++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 24 deletions(-) diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index d81cf1c72..ba3dcd1d8 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -2,6 +2,7 @@ package stmgr import ( "context" + "errors" "fmt" "sync" @@ -507,16 +508,7 @@ func (sm *StateManager) GetReceipt(ctx context.Context, msg cid.Cid, ts *types.T return nil, fmt.Errorf("failed to load message: %w", err) } - r, _, err := sm.tipsetExecutedMessage(ts, msg, m.VMMessage()) - if err != nil { - return nil, err - } - - if r != nil { - return r, nil - } - - _, r, _, err = sm.searchBackForMsg(ctx, ts, m) + _, r, _, err := sm.searchBackForMsg(ctx, ts, m) if err != nil { return nil, fmt.Errorf("failed to look back through chain for message: %w", err) } @@ -674,6 +666,18 @@ func (sm *StateManager) SearchForMessage(ctx context.Context, mcid cid.Cid) (*ty func (sm *StateManager) searchBackForMsg(ctx context.Context, from *types.TipSet, m types.ChainMsg) (*types.TipSet, *types.MessageReceipt, cid.Cid, error) { cur := from + curActor, err := sm.LoadActor(ctx, m.VMMessage().From, cur) + if err != nil { + return nil, nil, cid.Undef, xerrors.Errorf("failed to load initital tipset") + } + + mFromId, err := sm.LookupID(ctx, m.VMMessage().From, from) + if err != nil { + return nil, nil, cid.Undef, xerrors.Errorf("looking up From id address: %w", err) + } + + mNonce := m.VMMessage().Nonce + for { if cur.Height() == 0 { // it ain't here! @@ -686,32 +690,37 @@ func (sm *StateManager) searchBackForMsg(ctx context.Context, from *types.TipSet default: } - act, err := sm.LoadActor(ctx, m.VMMessage().From, cur) - if err != nil { - return nil, nil, cid.Cid{}, err - } - // we either have no messages from the sender, or the latest message we found has a lower nonce than the one being searched for, // either way, no reason to lookback, it ain't there - if act.Nonce == 0 || act.Nonce < m.VMMessage().Nonce { + if curActor == nil || curActor.Nonce == 0 || curActor.Nonce < mNonce { return nil, nil, cid.Undef, nil } - ts, err := sm.cs.LoadTipSet(cur.Parents()) + pts, err := sm.cs.LoadTipSet(cur.Parents()) if err != nil { - return nil, nil, cid.Undef, fmt.Errorf("failed to load tipset during msg wait searchback: %w", err) + return nil, nil, cid.Undef, xerrors.Errorf("failed to load tipset during msg wait searchback: %w", err) } - r, foundMsg, err := sm.tipsetExecutedMessage(ts, m.Cid(), m.VMMessage()) - if err != nil { - return nil, nil, cid.Undef, fmt.Errorf("checking for message execution during lookback: %w", err) + act, err := sm.LoadActor(ctx, mFromId, pts) + actorNoExist := errors.Is(err, types.ErrActorNotFound) + if err != nil && !actorNoExist { + return nil, nil, cid.Cid{}, xerrors.Errorf("failed to load the actor: %w", err) } - if r != nil { - return ts, r, foundMsg, nil + // check that between cur and parent tipset the nonce fell into range of our message + if actorNoExist || (curActor.Nonce > mNonce && act.Nonce <= mNonce) { + r, foundMsg, err := sm.tipsetExecutedMessage(cur, m.Cid(), m.VMMessage()) + if err != nil { + return nil, nil, cid.Undef, xerrors.Errorf("checking for message execution during lookback: %w", err) + } + + if r != nil { + return pts, r, foundMsg, nil + } } - cur = ts + cur = pts + curActor = act } } From 973f61bc105acbb7df6b394e092ae400c4c2d911 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 8 Oct 2020 22:49:36 +0200 Subject: [PATCH 630/885] Optimize chain and message sync Signed-off-by: Jakub Sztandera --- chain/sub/incoming.go | 17 +++++----- chain/sub/incoming_test.go | 63 ++++++++++++++++++++++++++++++++++++++ chain/sync.go | 22 ++++++++++--- 3 files changed, 88 insertions(+), 14 deletions(-) create mode 100644 chain/sub/incoming_test.go diff --git a/chain/sub/incoming.go b/chain/sub/incoming.go index 07b3343d2..d51c481d1 100644 --- a/chain/sub/incoming.go +++ b/chain/sub/incoming.go @@ -172,25 +172,24 @@ func fetchCids( cids []cid.Cid, cb func(int, blocks.Block) error, ) error { - fetchedBlocks := bserv.GetBlocks(ctx, cids) + + ctx, cancel := context.WithCancel(ctx) + defer cancel() cidIndex := make(map[cid.Cid]int) for i, c := range cids { cidIndex[c] = i } + if len(cids) != len(cidIndex) { + return fmt.Errorf("duplicate CIDs in fetchCids input") + } + + fetchedBlocks := bserv.GetBlocks(ctx, cids) for i := 0; i < len(cids); i++ { select { case block, ok := <-fetchedBlocks: if !ok { - // Closed channel, no more blocks fetched, check if we have all - // of the CIDs requested. - // FIXME: Review this check. We don't call the callback on the - // last index? - if i == len(cids)-1 { - break - } - return fmt.Errorf("failed to fetch all messages") } diff --git a/chain/sub/incoming_test.go b/chain/sub/incoming_test.go new file mode 100644 index 000000000..215439209 --- /dev/null +++ b/chain/sub/incoming_test.go @@ -0,0 +1,63 @@ +package sub + +import ( + "context" + "testing" + + address "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" +) + +type getter struct { + msgs []*types.Message +} + +func (g *getter) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) { panic("NYI") } + +func (g *getter) GetBlocks(ctx context.Context, ks []cid.Cid) <-chan blocks.Block { + ch := make(chan blocks.Block, len(g.msgs)) + for _, m := range g.msgs { + by, err := m.Serialize() + if err != nil { + panic(err) + } + b, err := blocks.NewBlockWithCid(by, m.Cid()) + if err != nil { + panic(err) + } + ch <- b + } + close(ch) + return ch +} + +func TestFetchCidsWithDedup(t *testing.T) { + msgs := []*types.Message{} + for i := 0; i < 10; i++ { + msgs = append(msgs, &types.Message{ + From: address.TestAddress, + To: address.TestAddress, + + Nonce: uint64(i), + }) + } + cids := []cid.Cid{} + for _, m := range msgs { + cids = append(cids, m.Cid()) + } + g := &getter{msgs} + + // the cids have a duplicate + res, err := FetchMessagesByCids(context.TODO(), g, append(cids, cids[0])) + + t.Logf("err: %+v", err) + t.Logf("res: %+v", res) + if err == nil { + t.Errorf("there should be an error") + } + if err == nil && (res[0] == nil || res[len(res)-1] == nil) { + t.Fatalf("there is a nil message: first %p, last %p", res[0], res[len(res)-1]) + } +} diff --git a/chain/sync.go b/chain/sync.go index 240d1edef..c280e3a40 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -217,6 +217,12 @@ func (syncer *Syncer) Stop() { // This should be called when connecting to new peers, and additionally // when receiving new blocks from the network func (syncer *Syncer) InformNewHead(from peer.ID, fts *store.FullTipSet) bool { + defer func() { + if err := recover(); err != nil { + log.Errorf("panic in InformNewHead: ", err) + } + }() + ctx := context.Background() if fts == nil { log.Errorf("got nil tipset in InformNewHead") @@ -1281,9 +1287,11 @@ func (syncer *Syncer) collectHeaders(ctx context.Context, incoming *types.TipSet blockSet := []*types.TipSet{incoming} + // Parent of the new (possibly better) tipset that we need to fetch next. at := incoming.Parents() - // we want to sync all the blocks until the height above the block we have + // we want to sync all the blocks until the height above our + // best tipset so far untilHeight := known.Height() + 1 ss.SetHeight(blockSet[len(blockSet)-1].Height()) @@ -1377,13 +1385,17 @@ loop: } base := blockSet[len(blockSet)-1] - if base.Parents() == known.Parents() { - // common case: receiving a block thats potentially part of the same tipset as our best block + if base.IsChildOf(known) { + // common case: receiving blocks that are building on top of our best tipset return blockSet, nil } - if types.CidArrsEqual(base.Parents().Cids(), known.Cids()) { - // common case: receiving blocks that are building on top of our best tipset + knownParent, err := syncer.store.LoadTipSet(known.Parents()) + if err != nil { + return nil, xerrors.Errorf("failed to load next local tipset: %w", err) + } + if base.IsChildOf(knownParent) { + // common case: receiving a block thats potentially part of the same tipset as our best block return blockSet, nil } From 0223582d4106e36d212827fb2f5aa9f4c9189b34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 8 Oct 2020 22:48:21 +0200 Subject: [PATCH 631/885] sync wait: Handle processed message offset --- cli/sync.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/cli/sync.go b/cli/sync.go index bee87cf70..dea96d14e 100644 --- a/cli/sync.go +++ b/cli/sync.go @@ -233,7 +233,13 @@ func SyncWait(ctx context.Context, napi api.FullNode) error { samples := 8 i := 0 - var app, lastApp uint64 + var firstApp, app, lastApp uint64 + + state, err := napi.SyncState(ctx) + if err != nil { + return err + } + firstApp = state.VMApplied for { state, err := napi.SyncState(ctx) @@ -286,10 +292,10 @@ func SyncWait(ctx context.Context, napi api.FullNode) error { if i%samples == 0 { lastApp = app - app = state.VMApplied + app = state.VMApplied - firstApp } if i > 0 { - fmt.Printf("Validated %d messages (%d per second)\n", state.VMApplied, (app-lastApp)*uint64(time.Second/tick)/uint64(samples)) + fmt.Printf("Validated %d messages (%d per second)\n", state.VMApplied-firstApp, (app-lastApp)*uint64(time.Second/tick)/uint64(samples)) lastLines++ } From a23ab123d794dace19775aa0aa2e1686b5552184 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=9E=97=E6=AC=A3?= Date: Tue, 1 Sep 2020 11:44:55 +0800 Subject: [PATCH 632/885] fix GetBestMiningCandidate bug Signed-off-by: Jakub Sztandera --- miner/miner.go | 1 + 1 file changed, 1 insertion(+) diff --git a/miner/miner.go b/miner/miner.go index a256e8b3a..aebcb7a25 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -327,6 +327,7 @@ func (m *Miner) GetBestMiningCandidate(ctx context.Context) (*MiningBase, error) } ltsw, err := m.api.ChainTipSetWeight(ctx, m.lastWork.TipSet.Key()) if err != nil { + m.lastWork = nil return nil, err } From 483510947096928612cadf6e3a91c1d6d466b13e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 9 Oct 2020 00:51:04 +0200 Subject: [PATCH 633/885] wallet: Post-merge fixes --- api/api_wallet.go | 3 ++- chain/messagesigner/messagesigner.go | 10 +++++----- chain/wallet/key.go | 2 +- chain/wallet/wallet.go | 2 +- cmd/lotus-wallet/logged.go | 2 +- 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/api/api_wallet.go b/api/api_wallet.go index 06b00a75b..9c289ca95 100644 --- a/api/api_wallet.go +++ b/api/api_wallet.go @@ -4,8 +4,9 @@ import ( "context" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/crypto" + "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/specs-actors/actors/crypto" ) type WalletAPI interface { diff --git a/chain/messagesigner/messagesigner.go b/chain/messagesigner/messagesigner.go index ac94d6a3e..1240485b8 100644 --- a/chain/messagesigner/messagesigner.go +++ b/chain/messagesigner/messagesigner.go @@ -3,12 +3,12 @@ package messagesigner import ( "bytes" "context" + "github.com/filecoin-project/lotus/api" "sync" "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/messagepool" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/chain/wallet" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" @@ -28,17 +28,17 @@ type mpoolAPI interface { // MessageSigner keeps track of nonces per address, and increments the nonce // when signing a message type MessageSigner struct { - wallet *wallet.Wallet + wallet api.WalletAPI lk sync.Mutex mpool mpoolAPI ds datastore.Batching } -func NewMessageSigner(wallet *wallet.Wallet, mpool *messagepool.MessagePool, ds dtypes.MetadataDS) *MessageSigner { +func NewMessageSigner(wallet api.WalletAPI, mpool *messagepool.MessagePool, ds dtypes.MetadataDS) *MessageSigner { return newMessageSigner(wallet, mpool, ds) } -func newMessageSigner(wallet *wallet.Wallet, mpool mpoolAPI, ds dtypes.MetadataDS) *MessageSigner { +func newMessageSigner(wallet api.WalletAPI, mpool mpoolAPI, ds dtypes.MetadataDS) *MessageSigner { ds = namespace.Wrap(ds, datastore.NewKey("/message-signer/")) return &MessageSigner{ wallet: wallet, @@ -61,7 +61,7 @@ func (ms *MessageSigner) SignMessage(ctx context.Context, msg *types.Message, cb // Sign the message with the nonce msg.Nonce = nonce - sig, err := ms.wallet.Sign(ctx, msg.From, msg.Cid().Bytes()) + sig, err := ms.wallet.WalletSign(ctx, msg.From, msg.Cid().Bytes()) if err != nil { return nil, xerrors.Errorf("failed to sign message: %w", err) } diff --git a/chain/wallet/key.go b/chain/wallet/key.go index 2d4ecafd7..4b746a17a 100644 --- a/chain/wallet/key.go +++ b/chain/wallet/key.go @@ -4,7 +4,7 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/specs-actors/actors/crypto" + "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/sigs" diff --git a/chain/wallet/wallet.go b/chain/wallet/wallet.go index b4c846d37..5f501307f 100644 --- a/chain/wallet/wallet.go +++ b/chain/wallet/wallet.go @@ -152,7 +152,7 @@ func (w *LocalWallet) tryFind(addr address.Address) (types.KeyInfo, error) { return ki, nil } -func (w *LocalWallet) Export(ctx context.Context, addr address.Address) (*types.KeyInfo, error) { +func (w *LocalWallet) WalletExport(ctx context.Context, addr address.Address) (*types.KeyInfo, error) { k, err := w.findKey(addr) if err != nil { return nil, xerrors.Errorf("failed to find key to export: %w", err) diff --git a/cmd/lotus-wallet/logged.go b/cmd/lotus-wallet/logged.go index e73daae90..4cdd62825 100644 --- a/cmd/lotus-wallet/logged.go +++ b/cmd/lotus-wallet/logged.go @@ -4,7 +4,7 @@ import ( "context" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/specs-actors/actors/crypto" + "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" From 114776f2c5e7beccd0dd865bfa9758e8239cdd10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 9 Oct 2020 01:04:59 +0200 Subject: [PATCH 634/885] wallet: Drop WalletSignMessage from WalletAPI --- api/api_wallet.go | 4 ++-- chain/wallet/wallet.go | 14 -------------- cmd/lotus-wallet/logged.go | 6 ------ node/impl/full/wallet.go | 15 +++++++++++++-- 4 files changed, 15 insertions(+), 24 deletions(-) diff --git a/api/api_wallet.go b/api/api_wallet.go index 9c289ca95..b3dbd5fa9 100644 --- a/api/api_wallet.go +++ b/api/api_wallet.go @@ -13,9 +13,9 @@ type WalletAPI interface { WalletNew(context.Context, crypto.SigType) (address.Address, error) WalletHas(context.Context, address.Address) (bool, error) WalletList(context.Context) ([]address.Address, error) + WalletSign(context.Context, address.Address, []byte) (*crypto.Signature, error) - // nonce keeping done by wallet app - WalletSignMessage(context.Context, address.Address, *types.Message) (*types.SignedMessage, error) + WalletExport(context.Context, address.Address) (*types.KeyInfo, error) WalletImport(context.Context, *types.KeyInfo) (address.Address, error) WalletDelete(context.Context, address.Address) error diff --git a/chain/wallet/wallet.go b/chain/wallet/wallet.go index 5f501307f..607380547 100644 --- a/chain/wallet/wallet.go +++ b/chain/wallet/wallet.go @@ -74,20 +74,6 @@ func (w *LocalWallet) WalletSign(ctx context.Context, addr address.Address, msg return sigs.Sign(ActSigType(ki.Type), ki.PrivateKey, msg) } -func (w *LocalWallet) WalletSignMessage(ctx context.Context, k address.Address, msg *types.Message) (*types.SignedMessage, error) { - mcid := msg.Cid() - - sig, err := w.WalletSign(ctx, k, mcid.Bytes()) - if err != nil { - return nil, xerrors.Errorf("failed to sign message: %w", err) - } - - return &types.SignedMessage{ - Message: *msg, - Signature: *sig, - }, nil -} - func (w *LocalWallet) findKey(addr address.Address) (*Key, error) { w.lk.Lock() defer w.lk.Unlock() diff --git a/cmd/lotus-wallet/logged.go b/cmd/lotus-wallet/logged.go index 4cdd62825..adafb7cc5 100644 --- a/cmd/lotus-wallet/logged.go +++ b/cmd/lotus-wallet/logged.go @@ -43,12 +43,6 @@ func (c *LoggedWallet) WalletSign(ctx context.Context, k address.Address, msg [] return c.under.WalletSign(ctx, k, msg) } -func (c *LoggedWallet) WalletSignMessage(ctx context.Context, k address.Address, msg *types.Message) (*types.SignedMessage, error) { - log.Infow("WalletSignMessage", "address", k) - - return c.under.WalletSignMessage(ctx, k, msg) -} - func (c *LoggedWallet) WalletExport(ctx context.Context, a address.Address) (*types.KeyInfo, error) { log.Infow("WalletExport", "address", a) diff --git a/node/impl/full/wallet.go b/node/impl/full/wallet.go index 6252e12a4..13880a03a 100644 --- a/node/impl/full/wallet.go +++ b/node/impl/full/wallet.go @@ -3,10 +3,10 @@ package full import ( "context" - "github.com/filecoin-project/go-address" "go.uber.org/fx" "golang.org/x/xerrors" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" @@ -48,7 +48,18 @@ func (a *WalletAPI) WalletSignMessage(ctx context.Context, k address.Address, ms if err != nil { return nil, xerrors.Errorf("failed to resolve ID address: %w", keyAddr) } - return a.WalletAPI.WalletSignMessage(ctx, keyAddr, msg) + + mcid := msg.Cid() + + sig, err := a.WalletAPI.WalletSign(ctx, k, mcid.Bytes()) + if err != nil { + return nil, xerrors.Errorf("failed to sign message: %w", err) + } + + return &types.SignedMessage{ + Message: *msg, + Signature: *sig, + }, nil } func (a *WalletAPI) WalletVerify(ctx context.Context, k address.Address, msg []byte, sig *crypto.Signature) (bool, error) { From f3dc730b05c5ff9d7161a731594bbba3fc37b90f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 9 Oct 2020 01:27:38 +0200 Subject: [PATCH 635/885] wallet: Add metadata to WalletAPI.WalletSign --- api/api_wallet.go | 26 +++++++++++++++++++++++++- api/apistruct/struct.go | 6 +++--- chain/gen/gen.go | 16 +++++++++++++--- chain/gen/mining.go | 7 ++++--- chain/messagesigner/messagesigner.go | 11 ++++++++++- chain/wallet/wallet.go | 2 +- cmd/lotus-wallet/logged.go | 4 ++-- markets/storageadapter/client.go | 11 ++++++++--- node/impl/full/state.go | 16 +++++++--------- node/impl/full/wallet.go | 10 ++++++++-- 10 files changed, 81 insertions(+), 28 deletions(-) diff --git a/api/api_wallet.go b/api/api_wallet.go index b3dbd5fa9..82ef726a0 100644 --- a/api/api_wallet.go +++ b/api/api_wallet.go @@ -9,12 +9,36 @@ import ( "github.com/filecoin-project/lotus/chain/types" ) +type MsgType string +const ( + MTUnknown = "unknown" + + // Signing message CID. MsgMeta.Extra contains raw cbor message bytes + MTChainMsg = "message" + + // Signing a blockheader. signing raw cbor block bytes (MsgMeta.Extra is empty) + MTBlock = "block" + + // Signing a deal proposal. signing raw cbor proposal bytes (MsgMeta.Extra is empty) + MTDealProposal = "dealproposal" + + // TODO: Deals, Vouchers, VRF +) + +type MsgMeta struct { + Type MsgType + + // Additional data related to what is signed. Should be verifiable with the + // signed bytes (e.g. CID(Extra).Bytes() == toSign) + Extra []byte +} + type WalletAPI interface { WalletNew(context.Context, crypto.SigType) (address.Address, error) WalletHas(context.Context, address.Address) (bool, error) WalletList(context.Context) ([]address.Address, error) - WalletSign(context.Context, address.Address, []byte) (*crypto.Signature, error) + WalletSign(ctx context.Context, signer address.Address, toSign []byte, meta MsgMeta) (*crypto.Signature, error) WalletExport(context.Context, address.Address) (*types.KeyInfo, error) WalletImport(context.Context, *types.KeyInfo) (address.Address, error) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 01de13e61..81fce92b5 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -366,7 +366,7 @@ type WalletStruct struct { WalletNew func(context.Context, crypto.SigType) (address.Address, error) `perm:"write"` WalletHas func(context.Context, address.Address) (bool, error) `perm:"write"` WalletList func(context.Context) ([]address.Address, error) `perm:"write"` - WalletSign func(context.Context, address.Address, []byte) (*crypto.Signature, error) `perm:"sign"` + WalletSign func(context.Context, address.Address, []byte, api.MsgMeta) (*crypto.Signature, error) `perm:"sign"` WalletSignMessage func(context.Context, address.Address, *types.Message) (*types.SignedMessage, error) `perm:"sign"` WalletExport func(context.Context, address.Address) (*types.KeyInfo, error) `perm:"admin"` WalletImport func(context.Context, *types.KeyInfo) (address.Address, error) `perm:"admin"` @@ -1397,8 +1397,8 @@ func (c *WalletStruct) WalletList(ctx context.Context) ([]address.Address, error return c.Internal.WalletList(ctx) } -func (c *WalletStruct) WalletSign(ctx context.Context, k address.Address, msg []byte) (*crypto.Signature, error) { - return c.Internal.WalletSign(ctx, k, msg) +func (c *WalletStruct) WalletSign(ctx context.Context, k address.Address, msg []byte, meta api.MsgMeta) (*crypto.Signature, error) { + return c.Internal.WalletSign(ctx, k, msg, meta) } func (c *WalletStruct) WalletSignMessage(ctx context.Context, k address.Address, msg *types.Message) (*types.SignedMessage, error) { diff --git a/chain/gen/gen.go b/chain/gen/gen.go index ee93ef60e..70c303818 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -374,7 +374,13 @@ func (cg *ChainGen) nextBlockProof(ctx context.Context, pts *types.TipSet, m add return nil, nil, nil, xerrors.Errorf("get miner worker: %w", err) } - vrfout, err := ComputeVRF(ctx, cg.w.WalletSign, worker, ticketRand) + sf := func(ctx context.Context, a address.Address, i []byte) (*crypto.Signature, error) { + return cg.w.WalletSign(ctx, a, i, api.MsgMeta{ + Type: api.MTUnknown, + }) + } + + vrfout, err := ComputeVRF(ctx, sf, worker, ticketRand) if err != nil { return nil, nil, nil, xerrors.Errorf("compute VRF: %w", err) } @@ -528,7 +534,9 @@ func getRandomMessages(cg *ChainGen) ([]*types.SignedMessage, error) { GasPremium: types.NewInt(0), } - sig, err := cg.w.WalletSign(context.TODO(), cg.banker, msg.Cid().Bytes()) + sig, err := cg.w.WalletSign(context.TODO(), cg.banker, msg.Cid().Bytes(), api.MsgMeta{ + Type: api.MTUnknown, // testing + }) if err != nil { return nil, err } @@ -588,7 +596,9 @@ func (mca mca) MinerGetBaseInfo(ctx context.Context, maddr address.Address, epoc } func (mca mca) WalletSign(ctx context.Context, a address.Address, v []byte) (*crypto.Signature, error) { - return mca.w.WalletSign(ctx, a, v) + return mca.w.WalletSign(ctx, a, v, api.MsgMeta{ + Type: api.MTUnknown, + }) } type WinningPoStProver interface { diff --git a/chain/gen/mining.go b/chain/gen/mining.go index 0c2f72590..37e8ca2c0 100644 --- a/chain/gen/mining.go +++ b/chain/gen/mining.go @@ -15,11 +15,10 @@ import ( "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" - "github.com/filecoin-project/lotus/chain/wallet" "github.com/filecoin-project/lotus/lib/sigs/bls" ) -func MinerCreateBlock(ctx context.Context, sm *stmgr.StateManager, w *wallet.LocalWallet, bt *api.BlockTemplate) (*types.FullBlock, error) { +func MinerCreateBlock(ctx context.Context, sm *stmgr.StateManager, w api.WalletAPI, bt *api.BlockTemplate) (*types.FullBlock, error) { pts, err := sm.ChainStore().LoadTipSet(bt.Parents) if err != nil { @@ -131,7 +130,9 @@ func MinerCreateBlock(ctx context.Context, sm *stmgr.StateManager, w *wallet.Loc return nil, xerrors.Errorf("failed to get signing bytes for block: %w", err) } - sig, err := w.WalletSign(ctx, waddr, nosigbytes) + sig, err := w.WalletSign(ctx, waddr, nosigbytes, api.MsgMeta{ + Type: api.MTBlock, + }) if err != nil { return nil, xerrors.Errorf("failed to sign new block: %w", err) } diff --git a/chain/messagesigner/messagesigner.go b/chain/messagesigner/messagesigner.go index 1240485b8..c7948e42e 100644 --- a/chain/messagesigner/messagesigner.go +++ b/chain/messagesigner/messagesigner.go @@ -61,7 +61,16 @@ func (ms *MessageSigner) SignMessage(ctx context.Context, msg *types.Message, cb // Sign the message with the nonce msg.Nonce = nonce - sig, err := ms.wallet.WalletSign(ctx, msg.From, msg.Cid().Bytes()) + + mb, err := msg.ToStorageBlock() + if err != nil { + return nil, xerrors.Errorf("serializing message: %w", err) + } + + sig, err := ms.wallet.WalletSign(ctx, msg.From, mb.Cid().Bytes(), api.MsgMeta{ + Type: api.MTChainMsg, + Extra: mb.RawData(), + }) if err != nil { return nil, xerrors.Errorf("failed to sign message: %w", err) } diff --git a/chain/wallet/wallet.go b/chain/wallet/wallet.go index 607380547..2dbe94b50 100644 --- a/chain/wallet/wallet.go +++ b/chain/wallet/wallet.go @@ -62,7 +62,7 @@ func KeyWallet(keys ...*Key) *LocalWallet { } } -func (w *LocalWallet) WalletSign(ctx context.Context, addr address.Address, msg []byte) (*crypto.Signature, error) { +func (w *LocalWallet) WalletSign(ctx context.Context, addr address.Address, msg []byte, meta api.MsgMeta) (*crypto.Signature, error) { ki, err := w.findKey(addr) if err != nil { return nil, err diff --git a/cmd/lotus-wallet/logged.go b/cmd/lotus-wallet/logged.go index adafb7cc5..9cd729943 100644 --- a/cmd/lotus-wallet/logged.go +++ b/cmd/lotus-wallet/logged.go @@ -37,10 +37,10 @@ func (c *LoggedWallet) WalletList(ctx context.Context) ([]address.Address, error return c.under.WalletList(ctx) } -func (c *LoggedWallet) WalletSign(ctx context.Context, k address.Address, msg []byte) (*crypto.Signature, error) { +func (c *LoggedWallet) WalletSign(ctx context.Context, k address.Address, msg []byte, meta api.MsgMeta) (*crypto.Signature, error) { log.Infow("WalletSign", "address", k) - return c.under.WalletSign(ctx, k, msg) + return c.under.WalletSign(ctx, k, msg, meta) } func (c *LoggedWallet) WalletExport(ctx context.Context, a address.Address) (*types.KeyInfo, error) { diff --git a/markets/storageadapter/client.go b/markets/storageadapter/client.go index 824693dac..22ee483ce 100644 --- a/markets/storageadapter/client.go +++ b/markets/storageadapter/client.go @@ -5,6 +5,7 @@ package storageadapter import ( "bytes" "context" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/go-state-types/big" miner0 "github.com/filecoin-project/specs-actors/actors/builtin" @@ -422,7 +423,9 @@ func (c *ClientNodeAdapter) SignProposal(ctx context.Context, signer address.Add return nil, err } - sig, err := c.Wallet.WalletSign(ctx, signer, buf) + sig, err := c.Wallet.WalletSign(ctx, signer, buf, api.MsgMeta{ + Type: api.MTDealProposal, + }) if err != nil { return nil, err } @@ -434,7 +437,7 @@ func (c *ClientNodeAdapter) SignProposal(ctx context.Context, signer address.Add } func (c *ClientNodeAdapter) GetDefaultWalletAddress(ctx context.Context) (address.Address, error) { - addr, err := c.Wallet.GetDefault() + addr, err := c.DefWallet.GetDefault() return addr, err } @@ -475,7 +478,9 @@ func (c *ClientNodeAdapter) SignBytes(ctx context.Context, signer address.Addres return nil, err } - localSignature, err := c.Wallet.WalletSign(ctx, signer, b) + localSignature, err := c.Wallet.WalletSign(ctx, signer, b, api.MsgMeta{ + Type: api.MTUnknown, // TODO: pass type here + }) if err != nil { return nil, err } diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 33ee75697..49754265d 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -5,14 +5,6 @@ import ( "context" "strconv" - "github.com/filecoin-project/lotus/chain/actors/builtin" - "github.com/filecoin-project/lotus/chain/actors/policy" - - "github.com/filecoin-project/lotus/chain/actors/builtin/verifreg" - - "github.com/filecoin-project/go-state-types/dline" - "github.com/filecoin-project/go-state-types/network" - cid "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" "go.uber.org/fx" @@ -22,14 +14,19 @@ import ( "github.com/filecoin-project/go-bitfield" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/go-state-types/dline" + "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/builtin/market" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/actors/builtin/multisig" "github.com/filecoin-project/lotus/chain/actors/builtin/power" "github.com/filecoin-project/lotus/chain/actors/builtin/reward" + "github.com/filecoin-project/lotus/chain/actors/builtin/verifreg" + "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/beacon" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/state" @@ -47,7 +44,8 @@ type StateAPI struct { // TODO: the wallet here is only needed because we have the MinerCreateBlock // API attached to the state API. It probably should live somewhere better - Wallet *wallet.LocalWallet + Wallet api.WalletAPI + DefWallet wallet.Default ProofVerifier ffiwrapper.Verifier StateManager *stmgr.StateManager diff --git a/node/impl/full/wallet.go b/node/impl/full/wallet.go index 13880a03a..2e2c48c5d 100644 --- a/node/impl/full/wallet.go +++ b/node/impl/full/wallet.go @@ -49,9 +49,15 @@ func (a *WalletAPI) WalletSignMessage(ctx context.Context, k address.Address, ms return nil, xerrors.Errorf("failed to resolve ID address: %w", keyAddr) } - mcid := msg.Cid() + mb, err := msg.ToStorageBlock() + if err != nil { + return nil, xerrors.Errorf("serializing message: %w", err) + } - sig, err := a.WalletAPI.WalletSign(ctx, k, mcid.Bytes()) + sig, err := a.WalletAPI.WalletSign(ctx, k, mb.Cid().Bytes(), api.MsgMeta{ + Type: api.MTChainMsg, + Extra: mb.RawData(), + }) if err != nil { return nil, xerrors.Errorf("failed to sign message: %w", err) } From 6e8efb9d2cea1e7d828790a8e42ebd030738654e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 9 Oct 2020 01:27:49 +0200 Subject: [PATCH 636/885] gofmt --- api/api_wallet.go | 1 + api/apistruct/struct.go | 16 ++++++++-------- chain/gen/gen.go | 6 +++--- chain/gen/mining.go | 2 +- chain/wallet/remotewallet/remote.go | 2 +- cmd/lotus-wallet/logged.go | 2 +- markets/storageadapter/client.go | 4 ++-- node/impl/full/state.go | 2 +- 8 files changed, 18 insertions(+), 17 deletions(-) diff --git a/api/api_wallet.go b/api/api_wallet.go index 82ef726a0..1213ffc1d 100644 --- a/api/api_wallet.go +++ b/api/api_wallet.go @@ -10,6 +10,7 @@ import ( ) type MsgType string + const ( MTUnknown = "unknown" diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 81fce92b5..d8c2c8094 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -363,14 +363,14 @@ type WorkerStruct struct { type WalletStruct struct { Internal struct { - WalletNew func(context.Context, crypto.SigType) (address.Address, error) `perm:"write"` - WalletHas func(context.Context, address.Address) (bool, error) `perm:"write"` - WalletList func(context.Context) ([]address.Address, error) `perm:"write"` - WalletSign func(context.Context, address.Address, []byte, api.MsgMeta) (*crypto.Signature, error) `perm:"sign"` - WalletSignMessage func(context.Context, address.Address, *types.Message) (*types.SignedMessage, error) `perm:"sign"` - WalletExport func(context.Context, address.Address) (*types.KeyInfo, error) `perm:"admin"` - WalletImport func(context.Context, *types.KeyInfo) (address.Address, error) `perm:"admin"` - WalletDelete func(context.Context, address.Address) error `perm:"write"` + WalletNew func(context.Context, crypto.SigType) (address.Address, error) `perm:"write"` + WalletHas func(context.Context, address.Address) (bool, error) `perm:"write"` + WalletList func(context.Context) ([]address.Address, error) `perm:"write"` + WalletSign func(context.Context, address.Address, []byte, api.MsgMeta) (*crypto.Signature, error) `perm:"sign"` + WalletSignMessage func(context.Context, address.Address, *types.Message) (*types.SignedMessage, error) `perm:"sign"` + WalletExport func(context.Context, address.Address) (*types.KeyInfo, error) `perm:"admin"` + WalletImport func(context.Context, *types.KeyInfo) (address.Address, error) `perm:"admin"` + WalletDelete func(context.Context, address.Address) error `perm:"write"` } } diff --git a/chain/gen/gen.go b/chain/gen/gen.go index 70c303818..d76835294 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -376,7 +376,7 @@ func (cg *ChainGen) nextBlockProof(ctx context.Context, pts *types.TipSet, m add sf := func(ctx context.Context, a address.Address, i []byte) (*crypto.Signature, error) { return cg.w.WalletSign(ctx, a, i, api.MsgMeta{ - Type: api.MTUnknown, + Type: api.MTUnknown, }) } @@ -535,7 +535,7 @@ func getRandomMessages(cg *ChainGen) ([]*types.SignedMessage, error) { } sig, err := cg.w.WalletSign(context.TODO(), cg.banker, msg.Cid().Bytes(), api.MsgMeta{ - Type: api.MTUnknown, // testing + Type: api.MTUnknown, // testing }) if err != nil { return nil, err @@ -597,7 +597,7 @@ func (mca mca) MinerGetBaseInfo(ctx context.Context, maddr address.Address, epoc func (mca mca) WalletSign(ctx context.Context, a address.Address, v []byte) (*crypto.Signature, error) { return mca.w.WalletSign(ctx, a, v, api.MsgMeta{ - Type: api.MTUnknown, + Type: api.MTUnknown, }) } diff --git a/chain/gen/mining.go b/chain/gen/mining.go index 37e8ca2c0..45a089452 100644 --- a/chain/gen/mining.go +++ b/chain/gen/mining.go @@ -131,7 +131,7 @@ func MinerCreateBlock(ctx context.Context, sm *stmgr.StateManager, w api.WalletA } sig, err := w.WalletSign(ctx, waddr, nosigbytes, api.MsgMeta{ - Type: api.MTBlock, + Type: api.MTBlock, }) if err != nil { return nil, xerrors.Errorf("failed to sign new block: %w", err) diff --git a/chain/wallet/remotewallet/remote.go b/chain/wallet/remotewallet/remote.go index 712f270b4..ca4e422d5 100644 --- a/chain/wallet/remotewallet/remote.go +++ b/chain/wallet/remotewallet/remote.go @@ -48,4 +48,4 @@ func SetupRemoteWallet(url string) func(mctx helpers.MetricsCtx, lc fx.Lifecycle return &RemoteWallet{wapi}, nil } -} \ No newline at end of file +} diff --git a/cmd/lotus-wallet/logged.go b/cmd/lotus-wallet/logged.go index 9cd729943..c713c2e87 100644 --- a/cmd/lotus-wallet/logged.go +++ b/cmd/lotus-wallet/logged.go @@ -59,4 +59,4 @@ func (c *LoggedWallet) WalletDelete(ctx context.Context, addr address.Address) e log.Infow("WalletDelete", "address", addr) return c.under.WalletDelete(ctx, addr) -} \ No newline at end of file +} diff --git a/markets/storageadapter/client.go b/markets/storageadapter/client.go index 22ee483ce..080fe36eb 100644 --- a/markets/storageadapter/client.go +++ b/markets/storageadapter/client.go @@ -424,7 +424,7 @@ func (c *ClientNodeAdapter) SignProposal(ctx context.Context, signer address.Add } sig, err := c.Wallet.WalletSign(ctx, signer, buf, api.MsgMeta{ - Type: api.MTDealProposal, + Type: api.MTDealProposal, }) if err != nil { return nil, err @@ -479,7 +479,7 @@ func (c *ClientNodeAdapter) SignBytes(ctx context.Context, signer address.Addres } localSignature, err := c.Wallet.WalletSign(ctx, signer, b, api.MsgMeta{ - Type: api.MTUnknown, // TODO: pass type here + Type: api.MTUnknown, // TODO: pass type here }) if err != nil { return nil, err diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 49754265d..c67b7a887 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -44,7 +44,7 @@ type StateAPI struct { // TODO: the wallet here is only needed because we have the MinerCreateBlock // API attached to the state API. It probably should live somewhere better - Wallet api.WalletAPI + Wallet api.WalletAPI DefWallet wallet.Default ProofVerifier ffiwrapper.Verifier From 7d2f06cfbdd008e8acc41f6a56e7c1ee43c6f12b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 9 Oct 2020 01:33:50 +0200 Subject: [PATCH 637/885] Fix WalletSign in tests --- chain/messagepool/selection_test.go | 20 +++++++++++--------- chain/messagesigner/messagesigner_test.go | 8 ++++---- chain/stmgr/forks_test.go | 19 ++++++++++--------- chain/sync_test.go | 4 ++-- chain/types/mock/chain.go | 3 ++- cmd/lotus-shed/keyinfo.go | 2 +- node/impl/full/wallet.go | 4 +++- 7 files changed, 33 insertions(+), 27 deletions(-) diff --git a/chain/messagepool/selection_test.go b/chain/messagepool/selection_test.go index 72fea4d2c..7dae583ad 100644 --- a/chain/messagepool/selection_test.go +++ b/chain/messagepool/selection_test.go @@ -13,6 +13,10 @@ import ( "sort" "testing" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" + logging "github.com/ipfs/go-log" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/lotus/build" @@ -21,12 +25,10 @@ import ( "github.com/filecoin-project/lotus/chain/types/mock" "github.com/filecoin-project/lotus/chain/wallet" "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/ipfs/go-cid" - "github.com/ipfs/go-datastore" + "github.com/filecoin-project/lotus/api" _ "github.com/filecoin-project/lotus/lib/sigs/bls" _ "github.com/filecoin-project/lotus/lib/sigs/secp" - logging "github.com/ipfs/go-log" ) func init() { @@ -45,7 +47,7 @@ func makeTestMessage(w *wallet.LocalWallet, from, to address.Address, nonce uint GasFeeCap: types.NewInt(100 + gasPrice), GasPremium: types.NewInt(gasPrice), } - sig, err := w.WalletSign(context.TODO(), from, msg.Cid().Bytes()) + sig, err := w.WalletSign(context.TODO(), from, msg.Cid().Bytes(), api.MsgMeta{}) if err != nil { panic(err) } @@ -744,7 +746,7 @@ func TestPriorityMessageSelection3(t *testing.T) { t.Fatal(err) } - a1, err := w1.GenerateKey(crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -754,7 +756,7 @@ func TestPriorityMessageSelection3(t *testing.T) { t.Fatal(err) } - a2, err := w2.GenerateKey(crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -1330,7 +1332,7 @@ readLoop: } actorMap := make(map[address.Address]address.Address) - actorWallets := make(map[address.Address]*wallet.Wallet) + actorWallets := make(map[address.Address]api.WalletAPI) for _, m := range msgs { baseNonce := baseNonces[m.Message.From] @@ -1342,7 +1344,7 @@ readLoop: t.Fatal(err) } - a, err := w.GenerateKey(crypto.SigTypeSecp256k1) + a, err := w.WalletNew(context.Background(), crypto.SigTypeSecp256k1) if err != nil { t.Fatal(err) } @@ -1360,7 +1362,7 @@ readLoop: m.Message.From = localActor m.Message.Nonce -= baseNonce - sig, err := w.Sign(context.TODO(), localActor, m.Message.Cid().Bytes()) + sig, err := w.WalletSign(context.TODO(), localActor, m.Message.Cid().Bytes(), api.MsgMeta{}) if err != nil { t.Fatal(err) } diff --git a/chain/messagesigner/messagesigner_test.go b/chain/messagesigner/messagesigner_test.go index 04869ff6d..f50bb2606 100644 --- a/chain/messagesigner/messagesigner_test.go +++ b/chain/messagesigner/messagesigner_test.go @@ -47,13 +47,13 @@ func TestMessageSignerSignMessage(t *testing.T) { ctx := context.Background() w, _ := wallet.NewWallet(wallet.NewMemKeyStore()) - from1, err := w.GenerateKey(crypto.SigTypeSecp256k1) + from1, err := w.WalletNew(ctx, crypto.SigTypeSecp256k1) require.NoError(t, err) - from2, err := w.GenerateKey(crypto.SigTypeSecp256k1) + from2, err := w.WalletNew(ctx, crypto.SigTypeSecp256k1) require.NoError(t, err) - to1, err := w.GenerateKey(crypto.SigTypeSecp256k1) + to1, err := w.WalletNew(ctx, crypto.SigTypeSecp256k1) require.NoError(t, err) - to2, err := w.GenerateKey(crypto.SigTypeSecp256k1) + to2, err := w.WalletNew(ctx, crypto.SigTypeSecp256k1) require.NoError(t, err) type msgSpec struct { diff --git a/chain/stmgr/forks_test.go b/chain/stmgr/forks_test.go index 3687e6b34..0388af6ad 100644 --- a/chain/stmgr/forks_test.go +++ b/chain/stmgr/forks_test.go @@ -6,15 +6,21 @@ import ( "io" "testing" + "github.com/ipfs/go-cid" + ipldcbor "github.com/ipfs/go-ipld-cbor" + logging "github.com/ipfs/go-log" + "github.com/stretchr/testify/require" + cbg "github.com/whyrusleeping/cbor-gen" + "golang.org/x/xerrors" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/cbor" "github.com/filecoin-project/specs-actors/actors/builtin" init0 "github.com/filecoin-project/specs-actors/actors/builtin/init" "github.com/filecoin-project/specs-actors/actors/runtime" - "github.com/stretchr/testify/require" - "golang.org/x/xerrors" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/aerrors" lotusinit "github.com/filecoin-project/lotus/chain/actors/builtin/init" @@ -25,11 +31,6 @@ import ( "github.com/filecoin-project/lotus/chain/vm" _ "github.com/filecoin-project/lotus/lib/sigs/bls" _ "github.com/filecoin-project/lotus/lib/sigs/secp" - - "github.com/ipfs/go-cid" - ipldcbor "github.com/ipfs/go-ipld-cbor" - logging "github.com/ipfs/go-log" - cbg "github.com/whyrusleeping/cbor-gen" ) func init() { @@ -186,7 +187,7 @@ func TestForkHeightTriggers(t *testing.T) { Params: enc, GasLimit: types.TestGasLimit, } - sig, err := cg.Wallet().WalletSign(ctx, cg.Banker(), m.Cid().Bytes()) + sig, err := cg.Wallet().WalletSign(ctx, cg.Banker(), m.Cid().Bytes(), api.MsgMeta{}) if err != nil { t.Fatal(err) } @@ -214,7 +215,7 @@ func TestForkHeightTriggers(t *testing.T) { } nonce++ - sig, err := cg.Wallet().WalletSign(ctx, cg.Banker(), m.Cid().Bytes()) + sig, err := cg.Wallet().WalletSign(ctx, cg.Banker(), m.Cid().Bytes(), api.MsgMeta{}) if err != nil { return nil, err } diff --git a/chain/sync_test.go b/chain/sync_test.go index 2c9f2f131..82d824e62 100644 --- a/chain/sync_test.go +++ b/chain/sync_test.go @@ -593,7 +593,7 @@ func TestDuplicateNonce(t *testing.T) { GasPremium: types.NewInt(0), } - sig, err := tu.g.Wallet().WalletSign(context.TODO(), tu.g.Banker(), msg.Cid().Bytes()) + sig, err := tu.g.Wallet().WalletSign(context.TODO(), tu.g.Banker(), msg.Cid().Bytes(), api.MsgMeta{}) require.NoError(t, err) return &types.SignedMessage{ @@ -685,7 +685,7 @@ func TestBadNonce(t *testing.T) { GasPremium: types.NewInt(0), } - sig, err := tu.g.Wallet().Sign(context.TODO(), tu.g.Banker(), msg.Cid().Bytes()) + sig, err := tu.g.Wallet().WalletSign(context.TODO(), tu.g.Banker(), msg.Cid().Bytes(), api.MsgMeta{}) require.NoError(t, err) return &types.SignedMessage{ diff --git a/chain/types/mock/chain.go b/chain/types/mock/chain.go index 7a9c82cba..2e1c0a976 100644 --- a/chain/types/mock/chain.go +++ b/chain/types/mock/chain.go @@ -9,6 +9,7 @@ import ( "github.com/filecoin-project/go-state-types/crypto" "github.com/ipfs/go-cid" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/wallet" @@ -33,7 +34,7 @@ func MkMessage(from, to address.Address, nonce uint64, w *wallet.LocalWallet) *t GasPremium: types.NewInt(1), } - sig, err := w.WalletSign(context.TODO(), from, msg.Cid().Bytes()) + sig, err := w.WalletSign(context.TODO(), from, msg.Cid().Bytes(), api.MsgMeta{}) if err != nil { panic(err) } diff --git a/cmd/lotus-shed/keyinfo.go b/cmd/lotus-shed/keyinfo.go index 09be67d1e..fdd1fcb49 100644 --- a/cmd/lotus-shed/keyinfo.go +++ b/cmd/lotus-shed/keyinfo.go @@ -105,7 +105,7 @@ var keyinfoVerifyCmd = &cli.Command{ return err } - if _, err := w.Import(&keyInfo); err != nil { + if _, err := w.WalletImport(cctx.Context, &keyInfo); err != nil { return err } diff --git a/node/impl/full/wallet.go b/node/impl/full/wallet.go index 2e2c48c5d..3bbb6556d 100644 --- a/node/impl/full/wallet.go +++ b/node/impl/full/wallet.go @@ -40,7 +40,9 @@ func (a *WalletAPI) WalletSign(ctx context.Context, k address.Address, msg []byt if err != nil { return nil, xerrors.Errorf("failed to resolve ID address: %w", keyAddr) } - return a.WalletSign(ctx, keyAddr, msg) + return a.WalletAPI.WalletSign(ctx, keyAddr, msg, api.MsgMeta{ + Type: api.MTUnknown, + }) } func (a *WalletAPI) WalletSignMessage(ctx context.Context, k address.Address, msg *types.Message) (*types.SignedMessage, error) { From 82b15d2637655508bc92141db0b739a012ff362c Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 8 Oct 2020 15:59:06 -0700 Subject: [PATCH 638/885] cleanup tests on finish --- chain/sync_test.go | 8 ++++---- node/test/builder.go | 17 +++++++++-------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/chain/sync_test.go b/chain/sync_test.go index 1b06f604b..318f6d82f 100644 --- a/chain/sync_test.go +++ b/chain/sync_test.go @@ -221,8 +221,7 @@ func (tu *syncTestUtil) addSourceNode(gen int) { sourceRepo, genesis, blocks := tu.repoWithChain(tu.t, gen) var out api.FullNode - // TODO: Don't ignore stop - _, err := node.New(tu.ctx, + stop, err := node.New(tu.ctx, node.FullAPI(&out), node.Online(), node.Repo(sourceRepo), @@ -232,6 +231,7 @@ func (tu *syncTestUtil) addSourceNode(gen int) { node.Override(new(modules.Genesis), modules.LoadGenesis(genesis)), ) require.NoError(tu.t, err) + tu.t.Cleanup(func() { _ = stop(context.Background()) }) lastTs := blocks[len(blocks)-1].Blocks for _, lastB := range lastTs { @@ -253,8 +253,7 @@ func (tu *syncTestUtil) addClientNode() int { var out api.FullNode - // TODO: Don't ignore stop - _, err := node.New(tu.ctx, + stop, err := node.New(tu.ctx, node.FullAPI(&out), node.Online(), node.Repo(repo.NewMemory(nil)), @@ -264,6 +263,7 @@ func (tu *syncTestUtil) addClientNode() int { node.Override(new(modules.Genesis), modules.LoadGenesis(tu.genesis)), ) require.NoError(tu.t, err) + tu.t.Cleanup(func() { _ = stop(context.Background()) }) tu.nds = append(tu.nds, out) return len(tu.nds) - 1 diff --git a/node/test/builder.go b/node/test/builder.go index a3455f376..d9ec04460 100644 --- a/node/test/builder.go +++ b/node/test/builder.go @@ -101,8 +101,7 @@ func CreateTestStorageNode(ctx context.Context, t *testing.T, waddr address.Addr var minerapi api.StorageMiner mineBlock := make(chan miner2.MineReq) - // TODO: use stop - _, err = node.New(ctx, + stop, err := node.New(ctx, node.StorageMiner(&minerapi), node.Online(), node.Repo(r), @@ -118,6 +117,7 @@ func CreateTestStorageNode(ctx context.Context, t *testing.T, waddr address.Addr if err != nil { t.Fatalf("failed to construct node: %v", err) } + t.Cleanup(func() { _ = stop(context.Background()) }) /*// Bootstrap with full node remoteAddrs, err := tnd.NetAddrsListen(ctx) @@ -138,7 +138,9 @@ func CreateTestStorageNode(ctx context.Context, t *testing.T, waddr address.Addr } func Builder(t *testing.T, nFull int, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { - ctx := context.Background() + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + mn := mocknet.New(ctx) fulls := make([]test.TestNode, nFull) @@ -214,9 +216,7 @@ func Builder(t *testing.T, nFull int, storage []test.StorageMiner, opts ...node. genesis = node.Override(new(modules.Genesis), modules.LoadGenesis(genbuf.Bytes())) } - var err error - // TODO: Don't ignore stop - _, err = node.New(ctx, + stop, err := node.New(ctx, node.FullAPI(&fulls[i].FullNode), node.Online(), node.Repo(repo.NewMemory(nil)), @@ -229,7 +229,7 @@ func Builder(t *testing.T, nFull int, storage []test.StorageMiner, opts ...node. if err != nil { t.Fatal(err) } - + t.Cleanup(func() { _ = stop(context.Background()) }) } for i, def := range storage { @@ -364,7 +364,7 @@ func MockSbBuilder(t *testing.T, nFull int, storage []test.StorageMiner, options var err error // TODO: Don't ignore stop - _, err = node.New(ctx, + stop, err := node.New(ctx, node.FullAPI(&fulls[i].FullNode), node.Online(), node.Repo(repo.NewMemory(nil)), @@ -379,6 +379,7 @@ func MockSbBuilder(t *testing.T, nFull int, storage []test.StorageMiner, options if err != nil { t.Fatalf("%+v", err) } + t.Cleanup(func() { _ = stop(context.Background()) }) } for i, def := range storage { From 791aa5c6c886fe12def84fc4c2910feb9dd434af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 9 Oct 2020 01:37:17 +0200 Subject: [PATCH 639/885] Add lotus-wallet to Makefile --- Makefile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Makefile b/Makefile index 56ab361ec..95a6af8a9 100644 --- a/Makefile +++ b/Makefile @@ -186,6 +186,12 @@ lotus-health: .PHONY: lotus-health BINS+=lotus-health +lotus-wallet: + rm -f lotus-wallet + go build -o lotus-stats ./cmd/lotus-wallet +.PHONY: lotus-wallet +BINS+=lotus-wallet + testground: go build -tags testground -o /dev/null ./cmd/lotus .PHONY: testground From 597d065e0510c240b545d30ee8ffe7fc230176ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 9 Oct 2020 01:39:06 +0200 Subject: [PATCH 640/885] Update go-jsonrpc with http(s) client support --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e9687c7b8..e1bf6cb3b 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,7 @@ require ( github.com/filecoin-project/go-data-transfer v0.6.7 github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f github.com/filecoin-project/go-fil-markets v0.7.1 - github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200822201400-474f4fdccc52 + github.com/filecoin-project/go-jsonrpc v0.1.2-0.20201008195726-68c6a2704e49 github.com/filecoin-project/go-multistore v0.0.3 github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 github.com/filecoin-project/go-paramfetch v0.0.2-0.20200701152213-3e0f0afdc261 diff --git a/go.sum b/go.sum index bab4e929d..ef954e66d 100644 --- a/go.sum +++ b/go.sum @@ -252,8 +252,8 @@ github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CW github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24= github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 h1:b3UDemBYN2HNfk3KOXNuxgTTxlWi3xVvbQP0IT38fvM= github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0/go.mod h1:7aWZdaQ1b16BVoQUYR+eEvrDCGJoPLxFpDynFjYfBjI= -github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200822201400-474f4fdccc52 h1:FXtCp0ybqdQL9knb3OGDpkNTaBbPxgkqPeWKotUwkH0= -github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200822201400-474f4fdccc52/go.mod h1:XBBpuKIMaXIIzeqzO1iucq4GvbF8CxmXRFoezRh+Cx4= +github.com/filecoin-project/go-jsonrpc v0.1.2-0.20201008195726-68c6a2704e49 h1:FSY245KeXFCUgyfFEu+bhrZNk8BGGJyfpSmQl2aiPU8= +github.com/filecoin-project/go-jsonrpc v0.1.2-0.20201008195726-68c6a2704e49/go.mod h1:XBBpuKIMaXIIzeqzO1iucq4GvbF8CxmXRFoezRh+Cx4= github.com/filecoin-project/go-multistore v0.0.3 h1:vaRBY4YiA2UZFPK57RNuewypB8u0DzzQwqsL0XarpnI= github.com/filecoin-project/go-multistore v0.0.3/go.mod h1:kaNqCC4IhU4B1uyr7YWFHd23TL4KM32aChS0jNkyUvQ= github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 h1:+/4aUeUoKr6AKfPE3mBhXA5spIV6UcKdTYDPNU2Tdmg= From 6a10af626c77d083119a00c20dc482376916cccb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 9 Oct 2020 01:50:33 +0200 Subject: [PATCH 641/885] Nicer message logging in lotus-wallet --- cmd/lotus-wallet/logged.go | 33 ++++++++++++++++++++++++++++++++- node/impl/full/wallet.go | 2 +- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/cmd/lotus-wallet/logged.go b/cmd/lotus-wallet/logged.go index c713c2e87..d3630c241 100644 --- a/cmd/lotus-wallet/logged.go +++ b/cmd/lotus-wallet/logged.go @@ -1,7 +1,11 @@ package main import ( + "bytes" "context" + "encoding/hex" + "github.com/ipfs/go-cid" + "golang.org/x/xerrors" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/crypto" @@ -38,7 +42,34 @@ func (c *LoggedWallet) WalletList(ctx context.Context) ([]address.Address, error } func (c *LoggedWallet) WalletSign(ctx context.Context, k address.Address, msg []byte, meta api.MsgMeta) (*crypto.Signature, error) { - log.Infow("WalletSign", "address", k) + switch meta.Type { + case api.MTChainMsg: + var cmsg types.Message + if err := cmsg.UnmarshalCBOR(bytes.NewReader(meta.Extra)); err != nil { + return nil, xerrors.Errorf("unmarshalling message: %w", err) + } + + _, bc, err := cid.CidFromBytes(msg) + if err != nil { + return nil, xerrors.Errorf("getting cid from signing bytes: %w", err) + } + + if !cmsg.Cid().Equals(bc) { + return nil, xerrors.Errorf("cid(meta.Extra).bytes() != msg") + } + + log.Infow("WalletSign", + "address", k, + "type", meta.Type, + "from", cmsg.From, + "to", cmsg.To, + "value", types.FIL(cmsg.Value), + "feecap", types.FIL(cmsg.RequiredFunds()), + "method", cmsg.Method, + "params", hex.EncodeToString(cmsg.Params)) + default: + log.Infow("WalletSign", "address", k, "type", meta.Type) + } return c.under.WalletSign(ctx, k, msg, meta) } diff --git a/node/impl/full/wallet.go b/node/impl/full/wallet.go index 3bbb6556d..1a32cd422 100644 --- a/node/impl/full/wallet.go +++ b/node/impl/full/wallet.go @@ -41,7 +41,7 @@ func (a *WalletAPI) WalletSign(ctx context.Context, k address.Address, msg []byt return nil, xerrors.Errorf("failed to resolve ID address: %w", keyAddr) } return a.WalletAPI.WalletSign(ctx, keyAddr, msg, api.MsgMeta{ - Type: api.MTUnknown, + Type: api.MTUnknown, }) } From 6c54d2445df3a307cfb90de5ff6e1fc22a8d08c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 9 Oct 2020 01:53:12 +0200 Subject: [PATCH 642/885] Appease the linter --- chain/messagesigner/messagesigner.go | 12 +++++++----- chain/wallet/remotewallet/remote.go | 3 ++- markets/storageadapter/client.go | 13 +++++++------ 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/chain/messagesigner/messagesigner.go b/chain/messagesigner/messagesigner.go index c7948e42e..e32a78405 100644 --- a/chain/messagesigner/messagesigner.go +++ b/chain/messagesigner/messagesigner.go @@ -3,18 +3,20 @@ package messagesigner import ( "bytes" "context" - "github.com/filecoin-project/lotus/api" "sync" - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/messagepool" - "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" logging "github.com/ipfs/go-log/v2" cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/messagepool" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/node/modules/dtypes" ) const dsKeyActorNonce = "ActorNextNonce" diff --git a/chain/wallet/remotewallet/remote.go b/chain/wallet/remotewallet/remote.go index ca4e422d5..8454ae6c9 100644 --- a/chain/wallet/remotewallet/remote.go +++ b/chain/wallet/remotewallet/remote.go @@ -2,9 +2,10 @@ package remotewallet import ( "context" + "net/http" + "go.uber.org/fx" "golang.org/x/xerrors" - "net/http" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/client" diff --git a/markets/storageadapter/client.go b/markets/storageadapter/client.go index 080fe36eb..e9a260afe 100644 --- a/markets/storageadapter/client.go +++ b/markets/storageadapter/client.go @@ -5,21 +5,23 @@ package storageadapter import ( "bytes" "context" - "github.com/filecoin-project/lotus/api" - - "github.com/filecoin-project/go-state-types/big" - miner0 "github.com/filecoin-project/specs-actors/actors/builtin" - market0 "github.com/filecoin-project/specs-actors/actors/builtin/market" "golang.org/x/xerrors" + "github.com/ipfs/go-cid" + "github.com/filecoin-project/go-address" cborutil "github.com/filecoin-project/go-cbor-util" "github.com/filecoin-project/go-fil-markets/shared" "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/go-state-types/exitcode" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin" + market0 "github.com/filecoin-project/specs-actors/actors/builtin/market" + + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/events" @@ -31,7 +33,6 @@ import ( "github.com/filecoin-project/lotus/lib/sigs" "github.com/filecoin-project/lotus/markets/utils" "github.com/filecoin-project/lotus/node/impl/full" - "github.com/ipfs/go-cid" ) type ClientNodeAdapter struct { From 3cd53ad9d9a17024a1a81846daf358a89313cc85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 9 Oct 2020 02:21:37 +0200 Subject: [PATCH 643/885] Use APIInfo parser for RemoteBacked wallet config --- Makefile | 2 +- chain/wallet/remotewallet/remote.go | 26 ++++++++------------------ cli/cmd.go | 29 ++++++++++++++++++++--------- 3 files changed, 29 insertions(+), 28 deletions(-) diff --git a/Makefile b/Makefile index 95a6af8a9..79f7fa81e 100644 --- a/Makefile +++ b/Makefile @@ -188,7 +188,7 @@ BINS+=lotus-health lotus-wallet: rm -f lotus-wallet - go build -o lotus-stats ./cmd/lotus-wallet + go build -o lotus-wallet ./cmd/lotus-wallet .PHONY: lotus-wallet BINS+=lotus-wallet diff --git a/chain/wallet/remotewallet/remote.go b/chain/wallet/remotewallet/remote.go index 8454ae6c9..61deec7ee 100644 --- a/chain/wallet/remotewallet/remote.go +++ b/chain/wallet/remotewallet/remote.go @@ -2,13 +2,13 @@ package remotewallet import ( "context" - "net/http" "go.uber.org/fx" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/client" + lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/node/modules/helpers" ) @@ -16,26 +16,16 @@ type RemoteWallet struct { api.WalletAPI } -func SetupRemoteWallet(url string) func(mctx helpers.MetricsCtx, lc fx.Lifecycle) (*RemoteWallet, error) { +func SetupRemoteWallet(info string) func(mctx helpers.MetricsCtx, lc fx.Lifecycle) (*RemoteWallet, error) { return func(mctx helpers.MetricsCtx, lc fx.Lifecycle) (*RemoteWallet, error) { - /*sp := strings.SplitN(env, ":", 2) - if len(sp) != 2 { - log.Warnf("invalid env(%s) value, missing token or address", envKey) - } else { - ma, err := multiaddr.NewMultiaddr(sp[1]) - if err != nil { - return APIInfo{}, xerrors.Errorf("could not parse multiaddr from env(%s): %w", envKey, err) - } - return APIInfo{ - Addr: ma, - Token: []byte(sp[0]), - }, nil - }*/ + ai := lcli.ParseApiInfo(info) - headers := http.Header{} - /*headers.Add("Authorization", "Bearer "+token)*/ + url, err := ai.DialArgs() + if err != nil { + return nil, err + } - wapi, closer, err := client.NewWalletRPC(mctx, url, headers) + wapi, closer, err := client.NewWalletRPC(mctx, url, ai.AuthHeader()) if err != nil { return nil, xerrors.Errorf("creating jsonrpc client: %w", err) } diff --git a/cli/cmd.go b/cli/cmd.go index edcb69adc..a3736f8d7 100644 --- a/cli/cmd.go +++ b/cli/cmd.go @@ -7,6 +7,7 @@ import ( "net/url" "os" "os/signal" + "regexp" "strings" "syscall" @@ -153,6 +154,24 @@ func envForRepoDeprecation(t repo.RepoType) string { } } +var ( + infoWithToken = regexp.MustCompile("^[a-zA-Z0-9\\-_]+?\\.[a-zA-Z0-9\\-_]+?\\.([a-zA-Z0-9\\-_]+)?:.+$") +) + +func ParseApiInfo(s string) APIInfo { + var tok []byte + if infoWithToken.Match([]byte(s)) { + sp := strings.SplitN(s, ":", 2) + tok = []byte(sp[0]) + s = sp[1] + } + + return APIInfo{ + Addr: s, + Token: tok, + } +} + func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (APIInfo, error) { // Check if there was a flag passed with the listen address of the API // server (only used by the tests) @@ -175,15 +194,7 @@ func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (APIInfo, error) { } } if ok { - sp := strings.SplitN(env, ":", 2) - if len(sp) != 2 { - log.Warnf("invalid env(%s) value, missing token or address", envKey) - } else { - return APIInfo{ - Addr: sp[1], - Token: []byte(sp[0]), - }, nil - } + return ParseApiInfo(env), nil } repoFlag := flagForRepo(t) From 6cc7559b046fc3d5410eb27e460e0aa38321751a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 9 Oct 2020 02:57:41 +0200 Subject: [PATCH 644/885] Fix APIInfo import cycle in tests --- chain/wallet/remotewallet/remote.go | 4 +- cli/cmd.go | 88 +++-------------------------- cli/util/apiinfo.go | 83 +++++++++++++++++++++++++++ cmd/lotus-shed/consensus.go | 3 +- 4 files changed, 95 insertions(+), 83 deletions(-) create mode 100644 cli/util/apiinfo.go diff --git a/chain/wallet/remotewallet/remote.go b/chain/wallet/remotewallet/remote.go index 61deec7ee..c7192f496 100644 --- a/chain/wallet/remotewallet/remote.go +++ b/chain/wallet/remotewallet/remote.go @@ -8,7 +8,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/client" - lcli "github.com/filecoin-project/lotus/cli" + cliutil "github.com/filecoin-project/lotus/cli/util" "github.com/filecoin-project/lotus/node/modules/helpers" ) @@ -18,7 +18,7 @@ type RemoteWallet struct { func SetupRemoteWallet(info string) func(mctx helpers.MetricsCtx, lc fx.Lifecycle) (*RemoteWallet, error) { return func(mctx helpers.MetricsCtx, lc fx.Lifecycle) (*RemoteWallet, error) { - ai := lcli.ParseApiInfo(info) + ai := cliutil.ParseApiInfo(info) url, err := ai.DialArgs() if err != nil { diff --git a/cli/cmd.go b/cli/cmd.go index a3736f8d7..fb0706997 100644 --- a/cli/cmd.go +++ b/cli/cmd.go @@ -4,17 +4,13 @@ import ( "context" "fmt" "net/http" - "net/url" "os" "os/signal" - "regexp" "strings" "syscall" logging "github.com/ipfs/go-log/v2" "github.com/mitchellh/go-homedir" - "github.com/multiformats/go-multiaddr" - manet "github.com/multiformats/go-multiaddr/net" "github.com/urfave/cli/v2" "golang.org/x/xerrors" @@ -22,6 +18,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/client" + cliutil "github.com/filecoin-project/lotus/cli/util" "github.com/filecoin-project/lotus/node/repo" ) @@ -48,57 +45,6 @@ func NewCliError(s string) error { // ApiConnector returns API instance type ApiConnector func() api.FullNode -type APIInfo struct { - Addr string - Token []byte -} - -func (a APIInfo) DialArgs() (string, error) { - ma, err := multiaddr.NewMultiaddr(a.Addr) - if err == nil { - _, addr, err := manet.DialArgs(ma) - if err != nil { - return "", err - } - - return "ws://" + addr + "/rpc/v0", nil - } - - _, err = url.Parse(a.Addr) - if err != nil { - return "", err - } - return a.Addr + "/rpc/v0", nil -} - -func (a APIInfo) Host() (string, error) { - ma, err := multiaddr.NewMultiaddr(a.Addr) - if err == nil { - _, addr, err := manet.DialArgs(ma) - if err != nil { - return "", err - } - - return addr, nil - } - - spec, err := url.Parse(a.Addr) - if err != nil { - return "", err - } - return spec.Host, nil -} - -func (a APIInfo) AuthHeader() http.Header { - if len(a.Token) != 0 { - headers := http.Header{} - headers.Add("Authorization", "Bearer "+string(a.Token)) - return headers - } - log.Warn("API Token not set and requested, capabilities might be limited.") - return nil -} - // The flag passed on the command line with the listen address of the API // server (only used by the tests) func flagForAPI(t repo.RepoType) string { @@ -154,25 +100,7 @@ func envForRepoDeprecation(t repo.RepoType) string { } } -var ( - infoWithToken = regexp.MustCompile("^[a-zA-Z0-9\\-_]+?\\.[a-zA-Z0-9\\-_]+?\\.([a-zA-Z0-9\\-_]+)?:.+$") -) - -func ParseApiInfo(s string) APIInfo { - var tok []byte - if infoWithToken.Match([]byte(s)) { - sp := strings.SplitN(s, ":", 2) - tok = []byte(sp[0]) - s = sp[1] - } - - return APIInfo{ - Addr: s, - Token: tok, - } -} - -func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (APIInfo, error) { +func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (cliutil.APIInfo, error) { // Check if there was a flag passed with the listen address of the API // server (only used by the tests) apiFlag := flagForAPI(t) @@ -180,7 +108,7 @@ func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (APIInfo, error) { strma := ctx.String(apiFlag) strma = strings.TrimSpace(strma) - return APIInfo{Addr: strma}, nil + return cliutil.APIInfo{Addr: strma}, nil } envKey := envForRepo(t) @@ -194,24 +122,24 @@ func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (APIInfo, error) { } } if ok { - return ParseApiInfo(env), nil + return cliutil.ParseApiInfo(env), nil } repoFlag := flagForRepo(t) p, err := homedir.Expand(ctx.String(repoFlag)) if err != nil { - return APIInfo{}, xerrors.Errorf("could not expand home dir (%s): %w", repoFlag, err) + return cliutil.APIInfo{}, xerrors.Errorf("could not expand home dir (%s): %w", repoFlag, err) } r, err := repo.NewFS(p) if err != nil { - return APIInfo{}, xerrors.Errorf("could not open repo at path: %s; %w", p, err) + return cliutil.APIInfo{}, xerrors.Errorf("could not open repo at path: %s; %w", p, err) } ma, err := r.APIEndpoint() if err != nil { - return APIInfo{}, xerrors.Errorf("could not get api endpoint: %w", err) + return cliutil.APIInfo{}, xerrors.Errorf("could not get api endpoint: %w", err) } token, err := r.APIToken() @@ -219,7 +147,7 @@ func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (APIInfo, error) { log.Warnf("Couldn't load CLI token, capabilities may be limited: %v", err) } - return APIInfo{ + return cliutil.APIInfo{ Addr: ma.String(), Token: token, }, nil diff --git a/cli/util/apiinfo.go b/cli/util/apiinfo.go new file mode 100644 index 000000000..1f9a83769 --- /dev/null +++ b/cli/util/apiinfo.go @@ -0,0 +1,83 @@ +package cliutil + +import ( + "net/http" + "net/url" + "regexp" + "strings" + + logging "github.com/ipfs/go-log/v2" + "github.com/multiformats/go-multiaddr" + manet "github.com/multiformats/go-multiaddr/net" +) + +var log = logging.Logger("cliutil") + +var ( + infoWithToken = regexp.MustCompile("^[a-zA-Z0-9\\-_]+?\\.[a-zA-Z0-9\\-_]+?\\.([a-zA-Z0-9\\-_]+)?:.+$") +) + +type APIInfo struct { + Addr string + Token []byte +} + +func ParseApiInfo(s string) APIInfo { + var tok []byte + if infoWithToken.Match([]byte(s)) { + sp := strings.SplitN(s, ":", 2) + tok = []byte(sp[0]) + s = sp[1] + } + + return APIInfo{ + Addr: s, + Token: tok, + } +} + +func (a APIInfo) DialArgs() (string, error) { + ma, err := multiaddr.NewMultiaddr(a.Addr) + if err == nil { + _, addr, err := manet.DialArgs(ma) + if err != nil { + return "", err + } + + return "ws://" + addr + "/rpc/v0", nil + } + + _, err = url.Parse(a.Addr) + if err != nil { + return "", err + } + return a.Addr + "/rpc/v0", nil +} + +func (a APIInfo) Host() (string, error) { + ma, err := multiaddr.NewMultiaddr(a.Addr) + if err == nil { + _, addr, err := manet.DialArgs(ma) + if err != nil { + return "", err + } + + return addr, nil + } + + spec, err := url.Parse(a.Addr) + if err != nil { + return "", err + } + return spec.Host, nil +} + +func (a APIInfo) AuthHeader() http.Header { + if len(a.Token) != 0 { + headers := http.Header{} + headers.Add("Authorization", "Bearer "+string(a.Token)) + return headers + } + log.Warn("API Token not set and requested, capabilities might be limited.") + return nil +} diff --git a/cmd/lotus-shed/consensus.go b/cmd/lotus-shed/consensus.go index 38a8cd8ef..1fe7756c1 100644 --- a/cmd/lotus-shed/consensus.go +++ b/cmd/lotus-shed/consensus.go @@ -15,6 +15,7 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" + cliutil "github.com/filecoin-project/lotus/cli/util" "github.com/libp2p/go-libp2p-core/peer" "github.com/multiformats/go-multiaddr" "github.com/urfave/cli/v2" @@ -111,7 +112,7 @@ var consensusCheckCmd = &cli.Command{ if err != nil { return err } - ainfo := lcli.APIInfo{Addr: apima.String()} + ainfo := cliutil.APIInfo{Addr: apima.String()} addr, err := ainfo.DialArgs() if err != nil { return err From 356137439f2cd0bcca2d11d8ab616d97b6eae064 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 9 Oct 2020 03:00:30 +0200 Subject: [PATCH 645/885] apistruct: Drop unused WalletSignMessage --- api/apistruct/struct.go | 19 +++++++------------ cmd/lotus-wallet/logged.go | 1 + 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index d8c2c8094..0265701d6 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -363,14 +363,13 @@ type WorkerStruct struct { type WalletStruct struct { Internal struct { - WalletNew func(context.Context, crypto.SigType) (address.Address, error) `perm:"write"` - WalletHas func(context.Context, address.Address) (bool, error) `perm:"write"` - WalletList func(context.Context) ([]address.Address, error) `perm:"write"` - WalletSign func(context.Context, address.Address, []byte, api.MsgMeta) (*crypto.Signature, error) `perm:"sign"` - WalletSignMessage func(context.Context, address.Address, *types.Message) (*types.SignedMessage, error) `perm:"sign"` - WalletExport func(context.Context, address.Address) (*types.KeyInfo, error) `perm:"admin"` - WalletImport func(context.Context, *types.KeyInfo) (address.Address, error) `perm:"admin"` - WalletDelete func(context.Context, address.Address) error `perm:"write"` + WalletNew func(context.Context, crypto.SigType) (address.Address, error) `perm:"write"` + WalletHas func(context.Context, address.Address) (bool, error) `perm:"write"` + WalletList func(context.Context) ([]address.Address, error) `perm:"write"` + WalletSign func(context.Context, address.Address, []byte, api.MsgMeta) (*crypto.Signature, error) `perm:"sign"` + WalletExport func(context.Context, address.Address) (*types.KeyInfo, error) `perm:"admin"` + WalletImport func(context.Context, *types.KeyInfo) (address.Address, error) `perm:"admin"` + WalletDelete func(context.Context, address.Address) error `perm:"write"` } } @@ -1401,10 +1400,6 @@ func (c *WalletStruct) WalletSign(ctx context.Context, k address.Address, msg [] return c.Internal.WalletSign(ctx, k, msg, meta) } -func (c *WalletStruct) WalletSignMessage(ctx context.Context, k address.Address, msg *types.Message) (*types.SignedMessage, error) { - return c.Internal.WalletSignMessage(ctx, k, msg) -} - func (c *WalletStruct) WalletExport(ctx context.Context, a address.Address) (*types.KeyInfo, error) { return c.Internal.WalletExport(ctx, a) } diff --git a/cmd/lotus-wallet/logged.go b/cmd/lotus-wallet/logged.go index d3630c241..3bcb3f867 100644 --- a/cmd/lotus-wallet/logged.go +++ b/cmd/lotus-wallet/logged.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/hex" + "github.com/ipfs/go-cid" "golang.org/x/xerrors" From d37389162e6ca2f03f1b198f05c88745784a57cf Mon Sep 17 00:00:00 2001 From: Animesh Date: Fri, 9 Oct 2020 09:55:38 +0530 Subject: [PATCH 646/885] Add docs --- documentation/en/api-methods.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index 630eae32e..f9678903b 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -37,6 +37,7 @@ * [ClientFindData](#ClientFindData) * [ClientGenCar](#ClientGenCar) * [ClientGetDealInfo](#ClientGetDealInfo) + * [ClientGetDealStatus](#ClientGetDealStatus) * [ClientGetDealUpdates](#ClientGetDealUpdates) * [ClientHasLocal](#ClientHasLocal) * [ClientImport](#ClientImport) @@ -963,6 +964,21 @@ Response: } ``` +### ClientGetDealStatus +ClientGetDealStatus returns status given a code + + +Perms: + +Inputs: +```json +[ + 42 +] +``` + +Response: `"string value"` + ### ClientGetDealUpdates ClientGetDealUpdates returns the status of updated deals From 4832cad1f1393cc57eb4f0407e3df6d3c5f29ca4 Mon Sep 17 00:00:00 2001 From: Animesh Date: Fri, 9 Oct 2020 10:18:32 +0530 Subject: [PATCH 647/885] sanity fix --- documentation/en/api-methods.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index f9678903b..ab298fe75 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -190,7 +190,7 @@ * [WalletSignMessage](#WalletSignMessage) * [WalletValidateAddress](#WalletValidateAddress) * [WalletVerify](#WalletVerify) -## +## ### Closing @@ -968,7 +968,7 @@ Response: ClientGetDealStatus returns status given a code -Perms: +Perms: Inputs: ```json From 9c357629662126c43d3b9cab9e40cb522477fc63 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Fri, 9 Oct 2020 03:10:40 -0400 Subject: [PATCH 648/885] Fix deletion of addresses from wallet --- chain/wallet/wallet.go | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/chain/wallet/wallet.go b/chain/wallet/wallet.go index 9438d2051..f821951b5 100644 --- a/chain/wallet/wallet.go +++ b/chain/wallet/wallet.go @@ -110,14 +110,12 @@ func (w *Wallet) tryFind(addr address.Address) (types.KeyInfo, error) { // We got an ErrKeyInfoNotFound error // Try again, this time with the testnet prefix - aChars := []rune(addr.String()) - prefixRunes := []rune(address.TestnetPrefix) - if len(prefixRunes) != 1 { - return types.KeyInfo{}, xerrors.Errorf("unexpected prefix length: %d", len(prefixRunes)) + tAddress, err := swapMainnetForTestnetPrefix(addr.String()) + if err != nil { + return types.KeyInfo{}, err } - aChars[0] = prefixRunes[0] - ki, err = w.keystore.Get(KNamePrefix + string(aChars)) + ki, err = w.keystore.Get(KNamePrefix + tAddress) if err != nil { return types.KeyInfo{}, err } @@ -291,6 +289,14 @@ func (w *Wallet) DeleteKey(addr address.Address) error { return xerrors.Errorf("failed to delete key %s: %w", addr, err) } + tAddr, err := swapMainnetForTestnetPrefix(addr.String()) + if err != nil { + return xerrors.Errorf("failed to swap prefixes: %w", err) + } + + // TODO: Does this always error in the not-found case? Just ignoring an error return for now. + _ = w.keystore.Delete(KNamePrefix + tAddr) + return nil } @@ -351,3 +357,14 @@ func ActSigType(typ string) crypto.SigType { return 0 } } + +func swapMainnetForTestnetPrefix(addr string) (string, error) { + aChars := []rune(addr) + prefixRunes := []rune(address.TestnetPrefix) + if len(prefixRunes) != 1 { + return "", xerrors.Errorf("unexpected prefix length: %d", len(prefixRunes)) + } + + aChars[0] = prefixRunes[0] + return string(aChars), nil +} From e19cd9ed01fbd5c633ac6542e8be6d1ffe2c77c4 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Tue, 29 Sep 2020 17:25:45 +0200 Subject: [PATCH 649/885] feat: lotus-lite - replace wallet StateManager with thin client to gateway --- api/api_gateway.go | 17 +++++++++++++++++ api/apistruct/struct.go | 32 ++++++++++++++++++++++++++++++++ api/client/client.go | 14 ++++++++++++++ chain/stmgr/stmgr.go | 7 +++++++ cli/cmd.go | 9 +++++++++ cmd/lotus-gateway/api.go | 33 ++++++++++++++++++++++----------- cmd/lotus/daemon.go | 28 ++++++++++++++++++++++++++++ node/builder.go | 1 + node/impl/full/wallet.go | 8 ++++---- node/modules/statemanager.go | 29 +++++++++++++++++++++++++++++ 10 files changed, 163 insertions(+), 15 deletions(-) create mode 100644 api/api_gateway.go create mode 100644 node/modules/statemanager.go diff --git a/api/api_gateway.go b/api/api_gateway.go new file mode 100644 index 000000000..6a220e9ea --- /dev/null +++ b/api/api_gateway.go @@ -0,0 +1,17 @@ +package api + +import ( + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" +) + +type GatewayAPI interface { + StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) + ChainHead(ctx context.Context) (*types.TipSet, error) + ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) + MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) + StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) +} diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 1a3ddda58..c71b6b560 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -361,6 +361,17 @@ type WorkerStruct struct { } } +type GatewayStruct struct { + Internal struct { + // TODO: does the gateway need perms? + StateGetActor func(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) + ChainHead func(ctx context.Context) (*types.TipSet, error) + ChainGetTipSet func(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) + MpoolPush func(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) + StateAccountKey func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + } +} + // CommonStruct func (c *CommonStruct) AuthVerify(ctx context.Context, token string) ([]auth.Permission, error) { @@ -1372,7 +1383,28 @@ func (w *WorkerStruct) Closing(ctx context.Context) (<-chan struct{}, error) { return w.Internal.Closing(ctx) } +func (g GatewayStruct) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) { + return g.Internal.StateGetActor(ctx, actor, ts) +} + +func (g GatewayStruct) ChainHead(ctx context.Context) (*types.TipSet, error) { + return g.Internal.ChainHead(ctx) +} + +func (g GatewayStruct) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) { + return g.Internal.ChainGetTipSet(ctx, tsk) +} + +func (g GatewayStruct) MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) { + return g.Internal.MpoolPush(ctx, sm) +} + +func (g GatewayStruct) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { + return g.Internal.StateAccountKey(ctx, addr, tsk) +} + var _ api.Common = &CommonStruct{} var _ api.FullNode = &FullNodeStruct{} var _ api.StorageMiner = &StorageMinerStruct{} var _ api.WorkerAPI = &WorkerStruct{} +var _ api.GatewayAPI = &GatewayStruct{} diff --git a/api/client/client.go b/api/client/client.go index cd915acf0..390ce93d7 100644 --- a/api/client/client.go +++ b/api/client/client.go @@ -82,3 +82,17 @@ func NewWorkerRPC(ctx context.Context, addr string, requestHeader http.Header) ( return &res, closer, err } + +// NewGatewayRPC creates a new http jsonrpc client for a gateway node. +func NewGatewayRPC(ctx context.Context, addr string, requestHeader http.Header, opts ...jsonrpc.Option) (api.GatewayAPI, jsonrpc.ClientCloser, error) { + var res apistruct.GatewayStruct + closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin", + []interface{}{ + &res.Internal, + }, + requestHeader, + opts..., + ) + + return &res, closer, err +} diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index ba3dcd1d8..476de1927 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -40,6 +40,11 @@ import ( var log = logging.Logger("statemgr") +type StateManagerAPI interface { + LoadActorTsk(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*types.Actor, error) + ResolveToKeyAddress(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) +} + type versionSpec struct { networkVersion network.Version atOrBelow abi.ChainEpoch @@ -1393,3 +1398,5 @@ func (sm *StateManager) GetMarketState(ctx context.Context, ts *types.TipSet) (m } return actState, nil } + +var _ StateManagerAPI = (*StateManager)(nil) diff --git a/cli/cmd.go b/cli/cmd.go index edcb69adc..e6475934b 100644 --- a/cli/cmd.go +++ b/cli/cmd.go @@ -289,6 +289,15 @@ func GetWorkerAPI(ctx *cli.Context) (api.WorkerAPI, jsonrpc.ClientCloser, error) return client.NewWorkerRPC(ctx.Context, addr, headers) } +func GetGatewayAPI(ctx *cli.Context) (api.GatewayAPI, jsonrpc.ClientCloser, error) { + addr, headers, err := GetRawAPI(ctx, repo.FullNode) + if err != nil { + return nil, nil, err + } + + return client.NewGatewayRPC(ctx.Context, addr, headers) +} + func DaemonContext(cctx *cli.Context) context.Context { if mtCtx, ok := cctx.App.Metadata[metadataTraceContext]; ok { return mtCtx.(context.Context) diff --git a/cmd/lotus-gateway/api.go b/cmd/lotus-gateway/api.go index 0a6365dbd..edf6525b1 100644 --- a/cmd/lotus-gateway/api.go +++ b/cmd/lotus-gateway/api.go @@ -49,17 +49,6 @@ func (a *GatewayAPI) checkTipset(ctx context.Context, ts types.TipSetKey) error return nil } -func (a *GatewayAPI) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) { - ctx, span := trace.StartSpan(ctx, "StateGetActor") - defer span.End() - - if err := a.checkTipset(ctx, ts); err != nil { - return nil, fmt.Errorf("bad tipset: %w", err) - } - - return a.api.StateGetActor(ctx, actor, ts) -} - func (a *GatewayAPI) ChainHead(ctx context.Context) (*types.TipSet, error) { ctx, span := trace.StartSpan(ctx, "ChainHead") defer span.End() @@ -88,3 +77,25 @@ func (a *GatewayAPI) MpoolPush(ctx context.Context, sm *types.SignedMessage) (ci return a.api.MpoolPushUntrusted(ctx, sm) } + +func (a *GatewayAPI) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) { + ctx, span := trace.StartSpan(ctx, "StateGetActor") + defer span.End() + + if err := a.checkTipset(ctx, ts); err != nil { + return nil, fmt.Errorf("bad tipset: %w", err) + } + + return a.api.StateGetActor(ctx, actor, ts) +} + +func (a *GatewayAPI) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { + ctx, span := trace.StartSpan(ctx, "StateAccountKey") + defer span.End() + + if err := a.checkTipset(ctx, tsk); err != nil { + return address.Undef, fmt.Errorf("bad tipset: %w", err) + } + + return a.api.StateAccountKey(ctx, addr, tsk) +} diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index a0f754a60..d57dddb79 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -15,6 +15,8 @@ import ( "runtime/pprof" "strings" + "go.uber.org/fx" + "github.com/filecoin-project/lotus/chain/types" paramfetch "github.com/filecoin-project/go-paramfetch" @@ -114,6 +116,10 @@ var DaemonCmd = &cli.Command{ Name: "halt-after-import", Usage: "halt the process after importing chain from file", }, + &cli.BoolFlag{ + Name: "lite", + Usage: "start lotus in lite mode", + }, &cli.StringFlag{ Name: "pprof", Usage: "specify name of file for writing cpu profile to", @@ -240,6 +246,27 @@ var DaemonCmd = &cli.Command{ shutdownChan := make(chan struct{}) + // If the daemon is started in "lite mode", replace the StateManager + // with a thin client to a gateway server + liteMode := node.Options() + if cctx.Bool("lite") { + gapi, closer, err := lcli.GetGatewayAPI(cctx) + if err != nil { + return err + } + + createRPCStateMgr := func(lc fx.Lifecycle) *modules.RPCStateManager { + lc.Append(fx.Hook{ + OnStop: func(ctx context.Context) error { + closer() + return nil + }, + }) + return modules.NewRPCStateManager(gapi) + } + liteMode = node.Override(new(stmgr.StateManagerAPI), createRPCStateMgr) + } + var api api.FullNode stop, err := node.New(ctx, @@ -251,6 +278,7 @@ var DaemonCmd = &cli.Command{ node.Repo(r), genesis, + liteMode, node.ApplyIf(func(s *node.Settings) bool { return cctx.IsSet("api") }, node.Override(node.SetApiEndpointKey, func(lr repo.LockedRepo) error { diff --git a/node/builder.go b/node/builder.go index 0d48ec130..50a20d4bb 100644 --- a/node/builder.go +++ b/node/builder.go @@ -261,6 +261,7 @@ func Online() Option { Override(new(*store.ChainStore), modules.ChainStore), Override(new(stmgr.UpgradeSchedule), stmgr.DefaultUpgradeSchedule()), Override(new(*stmgr.StateManager), stmgr.NewStateManagerWithUpgradeSchedule), + Override(new(stmgr.StateManagerAPI), From(new(*stmgr.StateManager))), Override(new(*wallet.Wallet), wallet.NewWallet), Override(new(*messagesigner.MessageSigner), messagesigner.NewMessageSigner), diff --git a/node/impl/full/wallet.go b/node/impl/full/wallet.go index b2ecdebbd..33f924846 100644 --- a/node/impl/full/wallet.go +++ b/node/impl/full/wallet.go @@ -19,8 +19,8 @@ import ( type WalletAPI struct { fx.In - StateManager *stmgr.StateManager - Wallet *wallet.Wallet + stmgr.StateManagerAPI + Wallet *wallet.Wallet } func (a *WalletAPI) WalletNew(ctx context.Context, typ crypto.SigType) (address.Address, error) { @@ -36,7 +36,7 @@ func (a *WalletAPI) WalletList(ctx context.Context) ([]address.Address, error) { } func (a *WalletAPI) WalletBalance(ctx context.Context, addr address.Address) (types.BigInt, error) { - act, err := a.StateManager.LoadActorTsk(ctx, addr, types.EmptyTSK) + act, err := a.StateManagerAPI.LoadActorTsk(ctx, addr, types.EmptyTSK) if xerrors.Is(err, types.ErrActorNotFound) { return big.Zero(), nil } else if err != nil { @@ -46,7 +46,7 @@ func (a *WalletAPI) WalletBalance(ctx context.Context, addr address.Address) (ty } func (a *WalletAPI) WalletSign(ctx context.Context, k address.Address, msg []byte) (*crypto.Signature, error) { - keyAddr, err := a.StateManager.ResolveToKeyAddress(ctx, k, nil) + keyAddr, err := a.StateManagerAPI.ResolveToKeyAddress(ctx, k, nil) if err != nil { return nil, xerrors.Errorf("failed to resolve ID address: %w", keyAddr) } diff --git a/node/modules/statemanager.go b/node/modules/statemanager.go new file mode 100644 index 000000000..b673f7268 --- /dev/null +++ b/node/modules/statemanager.go @@ -0,0 +1,29 @@ +package modules + +import ( + "context" + + "github.com/filecoin-project/lotus/api" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/stmgr" + "github.com/filecoin-project/lotus/chain/types" +) + +type RPCStateManager struct { + gapi api.GatewayAPI +} + +func NewRPCStateManager(api api.GatewayAPI) *RPCStateManager { + return &RPCStateManager{gapi: api} +} + +func (s *RPCStateManager) LoadActorTsk(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*types.Actor, error) { + return s.gapi.StateGetActor(ctx, addr, tsk) +} + +func (s *RPCStateManager) ResolveToKeyAddress(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) { + return s.gapi.StateAccountKey(ctx, addr, ts.Key()) +} + +var _ stmgr.StateManagerAPI = (*RPCStateManager)(nil) From 1ffdd7d5b3a83ab313066714f1e305cbe967931a Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Wed, 30 Sep 2020 10:43:59 +0200 Subject: [PATCH 650/885] fix: gateway - remove tracing (JsonRPC automatically traces all calls) --- cmd/lotus-gateway/api.go | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/cmd/lotus-gateway/api.go b/cmd/lotus-gateway/api.go index edf6525b1..22b30544e 100644 --- a/cmd/lotus-gateway/api.go +++ b/cmd/lotus-gateway/api.go @@ -9,8 +9,6 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" "github.com/ipfs/go-cid" - - "go.opencensus.io/trace" ) const LookbackCap = time.Hour @@ -50,17 +48,12 @@ func (a *GatewayAPI) checkTipset(ctx context.Context, ts types.TipSetKey) error } func (a *GatewayAPI) ChainHead(ctx context.Context) (*types.TipSet, error) { - ctx, span := trace.StartSpan(ctx, "ChainHead") - defer span.End() // TODO: cache and invalidate cache when timestamp is up (or have internal ChainNotify) return a.api.ChainHead(ctx) } func (a *GatewayAPI) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) { - ctx, span := trace.StartSpan(ctx, "ChainGetTipSet") - defer span.End() - if err := a.checkTipset(ctx, tsk); err != nil { return nil, fmt.Errorf("bad tipset: %w", err) } @@ -70,18 +63,12 @@ func (a *GatewayAPI) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (* } func (a *GatewayAPI) MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) { - ctx, span := trace.StartSpan(ctx, "MpoolPush") - defer span.End() - // TODO: additional anti-spam checks return a.api.MpoolPushUntrusted(ctx, sm) } func (a *GatewayAPI) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) { - ctx, span := trace.StartSpan(ctx, "StateGetActor") - defer span.End() - if err := a.checkTipset(ctx, ts); err != nil { return nil, fmt.Errorf("bad tipset: %w", err) } @@ -90,9 +77,6 @@ func (a *GatewayAPI) StateGetActor(ctx context.Context, actor address.Address, t } func (a *GatewayAPI) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { - ctx, span := trace.StartSpan(ctx, "StateAccountKey") - defer span.End() - if err := a.checkTipset(ctx, tsk); err != nil { return address.Undef, fmt.Errorf("bad tipset: %w", err) } From eec13ff8dc3144606b1a43e21d62f885ef45d8ce Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Wed, 30 Sep 2020 16:35:06 +0200 Subject: [PATCH 651/885] refactor: daemon - simplify gateway cleanup --- cmd/lotus/daemon.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index d57dddb79..54dcfe54c 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -15,8 +15,6 @@ import ( "runtime/pprof" "strings" - "go.uber.org/fx" - "github.com/filecoin-project/lotus/chain/types" paramfetch "github.com/filecoin-project/go-paramfetch" @@ -249,22 +247,24 @@ var DaemonCmd = &cli.Command{ // If the daemon is started in "lite mode", replace the StateManager // with a thin client to a gateway server liteMode := node.Options() - if cctx.Bool("lite") { + isLite := cctx.Bool("lite") + if isLite { gapi, closer, err := lcli.GetGatewayAPI(cctx) if err != nil { return err } - createRPCStateMgr := func(lc fx.Lifecycle) *modules.RPCStateManager { - lc.Append(fx.Hook{ - OnStop: func(ctx context.Context) error { - closer() - return nil - }, - }) - return modules.NewRPCStateManager(gapi) - } - liteMode = node.Override(new(stmgr.StateManagerAPI), createRPCStateMgr) + defer closer() + + liteMode = node.Options( + node.Override(new(api.GatewayAPI), gapi), + node.Override(new(stmgr.StateManagerAPI), modules.NewRPCStateManager), + node.Unset(node.RunHelloKey), + node.Unset(node.RunChainExchangeKey), + node.Unset(node.RunPeerMgrKey), + node.Unset(node.HandleIncomingBlocksKey), + node.Unset(node.HandleIncomingMessagesKey), + ) } var api api.FullNode From 00a14de0948ce078437338f2356c04159efd1087 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Wed, 30 Sep 2020 16:36:16 +0200 Subject: [PATCH 652/885] feat: gateway - StateLookupID --- api/api_gateway.go | 3 ++- api/apistruct/struct.go | 12 ++++++++---- chain/stmgr/stmgr.go | 1 + cmd/lotus-gateway/api.go | 26 ++++++++++++++++++-------- node/impl/full/multisig.go | 9 +++++---- node/impl/full/state.go | 6 +----- node/modules/statemanager.go | 4 ++++ 7 files changed, 39 insertions(+), 22 deletions(-) diff --git a/api/api_gateway.go b/api/api_gateway.go index 6a220e9ea..398a7518f 100644 --- a/api/api_gateway.go +++ b/api/api_gateway.go @@ -9,9 +9,10 @@ import ( ) type GatewayAPI interface { - StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) ChainHead(ctx context.Context) (*types.TipSet, error) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) + StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) } diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index c71b6b560..268fc4460 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -1383,10 +1383,6 @@ func (w *WorkerStruct) Closing(ctx context.Context) (<-chan struct{}, error) { return w.Internal.Closing(ctx) } -func (g GatewayStruct) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) { - return g.Internal.StateGetActor(ctx, actor, ts) -} - func (g GatewayStruct) ChainHead(ctx context.Context) (*types.TipSet, error) { return g.Internal.ChainHead(ctx) } @@ -1403,6 +1399,14 @@ func (g GatewayStruct) StateAccountKey(ctx context.Context, addr address.Address return g.Internal.StateAccountKey(ctx, addr, tsk) } +func (g GatewayStruct) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) { + return g.Internal.StateGetActor(ctx, actor, ts) +} + +func (g GatewayStruct) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { + panic("implement me") +} + var _ api.Common = &CommonStruct{} var _ api.FullNode = &FullNodeStruct{} var _ api.StorageMiner = &StorageMinerStruct{} diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index 476de1927..d3129ea20 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -42,6 +42,7 @@ var log = logging.Logger("statemgr") type StateManagerAPI interface { LoadActorTsk(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*types.Actor, error) + LookupID(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) ResolveToKeyAddress(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) } diff --git a/cmd/lotus-gateway/api.go b/cmd/lotus-gateway/api.go index 22b30544e..642ed4d86 100644 --- a/cmd/lotus-gateway/api.go +++ b/cmd/lotus-gateway/api.go @@ -68,14 +68,6 @@ func (a *GatewayAPI) MpoolPush(ctx context.Context, sm *types.SignedMessage) (ci return a.api.MpoolPushUntrusted(ctx, sm) } -func (a *GatewayAPI) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) { - if err := a.checkTipset(ctx, ts); err != nil { - return nil, fmt.Errorf("bad tipset: %w", err) - } - - return a.api.StateGetActor(ctx, actor, ts) -} - func (a *GatewayAPI) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { if err := a.checkTipset(ctx, tsk); err != nil { return address.Undef, fmt.Errorf("bad tipset: %w", err) @@ -83,3 +75,21 @@ func (a *GatewayAPI) StateAccountKey(ctx context.Context, addr address.Address, return a.api.StateAccountKey(ctx, addr, tsk) } + +func (a *GatewayAPI) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) { + if err := a.checkTipset(ctx, tsk); err != nil { + return nil, fmt.Errorf("bad tipset: %w", err) + } + + return a.api.StateGetActor(ctx, actor, tsk) +} + +func (a *GatewayAPI) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { + if err := a.checkTipset(ctx, tsk); err != nil { + return address.Undef, fmt.Errorf("bad tipset: %w", err) + } + + return a.api.StateLookupID(ctx, addr, tsk) +} + +var _ api.GatewayAPI = &GatewayAPI{} diff --git a/node/impl/full/multisig.go b/node/impl/full/multisig.go index 715689edc..489658412 100644 --- a/node/impl/full/multisig.go +++ b/node/impl/full/multisig.go @@ -3,6 +3,8 @@ package full import ( "context" + "github.com/filecoin-project/lotus/chain/stmgr" + "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-address" @@ -23,9 +25,8 @@ import ( type MsigAPI struct { fx.In - WalletAPI WalletAPI - StateAPI StateAPI - MpoolAPI MpoolAPI + StateManagerAPI stmgr.StateManagerAPI + MpoolAPI MpoolAPI } func (a *MsigAPI) messageBuilder(ctx context.Context, from address.Address) (multisig.MessageBuilder, error) { @@ -152,7 +153,7 @@ func (a *MsigAPI) msigApproveOrCancel(ctx context.Context, operation api.MsigPro } if proposer.Protocol() != address.ID { - proposerID, err := a.StateAPI.StateLookupID(ctx, proposer, types.EmptyTSK) + proposerID, err := a.StateManagerAPI.LookupID(ctx, proposer, nil) if err != nil { return cid.Undef, err } diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 56dfc0a14..37af5a9a4 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -382,12 +382,8 @@ func (a *StateAPI) StateLookupID(ctx context.Context, addr address.Address, tsk if err != nil { return address.Undef, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - state, err := a.stateForTs(ctx, ts) - if err != nil { - return address.Undef, err - } - return state.LookupID(addr) + return a.StateManager.LookupID(ctx, addr, ts) } func (a *StateAPI) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { diff --git a/node/modules/statemanager.go b/node/modules/statemanager.go index b673f7268..0ed054d45 100644 --- a/node/modules/statemanager.go +++ b/node/modules/statemanager.go @@ -22,6 +22,10 @@ func (s *RPCStateManager) LoadActorTsk(ctx context.Context, addr address.Address return s.gapi.StateGetActor(ctx, addr, tsk) } +func (s *RPCStateManager) LookupID(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) { + return s.gapi.StateLookupID(ctx, addr, ts.Key()) +} + func (s *RPCStateManager) ResolveToKeyAddress(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) { return s.gapi.StateAccountKey(ctx, addr, ts.Key()) } From 8fa4c0a970e58bd9cebbd4ba2614b033701b75c5 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Wed, 30 Sep 2020 17:59:18 +0200 Subject: [PATCH 653/885] feat: gateway - MpoolPush --- cmd/lotus/daemon.go | 3 +++ node/builder.go | 3 +++ node/impl/full/mpool.go | 7 ++++++- node/modules/pushmessage.go | 25 +++++++++++++++++++++++++ 4 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 node/modules/pushmessage.go diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 54dcfe54c..f83e74118 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -15,6 +15,8 @@ import ( "runtime/pprof" "strings" + "github.com/filecoin-project/lotus/node/impl/full" + "github.com/filecoin-project/lotus/chain/types" paramfetch "github.com/filecoin-project/go-paramfetch" @@ -259,6 +261,7 @@ var DaemonCmd = &cli.Command{ liteMode = node.Options( node.Override(new(api.GatewayAPI), gapi), node.Override(new(stmgr.StateManagerAPI), modules.NewRPCStateManager), + node.Override(new(full.PushMessageAPI), modules.NewRPCPushMessageAPI), node.Unset(node.RunHelloKey), node.Unset(node.RunChainExchangeKey), node.Unset(node.RunPeerMgrKey), diff --git a/node/builder.go b/node/builder.go index 50a20d4bb..8579131b9 100644 --- a/node/builder.go +++ b/node/builder.go @@ -6,6 +6,8 @@ import ( "os" "time" + "github.com/filecoin-project/lotus/node/impl/full" + logging "github.com/ipfs/go-log" ci "github.com/libp2p/go-libp2p-core/crypto" "github.com/libp2p/go-libp2p-core/host" @@ -262,6 +264,7 @@ func Online() Option { Override(new(stmgr.UpgradeSchedule), stmgr.DefaultUpgradeSchedule()), Override(new(*stmgr.StateManager), stmgr.NewStateManagerWithUpgradeSchedule), Override(new(stmgr.StateManagerAPI), From(new(*stmgr.StateManager))), + Override(new(full.PushMessageAPI), From(new(full.MpoolAPI))), Override(new(*wallet.Wallet), wallet.NewWallet), Override(new(*messagesigner.MessageSigner), messagesigner.NewMessageSigner), diff --git a/node/impl/full/mpool.go b/node/impl/full/mpool.go index 1f093606c..000891bb2 100644 --- a/node/impl/full/mpool.go +++ b/node/impl/full/mpool.go @@ -15,11 +15,16 @@ import ( "github.com/filecoin-project/lotus/node/modules/dtypes" ) +type PushMessageAPI interface { + MpoolPush(ctx context.Context, smsg *types.SignedMessage) (cid.Cid, error) +} + type MpoolAPI struct { fx.In WalletAPI GasAPI + PushMessageAPI MessageSigner *messagesigner.MessageSigner @@ -107,7 +112,7 @@ func (a *MpoolAPI) MpoolClear(ctx context.Context, local bool) error { } func (a *MpoolAPI) MpoolPush(ctx context.Context, smsg *types.SignedMessage) (cid.Cid, error) { - return a.Mpool.Push(smsg) + return a.PushMessageAPI.MpoolPush(ctx, smsg) } func (a *MpoolAPI) MpoolPushUntrusted(ctx context.Context, smsg *types.SignedMessage) (cid.Cid, error) { diff --git a/node/modules/pushmessage.go b/node/modules/pushmessage.go new file mode 100644 index 000000000..59af435a7 --- /dev/null +++ b/node/modules/pushmessage.go @@ -0,0 +1,25 @@ +package modules + +import ( + "context" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/node/impl/full" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lotus/chain/types" +) + +type RPCPushMessageAPI struct { + gapi api.GatewayAPI +} + +func NewRPCPushMessageAPI(api api.GatewayAPI) *RPCPushMessageAPI { + return &RPCPushMessageAPI{gapi: api} +} + +func (p *RPCPushMessageAPI) MpoolPush(ctx context.Context, smsg *types.SignedMessage) (cid.Cid, error) { + return p.gapi.MpoolPush(ctx, smsg) +} + +var _ full.PushMessageAPI = (*RPCPushMessageAPI)(nil) From f1b1d8cec6375f7d3decd47aa1c43f1582c1dfce Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Thu, 1 Oct 2020 12:38:14 +0200 Subject: [PATCH 654/885] fix: MpoolAPI DI --- node/builder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/builder.go b/node/builder.go index 8579131b9..6bdb46ca5 100644 --- a/node/builder.go +++ b/node/builder.go @@ -264,7 +264,7 @@ func Online() Option { Override(new(stmgr.UpgradeSchedule), stmgr.DefaultUpgradeSchedule()), Override(new(*stmgr.StateManager), stmgr.NewStateManagerWithUpgradeSchedule), Override(new(stmgr.StateManagerAPI), From(new(*stmgr.StateManager))), - Override(new(full.PushMessageAPI), From(new(full.MpoolAPI))), + Override(new(full.PushMessageAPI), new(full.MpoolAPI)), Override(new(*wallet.Wallet), wallet.NewWallet), Override(new(*messagesigner.MessageSigner), messagesigner.NewMessageSigner), From 2719adc1b1e28eb31c561ed0bcfd8610a30abe00 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Thu, 1 Oct 2020 17:51:27 +0200 Subject: [PATCH 655/885] feat: lite-mode - thin client for chan & state --- api/apistruct/struct.go | 7 +-- cmd/lotus/daemon.go | 4 +- node/builder.go | 5 +- node/impl/full/chain.go | 25 +++++++-- node/impl/full/mpool.go | 28 +++++++--- node/impl/full/state.go | 52 ++++++++++++++----- node/modules/pushmessage.go | 25 --------- .../{statemanager.go => rpcstatemanager.go} | 0 8 files changed, 93 insertions(+), 53 deletions(-) delete mode 100644 node/modules/pushmessage.go rename node/modules/{statemanager.go => rpcstatemanager.go} (100%) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 268fc4460..7ab50965a 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -364,11 +364,12 @@ type WorkerStruct struct { type GatewayStruct struct { Internal struct { // TODO: does the gateway need perms? - StateGetActor func(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) - ChainHead func(ctx context.Context) (*types.TipSet, error) ChainGetTipSet func(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) + ChainHead func(ctx context.Context) (*types.TipSet, error) MpoolPush func(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) StateAccountKey func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + StateGetActor func(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) + StateLookupID func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) } } @@ -1404,7 +1405,7 @@ func (g GatewayStruct) StateGetActor(ctx context.Context, actor address.Address, } func (g GatewayStruct) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { - panic("implement me") + return g.Internal.StateLookupID(ctx, addr, tsk) } var _ api.Common = &CommonStruct{} diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index f83e74118..1dc9b3c78 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -260,8 +260,10 @@ var DaemonCmd = &cli.Command{ liteMode = node.Options( node.Override(new(api.GatewayAPI), gapi), + node.Override(new(full.ChainModuleAPI), node.From(new(api.GatewayAPI))), + node.Override(new(full.MpoolModuleAPI), node.From(new(api.GatewayAPI))), + node.Override(new(full.StateModuleAPI), node.From(new(api.GatewayAPI))), node.Override(new(stmgr.StateManagerAPI), modules.NewRPCStateManager), - node.Override(new(full.PushMessageAPI), modules.NewRPCPushMessageAPI), node.Unset(node.RunHelloKey), node.Unset(node.RunChainExchangeKey), node.Unset(node.RunPeerMgrKey), diff --git a/node/builder.go b/node/builder.go index 6bdb46ca5..e8ce96ec8 100644 --- a/node/builder.go +++ b/node/builder.go @@ -264,10 +264,13 @@ func Online() Option { Override(new(stmgr.UpgradeSchedule), stmgr.DefaultUpgradeSchedule()), Override(new(*stmgr.StateManager), stmgr.NewStateManagerWithUpgradeSchedule), Override(new(stmgr.StateManagerAPI), From(new(*stmgr.StateManager))), - Override(new(full.PushMessageAPI), new(full.MpoolAPI)), Override(new(*wallet.Wallet), wallet.NewWallet), Override(new(*messagesigner.MessageSigner), messagesigner.NewMessageSigner), + Override(new(full.ChainModuleAPI), From(new(full.ChainModule))), + Override(new(full.StateModuleAPI), From(new(full.StateModule))), + Override(new(full.MpoolModuleAPI), From(new(full.MpoolModule))), + Override(new(dtypes.ChainGCLocker), blockstore.NewGCLocker), Override(new(dtypes.ChainGCBlockstore), modules.ChainGCBlockstore), Override(new(dtypes.ChainBitswap), modules.ChainBitswap), diff --git a/node/impl/full/chain.go b/node/impl/full/chain.go index aa2ae4df1..810c56c0b 100644 --- a/node/impl/full/chain.go +++ b/node/impl/full/chain.go @@ -39,10 +39,27 @@ import ( var log = logging.Logger("fullnode") +type ChainModuleAPI interface { + ChainHead(context.Context) (*types.TipSet, error) + ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) +} + +// ChainModule provides a default implementation of ChainModuleAPI. +// It can be swapped out with another implementation through Dependency +// Injection (for example with a thin RPC client). +type ChainModule struct { + fx.In + + Chain *store.ChainStore +} + +var _ ChainModuleAPI = (*ChainModule)(nil) + type ChainAPI struct { fx.In WalletAPI + ChainModuleAPI Chain *store.ChainStore } @@ -51,8 +68,8 @@ func (a *ChainAPI) ChainNotify(ctx context.Context) (<-chan []*api.HeadChange, e return a.Chain.SubHeadChanges(ctx), nil } -func (a *ChainAPI) ChainHead(context.Context) (*types.TipSet, error) { - return a.Chain.GetHeaviestTipSet(), nil +func (m *ChainModule) ChainHead(ctx context.Context) (*types.TipSet, error) { + return m.Chain.GetHeaviestTipSet(), nil } func (a *ChainAPI) ChainGetRandomnessFromTickets(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error) { @@ -77,8 +94,8 @@ func (a *ChainAPI) ChainGetBlock(ctx context.Context, msg cid.Cid) (*types.Block return a.Chain.GetBlock(msg) } -func (a *ChainAPI) ChainGetTipSet(ctx context.Context, key types.TipSetKey) (*types.TipSet, error) { - return a.Chain.LoadTipSet(key) +func (m *ChainModule) ChainGetTipSet(ctx context.Context, key types.TipSetKey) (*types.TipSet, error) { + return m.Chain.LoadTipSet(key) } func (a *ChainAPI) ChainGetBlockMessages(ctx context.Context, msg cid.Cid) (*api.BlockMessages, error) { diff --git a/node/impl/full/mpool.go b/node/impl/full/mpool.go index 000891bb2..916d5785b 100644 --- a/node/impl/full/mpool.go +++ b/node/impl/full/mpool.go @@ -4,27 +4,43 @@ import ( "context" "encoding/json" + "github.com/filecoin-project/lotus/chain/messagepool" + + "github.com/filecoin-project/lotus/chain/messagesigner" + + "github.com/filecoin-project/lotus/node/modules/dtypes" + "github.com/filecoin-project/go-address" "github.com/ipfs/go-cid" "go.uber.org/fx" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/messagesigner" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/node/modules/dtypes" ) -type PushMessageAPI interface { +type MpoolModuleAPI interface { MpoolPush(ctx context.Context, smsg *types.SignedMessage) (cid.Cid, error) } +// MpoolModule provides a default implementation of MpoolModuleAPI. +// It can be swapped out with another implementation through Dependency +// Injection (for example with a thin RPC client). +type MpoolModule struct { + fx.In + + Mpool *messagepool.MessagePool +} + +var _ MpoolModuleAPI = (*MpoolModule)(nil) + type MpoolAPI struct { fx.In + MpoolModuleAPI + WalletAPI GasAPI - PushMessageAPI MessageSigner *messagesigner.MessageSigner @@ -111,8 +127,8 @@ func (a *MpoolAPI) MpoolClear(ctx context.Context, local bool) error { return nil } -func (a *MpoolAPI) MpoolPush(ctx context.Context, smsg *types.SignedMessage) (cid.Cid, error) { - return a.PushMessageAPI.MpoolPush(ctx, smsg) +func (m *MpoolModule) MpoolPush(ctx context.Context, smsg *types.SignedMessage) (cid.Cid, error) { + return m.Mpool.Push(smsg) } func (a *MpoolAPI) MpoolPushUntrusted(ctx context.Context, smsg *types.SignedMessage) (cid.Cid, error) { diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 37af5a9a4..a391dffa1 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -42,6 +42,24 @@ import ( "github.com/filecoin-project/lotus/node/modules/dtypes" ) +type StateModuleAPI interface { + StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) + StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) +} + +// StateModule provides a default implementation of StateModuleAPI. +// It can be swapped out with another implementation through Dependency +// Injection (for example with a thin RPC client). +type StateModule struct { + fx.In + + StateManager *stmgr.StateManager + Chain *store.ChainStore +} + +var _ StateModuleAPI = (*StateModule)(nil) + type StateAPI struct { fx.In @@ -49,6 +67,8 @@ type StateAPI struct { // API attached to the state API. It probably should live somewhere better Wallet *wallet.Wallet + StateModuleAPI + ProofVerifier ffiwrapper.Verifier StateManager *stmgr.StateManager Chain *store.ChainStore @@ -349,27 +369,33 @@ func (a *StateAPI) StateReplay(ctx context.Context, tsk types.TipSetKey, mc cid. }, nil } -func (a *StateAPI) stateForTs(ctx context.Context, ts *types.TipSet) (*state.StateTree, error) { +func stateForTs(ctx context.Context, ts *types.TipSet, cstore *store.ChainStore, smgr *stmgr.StateManager) (*state.StateTree, error) { if ts == nil { - ts = a.Chain.GetHeaviestTipSet() + ts = cstore.GetHeaviestTipSet() } - st, _, err := a.StateManager.TipSetState(ctx, ts) + st, _, err := smgr.TipSetState(ctx, ts) if err != nil { return nil, err } - buf := bufbstore.NewBufferedBstore(a.Chain.Blockstore()) + buf := bufbstore.NewBufferedBstore(cstore.Blockstore()) cst := cbor.NewCborStore(buf) return state.LoadStateTree(cst, st) } +func (a *StateAPI) stateForTs(ctx context.Context, ts *types.TipSet) (*state.StateTree, error) { + return stateForTs(ctx, ts, a.Chain, a.StateManager) +} +func (m *StateModule) stateForTs(ctx context.Context, ts *types.TipSet) (*state.StateTree, error) { + return stateForTs(ctx, ts, m.Chain, m.StateManager) +} -func (a *StateAPI) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) { - ts, err := a.Chain.GetTipSetFromKey(tsk) +func (m *StateModule) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) { + ts, err := m.Chain.GetTipSetFromKey(tsk) if err != nil { return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - state, err := a.stateForTs(ctx, ts) + state, err := m.stateForTs(ctx, ts) if err != nil { return nil, xerrors.Errorf("computing tipset state failed: %w", err) } @@ -377,22 +403,22 @@ func (a *StateAPI) StateGetActor(ctx context.Context, actor address.Address, tsk return state.GetActor(actor) } -func (a *StateAPI) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { - ts, err := a.Chain.GetTipSetFromKey(tsk) +func (m *StateModule) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { + ts, err := m.Chain.GetTipSetFromKey(tsk) if err != nil { return address.Undef, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - return a.StateManager.LookupID(ctx, addr, ts) + return m.StateManager.LookupID(ctx, addr, ts) } -func (a *StateAPI) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { - ts, err := a.Chain.GetTipSetFromKey(tsk) +func (m *StateModule) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { + ts, err := m.Chain.GetTipSetFromKey(tsk) if err != nil { return address.Undef, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - return a.StateManager.ResolveToKeyAddress(ctx, addr, ts) + return m.StateManager.ResolveToKeyAddress(ctx, addr, ts) } func (a *StateAPI) StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*api.ActorState, error) { diff --git a/node/modules/pushmessage.go b/node/modules/pushmessage.go deleted file mode 100644 index 59af435a7..000000000 --- a/node/modules/pushmessage.go +++ /dev/null @@ -1,25 +0,0 @@ -package modules - -import ( - "context" - - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/node/impl/full" - "github.com/ipfs/go-cid" - - "github.com/filecoin-project/lotus/chain/types" -) - -type RPCPushMessageAPI struct { - gapi api.GatewayAPI -} - -func NewRPCPushMessageAPI(api api.GatewayAPI) *RPCPushMessageAPI { - return &RPCPushMessageAPI{gapi: api} -} - -func (p *RPCPushMessageAPI) MpoolPush(ctx context.Context, smsg *types.SignedMessage) (cid.Cid, error) { - return p.gapi.MpoolPush(ctx, smsg) -} - -var _ full.PushMessageAPI = (*RPCPushMessageAPI)(nil) diff --git a/node/modules/statemanager.go b/node/modules/rpcstatemanager.go similarity index 100% rename from node/modules/statemanager.go rename to node/modules/rpcstatemanager.go From be09a8a00acc3e41bbe1f08558c24a99785092c7 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Thu, 1 Oct 2020 17:54:28 +0200 Subject: [PATCH 656/885] feat: lite-mode - check that gateway API implements required chain, mpool, state methods --- cmd/lotus-gateway/api.go | 4 ++++ cmd/lotus/daemon.go | 6 ++---- node/builder.go | 4 ++-- node/impl/full/chain.go | 2 +- node/impl/full/mpool.go | 9 +++------ node/impl/full/multisig.go | 8 +++----- node/impl/full/wallet.go | 4 ++-- 7 files changed, 17 insertions(+), 20 deletions(-) diff --git a/cmd/lotus-gateway/api.go b/cmd/lotus-gateway/api.go index 642ed4d86..5f8e9adf9 100644 --- a/cmd/lotus-gateway/api.go +++ b/cmd/lotus-gateway/api.go @@ -8,6 +8,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/node/impl/full" "github.com/ipfs/go-cid" ) @@ -93,3 +94,6 @@ func (a *GatewayAPI) StateLookupID(ctx context.Context, addr address.Address, ts } var _ api.GatewayAPI = &GatewayAPI{} +var _ full.ChainModuleAPI = (*GatewayAPI)(nil) +var _ full.MpoolModuleAPI = (*GatewayAPI)(nil) +var _ full.StateModuleAPI = (*GatewayAPI)(nil) diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 1dc9b3c78..c05317458 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -15,10 +15,6 @@ import ( "runtime/pprof" "strings" - "github.com/filecoin-project/lotus/node/impl/full" - - "github.com/filecoin-project/lotus/chain/types" - paramfetch "github.com/filecoin-project/go-paramfetch" "github.com/mitchellh/go-homedir" "github.com/multiformats/go-multiaddr" @@ -34,6 +30,7 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/store" + "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" @@ -42,6 +39,7 @@ import ( "github.com/filecoin-project/lotus/lib/ulimit" "github.com/filecoin-project/lotus/metrics" "github.com/filecoin-project/lotus/node" + "github.com/filecoin-project/lotus/node/impl/full" "github.com/filecoin-project/lotus/node/modules" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/testing" diff --git a/node/builder.go b/node/builder.go index e8ce96ec8..d513fe626 100644 --- a/node/builder.go +++ b/node/builder.go @@ -6,8 +6,6 @@ import ( "os" "time" - "github.com/filecoin-project/lotus/node/impl/full" - logging "github.com/ipfs/go-log" ci "github.com/libp2p/go-libp2p-core/crypto" "github.com/libp2p/go-libp2p-core/host" @@ -61,6 +59,7 @@ import ( "github.com/filecoin-project/lotus/node/hello" "github.com/filecoin-project/lotus/node/impl" "github.com/filecoin-project/lotus/node/impl/common" + "github.com/filecoin-project/lotus/node/impl/full" "github.com/filecoin-project/lotus/node/modules" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/helpers" @@ -270,6 +269,7 @@ func Online() Option { Override(new(full.ChainModuleAPI), From(new(full.ChainModule))), Override(new(full.StateModuleAPI), From(new(full.StateModule))), Override(new(full.MpoolModuleAPI), From(new(full.MpoolModule))), + Override(new(stmgr.StateManagerAPI), From(new(*stmgr.StateManager))), Override(new(dtypes.ChainGCLocker), blockstore.NewGCLocker), Override(new(dtypes.ChainGCBlockstore), modules.ChainGCBlockstore), diff --git a/node/impl/full/chain.go b/node/impl/full/chain.go index 810c56c0b..88b84cad2 100644 --- a/node/impl/full/chain.go +++ b/node/impl/full/chain.go @@ -68,7 +68,7 @@ func (a *ChainAPI) ChainNotify(ctx context.Context) (<-chan []*api.HeadChange, e return a.Chain.SubHeadChanges(ctx), nil } -func (m *ChainModule) ChainHead(ctx context.Context) (*types.TipSet, error) { +func (m *ChainModule) ChainHead(context.Context) (*types.TipSet, error) { return m.Chain.GetHeaviestTipSet(), nil } diff --git a/node/impl/full/mpool.go b/node/impl/full/mpool.go index 916d5785b..fc32aecd2 100644 --- a/node/impl/full/mpool.go +++ b/node/impl/full/mpool.go @@ -4,19 +4,16 @@ import ( "context" "encoding/json" - "github.com/filecoin-project/lotus/chain/messagepool" - - "github.com/filecoin-project/lotus/chain/messagesigner" - - "github.com/filecoin-project/lotus/node/modules/dtypes" - "github.com/filecoin-project/go-address" "github.com/ipfs/go-cid" "go.uber.org/fx" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/messagepool" + "github.com/filecoin-project/lotus/chain/messagesigner" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/node/modules/dtypes" ) type MpoolModuleAPI interface { diff --git a/node/impl/full/multisig.go b/node/impl/full/multisig.go index 489658412..3b05e4493 100644 --- a/node/impl/full/multisig.go +++ b/node/impl/full/multisig.go @@ -3,8 +3,6 @@ package full import ( "context" - "github.com/filecoin-project/lotus/chain/stmgr" - "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-address" @@ -25,8 +23,8 @@ import ( type MsigAPI struct { fx.In - StateManagerAPI stmgr.StateManagerAPI - MpoolAPI MpoolAPI + StateAPI StateAPI + MpoolAPI MpoolAPI } func (a *MsigAPI) messageBuilder(ctx context.Context, from address.Address) (multisig.MessageBuilder, error) { @@ -153,7 +151,7 @@ func (a *MsigAPI) msigApproveOrCancel(ctx context.Context, operation api.MsigPro } if proposer.Protocol() != address.ID { - proposerID, err := a.StateManagerAPI.LookupID(ctx, proposer, nil) + proposerID, err := a.StateAPI.StateLookupID(ctx, proposer, types.EmptyTSK) if err != nil { return cid.Undef, err } diff --git a/node/impl/full/wallet.go b/node/impl/full/wallet.go index 33f924846..616c7b46e 100644 --- a/node/impl/full/wallet.go +++ b/node/impl/full/wallet.go @@ -19,8 +19,8 @@ import ( type WalletAPI struct { fx.In - stmgr.StateManagerAPI - Wallet *wallet.Wallet + StateManagerAPI stmgr.StateManagerAPI + Wallet *wallet.Wallet } func (a *WalletAPI) WalletNew(ctx context.Context, typ crypto.SigType) (address.Address, error) { From 767611247c0239f79473d59f02daaedea8a35b0c Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Fri, 2 Oct 2020 16:14:30 +0200 Subject: [PATCH 657/885] feat: add RPC for StateWaitMsg --- api/api_full.go | 4 ++ api/api_gateway.go | 3 ++ api/apistruct/struct.go | 27 +++++++--- chain/stmgr/stmgr.go | 28 ++++++++--- cmd/lotus-gateway/api.go | 86 +++++++++++++++++++++++--------- cmd/lotus/daemon.go | 2 +- markets/storageadapter/client.go | 2 +- node/impl/full/chain.go | 7 +-- node/impl/full/state.go | 15 ++++-- 9 files changed, 127 insertions(+), 47 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 601b14660..777e996cb 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -367,6 +367,10 @@ type FullNode interface { // StateWaitMsg looks back in the chain for a message. If not found, it blocks until the // message arrives on chain, and gets to the indicated confidence depth. StateWaitMsg(ctx context.Context, cid cid.Cid, confidence uint64) (*MsgLookup, error) + // StateWaitMsgLimited looks back up to limit epochs in the chain for a message. + // If not found, it blocks until the message arrives on chain, and gets to the + // indicated confidence depth. + StateWaitMsgLimited(ctx context.Context, cid cid.Cid, confidence uint64, limit abi.ChainEpoch) (*MsgLookup, error) // StateListMiners returns the addresses of every miner that has claimed power in the Power Actor StateListMiners(context.Context, types.TipSetKey) ([]address.Address, error) // StateListActors returns the addresses of every actor in the state diff --git a/api/api_gateway.go b/api/api_gateway.go index 398a7518f..3d2293ee1 100644 --- a/api/api_gateway.go +++ b/api/api_gateway.go @@ -4,6 +4,7 @@ import ( "context" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/types" "github.com/ipfs/go-cid" ) @@ -11,8 +12,10 @@ import ( type GatewayAPI interface { ChainHead(ctx context.Context) (*types.TipSet, error) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) + ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*MsgLookup, error) } diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 7ab50965a..b73379cfa 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -190,6 +190,7 @@ type FullNodeStruct struct { StateReadState func(context.Context, address.Address, types.TipSetKey) (*api.ActorState, error) `perm:"read"` StateMsgGasCost func(context.Context, cid.Cid, types.TipSetKey) (*api.MsgGasCost, error) `perm:"read"` StateWaitMsg func(ctx context.Context, cid cid.Cid, confidence uint64) (*api.MsgLookup, error) `perm:"read"` + StateWaitMsgLimited func(context.Context, cid.Cid, uint64, abi.ChainEpoch) (*api.MsgLookup, error) `perm:"read"` StateSearchMsg func(context.Context, cid.Cid) (*api.MsgLookup, error) `perm:"read"` StateListMiners func(context.Context, types.TipSetKey) ([]address.Address, error) `perm:"read"` StateListActors func(context.Context, types.TipSetKey) ([]address.Address, error) `perm:"read"` @@ -364,12 +365,14 @@ type WorkerStruct struct { type GatewayStruct struct { Internal struct { // TODO: does the gateway need perms? - ChainGetTipSet func(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) - ChainHead func(ctx context.Context) (*types.TipSet, error) - MpoolPush func(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) - StateAccountKey func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) - StateGetActor func(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) - StateLookupID func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + ChainGetTipSet func(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) + ChainGetTipSetByHeight func(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) + ChainHead func(ctx context.Context) (*types.TipSet, error) + MpoolPush func(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) + StateAccountKey func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + StateGetActor func(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) + StateLookupID func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + StateWaitMsg func(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error) } } @@ -866,6 +869,10 @@ func (c *FullNodeStruct) StateWaitMsg(ctx context.Context, msgc cid.Cid, confide return c.Internal.StateWaitMsg(ctx, msgc, confidence) } +func (c *FullNodeStruct) StateWaitMsgLimited(ctx context.Context, msgc cid.Cid, confidence uint64, limit abi.ChainEpoch) (*api.MsgLookup, error) { + return c.Internal.StateWaitMsgLimited(ctx, msgc, confidence, limit) +} + func (c *FullNodeStruct) StateSearchMsg(ctx context.Context, msgc cid.Cid) (*api.MsgLookup, error) { return c.Internal.StateSearchMsg(ctx, msgc) } @@ -1392,6 +1399,10 @@ func (g GatewayStruct) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) return g.Internal.ChainGetTipSet(ctx, tsk) } +func (g GatewayStruct) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) { + return g.Internal.ChainGetTipSetByHeight(ctx, h, tsk) +} + func (g GatewayStruct) MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) { return g.Internal.MpoolPush(ctx, sm) } @@ -1408,6 +1419,10 @@ func (g GatewayStruct) StateLookupID(ctx context.Context, addr address.Address, return g.Internal.StateLookupID(ctx, addr, tsk) } +func (g GatewayStruct) StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error) { + return g.Internal.StateWaitMsg(ctx, msg, confidence) +} + var _ api.Common = &CommonStruct{} var _ api.FullNode = &FullNodeStruct{} var _ api.StorageMiner = &StorageMinerStruct{} diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index d3129ea20..d6b6f4360 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -38,6 +38,8 @@ import ( "github.com/filecoin-project/lotus/chain/vm" ) +const LookbackNoLimit = abi.ChainEpoch(-1) + var log = logging.Logger("statemgr") type StateManagerAPI interface { @@ -514,7 +516,7 @@ func (sm *StateManager) GetReceipt(ctx context.Context, msg cid.Cid, ts *types.T return nil, fmt.Errorf("failed to load message: %w", err) } - _, r, _, err := sm.searchBackForMsg(ctx, ts, m) + _, r, _, err := sm.searchBackForMsg(ctx, ts, m, LookbackNoLimit) if err != nil { return nil, fmt.Errorf("failed to look back through chain for message: %w", err) } @@ -523,9 +525,9 @@ func (sm *StateManager) GetReceipt(ctx context.Context, msg cid.Cid, ts *types.T } // WaitForMessage blocks until a message appears on chain. It looks backwards in the chain to see if this has already -// happened. It guarantees that the message has been on chain for at least confidence epochs without being reverted -// before returning. -func (sm *StateManager) WaitForMessage(ctx context.Context, mcid cid.Cid, confidence uint64) (*types.TipSet, *types.MessageReceipt, cid.Cid, error) { +// happened, with an optional limit to how many epochs it will search. It guarantees that the message has been on +// chain for at least confidence epochs without being reverted before returning. +func (sm *StateManager) WaitForMessage(ctx context.Context, mcid cid.Cid, confidence uint64, lookbackLimit abi.ChainEpoch) (*types.TipSet, *types.MessageReceipt, cid.Cid, error) { ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -563,7 +565,7 @@ func (sm *StateManager) WaitForMessage(ctx context.Context, mcid cid.Cid, confid var backFm cid.Cid backSearchWait := make(chan struct{}) go func() { - fts, r, foundMsg, err := sm.searchBackForMsg(ctx, head[0].Val, msg) + fts, r, foundMsg, err := sm.searchBackForMsg(ctx, head[0].Val, msg, lookbackLimit) if err != nil { log.Warnf("failed to look back through chain for message: %w", err) return @@ -655,7 +657,7 @@ func (sm *StateManager) SearchForMessage(ctx context.Context, mcid cid.Cid) (*ty return head, r, foundMsg, nil } - fts, r, foundMsg, err := sm.searchBackForMsg(ctx, head, msg) + fts, r, foundMsg, err := sm.searchBackForMsg(ctx, head, msg, LookbackNoLimit) if err != nil { log.Warnf("failed to look back through chain for message %s", mcid) @@ -669,7 +671,15 @@ func (sm *StateManager) SearchForMessage(ctx context.Context, mcid cid.Cid) (*ty return fts, r, foundMsg, nil } -func (sm *StateManager) searchBackForMsg(ctx context.Context, from *types.TipSet, m types.ChainMsg) (*types.TipSet, *types.MessageReceipt, cid.Cid, error) { +// searchBackForMsg searches up to limit tipsets backwards from the given +// tipset for a message receipt. +// If limit is +// - 0 then no tipsets are searched +// - 5 then five tipset are searched +// - LookbackNoLimit then there is no limit +func (sm *StateManager) searchBackForMsg(ctx context.Context, from *types.TipSet, m types.ChainMsg, limit abi.ChainEpoch) (*types.TipSet, *types.MessageReceipt, cid.Cid, error) { + limitHeight := from.Height() - limit + noLimit := limit == LookbackNoLimit cur := from curActor, err := sm.LoadActor(ctx, m.VMMessage().From, cur) @@ -685,7 +695,9 @@ func (sm *StateManager) searchBackForMsg(ctx context.Context, from *types.TipSet mNonce := m.VMMessage().Nonce for { - if cur.Height() == 0 { + // If we've reached the genesis block, or we've reached the limit of + // how far back to look + if cur.Height() == 0 || !noLimit && cur.Height() <= limitHeight { // it ain't here! return nil, nil, cid.Undef, nil } diff --git a/cmd/lotus-gateway/api.go b/cmd/lotus-gateway/api.go index 5f8e9adf9..98c1a0fea 100644 --- a/cmd/lotus-gateway/api.go +++ b/cmd/lotus-gateway/api.go @@ -6,13 +6,18 @@ import ( "time" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/impl/full" "github.com/ipfs/go-cid" ) -const LookbackCap = time.Hour +const ( + LookbackCap = time.Hour + stateWaitLookbackLimit = abi.ChainEpoch(20) +) var ( ErrLookbackTooLong = fmt.Errorf("lookbacks of more than %s are disallowed", LookbackCap) @@ -22,26 +27,41 @@ type GatewayAPI struct { api api.FullNode } -func (a *GatewayAPI) getTipsetTimestamp(ctx context.Context, tsk types.TipSetKey) (time.Time, error) { +func (a *GatewayAPI) checkTipsetKey(ctx context.Context, tsk types.TipSetKey) error { if tsk.IsEmpty() { - return time.Now(), nil + return nil } ts, err := a.api.ChainGetTipSet(ctx, tsk) - if err != nil { - return time.Time{}, err - } - - return time.Unix(int64(ts.Blocks()[0].Timestamp), 0), nil -} - -func (a *GatewayAPI) checkTipset(ctx context.Context, ts types.TipSetKey) error { - when, err := a.getTipsetTimestamp(ctx, ts) if err != nil { return err } - if time.Since(when) > time.Hour { + return a.checkTipset(ts) +} + +func (a *GatewayAPI) checkTipset(ts *types.TipSet) error { + at := time.Unix(int64(ts.Blocks()[0].Timestamp), 0) + if err := a.checkTimestamp(at); err != nil { + return fmt.Errorf("bad tipset: %w", err) + } + return nil +} + +// TODO: write tests for this check +func (a *GatewayAPI) checkTipsetHeight(ts *types.TipSet, h abi.ChainEpoch) error { + tsBlock := ts.Blocks()[0] + heightDelta := time.Duration(uint64(tsBlock.Height-h)*build.BlockDelaySecs) * time.Second + timeAtHeight := time.Unix(int64(tsBlock.Timestamp), 0).Add(-heightDelta) + + if err := a.checkTimestamp(timeAtHeight); err != nil { + return fmt.Errorf("bad tipset height: %w", err) + } + return nil +} + +func (a *GatewayAPI) checkTimestamp(at time.Time) error { + if time.Since(at) > LookbackCap { return ErrLookbackTooLong } @@ -55,12 +75,26 @@ func (a *GatewayAPI) ChainHead(ctx context.Context) (*types.TipSet, error) { } func (a *GatewayAPI) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) { - if err := a.checkTipset(ctx, tsk); err != nil { - return nil, fmt.Errorf("bad tipset: %w", err) + return a.api.ChainGetTipSet(ctx, tsk) +} + +func (a *GatewayAPI) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) { + ts, err := a.api.ChainGetTipSet(ctx, tsk) + if err != nil { + return nil, err } - // TODO: since we're limiting lookbacks, should just cache this (could really even cache the json response bytes) - return a.api.ChainGetTipSet(ctx, tsk) + // Check if the tipset key refers to a tipset that's too far in the past + if err := a.checkTipset(ts); err != nil { + return nil, err + } + + // Check if the height is too far in the past + if err := a.checkTipsetHeight(ts, h); err != nil { + return nil, err + } + + return a.api.ChainGetTipSetByHeight(ctx, h, tsk) } func (a *GatewayAPI) MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) { @@ -70,30 +104,34 @@ func (a *GatewayAPI) MpoolPush(ctx context.Context, sm *types.SignedMessage) (ci } func (a *GatewayAPI) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { - if err := a.checkTipset(ctx, tsk); err != nil { - return address.Undef, fmt.Errorf("bad tipset: %w", err) + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return address.Undef, err } return a.api.StateAccountKey(ctx, addr, tsk) } func (a *GatewayAPI) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) { - if err := a.checkTipset(ctx, tsk); err != nil { - return nil, fmt.Errorf("bad tipset: %w", err) + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return nil, err } return a.api.StateGetActor(ctx, actor, tsk) } func (a *GatewayAPI) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { - if err := a.checkTipset(ctx, tsk); err != nil { - return address.Undef, fmt.Errorf("bad tipset: %w", err) + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return address.Undef, err } return a.api.StateLookupID(ctx, addr, tsk) } -var _ api.GatewayAPI = &GatewayAPI{} +func (a *GatewayAPI) StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error) { + return a.api.StateWaitMsgLimited(ctx, msg, confidence, stateWaitLookbackLimit) +} + +var _ api.GatewayAPI = (*GatewayAPI)(nil) var _ full.ChainModuleAPI = (*GatewayAPI)(nil) var _ full.MpoolModuleAPI = (*GatewayAPI)(nil) var _ full.StateModuleAPI = (*GatewayAPI)(nil) diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index c05317458..04463a839 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -244,7 +244,7 @@ var DaemonCmd = &cli.Command{ shutdownChan := make(chan struct{}) - // If the daemon is started in "lite mode", replace the StateManager + // If the daemon is started in "lite mode", replace key APIs // with a thin client to a gateway server liteMode := node.Options() isLite := cctx.Bool("lite") diff --git a/markets/storageadapter/client.go b/markets/storageadapter/client.go index 411c86ec9..647a1675d 100644 --- a/markets/storageadapter/client.go +++ b/markets/storageadapter/client.go @@ -187,7 +187,7 @@ func (c *ClientNodeAdapter) ValidatePublishedDeal(ctx context.Context, deal stor } // TODO: timeout - _, ret, _, err := c.sm.WaitForMessage(ctx, *deal.PublishMessage, build.MessageConfidence) + _, ret, _, err := c.sm.WaitForMessage(ctx, *deal.PublishMessage, build.MessageConfidence, stmgr.LookbackNoLimit) if err != nil { return 0, xerrors.Errorf("waiting for deal publish message: %w", err) } diff --git a/node/impl/full/chain.go b/node/impl/full/chain.go index 88b84cad2..5b4f41114 100644 --- a/node/impl/full/chain.go +++ b/node/impl/full/chain.go @@ -42,6 +42,7 @@ var log = logging.Logger("fullnode") type ChainModuleAPI interface { ChainHead(context.Context) (*types.TipSet, error) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) + ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) } // ChainModule provides a default implementation of ChainModuleAPI. @@ -197,12 +198,12 @@ func (a *ChainAPI) ChainGetParentReceipts(ctx context.Context, bcid cid.Cid) ([] return out, nil } -func (a *ChainAPI) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) { - ts, err := a.Chain.GetTipSetFromKey(tsk) +func (m *ChainModule) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) { + ts, err := m.Chain.GetTipSetFromKey(tsk) if err != nil { return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - return a.Chain.GetTipsetByHeight(ctx, h, ts, true) + return m.Chain.GetTipsetByHeight(ctx, h, ts, true) } func (a *ChainAPI) ChainReadObj(ctx context.Context, obj cid.Cid) ([]byte, error) { diff --git a/node/impl/full/state.go b/node/impl/full/state.go index a391dffa1..540d959ed 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -46,6 +46,7 @@ type StateModuleAPI interface { StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error) } // StateModule provides a default implementation of StateModuleAPI. @@ -475,22 +476,28 @@ func (a *StateAPI) MinerCreateBlock(ctx context.Context, bt *api.BlockTemplate) return &out, nil } -func (a *StateAPI) StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error) { - ts, recpt, found, err := a.StateManager.WaitForMessage(ctx, msg, confidence) +func (m *StateModule) StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error) { + return stateWaitMsgLimited(ctx, m.StateManager, m.Chain, msg, confidence, stmgr.LookbackNoLimit) +} +func (a *StateAPI) StateWaitMsgLimited(ctx context.Context, msg cid.Cid, confidence uint64, lookbackLimit abi.ChainEpoch) (*api.MsgLookup, error) { + return stateWaitMsgLimited(ctx, a.StateManager, a.Chain, msg, confidence, lookbackLimit) +} +func stateWaitMsgLimited(ctx context.Context, smgr *stmgr.StateManager, cstore *store.ChainStore, msg cid.Cid, confidence uint64, lookbackLimit abi.ChainEpoch) (*api.MsgLookup, error) { + ts, recpt, found, err := smgr.WaitForMessage(ctx, msg, confidence, lookbackLimit) if err != nil { return nil, err } var returndec interface{} if recpt.ExitCode == 0 && len(recpt.Return) > 0 { - cmsg, err := a.Chain.GetCMessage(msg) + cmsg, err := cstore.GetCMessage(msg) if err != nil { return nil, xerrors.Errorf("failed to load message after successful receipt search: %w", err) } vmsg := cmsg.VMMessage() - t, err := stmgr.GetReturnType(ctx, a.StateManager, vmsg.To, vmsg.Method, ts) + t, err := stmgr.GetReturnType(ctx, smgr, vmsg.To, vmsg.Method, ts) if err != nil { return nil, xerrors.Errorf("failed to get return type: %w", err) } From b32d25c562875a972714fd29c64bea3aaee636a1 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Mon, 5 Oct 2020 17:09:21 +0200 Subject: [PATCH 658/885] feat: add RPC for GasEstimateMessageGas --- api/api_gateway.go | 1 + api/apistruct/struct.go | 5 ++ cmd/lotus-gateway/api.go | 9 ++++ cmd/lotus/daemon.go | 1 + node/builder.go | 3 +- node/impl/full/gas.go | 103 +++++++++++++++++++++++++++++++-------- 6 files changed, 100 insertions(+), 22 deletions(-) diff --git a/api/api_gateway.go b/api/api_gateway.go index 3d2293ee1..8c06304e9 100644 --- a/api/api_gateway.go +++ b/api/api_gateway.go @@ -13,6 +13,7 @@ type GatewayAPI interface { ChainHead(ctx context.Context) (*types.TipSet, error) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) + GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index b73379cfa..38d177aec 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -368,6 +368,7 @@ type GatewayStruct struct { ChainGetTipSet func(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) ChainGetTipSetByHeight func(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) ChainHead func(ctx context.Context) (*types.TipSet, error) + GasEstimateMessageGas func(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) MpoolPush func(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) StateAccountKey func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) StateGetActor func(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) @@ -1403,6 +1404,10 @@ func (g GatewayStruct) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEp return g.Internal.ChainGetTipSetByHeight(ctx, h, tsk) } +func (g GatewayStruct) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) { + return g.Internal.GasEstimateMessageGas(ctx, msg, spec, tsk) +} + func (g GatewayStruct) MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) { return g.Internal.MpoolPush(ctx, sm) } diff --git a/cmd/lotus-gateway/api.go b/cmd/lotus-gateway/api.go index 98c1a0fea..8192c58ec 100644 --- a/cmd/lotus-gateway/api.go +++ b/cmd/lotus-gateway/api.go @@ -111,6 +111,14 @@ func (a *GatewayAPI) StateAccountKey(ctx context.Context, addr address.Address, return a.api.StateAccountKey(ctx, addr, tsk) } +func (a *GatewayAPI) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return nil, err + } + + return a.api.GasEstimateMessageGas(ctx, msg, spec, tsk) +} + func (a *GatewayAPI) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) { if err := a.checkTipsetKey(ctx, tsk); err != nil { return nil, err @@ -133,5 +141,6 @@ func (a *GatewayAPI) StateWaitMsg(ctx context.Context, msg cid.Cid, confidence u var _ api.GatewayAPI = (*GatewayAPI)(nil) var _ full.ChainModuleAPI = (*GatewayAPI)(nil) +var _ full.GasModuleAPI = (*GatewayAPI)(nil) var _ full.MpoolModuleAPI = (*GatewayAPI)(nil) var _ full.StateModuleAPI = (*GatewayAPI)(nil) diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 04463a839..495979ea9 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -259,6 +259,7 @@ var DaemonCmd = &cli.Command{ liteMode = node.Options( node.Override(new(api.GatewayAPI), gapi), node.Override(new(full.ChainModuleAPI), node.From(new(api.GatewayAPI))), + node.Override(new(full.GasModuleAPI), node.From(new(api.GatewayAPI))), node.Override(new(full.MpoolModuleAPI), node.From(new(api.GatewayAPI))), node.Override(new(full.StateModuleAPI), node.From(new(api.GatewayAPI))), node.Override(new(stmgr.StateManagerAPI), modules.NewRPCStateManager), diff --git a/node/builder.go b/node/builder.go index d513fe626..101ec3dbc 100644 --- a/node/builder.go +++ b/node/builder.go @@ -267,8 +267,9 @@ func Online() Option { Override(new(*messagesigner.MessageSigner), messagesigner.NewMessageSigner), Override(new(full.ChainModuleAPI), From(new(full.ChainModule))), - Override(new(full.StateModuleAPI), From(new(full.StateModule))), + Override(new(full.GasModuleAPI), From(new(full.GasModule))), Override(new(full.MpoolModuleAPI), From(new(full.MpoolModule))), + Override(new(full.StateModuleAPI), From(new(full.StateModule))), Override(new(stmgr.StateManagerAPI), From(new(*stmgr.StateManager))), Override(new(dtypes.ChainGCLocker), blockstore.NewGCLocker), diff --git a/node/impl/full/gas.go b/node/impl/full/gas.go index 3580ca26d..0cb1eb084 100644 --- a/node/impl/full/gas.go +++ b/node/impl/full/gas.go @@ -26,8 +26,27 @@ import ( "github.com/filecoin-project/lotus/chain/types" ) +type GasModuleAPI interface { + GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) +} + +// GasModule provides a default implementation of GasModuleAPI. +// It can be swapped out with another implementation through Dependency +// Injection (for example with a thin RPC client). +type GasModule struct { + fx.In + Stmgr *stmgr.StateManager + Chain *store.ChainStore + Mpool *messagepool.MessagePool +} + +var _ GasModuleAPI = (*GasModule)(nil) + type GasAPI struct { fx.In + + GasModuleAPI + Stmgr *stmgr.StateManager Chain *store.ChainStore Mpool *messagepool.MessagePool @@ -36,9 +55,24 @@ type GasAPI struct { const MinGasPremium = 100e3 const MaxSpendOnFeeDenom = 100 -func (a *GasAPI) GasEstimateFeeCap(ctx context.Context, msg *types.Message, maxqueueblks int64, - tsk types.TipSetKey) (types.BigInt, error) { - ts := a.Chain.GetHeaviestTipSet() +func (a *GasAPI) GasEstimateFeeCap( + ctx context.Context, + msg *types.Message, + maxqueueblks int64, + tsk types.TipSetKey, +) (types.BigInt, error) { + return gasEstimateFeeCap(a.Chain, msg, maxqueueblks) +} +func (m *GasModule) GasEstimateFeeCap( + ctx context.Context, + msg *types.Message, + maxqueueblks int64, + tsk types.TipSetKey, +) (types.BigInt, error) { + return gasEstimateFeeCap(m.Chain, msg, maxqueueblks) +} +func gasEstimateFeeCap(cstore *store.ChainStore, msg *types.Message, maxqueueblks int64) (types.BigInt, error) { + ts := cstore.GetHeaviestTipSet() parentBaseFee := ts.Blocks()[0].ParentBaseFee increaseFactor := math.Pow(1.+1./float64(build.BaseFeeMaxChangeDenom), float64(maxqueueblks)) @@ -82,9 +116,25 @@ func medianGasPremium(prices []gasMeta, blocks int) abi.TokenAmount { return premium } -func (a *GasAPI) GasEstimateGasPremium(ctx context.Context, nblocksincl uint64, - sender address.Address, gaslimit int64, _ types.TipSetKey) (types.BigInt, error) { - +func (a *GasAPI) GasEstimateGasPremium( + ctx context.Context, + nblocksincl uint64, + sender address.Address, + gaslimit int64, + _ types.TipSetKey, +) (types.BigInt, error) { + return gasEstimateGasPremium(a.Chain, nblocksincl) +} +func (m *GasModule) GasEstimateGasPremium( + ctx context.Context, + nblocksincl uint64, + sender address.Address, + gaslimit int64, + _ types.TipSetKey, +) (types.BigInt, error) { + return gasEstimateGasPremium(m.Chain, nblocksincl) +} +func gasEstimateGasPremium(cstore *store.ChainStore, nblocksincl uint64) (types.BigInt, error) { if nblocksincl == 0 { nblocksincl = 1 } @@ -92,20 +142,20 @@ func (a *GasAPI) GasEstimateGasPremium(ctx context.Context, nblocksincl uint64, var prices []gasMeta var blocks int - ts := a.Chain.GetHeaviestTipSet() + ts := cstore.GetHeaviestTipSet() for i := uint64(0); i < nblocksincl*2; i++ { if ts.Height() == 0 { break // genesis } - pts, err := a.Chain.LoadTipSet(ts.Parents()) + pts, err := cstore.LoadTipSet(ts.Parents()) if err != nil { return types.BigInt{}, err } blocks += len(pts.Blocks()) - msgs, err := a.Chain.MessagesForTipset(pts) + msgs, err := cstore.MessagesForTipset(pts) if err != nil { return types.BigInt{}, xerrors.Errorf("loading messages: %w", err) } @@ -142,19 +192,30 @@ func (a *GasAPI) GasEstimateGasPremium(ctx context.Context, nblocksincl uint64, } func (a *GasAPI) GasEstimateGasLimit(ctx context.Context, msgIn *types.Message, _ types.TipSetKey) (int64, error) { - + return gasEstimateGasLimit(ctx, a.Chain, a.Stmgr, a.Mpool, msgIn) +} +func (m *GasModule) GasEstimateGasLimit(ctx context.Context, msgIn *types.Message, _ types.TipSetKey) (int64, error) { + return gasEstimateGasLimit(ctx, m.Chain, m.Stmgr, m.Mpool, msgIn) +} +func gasEstimateGasLimit( + ctx context.Context, + cstore *store.ChainStore, + smgr *stmgr.StateManager, + mpool *messagepool.MessagePool, + msgIn *types.Message, +) (int64, error) { msg := *msgIn msg.GasLimit = build.BlockGasLimit msg.GasFeeCap = types.NewInt(uint64(build.MinimumBaseFee) + 1) msg.GasPremium = types.NewInt(1) - currTs := a.Chain.GetHeaviestTipSet() - fromA, err := a.Stmgr.ResolveToKeyAddress(ctx, msgIn.From, currTs) + currTs := cstore.GetHeaviestTipSet() + fromA, err := smgr.ResolveToKeyAddress(ctx, msgIn.From, currTs) if err != nil { return -1, xerrors.Errorf("getting key address: %w", err) } - pending, ts := a.Mpool.PendingFor(fromA) + pending, ts := mpool.PendingFor(fromA) priorMsgs := make([]types.ChainMsg, 0, len(pending)) for _, m := range pending { priorMsgs = append(priorMsgs, m) @@ -163,11 +224,11 @@ func (a *GasAPI) GasEstimateGasLimit(ctx context.Context, msgIn *types.Message, // Try calling until we find a height with no migration. var res *api.InvocResult for { - res, err = a.Stmgr.CallWithGas(ctx, &msg, priorMsgs, ts) + res, err = smgr.CallWithGas(ctx, &msg, priorMsgs, ts) if err != stmgr.ErrExpensiveFork { break } - ts, err = a.Chain.GetTipSetFromKey(ts.Parents()) + ts, err = cstore.GetTipSetFromKey(ts.Parents()) if err != nil { return -1, xerrors.Errorf("getting parent tipset: %w", err) } @@ -180,7 +241,7 @@ func (a *GasAPI) GasEstimateGasLimit(ctx context.Context, msgIn *types.Message, } // Special case for PaymentChannel collect, which is deleting actor - st, err := a.Stmgr.ParentState(ts) + st, err := smgr.ParentState(ts) if err != nil { _ = err // somewhat ignore it as it can happen and we just want to detect @@ -206,17 +267,17 @@ func (a *GasAPI) GasEstimateGasLimit(ctx context.Context, msgIn *types.Message, return res.MsgRct.GasUsed + 76e3, nil } -func (a *GasAPI) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, _ types.TipSetKey) (*types.Message, error) { +func (m *GasModule) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, _ types.TipSetKey) (*types.Message, error) { if msg.GasLimit == 0 { - gasLimit, err := a.GasEstimateGasLimit(ctx, msg, types.TipSetKey{}) + gasLimit, err := m.GasEstimateGasLimit(ctx, msg, types.TipSetKey{}) if err != nil { return nil, xerrors.Errorf("estimating gas used: %w", err) } - msg.GasLimit = int64(float64(gasLimit) * a.Mpool.GetConfig().GasLimitOverestimation) + msg.GasLimit = int64(float64(gasLimit) * m.Mpool.GetConfig().GasLimitOverestimation) } if msg.GasPremium == types.EmptyInt || types.BigCmp(msg.GasPremium, types.NewInt(0)) == 0 { - gasPremium, err := a.GasEstimateGasPremium(ctx, 2, msg.From, msg.GasLimit, types.TipSetKey{}) + gasPremium, err := m.GasEstimateGasPremium(ctx, 2, msg.From, msg.GasLimit, types.TipSetKey{}) if err != nil { return nil, xerrors.Errorf("estimating gas price: %w", err) } @@ -224,7 +285,7 @@ func (a *GasAPI) GasEstimateMessageGas(ctx context.Context, msg *types.Message, } if msg.GasFeeCap == types.EmptyInt || types.BigCmp(msg.GasFeeCap, types.NewInt(0)) == 0 { - feeCap, err := a.GasEstimateFeeCap(ctx, msg, 20, types.EmptyTSK) + feeCap, err := m.GasEstimateFeeCap(ctx, msg, 20, types.EmptyTSK) if err != nil { return nil, xerrors.Errorf("estimating fee cap: %w", err) } From 7b1bec91ed10aeeea3e93c2801edcaed574592d2 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Tue, 6 Oct 2020 11:03:27 +0200 Subject: [PATCH 659/885] feat: add tests for gateway ChainGetTipsetByHeight --- chain/types/mock/chain.go | 3 ++ cmd/lotus-gateway/api.go | 60 +++++++++++++++++++++++---------------- cmd/lotus-gateway/main.go | 2 +- 3 files changed, 39 insertions(+), 26 deletions(-) diff --git a/chain/types/mock/chain.go b/chain/types/mock/chain.go index 559630619..85437079c 100644 --- a/chain/types/mock/chain.go +++ b/chain/types/mock/chain.go @@ -59,9 +59,11 @@ func MkBlock(parents *types.TipSet, weightInc uint64, ticketNonce uint64) *types var pcids []cid.Cid var height abi.ChainEpoch weight := types.NewInt(weightInc) + var timestamp uint64 if parents != nil { pcids = parents.Cids() height = parents.Height() + 1 + timestamp = parents.MinTimestamp() + build.BlockDelaySecs weight = types.BigAdd(parents.Blocks()[0].ParentWeight, weight) } @@ -79,6 +81,7 @@ func MkBlock(parents *types.TipSet, weightInc uint64, ticketNonce uint64) *types ParentWeight: weight, Messages: c, Height: height, + Timestamp: timestamp, ParentStateRoot: pstateRoot, BlockSig: &crypto.Signature{Type: crypto.SigTypeBLS, Data: []byte("boo! im a signature")}, ParentBaseFee: types.NewInt(uint64(build.MinimumBaseFee)), diff --git a/cmd/lotus-gateway/api.go b/cmd/lotus-gateway/api.go index 8192c58ec..0400a4a30 100644 --- a/cmd/lotus-gateway/api.go +++ b/cmd/lotus-gateway/api.go @@ -23,8 +23,20 @@ var ( ErrLookbackTooLong = fmt.Errorf("lookbacks of more than %s are disallowed", LookbackCap) ) +type rpcAPI interface { + ChainHead(ctx context.Context) (*types.TipSet, error) + ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) + ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) + GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) + MpoolPushUntrusted(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) + StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) + StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + StateWaitMsgLimited(ctx context.Context, msg cid.Cid, confidence uint64, h abi.ChainEpoch) (*api.MsgLookup, error) +} + type GatewayAPI struct { - api api.FullNode + rpc rpcAPI } func (a *GatewayAPI) checkTipsetKey(ctx context.Context, tsk types.TipSetKey) error { @@ -32,7 +44,7 @@ func (a *GatewayAPI) checkTipsetKey(ctx context.Context, tsk types.TipSetKey) er return nil } - ts, err := a.api.ChainGetTipSet(ctx, tsk) + ts, err := a.rpc.ChainGetTipSet(ctx, tsk) if err != nil { return err } @@ -48,7 +60,6 @@ func (a *GatewayAPI) checkTipset(ts *types.TipSet) error { return nil } -// TODO: write tests for this check func (a *GatewayAPI) checkTipsetHeight(ts *types.TipSet, h abi.ChainEpoch) error { tsBlock := ts.Blocks()[0] heightDelta := time.Duration(uint64(tsBlock.Height-h)*build.BlockDelaySecs) * time.Second @@ -71,15 +82,15 @@ func (a *GatewayAPI) checkTimestamp(at time.Time) error { func (a *GatewayAPI) ChainHead(ctx context.Context) (*types.TipSet, error) { // TODO: cache and invalidate cache when timestamp is up (or have internal ChainNotify) - return a.api.ChainHead(ctx) + return a.rpc.ChainHead(ctx) } func (a *GatewayAPI) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) { - return a.api.ChainGetTipSet(ctx, tsk) + return a.rpc.ChainGetTipSet(ctx, tsk) } func (a *GatewayAPI) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) { - ts, err := a.api.ChainGetTipSet(ctx, tsk) + ts, err := a.rpc.ChainGetTipSet(ctx, tsk) if err != nil { return nil, err } @@ -94,21 +105,7 @@ func (a *GatewayAPI) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoc return nil, err } - return a.api.ChainGetTipSetByHeight(ctx, h, tsk) -} - -func (a *GatewayAPI) MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) { - // TODO: additional anti-spam checks - - return a.api.MpoolPushUntrusted(ctx, sm) -} - -func (a *GatewayAPI) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { - if err := a.checkTipsetKey(ctx, tsk); err != nil { - return address.Undef, err - } - - return a.api.StateAccountKey(ctx, addr, tsk) + return a.rpc.ChainGetTipSetByHeight(ctx, h, tsk) } func (a *GatewayAPI) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) { @@ -116,7 +113,20 @@ func (a *GatewayAPI) GasEstimateMessageGas(ctx context.Context, msg *types.Messa return nil, err } - return a.api.GasEstimateMessageGas(ctx, msg, spec, tsk) + return a.rpc.GasEstimateMessageGas(ctx, msg, spec, tsk) +} + +func (a *GatewayAPI) MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) { + // TODO: additional anti-spam checks + return a.rpc.MpoolPushUntrusted(ctx, sm) +} + +func (a *GatewayAPI) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return address.Undef, err + } + + return a.rpc.StateAccountKey(ctx, addr, tsk) } func (a *GatewayAPI) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) { @@ -124,7 +134,7 @@ func (a *GatewayAPI) StateGetActor(ctx context.Context, actor address.Address, t return nil, err } - return a.api.StateGetActor(ctx, actor, tsk) + return a.rpc.StateGetActor(ctx, actor, tsk) } func (a *GatewayAPI) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { @@ -132,11 +142,11 @@ func (a *GatewayAPI) StateLookupID(ctx context.Context, addr address.Address, ts return address.Undef, err } - return a.api.StateLookupID(ctx, addr, tsk) + return a.rpc.StateLookupID(ctx, addr, tsk) } func (a *GatewayAPI) StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error) { - return a.api.StateWaitMsgLimited(ctx, msg, confidence, stateWaitLookbackLimit) + return a.rpc.StateWaitMsgLimited(ctx, msg, confidence, stateWaitLookbackLimit) } var _ api.GatewayAPI = (*GatewayAPI)(nil) diff --git a/cmd/lotus-gateway/main.go b/cmd/lotus-gateway/main.go index c19599084..3c8eb4019 100644 --- a/cmd/lotus-gateway/main.go +++ b/cmd/lotus-gateway/main.go @@ -76,7 +76,7 @@ var runCmd = &cli.Command{ log.Info("Setting up API endpoint at " + address) rpcServer := jsonrpc.NewServer() - rpcServer.Register("Filecoin", &GatewayAPI{api: api}) + rpcServer.Register("Filecoin", &GatewayAPI{rpc: api}) mux.Handle("/rpc/v0", rpcServer) mux.PathPrefix("/").Handler(http.DefaultServeMux) From ef73b964fb40a916b921e484ee65111080e12de5 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Wed, 7 Oct 2020 11:26:15 +0200 Subject: [PATCH 660/885] feat: add end-to-end test for lite mode --- api/test/ccupgrade.go | 2 +- api/test/deals.go | 8 +- api/test/mining.go | 6 +- api/test/paych.go | 2 +- api/test/test.go | 22 ++- api/test/window_post.go | 4 +- chain/messagesigner/messagesigner.go | 11 +- chain/messagesigner/messagesigner_test.go | 2 +- cli/paych_test.go | 2 +- cmd/lotus-gateway/api_test.go | 183 ++++++++++++++++++++++ cmd/lotus-gateway/endtoend_test.go | 140 +++++++++++++++++ cmd/lotus-storage-miner/allinfo_test.go | 4 +- cmd/lotus/daemon.go | 16 +- node/builder.go | 18 +++ node/impl/full/mpool.go | 2 +- node/modules/mpoolnonceapi.go | 33 ++++ node/test/builder.go | 154 +++++++++++------- 17 files changed, 510 insertions(+), 99 deletions(-) create mode 100644 cmd/lotus-gateway/api_test.go create mode 100644 cmd/lotus-gateway/endtoend_test.go create mode 100644 node/modules/mpoolnonceapi.go diff --git a/api/test/ccupgrade.go b/api/test/ccupgrade.go index 97fb665ed..38342d7fe 100644 --- a/api/test/ccupgrade.go +++ b/api/test/ccupgrade.go @@ -37,7 +37,7 @@ func TestCCUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration) { func testCCUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration, upgradeHeight abi.ChainEpoch) { ctx := context.Background() - n, sn := b(t, 1, OneMiner, node.Override(new(stmgr.UpgradeSchedule), stmgr.UpgradeSchedule{{ + n, sn := b(t, OneFull, OneMiner, node.Override(new(stmgr.UpgradeSchedule), stmgr.UpgradeSchedule{{ Network: build.ActorUpgradeNetworkVersion, Height: upgradeHeight, Migration: stmgr.UpgradeActorsV2, diff --git a/api/test/deals.go b/api/test/deals.go index aa5bfa716..8b4a7fe8b 100644 --- a/api/test/deals.go +++ b/api/test/deals.go @@ -48,7 +48,7 @@ func TestDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration, carExport _ = os.Setenv("BELLMAN_NO_GPU", "1") ctx := context.Background() - n, sn := b(t, 1, OneMiner) + n, sn := b(t, OneFull, OneMiner) client := n[0].FullNode.(*impl.FullNodeAPI) miner := sn[0] @@ -85,7 +85,7 @@ func TestDoubleDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration) { _ = os.Setenv("BELLMAN_NO_GPU", "1") ctx := context.Background() - n, sn := b(t, 1, OneMiner) + n, sn := b(t, OneFull, OneMiner) client := n[0].FullNode.(*impl.FullNodeAPI) miner := sn[0] @@ -149,7 +149,7 @@ func TestFastRetrievalDealFlow(t *testing.T, b APIBuilder, blocktime time.Durati _ = os.Setenv("BELLMAN_NO_GPU", "1") ctx := context.Background() - n, sn := b(t, 1, OneMiner) + n, sn := b(t, OneFull, OneMiner) client := n[0].FullNode.(*impl.FullNodeAPI) miner := sn[0] @@ -204,7 +204,7 @@ func TestSenondDealRetrieval(t *testing.T, b APIBuilder, blocktime time.Duration _ = os.Setenv("BELLMAN_NO_GPU", "1") ctx := context.Background() - n, sn := b(t, 1, OneMiner) + n, sn := b(t, OneFull, OneMiner) client := n[0].FullNode.(*impl.FullNodeAPI) miner := sn[0] diff --git a/api/test/mining.go b/api/test/mining.go index e19774a76..8147c224b 100644 --- a/api/test/mining.go +++ b/api/test/mining.go @@ -25,7 +25,7 @@ var log = logging.Logger("apitest") func (ts *testSuite) testMining(t *testing.T) { ctx := context.Background() - apis, sn := ts.makeNodes(t, 1, OneMiner) + apis, sn := ts.makeNodes(t, OneFull, OneMiner) api := apis[0] newHeads, err := api.ChainNotify(ctx) @@ -54,7 +54,7 @@ func (ts *testSuite) testMiningReal(t *testing.T) { }() ctx := context.Background() - apis, sn := ts.makeNodes(t, 1, OneMiner) + apis, sn := ts.makeNodes(t, OneFull, OneMiner) api := apis[0] newHeads, err := api.ChainNotify(ctx) @@ -93,7 +93,7 @@ func TestDealMining(t *testing.T, b APIBuilder, blocktime time.Duration, carExpo // test making a deal with a fresh miner, and see if it starts to mine ctx := context.Background() - n, sn := b(t, 1, []StorageMiner{ + n, sn := b(t, OneFull, []StorageMiner{ {Full: 0, Preseal: PresealGenesis}, {Full: 0, Preseal: 0}, // TODO: Add support for miners on non-first full node }) diff --git a/api/test/paych.go b/api/test/paych.go index e95773b6a..a8ccebdde 100644 --- a/api/test/paych.go +++ b/api/test/paych.go @@ -33,7 +33,7 @@ func TestPaymentChannels(t *testing.T, b APIBuilder, blocktime time.Duration) { _ = os.Setenv("BELLMAN_NO_GPU", "1") ctx := context.Background() - n, sn := b(t, 2, OneMiner) + n, sn := b(t, TwoFull, OneMiner) paymentCreator := n[0] paymentReceiver := n[1] diff --git a/api/test/test.go b/api/test/test.go index 853267eff..63e50bd73 100644 --- a/api/test/test.go +++ b/api/test/test.go @@ -40,12 +40,14 @@ type StorageMiner struct { Preseal int } +type OptionGenerator func([]TestNode) node.Option + // APIBuilder is a function which is invoked in test suite to provide // test nodes and networks // // storage array defines storage nodes, numbers in the array specify full node // index the storage node 'belongs' to -type APIBuilder func(t *testing.T, nFull int, storage []StorageMiner, opts ...node.Option) ([]TestNode, []TestStorageNode) +type APIBuilder func(t *testing.T, full []OptionGenerator, storage []StorageMiner, opts ...node.Option) ([]TestNode, []TestStorageNode) type testSuite struct { makeNodes APIBuilder } @@ -63,13 +65,25 @@ func TestApis(t *testing.T, b APIBuilder) { t.Run("testMiningReal", ts.testMiningReal) } +func DefaultFullOpts(nFull int) []OptionGenerator { + full := make([]OptionGenerator, nFull) + for i := range full { + full[i] = func(nodes []TestNode) node.Option { + return node.Options() + } + } + return full +} + var OneMiner = []StorageMiner{{Full: 0, Preseal: PresealGenesis}} +var OneFull = DefaultFullOpts(1) +var TwoFull = DefaultFullOpts(2) func (ts *testSuite) testVersion(t *testing.T) { build.RunningNodeType = build.NodeFull ctx := context.Background() - apis, _ := ts.makeNodes(t, 1, OneMiner) + apis, _ := ts.makeNodes(t, OneFull, OneMiner) api := apis[0] v, err := api.Version(ctx) @@ -81,7 +95,7 @@ func (ts *testSuite) testVersion(t *testing.T) { func (ts *testSuite) testID(t *testing.T) { ctx := context.Background() - apis, _ := ts.makeNodes(t, 1, OneMiner) + apis, _ := ts.makeNodes(t, OneFull, OneMiner) api := apis[0] id, err := api.ID(ctx) @@ -93,7 +107,7 @@ func (ts *testSuite) testID(t *testing.T) { func (ts *testSuite) testConnectTwo(t *testing.T) { ctx := context.Background() - apis, _ := ts.makeNodes(t, 2, OneMiner) + apis, _ := ts.makeNodes(t, TwoFull, OneMiner) p, err := apis[0].NetPeers(ctx) if err != nil { diff --git a/api/test/window_post.go b/api/test/window_post.go index eadcdbb05..c45d5e1bc 100644 --- a/api/test/window_post.go +++ b/api/test/window_post.go @@ -34,7 +34,7 @@ func init() { func TestPledgeSector(t *testing.T, b APIBuilder, blocktime time.Duration, nSectors int) { ctx := context.Background() - n, sn := b(t, 1, OneMiner) + n, sn := b(t, OneFull, OneMiner) client := n[0].FullNode.(*impl.FullNodeAPI) miner := sn[0] @@ -133,7 +133,7 @@ func testWindowPostUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration, ctx, cancel := context.WithCancel(context.Background()) defer cancel() - n, sn := b(t, 1, OneMiner, node.Override(new(stmgr.UpgradeSchedule), stmgr.UpgradeSchedule{{ + n, sn := b(t, OneFull, OneMiner, node.Override(new(stmgr.UpgradeSchedule), stmgr.UpgradeSchedule{{ Network: build.ActorUpgradeNetworkVersion, Height: upgradeHeight, Migration: stmgr.UpgradeActorsV2, diff --git a/chain/messagesigner/messagesigner.go b/chain/messagesigner/messagesigner.go index ac94d6a3e..1fe8f9565 100644 --- a/chain/messagesigner/messagesigner.go +++ b/chain/messagesigner/messagesigner.go @@ -6,7 +6,6 @@ import ( "sync" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/messagepool" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/wallet" "github.com/filecoin-project/lotus/node/modules/dtypes" @@ -21,7 +20,7 @@ const dsKeyActorNonce = "ActorNextNonce" var log = logging.Logger("messagesigner") -type mpoolAPI interface { +type MpoolNonceAPI interface { GetNonce(address.Address) (uint64, error) } @@ -30,15 +29,11 @@ type mpoolAPI interface { type MessageSigner struct { wallet *wallet.Wallet lk sync.Mutex - mpool mpoolAPI + mpool MpoolNonceAPI ds datastore.Batching } -func NewMessageSigner(wallet *wallet.Wallet, mpool *messagepool.MessagePool, ds dtypes.MetadataDS) *MessageSigner { - return newMessageSigner(wallet, mpool, ds) -} - -func newMessageSigner(wallet *wallet.Wallet, mpool mpoolAPI, ds dtypes.MetadataDS) *MessageSigner { +func NewMessageSigner(wallet *wallet.Wallet, mpool MpoolNonceAPI, ds dtypes.MetadataDS) *MessageSigner { ds = namespace.Wrap(ds, datastore.NewKey("/message-signer/")) return &MessageSigner{ wallet: wallet, diff --git a/chain/messagesigner/messagesigner_test.go b/chain/messagesigner/messagesigner_test.go index 04869ff6d..de74b8a48 100644 --- a/chain/messagesigner/messagesigner_test.go +++ b/chain/messagesigner/messagesigner_test.go @@ -177,7 +177,7 @@ func TestMessageSignerSignMessage(t *testing.T) { t.Run(tt.name, func(t *testing.T) { mpool := newMockMpool() ds := ds_sync.MutexWrap(datastore.NewMapDatastore()) - ms := newMessageSigner(w, mpool, ds) + ms := NewMessageSigner(w, mpool, ds) for _, m := range tt.msgs { if len(m.mpoolNonce) == 1 { diff --git a/cli/paych_test.go b/cli/paych_test.go index 18782b4e8..2aa5c6009 100644 --- a/cli/paych_test.go +++ b/cli/paych_test.go @@ -390,7 +390,7 @@ func checkVoucherOutput(t *testing.T, list string, vouchers []voucherSpec) { } func startTwoNodesOneMiner(ctx context.Context, t *testing.T, blocktime time.Duration) ([]test.TestNode, []address.Address) { - n, sn := builder.RPCMockSbBuilder(t, 2, test.OneMiner) + n, sn := builder.RPCMockSbBuilder(t, test.TwoFull, test.OneMiner) paymentCreator := n[0] paymentReceiver := n[1] diff --git a/cmd/lotus-gateway/api_test.go b/cmd/lotus-gateway/api_test.go new file mode 100644 index 000000000..ab42c9310 --- /dev/null +++ b/cmd/lotus-gateway/api_test.go @@ -0,0 +1,183 @@ +package main + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/filecoin-project/lotus/build" + + "github.com/stretchr/testify/require" + + "github.com/filecoin-project/lotus/chain/types/mock" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" +) + +func TestGatewayAPIChainGetTipSetByHeight(t *testing.T) { + ctx := context.Background() + + lookbackTimestamp := uint64(time.Now().Unix()) - uint64(LookbackCap.Seconds()) + type args struct { + h abi.ChainEpoch + tskh abi.ChainEpoch + genesisTS uint64 + } + tests := []struct { + name string + args args + expErr bool + }{{ + name: "basic", + args: args{ + h: abi.ChainEpoch(1), + tskh: abi.ChainEpoch(5), + }, + }, { + name: "genesis", + args: args{ + h: abi.ChainEpoch(0), + tskh: abi.ChainEpoch(5), + }, + }, { + name: "same epoch as tipset", + args: args{ + h: abi.ChainEpoch(5), + tskh: abi.ChainEpoch(5), + }, + }, { + name: "tipset too old", + args: args{ + // Tipset height is 5, genesis is at LookbackCap - 10 epochs. + // So resulting tipset height will be 5 epochs earlier than LookbackCap. + h: abi.ChainEpoch(1), + tskh: abi.ChainEpoch(5), + genesisTS: lookbackTimestamp - build.BlockDelaySecs*10, + }, + expErr: true, + }, { + name: "lookup height too old", + args: args{ + // Tipset height is 5, lookup height is 1, genesis is at LookbackCap - 3 epochs. + // So + // - lookup height will be 2 epochs earlier than LookbackCap. + // - tipset height will be 2 epochs later than LookbackCap. + h: abi.ChainEpoch(1), + tskh: abi.ChainEpoch(5), + genesisTS: lookbackTimestamp - build.BlockDelaySecs*3, + }, + expErr: true, + }, { + name: "tipset and lookup height within acceptable range", + args: args{ + // Tipset height is 5, lookup height is 1, genesis is at LookbackCap. + // So + // - lookup height will be 1 epoch later than LookbackCap. + // - tipset height will be 5 epochs later than LookbackCap. + h: abi.ChainEpoch(1), + tskh: abi.ChainEpoch(5), + genesisTS: lookbackTimestamp, + }, + }} + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + mock := &mockRPCAPI{} + a := &GatewayAPI{rpc: mock} + + // Create tipsets from genesis up to tskh and return the highest + ts := mock.createTipSets(tt.args.tskh, tt.args.genesisTS) + + got, err := a.ChainGetTipSetByHeight(ctx, tt.args.h, ts.Key()) + if tt.expErr { + require.Error(t, err) + } else { + require.NoError(t, err) + require.Equal(t, tt.args.h, got.Height()) + } + }) + } +} + +type mockRPCAPI struct { + lk sync.RWMutex + tipsets []*types.TipSet +} + +func (m *mockRPCAPI) ChainHead(ctx context.Context) (*types.TipSet, error) { + m.lk.RLock() + defer m.lk.RUnlock() + + return m.tipsets[len(m.tipsets)-1], nil +} + +func (m *mockRPCAPI) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) { + m.lk.RLock() + defer m.lk.RUnlock() + + for _, ts := range m.tipsets { + if ts.Key() == tsk { + return ts, nil + } + } + + return nil, nil +} + +// createTipSets creates tipsets from genesis up to tskh and returns the highest +func (m *mockRPCAPI) createTipSets(h abi.ChainEpoch, genesisTimestamp uint64) *types.TipSet { + m.lk.Lock() + defer m.lk.Unlock() + + targeth := h + 1 // add one for genesis block + if genesisTimestamp == 0 { + genesisTimestamp = uint64(time.Now().Unix()) - build.BlockDelaySecs*uint64(targeth) + } + var currts *types.TipSet + for currh := abi.ChainEpoch(0); currh < targeth; currh++ { + blks := mock.MkBlock(currts, 1, 1) + if currh == 0 { + blks.Timestamp = genesisTimestamp + } + currts = mock.TipSet(blks) + m.tipsets = append(m.tipsets, currts) + } + + return m.tipsets[len(m.tipsets)-1] +} + +func (m *mockRPCAPI) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) { + m.lk.Lock() + defer m.lk.Unlock() + + return m.tipsets[h], nil +} + +func (m *mockRPCAPI) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) { + panic("implement me") +} + +func (m *mockRPCAPI) MpoolPushUntrusted(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) { + panic("implement me") +} + +func (m *mockRPCAPI) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { + panic("implement me") +} + +func (m *mockRPCAPI) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) { + panic("implement me") +} + +func (m *mockRPCAPI) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { + panic("implement me") +} + +func (m *mockRPCAPI) StateWaitMsgLimited(ctx context.Context, msg cid.Cid, confidence uint64, h abi.ChainEpoch) (*api.MsgLookup, error) { + panic("implement me") +} diff --git a/cmd/lotus-gateway/endtoend_test.go b/cmd/lotus-gateway/endtoend_test.go new file mode 100644 index 000000000..e92c3248b --- /dev/null +++ b/cmd/lotus-gateway/endtoend_test.go @@ -0,0 +1,140 @@ +package main + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + + "github.com/filecoin-project/lotus/api" + + "github.com/filecoin-project/lotus/node" + + "github.com/filecoin-project/lotus/api/client" + + "github.com/filecoin-project/go-jsonrpc" + + "github.com/filecoin-project/lotus/chain/wallet" + + "github.com/stretchr/testify/require" + + "github.com/filecoin-project/go-state-types/abi" + builder "github.com/filecoin-project/lotus/node/test" + + "github.com/filecoin-project/lotus/api/test" + "github.com/filecoin-project/lotus/chain/actors/policy" + "github.com/filecoin-project/lotus/chain/types" +) + +func init() { + policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg2KiBV1) + policy.SetConsensusMinerMinPower(abi.NewStoragePower(2048)) + policy.SetMinVerifiedDealSize(abi.NewStoragePower(256)) +} + +func TestEndToEnd(t *testing.T) { + _ = os.Setenv("BELLMAN_NO_GPU", "1") + + blocktime := 5 * time.Millisecond + ctx := context.Background() + full, lite, closer := startNodes(ctx, t, blocktime) + defer closer() + + // The full node starts with a wallet + fullWalletAddr, err := full.WalletDefaultAddress(ctx) + require.NoError(t, err) + + // Check the full node's wallet balance from the lite node + balance, err := lite.WalletBalance(ctx, fullWalletAddr) + require.NoError(t, err) + fmt.Println(balance) + + // Create a wallet on the lite node + liteWalletAddr, err := lite.WalletNew(ctx, wallet.ActSigType("secp256k1")) + require.NoError(t, err) + + // Send some funds from the full node to the lite node + err = sendFunds(ctx, t, full, fullWalletAddr, liteWalletAddr, types.NewInt(1e18)) + require.NoError(t, err) + + // Send some funds from the lite node back to the full node + err = sendFunds(ctx, t, lite, liteWalletAddr, fullWalletAddr, types.NewInt(100)) + require.NoError(t, err) + + data := []byte("hello") + sig, err := lite.WalletSign(ctx, liteWalletAddr, data) + require.NoError(t, err) + + ok, err := lite.WalletVerify(ctx, liteWalletAddr, data, sig) + require.NoError(t, err) + require.True(t, ok) +} + +func sendFunds(ctx context.Context, t *testing.T, fromNode test.TestNode, fromAddr address.Address, toAddr address.Address, amt types.BigInt) error { + msg := &types.Message{ + From: fromAddr, + To: toAddr, + Value: amt, + } + + sm, err := fromNode.MpoolPushMessage(ctx, msg, nil) + if err != nil { + return err + } + + res, err := fromNode.StateWaitMsg(ctx, sm.Cid(), 1) + if err != nil { + return err + } + if res.Receipt.ExitCode != 0 { + return xerrors.Errorf("send funds failed with exit code %d", res.Receipt.ExitCode) + } + + return nil +} + +func startNodes(ctx context.Context, t *testing.T, blocktime time.Duration) (test.TestNode, test.TestNode, jsonrpc.ClientCloser) { + var closer jsonrpc.ClientCloser + + opts := append( + // Full node + test.OneFull, + // Lite node + func(nodes []test.TestNode) node.Option { + fullNode := nodes[0] + + addr, err := builder.WSMultiAddrToString(fullNode.ListenAddr) + require.NoError(t, err) + + // Create a gateway API that connects to the full node + var gapi api.GatewayAPI + gapi, closer, err = client.NewGatewayRPC(ctx, addr, nil) + require.NoError(t, err) + return node.LiteModeOverrides(gapi) + }, + ) + n, sn := builder.RPCMockSbBuilderWithOpts(t, opts, test.OneMiner) + + full := n[0] + lite := n[1] + miner := sn[0] + + // Get the listener address for the full node + fullAddr, err := full.NetAddrsListen(ctx) + require.NoError(t, err) + + // Connect the miner and the full node + err = miner.NetConnect(ctx, fullAddr) + require.NoError(t, err) + + // Start mining blocks + bm := test.NewBlockMiner(ctx, t, miner, blocktime) + bm.MineBlocks() + + return full, lite, closer +} diff --git a/cmd/lotus-storage-miner/allinfo_test.go b/cmd/lotus-storage-miner/allinfo_test.go index 8f744c4b3..fbe1d1617 100644 --- a/cmd/lotus-storage-miner/allinfo_test.go +++ b/cmd/lotus-storage-miner/allinfo_test.go @@ -64,8 +64,8 @@ func TestMinerAllInfo(t *testing.T) { require.NoError(t, infoAllCmd.Action(cctx)) } - bp := func(t *testing.T, nFull int, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { - n, sn = builder.Builder(t, nFull, storage, opts...) + bp := func(t *testing.T, fullOpts []test.OptionGenerator, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { + n, sn = builder.Builder(t, fullOpts, storage, opts...) t.Run("pre-info-all", run) diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 495979ea9..8c154d7d7 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -39,7 +39,6 @@ import ( "github.com/filecoin-project/lotus/lib/ulimit" "github.com/filecoin-project/lotus/metrics" "github.com/filecoin-project/lotus/node" - "github.com/filecoin-project/lotus/node/impl/full" "github.com/filecoin-project/lotus/node/modules" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/testing" @@ -255,20 +254,7 @@ var DaemonCmd = &cli.Command{ } defer closer() - - liteMode = node.Options( - node.Override(new(api.GatewayAPI), gapi), - node.Override(new(full.ChainModuleAPI), node.From(new(api.GatewayAPI))), - node.Override(new(full.GasModuleAPI), node.From(new(api.GatewayAPI))), - node.Override(new(full.MpoolModuleAPI), node.From(new(api.GatewayAPI))), - node.Override(new(full.StateModuleAPI), node.From(new(api.GatewayAPI))), - node.Override(new(stmgr.StateManagerAPI), modules.NewRPCStateManager), - node.Unset(node.RunHelloKey), - node.Unset(node.RunChainExchangeKey), - node.Unset(node.RunPeerMgrKey), - node.Unset(node.HandleIncomingBlocksKey), - node.Unset(node.HandleIncomingMessagesKey), - ) + liteMode = node.LiteModeOverrides(gapi) } var api api.FullNode diff --git a/node/builder.go b/node/builder.go index 101ec3dbc..d521fdb94 100644 --- a/node/builder.go +++ b/node/builder.go @@ -264,6 +264,7 @@ func Online() Option { Override(new(*stmgr.StateManager), stmgr.NewStateManagerWithUpgradeSchedule), Override(new(stmgr.StateManagerAPI), From(new(*stmgr.StateManager))), Override(new(*wallet.Wallet), wallet.NewWallet), + Override(new(messagesigner.MpoolNonceAPI), From(new(*messagepool.MessagePool))), Override(new(*messagesigner.MessageSigner), messagesigner.NewMessageSigner), Override(new(full.ChainModuleAPI), From(new(full.ChainModule))), @@ -401,6 +402,23 @@ func StorageMiner(out *api.StorageMiner) Option { ) } +func LiteModeOverrides(gapi api.GatewayAPI) Option { + return Options( + Override(new(messagesigner.MpoolNonceAPI), From(new(modules.MpoolNonceAPI))), + Override(new(api.GatewayAPI), gapi), + Override(new(full.ChainModuleAPI), From(new(api.GatewayAPI))), + Override(new(full.GasModuleAPI), From(new(api.GatewayAPI))), + Override(new(full.MpoolModuleAPI), From(new(api.GatewayAPI))), + Override(new(full.StateModuleAPI), From(new(api.GatewayAPI))), + Override(new(stmgr.StateManagerAPI), modules.NewRPCStateManager), + Unset(RunHelloKey), + Unset(RunChainExchangeKey), + Unset(RunPeerMgrKey), + Unset(HandleIncomingBlocksKey), + Unset(HandleIncomingMessagesKey), + ) +} + // Config sets up constructors based on the provided Config func ConfigCommon(cfg *config.Common) Option { return Options( diff --git a/node/impl/full/mpool.go b/node/impl/full/mpool.go index fc32aecd2..8ad209f3f 100644 --- a/node/impl/full/mpool.go +++ b/node/impl/full/mpool.go @@ -180,7 +180,7 @@ func (a *MpoolAPI) MpoolPushMessage(ctx context.Context, msg *types.Message, spe // Sign and push the message return a.MessageSigner.SignMessage(ctx, msg, func(smsg *types.SignedMessage) error { - if _, err := a.Mpool.Push(smsg); err != nil { + if _, err := a.MpoolModuleAPI.MpoolPush(ctx, smsg); err != nil { return xerrors.Errorf("mpool push: failed to push message: %w", err) } return nil diff --git a/node/modules/mpoolnonceapi.go b/node/modules/mpoolnonceapi.go new file mode 100644 index 000000000..294f4d954 --- /dev/null +++ b/node/modules/mpoolnonceapi.go @@ -0,0 +1,33 @@ +package modules + +import ( + "context" + + "go.uber.org/fx" + + "github.com/filecoin-project/lotus/node/impl/full" + + "github.com/filecoin-project/lotus/chain/messagesigner" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/go-address" +) + +// MpoolNonceAPI substitutes the mpool nonce with an implementation that +// doesn't rely on the mpool - it just gets the nonce from actor state +type MpoolNonceAPI struct { + fx.In + + StateAPI full.StateAPI +} + +// GetNonce gets the nonce from actor state +func (a *MpoolNonceAPI) GetNonce(addr address.Address) (uint64, error) { + act, err := a.StateAPI.StateGetActor(context.Background(), addr, types.EmptyTSK) + if err != nil { + return 0, err + } + return act.Nonce, nil +} + +var _ messagesigner.MpoolNonceAPI = (*MpoolNonceAPI)(nil) diff --git a/node/test/builder.go b/node/test/builder.go index d9ec04460..f38201697 100644 --- a/node/test/builder.go +++ b/node/test/builder.go @@ -7,10 +7,13 @@ import ( "io/ioutil" "net" "net/http/httptest" + "strings" "sync" "testing" "time" + "golang.org/x/xerrors" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-jsonrpc" "github.com/filecoin-project/go-state-types/abi" @@ -117,6 +120,7 @@ func CreateTestStorageNode(ctx context.Context, t *testing.T, waddr address.Addr if err != nil { t.Fatalf("failed to construct node: %v", err) } + t.Cleanup(func() { _ = stop(context.Background()) }) /*// Bootstrap with full node @@ -137,13 +141,33 @@ func CreateTestStorageNode(ctx context.Context, t *testing.T, waddr address.Addr return test.TestStorageNode{StorageMiner: minerapi, MineOne: mineOne} } -func Builder(t *testing.T, nFull int, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { +func Builder(t *testing.T, fullOpts []test.OptionGenerator, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { + return mockBuilderOpts(t, fullOpts, storage, opts, false) +} + +func MockSbBuilder(t *testing.T, fullOpts []test.OptionGenerator, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { + return mockSbBuilderOpts(t, fullOpts, storage, opts, false) +} + +func RPCBuilder(t *testing.T, fullOpts []test.OptionGenerator, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { + return mockBuilderOpts(t, fullOpts, storage, opts, true) +} + +func RPCMockSbBuilder(t *testing.T, fullOpts []test.OptionGenerator, storage []test.StorageMiner) ([]test.TestNode, []test.TestStorageNode) { + return mockSbBuilderOpts(t, fullOpts, storage, []node.Option{}, true) +} + +func RPCMockSbBuilderWithOpts(t *testing.T, fullOpts []test.OptionGenerator, storage []test.StorageMiner) ([]test.TestNode, []test.TestStorageNode) { + return mockSbBuilderOpts(t, fullOpts, storage, []node.Option{}, true) +} + +func mockBuilderOpts(t *testing.T, fullOpts []test.OptionGenerator, storage []test.StorageMiner, opts []node.Option, rpc bool) ([]test.TestNode, []test.TestStorageNode) { ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) mn := mocknet.New(ctx) - fulls := make([]test.TestNode, nFull) + fulls := make([]test.TestNode, len(fullOpts)) storers := make([]test.TestStorageNode, len(storage)) pk, _, err := crypto.GenerateEd25519Key(rand.Reader) @@ -208,7 +232,7 @@ func Builder(t *testing.T, nFull int, storage []test.StorageMiner, opts ...node. // END PRESEAL SECTION - for i := 0; i < nFull; i++ { + for i := 0; i < len(fullOpts); i++ { var genesis node.Option if i == 0 { genesis = node.Override(new(modules.Genesis), testing2.MakeGenesisMem(&genbuf, *templ)) @@ -224,12 +248,18 @@ func Builder(t *testing.T, nFull int, storage []test.StorageMiner, opts ...node. node.Test(), genesis, + fullOpts[i](fulls), node.Options(opts...), ) if err != nil { t.Fatal(err) } + t.Cleanup(func() { _ = stop(context.Background()) }) + + if rpc { + fulls[i] = fullRpc(t, fulls[i]) + } } for i, def := range storage { @@ -261,6 +291,9 @@ func Builder(t *testing.T, nFull int, storage []test.StorageMiner, opts ...node. psd := presealDirs[i] */ + if rpc { + storers[i] = storerRpc(t, storers[i]) + } } if err := mn.LinkAll(); err != nil { @@ -286,11 +319,13 @@ func Builder(t *testing.T, nFull int, storage []test.StorageMiner, opts ...node. return fulls, storers } -func MockSbBuilder(t *testing.T, nFull int, storage []test.StorageMiner, options ...node.Option) ([]test.TestNode, []test.TestStorageNode) { - ctx := context.Background() +func mockSbBuilderOpts(t *testing.T, fullOpts []test.OptionGenerator, storage []test.StorageMiner, options []node.Option, rpc bool) ([]test.TestNode, []test.TestStorageNode) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + mn := mocknet.New(ctx) - fulls := make([]test.TestNode, nFull) + fulls := make([]test.TestNode, len(fullOpts)) storers := make([]test.TestStorageNode, len(storage)) var genbuf bytes.Buffer @@ -354,7 +389,7 @@ func MockSbBuilder(t *testing.T, nFull int, storage []test.StorageMiner, options // END PRESEAL SECTION - for i := 0; i < nFull; i++ { + for i := 0; i < len(fullOpts); i++ { var genesis node.Option if i == 0 { genesis = node.Override(new(modules.Genesis), testing2.MakeGenesisMem(&genbuf, *templ)) @@ -374,12 +409,18 @@ func MockSbBuilder(t *testing.T, nFull int, storage []test.StorageMiner, options node.Override(new(ffiwrapper.Verifier), mock.MockVerifier), genesis, + fullOpts[i](fulls), node.Options(options...), ) if err != nil { t.Fatalf("%+v", err) } + t.Cleanup(func() { _ = stop(context.Background()) }) + + if rpc { + fulls[i] = fullRpc(t, fulls[i]) + } } for i, def := range storage { @@ -414,6 +455,10 @@ func MockSbBuilder(t *testing.T, nFull int, storage []test.StorageMiner, options node.Override(new(ffiwrapper.Verifier), mock.MockVerifier), node.Unset(new(*sectorstorage.Manager)), )) + + if rpc { + storers[i] = storerRpc(t, storers[i]) + } } if err := mn.LinkAll(); err != nil { @@ -438,69 +483,66 @@ func MockSbBuilder(t *testing.T, nFull int, storage []test.StorageMiner, options return fulls, storers } -func RPCBuilder(t *testing.T, nFull int, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { - return rpcWithBuilder(t, Builder, nFull, storage, opts...) +func fullRpc(t *testing.T, nd test.TestNode) test.TestNode { + ma, listenAddr, err := CreateRPCServer(nd) + require.NoError(t, err) + + var full test.TestNode + full.FullNode, _, err = client.NewFullNodeRPC(context.Background(), listenAddr, nil) + require.NoError(t, err) + + full.ListenAddr = ma + return full } -func RPCMockSbBuilder(t *testing.T, nFull int, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { - return rpcWithBuilder(t, MockSbBuilder, nFull, storage, opts...) +func storerRpc(t *testing.T, nd test.TestStorageNode) test.TestStorageNode { + ma, listenAddr, err := CreateRPCServer(nd) + require.NoError(t, err) + + var storer test.TestStorageNode + storer.StorageMiner, _, err = client.NewStorageMinerRPC(context.Background(), listenAddr, nil) + require.NoError(t, err) + + storer.ListenAddr = ma + storer.MineOne = nd.MineOne + return storer } -func rpcWithBuilder(t *testing.T, b test.APIBuilder, nFull int, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { - fullApis, storaApis := b(t, nFull, storage, opts...) - fulls := make([]test.TestNode, nFull) - storers := make([]test.TestStorageNode, len(storage)) +func CreateRPCServer(handler interface{}) (multiaddr.Multiaddr, string, error) { + rpcServer := jsonrpc.NewServer() + rpcServer.Register("Filecoin", handler) + testServ := httptest.NewServer(rpcServer) // todo: close - for i, a := range fullApis { - rpcServer := jsonrpc.NewServer() - rpcServer.Register("Filecoin", a) - testServ := httptest.NewServer(rpcServer) // todo: close - - addr := testServ.Listener.Addr() - listenAddr := "ws://" + addr.String() - var err error - fulls[i].FullNode, _, err = client.NewFullNodeRPC(context.Background(), listenAddr, nil) - if err != nil { - t.Fatal(err) - } - ma, err := parseWSSMultiAddr(addr) - if err != nil { - t.Fatal(err) - } - fulls[i].ListenAddr = ma + addr := testServ.Listener.Addr() + listenAddr := "ws://" + addr.String() + ma, err := parseWSMultiAddr(addr) + if err != nil { + return nil, "", err } - - for i, a := range storaApis { - rpcServer := jsonrpc.NewServer() - rpcServer.Register("Filecoin", a) - testServ := httptest.NewServer(rpcServer) // todo: close - - addr := testServ.Listener.Addr() - listenAddr := "ws://" + addr.String() - var err error - storers[i].StorageMiner, _, err = client.NewStorageMinerRPC(context.Background(), listenAddr, nil) - if err != nil { - t.Fatal(err) - } - ma, err := parseWSSMultiAddr(addr) - if err != nil { - t.Fatal(err) - } - storers[i].ListenAddr = ma - storers[i].MineOne = a.MineOne - } - - return fulls, storers + return ma, listenAddr, err } -func parseWSSMultiAddr(addr net.Addr) (multiaddr.Multiaddr, error) { +func parseWSMultiAddr(addr net.Addr) (multiaddr.Multiaddr, error) { host, port, err := net.SplitHostPort(addr.String()) if err != nil { return nil, err } - ma, err := multiaddr.NewMultiaddr("/ip4/" + host + "/" + addr.Network() + "/" + port + "/wss") + ma, err := multiaddr.NewMultiaddr("/ip4/" + host + "/" + addr.Network() + "/" + port + "/ws") if err != nil { return nil, err } return ma, nil } + +func WSMultiAddrToString(addr multiaddr.Multiaddr) (string, error) { + parts := strings.Split(addr.String(), "/") + if len(parts) != 6 || parts[0] != "" { + return "", xerrors.Errorf("Malformed ws multiaddr %s", addr) + } + + host := parts[2] + port := parts[4] + proto := parts[5] + + return proto + "://" + host + ":" + port + "/rpc/v0", nil +} From e9e27cbbb0834c93d5b6df949382ac28d07e69ed Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Wed, 7 Oct 2020 13:59:32 +0200 Subject: [PATCH 661/885] feat: put gateway in middle of end-to-end test for lite mode --- cmd/lotus-gateway/api.go | 28 +++++++++++++++------------- cmd/lotus-gateway/api_test.go | 26 +++++++++++++------------- cmd/lotus-gateway/endtoend_test.go | 17 ++++++++++++++--- cmd/lotus-gateway/main.go | 2 +- node/test/builder.go | 4 ---- 5 files changed, 43 insertions(+), 34 deletions(-) diff --git a/cmd/lotus-gateway/api.go b/cmd/lotus-gateway/api.go index 0400a4a30..373c02eaa 100644 --- a/cmd/lotus-gateway/api.go +++ b/cmd/lotus-gateway/api.go @@ -23,7 +23,9 @@ var ( ErrLookbackTooLong = fmt.Errorf("lookbacks of more than %s are disallowed", LookbackCap) ) -type rpcAPI interface { +// gatewayDepsAPI defines the API methods that the GatewayAPI depends on +// (to make it easy to mock for tests) +type gatewayDepsAPI interface { ChainHead(ctx context.Context) (*types.TipSet, error) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) @@ -36,7 +38,7 @@ type rpcAPI interface { } type GatewayAPI struct { - rpc rpcAPI + api gatewayDepsAPI } func (a *GatewayAPI) checkTipsetKey(ctx context.Context, tsk types.TipSetKey) error { @@ -44,7 +46,7 @@ func (a *GatewayAPI) checkTipsetKey(ctx context.Context, tsk types.TipSetKey) er return nil } - ts, err := a.rpc.ChainGetTipSet(ctx, tsk) + ts, err := a.api.ChainGetTipSet(ctx, tsk) if err != nil { return err } @@ -82,15 +84,15 @@ func (a *GatewayAPI) checkTimestamp(at time.Time) error { func (a *GatewayAPI) ChainHead(ctx context.Context) (*types.TipSet, error) { // TODO: cache and invalidate cache when timestamp is up (or have internal ChainNotify) - return a.rpc.ChainHead(ctx) + return a.api.ChainHead(ctx) } func (a *GatewayAPI) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) { - return a.rpc.ChainGetTipSet(ctx, tsk) + return a.api.ChainGetTipSet(ctx, tsk) } func (a *GatewayAPI) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) { - ts, err := a.rpc.ChainGetTipSet(ctx, tsk) + ts, err := a.api.ChainGetTipSet(ctx, tsk) if err != nil { return nil, err } @@ -105,7 +107,7 @@ func (a *GatewayAPI) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoc return nil, err } - return a.rpc.ChainGetTipSetByHeight(ctx, h, tsk) + return a.api.ChainGetTipSetByHeight(ctx, h, tsk) } func (a *GatewayAPI) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) { @@ -113,12 +115,12 @@ func (a *GatewayAPI) GasEstimateMessageGas(ctx context.Context, msg *types.Messa return nil, err } - return a.rpc.GasEstimateMessageGas(ctx, msg, spec, tsk) + return a.api.GasEstimateMessageGas(ctx, msg, spec, tsk) } func (a *GatewayAPI) MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) { // TODO: additional anti-spam checks - return a.rpc.MpoolPushUntrusted(ctx, sm) + return a.api.MpoolPushUntrusted(ctx, sm) } func (a *GatewayAPI) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { @@ -126,7 +128,7 @@ func (a *GatewayAPI) StateAccountKey(ctx context.Context, addr address.Address, return address.Undef, err } - return a.rpc.StateAccountKey(ctx, addr, tsk) + return a.api.StateAccountKey(ctx, addr, tsk) } func (a *GatewayAPI) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) { @@ -134,7 +136,7 @@ func (a *GatewayAPI) StateGetActor(ctx context.Context, actor address.Address, t return nil, err } - return a.rpc.StateGetActor(ctx, actor, tsk) + return a.api.StateGetActor(ctx, actor, tsk) } func (a *GatewayAPI) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { @@ -142,11 +144,11 @@ func (a *GatewayAPI) StateLookupID(ctx context.Context, addr address.Address, ts return address.Undef, err } - return a.rpc.StateLookupID(ctx, addr, tsk) + return a.api.StateLookupID(ctx, addr, tsk) } func (a *GatewayAPI) StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error) { - return a.rpc.StateWaitMsgLimited(ctx, msg, confidence, stateWaitLookbackLimit) + return a.api.StateWaitMsgLimited(ctx, msg, confidence, stateWaitLookbackLimit) } var _ api.GatewayAPI = (*GatewayAPI)(nil) diff --git a/cmd/lotus-gateway/api_test.go b/cmd/lotus-gateway/api_test.go index ab42c9310..2df2c9ba4 100644 --- a/cmd/lotus-gateway/api_test.go +++ b/cmd/lotus-gateway/api_test.go @@ -87,8 +87,8 @@ func TestGatewayAPIChainGetTipSetByHeight(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - mock := &mockRPCAPI{} - a := &GatewayAPI{rpc: mock} + mock := &mockGatewayDepsAPI{} + a := &GatewayAPI{api: mock} // Create tipsets from genesis up to tskh and return the highest ts := mock.createTipSets(tt.args.tskh, tt.args.genesisTS) @@ -104,19 +104,19 @@ func TestGatewayAPIChainGetTipSetByHeight(t *testing.T) { } } -type mockRPCAPI struct { +type mockGatewayDepsAPI struct { lk sync.RWMutex tipsets []*types.TipSet } -func (m *mockRPCAPI) ChainHead(ctx context.Context) (*types.TipSet, error) { +func (m *mockGatewayDepsAPI) ChainHead(ctx context.Context) (*types.TipSet, error) { m.lk.RLock() defer m.lk.RUnlock() return m.tipsets[len(m.tipsets)-1], nil } -func (m *mockRPCAPI) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) { +func (m *mockGatewayDepsAPI) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) { m.lk.RLock() defer m.lk.RUnlock() @@ -130,7 +130,7 @@ func (m *mockRPCAPI) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (* } // createTipSets creates tipsets from genesis up to tskh and returns the highest -func (m *mockRPCAPI) createTipSets(h abi.ChainEpoch, genesisTimestamp uint64) *types.TipSet { +func (m *mockGatewayDepsAPI) createTipSets(h abi.ChainEpoch, genesisTimestamp uint64) *types.TipSet { m.lk.Lock() defer m.lk.Unlock() @@ -151,33 +151,33 @@ func (m *mockRPCAPI) createTipSets(h abi.ChainEpoch, genesisTimestamp uint64) *t return m.tipsets[len(m.tipsets)-1] } -func (m *mockRPCAPI) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) { +func (m *mockGatewayDepsAPI) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) { m.lk.Lock() defer m.lk.Unlock() return m.tipsets[h], nil } -func (m *mockRPCAPI) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) { +func (m *mockGatewayDepsAPI) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) { panic("implement me") } -func (m *mockRPCAPI) MpoolPushUntrusted(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) { +func (m *mockGatewayDepsAPI) MpoolPushUntrusted(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) { panic("implement me") } -func (m *mockRPCAPI) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { +func (m *mockGatewayDepsAPI) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { panic("implement me") } -func (m *mockRPCAPI) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) { +func (m *mockGatewayDepsAPI) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) { panic("implement me") } -func (m *mockRPCAPI) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { +func (m *mockGatewayDepsAPI) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { panic("implement me") } -func (m *mockRPCAPI) StateWaitMsgLimited(ctx context.Context, msg cid.Cid, confidence uint64, h abi.ChainEpoch) (*api.MsgLookup, error) { +func (m *mockGatewayDepsAPI) StateWaitMsgLimited(ctx context.Context, msg cid.Cid, confidence uint64, h abi.ChainEpoch) (*api.MsgLookup, error) { panic("implement me") } diff --git a/cmd/lotus-gateway/endtoend_test.go b/cmd/lotus-gateway/endtoend_test.go index e92c3248b..317e676d7 100644 --- a/cmd/lotus-gateway/endtoend_test.go +++ b/cmd/lotus-gateway/endtoend_test.go @@ -37,6 +37,8 @@ func init() { policy.SetMinVerifiedDealSize(abi.NewStoragePower(256)) } +// TestEndToEnd tests that API calls can be made on a lite node that is +// connected through a gateway to a full API node func TestEndToEnd(t *testing.T) { _ = os.Setenv("BELLMAN_NO_GPU", "1") @@ -66,10 +68,12 @@ func TestEndToEnd(t *testing.T) { err = sendFunds(ctx, t, lite, liteWalletAddr, fullWalletAddr, types.NewInt(100)) require.NoError(t, err) + // Sign some data with the lite node wallet address data := []byte("hello") sig, err := lite.WalletSign(ctx, liteWalletAddr, data) require.NoError(t, err) + // Verify the signature ok, err := lite.WalletVerify(ctx, liteWalletAddr, data, sig) require.NoError(t, err) require.True(t, ok) @@ -101,6 +105,10 @@ func sendFunds(ctx context.Context, t *testing.T, fromNode test.TestNode, fromAd func startNodes(ctx context.Context, t *testing.T, blocktime time.Duration) (test.TestNode, test.TestNode, jsonrpc.ClientCloser) { var closer jsonrpc.ClientCloser + // Create one miner and two full nodes. + // - Put a gateway server in front of full node 1 + // - Start full node 2 in lite mode + // - Connect lite node -> gateway server -> full node opts := append( // Full node test.OneFull, @@ -108,17 +116,20 @@ func startNodes(ctx context.Context, t *testing.T, blocktime time.Duration) (tes func(nodes []test.TestNode) node.Option { fullNode := nodes[0] - addr, err := builder.WSMultiAddrToString(fullNode.ListenAddr) + // Create a gateway server in front of the full node + _, addr, err := builder.CreateRPCServer(&GatewayAPI{api: fullNode}) require.NoError(t, err) - // Create a gateway API that connects to the full node + // Create a gateway client API that connects to the gateway server var gapi api.GatewayAPI gapi, closer, err = client.NewGatewayRPC(ctx, addr, nil) require.NoError(t, err) + + // Override this node with lite-mode options return node.LiteModeOverrides(gapi) }, ) - n, sn := builder.RPCMockSbBuilderWithOpts(t, opts, test.OneMiner) + n, sn := builder.RPCMockSbBuilder(t, opts, test.OneMiner) full := n[0] lite := n[1] diff --git a/cmd/lotus-gateway/main.go b/cmd/lotus-gateway/main.go index 3c8eb4019..c19599084 100644 --- a/cmd/lotus-gateway/main.go +++ b/cmd/lotus-gateway/main.go @@ -76,7 +76,7 @@ var runCmd = &cli.Command{ log.Info("Setting up API endpoint at " + address) rpcServer := jsonrpc.NewServer() - rpcServer.Register("Filecoin", &GatewayAPI{rpc: api}) + rpcServer.Register("Filecoin", &GatewayAPI{api: api}) mux.Handle("/rpc/v0", rpcServer) mux.PathPrefix("/").Handler(http.DefaultServeMux) diff --git a/node/test/builder.go b/node/test/builder.go index f38201697..ebf96e98f 100644 --- a/node/test/builder.go +++ b/node/test/builder.go @@ -157,10 +157,6 @@ func RPCMockSbBuilder(t *testing.T, fullOpts []test.OptionGenerator, storage []t return mockSbBuilderOpts(t, fullOpts, storage, []node.Option{}, true) } -func RPCMockSbBuilderWithOpts(t *testing.T, fullOpts []test.OptionGenerator, storage []test.StorageMiner) ([]test.TestNode, []test.TestStorageNode) { - return mockSbBuilderOpts(t, fullOpts, storage, []node.Option{}, true) -} - func mockBuilderOpts(t *testing.T, fullOpts []test.OptionGenerator, storage []test.StorageMiner, opts []node.Option, rpc bool) ([]test.TestNode, []test.TestStorageNode) { ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) From b2834baa4b2818a7773859b47f70e4ccc3975993 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Wed, 7 Oct 2020 18:14:12 +0200 Subject: [PATCH 662/885] feat: add msig available balance and vested to lite mode --- api/api_gateway.go | 2 + api/apistruct/struct.go | 28 +++++++++---- cmd/lotus-gateway/api.go | 21 ++++++++++ cmd/lotus-gateway/api_test.go | 8 ++++ cmd/lotus-gateway/endtoend_test.go | 66 ++++++++++++++++++++++++++++++ node/impl/full/state.go | 20 +++++---- 6 files changed, 127 insertions(+), 18 deletions(-) diff --git a/api/api_gateway.go b/api/api_gateway.go index 8c06304e9..95d28887d 100644 --- a/api/api_gateway.go +++ b/api/api_gateway.go @@ -15,6 +15,8 @@ type GatewayAPI interface { ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) + MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) + MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 38d177aec..df84eac80 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -365,15 +365,17 @@ type WorkerStruct struct { type GatewayStruct struct { Internal struct { // TODO: does the gateway need perms? - ChainGetTipSet func(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) - ChainGetTipSetByHeight func(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) - ChainHead func(ctx context.Context) (*types.TipSet, error) - GasEstimateMessageGas func(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) - MpoolPush func(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) - StateAccountKey func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) - StateGetActor func(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) - StateLookupID func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) - StateWaitMsg func(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error) + ChainGetTipSet func(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) + ChainGetTipSetByHeight func(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) + ChainHead func(ctx context.Context) (*types.TipSet, error) + GasEstimateMessageGas func(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) + MpoolPush func(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) + MsigGetAvailableBalance func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) + MsigGetVested func(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error) + StateAccountKey func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + StateGetActor func(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) + StateLookupID func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + StateWaitMsg func(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error) } } @@ -1412,6 +1414,14 @@ func (g GatewayStruct) MpoolPush(ctx context.Context, sm *types.SignedMessage) ( return g.Internal.MpoolPush(ctx, sm) } +func (g GatewayStruct) MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) { + return g.Internal.MsigGetAvailableBalance(ctx, addr, tsk) +} + +func (g GatewayStruct) MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error) { + return g.Internal.MsigGetVested(ctx, addr, start, end) +} + func (g GatewayStruct) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { return g.Internal.StateAccountKey(ctx, addr, tsk) } diff --git a/cmd/lotus-gateway/api.go b/cmd/lotus-gateway/api.go index 373c02eaa..d5fac0a06 100644 --- a/cmd/lotus-gateway/api.go +++ b/cmd/lotus-gateway/api.go @@ -31,6 +31,8 @@ type gatewayDepsAPI interface { ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) MpoolPushUntrusted(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) + MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) + MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) @@ -123,6 +125,25 @@ func (a *GatewayAPI) MpoolPush(ctx context.Context, sm *types.SignedMessage) (ci return a.api.MpoolPushUntrusted(ctx, sm) } +func (a *GatewayAPI) MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return types.NewInt(0), err + } + + return a.api.MsigGetAvailableBalance(ctx, addr, tsk) +} + +func (a *GatewayAPI) MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error) { + if err := a.checkTipsetKey(ctx, start); err != nil { + return types.NewInt(0), err + } + if err := a.checkTipsetKey(ctx, end); err != nil { + return types.NewInt(0), err + } + + return a.api.MsigGetVested(ctx, addr, start, end) +} + func (a *GatewayAPI) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { if err := a.checkTipsetKey(ctx, tsk); err != nil { return address.Undef, err diff --git a/cmd/lotus-gateway/api_test.go b/cmd/lotus-gateway/api_test.go index 2df2c9ba4..f34f887f5 100644 --- a/cmd/lotus-gateway/api_test.go +++ b/cmd/lotus-gateway/api_test.go @@ -166,6 +166,14 @@ func (m *mockGatewayDepsAPI) MpoolPushUntrusted(ctx context.Context, sm *types.S panic("implement me") } +func (m *mockGatewayDepsAPI) MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) { + panic("implement me") +} + +func (m *mockGatewayDepsAPI) MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error) { + panic("implement me") +} + func (m *mockGatewayDepsAPI) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { panic("implement me") } diff --git a/cmd/lotus-gateway/endtoend_test.go b/cmd/lotus-gateway/endtoend_test.go index 317e676d7..ad62be3fb 100644 --- a/cmd/lotus-gateway/endtoend_test.go +++ b/cmd/lotus-gateway/endtoend_test.go @@ -1,12 +1,17 @@ package main import ( + "bytes" "context" "fmt" "os" "testing" "time" + "github.com/filecoin-project/specs-actors/actors/builtin/multisig" + + init0 "github.com/filecoin-project/specs-actors/actors/builtin/init" + "golang.org/x/xerrors" "github.com/filecoin-project/go-address" @@ -77,6 +82,67 @@ func TestEndToEnd(t *testing.T) { ok, err := lite.WalletVerify(ctx, liteWalletAddr, data, sig) require.NoError(t, err) require.True(t, ok) + + // Create some wallets on the lite node to use for testing multisig + var walletAddrs []address.Address + for i := 0; i < 4; i++ { + addr, err := lite.WalletNew(ctx, wallet.ActSigType("secp256k1")) + require.NoError(t, err) + + walletAddrs = append(walletAddrs, addr) + + err = sendFunds(ctx, t, lite, liteWalletAddr, addr, types.NewInt(1e15)) + require.NoError(t, err) + } + + // Create an msig with three of the addresses and threshold of two sigs + msigAddrs := walletAddrs[:3] + amt := types.NewInt(1000) + addProposal, err := lite.MsigCreate(ctx, 2, msigAddrs, abi.ChainEpoch(50), amt, liteWalletAddr, types.NewInt(0)) + require.NoError(t, err) + + res, err := lite.StateWaitMsg(ctx, addProposal, 1) + require.NoError(t, err) + require.EqualValues(t, 0, res.Receipt.ExitCode) + + var execReturn init0.ExecReturn + err = execReturn.UnmarshalCBOR(bytes.NewReader(res.Receipt.Return)) + require.NoError(t, err) + + // Get available balance of msig: should be greater than zero and less + // than initial amount + msig := execReturn.IDAddress + msigBalance, err := lite.MsigGetAvailableBalance(ctx, msig, types.EmptyTSK) + require.NoError(t, err) + require.Greater(t, msigBalance.Int64(), int64(0)) + require.Less(t, msigBalance.Int64(), amt.Int64()) + + // Propose to add a new address to the msig + addProposal, err = lite.MsigAddPropose(ctx, msig, walletAddrs[0], walletAddrs[3], false) + require.NoError(t, err) + + res, err = lite.StateWaitMsg(ctx, addProposal, 1) + require.NoError(t, err) + require.EqualValues(t, 0, res.Receipt.ExitCode) + + var proposeReturn multisig.ProposeReturn + err = proposeReturn.UnmarshalCBOR(bytes.NewReader(res.Receipt.Return)) + require.NoError(t, err) + + // Approve proposal (proposer is first (implicit) signer, approver is + // second signer + txnID := uint64(proposeReturn.TxnID) + approval1, err := lite.MsigAddApprove(ctx, msig, walletAddrs[1], txnID, walletAddrs[0], walletAddrs[3], false) + require.NoError(t, err) + + res, err = lite.StateWaitMsg(ctx, approval1, 1) + require.NoError(t, err) + require.EqualValues(t, 0, res.Receipt.ExitCode) + + var approveReturn multisig.ApproveReturn + err = approveReturn.UnmarshalCBOR(bytes.NewReader(res.Receipt.Return)) + require.NoError(t, err) + require.True(t, approveReturn.Applied) } func sendFunds(ctx context.Context, t *testing.T, fromNode test.TestNode, fromAddr address.Address, toAddr address.Address, amt types.BigInt) error { diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 540d959ed..34724e205 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -47,6 +47,8 @@ type StateModuleAPI interface { StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error) + MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) + MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error) } // StateModule provides a default implementation of StateModuleAPI. @@ -828,17 +830,17 @@ func (a *StateAPI) StateCompute(ctx context.Context, height abi.ChainEpoch, msgs }, nil } -func (a *StateAPI) MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) { - ts, err := a.Chain.GetTipSetFromKey(tsk) +func (m *StateModule) MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) { + ts, err := m.Chain.GetTipSetFromKey(tsk) if err != nil { return types.EmptyInt, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - act, err := a.StateManager.LoadActor(ctx, addr, ts) + act, err := m.StateManager.LoadActor(ctx, addr, ts) if err != nil { return types.EmptyInt, xerrors.Errorf("failed to load multisig actor: %w", err) } - msas, err := multisig.Load(a.Chain.Store(ctx), act) + msas, err := multisig.Load(m.Chain.Store(ctx), act) if err != nil { return types.EmptyInt, xerrors.Errorf("failed to load multisig actor state: %w", err) } @@ -887,13 +889,13 @@ func (a *StateAPI) MsigGetVestingSchedule(ctx context.Context, addr address.Addr }, nil } -func (a *StateAPI) MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error) { - startTs, err := a.Chain.GetTipSetFromKey(start) +func (m *StateModule) MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error) { + startTs, err := m.Chain.GetTipSetFromKey(start) if err != nil { return types.EmptyInt, xerrors.Errorf("loading start tipset %s: %w", start, err) } - endTs, err := a.Chain.GetTipSetFromKey(end) + endTs, err := m.Chain.GetTipSetFromKey(end) if err != nil { return types.EmptyInt, xerrors.Errorf("loading end tipset %s: %w", end, err) } @@ -904,12 +906,12 @@ func (a *StateAPI) MsigGetVested(ctx context.Context, addr address.Address, star return big.Zero(), nil } - act, err := a.StateManager.LoadActor(ctx, addr, endTs) + act, err := m.StateManager.LoadActor(ctx, addr, endTs) if err != nil { return types.EmptyInt, xerrors.Errorf("failed to load multisig actor at end epoch: %w", err) } - msas, err := multisig.Load(a.Chain.Store(ctx), act) + msas, err := multisig.Load(m.Chain.Store(ctx), act) if err != nil { return types.EmptyInt, xerrors.Errorf("failed to load multisig actor state: %w", err) } From d69e4c7cf2fc2c3e64cd83c1a15363e31c9b1ec1 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Thu, 8 Oct 2020 11:24:31 +0200 Subject: [PATCH 663/885] refactor: lite-mode - simplify organization of dep injection --- api/test/test.go | 20 +++++-- chain/sync_test.go | 4 +- cmd/lotus-gateway/endtoend_test.go | 48 ++++++++--------- cmd/lotus/daemon.go | 21 ++++---- node/builder.go | 87 +++++++++++++++--------------- node/test/builder.go | 24 ++++----- 6 files changed, 105 insertions(+), 99 deletions(-) diff --git a/api/test/test.go b/api/test/test.go index 63e50bd73..014859865 100644 --- a/api/test/test.go +++ b/api/test/test.go @@ -35,6 +35,7 @@ var PresealGenesis = -1 const GenesisPreseals = 2 +// Options for setting up a mock storage miner type StorageMiner struct { Full int Preseal int @@ -42,12 +43,19 @@ type StorageMiner struct { type OptionGenerator func([]TestNode) node.Option +// Options for setting up a mock full node +type FullNodeOpts struct { + Lite bool // run node in "lite" mode + Opts OptionGenerator // generate dependency injection options +} + // APIBuilder is a function which is invoked in test suite to provide // test nodes and networks // +// fullOpts array defines options for each full node // storage array defines storage nodes, numbers in the array specify full node // index the storage node 'belongs' to -type APIBuilder func(t *testing.T, full []OptionGenerator, storage []StorageMiner, opts ...node.Option) ([]TestNode, []TestStorageNode) +type APIBuilder func(t *testing.T, full []FullNodeOpts, storage []StorageMiner, opts ...node.Option) ([]TestNode, []TestStorageNode) type testSuite struct { makeNodes APIBuilder } @@ -65,11 +73,13 @@ func TestApis(t *testing.T, b APIBuilder) { t.Run("testMiningReal", ts.testMiningReal) } -func DefaultFullOpts(nFull int) []OptionGenerator { - full := make([]OptionGenerator, nFull) +func DefaultFullOpts(nFull int) []FullNodeOpts { + full := make([]FullNodeOpts, nFull) for i := range full { - full[i] = func(nodes []TestNode) node.Option { - return node.Options() + full[i] = FullNodeOpts{ + Opts: func(nodes []TestNode) node.Option { + return node.Options() + }, } } return full diff --git a/chain/sync_test.go b/chain/sync_test.go index 318f6d82f..1fe90a071 100644 --- a/chain/sync_test.go +++ b/chain/sync_test.go @@ -222,7 +222,7 @@ func (tu *syncTestUtil) addSourceNode(gen int) { var out api.FullNode stop, err := node.New(tu.ctx, - node.FullAPI(&out), + node.FullAPI(&out, false), node.Online(), node.Repo(sourceRepo), node.MockHost(tu.mn), @@ -254,7 +254,7 @@ func (tu *syncTestUtil) addClientNode() int { var out api.FullNode stop, err := node.New(tu.ctx, - node.FullAPI(&out), + node.FullAPI(&out, false), node.Online(), node.Repo(repo.NewMemory(nil)), node.MockHost(tu.mn), diff --git a/cmd/lotus-gateway/endtoend_test.go b/cmd/lotus-gateway/endtoend_test.go index ad62be3fb..206034968 100644 --- a/cmd/lotus-gateway/endtoend_test.go +++ b/cmd/lotus-gateway/endtoend_test.go @@ -8,32 +8,23 @@ import ( "testing" "time" + init0 "github.com/filecoin-project/specs-actors/actors/builtin/init" "github.com/filecoin-project/specs-actors/actors/builtin/multisig" - init0 "github.com/filecoin-project/specs-actors/actors/builtin/init" - + "github.com/stretchr/testify/require" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" - - "github.com/filecoin-project/lotus/api" - - "github.com/filecoin-project/lotus/node" - - "github.com/filecoin-project/lotus/api/client" - "github.com/filecoin-project/go-jsonrpc" - - "github.com/filecoin-project/lotus/chain/wallet" - - "github.com/stretchr/testify/require" - "github.com/filecoin-project/go-state-types/abi" - builder "github.com/filecoin-project/lotus/node/test" - + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/api/client" "github.com/filecoin-project/lotus/api/test" "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/wallet" + "github.com/filecoin-project/lotus/node" + builder "github.com/filecoin-project/lotus/node/test" ) func init() { @@ -179,20 +170,23 @@ func startNodes(ctx context.Context, t *testing.T, blocktime time.Duration) (tes // Full node test.OneFull, // Lite node - func(nodes []test.TestNode) node.Option { - fullNode := nodes[0] + test.FullNodeOpts{ + Lite: true, + Opts: func(nodes []test.TestNode) node.Option { + fullNode := nodes[0] - // Create a gateway server in front of the full node - _, addr, err := builder.CreateRPCServer(&GatewayAPI{api: fullNode}) - require.NoError(t, err) + // Create a gateway server in front of the full node + _, addr, err := builder.CreateRPCServer(&GatewayAPI{api: fullNode}) + require.NoError(t, err) - // Create a gateway client API that connects to the gateway server - var gapi api.GatewayAPI - gapi, closer, err = client.NewGatewayRPC(ctx, addr, nil) - require.NoError(t, err) + // Create a gateway client API that connects to the gateway server + var gapi api.GatewayAPI + gapi, closer, err = client.NewGatewayRPC(ctx, addr, nil) + require.NoError(t, err) - // Override this node with lite-mode options - return node.LiteModeOverrides(gapi) + // Provide the gateway API to dependency injection + return node.Override(new(api.GatewayAPI), gapi) + }, }, ) n, sn := builder.RPCMockSbBuilder(t, opts, test.OneMiner) diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 8c154d7d7..084f75ef2 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -136,6 +136,8 @@ var DaemonCmd = &cli.Command{ }, }, Action: func(cctx *cli.Context) error { + isLite := cctx.Bool("lite") + err := runmetrics.Enable(runmetrics.RunMetricOptions{ EnableCPU: true, EnableMemory: true, @@ -195,8 +197,10 @@ var DaemonCmd = &cli.Command{ return xerrors.Errorf("repo init error: %w", err) } - if err := paramfetch.GetParams(lcli.ReqContext(cctx), build.ParametersJSON(), 0); err != nil { - return xerrors.Errorf("fetching proof parameters: %w", err) + if !isLite { + if err := paramfetch.GetParams(lcli.ReqContext(cctx), build.ParametersJSON(), 0); err != nil { + return xerrors.Errorf("fetching proof parameters: %w", err) + } } var genBytes []byte @@ -243,10 +247,9 @@ var DaemonCmd = &cli.Command{ shutdownChan := make(chan struct{}) - // If the daemon is started in "lite mode", replace key APIs - // with a thin client to a gateway server - liteMode := node.Options() - isLite := cctx.Bool("lite") + // If the daemon is started in "lite mode", provide a GatewayAPI + // for RPC calls + liteModeDeps := node.Options() if isLite { gapi, closer, err := lcli.GetGatewayAPI(cctx) if err != nil { @@ -254,13 +257,13 @@ var DaemonCmd = &cli.Command{ } defer closer() - liteMode = node.LiteModeOverrides(gapi) + liteModeDeps = node.Override(new(api.GatewayAPI), gapi) } var api api.FullNode stop, err := node.New(ctx, - node.FullAPI(&api), + node.FullAPI(&api, isLite), node.Override(new(dtypes.Bootstrapper), isBootstrapper), node.Override(new(dtypes.ShutdownChan), shutdownChan), @@ -268,7 +271,7 @@ var DaemonCmd = &cli.Command{ node.Repo(r), genesis, - liteMode, + liteModeDeps, node.ApplyIf(func(s *node.Settings) bool { return cctx.IsSet("api") }, node.Override(node.SetApiEndpointKey, func(lr repo.LockedRepo) error { diff --git a/node/builder.go b/node/builder.go index d521fdb94..452104ba8 100644 --- a/node/builder.go +++ b/node/builder.go @@ -6,6 +6,13 @@ import ( "os" "time" + "github.com/filecoin-project/lotus/chain" + "github.com/filecoin-project/lotus/chain/exchange" + "github.com/filecoin-project/lotus/chain/store" + "github.com/filecoin-project/lotus/chain/vm" + "github.com/filecoin-project/lotus/chain/wallet" + "github.com/filecoin-project/lotus/node/hello" + logging "github.com/ipfs/go-log" ci "github.com/libp2p/go-libp2p-core/crypto" "github.com/libp2p/go-libp2p-core/host" @@ -29,9 +36,7 @@ import ( storage2 "github.com/filecoin-project/specs-storage/storage" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain" "github.com/filecoin-project/lotus/chain/beacon" - "github.com/filecoin-project/lotus/chain/exchange" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/gen/slashfilter" "github.com/filecoin-project/lotus/chain/market" @@ -39,10 +44,7 @@ import ( "github.com/filecoin-project/lotus/chain/messagesigner" "github.com/filecoin-project/lotus/chain/metrics" "github.com/filecoin-project/lotus/chain/stmgr" - "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/chain/vm" - "github.com/filecoin-project/lotus/chain/wallet" sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/extern/sector-storage/stores" @@ -56,7 +58,6 @@ import ( "github.com/filecoin-project/lotus/markets/storageadapter" "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node/config" - "github.com/filecoin-project/lotus/node/hello" "github.com/filecoin-project/lotus/node/impl" "github.com/filecoin-project/lotus/node/impl/common" "github.com/filecoin-project/lotus/node/impl/full" @@ -159,7 +160,7 @@ type Settings struct { Online bool // Online option applied Config bool // Config option applied - + Lite bool // Start node in "lite" mode } func defaults() []Option { @@ -233,6 +234,10 @@ func isType(t repo.RepoType) func(s *Settings) bool { // Online sets up basic libp2p node func Online() Option { + isFullOrLiteNode := func(s *Settings) bool { return s.nodeType == repo.FullNode } + isFullNode := func(s *Settings) bool { return s.nodeType == repo.FullNode && !s.Lite } + isLiteNode := func(s *Settings) bool { return s.nodeType == repo.FullNode && s.Lite } + return Options( // make sure that online is applied before Config. // This is important because Config overrides some of Online units @@ -246,17 +251,14 @@ func Online() Option { // common Override(new(*slashfilter.SlashFilter), modules.NewSlashFilter), - // Full node - - ApplyIf(isType(repo.FullNode), + // Full node or lite node + ApplyIf(isFullOrLiteNode, // TODO: Fix offline mode Override(new(dtypes.BootstrapPeers), modules.BuiltinBootstrap), Override(new(dtypes.DrandBootstrap), modules.DrandBootstrap), Override(new(dtypes.DrandSchedule), modules.BuiltinDrandConfig), - Override(HandleIncomingMessagesKey, modules.HandleIncomingMessages), - Override(new(ffiwrapper.Verifier), ffiwrapper.ProofVerifier), Override(new(vm.SyscallBuilder), vm.Syscalls), Override(new(*store.ChainStore), modules.ChainStore), @@ -264,15 +266,8 @@ func Online() Option { Override(new(*stmgr.StateManager), stmgr.NewStateManagerWithUpgradeSchedule), Override(new(stmgr.StateManagerAPI), From(new(*stmgr.StateManager))), Override(new(*wallet.Wallet), wallet.NewWallet), - Override(new(messagesigner.MpoolNonceAPI), From(new(*messagepool.MessagePool))), Override(new(*messagesigner.MessageSigner), messagesigner.NewMessageSigner), - Override(new(full.ChainModuleAPI), From(new(full.ChainModule))), - Override(new(full.GasModuleAPI), From(new(full.GasModule))), - Override(new(full.MpoolModuleAPI), From(new(full.MpoolModule))), - Override(new(full.StateModuleAPI), From(new(full.StateModule))), - Override(new(stmgr.StateManagerAPI), From(new(*stmgr.StateManager))), - Override(new(dtypes.ChainGCLocker), blockstore.NewGCLocker), Override(new(dtypes.ChainGCBlockstore), modules.ChainGCBlockstore), Override(new(dtypes.ChainBitswap), modules.ChainBitswap), @@ -297,12 +292,6 @@ func Online() Option { Override(new(dtypes.Graphsync), modules.Graphsync), Override(new(*dtypes.MpoolLocker), new(dtypes.MpoolLocker)), - - Override(RunHelloKey, modules.RunHello), - Override(RunChainExchangeKey, modules.RunChainExchange), - Override(RunPeerMgrKey, modules.RunPeerMgr), - Override(HandleIncomingBlocksKey, modules.HandleIncomingBlocks), - Override(new(*discoveryimpl.Local), modules.NewLocalDiscovery), Override(new(discovery.PeerResolver), modules.RetrievalResolver), @@ -321,8 +310,34 @@ func Online() Option { Override(SettlePaymentChannelsKey, settler.SettlePaymentChannels), ), + // Lite node + ApplyIf(isLiteNode, + Override(new(messagesigner.MpoolNonceAPI), From(new(modules.MpoolNonceAPI))), + Override(new(full.ChainModuleAPI), From(new(api.GatewayAPI))), + Override(new(full.GasModuleAPI), From(new(api.GatewayAPI))), + Override(new(full.MpoolModuleAPI), From(new(api.GatewayAPI))), + Override(new(full.StateModuleAPI), From(new(api.GatewayAPI))), + Override(new(stmgr.StateManagerAPI), modules.NewRPCStateManager), + ), + + // Full node + ApplyIf(isFullNode, + Override(new(messagesigner.MpoolNonceAPI), From(new(*messagepool.MessagePool))), + Override(new(full.ChainModuleAPI), From(new(full.ChainModule))), + Override(new(full.GasModuleAPI), From(new(full.GasModule))), + Override(new(full.MpoolModuleAPI), From(new(full.MpoolModule))), + Override(new(full.StateModuleAPI), From(new(full.StateModule))), + Override(new(stmgr.StateManagerAPI), From(new(*stmgr.StateManager))), + + Override(RunHelloKey, modules.RunHello), + Override(RunChainExchangeKey, modules.RunChainExchange), + Override(RunPeerMgrKey, modules.RunPeerMgr), + Override(HandleIncomingMessagesKey, modules.HandleIncomingMessages), + Override(HandleIncomingBlocksKey, modules.HandleIncomingBlocks), + ), + // miner - ApplyIf(func(s *Settings) bool { return s.nodeType == repo.StorageMiner }, + ApplyIf(isType(repo.StorageMiner), Override(new(api.Common), From(new(common.CommonAPI))), Override(new(sectorstorage.StorageAuth), modules.StorageAuth), @@ -402,23 +417,6 @@ func StorageMiner(out *api.StorageMiner) Option { ) } -func LiteModeOverrides(gapi api.GatewayAPI) Option { - return Options( - Override(new(messagesigner.MpoolNonceAPI), From(new(modules.MpoolNonceAPI))), - Override(new(api.GatewayAPI), gapi), - Override(new(full.ChainModuleAPI), From(new(api.GatewayAPI))), - Override(new(full.GasModuleAPI), From(new(api.GatewayAPI))), - Override(new(full.MpoolModuleAPI), From(new(api.GatewayAPI))), - Override(new(full.StateModuleAPI), From(new(api.GatewayAPI))), - Override(new(stmgr.StateManagerAPI), modules.NewRPCStateManager), - Unset(RunHelloKey), - Unset(RunChainExchangeKey), - Unset(RunPeerMgrKey), - Unset(HandleIncomingBlocksKey), - Unset(HandleIncomingMessagesKey), - ) -} - // Config sets up constructors based on the provided Config func ConfigCommon(cfg *config.Common) Option { return Options( @@ -533,10 +531,11 @@ func Repo(r repo.Repo) Option { } } -func FullAPI(out *api.FullNode) Option { +func FullAPI(out *api.FullNode, lite bool) Option { return Options( func(s *Settings) error { s.nodeType = repo.FullNode + s.Lite = lite return nil }, func(s *Settings) error { diff --git a/node/test/builder.go b/node/test/builder.go index ebf96e98f..cdb0b43dd 100644 --- a/node/test/builder.go +++ b/node/test/builder.go @@ -141,23 +141,23 @@ func CreateTestStorageNode(ctx context.Context, t *testing.T, waddr address.Addr return test.TestStorageNode{StorageMiner: minerapi, MineOne: mineOne} } -func Builder(t *testing.T, fullOpts []test.OptionGenerator, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { +func Builder(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { return mockBuilderOpts(t, fullOpts, storage, opts, false) } -func MockSbBuilder(t *testing.T, fullOpts []test.OptionGenerator, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { +func MockSbBuilder(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { return mockSbBuilderOpts(t, fullOpts, storage, opts, false) } -func RPCBuilder(t *testing.T, fullOpts []test.OptionGenerator, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { +func RPCBuilder(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { return mockBuilderOpts(t, fullOpts, storage, opts, true) } -func RPCMockSbBuilder(t *testing.T, fullOpts []test.OptionGenerator, storage []test.StorageMiner) ([]test.TestNode, []test.TestStorageNode) { +func RPCMockSbBuilder(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner) ([]test.TestNode, []test.TestStorageNode) { return mockSbBuilderOpts(t, fullOpts, storage, []node.Option{}, true) } -func mockBuilderOpts(t *testing.T, fullOpts []test.OptionGenerator, storage []test.StorageMiner, opts []node.Option, rpc bool) ([]test.TestNode, []test.TestStorageNode) { +func mockBuilderOpts(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner, opts []node.Option, rpc bool) ([]test.TestNode, []test.TestStorageNode) { ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) @@ -237,14 +237,15 @@ func mockBuilderOpts(t *testing.T, fullOpts []test.OptionGenerator, storage []te } stop, err := node.New(ctx, - node.FullAPI(&fulls[i].FullNode), + node.FullAPI(&fulls[i].FullNode, fullOpts[i].Lite), node.Online(), node.Repo(repo.NewMemory(nil)), node.MockHost(mn), node.Test(), genesis, - fullOpts[i](fulls), + + fullOpts[i].Opts(fulls), node.Options(opts...), ) if err != nil { @@ -315,7 +316,7 @@ func mockBuilderOpts(t *testing.T, fullOpts []test.OptionGenerator, storage []te return fulls, storers } -func mockSbBuilderOpts(t *testing.T, fullOpts []test.OptionGenerator, storage []test.StorageMiner, options []node.Option, rpc bool) ([]test.TestNode, []test.TestStorageNode) { +func mockSbBuilderOpts(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner, options []node.Option, rpc bool) ([]test.TestNode, []test.TestStorageNode) { ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) @@ -393,10 +394,8 @@ func mockSbBuilderOpts(t *testing.T, fullOpts []test.OptionGenerator, storage [] genesis = node.Override(new(modules.Genesis), modules.LoadGenesis(genbuf.Bytes())) } - var err error - // TODO: Don't ignore stop stop, err := node.New(ctx, - node.FullAPI(&fulls[i].FullNode), + node.FullAPI(&fulls[i].FullNode, fullOpts[i].Lite), node.Online(), node.Repo(repo.NewMemory(nil)), node.MockHost(mn), @@ -405,7 +404,8 @@ func mockSbBuilderOpts(t *testing.T, fullOpts []test.OptionGenerator, storage [] node.Override(new(ffiwrapper.Verifier), mock.MockVerifier), genesis, - fullOpts[i](fulls), + + fullOpts[i].Opts(fulls), node.Options(options...), ) if err != nil { From 200a95f82420757840b5c28a7e7a27c43e347ff1 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Thu, 8 Oct 2020 12:21:15 +0200 Subject: [PATCH 664/885] refactor: unify test builder full node options --- api/test/ccupgrade.go | 9 +-------- api/test/test.go | 17 ++++++++++++++++- api/test/window_post.go | 8 +------- cmd/lotus-storage-miner/allinfo_test.go | 6 ++---- node/test/builder.go | 20 +++++++++----------- 5 files changed, 29 insertions(+), 31 deletions(-) diff --git a/api/test/ccupgrade.go b/api/test/ccupgrade.go index 38342d7fe..4a860c661 100644 --- a/api/test/ccupgrade.go +++ b/api/test/ccupgrade.go @@ -12,10 +12,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/node" "github.com/filecoin-project/lotus/node/impl" ) @@ -37,11 +34,7 @@ func TestCCUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration) { func testCCUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration, upgradeHeight abi.ChainEpoch) { ctx := context.Background() - n, sn := b(t, OneFull, OneMiner, node.Override(new(stmgr.UpgradeSchedule), stmgr.UpgradeSchedule{{ - Network: build.ActorUpgradeNetworkVersion, - Height: upgradeHeight, - Migration: stmgr.UpgradeActorsV2, - }})) + n, sn := b(t, []FullNodeOpts{FullNodeWithUpgradeAt(upgradeHeight)}, OneMiner) client := n[0].FullNode.(*impl.FullNodeAPI) miner := sn[0] diff --git a/api/test/test.go b/api/test/test.go index 014859865..35b397740 100644 --- a/api/test/test.go +++ b/api/test/test.go @@ -4,11 +4,14 @@ import ( "context" "testing" + "github.com/filecoin-project/lotus/chain/stmgr" + "github.com/multiformats/go-multiaddr" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/miner" @@ -55,7 +58,7 @@ type FullNodeOpts struct { // fullOpts array defines options for each full node // storage array defines storage nodes, numbers in the array specify full node // index the storage node 'belongs' to -type APIBuilder func(t *testing.T, full []FullNodeOpts, storage []StorageMiner, opts ...node.Option) ([]TestNode, []TestStorageNode) +type APIBuilder func(t *testing.T, full []FullNodeOpts, storage []StorageMiner) ([]TestNode, []TestStorageNode) type testSuite struct { makeNodes APIBuilder } @@ -89,6 +92,18 @@ var OneMiner = []StorageMiner{{Full: 0, Preseal: PresealGenesis}} var OneFull = DefaultFullOpts(1) var TwoFull = DefaultFullOpts(2) +var FullNodeWithUpgradeAt = func(upgradeHeight abi.ChainEpoch) FullNodeOpts { + return FullNodeOpts{ + Opts: func(nodes []TestNode) node.Option { + return node.Override(new(stmgr.UpgradeSchedule), stmgr.UpgradeSchedule{{ + Network: build.ActorUpgradeNetworkVersion, + Height: upgradeHeight, + Migration: stmgr.UpgradeActorsV2, + }}) + }, + } +} + func (ts *testSuite) testVersion(t *testing.T) { build.RunningNodeType = build.NodeFull diff --git a/api/test/window_post.go b/api/test/window_post.go index c45d5e1bc..7bc56a562 100644 --- a/api/test/window_post.go +++ b/api/test/window_post.go @@ -15,11 +15,9 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/extern/sector-storage/mock" sealing "github.com/filecoin-project/lotus/extern/storage-sealing" - "github.com/filecoin-project/lotus/node" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/types" bminer "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node/impl" @@ -133,11 +131,7 @@ func testWindowPostUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration, ctx, cancel := context.WithCancel(context.Background()) defer cancel() - n, sn := b(t, OneFull, OneMiner, node.Override(new(stmgr.UpgradeSchedule), stmgr.UpgradeSchedule{{ - Network: build.ActorUpgradeNetworkVersion, - Height: upgradeHeight, - Migration: stmgr.UpgradeActorsV2, - }})) + n, sn := b(t, []FullNodeOpts{FullNodeWithUpgradeAt(upgradeHeight)}, OneMiner) client := n[0].FullNode.(*impl.FullNodeAPI) miner := sn[0] diff --git a/cmd/lotus-storage-miner/allinfo_test.go b/cmd/lotus-storage-miner/allinfo_test.go index fbe1d1617..a458c024b 100644 --- a/cmd/lotus-storage-miner/allinfo_test.go +++ b/cmd/lotus-storage-miner/allinfo_test.go @@ -5,8 +5,6 @@ import ( "testing" "time" - "github.com/filecoin-project/lotus/node" - logging "github.com/ipfs/go-log/v2" "github.com/stretchr/testify/require" "github.com/urfave/cli/v2" @@ -64,8 +62,8 @@ func TestMinerAllInfo(t *testing.T) { require.NoError(t, infoAllCmd.Action(cctx)) } - bp := func(t *testing.T, fullOpts []test.OptionGenerator, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { - n, sn = builder.Builder(t, fullOpts, storage, opts...) + bp := func(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner) ([]test.TestNode, []test.TestStorageNode) { + n, sn = builder.Builder(t, fullOpts, storage) t.Run("pre-info-all", run) diff --git a/node/test/builder.go b/node/test/builder.go index cdb0b43dd..289fae841 100644 --- a/node/test/builder.go +++ b/node/test/builder.go @@ -141,23 +141,23 @@ func CreateTestStorageNode(ctx context.Context, t *testing.T, waddr address.Addr return test.TestStorageNode{StorageMiner: minerapi, MineOne: mineOne} } -func Builder(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { - return mockBuilderOpts(t, fullOpts, storage, opts, false) +func Builder(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner) ([]test.TestNode, []test.TestStorageNode) { + return mockBuilderOpts(t, fullOpts, storage, false) } -func MockSbBuilder(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { - return mockSbBuilderOpts(t, fullOpts, storage, opts, false) +func MockSbBuilder(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner) ([]test.TestNode, []test.TestStorageNode) { + return mockSbBuilderOpts(t, fullOpts, storage, false) } -func RPCBuilder(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) { - return mockBuilderOpts(t, fullOpts, storage, opts, true) +func RPCBuilder(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner) ([]test.TestNode, []test.TestStorageNode) { + return mockBuilderOpts(t, fullOpts, storage, true) } func RPCMockSbBuilder(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner) ([]test.TestNode, []test.TestStorageNode) { - return mockSbBuilderOpts(t, fullOpts, storage, []node.Option{}, true) + return mockSbBuilderOpts(t, fullOpts, storage, true) } -func mockBuilderOpts(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner, opts []node.Option, rpc bool) ([]test.TestNode, []test.TestStorageNode) { +func mockBuilderOpts(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner, rpc bool) ([]test.TestNode, []test.TestStorageNode) { ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) @@ -246,7 +246,6 @@ func mockBuilderOpts(t *testing.T, fullOpts []test.FullNodeOpts, storage []test. genesis, fullOpts[i].Opts(fulls), - node.Options(opts...), ) if err != nil { t.Fatal(err) @@ -316,7 +315,7 @@ func mockBuilderOpts(t *testing.T, fullOpts []test.FullNodeOpts, storage []test. return fulls, storers } -func mockSbBuilderOpts(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner, options []node.Option, rpc bool) ([]test.TestNode, []test.TestStorageNode) { +func mockSbBuilderOpts(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner, rpc bool) ([]test.TestNode, []test.TestStorageNode) { ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) @@ -406,7 +405,6 @@ func mockSbBuilderOpts(t *testing.T, fullOpts []test.FullNodeOpts, storage []tes genesis, fullOpts[i].Opts(fulls), - node.Options(options...), ) if err != nil { t.Fatalf("%+v", err) From 7f7c9e978fc7641eecdb6c7271b468d32f217b4f Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Thu, 8 Oct 2020 13:25:51 +0200 Subject: [PATCH 665/885] feat: hide lite mode flag --- cmd/lotus/daemon.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 084f75ef2..298cb7a67 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -114,8 +114,9 @@ var DaemonCmd = &cli.Command{ Usage: "halt the process after importing chain from file", }, &cli.BoolFlag{ - Name: "lite", - Usage: "start lotus in lite mode", + Name: "lite", + Usage: "start lotus in lite mode", + Hidden: true, }, &cli.StringFlag{ Name: "pprof", From 4cca3f19a361a5e02564208f35096722cbcba387 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Fri, 9 Oct 2020 09:40:25 -0700 Subject: [PATCH 666/885] add an api for removing multisig signers --- api/api_full.go | 22 ++++++++++-- api/apistruct/struct.go | 16 +++++++-- cli/multisig.go | 4 +-- node/impl/full/multisig.go | 73 ++++++++++++++++++++++++++++++++++---- 4 files changed, 101 insertions(+), 14 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 601b14660..7b854a3e4 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -432,12 +432,21 @@ type FullNode interface { // It takes the following params: , , , // , , MsigPropose(context.Context, address.Address, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error) - // MsigApprove approves a previously-proposed multisig message + + // MsigApprove approves a previously-proposed multisig message by transaction ID + // It takes the following params: , + MsigApprove(context.Context, address.Address, uint64, address.Address) (cid.Cid, error) + + // MsigApproveTxnHash approves a previously-proposed multisig message, specified + // using both transaction ID and a hash of the parameters used in the + // proposal. This method of approval can be used to ensure you only approve + // exactly the transaction you think you are. // It takes the following params: , , , , , // , , - MsigApprove(context.Context, address.Address, uint64, address.Address, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error) + MsigApproveTxnHash(context.Context, address.Address, uint64, address.Address, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error) + // MsigCancel cancels a previously-proposed multisig message - // It takes the following params: , , , , + // It takes the following params: , , , , // , , MsigCancel(context.Context, address.Address, uint64, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error) // MsigAddPropose proposes adding a signer in the multisig @@ -465,6 +474,13 @@ type FullNode interface { // , MsigSwapCancel(context.Context, address.Address, address.Address, uint64, address.Address, address.Address) (cid.Cid, error) + // MsigRemoveSigner proposes the removal of a signer from the multisig. + // It accepts the multisig to make the change on, the proposer address to + // send the message from, the address to be removed, and a boolean + // indicating whether or not the signing threshold should be lowered by one + // along with the address removal. + MsigRemoveSigner(ctx context.Context, msig address.Address, proposer address.Address, toRemove address.Address, decrease bool) (cid.Cid, error) + MarketEnsureAvailable(context.Context, address.Address, address.Address, types.BigInt) (cid.Cid, error) // MarketFreeBalance diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 1a3ddda58..92e6ffe27 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -216,7 +216,8 @@ type FullNodeStruct struct { MsigGetVested func(context.Context, address.Address, types.TipSetKey, types.TipSetKey) (types.BigInt, error) `perm:"read"` MsigCreate func(context.Context, uint64, []address.Address, abi.ChainEpoch, types.BigInt, address.Address, types.BigInt) (cid.Cid, error) `perm:"sign"` MsigPropose func(context.Context, address.Address, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error) `perm:"sign"` - MsigApprove func(context.Context, address.Address, uint64, address.Address, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error) `perm:"sign"` + MsigApprove func(context.Context, address.Address, uint64, address.Address) (cid.Cid, error) `perm:"sign"` + MsigApproveTxnHash func(context.Context, address.Address, uint64, address.Address, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error) `perm:"sign"` MsigCancel func(context.Context, address.Address, uint64, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error) `perm:"sign"` MsigAddPropose func(context.Context, address.Address, address.Address, address.Address, bool) (cid.Cid, error) `perm:"sign"` MsigAddApprove func(context.Context, address.Address, address.Address, uint64, address.Address, address.Address, bool) (cid.Cid, error) `perm:"sign"` @@ -224,6 +225,7 @@ type FullNodeStruct struct { MsigSwapPropose func(context.Context, address.Address, address.Address, address.Address, address.Address) (cid.Cid, error) `perm:"sign"` MsigSwapApprove func(context.Context, address.Address, address.Address, uint64, address.Address, address.Address, address.Address) (cid.Cid, error) `perm:"sign"` MsigSwapCancel func(context.Context, address.Address, address.Address, uint64, address.Address, address.Address) (cid.Cid, error) `perm:"sign"` + MsigRemoveSigner func(ctx context.Context, msig address.Address, proposer address.Address, toRemove address.Address, decrease bool) (cid.Cid, error) `perm:"sign"` MarketEnsureAvailable func(context.Context, address.Address, address.Address, types.BigInt) (cid.Cid, error) `perm:"sign"` @@ -950,8 +952,12 @@ func (c *FullNodeStruct) MsigPropose(ctx context.Context, msig address.Address, return c.Internal.MsigPropose(ctx, msig, to, amt, src, method, params) } -func (c *FullNodeStruct) MsigApprove(ctx context.Context, msig address.Address, txID uint64, proposer address.Address, to address.Address, amt types.BigInt, src address.Address, method uint64, params []byte) (cid.Cid, error) { - return c.Internal.MsigApprove(ctx, msig, txID, proposer, to, amt, src, method, params) +func (c *FullNodeStruct) MsigApprove(ctx context.Context, msig address.Address, txID uint64, signer address.Address) (cid.Cid, error) { + return c.Internal.MsigApprove(ctx, msig, txID, signer) +} + +func (c *FullNodeStruct) MsigApproveTxnHash(ctx context.Context, msig address.Address, txID uint64, proposer address.Address, to address.Address, amt types.BigInt, src address.Address, method uint64, params []byte) (cid.Cid, error) { + return c.Internal.MsigApproveTxnHash(ctx, msig, txID, proposer, to, amt, src, method, params) } func (c *FullNodeStruct) MsigCancel(ctx context.Context, msig address.Address, txID uint64, to address.Address, amt types.BigInt, src address.Address, method uint64, params []byte) (cid.Cid, error) { @@ -982,6 +988,10 @@ func (c *FullNodeStruct) MsigSwapCancel(ctx context.Context, msig address.Addres return c.Internal.MsigSwapCancel(ctx, msig, src, txID, oldAdd, newAdd) } +func (c *FullNodeStruct) MsigRemoveSigner(ctx context.Context, msig address.Address, proposer address.Address, toRemove address.Address, decrease bool) (cid.Cid, error) { + return c.Internal.MsigRemoveSigner(ctx, msig, proposer, toRemove, decrease) +} + func (c *FullNodeStruct) MarketEnsureAvailable(ctx context.Context, addr, wallet address.Address, amt types.BigInt) (cid.Cid, error) { return c.Internal.MarketEnsureAvailable(ctx, addr, wallet, amt) } diff --git a/cli/multisig.go b/cli/multisig.go index 6b3867cb3..b692fad5a 100644 --- a/cli/multisig.go +++ b/cli/multisig.go @@ -484,7 +484,7 @@ var msigApproveCmd = &cli.Command{ from = defaddr } - msgCid, err := api.MsigApprove(ctx, msig, txid, proposer, dest, types.BigInt(value), from, method, params) + msgCid, err := api.MsigApproveTxnHash(ctx, msig, txid, proposer, dest, types.BigInt(value), from, method, params) if err != nil { return err } @@ -1129,7 +1129,7 @@ var msigLockApproveCmd = &cli.Command{ return actErr } - msgCid, err := api.MsigApprove(ctx, msig, txid, prop, msig, big.Zero(), from, uint64(builtin2.MethodsMultisig.LockBalance), params) + msgCid, err := api.MsigApproveTxnHash(ctx, msig, txid, prop, msig, big.Zero(), from, uint64(builtin2.MethodsMultisig.LockBalance), params) if err != nil { return err } diff --git a/node/impl/full/multisig.go b/node/impl/full/multisig.go index 715689edc..d52f29a30 100644 --- a/node/impl/full/multisig.go +++ b/node/impl/full/multisig.go @@ -95,7 +95,7 @@ func (a *MsigAPI) MsigAddApprove(ctx context.Context, msig address.Address, src return cid.Undef, actErr } - return a.MsigApprove(ctx, msig, txID, proposer, msig, big.Zero(), src, uint64(builtin0.MethodsMultisig.AddSigner), enc) + return a.MsigApproveTxnHash(ctx, msig, txID, proposer, msig, big.Zero(), src, uint64(builtin0.MethodsMultisig.AddSigner), enc) } func (a *MsigAPI) MsigAddCancel(ctx context.Context, msig address.Address, src address.Address, txID uint64, newAdd address.Address, inc bool) (cid.Cid, error) { @@ -122,7 +122,7 @@ func (a *MsigAPI) MsigSwapApprove(ctx context.Context, msig address.Address, src return cid.Undef, actErr } - return a.MsigApprove(ctx, msig, txID, proposer, msig, big.Zero(), src, uint64(builtin0.MethodsMultisig.SwapSigner), enc) + return a.MsigApproveTxnHash(ctx, msig, txID, proposer, msig, big.Zero(), src, uint64(builtin0.MethodsMultisig.SwapSigner), enc) } func (a *MsigAPI) MsigSwapCancel(ctx context.Context, msig address.Address, src address.Address, txID uint64, oldAdd address.Address, newAdd address.Address) (cid.Cid, error) { @@ -134,15 +134,64 @@ func (a *MsigAPI) MsigSwapCancel(ctx context.Context, msig address.Address, src return a.MsigCancel(ctx, msig, txID, msig, big.Zero(), src, uint64(builtin0.MethodsMultisig.SwapSigner), enc) } -func (a *MsigAPI) MsigApprove(ctx context.Context, msig address.Address, txID uint64, proposer address.Address, to address.Address, amt types.BigInt, src address.Address, method uint64, params []byte) (cid.Cid, error) { - return a.msigApproveOrCancel(ctx, api.MsigApprove, msig, txID, proposer, to, amt, src, method, params) +func (a *MsigAPI) MsigApprove(ctx context.Context, msig address.Address, txID uint64, src address.Address) (cid.Cid, error) { + return a.msigApproveOrCancelSimple(ctx, api.MsigApprove, msig, txID, src) +} + +func (a *MsigAPI) MsigApproveTxnHash(ctx context.Context, msig address.Address, txID uint64, proposer address.Address, to address.Address, amt types.BigInt, src address.Address, method uint64, params []byte) (cid.Cid, error) { + return a.msigApproveOrCancelTxnHash(ctx, api.MsigApprove, msig, txID, proposer, to, amt, src, method, params) } func (a *MsigAPI) MsigCancel(ctx context.Context, msig address.Address, txID uint64, to address.Address, amt types.BigInt, src address.Address, method uint64, params []byte) (cid.Cid, error) { - return a.msigApproveOrCancel(ctx, api.MsigCancel, msig, txID, src, to, amt, src, method, params) + return a.msigApproveOrCancelTxnHash(ctx, api.MsigCancel, msig, txID, src, to, amt, src, method, params) } -func (a *MsigAPI) msigApproveOrCancel(ctx context.Context, operation api.MsigProposeResponse, msig address.Address, txID uint64, proposer address.Address, to address.Address, amt types.BigInt, src address.Address, method uint64, params []byte) (cid.Cid, error) { +func (a *MsigAPI) MsigRemoveSigner(ctx context.Context, msig address.Address, proposer address.Address, toRemove address.Address, decrease bool) (cid.Cid, error) { + enc, actErr := serializeRemoveParams(toRemove, decrease) + if actErr != nil { + return cid.Undef, actErr + } + + return a.MsigPropose(ctx, msig, msig, types.NewInt(0), proposer, uint64(builtin0.MethodsMultisig.RemoveSigner), enc) +} + +func (a *MsigAPI) msigApproveOrCancelSimple(ctx context.Context, operation api.MsigProposeResponse, msig address.Address, txID uint64, src address.Address) (cid.Cid, error) { + if msig == address.Undef { + return cid.Undef, xerrors.Errorf("must provide multisig address") + } + + if src == address.Undef { + return cid.Undef, xerrors.Errorf("must provide source address") + } + + mb, err := a.messageBuilder(ctx, src) + if err != nil { + return cid.Undef, err + } + + var msg *types.Message + switch operation { + case api.MsigApprove: + msg, err = mb.Approve(msig, txID, nil) + case api.MsigCancel: + msg, err = mb.Cancel(msig, txID, nil) + default: + return cid.Undef, xerrors.Errorf("Invalid operation for msigApproveOrCancel") + } + if err != nil { + return cid.Undef, err + } + + smsg, err := a.MpoolAPI.MpoolPushMessage(ctx, msg, nil) + if err != nil { + return cid.Undef, err + } + + return smsg.Cid(), nil + +} + +func (a *MsigAPI) msigApproveOrCancelTxnHash(ctx context.Context, operation api.MsigProposeResponse, msig address.Address, txID uint64, proposer address.Address, to address.Address, amt types.BigInt, src address.Address, method uint64, params []byte) (cid.Cid, error) { if msig == address.Undef { return cid.Undef, xerrors.Errorf("must provide multisig address") } @@ -216,3 +265,15 @@ func serializeSwapParams(old address.Address, new address.Address) ([]byte, erro return enc, nil } + +func serializeRemoveParams(rem address.Address, dec bool) ([]byte, error) { + enc, actErr := actors.SerializeParams(&multisig0.RemoveSignerParams{ + Signer: rem, + Decrease: dec, + }) + if actErr != nil { + return nil, actErr + } + + return enc, nil +} From ab8286fa381c63d13e3e79609ba7659e2974e429 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 9 Oct 2020 18:43:22 +0200 Subject: [PATCH 667/885] Fix docsgen, lotus-soup build --- chain/sync_test.go | 4 +-- cmd/lotus/daemon.go | 2 +- documentation/en/api-methods.md | 44 +++++++++++++++++++++++++++++++++ node/builder.go | 13 ++++++++-- node/test/builder.go | 4 +-- 5 files changed, 60 insertions(+), 7 deletions(-) diff --git a/chain/sync_test.go b/chain/sync_test.go index 1fe90a071..318f6d82f 100644 --- a/chain/sync_test.go +++ b/chain/sync_test.go @@ -222,7 +222,7 @@ func (tu *syncTestUtil) addSourceNode(gen int) { var out api.FullNode stop, err := node.New(tu.ctx, - node.FullAPI(&out, false), + node.FullAPI(&out), node.Online(), node.Repo(sourceRepo), node.MockHost(tu.mn), @@ -254,7 +254,7 @@ func (tu *syncTestUtil) addClientNode() int { var out api.FullNode stop, err := node.New(tu.ctx, - node.FullAPI(&out, false), + node.FullAPI(&out), node.Online(), node.Repo(repo.NewMemory(nil)), node.MockHost(tu.mn), diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 298cb7a67..5964a2458 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -264,7 +264,7 @@ var DaemonCmd = &cli.Command{ var api api.FullNode stop, err := node.New(ctx, - node.FullAPI(&api, isLite), + node.FullAPI(&api, node.Lite(isLite)), node.Override(new(dtypes.Bootstrapper), isBootstrapper), node.Override(new(dtypes.ShutdownChan), shutdownChan), diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index 8a288a4bf..cc6d92e4f 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -167,6 +167,7 @@ * [StateVerifiedRegistryRootKey](#StateVerifiedRegistryRootKey) * [StateVerifierStatus](#StateVerifierStatus) * [StateWaitMsg](#StateWaitMsg) + * [StateWaitMsgLimited](#StateWaitMsgLimited) * [Sync](#Sync) * [SyncCheckBad](#SyncCheckBad) * [SyncCheckpoint](#SyncCheckpoint) @@ -4322,6 +4323,49 @@ Response: } ``` +### StateWaitMsgLimited +StateWaitMsgLimited looks back up to limit epochs in the chain for a message. +If not found, it blocks until the message arrives on chain, and gets to the +indicated confidence depth. + + +Perms: read + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + 42, + 10101 +] +``` + +Response: +```json +{ + "Message": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Receipt": { + "ExitCode": 0, + "Return": "Ynl0ZSBhcnJheQ==", + "GasUsed": 9 + }, + "ReturnDec": {}, + "TipSet": [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ], + "Height": 10101 +} +``` + ## Sync The Sync method group contains methods for interacting with and observing the lotus sync service. diff --git a/node/builder.go b/node/builder.go index 452104ba8..71295dbc8 100644 --- a/node/builder.go +++ b/node/builder.go @@ -531,13 +531,22 @@ func Repo(r repo.Repo) Option { } } -func FullAPI(out *api.FullNode, lite bool) Option { +type FullOption = Option + +func Lite(enable bool) FullOption { + return func(s *Settings) error { + s.Lite = enable + return nil + } +} + +func FullAPI(out *api.FullNode, fopts ...FullOption) Option { return Options( func(s *Settings) error { s.nodeType = repo.FullNode - s.Lite = lite return nil }, + Options(fopts...), func(s *Settings) error { resAPI := &impl.FullNodeAPI{} s.invokes[ExtractApiKey] = fx.Populate(resAPI) diff --git a/node/test/builder.go b/node/test/builder.go index 289fae841..4aa8a55ea 100644 --- a/node/test/builder.go +++ b/node/test/builder.go @@ -237,7 +237,7 @@ func mockBuilderOpts(t *testing.T, fullOpts []test.FullNodeOpts, storage []test. } stop, err := node.New(ctx, - node.FullAPI(&fulls[i].FullNode, fullOpts[i].Lite), + node.FullAPI(&fulls[i].FullNode, node.Lite(fullOpts[i].Lite)), node.Online(), node.Repo(repo.NewMemory(nil)), node.MockHost(mn), @@ -394,7 +394,7 @@ func mockSbBuilderOpts(t *testing.T, fullOpts []test.FullNodeOpts, storage []tes } stop, err := node.New(ctx, - node.FullAPI(&fulls[i].FullNode, fullOpts[i].Lite), + node.FullAPI(&fulls[i].FullNode, node.Lite(fullOpts[i].Lite)), node.Online(), node.Repo(repo.NewMemory(nil)), node.MockHost(mn), From f87d325b6fecab5a50bce36dc69d3c8f0e1b09db Mon Sep 17 00:00:00 2001 From: Ian Davis Date: Fri, 9 Oct 2020 17:23:40 +0100 Subject: [PATCH 668/885] fix: spec actors 2 reward shim uses correct field for CumsumRealized --- chain/actors/builtin/reward/v0.go | 2 +- chain/actors/builtin/reward/v2.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/chain/actors/builtin/reward/v0.go b/chain/actors/builtin/reward/v0.go index 0efd0b482..6a6e6d12e 100644 --- a/chain/actors/builtin/reward/v0.go +++ b/chain/actors/builtin/reward/v0.go @@ -57,7 +57,7 @@ func (s *state0) CumsumBaseline() (abi.StoragePower, error) { } func (s *state0) CumsumRealized() (abi.StoragePower, error) { - return s.State.CumsumBaseline, nil + return s.State.CumsumRealized, nil } func (s *state0) InitialPledgeForPower(sectorWeight abi.StoragePower, networkTotalPledge abi.TokenAmount, networkQAPower *builtin.FilterEstimate, circSupply abi.TokenAmount) (abi.TokenAmount, error) { diff --git a/chain/actors/builtin/reward/v2.go b/chain/actors/builtin/reward/v2.go index ec0709c39..b7cb49102 100644 --- a/chain/actors/builtin/reward/v2.go +++ b/chain/actors/builtin/reward/v2.go @@ -60,7 +60,7 @@ func (s *state2) CumsumBaseline() (abi.StoragePower, error) { } func (s *state2) CumsumRealized() (abi.StoragePower, error) { - return s.State.CumsumBaseline, nil + return s.State.CumsumRealized, nil } func (s *state2) InitialPledgeForPower(qaPower abi.StoragePower, networkTotalPledge abi.TokenAmount, networkQAPower *builtin.FilterEstimate, circSupply abi.TokenAmount) (abi.TokenAmount, error) { From 105aa40007f92bfd1b2e057731da76381d2ed6a2 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 9 Oct 2020 11:06:49 -0700 Subject: [PATCH 669/885] simplify message syncing logic 1. Allow duplicate blocks from bitswap. This shouldn't happen, but there's no reason to bail (just log loudly). 2. Simplify logic to very explicitly check to make sure we're fetching every block. --- chain/sub/incoming.go | 49 +++++++++++++++++++------------------------ 1 file changed, 22 insertions(+), 27 deletions(-) diff --git a/chain/sub/incoming.go b/chain/sub/incoming.go index d51c481d1..68ee5e20c 100644 --- a/chain/sub/incoming.go +++ b/chain/sub/incoming.go @@ -120,12 +120,6 @@ func FetchMessagesByCids( return err } - // FIXME: We already sort in `fetchCids`, we are duplicating too much work, - // we don't need to pass the index. - if out[i] != nil { - return fmt.Errorf("received duplicate message") - } - out[i] = msg return nil }) @@ -149,10 +143,6 @@ func FetchSignedMessagesByCids( return err } - if out[i] != nil { - return fmt.Errorf("received duplicate message") - } - out[i] = smsg return nil }) @@ -184,24 +174,29 @@ func fetchCids( return fmt.Errorf("duplicate CIDs in fetchCids input") } - fetchedBlocks := bserv.GetBlocks(ctx, cids) - - for i := 0; i < len(cids); i++ { - select { - case block, ok := <-fetchedBlocks: - if !ok { - return fmt.Errorf("failed to fetch all messages") - } - - ix, ok := cidIndex[block.Cid()] - if !ok { - return fmt.Errorf("received message we didnt ask for") - } - - if err := cb(ix, block); err != nil { - return err - } + for block := range bserv.GetBlocks(ctx, cids) { + ix, ok := cidIndex[block.Cid()] + if !ok { + // Ignore duplicate/unexpected blocks. This shouldn't + // happen, but we can be safe. + log.Errorw("received duplicate/unexpected block when syncing", "cid", block.Cid()) + continue } + + // Record that we've received the block. + delete(cidIndex, block.Cid()) + + if err := cb(ix, block); err != nil { + return err + } + } + + if len(cidIndex) > 0 { + err := ctx.Err() + if err == nil { + err = fmt.Errorf("failed to fetch %d messages for unknown reasons", len(cidIndex)) + } + return err } return nil From 9f9d5cb2a5645e16e269160f32852537c9502094 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 9 Oct 2020 11:50:16 -0700 Subject: [PATCH 670/885] fix a race in the payment channel status test We should not be clobbering the cmd variable. --- cli/paych_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/paych_test.go b/cli/paych_test.go index 2aa5c6009..862ca2e74 100644 --- a/cli/paych_test.go +++ b/cli/paych_test.go @@ -121,13 +121,13 @@ func TestPaymentChannelStatus(t *testing.T) { create := make(chan string) go func() { // creator: paych add-funds - cmd = []string{creatorAddr.String(), receiverAddr.String(), fmt.Sprintf("%d", channelAmt)} + cmd := []string{creatorAddr.String(), receiverAddr.String(), fmt.Sprintf("%d", channelAmt)} create <- creatorCLI.runCmd(paychAddFundsCmd, cmd) }() // Wait for the output to stop being "Channel does not exist" for regexp.MustCompile(noChannelState).MatchString(out) { - cmd = []string{creatorAddr.String(), receiverAddr.String()} + cmd := []string{creatorAddr.String(), receiverAddr.String()} out = creatorCLI.runCmd(paychStatusByFromToCmd, cmd) } fmt.Println(out) From d80e7eda932a2a28ec6c5cd6fdb0acdc1c22c966 Mon Sep 17 00:00:00 2001 From: Ingar Shu Date: Fri, 9 Oct 2020 13:18:43 -0700 Subject: [PATCH 671/885] Revert cached "available" amount if the AddFunds message send fails --- chain/market/fundmgr.go | 6 ++++-- chain/market/fundmgr_test.go | 38 +++++++++++++++++++++++------------- 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/chain/market/fundmgr.go b/chain/market/fundmgr.go index aef3b98eb..8b5f85a12 100644 --- a/chain/market/fundmgr.go +++ b/chain/market/fundmgr.go @@ -120,9 +120,10 @@ func (fm *FundMgr) EnsureAvailable(ctx context.Context, addr, wallet address.Add return cid.Undef, err } fm.lk.Lock() + defer fm.lk.Unlock() + bal, err := fm.api.StateMarketBalance(ctx, addr, types.EmptyTSK) if err != nil { - fm.lk.Unlock() return cid.Undef, err } @@ -138,7 +139,6 @@ func (fm *FundMgr) EnsureAvailable(ctx context.Context, addr, wallet address.Add toAdd = types.NewInt(0) } fm.available[idAddr] = big.Add(avail, toAdd) - fm.lk.Unlock() log.Infof("Funds operation w/ Expected Balance: %s, In State: %s, Requested: %s, Adding: %s", avail.String(), stateAvail.String(), amt.String(), toAdd.String()) @@ -148,6 +148,7 @@ func (fm *FundMgr) EnsureAvailable(ctx context.Context, addr, wallet address.Add params, err := actors.SerializeParams(&addr) if err != nil { + fm.available[idAddr] = avail return cid.Undef, err } @@ -159,6 +160,7 @@ func (fm *FundMgr) EnsureAvailable(ctx context.Context, addr, wallet address.Add Params: params, }, nil) if err != nil { + fm.available[idAddr] = avail return cid.Undef, err } diff --git a/chain/market/fundmgr_test.go b/chain/market/fundmgr_test.go index b05db55d8..f5936f73d 100644 --- a/chain/market/fundmgr_test.go +++ b/chain/market/fundmgr_test.go @@ -57,9 +57,10 @@ func addFundsMsg(toAdd abi.TokenAmount, addr address.Address, wallet address.Add } type expectedResult struct { - addAmt abi.TokenAmount - shouldAdd bool - err error + addAmt abi.TokenAmount + shouldAdd bool + err error + cachedAvailable abi.TokenAmount } func TestAddFunds(t *testing.T) { @@ -88,8 +89,9 @@ func TestAddFunds(t *testing.T) { addAmounts: []abi.TokenAmount{abi.NewTokenAmount(100)}, expectedResults: []expectedResult{ { - shouldAdd: false, - err: nil, + shouldAdd: false, + err: nil, + cachedAvailable: abi.NewTokenAmount(100), }, }, }, @@ -102,18 +104,21 @@ func TestAddFunds(t *testing.T) { err: nil, }, { - addAmt: abi.NewTokenAmount(100), - shouldAdd: true, - err: nil, + addAmt: abi.NewTokenAmount(100), + shouldAdd: true, + err: nil, + cachedAvailable: abi.NewTokenAmount(200), }, { - addAmt: abi.NewTokenAmount(50), - shouldAdd: true, - err: nil, + addAmt: abi.NewTokenAmount(50), + shouldAdd: true, + err: nil, + cachedAvailable: abi.NewTokenAmount(250), }, { - shouldAdd: false, - err: nil, + shouldAdd: false, + err: nil, + cachedAvailable: abi.NewTokenAmount(250), }, }, }, @@ -132,7 +137,8 @@ func TestAddFunds(t *testing.T) { addAmounts: []abi.TokenAmount{abi.NewTokenAmount(100)}, expectedResults: []expectedResult{ { - err: errors.New("something went wrong"), + err: errors.New("something went wrong"), + cachedAvailable: abi.NewTokenAmount(0), }, }, }, @@ -183,6 +189,10 @@ func TestAddFunds(t *testing.T) { } else { require.EqualError(t, err, expected.err.Error()) } + + if !expected.cachedAvailable.Nil() { + require.Equal(t, expected.cachedAvailable, fundMgr.available[addr]) + } } }) } From 748d2e82a766e4db46e1d6aabc4f6778ddc063e7 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 9 Oct 2020 12:52:04 -0700 Subject: [PATCH 672/885] unshare the journal Motivation: * Run lotus with the race detector enabled (primary motivation). * Allow multiple lotus nodes in a process (not a high priority). Previously, the journal was shared between all lotus instances, but it was initialized for every new node. This caused safety problems in tests (at a minimum). This patch explicitly passes the journal to all services that need it. --- chain/gen/gen.go | 6 ++++-- chain/gen/genesis/genesis.go | 8 ++++++-- chain/messagepool/messagepool.go | 18 ++++++++++++------ chain/messagepool/messagepool_test.go | 18 +++++++++--------- chain/messagepool/repub.go | 3 +-- chain/messagepool/repub_test.go | 2 +- chain/messagepool/selection_test.go | 2 +- chain/store/index_test.go | 2 +- chain/store/store.go | 11 ++++++++--- chain/store/store_test.go | 4 ++-- cmd/lotus-bench/import.go | 2 +- cmd/lotus-shed/balances.go | 4 ++-- cmd/lotus-shed/export.go | 2 +- cmd/lotus-shed/genesis-verify.go | 2 +- cmd/lotus-shed/pruning.go | 2 +- cmd/lotus-storage-miner/init.go | 7 +++---- cmd/lotus/daemon.go | 7 ++++++- conformance/driver.go | 2 +- journal/env.go | 19 +++++++++++++++++++ journal/global.go | 9 --------- markets/journal.go | 16 ++++++++-------- miner/miner.go | 8 +++++--- miner/testminer.go | 2 ++ node/builder.go | 18 +----------------- node/modules/chain.go | 9 +++++---- node/modules/client.go | 12 ++++++------ node/modules/storageminer.go | 22 ++++++++++++---------- node/modules/testing/genesis.go | 13 +++++++------ storage/miner.go | 9 ++++++--- storage/wdpost_run.go | 13 ++++++------- storage/wdpost_run_test.go | 2 ++ storage/wdpost_sched.go | 14 ++++++++------ 32 files changed, 148 insertions(+), 120 deletions(-) create mode 100644 journal/env.go delete mode 100644 journal/global.go diff --git a/chain/gen/gen.go b/chain/gen/gen.go index 3ebd127f3..a1d9a3f7e 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -39,6 +39,7 @@ import ( "github.com/filecoin-project/lotus/cmd/lotus-seed/seed" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/genesis" + "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/lib/blockstore" "github.com/filecoin-project/lotus/lib/sigs" "github.com/filecoin-project/lotus/node/repo" @@ -122,6 +123,7 @@ var DefaultRemainderAccountActor = genesis.Actor{ } func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) { + j := journal.NilJournal() // TODO: we really shouldn't modify a global variable here. policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg2KiBV1) @@ -229,12 +231,12 @@ func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) { Timestamp: uint64(build.Clock.Now().Add(-500 * time.Duration(build.BlockDelaySecs) * time.Second).Unix()), } - genb, err := genesis2.MakeGenesisBlock(context.TODO(), bs, sys, tpl) + genb, err := genesis2.MakeGenesisBlock(context.TODO(), j, bs, sys, tpl) if err != nil { return nil, xerrors.Errorf("make genesis block failed: %w", err) } - cs := store.NewChainStore(bs, ds, sys) + cs := store.NewChainStore(bs, ds, sys, j) genfb := &types.FullBlock{Header: genb.Genesis} gents := store.NewFullTipSet([]*types.FullBlock{genfb}) diff --git a/chain/gen/genesis/genesis.go b/chain/gen/genesis/genesis.go index 9f15ecaed..6a1090784 100644 --- a/chain/gen/genesis/genesis.go +++ b/chain/gen/genesis/genesis.go @@ -7,6 +7,7 @@ import ( "fmt" "github.com/filecoin-project/lotus/chain/actors/builtin" + "github.com/filecoin-project/lotus/journal" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" @@ -466,7 +467,10 @@ func VerifyPreSealedData(ctx context.Context, cs *store.ChainStore, stateroot ci return st, nil } -func MakeGenesisBlock(ctx context.Context, bs bstore.Blockstore, sys vm.SyscallBuilder, template genesis.Template) (*GenesisBootstrap, error) { +func MakeGenesisBlock(ctx context.Context, j journal.Journal, bs bstore.Blockstore, sys vm.SyscallBuilder, template genesis.Template) (*GenesisBootstrap, error) { + if j == nil { + j = journal.NilJournal() + } st, keyIDs, err := MakeInitialStateTree(ctx, bs, template) if err != nil { return nil, xerrors.Errorf("make initial state tree failed: %w", err) @@ -478,7 +482,7 @@ func MakeGenesisBlock(ctx context.Context, bs bstore.Blockstore, sys vm.SyscallB } // temp chainstore - cs := store.NewChainStore(bs, datastore.NewMapDatastore(), sys) + cs := store.NewChainStore(bs, datastore.NewMapDatastore(), sys, j) // Verify PreSealed Data stateroot, err = VerifyPreSealedData(ctx, cs, stateroot, template, keyIDs) diff --git a/chain/messagepool/messagepool.go b/chain/messagepool/messagepool.go index 83aa5c6b7..d3c638b22 100644 --- a/chain/messagepool/messagepool.go +++ b/chain/messagepool/messagepool.go @@ -159,6 +159,7 @@ type MessagePool struct { sigValCache *lru.TwoQueueCache evtTypes [3]journal.EventType + journal journal.Journal } type msgSet struct { @@ -316,7 +317,7 @@ func (ms *msgSet) getRequiredFunds(nonce uint64) types.BigInt { return types.BigInt{Int: requiredFunds} } -func New(api Provider, ds dtypes.MetadataDS, netName dtypes.NetworkName) (*MessagePool, error) { +func New(api Provider, ds dtypes.MetadataDS, netName dtypes.NetworkName, j journal.Journal) (*MessagePool, error) { cache, _ := lru.New2Q(build.BlsSignatureCacheSize) verifcache, _ := lru.New2Q(build.VerifSigCacheSize) @@ -325,6 +326,10 @@ func New(api Provider, ds dtypes.MetadataDS, netName dtypes.NetworkName) (*Messa return nil, xerrors.Errorf("error loading mpool config: %w", err) } + if j == nil { + j = journal.NilJournal() + } + mp := &MessagePool{ ds: ds, addSema: make(chan struct{}, 1), @@ -344,10 +349,11 @@ func New(api Provider, ds dtypes.MetadataDS, netName dtypes.NetworkName) (*Messa netName: netName, cfg: cfg, evtTypes: [...]journal.EventType{ - evtTypeMpoolAdd: journal.J.RegisterEventType("mpool", "add"), - evtTypeMpoolRemove: journal.J.RegisterEventType("mpool", "remove"), - evtTypeMpoolRepub: journal.J.RegisterEventType("mpool", "repub"), + evtTypeMpoolAdd: j.RegisterEventType("mpool", "add"), + evtTypeMpoolRemove: j.RegisterEventType("mpool", "remove"), + evtTypeMpoolRepub: j.RegisterEventType("mpool", "repub"), }, + journal: j, } // enable initial prunes @@ -744,7 +750,7 @@ func (mp *MessagePool) addLocked(m *types.SignedMessage, strict, untrusted bool) Message: m, }, localUpdates) - journal.J.RecordEvent(mp.evtTypes[evtTypeMpoolAdd], func() interface{} { + mp.journal.RecordEvent(mp.evtTypes[evtTypeMpoolAdd], func() interface{} { return MessagePoolEvt{ Action: "add", Messages: []MessagePoolEvtMessage{{Message: m.Message, CID: m.Cid()}}, @@ -865,7 +871,7 @@ func (mp *MessagePool) remove(from address.Address, nonce uint64, applied bool) Message: m, }, localUpdates) - journal.J.RecordEvent(mp.evtTypes[evtTypeMpoolRemove], func() interface{} { + mp.journal.RecordEvent(mp.evtTypes[evtTypeMpoolRemove], func() interface{} { return MessagePoolEvt{ Action: "remove", Messages: []MessagePoolEvtMessage{{Message: m.Message, CID: m.Cid()}}} diff --git a/chain/messagepool/messagepool_test.go b/chain/messagepool/messagepool_test.go index a4aa059ca..afb1efde2 100644 --- a/chain/messagepool/messagepool_test.go +++ b/chain/messagepool/messagepool_test.go @@ -225,7 +225,7 @@ func TestMessagePool(t *testing.T) { ds := datastore.NewMapDatastore() - mp, err := New(tma, ds, "mptest") + mp, err := New(tma, ds, "mptest", nil) if err != nil { t.Fatal(err) } @@ -266,7 +266,7 @@ func TestMessagePoolMessagesInEachBlock(t *testing.T) { ds := datastore.NewMapDatastore() - mp, err := New(tma, ds, "mptest") + mp, err := New(tma, ds, "mptest", nil) if err != nil { t.Fatal(err) } @@ -315,7 +315,7 @@ func TestRevertMessages(t *testing.T) { ds := datastore.NewMapDatastore() - mp, err := New(tma, ds, "mptest") + mp, err := New(tma, ds, "mptest", nil) if err != nil { t.Fatal(err) } @@ -378,7 +378,7 @@ func TestPruningSimple(t *testing.T) { ds := datastore.NewMapDatastore() - mp, err := New(tma, ds, "mptest") + mp, err := New(tma, ds, "mptest", nil) if err != nil { t.Fatal(err) } @@ -422,7 +422,7 @@ func TestLoadLocal(t *testing.T) { tma := newTestMpoolAPI() ds := datastore.NewMapDatastore() - mp, err := New(tma, ds, "mptest") + mp, err := New(tma, ds, "mptest", nil) if err != nil { t.Fatal(err) } @@ -465,7 +465,7 @@ func TestLoadLocal(t *testing.T) { t.Fatal(err) } - mp, err = New(tma, ds, "mptest") + mp, err = New(tma, ds, "mptest", nil) if err != nil { t.Fatal(err) } @@ -494,7 +494,7 @@ func TestClearAll(t *testing.T) { tma := newTestMpoolAPI() ds := datastore.NewMapDatastore() - mp, err := New(tma, ds, "mptest") + mp, err := New(tma, ds, "mptest", nil) if err != nil { t.Fatal(err) } @@ -548,7 +548,7 @@ func TestClearNonLocal(t *testing.T) { tma := newTestMpoolAPI() ds := datastore.NewMapDatastore() - mp, err := New(tma, ds, "mptest") + mp, err := New(tma, ds, "mptest", nil) if err != nil { t.Fatal(err) } @@ -609,7 +609,7 @@ func TestUpdates(t *testing.T) { tma := newTestMpoolAPI() ds := datastore.NewMapDatastore() - mp, err := New(tma, ds, "mptest") + mp, err := New(tma, ds, "mptest", nil) if err != nil { t.Fatal(err) } diff --git a/chain/messagepool/repub.go b/chain/messagepool/repub.go index 672119ba9..cdd169e1d 100644 --- a/chain/messagepool/repub.go +++ b/chain/messagepool/repub.go @@ -11,7 +11,6 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/messagepool/gasguess" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/journal" "github.com/ipfs/go-cid" ) @@ -148,7 +147,7 @@ loop: } if len(msgs) > 0 { - journal.J.RecordEvent(mp.evtTypes[evtTypeMpoolRepub], func() interface{} { + mp.journal.RecordEvent(mp.evtTypes[evtTypeMpoolRepub], func() interface{} { msgsEv := make([]MessagePoolEvtMessage, 0, len(msgs)) for _, m := range msgs { msgsEv = append(msgsEv, MessagePoolEvtMessage{Message: m.Message, CID: m.Cid()}) diff --git a/chain/messagepool/repub_test.go b/chain/messagepool/repub_test.go index 491f484f5..5cb99f836 100644 --- a/chain/messagepool/repub_test.go +++ b/chain/messagepool/repub_test.go @@ -21,7 +21,7 @@ func TestRepubMessages(t *testing.T) { tma := newTestMpoolAPI() ds := datastore.NewMapDatastore() - mp, err := New(tma, ds, "mptest") + mp, err := New(tma, ds, "mptest", nil) if err != nil { t.Fatal(err) } diff --git a/chain/messagepool/selection_test.go b/chain/messagepool/selection_test.go index ea19dad9c..ca95b9621 100644 --- a/chain/messagepool/selection_test.go +++ b/chain/messagepool/selection_test.go @@ -58,7 +58,7 @@ func makeTestMessage(w *wallet.Wallet, from, to address.Address, nonce uint64, g func makeTestMpool() (*MessagePool, *testMpoolAPI) { tma := newTestMpoolAPI() ds := datastore.NewMapDatastore() - mp, err := New(tma, ds, "test") + mp, err := New(tma, ds, "test", nil) if err != nil { panic(err) } diff --git a/chain/store/index_test.go b/chain/store/index_test.go index 63e08070c..5283d10dc 100644 --- a/chain/store/index_test.go +++ b/chain/store/index_test.go @@ -31,7 +31,7 @@ func TestIndexSeeks(t *testing.T) { ctx := context.TODO() nbs := blockstore.NewTemporarySync() - cs := store.NewChainStore(nbs, syncds.MutexWrap(datastore.NewMapDatastore()), nil) + cs := store.NewChainStore(nbs, syncds.MutexWrap(datastore.NewMapDatastore()), nil, nil) _, err = cs.Import(bytes.NewReader(gencar)) if err != nil { diff --git a/chain/store/store.go b/chain/store/store.go index aac28e5d3..39f3d5c48 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -124,11 +124,15 @@ type ChainStore struct { vmcalls vm.SyscallBuilder evtTypes [1]journal.EventType + journal journal.Journal } -func NewChainStore(bs bstore.Blockstore, ds dstore.Batching, vmcalls vm.SyscallBuilder) *ChainStore { +func NewChainStore(bs bstore.Blockstore, ds dstore.Batching, vmcalls vm.SyscallBuilder, j journal.Journal) *ChainStore { c, _ := lru.NewARC(DefaultMsgMetaCacheSize) tsc, _ := lru.NewARC(DefaultTipSetCacheSize) + if j == nil { + j = journal.NilJournal() + } cs := &ChainStore{ bs: bs, ds: ds, @@ -137,10 +141,11 @@ func NewChainStore(bs bstore.Blockstore, ds dstore.Batching, vmcalls vm.SyscallB mmCache: c, tsCache: tsc, vmcalls: vmcalls, + journal: j, } cs.evtTypes = [1]journal.EventType{ - evtTypeHeadChange: journal.J.RegisterEventType("sync", "head_change"), + evtTypeHeadChange: j.RegisterEventType("sync", "head_change"), } ci := NewChainIndex(cs.LoadTipSet) @@ -379,7 +384,7 @@ func (cs *ChainStore) reorgWorker(ctx context.Context, initialNotifees []ReorgNo continue } - journal.J.RecordEvent(cs.evtTypes[evtTypeHeadChange], func() interface{} { + cs.journal.RecordEvent(cs.evtTypes[evtTypeHeadChange], func() interface{} { return HeadChangeEvt{ From: r.old.Key(), FromHeight: r.old.Height(), diff --git a/chain/store/store_test.go b/chain/store/store_test.go index b7adfb595..326899f90 100644 --- a/chain/store/store_test.go +++ b/chain/store/store_test.go @@ -62,7 +62,7 @@ func BenchmarkGetRandomness(b *testing.B) { bs := blockstore.NewBlockstore(bds) - cs := store.NewChainStore(bs, mds, nil) + cs := store.NewChainStore(bs, mds, nil, nil) b.ResetTimer() @@ -96,7 +96,7 @@ func TestChainExportImport(t *testing.T) { } nbs := blockstore.NewTemporary() - cs := store.NewChainStore(nbs, datastore.NewMapDatastore(), nil) + cs := store.NewChainStore(nbs, datastore.NewMapDatastore(), nil, nil) root, err := cs.Import(buf) if err != nil { diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index 3d93b0e5e..3f99d0453 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -183,7 +183,7 @@ var importBenchCmd = &cli.Command{ return nil } - cs := store.NewChainStore(bs, ds, vm.Syscalls(verifier)) + cs := store.NewChainStore(bs, ds, vm.Syscalls(verifier), nil) stm := stmgr.NewStateManager(cs) if cctx.Bool("global-profile") { diff --git a/cmd/lotus-shed/balances.go b/cmd/lotus-shed/balances.go index 26450bdb8..c2099bb2d 100644 --- a/cmd/lotus-shed/balances.go +++ b/cmd/lotus-shed/balances.go @@ -173,7 +173,7 @@ var chainBalanceStateCmd = &cli.Command{ bs := blockstore.NewBlockstore(ds) - cs := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier)) + cs := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier), nil) cst := cbor.NewCborStore(bs) store := adt.WrapStore(ctx, cst) @@ -343,7 +343,7 @@ var chainPledgeCmd = &cli.Command{ bs := blockstore.NewBlockstore(ds) - cs := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier)) + cs := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier), nil) cst := cbor.NewCborStore(bs) store := adt.WrapStore(ctx, cst) diff --git a/cmd/lotus-shed/export.go b/cmd/lotus-shed/export.go index c12cbd82d..3be49f0e0 100644 --- a/cmd/lotus-shed/export.go +++ b/cmd/lotus-shed/export.go @@ -83,7 +83,7 @@ var exportChainCmd = &cli.Command{ bs := blockstore.NewBlockstore(ds) - cs := store.NewChainStore(bs, mds, nil) + cs := store.NewChainStore(bs, mds, nil, nil) if err := cs.Load(); err != nil { return err } diff --git a/cmd/lotus-shed/genesis-verify.go b/cmd/lotus-shed/genesis-verify.go index 9a47d6561..4b197c58f 100644 --- a/cmd/lotus-shed/genesis-verify.go +++ b/cmd/lotus-shed/genesis-verify.go @@ -52,7 +52,7 @@ var genesisVerifyCmd = &cli.Command{ } bs := blockstore.NewBlockstore(datastore.NewMapDatastore()) - cs := store.NewChainStore(bs, datastore.NewMapDatastore(), nil) + cs := store.NewChainStore(bs, datastore.NewMapDatastore(), nil, nil) cf := cctx.Args().Get(0) f, err := os.Open(cf) diff --git a/cmd/lotus-shed/pruning.go b/cmd/lotus-shed/pruning.go index 79158c3a3..6cf4f8c6f 100644 --- a/cmd/lotus-shed/pruning.go +++ b/cmd/lotus-shed/pruning.go @@ -162,7 +162,7 @@ var stateTreePruneCmd = &cli.Command{ bs := blockstore.NewBlockstore(ds) - cs := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier)) + cs := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier), nil) if err := cs.Load(); err != nil { return fmt.Errorf("loading chainstore: %w", err) } diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index 9218faa77..0f830023f 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -464,13 +464,12 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api lapi.FullNode, return err } - if jrnl, err := journal.OpenFSJournal(lr, journal.DefaultDisabledEvents); err == nil { - journal.J = jrnl - } else { + j, err := journal.OpenFSJournal(lr, journal.EnvDisabledEvents()) + if err != nil { return fmt.Errorf("failed to open filesystem journal: %w", err) } - m := miner.NewMiner(api, epp, a, slashfilter.New(mds)) + m := miner.NewMiner(api, epp, a, slashfilter.New(mds), j) { if err := m.Start(ctx); err != nil { return xerrors.Errorf("failed to start up genesis miner: %w", err) diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 5964a2458..a1f301572 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -34,6 +34,7 @@ import ( "github.com/filecoin-project/lotus/chain/vm" lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" + "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/lib/blockstore" "github.com/filecoin-project/lotus/lib/peermgr" "github.com/filecoin-project/lotus/lib/ulimit" @@ -410,7 +411,11 @@ func ImportChain(r repo.Repo, fname string, snapshot bool) (err error) { bs := blockstore.NewBlockstore(ds) - cst := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier)) + j, err := journal.OpenFSJournal(lr, journal.EnvDisabledEvents()) + if err != nil { + return xerrors.Errorf("failed to open journal: %w", err) + } + cst := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier), j) log.Infof("importing chain from %s...", fname) diff --git a/conformance/driver.go b/conformance/driver.go index 940c137cf..f49022b9c 100644 --- a/conformance/driver.go +++ b/conformance/driver.go @@ -87,7 +87,7 @@ func (d *Driver) ExecuteTipset(bs blockstore.Blockstore, ds ds.Batching, preroot syscalls = vm.Syscalls(ffiwrapper.ProofVerifier) vmRand = NewFixedRand() - cs = store.NewChainStore(bs, ds, syscalls) + cs = store.NewChainStore(bs, ds, syscalls, nil) sm = stmgr.NewStateManager(cs) ) diff --git a/journal/env.go b/journal/env.go new file mode 100644 index 000000000..68380844e --- /dev/null +++ b/journal/env.go @@ -0,0 +1,19 @@ +package journal + +import ( + "os" +) + +// envJournalDisabledEvents is the environment variable through which disabled +// journal events can be customized. +const envDisabledEvents = "LOTUS_JOURNAL_DISABLED_EVENTS" + +func EnvDisabledEvents() DisabledEvents { + if env, ok := os.LookupEnv(envDisabledEvents); ok { + if ret, err := ParseDisabledEvents(env); err == nil { + return ret + } + } + // fallback if env variable is not set, or if it failed to parse. + return DefaultDisabledEvents +} diff --git a/journal/global.go b/journal/global.go deleted file mode 100644 index b4d0e0a1b..000000000 --- a/journal/global.go +++ /dev/null @@ -1,9 +0,0 @@ -package journal - -var ( - // J is a globally accessible Journal. It starts being NilJournal, and early - // during the Lotus initialization routine, it is reset to whichever Journal - // is configured (by default, the filesystem journal). Components can safely - // record in the journal by calling: journal.J.RecordEvent(...). - J Journal = NilJournal() // nolint -) diff --git a/markets/journal.go b/markets/journal.go index a73d28011..9c9c5be9c 100644 --- a/markets/journal.go +++ b/markets/journal.go @@ -28,9 +28,9 @@ type RetrievalProviderEvt struct { } // StorageClientJournaler records journal events from the storage client. -func StorageClientJournaler(evtType journal.EventType) func(event storagemarket.ClientEvent, deal storagemarket.ClientDeal) { +func StorageClientJournaler(j journal.Journal, evtType journal.EventType) func(event storagemarket.ClientEvent, deal storagemarket.ClientDeal) { return func(event storagemarket.ClientEvent, deal storagemarket.ClientDeal) { - journal.J.RecordEvent(evtType, func() interface{} { + j.RecordEvent(evtType, func() interface{} { return StorageClientEvt{ Event: storagemarket.ClientEvents[event], Deal: deal, @@ -40,9 +40,9 @@ func StorageClientJournaler(evtType journal.EventType) func(event storagemarket. } // StorageProviderJournaler records journal events from the storage provider. -func StorageProviderJournaler(evtType journal.EventType) func(event storagemarket.ProviderEvent, deal storagemarket.MinerDeal) { +func StorageProviderJournaler(j journal.Journal, evtType journal.EventType) func(event storagemarket.ProviderEvent, deal storagemarket.MinerDeal) { return func(event storagemarket.ProviderEvent, deal storagemarket.MinerDeal) { - journal.J.RecordEvent(evtType, func() interface{} { + j.RecordEvent(evtType, func() interface{} { return StorageProviderEvt{ Event: storagemarket.ProviderEvents[event], Deal: deal, @@ -52,9 +52,9 @@ func StorageProviderJournaler(evtType journal.EventType) func(event storagemarke } // RetrievalClientJournaler records journal events from the retrieval client. -func RetrievalClientJournaler(evtType journal.EventType) func(event retrievalmarket.ClientEvent, deal retrievalmarket.ClientDealState) { +func RetrievalClientJournaler(j journal.Journal, evtType journal.EventType) func(event retrievalmarket.ClientEvent, deal retrievalmarket.ClientDealState) { return func(event retrievalmarket.ClientEvent, deal retrievalmarket.ClientDealState) { - journal.J.RecordEvent(evtType, func() interface{} { + j.RecordEvent(evtType, func() interface{} { return RetrievalClientEvt{ Event: retrievalmarket.ClientEvents[event], Deal: deal, @@ -64,9 +64,9 @@ func RetrievalClientJournaler(evtType journal.EventType) func(event retrievalmar } // RetrievalProviderJournaler records journal events from the retrieval provider. -func RetrievalProviderJournaler(evtType journal.EventType) func(event retrievalmarket.ProviderEvent, deal retrievalmarket.ProviderDealState) { +func RetrievalProviderJournaler(j journal.Journal, evtType journal.EventType) func(event retrievalmarket.ProviderEvent, deal retrievalmarket.ProviderDealState) { return func(event retrievalmarket.ProviderEvent, deal retrievalmarket.ProviderDealState) { - journal.J.RecordEvent(evtType, func() interface{} { + j.RecordEvent(evtType, func() interface{} { return RetrievalProviderEvt{ Event: retrievalmarket.ProviderEvents[event], Deal: deal, diff --git a/miner/miner.go b/miner/miner.go index aebcb7a25..73985a649 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -49,7 +49,7 @@ func randTimeOffset(width time.Duration) time.Duration { return val - (width / 2) } -func NewMiner(api api.FullNode, epp gen.WinningPoStProver, addr address.Address, sf *slashfilter.SlashFilter) *Miner { +func NewMiner(api api.FullNode, epp gen.WinningPoStProver, addr address.Address, sf *slashfilter.SlashFilter, j journal.Journal) *Miner { arc, err := lru.NewARC(10000) if err != nil { panic(err) @@ -74,8 +74,9 @@ func NewMiner(api api.FullNode, epp gen.WinningPoStProver, addr address.Address, sf: sf, minedBlockHeights: arc, evtTypes: [...]journal.EventType{ - evtTypeBlockMined: journal.J.RegisterEventType("miner", "block_mined"), + evtTypeBlockMined: j.RegisterEventType("miner", "block_mined"), }, + journal: j, } } @@ -97,6 +98,7 @@ type Miner struct { minedBlockHeights *lru.ARCCache evtTypes [1]journal.EventType + journal journal.Journal } func (m *Miner) Address() address.Address { @@ -239,7 +241,7 @@ minerLoop: onDone(b != nil, h, nil) if b != nil { - journal.J.RecordEvent(m.evtTypes[evtTypeBlockMined], func() interface{} { + m.journal.RecordEvent(m.evtTypes[evtTypeBlockMined], func() interface{} { return map[string]interface{}{ "parents": base.TipSet.Cids(), "nulls": base.NullRounds, diff --git a/miner/testminer.go b/miner/testminer.go index 64e3b3a62..5f461d884 100644 --- a/miner/testminer.go +++ b/miner/testminer.go @@ -11,6 +11,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/gen/slashfilter" + "github.com/filecoin-project/lotus/journal" ) type MineReq struct { @@ -32,6 +33,7 @@ func NewTestMiner(nextCh <-chan MineReq, addr address.Address) func(api.FullNode minedBlockHeights: arc, address: addr, sf: slashfilter.New(ds.NewMapDatastore()), + journal: journal.NilJournal(), } if err := m.Start(context.TODO()); err != nil { diff --git a/node/builder.go b/node/builder.go index 71295dbc8..a2e3c981e 100644 --- a/node/builder.go +++ b/node/builder.go @@ -3,7 +3,6 @@ package node import ( "context" "errors" - "os" "time" "github.com/filecoin-project/lotus/chain" @@ -73,10 +72,6 @@ import ( "github.com/filecoin-project/lotus/storage/sectorblocks" ) -// EnvJournalDisabledEvents is the environment variable through which disabled -// journal events can be customized. -const EnvJournalDisabledEvents = "LOTUS_JOURNAL_DISABLED_EVENTS" - //nolint:deadcode,varcheck var log = logging.Logger("builder") @@ -166,19 +161,8 @@ type Settings struct { func defaults() []Option { return []Option{ // global system journal. - Override(new(journal.DisabledEvents), func() journal.DisabledEvents { - if env, ok := os.LookupEnv(EnvJournalDisabledEvents); ok { - if ret, err := journal.ParseDisabledEvents(env); err == nil { - return ret - } - } - // fallback if env variable is not set, or if it failed to parse. - return journal.DefaultDisabledEvents - }), + Override(new(journal.DisabledEvents), journal.EnvDisabledEvents), Override(new(journal.Journal), modules.OpenFilesystemJournal), - Override(InitJournalKey, func(j journal.Journal) { - journal.J = j // eagerly sets the global journal through fx.Invoke. - }), Override(new(helpers.MetricsCtx), context.Background), Override(new(record.Validator), modules.RecordValidator), diff --git a/node/modules/chain.go b/node/modules/chain.go index 66f54a76a..f563b4cdd 100644 --- a/node/modules/chain.go +++ b/node/modules/chain.go @@ -17,6 +17,7 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" + "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain" @@ -51,9 +52,9 @@ func ChainBitswap(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, rt r return exch } -func MessagePool(lc fx.Lifecycle, sm *stmgr.StateManager, ps *pubsub.PubSub, ds dtypes.MetadataDS, nn dtypes.NetworkName) (*messagepool.MessagePool, error) { +func MessagePool(lc fx.Lifecycle, sm *stmgr.StateManager, ps *pubsub.PubSub, ds dtypes.MetadataDS, nn dtypes.NetworkName, j journal.Journal) (*messagepool.MessagePool, error) { mpp := messagepool.NewProvider(sm, ps) - mp, err := messagepool.New(mpp, ds, nn) + mp, err := messagepool.New(mpp, ds, nn, j) if err != nil { return nil, xerrors.Errorf("constructing mpool: %w", err) } @@ -88,8 +89,8 @@ func ChainBlockService(bs dtypes.ChainBlockstore, rem dtypes.ChainBitswap) dtype return blockservice.New(bs, rem) } -func ChainStore(lc fx.Lifecycle, bs dtypes.ChainBlockstore, ds dtypes.MetadataDS, syscalls vm.SyscallBuilder) *store.ChainStore { - chain := store.NewChainStore(bs, ds, syscalls) +func ChainStore(lc fx.Lifecycle, bs dtypes.ChainBlockstore, ds dtypes.MetadataDS, syscalls vm.SyscallBuilder, j journal.Journal) *store.ChainStore { + chain := store.NewChainStore(bs, ds, syscalls, j) if err := chain.Load(); err != nil { log.Warnf("loading chain state from disk: %s", err) diff --git a/node/modules/client.go b/node/modules/client.go index 6972ca36e..d012e4539 100644 --- a/node/modules/client.go +++ b/node/modules/client.go @@ -113,7 +113,7 @@ func NewClientDealFunds(ds dtypes.MetadataDS) (ClientDealFunds, error) { return funds.NewDealFunds(ds, datastore.NewKey("/marketfunds/client")) } -func StorageClient(lc fx.Lifecycle, h host.Host, ibs dtypes.ClientBlockstore, mds dtypes.ClientMultiDstore, r repo.LockedRepo, dataTransfer dtypes.ClientDataTransfer, discovery *discoveryimpl.Local, deals dtypes.ClientDatastore, scn storagemarket.StorageClientNode, dealFunds ClientDealFunds) (storagemarket.StorageClient, error) { +func StorageClient(lc fx.Lifecycle, h host.Host, ibs dtypes.ClientBlockstore, mds dtypes.ClientMultiDstore, r repo.LockedRepo, dataTransfer dtypes.ClientDataTransfer, discovery *discoveryimpl.Local, deals dtypes.ClientDatastore, scn storagemarket.StorageClientNode, dealFunds ClientDealFunds, j journal.Journal) (storagemarket.StorageClient, error) { net := smnet.NewFromLibp2pHost(h) c, err := storageimpl.NewClient(net, ibs, mds, dataTransfer, discovery, deals, scn, dealFunds, storageimpl.DealPollingInterval(time.Second)) if err != nil { @@ -124,8 +124,8 @@ func StorageClient(lc fx.Lifecycle, h host.Host, ibs dtypes.ClientBlockstore, md OnStart: func(ctx context.Context) error { c.SubscribeToEvents(marketevents.StorageClientLogger) - evtType := journal.J.RegisterEventType("markets/storage/client", "state_change") - c.SubscribeToEvents(markets.StorageClientJournaler(evtType)) + evtType := j.RegisterEventType("markets/storage/client", "state_change") + c.SubscribeToEvents(markets.StorageClientJournaler(j, evtType)) return c.Start(ctx) }, @@ -137,7 +137,7 @@ func StorageClient(lc fx.Lifecycle, h host.Host, ibs dtypes.ClientBlockstore, md } // RetrievalClient creates a new retrieval client attached to the client blockstore -func RetrievalClient(lc fx.Lifecycle, h host.Host, mds dtypes.ClientMultiDstore, dt dtypes.ClientDataTransfer, payAPI payapi.PaychAPI, resolver discovery.PeerResolver, ds dtypes.MetadataDS, chainAPI full.ChainAPI, stateAPI full.StateAPI) (retrievalmarket.RetrievalClient, error) { +func RetrievalClient(lc fx.Lifecycle, h host.Host, mds dtypes.ClientMultiDstore, dt dtypes.ClientDataTransfer, payAPI payapi.PaychAPI, resolver discovery.PeerResolver, ds dtypes.MetadataDS, chainAPI full.ChainAPI, stateAPI full.StateAPI, j journal.Journal) (retrievalmarket.RetrievalClient, error) { adapter := retrievaladapter.NewRetrievalClientNode(payAPI, chainAPI, stateAPI) network := rmnet.NewFromLibp2pHost(h) sc := storedcounter.New(ds, datastore.NewKey("/retr")) @@ -150,8 +150,8 @@ func RetrievalClient(lc fx.Lifecycle, h host.Host, mds dtypes.ClientMultiDstore, OnStart: func(ctx context.Context) error { client.SubscribeToEvents(marketevents.RetrievalClientLogger) - evtType := journal.J.RegisterEventType("markets/retrieval/client", "state_change") - client.SubscribeToEvents(markets.RetrievalClientJournaler(evtType)) + evtType := j.RegisterEventType("markets/retrieval/client", "state_change") + client.SubscribeToEvents(markets.RetrievalClientJournaler(j, evtType)) return client.Start(ctx) }, diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 95bfb6c11..9011c4821 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -161,6 +161,7 @@ type StorageMinerParams struct { SectorIDCounter sealing.SectorIDCounter Verifier ffiwrapper.Verifier GetSealingConfigFn dtypes.GetSealingConfigFunc + Journal journal.Journal } func StorageMiner(fc config.MinerFeeConfig) func(params StorageMinerParams) (*storage.Miner, error) { @@ -175,6 +176,7 @@ func StorageMiner(fc config.MinerFeeConfig) func(params StorageMinerParams) (*st sc = params.SectorIDCounter verif = params.Verifier gsd = params.GetSealingConfigFn + j = params.Journal ) maddr, err := minerAddrFromDS(ds) @@ -194,12 +196,12 @@ func StorageMiner(fc config.MinerFeeConfig) func(params StorageMinerParams) (*st return nil, err } - fps, err := storage.NewWindowedPoStScheduler(api, fc, sealer, sealer, maddr, worker) + fps, err := storage.NewWindowedPoStScheduler(api, fc, sealer, sealer, j, maddr, worker) if err != nil { return nil, err } - sm, err := storage.NewMiner(api, maddr, worker, h, ds, sealer, sc, verif, gsd, fc) + sm, err := storage.NewMiner(api, maddr, worker, h, ds, sealer, sc, verif, gsd, fc, j) if err != nil { return nil, err } @@ -216,15 +218,15 @@ func StorageMiner(fc config.MinerFeeConfig) func(params StorageMinerParams) (*st } } -func HandleRetrieval(host host.Host, lc fx.Lifecycle, m retrievalmarket.RetrievalProvider) { +func HandleRetrieval(host host.Host, lc fx.Lifecycle, m retrievalmarket.RetrievalProvider, j journal.Journal) { m.OnReady(marketevents.ReadyLogger("retrieval provider")) lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { m.SubscribeToEvents(marketevents.RetrievalProviderLogger) - evtType := journal.J.RegisterEventType("markets/retrieval/provider", "state_change") - m.SubscribeToEvents(markets.RetrievalProviderJournaler(evtType)) + evtType := j.RegisterEventType("markets/retrieval/provider", "state_change") + m.SubscribeToEvents(markets.RetrievalProviderJournaler(j, evtType)) return m.Start(ctx) }, @@ -234,15 +236,15 @@ func HandleRetrieval(host host.Host, lc fx.Lifecycle, m retrievalmarket.Retrieva }) } -func HandleDeals(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, h storagemarket.StorageProvider) { +func HandleDeals(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, h storagemarket.StorageProvider, j journal.Journal) { ctx := helpers.LifecycleCtx(mctx, lc) h.OnReady(marketevents.ReadyLogger("storage provider")) lc.Append(fx.Hook{ OnStart: func(context.Context) error { h.SubscribeToEvents(marketevents.StorageProviderLogger) - evtType := journal.J.RegisterEventType("markets/storage/provider", "state_change") - h.SubscribeToEvents(markets.StorageProviderJournaler(evtType)) + evtType := j.RegisterEventType("markets/storage/provider", "state_change") + h.SubscribeToEvents(markets.StorageProviderJournaler(j, evtType)) return h.Start(ctx) }, @@ -354,13 +356,13 @@ func StagingGraphsync(mctx helpers.MetricsCtx, lc fx.Lifecycle, ibs dtypes.Stagi return gs } -func SetupBlockProducer(lc fx.Lifecycle, ds dtypes.MetadataDS, api lapi.FullNode, epp gen.WinningPoStProver, sf *slashfilter.SlashFilter) (*miner.Miner, error) { +func SetupBlockProducer(lc fx.Lifecycle, ds dtypes.MetadataDS, api lapi.FullNode, epp gen.WinningPoStProver, sf *slashfilter.SlashFilter, j journal.Journal) (*miner.Miner, error) { minerAddr, err := minerAddrFromDS(ds) if err != nil { return nil, err } - m := miner.NewMiner(api, epp, minerAddr, sf) + m := miner.NewMiner(api, epp, minerAddr, sf, j) lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { diff --git a/node/modules/testing/genesis.go b/node/modules/testing/genesis.go index 40df68bf3..fa9e0cff7 100644 --- a/node/modules/testing/genesis.go +++ b/node/modules/testing/genesis.go @@ -23,17 +23,18 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" "github.com/filecoin-project/lotus/genesis" + "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/node/modules" "github.com/filecoin-project/lotus/node/modules/dtypes" ) var glog = logging.Logger("genesis") -func MakeGenesisMem(out io.Writer, template genesis.Template) func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder) modules.Genesis { - return func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder) modules.Genesis { +func MakeGenesisMem(out io.Writer, template genesis.Template) func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder, j journal.Journal) modules.Genesis { + return func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder, j journal.Journal) modules.Genesis { return func() (*types.BlockHeader, error) { glog.Warn("Generating new random genesis block, note that this SHOULD NOT happen unless you are setting up new network") - b, err := genesis2.MakeGenesisBlock(context.TODO(), bs, syscalls, template) + b, err := genesis2.MakeGenesisBlock(context.TODO(), j, bs, syscalls, template) if err != nil { return nil, xerrors.Errorf("make genesis block failed: %w", err) } @@ -50,8 +51,8 @@ func MakeGenesisMem(out io.Writer, template genesis.Template) func(bs dtypes.Cha } } -func MakeGenesis(outFile, genesisTemplate string) func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder) modules.Genesis { - return func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder) modules.Genesis { +func MakeGenesis(outFile, genesisTemplate string) func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder, j journal.Journal) modules.Genesis { + return func(bs dtypes.ChainBlockstore, syscalls vm.SyscallBuilder, j journal.Journal) modules.Genesis { return func() (*types.BlockHeader, error) { glog.Warn("Generating new random genesis block, note that this SHOULD NOT happen unless you are setting up new network") genesisTemplate, err := homedir.Expand(genesisTemplate) @@ -73,7 +74,7 @@ func MakeGenesis(outFile, genesisTemplate string) func(bs dtypes.ChainBlockstore template.Timestamp = uint64(build.Clock.Now().Unix()) } - b, err := genesis2.MakeGenesisBlock(context.TODO(), bs, syscalls, template) + b, err := genesis2.MakeGenesisBlock(context.TODO(), j, bs, syscalls, template) if err != nil { return nil, xerrors.Errorf("make genesis block: %w", err) } diff --git a/storage/miner.go b/storage/miner.go index c1b50fe89..74a048c8e 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -57,6 +57,8 @@ type Miner struct { sealing *sealing.Sealing sealingEvtType journal.EventType + + journal journal.Journal } // SealingStateEvt is a journal event that records a sector state transition. @@ -110,7 +112,7 @@ type storageMinerApi interface { WalletHas(context.Context, address.Address) (bool, error) } -func NewMiner(api storageMinerApi, maddr, worker address.Address, h host.Host, ds datastore.Batching, sealer sectorstorage.SectorManager, sc sealing.SectorIDCounter, verif ffiwrapper.Verifier, gsd dtypes.GetSealingConfigFunc, feeCfg config.MinerFeeConfig) (*Miner, error) { +func NewMiner(api storageMinerApi, maddr, worker address.Address, h host.Host, ds datastore.Batching, sealer sectorstorage.SectorManager, sc sealing.SectorIDCounter, verif ffiwrapper.Verifier, gsd dtypes.GetSealingConfigFunc, feeCfg config.MinerFeeConfig, journal journal.Journal) (*Miner, error) { m := &Miner{ api: api, feeCfg: feeCfg, @@ -123,7 +125,8 @@ func NewMiner(api storageMinerApi, maddr, worker address.Address, h host.Host, d maddr: maddr, worker: worker, getSealConfig: gsd, - sealingEvtType: journal.J.RegisterEventType("storage", "sealing_states"), + journal: journal, + sealingEvtType: journal.RegisterEventType("storage", "sealing_states"), } return m, nil @@ -156,7 +159,7 @@ func (m *Miner) Run(ctx context.Context) error { } func (m *Miner) handleSealingNotifications(before, after sealing.SectorInfo) { - journal.J.RecordEvent(m.sealingEvtType, func() interface{} { + m.journal.RecordEvent(m.sealingEvtType, func() interface{} { return SealingStateEvt{ SectorNumber: before.SectorNumber, SectorType: before.SectorType, diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 6aa3d5188..d4ed4d64c 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -26,11 +26,10 @@ import ( "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/journal" ) func (s *WindowPoStScheduler) failPost(err error, ts *types.TipSet, deadline *dline.Info) { - journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} { + s.journal.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} { c := evtCommon{Error: err} if ts != nil { c.Deadline = deadline @@ -54,7 +53,7 @@ func (s *WindowPoStScheduler) failPost(err error, ts *types.TipSet, deadline *dl // recordProofsEvent records a successful proofs_processed event in the // journal, even if it was a noop (no partitions). func (s *WindowPoStScheduler) recordProofsEvent(partitions []miner.PoStPartition, mcid cid.Cid) { - journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStProofs], func() interface{} { + s.journal.RecordEvent(s.evtTypes[evtTypeWdPoStProofs], func() interface{} { return &WdPoStProofsProcessedEvt{ evtCommon: s.getEvtCommon(nil), Partitions: partitions, @@ -74,7 +73,7 @@ func (s *WindowPoStScheduler) startGeneratePoST( go func() { defer abort() - journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} { + s.journal.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} { return WdPoStSchedulerEvt{ evtCommon: s.getEvtCommon(nil), State: SchedulerStateStarted, @@ -125,7 +124,7 @@ func (s *WindowPoStScheduler) startSubmitPoST( err := s.runSubmitPoST(ctx, ts, deadline, posts) if err == nil { - journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} { + s.journal.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} { return WdPoStSchedulerEvt{ evtCommon: s.getEvtCommon(nil), State: SchedulerStateSucceeded, @@ -439,7 +438,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty log.Errorf("checking sector recoveries: %v", err) } - journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStRecoveries], func() interface{} { + s.journal.RecordEvent(s.evtTypes[evtTypeWdPoStRecoveries], func() interface{} { j := WdPoStRecoveriesProcessedEvt{ evtCommon: s.getEvtCommon(err), Declarations: recoveries, @@ -458,7 +457,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty log.Errorf("checking sector faults: %v", err) } - journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStFaults], func() interface{} { + s.journal.RecordEvent(s.evtTypes[evtTypeWdPoStFaults], func() interface{} { return WdPoStFaultsProcessedEvt{ evtCommon: s.getEvtCommon(err), Declarations: faults, diff --git a/storage/wdpost_run_test.go b/storage/wdpost_run_test.go index 09b9aee5c..dd7ac4c24 100644 --- a/storage/wdpost_run_test.go +++ b/storage/wdpost_run_test.go @@ -25,6 +25,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/journal" ) type mockStorageMinerAPI struct { @@ -168,6 +169,7 @@ func TestWDPostDoPost(t *testing.T) { proofType: proofType, actor: postAct, worker: workerAct, + journal: journal.NilJournal(), } di := &dline.Info{ diff --git a/storage/wdpost_sched.go b/storage/wdpost_sched.go index ee380fbaf..99bea2e66 100644 --- a/storage/wdpost_sched.go +++ b/storage/wdpost_sched.go @@ -35,12 +35,13 @@ type WindowPoStScheduler struct { worker address.Address evtTypes [4]journal.EventType + journal journal.Journal // failed abi.ChainEpoch // eps // failLk sync.Mutex } -func NewWindowedPoStScheduler(api storageMinerApi, fc config.MinerFeeConfig, sb storage.Prover, ft sectorstorage.FaultTracker, actor address.Address, worker address.Address) (*WindowPoStScheduler, error) { +func NewWindowedPoStScheduler(api storageMinerApi, fc config.MinerFeeConfig, sb storage.Prover, ft sectorstorage.FaultTracker, j journal.Journal, actor address.Address, worker address.Address) (*WindowPoStScheduler, error) { mi, err := api.StateMinerInfo(context.TODO(), actor, types.EmptyTSK) if err != nil { return nil, xerrors.Errorf("getting sector size: %w", err) @@ -62,11 +63,12 @@ func NewWindowedPoStScheduler(api storageMinerApi, fc config.MinerFeeConfig, sb actor: actor, worker: worker, evtTypes: [...]journal.EventType{ - evtTypeWdPoStScheduler: journal.J.RegisterEventType("wdpost", "scheduler"), - evtTypeWdPoStProofs: journal.J.RegisterEventType("wdpost", "proofs_processed"), - evtTypeWdPoStRecoveries: journal.J.RegisterEventType("wdpost", "recoveries_processed"), - evtTypeWdPoStFaults: journal.J.RegisterEventType("wdpost", "faults_processed"), + evtTypeWdPoStScheduler: j.RegisterEventType("wdpost", "scheduler"), + evtTypeWdPoStProofs: j.RegisterEventType("wdpost", "proofs_processed"), + evtTypeWdPoStRecoveries: j.RegisterEventType("wdpost", "recoveries_processed"), + evtTypeWdPoStFaults: j.RegisterEventType("wdpost", "faults_processed"), }, + journal: j, }, nil } @@ -166,7 +168,7 @@ func (s *WindowPoStScheduler) update(ctx context.Context, revert, apply *types.T // onAbort is called when generating proofs or submitting proofs is aborted func (s *WindowPoStScheduler) onAbort(ts *types.TipSet, deadline *dline.Info) { - journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} { + s.journal.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} { c := evtCommon{} if ts != nil { c.Deadline = deadline From c0182f8301b9773f3441f25c5dc9b938d4e08aa8 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 9 Oct 2020 13:59:56 -0700 Subject: [PATCH 673/885] return the correct err from iterFullTipset --- chain/sync.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chain/sync.go b/chain/sync.go index c280e3a40..369c65d33 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -1526,7 +1526,7 @@ func (syncer *Syncer) iterFullTipsets(ctx context.Context, headers []*types.TipS ss.SetStage(api.StageMessages) if batchErr != nil { - return xerrors.Errorf("failed to fetch messages: %w", err) + return xerrors.Errorf("failed to fetch messages: %w", batchErr) } for bsi := 0; bsi < len(bstout); bsi++ { From f773ab4b2824b2549f7768dc83ae200aa00ddaa2 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Fri, 9 Oct 2020 17:14:21 -0400 Subject: [PATCH 674/885] Correct docs --- api/api_full.go | 3 +-- documentation/en/api-methods.md | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 777e996cb..7ebc8791a 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -422,8 +422,7 @@ type FullNode interface { // MsigGetAvailableBalance returns the portion of a multisig's balance that can be withdrawn or spent MsigGetAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) - // MsigGetLockedBalance returns the locked balance of an msig at a vien epoch. - // The return may be greater than the multisig actor's actual balance. + // MsigGetVestingSchedule returns the vesting details of a given multisig. MsigGetVestingSchedule(context.Context, address.Address, types.TipSetKey) (MsigVesting, error) // MsigGetVested returns the amount of FIL that vested in a multisig in a certain period. // It takes the following params: , , diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index cc6d92e4f..b3a9627e0 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -2146,8 +2146,7 @@ Inputs: Response: `"0"` ### MsigGetVestingSchedule -MsigGetLockedBalance returns the locked balance of an msig at a vien epoch. -The return may be greater than the multisig actor's actual balance. +MsigGetVestingSchedule returns the vesting details of a given multisig. Perms: read From e0dcb0ec4930b465948af8ec04dc9ce6835c8087 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Fri, 9 Oct 2020 22:20:15 +0100 Subject: [PATCH 675/885] tvx: trace puts to blockstore for inclusion in CAR. --- cmd/tvx/stores.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/cmd/tvx/stores.go b/cmd/tvx/stores.go index 93e0d215f..4f574c175 100644 --- a/cmd/tvx/stores.go +++ b/cmd/tvx/stores.go @@ -87,7 +87,7 @@ type proxyingBlockstore struct { ctx context.Context api api.FullNode - lk sync.RWMutex + lk sync.Mutex tracing bool traced map[cid.Cid]struct{} @@ -113,11 +113,11 @@ func (pb *proxyingBlockstore) FinishTracing() map[cid.Cid]struct{} { } func (pb *proxyingBlockstore) Get(cid cid.Cid) (blocks.Block, error) { - pb.lk.RLock() + pb.lk.Lock() if pb.tracing { pb.traced[cid] = struct{}{} } - pb.lk.RUnlock() + pb.lk.Unlock() if block, err := pb.Blockstore.Get(cid); err == nil { return block, err @@ -140,3 +140,12 @@ func (pb *proxyingBlockstore) Get(cid cid.Cid) (blocks.Block, error) { return block, nil } + +func (pb *proxyingBlockstore) Put(block blocks.Block) error { + pb.lk.Lock() + if pb.tracing { + pb.traced[block.Cid()] = struct{}{} + } + pb.lk.Unlock() + return pb.Blockstore.Put(block) +} From 283fd054e89ba9119088e85b30bd27f71992ecd9 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 9 Oct 2020 15:35:44 -0700 Subject: [PATCH 676/885] fix a race when retrieving pieces We'd read the deal ID without synchronizing. This could (and probably did given the history of flaky deal tests) cause us to miss events. This patch also makes sure to always unsubscribe from events, even on error. --- node/impl/client/client.go | 40 +++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/node/impl/client/client.go b/node/impl/client/client.go index f146dcea3..fe83fda7b 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -462,13 +462,19 @@ type retrievalSubscribeEvent struct { state rm.ClientDealState } -func readSubscribeEvents(ctx context.Context, subscribeEvents chan retrievalSubscribeEvent, events chan marketevents.RetrievalEvent) error { +func readSubscribeEvents(ctx context.Context, dealID retrievalmarket.DealID, subscribeEvents chan retrievalSubscribeEvent, events chan marketevents.RetrievalEvent) error { for { var subscribeEvent retrievalSubscribeEvent select { case <-ctx.Done(): return xerrors.New("Retrieval Timed Out") case subscribeEvent = <-subscribeEvents: + if subscribeEvent.state.ID != dealID { + // we can't check the deal ID ahead of time because: + // 1. We need to subscribe before retrieving. + // 2. We won't know the deal ID until after retrieving. + continue + } } select { @@ -531,19 +537,6 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref return err }*/ - var dealID retrievalmarket.DealID - subscribeEvents := make(chan retrievalSubscribeEvent, 1) - subscribeCtx, cancel := context.WithCancel(ctx) - defer cancel() - unsubscribe := a.Retrieval.SubscribeToEvents(func(event rm.ClientEvent, state rm.ClientDealState) { - if state.PayloadCID.Equals(order.Root) && state.ID == dealID { - select { - case <-subscribeCtx.Done(): - case subscribeEvents <- retrievalSubscribeEvent{event, state}: - } - } - }) - ppb := types.BigDiv(order.Total, types.NewInt(order.Size)) params, err := rm.NewParamsV1(ppb, order.PaymentInterval, order.PaymentIntervalIncrease, shared.AllSelector(), order.Piece, order.UnsealPrice) @@ -562,7 +555,21 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref _ = a.RetrievalStoreMgr.ReleaseStore(store) }() - dealID, err = a.Retrieval.Retrieve( + // Subscribe to events before retrieving to avoid losing events. + subscribeEvents := make(chan retrievalSubscribeEvent, 1) + subscribeCtx, cancel := context.WithCancel(ctx) + defer cancel() + unsubscribe := a.Retrieval.SubscribeToEvents(func(event rm.ClientEvent, state rm.ClientDealState) { + // We'll check the deal IDs inside readSubscribeEvents. + if state.PayloadCID.Equals(order.Root) { + select { + case <-subscribeCtx.Done(): + case subscribeEvents <- retrievalSubscribeEvent{event, state}: + } + } + }) + + dealID, err := a.Retrieval.Retrieve( ctx, order.Root, params, @@ -573,11 +580,12 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref store.StoreID()) if err != nil { + unsubscribe() finish(xerrors.Errorf("Retrieve failed: %w", err)) return } - err = readSubscribeEvents(ctx, subscribeEvents, events) + err = readSubscribeEvents(ctx, dealID, subscribeEvents, events) unsubscribe() if err != nil { From 83dfc460d490721dafa237717c96c8918cf1262e Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 9 Oct 2020 15:39:41 -0700 Subject: [PATCH 677/885] fix race in unseal 1. Remove an invalid error check. 2. Make sure to shadow the outer error type from within the goroutine instead or reading the outer type. This may have been causing test issues (caught in TestMinerAllInfo with the race detector). --- extern/sector-storage/ffiwrapper/sealer_cgo.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/extern/sector-storage/ffiwrapper/sealer_cgo.go b/extern/sector-storage/ffiwrapper/sealer_cgo.go index d75501838..bae6cafb4 100644 --- a/extern/sector-storage/ffiwrapper/sealer_cgo.go +++ b/extern/sector-storage/ffiwrapper/sealer_cgo.go @@ -290,10 +290,6 @@ func (sb *Sealer) UnsealPiece(ctx context.Context, sector abi.SectorID, offset s defer opr.Close() // nolint padwriter := fr32.NewPadWriter(out) - if err != nil { - perr = xerrors.Errorf("creating new padded writer: %w", err) - return - } bsize := uint64(size.Padded()) if bsize > uint64(runtime.NumCPU())*fr32.MTTresh { @@ -302,7 +298,7 @@ func (sb *Sealer) UnsealPiece(ctx context.Context, sector abi.SectorID, offset s bw := bufio.NewWriterSize(padwriter, int(abi.PaddedPieceSize(bsize).Unpadded())) - _, err = io.CopyN(bw, opr, int64(size)) + _, err := io.CopyN(bw, opr, int64(size)) if err != nil { perr = xerrors.Errorf("copying data: %w", err) return From f32b3bf99b78821ff15dc66e92e3e73ee817996c Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 9 Oct 2020 17:21:50 -0700 Subject: [PATCH 678/885] fix a race in tipset cache usage This tipset cache is shared between multiple services and is called from multiple places. --- chain/events/tscache.go | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/chain/events/tscache.go b/chain/events/tscache.go index d47c71480..44699684e 100644 --- a/chain/events/tscache.go +++ b/chain/events/tscache.go @@ -2,6 +2,7 @@ package events import ( "context" + "sync" "github.com/filecoin-project/go-state-types/abi" "golang.org/x/xerrors" @@ -17,6 +18,8 @@ type tsCacheAPI interface { // tipSetCache implements a simple ring-buffer cache to keep track of recent // tipsets type tipSetCache struct { + mu sync.RWMutex + cache []*types.TipSet start int len int @@ -35,6 +38,9 @@ func newTSCache(cap abi.ChainEpoch, storage tsCacheAPI) *tipSetCache { } func (tsc *tipSetCache) add(ts *types.TipSet) error { + tsc.mu.Lock() + defer tsc.mu.Unlock() + if tsc.len > 0 { if tsc.cache[tsc.start].Height() >= ts.Height() { return xerrors.Errorf("tipSetCache.add: expected new tipset height to be at least %d, was %d", tsc.cache[tsc.start].Height()+1, ts.Height()) @@ -65,6 +71,13 @@ func (tsc *tipSetCache) add(ts *types.TipSet) error { } func (tsc *tipSetCache) revert(ts *types.TipSet) error { + tsc.mu.Lock() + defer tsc.mu.Unlock() + + return tsc.revertUnlocked(ts) +} + +func (tsc *tipSetCache) revertUnlocked(ts *types.TipSet) error { if tsc.len == 0 { return nil // this can happen, and it's fine } @@ -77,7 +90,7 @@ func (tsc *tipSetCache) revert(ts *types.TipSet) error { tsc.start = normalModulo(tsc.start-1, len(tsc.cache)) tsc.len-- - _ = tsc.revert(nil) // revert null block gap + _ = tsc.revertUnlocked(nil) // revert null block gap return nil } @@ -95,7 +108,10 @@ func (tsc *tipSetCache) getNonNull(height abi.ChainEpoch) (*types.TipSet, error) } func (tsc *tipSetCache) get(height abi.ChainEpoch) (*types.TipSet, error) { + tsc.mu.RLock() + if tsc.len == 0 { + tsc.mu.RUnlock() log.Warnf("tipSetCache.get: cache is empty, requesting from storage (h=%d)", height) return tsc.storage.ChainGetTipSetByHeight(context.TODO(), height, types.EmptyTSK) } @@ -103,6 +119,7 @@ func (tsc *tipSetCache) get(height abi.ChainEpoch) (*types.TipSet, error) { headH := tsc.cache[tsc.start].Height() if height > headH { + tsc.mu.RUnlock() return nil, xerrors.Errorf("tipSetCache.get: requested tipset not in cache (req: %d, cache head: %d)", height, headH) } @@ -116,15 +133,20 @@ func (tsc *tipSetCache) get(height abi.ChainEpoch) (*types.TipSet, error) { } if height < tail.Height() { + tsc.mu.RUnlock() log.Warnf("tipSetCache.get: requested tipset not in cache, requesting from storage (h=%d; tail=%d)", height, tail.Height()) return tsc.storage.ChainGetTipSetByHeight(context.TODO(), height, tail.Key()) } - return tsc.cache[normalModulo(tsc.start-int(headH-height), clen)], nil + ts := tsc.cache[normalModulo(tsc.start-int(headH-height), clen)] + tsc.mu.RUnlock() + return ts, nil } func (tsc *tipSetCache) best() (*types.TipSet, error) { + tsc.mu.RLock() best := tsc.cache[tsc.start] + tsc.mu.RUnlock() if best == nil { return tsc.storage.ChainHead(context.TODO()) } From 6610a247cfebb4a213e92fc2bb85dcd23e87c189 Mon Sep 17 00:00:00 2001 From: Peter Rabbitson Date: Fri, 9 Oct 2020 22:41:34 +0200 Subject: [PATCH 679/885] Dump the block validation cache whenever we perform an import This solves a problem with folks resurrecting long-out-of-sync nodes via snapshot imports. The interface switch to Batching is necessary: startup is too long otherwise ( 8 minutes just to clear everything on a relatively old node ) --- chain/store/store.go | 42 +++++++++++++++++++++++++++++++++++++++++- cmd/lotus/daemon.go | 6 +++++- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/chain/store/store.go b/chain/store/store.go index 39f3d5c48..22b305afb 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -8,6 +8,7 @@ import ( "io" "os" "strconv" + "strings" "sync" "golang.org/x/sync/errgroup" @@ -38,7 +39,9 @@ import ( lru "github.com/hashicorp/golang-lru" block "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" dstore "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/query" cbor "github.com/ipfs/go-ipld-cbor" logging "github.com/ipfs/go-log/v2" car "github.com/ipld/go-car" @@ -102,7 +105,7 @@ type HeadChangeEvt struct { // 2. a block => messages references cache. type ChainStore struct { bs bstore.Blockstore - ds dstore.Datastore + ds dstore.Batching heaviestLk sync.Mutex heaviest *types.TipSet @@ -446,6 +449,43 @@ func (cs *ChainStore) takeHeaviestTipSet(ctx context.Context, ts *types.TipSet) return nil } +// FlushValidationCache removes all results of block validation from the +// chain metadata store. Usually the first step after a new chain import. +func (cs *ChainStore) FlushValidationCache() error { + log.Infof("clearing block validation cache...") + + dsWalk, err := cs.ds.Query(query.Query{KeysOnly: true}) + if err != nil { + return xerrors.Errorf("failed to initialize key listing query: %w", err) + } + + allKeys, err := dsWalk.Rest() + if err != nil { + return xerrors.Errorf("failed to run key listing query: %w", err) + } + + batch, err := cs.ds.Batch() + if err != nil { + return xerrors.Errorf("failed to open a DS batch: %w", err) + } + + delCnt := 0 + for _, k := range allKeys { + if strings.HasPrefix(k.Key, blockValidationCacheKeyPrefix.String()) { + delCnt++ + batch.Delete(datastore.RawKey(k.Key)) + } + } + + if err := batch.Commit(); err != nil { + return xerrors.Errorf("failed to commit the DS batch: %w", err) + } + + log.Infof("%d block validation entries cleared.", delCnt) + + return nil +} + // SetHead sets the chainstores current 'best' head node. // This should only be called if something is broken and needs fixing func (cs *ChainStore) SetHead(ts *types.TipSet) error { diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index a1f301572..4ff63be11 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -436,6 +436,10 @@ func ImportChain(r repo.Repo, fname string, snapshot bool) (err error) { return xerrors.Errorf("importing chain failed: %w", err) } + if err := cst.FlushValidationCache(); err != nil { + return xerrors.Errorf("flushing validation cache failed: %w", err) + } + gb, err := cst.GetTipsetByHeight(context.TODO(), 0, ts, true) if err != nil { return err @@ -455,7 +459,7 @@ func ImportChain(r repo.Repo, fname string, snapshot bool) (err error) { } } - log.Info("accepting %s as new head", ts.Cids()) + log.Infof("accepting %s as new head", ts.Cids()) if err := cst.SetHead(ts); err != nil { return err } From 85abca4b16231c7693b7b3ca142113e737c19240 Mon Sep 17 00:00:00 2001 From: Peter Rabbitson Date: Sat, 10 Oct 2020 04:00:12 +0200 Subject: [PATCH 680/885] Delint --- chain/store/store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chain/store/store.go b/chain/store/store.go index 22b305afb..6fb689b3e 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -473,7 +473,7 @@ func (cs *ChainStore) FlushValidationCache() error { for _, k := range allKeys { if strings.HasPrefix(k.Key, blockValidationCacheKeyPrefix.String()) { delCnt++ - batch.Delete(datastore.RawKey(k.Key)) + batch.Delete(datastore.RawKey(k.Key)) // nolint:errcheck } } From f3abd1ae82d0dbd500b4fa1ccc32e6a6fe54d020 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 9 Oct 2020 20:32:09 -0700 Subject: [PATCH 681/885] make pledge test pass with the race detector --- api/test/window_post.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/api/test/window_post.go b/api/test/window_post.go index 7bc56a562..28639cda8 100644 --- a/api/test/window_post.go +++ b/api/test/window_post.go @@ -3,6 +3,7 @@ package test import ( "context" "fmt" + "sync/atomic" "os" "strings" @@ -31,7 +32,9 @@ func init() { } func TestPledgeSector(t *testing.T, b APIBuilder, blocktime time.Duration, nSectors int) { - ctx := context.Background() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + n, sn := b(t, OneFull, OneMiner) client := n[0].FullNode.(*impl.FullNodeAPI) miner := sn[0] @@ -46,11 +49,11 @@ func TestPledgeSector(t *testing.T, b APIBuilder, blocktime time.Duration, nSect } build.Clock.Sleep(time.Second) - mine := true + mine := int64(1) done := make(chan struct{}) go func() { defer close(done) - for mine { + for atomic.LoadInt64(&mine) != 0 { build.Clock.Sleep(blocktime) if err := sn[0].MineOne(ctx, bminer.MineReq{Done: func(bool, abi.ChainEpoch, error) { @@ -62,7 +65,7 @@ func TestPledgeSector(t *testing.T, b APIBuilder, blocktime time.Duration, nSect pledgeSectors(t, ctx, miner, nSectors, 0, nil) - mine = false + atomic.StoreInt64(&mine, 0) <-done } From d84f5dababc6b795e8b118eda7e77d47d1877767 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 9 Oct 2020 21:38:47 -0700 Subject: [PATCH 682/885] remove upgrade invariant checks These checks may be _too_ strict and could cause the upgrade to fail unnecessarily. --- chain/stmgr/forks.go | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index a61f70b44..fba92ee3f 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -24,7 +24,6 @@ import ( "github.com/filecoin-project/specs-actors/actors/migration/nv3" m2 "github.com/filecoin-project/specs-actors/v2/actors/migration" - states2 "github.com/filecoin-project/specs-actors/v2/actors/states" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors/adt" @@ -580,19 +579,6 @@ func UpgradeActorsV2(ctx context.Context, sm *StateManager, cb ExecCallback, roo return cid.Undef, xerrors.Errorf("upgrading to actors v2: %w", err) } - newStateTree, err := states2.LoadTree(store, newHamtRoot) - if err != nil { - return cid.Undef, xerrors.Errorf("failed to load new state tree: %w", err) - } - - // Check all state-tree invariants. - if msgs, err := states2.CheckStateInvariants(newStateTree, types.TotalFilecoinInt); err != nil { - return cid.Undef, xerrors.Errorf("failed to check new state tree: %w", err) - } else if !msgs.IsEmpty() { - // This error is going to be really nasty. - return cid.Undef, xerrors.Errorf("network upgrade failed: %v", msgs.Messages()) - } - newRoot, err := store.Put(ctx, &types.StateRoot{ Version: types.StateTreeVersion1, Actors: newHamtRoot, From 10d464a1ebb90fe612164645885704b17ef54b47 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 9 Oct 2020 21:39:55 -0700 Subject: [PATCH 683/885] upgrade to specs-actors v2.0.3 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e1bf6cb3b..eb79a496c 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,7 @@ require ( github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b github.com/filecoin-project/specs-actors v0.9.12 - github.com/filecoin-project/specs-actors/v2 v2.0.1 + github.com/filecoin-project/specs-actors/v2 v2.0.3 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 github.com/filecoin-project/test-vectors/schema v0.0.4 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 diff --git a/go.sum b/go.sum index ef954e66d..e13359f6c 100644 --- a/go.sum +++ b/go.sum @@ -276,8 +276,8 @@ github.com/filecoin-project/specs-actors v0.9.4/go.mod h1:BStZQzx5x7TmCkLv0Bpa07 github.com/filecoin-project/specs-actors v0.9.7/go.mod h1:wM2z+kwqYgXn5Z7scV1YHLyd1Q1cy0R8HfTIWQ0BFGU= github.com/filecoin-project/specs-actors v0.9.12 h1:iIvk58tuMtmloFNHhAOQHG+4Gci6Lui0n7DYQGi3cJk= github.com/filecoin-project/specs-actors v0.9.12/go.mod h1:TS1AW/7LbG+615j4NsjMK1qlpAwaFsG9w0V2tg2gSao= -github.com/filecoin-project/specs-actors/v2 v2.0.1 h1:bf08x6tqCDfClzrv2q/rmt/A/UbBOy1KgaoogQEcLhU= -github.com/filecoin-project/specs-actors/v2 v2.0.1/go.mod h1:v2NZVYinNIKA9acEMBm5wWXxqv5+frFEbekBFemYghY= +github.com/filecoin-project/specs-actors/v2 v2.0.3 h1:Niy6xncgi8bI8aBCt1McdZfATBfG4Uxytt8KW4s3bAc= +github.com/filecoin-project/specs-actors/v2 v2.0.3/go.mod h1:v2NZVYinNIKA9acEMBm5wWXxqv5+frFEbekBFemYghY= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= github.com/filecoin-project/test-vectors/schema v0.0.4 h1:QTRd0gb/NP4ZOTM7Dib5U3xE1/ToGDKnYLfxkC3t/m8= From f417d39a0831568d7c9fd0d84353009ab82b0c6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 10 Oct 2020 08:55:33 +0200 Subject: [PATCH 684/885] Bump the version to 0.9.1 --- CHANGELOG.md | 20 ++++++++++++++++++++ build/version.go | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f6f0b3a0..d397762a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Lotus changelog +# 0.9.1 / 2020-10-10 + +This release fixes an issue which may cause the actors v2 migration to compute the state incorrectly when more than one migration is running in parallel. + +## Changes + +- Make concurrent actor migrations safe (https://github.com/filecoin-project/lotus/pull/4293) +- Remote wallet backends (https://github.com/filecoin-project/lotus/pull/3583) +- Track funds in FundMgr correctly in case of AddFunds failing (https://github.com/filecoin-project/lotus/pull/4273) +- Partial lite-node mode (https://github.com/filecoin-project/lotus/pull/4095) +- Fix potential infinite loop in GetBestMiningCandidate (https://github.com/filecoin-project/lotus/pull/3444) +- sync wait: Handle processed message offset (https://github.com/filecoin-project/lotus/pull/4253) +- Add some new endpoints for querying Msig info (https://github.com/filecoin-project/lotus/pull/4250) +- Update markets v0.7.1 (https://github.com/filecoin-project/lotus/pull/4254) +- Optimize SearchForMessage and GetReceipt (https://github.com/filecoin-project/lotus/pull/4246) +- Use FIL instead of attoFIL in CLI more consistently (https://github.com/filecoin-project/lotus/pull/4249) +- fix: clash between daemon --api flag and cli tests (https://github.com/filecoin-project/lotus/pull/4241) +- add more info to chain sync lookback failure (https://github.com/filecoin-project/lotus/pull/4245) +- Add message counts to inspect chain output (https://github.com/filecoin-project/lotus/pull/4230) + # 0.9.0 / 2020-10-07 This consensus-breaking release of Lotus upgrades the actors version to v2.0.0. This requires migrating actor state from v0 to v2. The changes that break consensus are: diff --git a/build/version.go b/build/version.go index 1602551e4..5baed1fb7 100644 --- a/build/version.go +++ b/build/version.go @@ -29,7 +29,7 @@ func buildType() string { } // BuildVersion is the local build version, set by build system -const BuildVersion = "0.9.0" +const BuildVersion = "0.9.1" func UserVersion() string { return BuildVersion + buildType() + CurrentCommit From 18e58467f8cbfb9e12d74524c03ae4d139800ab3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 10 Oct 2020 10:26:42 +0200 Subject: [PATCH 685/885] sync unmark-bad --all --- api/api_full.go | 3 +++ api/apistruct/struct.go | 5 +++++ chain/badtscache.go | 4 ++++ chain/sync.go | 4 ++++ cli/sync.go | 10 ++++++++++ node/impl/full/sync.go | 6 ++++++ 6 files changed, 32 insertions(+) diff --git a/api/api_full.go b/api/api_full.go index 7ebc8791a..1311dac6a 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -172,6 +172,9 @@ type FullNode interface { // SyncUnmarkBad unmarks a blocks as bad, making it possible to be validated and synced again. SyncUnmarkBad(ctx context.Context, bcid cid.Cid) error + // SyncUnmarkAllBad purges bad block cache, making it possible to sync to chains previously marked as bad + SyncUnmarkAllBad(ctx context.Context) error + // SyncCheckBad checks if a block was marked as bad, and if it was, returns // the reason. SyncCheckBad(ctx context.Context, bcid cid.Cid) (string, error) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 348e8a6de..6bfbf43cb 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -111,6 +111,7 @@ type FullNodeStruct struct { SyncCheckpoint func(ctx context.Context, key types.TipSetKey) error `perm:"admin"` SyncMarkBad func(ctx context.Context, bcid cid.Cid) error `perm:"admin"` SyncUnmarkBad func(ctx context.Context, bcid cid.Cid) error `perm:"admin"` + SyncUnmarkAllBad func(ctx context.Context) error `perm:"admin"` SyncCheckBad func(ctx context.Context, bcid cid.Cid) (string, error) `perm:"read"` SyncValidateTipset func(ctx context.Context, tsk types.TipSetKey) (bool, error) `perm:"read"` @@ -780,6 +781,10 @@ func (c *FullNodeStruct) SyncUnmarkBad(ctx context.Context, bcid cid.Cid) error return c.Internal.SyncUnmarkBad(ctx, bcid) } +func (c *FullNodeStruct) SyncUnmarkAllBad(ctx context.Context) error { + return c.Internal.SyncUnmarkAllBad(ctx) +} + func (c *FullNodeStruct) SyncCheckBad(ctx context.Context, bcid cid.Cid) (string, error) { return c.Internal.SyncCheckBad(ctx, bcid) } diff --git a/chain/badtscache.go b/chain/badtscache.go index 3c5bf05ef..42564128e 100644 --- a/chain/badtscache.go +++ b/chain/badtscache.go @@ -60,6 +60,10 @@ func (bts *BadBlockCache) Remove(c cid.Cid) { bts.badBlocks.Remove(c) } +func (bts *BadBlockCache) Purge() { + bts.badBlocks.Purge() +} + func (bts *BadBlockCache) Has(c cid.Cid) (BadBlockReason, bool) { rval, ok := bts.badBlocks.Get(c) if !ok { diff --git a/chain/sync.go b/chain/sync.go index 369c65d33..23688ecb7 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -1740,6 +1740,10 @@ func (syncer *Syncer) UnmarkBad(blk cid.Cid) { syncer.bad.Remove(blk) } +func (syncer *Syncer) UnmarkAllBad() { + syncer.bad.Purge() +} + func (syncer *Syncer) CheckBadBlockCache(blk cid.Cid) (string, bool) { bbr, ok := syncer.bad.Has(blk) return bbr.String(), ok diff --git a/cli/sync.go b/cli/sync.go index dea96d14e..a5178441d 100644 --- a/cli/sync.go +++ b/cli/sync.go @@ -124,6 +124,12 @@ var syncMarkBadCmd = &cli.Command{ var syncUnmarkBadCmd = &cli.Command{ Name: "unmark-bad", Usage: "Unmark the given block as bad, makes it possible to sync to a chain containing it", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "all", + Usage: "drop the entire bad block cache", + }, + }, ArgsUsage: "[blockCid]", Action: func(cctx *cli.Context) error { napi, closer, err := GetFullNodeAPI(cctx) @@ -133,6 +139,10 @@ var syncUnmarkBadCmd = &cli.Command{ defer closer() ctx := ReqContext(cctx) + if cctx.Bool("all") { + return napi.SyncUnmarkAllBad(ctx) + } + if !cctx.Args().Present() { return fmt.Errorf("must specify block cid to unmark") } diff --git a/node/impl/full/sync.go b/node/impl/full/sync.go index 1bd3af415..05d4c9cb7 100644 --- a/node/impl/full/sync.go +++ b/node/impl/full/sync.go @@ -118,6 +118,12 @@ func (a *SyncAPI) SyncUnmarkBad(ctx context.Context, bcid cid.Cid) error { return nil } +func (a *SyncAPI) SyncUnmarkAllBad(ctx context.Context) error { + log.Warnf("Dropping bad block cache") + a.Syncer.UnmarkAllBad() + return nil +} + func (a *SyncAPI) SyncCheckBad(ctx context.Context, bcid cid.Cid) (string, error) { reason, ok := a.Syncer.CheckBadBlockCache(bcid) if !ok { From 388c9afce0f6a43be99454bb6229eadbddb1fa23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 10 Oct 2020 12:28:59 +0200 Subject: [PATCH 686/885] docsgen --- documentation/en/api-methods.md | 59 +++++++++++++++++++++++++++++++-- 1 file changed, 57 insertions(+), 2 deletions(-) diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index 8a288a4bf..e921d3e45 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -83,12 +83,14 @@ * [MsigAddCancel](#MsigAddCancel) * [MsigAddPropose](#MsigAddPropose) * [MsigApprove](#MsigApprove) + * [MsigApproveTxnHash](#MsigApproveTxnHash) * [MsigCancel](#MsigCancel) * [MsigCreate](#MsigCreate) * [MsigGetAvailableBalance](#MsigGetAvailableBalance) * [MsigGetVested](#MsigGetVested) * [MsigGetVestingSchedule](#MsigGetVestingSchedule) * [MsigPropose](#MsigPropose) + * [MsigRemoveSigner](#MsigRemoveSigner) * [MsigSwapApprove](#MsigSwapApprove) * [MsigSwapCancel](#MsigSwapCancel) * [MsigSwapPropose](#MsigSwapPropose) @@ -2006,7 +2008,33 @@ Response: ``` ### MsigApprove -MsigApprove approves a previously-proposed multisig message +MsigApprove approves a previously-proposed multisig message by transaction ID +It takes the following params: , + + +Perms: sign + +Inputs: +```json +[ + "f01234", + 42, + "f01234" +] +``` + +Response: +```json +{ + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" +} +``` + +### MsigApproveTxnHash +MsigApproveTxnHash approves a previously-proposed multisig message, specified +using both transaction ID and a hash of the parameters used in the +proposal. This method of approval can be used to ensure you only approve +exactly the transaction you think you are. It takes the following params: , , , , , , , @@ -2036,7 +2064,7 @@ Response: ### MsigCancel MsigCancel cancels a previously-proposed multisig message -It takes the following params: , , , , +It takes the following params: , , , , , , @@ -2202,6 +2230,33 @@ Response: } ``` +### MsigRemoveSigner +MsigRemoveSigner proposes the removal of a signer from the multisig. +It accepts the multisig to make the change on, the proposer address to +send the message from, the address to be removed, and a boolean +indicating whether or not the signing threshold should be lowered by one +along with the address removal. + + +Perms: sign + +Inputs: +```json +[ + "f01234", + "f01234", + "f01234", + true +] +``` + +Response: +```json +{ + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" +} +``` + ### MsigSwapApprove MsigSwapApprove approves a previously proposed SwapSigner It takes the following params: , , , From 23f5a99117226fba70fea235c421f1d7b6f26745 Mon Sep 17 00:00:00 2001 From: Peter Rabbitson Date: Sat, 10 Oct 2020 15:36:32 +0200 Subject: [PATCH 687/885] Add comment clarifying current codepath --- chain/store/store.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/chain/store/store.go b/chain/store/store.go index 6fb689b3e..c6fc0cbef 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -454,7 +454,17 @@ func (cs *ChainStore) takeHeaviestTipSet(ctx context.Context, ts *types.TipSet) func (cs *ChainStore) FlushValidationCache() error { log.Infof("clearing block validation cache...") - dsWalk, err := cs.ds.Query(query.Query{KeysOnly: true}) + dsWalk, err := cs.ds.Query(query.Query{ + // Potential TODO: the validation cache is not a namespace on its own + // but is rather constructed as prefixed-key `foo:bar` via .Instance(), which + // in turn does not work with the filter, which can match only on `foo/bar` + // + // If this is addressed (blockcache goes into its own sub-namespace) then + // strings.HasPrefix(...) below can be skipped + // + //Prefix: blockValidationCacheKeyPrefix.String() + KeysOnly: true, + }) if err != nil { return xerrors.Errorf("failed to initialize key listing query: %w", err) } From a80e98dc051bca1ff87e7569c8750fbeecebb42a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 10 Oct 2020 15:38:14 +0200 Subject: [PATCH 688/885] cli: Don't output errors to stdout --- cli/helper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/helper.go b/cli/helper.go index 70a168145..9398ead71 100644 --- a/cli/helper.go +++ b/cli/helper.go @@ -35,7 +35,7 @@ func RunApp(app *cli.App) { if os.Getenv("LOTUS_DEV") != "" { log.Warnf("%+v", err) } else { - fmt.Printf("ERROR: %s\n\n", err) + fmt.Fprintf(os.Stderr, "ERROR: %s\n\n", err) // nolint:errcheck } var phe *PrintHelpErr if xerrors.As(err, &phe) { From 4fa6afad132704c699b351b7c2b05ec48057c629 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 10 Oct 2020 15:44:22 +0200 Subject: [PATCH 689/885] Fix panic in wallet export when key is not found --- chain/wallet/wallet.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/chain/wallet/wallet.go b/chain/wallet/wallet.go index 3efbf8ab7..57e63625b 100644 --- a/chain/wallet/wallet.go +++ b/chain/wallet/wallet.go @@ -141,6 +141,9 @@ func (w *LocalWallet) WalletExport(ctx context.Context, addr address.Address) (* if err != nil { return nil, xerrors.Errorf("failed to find key to export: %w", err) } + if k == nil { + return nil, xerrors.Errorf("key not found") + } return &k.KeyInfo, nil } @@ -274,6 +277,9 @@ func (w *LocalWallet) WalletDelete(ctx context.Context, addr address.Address) er if err != nil { return xerrors.Errorf("failed to delete key %s : %w", addr, err) } + if k == nil { + return nil // already not there + } if err := w.keystore.Put(KTrashPrefix+k.Address.String(), k.KeyInfo); err != nil { return xerrors.Errorf("failed to mark key %s as trashed: %w", addr, err) From 7245ac2b699263d91321c4286f9b1a51eac4a91c Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Sat, 10 Oct 2020 08:31:04 -0700 Subject: [PATCH 690/885] fix a race in the sync manager 1. SyncerState contains a mutex and should never be copied. Honestly, this case was probably fine but it's just as easy to create a separate snapshot type and easier to reason about. 2. We need to initialize the syncStates array once at start, before accessing it. By each syncer state inside each worker, we were racing with calls to `State()`. Again, this was probably benign, but I don't trust optimizing compilers. --- chain/sync.go | 2 +- chain/sync_manager.go | 15 ++++++++------ chain/syncstate.go | 46 ++++++++++++++++++++----------------------- 3 files changed, 31 insertions(+), 32 deletions(-) diff --git a/chain/sync.go b/chain/sync.go index 369c65d33..f71100621 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -1726,7 +1726,7 @@ func VerifyElectionPoStVRF(ctx context.Context, worker address.Address, rand []b return gen.VerifyVRF(ctx, worker, rand, evrf) } -func (syncer *Syncer) State() []SyncerState { +func (syncer *Syncer) State() []SyncerStateSnapshot { return syncer.syncmgr.State() } diff --git a/chain/sync_manager.go b/chain/sync_manager.go index 811092bc7..c7fdea726 100644 --- a/chain/sync_manager.go +++ b/chain/sync_manager.go @@ -38,7 +38,7 @@ type SyncManager interface { SetPeerHead(ctx context.Context, p peer.ID, ts *types.TipSet) // State retrieves the state of the sync workers. - State() []SyncerState + State() []SyncerStateSnapshot } type syncManager struct { @@ -79,7 +79,7 @@ type syncResult struct { const syncWorkerCount = 3 func NewSyncManager(sync SyncFunc) SyncManager { - return &syncManager{ + sm := &syncManager{ bspThresh: 1, peerHeads: make(map[peer.ID]*types.TipSet), syncTargets: make(chan *types.TipSet), @@ -90,6 +90,10 @@ func NewSyncManager(sync SyncFunc) SyncManager { doSync: sync, stop: make(chan struct{}), } + for i := range sm.syncStates { + sm.syncStates[i] = new(SyncerState) + } + return sm } func (sm *syncManager) Start() { @@ -128,8 +132,8 @@ func (sm *syncManager) SetPeerHead(ctx context.Context, p peer.ID, ts *types.Tip sm.incomingTipSets <- ts } -func (sm *syncManager) State() []SyncerState { - ret := make([]SyncerState, 0, len(sm.syncStates)) +func (sm *syncManager) State() []SyncerStateSnapshot { + ret := make([]SyncerStateSnapshot, 0, len(sm.syncStates)) for _, s := range sm.syncStates { ret = append(ret, s.Snapshot()) } @@ -405,8 +409,7 @@ func (sm *syncManager) scheduleWorkSent() { } func (sm *syncManager) syncWorker(id int) { - ss := &SyncerState{} - sm.syncStates[id] = ss + ss := sm.syncStates[id] for { select { case ts, ok := <-sm.syncTargets: diff --git a/chain/syncstate.go b/chain/syncstate.go index 06cd5d91e..26f9f1c39 100644 --- a/chain/syncstate.go +++ b/chain/syncstate.go @@ -11,8 +11,7 @@ import ( "github.com/filecoin-project/lotus/chain/types" ) -type SyncerState struct { - lk sync.Mutex +type SyncerStateSnapshot struct { Target *types.TipSet Base *types.TipSet Stage api.SyncStateStage @@ -22,6 +21,11 @@ type SyncerState struct { End time.Time } +type SyncerState struct { + lk sync.Mutex + data SyncerStateSnapshot +} + func (ss *SyncerState) SetStage(v api.SyncStateStage) { if ss == nil { return @@ -29,9 +33,9 @@ func (ss *SyncerState) SetStage(v api.SyncStateStage) { ss.lk.Lock() defer ss.lk.Unlock() - ss.Stage = v + ss.data.Stage = v if v == api.StageSyncComplete { - ss.End = build.Clock.Now() + ss.data.End = build.Clock.Now() } } @@ -42,13 +46,13 @@ func (ss *SyncerState) Init(base, target *types.TipSet) { ss.lk.Lock() defer ss.lk.Unlock() - ss.Target = target - ss.Base = base - ss.Stage = api.StageHeaders - ss.Height = 0 - ss.Message = "" - ss.Start = build.Clock.Now() - ss.End = time.Time{} + ss.data.Target = target + ss.data.Base = base + ss.data.Stage = api.StageHeaders + ss.data.Height = 0 + ss.data.Message = "" + ss.data.Start = build.Clock.Now() + ss.data.End = time.Time{} } func (ss *SyncerState) SetHeight(h abi.ChainEpoch) { @@ -58,7 +62,7 @@ func (ss *SyncerState) SetHeight(h abi.ChainEpoch) { ss.lk.Lock() defer ss.lk.Unlock() - ss.Height = h + ss.data.Height = h } func (ss *SyncerState) Error(err error) { @@ -68,21 +72,13 @@ func (ss *SyncerState) Error(err error) { ss.lk.Lock() defer ss.lk.Unlock() - ss.Message = err.Error() - ss.Stage = api.StageSyncErrored - ss.End = build.Clock.Now() + ss.data.Message = err.Error() + ss.data.Stage = api.StageSyncErrored + ss.data.End = build.Clock.Now() } -func (ss *SyncerState) Snapshot() SyncerState { +func (ss *SyncerState) Snapshot() SyncerStateSnapshot { ss.lk.Lock() defer ss.lk.Unlock() - return SyncerState{ - Base: ss.Base, - Target: ss.Target, - Stage: ss.Stage, - Height: ss.Height, - Message: ss.Message, - Start: ss.Start, - End: ss.End, - } + return ss.data } From c463582528150150ae5571901e1005f5475d89ff Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Sat, 10 Oct 2020 08:33:06 -0700 Subject: [PATCH 691/885] fix a race and optimize hello messages LifecycleCtx can _only_ be called during startup as it appends an fx hook. Worse, this was causing us to append an fx hook on every single hello message, leaking memory (and probably causing other shutdown issues...). --- node/modules/services.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/node/modules/services.go b/node/modules/services.go index 4ee0abacc..e0a7c2eda 100644 --- a/node/modules/services.go +++ b/node/modules/services.go @@ -42,11 +42,13 @@ func RunHello(mctx helpers.MetricsCtx, lc fx.Lifecycle, h host.Host, svc *hello. return xerrors.Errorf("failed to subscribe to event bus: %w", err) } + ctx := helpers.LifecycleCtx(mctx, lc) + go func() { for evt := range sub.Out() { pic := evt.(event.EvtPeerIdentificationCompleted) go func() { - if err := svc.SayHello(helpers.LifecycleCtx(mctx, lc), pic.Peer); err != nil { + if err := svc.SayHello(ctx, pic.Peer); err != nil { protos, _ := h.Peerstore().GetProtocols(pic.Peer) agent, _ := h.Peerstore().Get(pic.Peer, "AgentVersion") if protosContains(protos, hello.ProtocolID) { From 32d95fc97a01eb02d17665dfe329c89f6f4d6ce3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 10 Oct 2020 20:46:06 +0200 Subject: [PATCH 692/885] docsgen, gofmt --- api/apistruct/struct.go | 2 +- cli/sync.go | 6 +++--- documentation/en/api-methods.md | 11 +++++++++++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 6bfbf43cb..5ff83ed09 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -111,7 +111,7 @@ type FullNodeStruct struct { SyncCheckpoint func(ctx context.Context, key types.TipSetKey) error `perm:"admin"` SyncMarkBad func(ctx context.Context, bcid cid.Cid) error `perm:"admin"` SyncUnmarkBad func(ctx context.Context, bcid cid.Cid) error `perm:"admin"` - SyncUnmarkAllBad func(ctx context.Context) error `perm:"admin"` + SyncUnmarkAllBad func(ctx context.Context) error `perm:"admin"` SyncCheckBad func(ctx context.Context, bcid cid.Cid) (string, error) `perm:"read"` SyncValidateTipset func(ctx context.Context, tsk types.TipSetKey) (bool, error) `perm:"read"` diff --git a/cli/sync.go b/cli/sync.go index a5178441d..ea066cbda 100644 --- a/cli/sync.go +++ b/cli/sync.go @@ -122,11 +122,11 @@ var syncMarkBadCmd = &cli.Command{ } var syncUnmarkBadCmd = &cli.Command{ - Name: "unmark-bad", - Usage: "Unmark the given block as bad, makes it possible to sync to a chain containing it", + Name: "unmark-bad", + Usage: "Unmark the given block as bad, makes it possible to sync to a chain containing it", Flags: []cli.Flag{ &cli.BoolFlag{ - Name: "all", + Name: "all", Usage: "drop the entire bad block cache", }, }, diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index b3a9627e0..0774db1c8 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -175,6 +175,7 @@ * [SyncMarkBad](#SyncMarkBad) * [SyncState](#SyncState) * [SyncSubmitBlock](#SyncSubmitBlock) + * [SyncUnmarkAllBad](#SyncUnmarkAllBad) * [SyncUnmarkBad](#SyncUnmarkBad) * [SyncValidateTipset](#SyncValidateTipset) * [Wallet](#Wallet) @@ -4546,6 +4547,16 @@ Inputs: Response: `{}` +### SyncUnmarkAllBad +SyncUnmarkAllBad purges bad block cache, making it possible to sync to chains previously marked as bad + + +Perms: admin + +Inputs: `null` + +Response: `{}` + ### SyncUnmarkBad SyncUnmarkBad unmarks a blocks as bad, making it possible to be validated and synced again. From 85be11fb7515d997460b12cb91ed212d7e3dc3ac Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Sun, 11 Oct 2020 18:10:02 +0200 Subject: [PATCH 693/885] Add propose remove Signed-off-by: Jakub Sztandera --- chain/actors/builtin/multisig/message.go | 1 + cli/multisig.go | 79 ++++++++++++++++++++++++ 2 files changed, 80 insertions(+) diff --git a/chain/actors/builtin/multisig/message.go b/chain/actors/builtin/multisig/message.go index b19287432..5beb5319d 100644 --- a/chain/actors/builtin/multisig/message.go +++ b/chain/actors/builtin/multisig/message.go @@ -45,6 +45,7 @@ type MessageBuilder interface { // this type is the same between v0 and v2 type ProposalHashData = multisig2.ProposalHashData +type ProposeReturn = multisig2.ProposeReturn func txnParams(id uint64, data *ProposalHashData) ([]byte, error) { params := multisig2.TxnIDParams{ID: multisig2.TxnID(id)} diff --git a/cli/multisig.go b/cli/multisig.go index b692fad5a..90074c64d 100644 --- a/cli/multisig.go +++ b/cli/multisig.go @@ -41,6 +41,7 @@ var multisigCmd = &cli.Command{ msigCreateCmd, msigInspectCmd, msigProposeCmd, + msigRemoveProposeCmd, msigApproveCmd, msigAddProposeCmd, msigAddApproveCmd, @@ -504,6 +505,84 @@ var msigApproveCmd = &cli.Command{ }, } +var msigRemoveProposeCmd = &cli.Command{ + Name: "propose-remove", + Usage: "Propose to remove a signer", + ArgsUsage: "[multisigAddress signer]", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "decrease-threshold", + Usage: "whether the number of required signers should be decreased", + }, + &cli.StringFlag{ + Name: "from", + Usage: "account to send the propose message from", + }, + }, + Action: func(cctx *cli.Context) error { + if cctx.Args().Len() != 2 { + return ShowHelp(cctx, fmt.Errorf("must pass multisig address and signer address")) + } + + api, closer, err := GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + ctx := ReqContext(cctx) + + msig, err := address.NewFromString(cctx.Args().Get(0)) + if err != nil { + return err + } + + addr, err := address.NewFromString(cctx.Args().Get(1)) + if err != nil { + return err + } + + var from address.Address + if cctx.IsSet("from") { + f, err := address.NewFromString(cctx.String("from")) + if err != nil { + return err + } + from = f + } else { + defaddr, err := api.WalletDefaultAddress(ctx) + if err != nil { + return err + } + from = defaddr + } + + msgCid, err := api.MsigRemoveSigner(ctx, msig, from, addr, cctx.Bool("decrease-threshold")) + if err != nil { + return err + } + + fmt.Println("sent remove proposal in message: ", msgCid) + + wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) + if err != nil { + return err + } + + if wait.Receipt.ExitCode != 0 { + return fmt.Errorf("add proposal returned exit %d", wait.Receipt.ExitCode) + } + + var ret multisig.ProposeReturn + err = ret.UnmarshalCBOR(bytes.NewReader(wait.Receipt.Return)) + if err != nil { + return xerrors.Errorf("decoding proposal return: %w", err) + } + fmt.Printf("TxnID: %d", ret.TxnID) + + return nil + }, +} + var msigAddProposeCmd = &cli.Command{ Name: "add-propose", Usage: "Propose to add a signer", From a36ec27230d1713aed62dcedaa6b1d19e9b2e723 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Sun, 11 Oct 2020 21:14:51 +0200 Subject: [PATCH 694/885] Look at block base fee Signed-off-by: Jakub Sztandera --- cmd/lotus-pcr/main.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cmd/lotus-pcr/main.go b/cmd/lotus-pcr/main.go index 5491e4af2..b5da05883 100644 --- a/cmd/lotus-pcr/main.go +++ b/cmd/lotus-pcr/main.go @@ -926,6 +926,11 @@ func (r *refunder) processTipsetStorageMinerActor(ctx context.Context, tipset *t return false, messageMethod, types.NewInt(0), nil } + if tipset.Blocks()[0].ParentBaseFee.GreaterThan(r.proveFeeCapMax) { + log.Debugw("skipping high base fee message", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "basefee", tipset.Blocks()[0].ParentBaseFee, "fee_cap_max", r.proveFeeCapMax) + return false, messageMethod, types.NewInt(0), nil + } + var sn abi.SectorNumber var proveCommitSector miner0.ProveCommitSectorParams @@ -982,6 +987,11 @@ func (r *refunder) processTipsetStorageMinerActor(ctx context.Context, tipset *t return false, messageMethod, types.NewInt(0), nil } + if tipset.Blocks()[0].ParentBaseFee.GreaterThan(r.preFeeCapMax) { + log.Debugw("skipping high base fee message", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "basefee", tipset.Blocks()[0].ParentBaseFee, "fee_cap_max", r.preFeeCapMax) + return false, messageMethod, types.NewInt(0), nil + } + var precommitInfo miner.SectorPreCommitInfo if err := precommitInfo.UnmarshalCBOR(bytes.NewBuffer(m.Params)); err != nil { log.Warnw("failed to decode precommit params", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To) From 23470bc38ff8502660c107418d323554d9d6fb83 Mon Sep 17 00:00:00 2001 From: Travis Person Date: Sun, 11 Oct 2020 20:27:36 +0000 Subject: [PATCH 695/885] lotus-stats: optmize getting miner power --- tools/stats/metrics.go | 42 +++++++++++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/tools/stats/metrics.go b/tools/stats/metrics.go index aee61b2aa..8dc79ea44 100644 --- a/tools/stats/metrics.go +++ b/tools/stats/metrics.go @@ -16,6 +16,9 @@ import ( "github.com/filecoin-project/lotus/chain/actors/builtin/reward" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/actors/builtin" + "github.com/filecoin-project/specs-actors/actors/builtin/power" + "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/ipfs/go-cid" "github.com/multiformats/go-multihash" @@ -269,22 +272,47 @@ func RecordTipsetStatePoints(ctx context.Context, api api.FullNode, pl *PointLis p = NewPoint("chain.power", totalPower.TotalPower.QualityAdjPower.Int64()) pl.AddPoint(p) - miners, err := api.StateListMiners(ctx, tipset.Key()) + powerActor, err := api.StateGetActor(ctx, builtin.StoragePowerActorAddr, tipset.Key()) if err != nil { return err } - for _, addr := range miners { - mp, err := api.StateMinerPower(ctx, addr, tipset.Key()) + powerRaw, err := api.ChainReadObj(ctx, powerActor.Head) + if err != nil { + return err + } + + var powerActorState power.State + + if err := powerActorState.UnmarshalCBOR(bytes.NewReader(powerRaw)); err != nil { + return fmt.Errorf("failed to unmarshal power actor state: %w", err) + } + + s := &ApiIpldStore{ctx, api} + mp, err := adt.AsMap(s, powerActorState.Claims) + if err != nil { + return err + } + + var claim power.Claim + err = mp.ForEach(&claim, func(key string) error { + addr, err := address.NewFromBytes([]byte(key)) if err != nil { return err } - if !mp.MinerPower.QualityAdjPower.IsZero() { - p = NewPoint("chain.miner_power", mp.MinerPower.QualityAdjPower.Int64()) - p.AddTag("miner", addr.String()) - pl.AddPoint(p) + if claim.QualityAdjPower.Int64() == 0 { + return nil } + + p = NewPoint("chain.miner_power", claim.QualityAdjPower.Int64()) + p.AddTag("miner", addr.String()) + pl.AddPoint(p) + + return nil + }) + if err != nil { + return err } return nil From b35f9b56b42ae7415284f5ced424af21923d693e Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Fri, 9 Oct 2020 20:17:04 -0700 Subject: [PATCH 696/885] WIP: adding in ledger support Signed-off-by: Jakub Sztandera --- chain/messagepool/messagepool.go | 3 + chain/wallet/ledger/ledger.go | 139 +++++++++++++++++++++ cmd/lotus-shed/ledger.go | 205 +++++++++++++++++++++++++++++++ cmd/lotus-shed/main.go | 1 + cmd/lotus-wallet/main.go | 11 +- go.mod | 1 + go.sum | 7 ++ 7 files changed, 366 insertions(+), 1 deletion(-) create mode 100644 chain/wallet/ledger/ledger.go create mode 100644 cmd/lotus-shed/ledger.go diff --git a/chain/messagepool/messagepool.go b/chain/messagepool/messagepool.go index d3c638b22..2a4704c3b 100644 --- a/chain/messagepool/messagepool.go +++ b/chain/messagepool/messagepool.go @@ -1249,6 +1249,9 @@ func (mp *MessagePool) Updates(ctx context.Context) (<-chan api.MpoolUpdate, err } func (mp *MessagePool) loadLocal() error { + return nil + // TODO: this causes super slow startup... + res, err := mp.localMsgs.Query(query.Query{}) if err != nil { return xerrors.Errorf("query local messages: %w", err) diff --git a/chain/wallet/ledger/ledger.go b/chain/wallet/ledger/ledger.go new file mode 100644 index 000000000..36b0fb040 --- /dev/null +++ b/chain/wallet/ledger/ledger.go @@ -0,0 +1,139 @@ +package ledgerwallet + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/crypto" + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/query" + ledgerfil "github.com/whyrusleeping/ledger-filecoin-go" +) + +type LedgerWallet struct { + ds datastore.Datastore +} + +func NewWallet(ds datastore.Datastore) *LedgerWallet { + return &LedgerWallet{ds} +} + +type LedgerKeyInfo struct { + Address address.Address + Path []uint32 +} + +var _ (api.WalletAPI) = (*LedgerWallet)(nil) + +func (lw LedgerWallet) WalletSign(ctx context.Context, signer address.Address, toSign []byte, meta api.MsgMeta) (*crypto.Signature, error) { + ki, err := lw.getKeyInfo(signer) + if err != nil { + return nil, err + } + + fl, err := ledgerfil.FindLedgerFilecoinApp() + if err != nil { + return nil, err + } + defer fl.Close() + if meta.Type != api.MTChainMsg { + return nil, fmt.Errorf("ledger can only sign chain messages") + } + + // TODO: assert meta matches the 'toSign' bits + + sig, err := fl.SignSECP256K1(ki.Path, meta.Extra) + if err != nil { + return nil, err + } + + return &crypto.Signature{ + Type: crypto.SigTypeSecp256k1, + Data: sig.SignatureBytes(), + }, nil +} + +func (lw LedgerWallet) getKeyInfo(addr address.Address) (*LedgerKeyInfo, error) { + kib, err := lw.ds.Get(keyForAddr(addr)) + if err != nil { + return nil, err + } + + var out LedgerKeyInfo + if err := json.Unmarshal(kib, &out); err != nil { + return nil, err + } + + return &out, nil +} + +func (lw LedgerWallet) WalletDelete(ctx context.Context, k address.Address) error { + return lw.ds.Delete(keyForAddr(k)) +} + +func (lw LedgerWallet) WalletExport(ctx context.Context, k address.Address) (*types.KeyInfo, error) { + return nil, fmt.Errorf("cannot export keys from ledger wallets") +} + +func (lw LedgerWallet) WalletHas(ctx context.Context, k address.Address) (bool, error) { + _, err := lw.ds.Get(keyForAddr(k)) + if err == nil { + return true, nil + } + if err == datastore.ErrNotFound { + return false, nil + } + return false, err +} + +func (lw LedgerWallet) WalletImport(ctx context.Context, kinfo *types.KeyInfo) (address.Address, error) { + var ki LedgerKeyInfo + if err := json.Unmarshal(kinfo.PrivateKey, &ki); err != nil { + return address.Undef, err + } + + if ki.Address == address.Undef { + return address.Undef, fmt.Errorf("no address given in imported key info") + } + + if err := lw.ds.Put(keyForAddr(ki.Address), kinfo.PrivateKey); err != nil { + return address.Undef, err + } + + return ki.Address, nil +} + +func (lw LedgerWallet) WalletList(ctx context.Context) ([]address.Address, error) { + res, err := lw.ds.Query(query.Query{Prefix: "ledgerkey/"}) + if err != nil { + return nil, err + } + + var out []address.Address + for { + res, ok := res.NextSync() + if !ok { + break + } + + var ki LedgerKeyInfo + if err := json.Unmarshal(res.Value, &ki); err != nil { + return nil, err + } + + out = append(out, ki.Address) + } + return out, nil +} + +func (lw LedgerWallet) WalletNew(ctx context.Context, t crypto.SigType) (address.Address, error) { + return address.Undef, fmt.Errorf("cannot create new address on ledger") +} + +func keyForAddr(addr address.Address) datastore.Key { + return datastore.NewKey("ledgerkey/" + addr.String()) +} diff --git a/cmd/lotus-shed/ledger.go b/cmd/lotus-shed/ledger.go new file mode 100644 index 000000000..4bb61e5e0 --- /dev/null +++ b/cmd/lotus-shed/ledger.go @@ -0,0 +1,205 @@ +package main + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/wallet" + ledgerwallet "github.com/filecoin-project/lotus/chain/wallet/ledger" + "github.com/urfave/cli/v2" + ledgerfil "github.com/whyrusleeping/ledger-filecoin-go" +) + +var ledgerCmd = &cli.Command{ + Name: "ledger", + Usage: "Ledger interactions", + Flags: []cli.Flag{}, + Subcommands: []*cli.Command{ + ledgerListAddressesCmd, + ledgerKeyInfoCmd, + ledgerSignTestCmd, + }, +} + +var ledgerListAddressesCmd = &cli.Command{ + Name: "list", + Action: func(cctx *cli.Context) error { + + fl, err := ledgerfil.FindLedgerFilecoinApp() + if err != nil { + return err + } + + for i := 0; i < 20; i++ { + p := []uint32{0x80000000 + 44, 0x80000000 + 461, 0x80000000, 0, uint32(i)} + pubk, err := fl.GetPublicKeySECP256K1(p) + if err != nil { + return err + } + + addr, err := address.NewSecp256k1Address(pubk) + if err != nil { + return err + } + + fmt.Printf("%s: %s\n", addr, printHDPath(p)) + } + + return nil + }, +} + +func parseHDPath(s string) ([]uint32, error) { + parts := strings.Split(s, "/") + if parts[0] != "m" { + return nil, fmt.Errorf("expected HD path to start with 'm'") + } + + var out []uint32 + for _, p := range parts[1:] { + var hard bool + if strings.HasSuffix(p, "'") { + p = p[:len(p)-1] + hard = true + } + + v, err := strconv.ParseUint(p, 10, 32) + if err != nil { + return nil, err + } + if v >= 0x80000000 { + return nil, fmt.Errorf("path element %s too large", p) + } + + if hard { + v += 0x80000000 + } + out = append(out, uint32(v)) + } + return out, nil +} + +func printHDPath(pth []uint32) string { + s := "m/" + for _, p := range pth { + var hard bool + if p >= 0x80000000 { + p -= 0x80000000 + } + s += fmt.Sprint(p) + if hard { + s += "'" + } + s += "/" + } + + return strings.TrimRight(s, "/") +} + +func numlist(p []uint32) string { + var out []string + for _, v := range p { + out = append(out, fmt.Sprint(v)) + } + return strings.Join(out, ",") +} + +var ledgerKeyInfoCmd = &cli.Command{ + Name: "key-info", + Action: func(cctx *cli.Context) error { + if !cctx.Args().Present() { + return cli.ShowCommandHelp(cctx, cctx.Command.Name) + } + + fl, err := ledgerfil.FindLedgerFilecoinApp() + if err != nil { + return err + } + + p, err := parseHDPath(cctx.Args().First()) + if err != nil { + return err + } + + pubk, _, addr, err := fl.GetAddressPubKeySECP256K1(p) + if err != nil { + return err + } + fmt.Println(addr) + fmt.Println(pubk) + + a, err := address.NewFromString(addr) + if err != nil { + return err + } + + var pd ledgerwallet.LedgerKeyInfo + pd.Address = a + pd.Path = p + + b, err := json.Marshal(pd) + if err != nil { + return err + } + + var ki types.KeyInfo + ki.Type = wallet.KTSecp256k1 + ki.PrivateKey = b + + out, err := json.Marshal(ki) + if err != nil { + return err + } + + fmt.Println(string(out)) + + return nil + }, +} + +var ledgerSignTestCmd = &cli.Command{ + Name: "sign", + Action: func(cctx *cli.Context) error { + if !cctx.Args().Present() { + return cli.ShowCommandHelp(cctx, cctx.Command.Name) + } + + fl, err := ledgerfil.FindLedgerFilecoinApp() + if err != nil { + return err + } + + p, err := parseHDPath(cctx.Args().First()) + if err != nil { + return err + } + + addr, err := address.NewFromString("f1xc3hws5n6y5m3m44gzb3gyjzhups6wzmhe663ji") + if err != nil { + return err + } + + m := &types.Message{ + To: addr, + From: addr, + } + + b, err := m.ToStorageBlock() + if err != nil { + return err + } + + sig, err := fl.SignSECP256K1(p, b.RawData()) + if err != nil { + return err + } + + fmt.Println(sig.SignatureBytes()) + + return nil + }, +} diff --git a/cmd/lotus-shed/main.go b/cmd/lotus-shed/main.go index 4542551db..29a105355 100644 --- a/cmd/lotus-shed/main.go +++ b/cmd/lotus-shed/main.go @@ -41,6 +41,7 @@ func main() { syncCmd, stateTreePruneCmd, datastoreCmd, + ledgerCmd, } app := &cli.App{ diff --git a/cmd/lotus-wallet/main.go b/cmd/lotus-wallet/main.go index ae3580a59..fbedffed0 100644 --- a/cmd/lotus-wallet/main.go +++ b/cmd/lotus-wallet/main.go @@ -13,6 +13,7 @@ import ( "github.com/filecoin-project/go-jsonrpc" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/wallet" + ledgerwallet "github.com/filecoin-project/lotus/chain/wallet/ledger" lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/lib/lotuslog" "github.com/filecoin-project/lotus/node/repo" @@ -98,6 +99,14 @@ var runCmd = &cli.Command{ if err != nil { return err } + _ = w + + ds, err := lr.Datastore("/metadata") + if err != nil { + return err + } + + lw := ledgerwallet.NewWallet(ds) address := cctx.String("listen") mux := mux.NewRouter() @@ -105,7 +114,7 @@ var runCmd = &cli.Command{ log.Info("Setting up API endpoint at " + address) rpcServer := jsonrpc.NewServer() - rpcServer.Register("Filecoin", &LoggedWallet{under: w}) + rpcServer.Register("Filecoin", &LoggedWallet{under: lw}) mux.Handle("/rpc/v0", rpcServer) mux.PathPrefix("/").Handler(http.DefaultServeMux) // pprof diff --git a/go.mod b/go.mod index eb79a496c..d6a4768b7 100644 --- a/go.mod +++ b/go.mod @@ -120,6 +120,7 @@ require ( github.com/urfave/cli/v2 v2.2.0 github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba github.com/whyrusleeping/cbor-gen v0.0.0-20200826160007-0b9f6c5fb163 + github.com/whyrusleeping/ledger-filecoin-go v0.9.1-0.20201010031517-c3dcc1bddce4 github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542 diff --git a/go.sum b/go.sum index e13359f6c..fe7aacbf9 100644 --- a/go.sum +++ b/go.sum @@ -1422,6 +1422,8 @@ github.com/whyrusleeping/go-smux-multiplex v3.0.16+incompatible/go.mod h1:34LEDb github.com/whyrusleeping/go-smux-multistream v2.0.2+incompatible/go.mod h1:dRWHHvc4HDQSHh9gbKEBbUZ+f2Q8iZTPG3UOGYODxSQ= github.com/whyrusleeping/go-smux-yamux v2.0.8+incompatible/go.mod h1:6qHUzBXUbB9MXmw3AUdB52L8sEb/hScCqOdW2kj/wuI= github.com/whyrusleeping/go-smux-yamux v2.0.9+incompatible/go.mod h1:6qHUzBXUbB9MXmw3AUdB52L8sEb/hScCqOdW2kj/wuI= +github.com/whyrusleeping/ledger-filecoin-go v0.9.1-0.20201010031517-c3dcc1bddce4 h1:NwiwjQDB3CzQ5XH0rdMh1oQqzJH7O2PSLWxif/w3zsY= +github.com/whyrusleeping/ledger-filecoin-go v0.9.1-0.20201010031517-c3dcc1bddce4/go.mod h1:K+EVq8d5QcQ2At5VECsA+SNZvWefyBXh8TnIsxo1OvQ= github.com/whyrusleeping/mafmt v1.2.8/go.mod h1:faQJFPbLSxzD9xpA02ttW/tS9vZykNvXwGvqIpk20FA= github.com/whyrusleeping/mdns v0.0.0-20180901202407-ef14215e6b30/go.mod h1:j4l84WPFclQPj320J9gp0XwNKBb3U0zt5CBqjPp22G4= github.com/whyrusleeping/mdns v0.0.0-20190826153040-b9b60ed33aa9/go.mod h1:j4l84WPFclQPj320J9gp0XwNKBb3U0zt5CBqjPp22G4= @@ -1443,6 +1445,10 @@ github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542 h1:oWgZJmC1DorFZD github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542/go.mod h1:7T39/ZMvaSEZlBPoYfVFmsBLmUl3uz9IuzWj/U6FtvQ= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/zondax/hid v0.9.0 h1:eiT3P6vNxAEVxXMw66eZUAAnU2zD33JBkfG/EnfAKl8= +github.com/zondax/hid v0.9.0/go.mod h1:l5wttcP0jwtdLjqjMMWFVEE7d1zO0jvSPA9OPZxWpEM= +github.com/zondax/ledger-go v0.12.1 h1:hYRcyznPRJp+5mzF2sazTLP2nGvGjYDD2VzhHhFomLU= +github.com/zondax/ledger-go v0.12.1/go.mod h1:KatxXrVDzgWwbssUWsF5+cOJHXPvzQ09YSlzGNuhOEo= go.dedis.ch/fixbuf v1.0.3 h1:hGcV9Cd/znUxlusJ64eAlExS+5cJDIyTyEG+otu5wQs= go.dedis.ch/fixbuf v1.0.3/go.mod h1:yzJMt34Wa5xD37V5RTdmp38cz3QhMagdGoem9anUalw= go.dedis.ch/kyber/v3 v3.0.4/go.mod h1:OzvaEnPvKlyrWyp3kGXlFdp7ap1VC6RkZDTaPikqhsQ= @@ -1515,6 +1521,7 @@ golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200117160349-530e935923ad/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200128174031-69ecbb4d6d5d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= From 99791a695373c78ec93d985adb5895358406585b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 10 Oct 2020 13:03:39 +0200 Subject: [PATCH 697/885] lotus-wallet: ledger flag Signed-off-by: Jakub Sztandera --- cmd/lotus-wallet/main.go | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/cmd/lotus-wallet/main.go b/cmd/lotus-wallet/main.go index fbedffed0..2d87998a4 100644 --- a/cmd/lotus-wallet/main.go +++ b/cmd/lotus-wallet/main.go @@ -2,6 +2,7 @@ package main import ( "context" + "github.com/filecoin-project/lotus/api" "net" "net/http" "os" @@ -61,6 +62,10 @@ var runCmd = &cli.Command{ Usage: "host address and port the wallet api will listen on", Value: "0.0.0.0:1777", }, + &cli.BoolFlag{ + Name: "ledger", + Usage: "use a ledger device instead of an on-disk wallet", + }, }, Action: func(cctx *cli.Context) error { log.Info("Starting lotus wallet") @@ -95,18 +100,21 @@ var runCmd = &cli.Command{ return err } - w, err := wallet.NewWallet(ks) - if err != nil { - return err - } - _ = w + var w api.WalletAPI + if !cctx.Bool("ledger") { + w, err = wallet.NewWallet(ks) + if err != nil { + return err + } + } else { + ds, err := lr.Datastore("/metadata") + if err != nil { + return err + } - ds, err := lr.Datastore("/metadata") - if err != nil { - return err + w = ledgerwallet.NewWallet(ds) } - lw := ledgerwallet.NewWallet(ds) address := cctx.String("listen") mux := mux.NewRouter() @@ -114,7 +122,7 @@ var runCmd = &cli.Command{ log.Info("Setting up API endpoint at " + address) rpcServer := jsonrpc.NewServer() - rpcServer.Register("Filecoin", &LoggedWallet{under: lw}) + rpcServer.Register("Filecoin", &LoggedWallet{under: w}) mux.Handle("/rpc/v0", rpcServer) mux.PathPrefix("/").Handler(http.DefaultServeMux) // pprof From 66d6113340e943fe9aca2b0227312ac521b059e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 10 Oct 2020 13:06:02 +0200 Subject: [PATCH 698/885] ledgerwallet: Validate signing bytes Signed-off-by: Jakub Sztandera --- chain/wallet/ledger/ledger.go | 21 +++++++++++++++++++-- cmd/lotus-wallet/main.go | 6 +++--- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/chain/wallet/ledger/ledger.go b/chain/wallet/ledger/ledger.go index 36b0fb040..71ec067c8 100644 --- a/chain/wallet/ledger/ledger.go +++ b/chain/wallet/ledger/ledger.go @@ -1,9 +1,12 @@ package ledgerwallet import ( + "bytes" "context" "encoding/json" "fmt" + "github.com/ipfs/go-cid" + "golang.org/x/xerrors" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/crypto" @@ -27,7 +30,7 @@ type LedgerKeyInfo struct { Path []uint32 } -var _ (api.WalletAPI) = (*LedgerWallet)(nil) +var _ api.WalletAPI = (*LedgerWallet)(nil) func (lw LedgerWallet) WalletSign(ctx context.Context, signer address.Address, toSign []byte, meta api.MsgMeta) (*crypto.Signature, error) { ki, err := lw.getKeyInfo(signer) @@ -44,7 +47,21 @@ func (lw LedgerWallet) WalletSign(ctx context.Context, signer address.Address, t return nil, fmt.Errorf("ledger can only sign chain messages") } - // TODO: assert meta matches the 'toSign' bits + { + var cmsg types.Message + if err := cmsg.UnmarshalCBOR(bytes.NewReader(meta.Extra)); err != nil { + return nil, xerrors.Errorf("unmarshalling message: %w", err) + } + + _, bc, err := cid.CidFromBytes(toSign) + if err != nil { + return nil, xerrors.Errorf("getting cid from signing bytes: %w", err) + } + + if !cmsg.Cid().Equals(bc) { + return nil, xerrors.Errorf("cid(meta.Extra).bytes() != toSign") + } + } sig, err := fl.SignSECP256K1(ki.Path, meta.Extra) if err != nil { diff --git a/cmd/lotus-wallet/main.go b/cmd/lotus-wallet/main.go index 2d87998a4..ffaddc7e3 100644 --- a/cmd/lotus-wallet/main.go +++ b/cmd/lotus-wallet/main.go @@ -2,7 +2,6 @@ package main import ( "context" - "github.com/filecoin-project/lotus/api" "net" "net/http" "os" @@ -12,6 +11,8 @@ import ( "github.com/urfave/cli/v2" "github.com/filecoin-project/go-jsonrpc" + + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/wallet" ledgerwallet "github.com/filecoin-project/lotus/chain/wallet/ledger" @@ -63,7 +64,7 @@ var runCmd = &cli.Command{ Value: "0.0.0.0:1777", }, &cli.BoolFlag{ - Name: "ledger", + Name: "ledger", Usage: "use a ledger device instead of an on-disk wallet", }, }, @@ -115,7 +116,6 @@ var runCmd = &cli.Command{ w = ledgerwallet.NewWallet(ds) } - address := cctx.String("listen") mux := mux.NewRouter() From 2c9215723344c106f9508df7e7c13fde6def76d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 10 Oct 2020 13:41:19 +0200 Subject: [PATCH 699/885] shed ledger: Some UX improvements Signed-off-by: Jakub Sztandera --- cmd/lotus-shed/ledger.go | 55 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 5 deletions(-) diff --git a/cmd/lotus-shed/ledger.go b/cmd/lotus-shed/ledger.go index 4bb61e5e0..fd7ac1b51 100644 --- a/cmd/lotus-shed/ledger.go +++ b/cmd/lotus-shed/ledger.go @@ -7,11 +7,15 @@ import ( "strings" "github.com/filecoin-project/go-address" + "github.com/urfave/cli/v2" + ledgerfil "github.com/whyrusleeping/ledger-filecoin-go" + "golang.org/x/xerrors" + + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/wallet" ledgerwallet "github.com/filecoin-project/lotus/chain/wallet/ledger" - "github.com/urfave/cli/v2" - ledgerfil "github.com/whyrusleeping/ledger-filecoin-go" + lcli "github.com/filecoin-project/lotus/cli" ) var ledgerCmd = &cli.Command{ @@ -27,7 +31,26 @@ var ledgerCmd = &cli.Command{ var ledgerListAddressesCmd = &cli.Command{ Name: "list", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "print-balances", + Usage: "print balances", + Aliases: []string{"b"}, + }, + }, Action: func(cctx *cli.Context) error { + var api api.FullNode + if cctx.Bool("print-balances") { + a, closer, err := lcli.GetFullNodeAPI(cctx) + if err != nil { + return err + } + + api = a + + defer closer() + } + ctx := lcli.ReqContext(cctx) fl, err := ledgerfil.FindLedgerFilecoinApp() if err != nil { @@ -35,6 +58,10 @@ var ledgerListAddressesCmd = &cli.Command{ } for i := 0; i < 20; i++ { + if err := ctx.Err(); err != nil { + return err + } + p := []uint32{0x80000000 + 44, 0x80000000 + 461, 0x80000000, 0, uint32(i)} pubk, err := fl.GetPublicKeySECP256K1(p) if err != nil { @@ -46,7 +73,17 @@ var ledgerListAddressesCmd = &cli.Command{ return err } - fmt.Printf("%s: %s\n", addr, printHDPath(p)) + if cctx.Bool("print-balances") && api != nil { // api check makes linter happier + b, err := api.WalletBalance(ctx, addr) + if err != nil { + return xerrors.Errorf("getting balance: %w", err) + } + + fmt.Printf("%s %s %s\n", addr, printHDPath(p), types.FIL(b)) + } else { + fmt.Printf("%s %s\n", addr, printHDPath(p)) + } + } return nil @@ -110,6 +147,12 @@ func numlist(p []uint32) string { var ledgerKeyInfoCmd = &cli.Command{ Name: "key-info", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "verbose", + Aliases: []string{"v"}, + }, + }, Action: func(cctx *cli.Context) error { if !cctx.Args().Present() { return cli.ShowCommandHelp(cctx, cctx.Command.Name) @@ -129,8 +172,10 @@ var ledgerKeyInfoCmd = &cli.Command{ if err != nil { return err } - fmt.Println(addr) - fmt.Println(pubk) + if cctx.Bool("verbose") { + fmt.Println(addr) + fmt.Println(pubk) + } a, err := address.NewFromString(addr) if err != nil { From 8ae66d6c6a2aa27105566bbdf41dd569b2e4a109 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 10 Oct 2020 14:06:03 +0200 Subject: [PATCH 700/885] Fix lint warnings Signed-off-by: Jakub Sztandera --- chain/wallet/ledger/ledger.go | 12 +++++++----- cmd/lotus-shed/ledger.go | 14 +++----------- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/chain/wallet/ledger/ledger.go b/chain/wallet/ledger/ledger.go index 71ec067c8..94a828603 100644 --- a/chain/wallet/ledger/ledger.go +++ b/chain/wallet/ledger/ledger.go @@ -5,16 +5,18 @@ import ( "context" "encoding/json" "fmt" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/query" + ledgerfil "github.com/whyrusleeping/ledger-filecoin-go" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/crypto" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" - "github.com/ipfs/go-datastore" - "github.com/ipfs/go-datastore/query" - ledgerfil "github.com/whyrusleeping/ledger-filecoin-go" ) type LedgerWallet struct { @@ -42,7 +44,7 @@ func (lw LedgerWallet) WalletSign(ctx context.Context, signer address.Address, t if err != nil { return nil, err } - defer fl.Close() + defer fl.Close() // nolint:errcheck if meta.Type != api.MTChainMsg { return nil, fmt.Errorf("ledger can only sign chain messages") } @@ -82,7 +84,7 @@ func (lw LedgerWallet) getKeyInfo(addr address.Address) (*LedgerKeyInfo, error) var out LedgerKeyInfo if err := json.Unmarshal(kib, &out); err != nil { - return nil, err + return nil, xerrors.Errorf("unmarshalling ledger key info: %w", err) } return &out, nil diff --git a/cmd/lotus-shed/ledger.go b/cmd/lotus-shed/ledger.go index fd7ac1b51..8111cc6bf 100644 --- a/cmd/lotus-shed/ledger.go +++ b/cmd/lotus-shed/ledger.go @@ -33,8 +33,8 @@ var ledgerListAddressesCmd = &cli.Command{ Name: "list", Flags: []cli.Flag{ &cli.BoolFlag{ - Name: "print-balances", - Usage: "print balances", + Name: "print-balances", + Usage: "print balances", Aliases: []string{"b"}, }, }, @@ -137,19 +137,11 @@ func printHDPath(pth []uint32) string { return strings.TrimRight(s, "/") } -func numlist(p []uint32) string { - var out []string - for _, v := range p { - out = append(out, fmt.Sprint(v)) - } - return strings.Join(out, ",") -} - var ledgerKeyInfoCmd = &cli.Command{ Name: "key-info", Flags: []cli.Flag{ &cli.BoolFlag{ - Name: "verbose", + Name: "verbose", Aliases: []string{"v"}, }, }, From bb6a354b6269949529bc9791b62ec34c7562d74d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 10 Oct 2020 14:17:44 +0200 Subject: [PATCH 701/885] chain cli: decode params command Signed-off-by: Jakub Sztandera --- cli/chain.go | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/cli/chain.go b/cli/chain.go index 763752f23..4dfcf3f36 100644 --- a/cli/chain.go +++ b/cli/chain.go @@ -3,6 +3,7 @@ package cli import ( "bytes" "context" + "encoding/hex" "encoding/json" "fmt" "os" @@ -53,6 +54,7 @@ var chainCmd = &cli.Command{ slashConsensusFault, chainGasPriceCmd, chainInspectUsage, + chainDecodeCmd, }, } @@ -1233,3 +1235,68 @@ var chainGasPriceCmd = &cli.Command{ return nil }, } + +var chainDecodeCmd = &cli.Command{ + Name: "decode", + Usage: "decode various types", + Subcommands: []*cli.Command{ + chainDecodeParamsCmd, + }, +} + +var chainDecodeParamsCmd = &cli.Command{ + Name: "params", + Usage: "Decode message params", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "tipset", + }, + }, + ArgsUsage: "[toAddr method hexParams]", + Action: func(cctx *cli.Context) error { + api, closer, err := GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + ctx := ReqContext(cctx) + + if cctx.Args().Len() != 3 { + return ShowHelp(cctx, fmt.Errorf("incorrect number of arguments")) + } + + to, err := address.NewFromString(cctx.Args().First()) + if err != nil { + return xerrors.Errorf("parsing toAddr: %w", err) + } + + method, err := strconv.ParseInt(cctx.Args().Get(1), 10, 64) + if err != nil { + return xerrors.Errorf("parsing method id: %w", err) + } + + params, err := hex.DecodeString(cctx.Args().Get(2)) + if err != nil { + return xerrors.Errorf("parsing hex params: %w", err) + } + + ts, err := LoadTipSet(ctx, cctx, api) + if err != nil { + return err + } + + act, err := api.StateGetActor(ctx, to, ts.Key()) + if err != nil { + return xerrors.Errorf("getting actor: %w", err) + } + + pstr, err := jsonParams(act.Code, abi.MethodNum(method), params) + if err != nil { + return err + } + + fmt.Println(pstr) + + return nil + }, +} From 753d7b71a9e3ac330ec29fd9c41a9eba824bd8ec Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Sat, 10 Oct 2020 22:59:40 +0200 Subject: [PATCH 702/885] Fix hardened path printing, show 20 addresses after empty Signed-off-by: Jakub Sztandera --- cmd/lotus-shed/ledger.go | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/cmd/lotus-shed/ledger.go b/cmd/lotus-shed/ledger.go index 8111cc6bf..84d582dea 100644 --- a/cmd/lotus-shed/ledger.go +++ b/cmd/lotus-shed/ledger.go @@ -29,6 +29,8 @@ var ledgerCmd = &cli.Command{ }, } +const hdHard = 0x80000000 + var ledgerListAddressesCmd = &cli.Command{ Name: "list", Flags: []cli.Flag{ @@ -57,12 +59,13 @@ var ledgerListAddressesCmd = &cli.Command{ return err } - for i := 0; i < 20; i++ { + end := 20 + for i := 0; i < end; i++ { if err := ctx.Err(); err != nil { return err } - p := []uint32{0x80000000 + 44, 0x80000000 + 461, 0x80000000, 0, uint32(i)} + p := []uint32{hdHard | 44, hdHard | 461, hdHard | 0, 0, uint32(i)} pubk, err := fl.GetPublicKeySECP256K1(p) if err != nil { return err @@ -78,6 +81,9 @@ var ledgerListAddressesCmd = &cli.Command{ if err != nil { return xerrors.Errorf("getting balance: %w", err) } + if !b.IsZero() { + end = i + 21 // BIP32 spec, stop after 20 empty addresses + } fmt.Printf("%s %s %s\n", addr, printHDPath(p), types.FIL(b)) } else { @@ -108,12 +114,12 @@ func parseHDPath(s string) ([]uint32, error) { if err != nil { return nil, err } - if v >= 0x80000000 { + if v >= hdHard { return nil, fmt.Errorf("path element %s too large", p) } if hard { - v += 0x80000000 + v += hdHard } out = append(out, uint32(v)) } @@ -121,20 +127,20 @@ func parseHDPath(s string) ([]uint32, error) { } func printHDPath(pth []uint32) string { - s := "m/" + s := "m" for _, p := range pth { - var hard bool - if p >= 0x80000000 { - p -= 0x80000000 - } + s += "/" + + hard := p&hdHard != 0 + p &^= hdHard // remove hdHard bit + s += fmt.Sprint(p) if hard { s += "'" } - s += "/" } - return strings.TrimRight(s, "/") + return s } var ledgerKeyInfoCmd = &cli.Command{ @@ -164,6 +170,7 @@ var ledgerKeyInfoCmd = &cli.Command{ if err != nil { return err } + if cctx.Bool("verbose") { fmt.Println(addr) fmt.Println(pubk) From 7ee6bafd2312f8894eee66bd2c57fe3015118ede Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Sat, 10 Oct 2020 23:13:22 +0200 Subject: [PATCH 703/885] Fix lint warn Signed-off-by: Jakub Sztandera --- cmd/lotus-shed/ledger.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/lotus-shed/ledger.go b/cmd/lotus-shed/ledger.go index 84d582dea..f8267ce12 100644 --- a/cmd/lotus-shed/ledger.go +++ b/cmd/lotus-shed/ledger.go @@ -65,7 +65,7 @@ var ledgerListAddressesCmd = &cli.Command{ return err } - p := []uint32{hdHard | 44, hdHard | 461, hdHard | 0, 0, uint32(i)} + p := []uint32{hdHard | 44, hdHard | 461, hdHard, 0, uint32(i)} pubk, err := fl.GetPublicKeySECP256K1(p) if err != nil { return err From 6a232e7214ae6e21c2b6c7f5ac5735b694cc47a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sun, 11 Oct 2020 00:08:12 +0200 Subject: [PATCH 704/885] Basic multi-wallet support Signed-off-by: Jakub Sztandera --- chain/wallet/ledger/ledger.go | 15 ++- chain/wallet/multi.go | 160 ++++++++++++++++++++++++++++ chain/wallet/remotewallet/remote.go | 8 ++ chain/wallet/wallet.go | 21 ++++ node/builder.go | 13 ++- node/config/def.go | 2 + 6 files changed, 214 insertions(+), 5 deletions(-) create mode 100644 chain/wallet/multi.go diff --git a/chain/wallet/ledger/ledger.go b/chain/wallet/ledger/ledger.go index 94a828603..231b3faae 100644 --- a/chain/wallet/ledger/ledger.go +++ b/chain/wallet/ledger/ledger.go @@ -17,13 +17,14 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/node/modules/dtypes" ) type LedgerWallet struct { ds datastore.Datastore } -func NewWallet(ds datastore.Datastore) *LedgerWallet { +func NewWallet(ds dtypes.MetadataDS) *LedgerWallet { return &LedgerWallet{ds} } @@ -127,7 +128,7 @@ func (lw LedgerWallet) WalletImport(ctx context.Context, kinfo *types.KeyInfo) ( } func (lw LedgerWallet) WalletList(ctx context.Context) ([]address.Address, error) { - res, err := lw.ds.Query(query.Query{Prefix: "ledgerkey/"}) + res, err := lw.ds.Query(query.Query{Prefix: "/ledgerkey/"}) if err != nil { return nil, err } @@ -153,6 +154,14 @@ func (lw LedgerWallet) WalletNew(ctx context.Context, t crypto.SigType) (address return address.Undef, fmt.Errorf("cannot create new address on ledger") } +func (lw *LedgerWallet) Get() api.WalletAPI { + if lw == nil { + return nil + } + + return lw +} + func keyForAddr(addr address.Address) datastore.Key { - return datastore.NewKey("ledgerkey/" + addr.String()) + return datastore.NewKey("/ledgerkey/" + addr.String()) } diff --git a/chain/wallet/multi.go b/chain/wallet/multi.go new file mode 100644 index 000000000..bff8d7003 --- /dev/null +++ b/chain/wallet/multi.go @@ -0,0 +1,160 @@ +package wallet + +import ( + "context" + + "go.uber.org/fx" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/crypto" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/types" + ledgerwallet "github.com/filecoin-project/lotus/chain/wallet/ledger" + "github.com/filecoin-project/lotus/chain/wallet/remotewallet" +) + +type MultiWallet struct { + fx.In // "constructed" with fx.In instead of normal constructor + + Local *LocalWallet `optional:"true"` + Remote *remotewallet.RemoteWallet `optional:"true"` + Ledger *ledgerwallet.LedgerWallet `optional:"true"` +} + +type getif interface { + api.WalletAPI + + // workaround for the fact that iface(*struct(nil)) != nil + Get() api.WalletAPI +} + +func firstNonNil(wallets ...getif) api.WalletAPI { + for _, w := range wallets { + if w.Get() != nil { + return w + } + } + + return nil +} + +func nonNil(wallets ...getif) []api.WalletAPI { + var out []api.WalletAPI + for _, w := range wallets { + if w.Get() == nil { + continue + } + + out = append(out, w) + } + + return out +} + +func (m MultiWallet) find(ctx context.Context, address address.Address, wallets ...getif) (api.WalletAPI, error) { + ws := nonNil(wallets...) + + for _, w := range ws { + have, err := w.WalletHas(ctx, address) + if err != nil { + return nil, err + } + + if have { + return w, nil + } + } + + return nil, nil +} + +func (m MultiWallet) WalletNew(ctx context.Context, sigType crypto.SigType) (address.Address, error) { + w := firstNonNil(m.Remote, m.Local) + if w == nil { + return address.Undef, xerrors.Errorf("no wallet backends configured") + } + + return w.WalletNew(ctx, sigType) +} + +func (m MultiWallet) WalletHas(ctx context.Context, address address.Address) (bool, error) { + w, err := m.find(ctx, address, m.Remote, m.Ledger, m.Local) + return w != nil, err +} + +func (m MultiWallet) WalletList(ctx context.Context) ([]address.Address, error) { + var out []address.Address + seen := map[address.Address]struct{}{} + + ws := nonNil(m.Remote, m.Ledger, m.Local) + for _, w := range ws { + l, err := w.WalletList(ctx) + if err != nil { + return nil, err + } + + for _, a := range l { + if _, ok := seen[a]; ok { + continue + } + seen[a] = struct{}{} + + out = append(out, a) + } + } + + return out, nil +} + +func (m MultiWallet) WalletSign(ctx context.Context, signer address.Address, toSign []byte, meta api.MsgMeta) (*crypto.Signature, error) { + w, err := m.find(ctx, signer, m.Remote, m.Ledger, m.Local) + if err != nil { + return nil, err + } + if w == nil { + return nil, xerrors.Errorf("key not found") + } + + return w.WalletSign(ctx, signer, toSign, meta) +} + +func (m MultiWallet) WalletExport(ctx context.Context, address address.Address) (*types.KeyInfo, error) { + w, err := m.find(ctx, address, m.Remote, m.Local) + if err != nil { + return nil, err + } + if w == nil { + return nil, xerrors.Errorf("key not found") + } + + return w.WalletExport(ctx, address) +} + +func (m MultiWallet) WalletImport(ctx context.Context, info *types.KeyInfo) (address.Address, error) { + w := firstNonNil(m.Remote, m.Ledger, m.Local) + if w == nil { + return address.Undef, xerrors.Errorf("no wallet backends configured") + } + + return w.WalletImport(ctx, info) +} + +func (m MultiWallet) WalletDelete(ctx context.Context, address address.Address) error { + for { + w, err := m.find(ctx, address, m.Remote, m.Ledger, m.Local) + if err != nil { + return err + } + if w == nil { + return nil + } + + if err := w.WalletDelete(ctx, address); err != nil { + return err + } + } +} + +var _ api.WalletAPI = MultiWallet{} diff --git a/chain/wallet/remotewallet/remote.go b/chain/wallet/remotewallet/remote.go index c7192f496..aa4427132 100644 --- a/chain/wallet/remotewallet/remote.go +++ b/chain/wallet/remotewallet/remote.go @@ -40,3 +40,11 @@ func SetupRemoteWallet(info string) func(mctx helpers.MetricsCtx, lc fx.Lifecycl return &RemoteWallet{wapi}, nil } } + +func (w *RemoteWallet) Get() api.WalletAPI { + if w == nil { + return nil + } + + return w +} diff --git a/chain/wallet/wallet.go b/chain/wallet/wallet.go index 57e63625b..66529d889 100644 --- a/chain/wallet/wallet.go +++ b/chain/wallet/wallet.go @@ -300,6 +300,14 @@ func (w *LocalWallet) WalletDelete(ctx context.Context, addr address.Address) er return nil } +func (w *LocalWallet) Get() api.WalletAPI { + if w == nil { + return nil + } + + return w +} + var _ api.WalletAPI = &LocalWallet{} func swapMainnetForTestnetPrefix(addr string) (string, error) { @@ -312,3 +320,16 @@ func swapMainnetForTestnetPrefix(addr string) (string, error) { aChars[0] = prefixRunes[0] return string(aChars), nil } + +type nilDefault struct{} + +func (n nilDefault) GetDefault() (address.Address, error) { + return address.Undef, nil +} + +func (n nilDefault) SetDefault(a address.Address) error { + return xerrors.Errorf("not supported; local wallet disabled") +} + +var NilDefault nilDefault +var _ Default = NilDefault diff --git a/node/builder.go b/node/builder.go index b91172386..f96505eec 100644 --- a/node/builder.go +++ b/node/builder.go @@ -44,6 +44,7 @@ import ( "github.com/filecoin-project/lotus/chain/metrics" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/types" + ledgerwallet "github.com/filecoin-project/lotus/chain/wallet/ledger" "github.com/filecoin-project/lotus/chain/wallet/remotewallet" sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" @@ -251,8 +252,8 @@ func Online() Option { Override(new(*stmgr.StateManager), stmgr.NewStateManagerWithUpgradeSchedule), Override(new(stmgr.StateManagerAPI), From(new(*stmgr.StateManager))), Override(new(*wallet.LocalWallet), wallet.NewWallet), - Override(new(api.WalletAPI), From(new(*wallet.LocalWallet))), Override(new(wallet.Default), From(new(*wallet.LocalWallet))), + Override(new(api.WalletAPI), From(new(wallet.MultiWallet))), Override(new(*messagesigner.MessageSigner), messagesigner.NewMessageSigner), Override(new(dtypes.ChainGCLocker), blockstore.NewGCLocker), @@ -459,8 +460,16 @@ func ConfigFullNode(c interface{}) Option { If(cfg.Metrics.HeadNotifs, Override(HeadMetricsKey, metrics.SendHeadNotifs(cfg.Metrics.Nickname)), ), + If(cfg.Wallet.RemoteBackend != "", - Override(new(api.WalletAPI), remotewallet.SetupRemoteWallet(cfg.Wallet.RemoteBackend)), + Override(new(*remotewallet.RemoteWallet), remotewallet.SetupRemoteWallet(cfg.Wallet.RemoteBackend)), + ), + If(cfg.Wallet.EnableLedger, + Override(new(*ledgerwallet.LedgerWallet), ledgerwallet.NewWallet), + ), + If(cfg.Wallet.DisableLocal, + Unset(new(*wallet.LocalWallet)), + Override(new(wallet.Default), wallet.NilDefault), ), ) } diff --git a/node/config/def.go b/node/config/def.go index 63340cfd5..7eb7e19f0 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -110,6 +110,8 @@ type Client struct { type Wallet struct { RemoteBackend string + EnableLedger bool + DisableLocal bool } func defCommon() Common { From 2b21fdef3348dcc5799a9ecc21c04cfdd40e9cc1 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Sun, 11 Oct 2020 20:12:01 +0200 Subject: [PATCH 705/885] Refactor from crypto.SigType to types.KeyType Signed-off-by: Jakub Sztandera --- api/api_full.go | 4 +- api/api_wallet.go | 2 +- api/apistruct/struct.go | 8 ++-- api/docgen/docgen.go | 1 + api/test/paych.go | 3 +- chain/gen/gen.go | 4 +- chain/messagepool/messagepool_test.go | 25 ++++++------ chain/messagepool/repub_test.go | 6 +-- chain/messagepool/selection_test.go | 43 +++++++++++---------- chain/messagesigner/messagesigner_test.go | 9 ++--- chain/types/keystore.go | 46 ++++++++++++++++++++++- chain/vectors/gen/main.go | 5 +-- chain/wallet/key.go | 30 ++++++++------- chain/wallet/ledger/ledger.go | 2 +- chain/wallet/multi.go | 11 ++++-- chain/wallet/wallet.go | 4 +- cli/paych_test.go | 3 +- cli/wallet.go | 7 ++-- cmd/chain-noise/main.go | 4 +- cmd/lotus-gateway/endtoend_test.go | 5 +-- cmd/lotus-keygen/main.go | 8 ++-- cmd/lotus-seed/seed/seed.go | 3 +- cmd/lotus-shed/keyinfo.go | 20 +++++----- cmd/lotus-shed/ledger.go | 3 +- cmd/lotus-storage-miner/init.go | 3 +- cmd/lotus-wallet/logged.go | 9 +---- cmd/lotus-wallet/main.go | 19 ++++++---- documentation/en/api-methods.md | 8 ++-- node/modules/lp2p/libp2p.go | 4 +- scripts/dev/api.bash | 4 +- storage/mockstorage/preseal.go | 3 +- 31 files changed, 173 insertions(+), 133 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index a2fe94ee9..f42c8d03f 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -229,7 +229,9 @@ type FullNode interface { // MethodGroup: Wallet // WalletNew creates a new address in the wallet with the given sigType. - WalletNew(context.Context, crypto.SigType) (address.Address, error) + // Available key types: bls, secp256k1, secp256k1-ledger + // Support for numerical types: 1 - secp256k1, 2 - BLS is deprecated + WalletNew(context.Context, types.KeyType) (address.Address, error) // WalletHas indicates whether the given address is in the wallet. WalletHas(context.Context, address.Address) (bool, error) // WalletList lists all the addresses in the wallet. diff --git a/api/api_wallet.go b/api/api_wallet.go index 1213ffc1d..88ad8f43a 100644 --- a/api/api_wallet.go +++ b/api/api_wallet.go @@ -35,7 +35,7 @@ type MsgMeta struct { } type WalletAPI interface { - WalletNew(context.Context, crypto.SigType) (address.Address, error) + WalletNew(context.Context, types.KeyType) (address.Address, error) WalletHas(context.Context, address.Address) (bool, error) WalletList(context.Context) ([]address.Address, error) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 22d50e726..cef335939 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -133,7 +133,7 @@ type FullNodeStruct struct { MinerGetBaseInfo func(context.Context, address.Address, abi.ChainEpoch, types.TipSetKey) (*api.MiningBaseInfo, error) `perm:"read"` MinerCreateBlock func(context.Context, *api.BlockTemplate) (*types.BlockMsg, error) `perm:"write"` - WalletNew func(context.Context, crypto.SigType) (address.Address, error) `perm:"write"` + WalletNew func(context.Context, types.KeyType) (address.Address, error) `perm:"write"` WalletHas func(context.Context, address.Address) (bool, error) `perm:"write"` WalletList func(context.Context) ([]address.Address, error) `perm:"write"` WalletBalance func(context.Context, address.Address) (types.BigInt, error) `perm:"read"` @@ -384,7 +384,7 @@ type GatewayStruct struct { type WalletStruct struct { Internal struct { - WalletNew func(context.Context, crypto.SigType) (address.Address, error) `perm:"write"` + WalletNew func(context.Context, types.KeyType) (address.Address, error) `perm:"write"` WalletHas func(context.Context, address.Address) (bool, error) `perm:"write"` WalletList func(context.Context) ([]address.Address, error) `perm:"write"` WalletSign func(context.Context, address.Address, []byte, api.MsgMeta) (*crypto.Signature, error) `perm:"sign"` @@ -631,7 +631,7 @@ func (c *FullNodeStruct) ChainGetTipSetByHeight(ctx context.Context, h abi.Chain return c.Internal.ChainGetTipSetByHeight(ctx, h, tsk) } -func (c *FullNodeStruct) WalletNew(ctx context.Context, typ crypto.SigType) (address.Address, error) { +func (c *FullNodeStruct) WalletNew(ctx context.Context, typ types.KeyType) (address.Address, error) { return c.Internal.WalletNew(ctx, typ) } @@ -1465,7 +1465,7 @@ func (g GatewayStruct) StateWaitMsg(ctx context.Context, msg cid.Cid, confidence return g.Internal.StateWaitMsg(ctx, msg, confidence) } -func (c *WalletStruct) WalletNew(ctx context.Context, typ crypto.SigType) (address.Address, error) { +func (c *WalletStruct) WalletNew(ctx context.Context, typ types.KeyType) (address.Address, error) { return c.Internal.WalletNew(ctx, typ) } diff --git a/api/docgen/docgen.go b/api/docgen/docgen.go index ced536cc3..dc6004121 100644 --- a/api/docgen/docgen.go +++ b/api/docgen/docgen.go @@ -87,6 +87,7 @@ func init() { addExample(abi.RegisteredPoStProof_StackedDrgWindow32GiBV1) addExample(abi.ChainEpoch(10101)) addExample(crypto.SigTypeBLS) + addExample(types.KTBLS) addExample(int64(9)) addExample(12.3) addExample(123) diff --git a/api/test/paych.go b/api/test/paych.go index a8ccebdde..43401554d 100644 --- a/api/test/paych.go +++ b/api/test/paych.go @@ -26,7 +26,6 @@ import ( "github.com/filecoin-project/lotus/chain/events" "github.com/filecoin-project/lotus/chain/events/state" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/chain/wallet" ) func TestPaymentChannels(t *testing.T, b APIBuilder, blocktime time.Duration) { @@ -58,7 +57,7 @@ func TestPaymentChannels(t *testing.T, b APIBuilder, blocktime time.Duration) { bm.MineBlocks() // send some funds to register the receiver - receiverAddr, err := paymentReceiver.WalletNew(ctx, wallet.ActSigType("secp256k1")) + receiverAddr, err := paymentReceiver.WalletNew(ctx, types.KTSecp256k1) if err != nil { t.Fatal(err) } diff --git a/chain/gen/gen.go b/chain/gen/gen.go index 9133f14b4..288c84219 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -155,14 +155,14 @@ func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) { return nil, xerrors.Errorf("creating memrepo wallet failed: %w", err) } - banker, err := w.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + banker, err := w.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { return nil, xerrors.Errorf("failed to generate banker key: %w", err) } receievers := make([]address.Address, msgsPerBlock) for r := range receievers { - receievers[r], err = w.WalletNew(context.Background(), crypto.SigTypeBLS) + receievers[r], err = w.WalletNew(context.Background(), types.KTBLS) if err != nil { return nil, xerrors.Errorf("failed to generate receiver key: %w", err) } diff --git a/chain/messagepool/messagepool_test.go b/chain/messagepool/messagepool_test.go index 063e0bbab..8f6613cad 100644 --- a/chain/messagepool/messagepool_test.go +++ b/chain/messagepool/messagepool_test.go @@ -8,7 +8,6 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/lotus/chain/messagepool/gasguess" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/types/mock" @@ -232,7 +231,7 @@ func TestMessagePool(t *testing.T) { a := tma.nextBlock() - sender, err := w.WalletNew(context.Background(), crypto.SigTypeBLS) + sender, err := w.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -273,7 +272,7 @@ func TestMessagePoolMessagesInEachBlock(t *testing.T) { a := tma.nextBlock() - sender, err := w.WalletNew(context.Background(), crypto.SigTypeBLS) + sender, err := w.WalletNew(context.Background(), types.KTBLS) if err != nil { t.Fatal(err) } @@ -323,7 +322,7 @@ func TestRevertMessages(t *testing.T) { a := tma.nextBlock() b := tma.nextBlock() - sender, err := w.WalletNew(context.Background(), crypto.SigTypeBLS) + sender, err := w.WalletNew(context.Background(), types.KTBLS) if err != nil { t.Fatal(err) } @@ -386,7 +385,7 @@ func TestPruningSimple(t *testing.T) { a := tma.nextBlock() tma.applyBlock(t, a) - sender, err := w.WalletNew(context.Background(), crypto.SigTypeBLS) + sender, err := w.WalletNew(context.Background(), types.KTBLS) if err != nil { t.Fatal(err) } @@ -433,7 +432,7 @@ func TestLoadLocal(t *testing.T) { t.Fatal(err) } - a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -443,7 +442,7 @@ func TestLoadLocal(t *testing.T) { t.Fatal(err) } - a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -505,7 +504,7 @@ func TestClearAll(t *testing.T) { t.Fatal(err) } - a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -515,7 +514,7 @@ func TestClearAll(t *testing.T) { t.Fatal(err) } - a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -559,7 +558,7 @@ func TestClearNonLocal(t *testing.T) { t.Fatal(err) } - a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -569,7 +568,7 @@ func TestClearNonLocal(t *testing.T) { t.Fatal(err) } - a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -620,7 +619,7 @@ func TestUpdates(t *testing.T) { t.Fatal(err) } - a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -630,7 +629,7 @@ func TestUpdates(t *testing.T) { t.Fatal(err) } - a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } diff --git a/chain/messagepool/repub_test.go b/chain/messagepool/repub_test.go index 3e1252eec..9cadf24c7 100644 --- a/chain/messagepool/repub_test.go +++ b/chain/messagepool/repub_test.go @@ -5,8 +5,8 @@ import ( "testing" "time" - "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/lotus/chain/messagepool/gasguess" + "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/wallet" "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/ipfs/go-datastore" @@ -33,7 +33,7 @@ func TestRepubMessages(t *testing.T) { t.Fatal(err) } - a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -43,7 +43,7 @@ func TestRepubMessages(t *testing.T) { t.Fatal(err) } - a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } diff --git a/chain/messagepool/selection_test.go b/chain/messagepool/selection_test.go index 9e4fe39e5..37c2f2dc0 100644 --- a/chain/messagepool/selection_test.go +++ b/chain/messagepool/selection_test.go @@ -18,7 +18,6 @@ import ( logging "github.com/ipfs/go-log" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/messagepool/gasguess" "github.com/filecoin-project/lotus/chain/types" @@ -77,7 +76,7 @@ func TestMessageChains(t *testing.T) { t.Fatal(err) } - a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -87,7 +86,7 @@ func TestMessageChains(t *testing.T) { t.Fatal(err) } - a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -315,7 +314,7 @@ func TestMessageChainSkipping(t *testing.T) { t.Fatal(err) } - a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -325,7 +324,7 @@ func TestMessageChainSkipping(t *testing.T) { t.Fatal(err) } - a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -391,7 +390,7 @@ func TestBasicMessageSelection(t *testing.T) { t.Fatal(err) } - a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -401,7 +400,7 @@ func TestBasicMessageSelection(t *testing.T) { t.Fatal(err) } - a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -535,7 +534,7 @@ func TestMessageSelectionTrimming(t *testing.T) { t.Fatal(err) } - a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -545,7 +544,7 @@ func TestMessageSelectionTrimming(t *testing.T) { t.Fatal(err) } - a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -598,7 +597,7 @@ func TestPriorityMessageSelection(t *testing.T) { t.Fatal(err) } - a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -608,7 +607,7 @@ func TestPriorityMessageSelection(t *testing.T) { t.Fatal(err) } - a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -677,7 +676,7 @@ func TestPriorityMessageSelection2(t *testing.T) { t.Fatal(err) } - a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -687,7 +686,7 @@ func TestPriorityMessageSelection2(t *testing.T) { t.Fatal(err) } - a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -746,7 +745,7 @@ func TestPriorityMessageSelection3(t *testing.T) { t.Fatal(err) } - a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -756,7 +755,7 @@ func TestPriorityMessageSelection3(t *testing.T) { t.Fatal(err) } - a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -843,7 +842,7 @@ func TestOptimalMessageSelection1(t *testing.T) { t.Fatal(err) } - a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -853,7 +852,7 @@ func TestOptimalMessageSelection1(t *testing.T) { t.Fatal(err) } - a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -910,7 +909,7 @@ func TestOptimalMessageSelection2(t *testing.T) { t.Fatal(err) } - a1, err := w1.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a1, err := w1.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -920,7 +919,7 @@ func TestOptimalMessageSelection2(t *testing.T) { t.Fatal(err) } - a2, err := w2.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a2, err := w2.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -994,7 +993,7 @@ func TestOptimalMessageSelection3(t *testing.T) { t.Fatal(err) } - a, err := w.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a, err := w.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -1074,7 +1073,7 @@ func testCompetitiveMessageSelection(t *testing.T, rng *rand.Rand, getPremium fu t.Fatal(err) } - a, err := w.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a, err := w.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } @@ -1344,7 +1343,7 @@ readLoop: t.Fatal(err) } - a, err := w.WalletNew(context.Background(), crypto.SigTypeSecp256k1) + a, err := w.WalletNew(context.Background(), types.KTSecp256k1) if err != nil { t.Fatal(err) } diff --git a/chain/messagesigner/messagesigner_test.go b/chain/messagesigner/messagesigner_test.go index 0dafce9a1..5eebd36da 100644 --- a/chain/messagesigner/messagesigner_test.go +++ b/chain/messagesigner/messagesigner_test.go @@ -9,7 +9,6 @@ import ( "github.com/filecoin-project/lotus/chain/wallet" - "github.com/filecoin-project/go-state-types/crypto" "github.com/stretchr/testify/require" ds_sync "github.com/ipfs/go-datastore/sync" @@ -47,13 +46,13 @@ func TestMessageSignerSignMessage(t *testing.T) { ctx := context.Background() w, _ := wallet.NewWallet(wallet.NewMemKeyStore()) - from1, err := w.WalletNew(ctx, crypto.SigTypeSecp256k1) + from1, err := w.WalletNew(ctx, types.KTSecp256k1) require.NoError(t, err) - from2, err := w.WalletNew(ctx, crypto.SigTypeSecp256k1) + from2, err := w.WalletNew(ctx, types.KTSecp256k1) require.NoError(t, err) - to1, err := w.WalletNew(ctx, crypto.SigTypeSecp256k1) + to1, err := w.WalletNew(ctx, types.KTSecp256k1) require.NoError(t, err) - to2, err := w.WalletNew(ctx, crypto.SigTypeSecp256k1) + to2, err := w.WalletNew(ctx, types.KTSecp256k1) require.NoError(t, err) type msgSpec struct { diff --git a/chain/types/keystore.go b/chain/types/keystore.go index 76eb5f296..107c1fbe3 100644 --- a/chain/types/keystore.go +++ b/chain/types/keystore.go @@ -1,7 +1,10 @@ package types import ( + "encoding/json" "fmt" + + "github.com/filecoin-project/go-state-types/crypto" ) var ( @@ -9,9 +12,50 @@ var ( ErrKeyExists = fmt.Errorf("key already exists") ) +// KeyType defines a type of a key +type KeyType string + +func (kt *KeyType) UnmarshalJSON(bb []byte) error { + { + // first option, try unmarshaling as string + var s string + err := json.Unmarshal(bb, &s) + if err == nil { + *kt = KeyType(s) + return nil + } + } + + { + var b byte + err := json.Unmarshal(bb, &b) + if err != nil { + return fmt.Errorf("could not unmarshal KeyType either as string nor integer: %w", err) + } + bst := crypto.SigType(b) + + switch bst { + case crypto.SigTypeBLS: + *kt = KTBLS + case crypto.SigTypeSecp256k1: + *kt = KTSecp256k1 + default: + return fmt.Errorf("unknown sigtype: %d", bst) + } + log.Warnf("deprecation: integer style 'KeyType' is deprecated, switch to string style") + return nil + } +} + +const ( + KTBLS KeyType = "bls" + KTSecp256k1 KeyType = "secp256k1" + KTSecp256k1Ledger KeyType = "secp256k1-ledger" +) + // KeyInfo is used for storing keys in KeyStore type KeyInfo struct { - Type string + Type KeyType PrivateKey []byte } diff --git a/chain/vectors/gen/main.go b/chain/vectors/gen/main.go index 096548e04..757227d0d 100644 --- a/chain/vectors/gen/main.go +++ b/chain/vectors/gen/main.go @@ -11,7 +11,6 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/types" @@ -62,7 +61,7 @@ func MakeMessageSigningVectors() []vectors.MessageSigningVector { panic(err) } - blsk, err := w.WalletNew(context.Background(), crypto.SigTypeBLS) + blsk, err := w.WalletNew(context.Background(), types.KTBLS) if err != nil { panic(err) } @@ -86,7 +85,7 @@ func MakeMessageSigningVectors() []vectors.MessageSigningVector { Signature: &bmsg.Signature, } - secpk, err := w.WalletNew(context.Background(), crypto.SigTypeBLS) + secpk, err := w.WalletNew(context.Background(), types.KTBLS) if err != nil { panic(err) } diff --git a/chain/wallet/key.go b/chain/wallet/key.go index 4b746a17a..ac9fc436f 100644 --- a/chain/wallet/key.go +++ b/chain/wallet/key.go @@ -10,13 +10,17 @@ import ( "github.com/filecoin-project/lotus/lib/sigs" ) -func GenerateKey(typ crypto.SigType) (*Key, error) { - pk, err := sigs.Generate(typ) +func GenerateKey(typ types.KeyType) (*Key, error) { + ctyp := ActSigType(typ) + if ctyp == crypto.SigTypeUnknown { + return nil, xerrors.Errorf("unknown key type: %s", typ) + } + pk, err := sigs.Generate(ctyp) if err != nil { return nil, err } ki := types.KeyInfo{ - Type: kstoreSigType(typ), + Type: typ, PrivateKey: pk, } return NewKey(ki) @@ -41,41 +45,41 @@ func NewKey(keyinfo types.KeyInfo) (*Key, error) { } switch k.Type { - case KTSecp256k1: + case types.KTSecp256k1: k.Address, err = address.NewSecp256k1Address(k.PublicKey) if err != nil { return nil, xerrors.Errorf("converting Secp256k1 to address: %w", err) } - case KTBLS: + case types.KTBLS: k.Address, err = address.NewBLSAddress(k.PublicKey) if err != nil { return nil, xerrors.Errorf("converting BLS to address: %w", err) } default: - return nil, xerrors.Errorf("unknown key type") + return nil, xerrors.Errorf("unknown key type: %s", k.Type) } return k, nil } -func kstoreSigType(typ crypto.SigType) string { +func kstoreSigType(typ crypto.SigType) types.KeyType { switch typ { case crypto.SigTypeBLS: - return KTBLS + return types.KTBLS case crypto.SigTypeSecp256k1: - return KTSecp256k1 + return types.KTSecp256k1 default: return "" } } -func ActSigType(typ string) crypto.SigType { +func ActSigType(typ types.KeyType) crypto.SigType { switch typ { - case KTBLS: + case types.KTBLS: return crypto.SigTypeBLS - case KTSecp256k1: + case types.KTSecp256k1: return crypto.SigTypeSecp256k1 default: - return 0 + return crypto.SigTypeUnknown } } diff --git a/chain/wallet/ledger/ledger.go b/chain/wallet/ledger/ledger.go index 231b3faae..b6b5683cd 100644 --- a/chain/wallet/ledger/ledger.go +++ b/chain/wallet/ledger/ledger.go @@ -150,7 +150,7 @@ func (lw LedgerWallet) WalletList(ctx context.Context) ([]address.Address, error return out, nil } -func (lw LedgerWallet) WalletNew(ctx context.Context, t crypto.SigType) (address.Address, error) { +func (lw LedgerWallet) WalletNew(ctx context.Context, t types.KeyType) (address.Address, error) { return address.Undef, fmt.Errorf("cannot create new address on ledger") } diff --git a/chain/wallet/multi.go b/chain/wallet/multi.go index bff8d7003..6eac03367 100644 --- a/chain/wallet/multi.go +++ b/chain/wallet/multi.go @@ -18,7 +18,7 @@ import ( type MultiWallet struct { fx.In // "constructed" with fx.In instead of normal constructor - Local *LocalWallet `optional:"true"` + Local *LocalWallet `optional:"true"` Remote *remotewallet.RemoteWallet `optional:"true"` Ledger *ledgerwallet.LedgerWallet `optional:"true"` } @@ -70,7 +70,7 @@ func (m MultiWallet) find(ctx context.Context, address address.Address, wallets return nil, nil } -func (m MultiWallet) WalletNew(ctx context.Context, sigType crypto.SigType) (address.Address, error) { +func (m MultiWallet) WalletNew(ctx context.Context, sigType types.KeyType) (address.Address, error) { w := firstNonNil(m.Remote, m.Local) if w == nil { return address.Undef, xerrors.Errorf("no wallet backends configured") @@ -133,7 +133,12 @@ func (m MultiWallet) WalletExport(ctx context.Context, address address.Address) } func (m MultiWallet) WalletImport(ctx context.Context, info *types.KeyInfo) (address.Address, error) { - w := firstNonNil(m.Remote, m.Ledger, m.Local) + var local getif = m.Local + if info.Type == types.KTSecp256k1Ledger { + local = m.Ledger + } + + w := firstNonNil(m.Remote, local) if w == nil { return address.Undef, xerrors.Errorf("no wallet backends configured") } diff --git a/chain/wallet/wallet.go b/chain/wallet/wallet.go index 66529d889..925547ea8 100644 --- a/chain/wallet/wallet.go +++ b/chain/wallet/wallet.go @@ -26,8 +26,6 @@ const ( KNamePrefix = "wallet-" KTrashPrefix = "trash-" KDefault = "default" - KTBLS = "bls" - KTSecp256k1 = "secp256k1" ) type LocalWallet struct { @@ -236,7 +234,7 @@ func (w *LocalWallet) SetDefault(a address.Address) error { return nil } -func (w *LocalWallet) WalletNew(ctx context.Context, typ crypto.SigType) (address.Address, error) { +func (w *LocalWallet) WalletNew(ctx context.Context, typ types.KeyType) (address.Address, error) { w.lk.Lock() defer w.lk.Unlock() diff --git a/cli/paych_test.go b/cli/paych_test.go index 862ca2e74..b94ba27e7 100644 --- a/cli/paych_test.go +++ b/cli/paych_test.go @@ -29,7 +29,6 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/events" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/chain/wallet" builder "github.com/filecoin-project/lotus/node/test" ) @@ -415,7 +414,7 @@ func startTwoNodesOneMiner(ctx context.Context, t *testing.T, blocktime time.Dur bm.MineBlocks() // Send some funds to register the receiver - receiverAddr, err := paymentReceiver.WalletNew(ctx, wallet.ActSigType("secp256k1")) + receiverAddr, err := paymentReceiver.WalletNew(ctx, types.KTSecp256k1) if err != nil { t.Fatal(err) } diff --git a/cli/wallet.go b/cli/wallet.go index aa5b9bed3..3d6abc357 100644 --- a/cli/wallet.go +++ b/cli/wallet.go @@ -17,7 +17,6 @@ import ( "github.com/filecoin-project/go-state-types/crypto" types "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/chain/wallet" "github.com/filecoin-project/lotus/lib/tablewriter" ) @@ -55,7 +54,7 @@ var walletNew = &cli.Command{ t = "secp256k1" } - nk, err := api.WalletNew(ctx, wallet.ActSigType(t)) + nk, err := api.WalletNew(ctx, types.KeyType(t)) if err != nil { return err } @@ -329,9 +328,9 @@ var walletImport = &cli.Command{ ki.PrivateKey = gk.PrivateKey switch gk.SigType { case 1: - ki.Type = wallet.KTSecp256k1 + ki.Type = types.KTSecp256k1 case 2: - ki.Type = wallet.KTBLS + ki.Type = types.KTBLS default: return fmt.Errorf("unrecognized key type: %d", gk.SigType) } diff --git a/cmd/chain-noise/main.go b/cmd/chain-noise/main.go index 7b9824016..81586e1b2 100644 --- a/cmd/chain-noise/main.go +++ b/cmd/chain-noise/main.go @@ -7,8 +7,6 @@ import ( "os" "time" - "github.com/filecoin-project/go-state-types/crypto" - "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" @@ -61,7 +59,7 @@ var runCmd = &cli.Command{ func sendSmallFundsTxs(ctx context.Context, api api.FullNode, from address.Address, rate int) error { var sendSet []address.Address for i := 0; i < 20; i++ { - naddr, err := api.WalletNew(ctx, crypto.SigTypeSecp256k1) + naddr, err := api.WalletNew(ctx, types.KTSecp256k1) if err != nil { return err } diff --git a/cmd/lotus-gateway/endtoend_test.go b/cmd/lotus-gateway/endtoend_test.go index 206034968..eebf56a9f 100644 --- a/cmd/lotus-gateway/endtoend_test.go +++ b/cmd/lotus-gateway/endtoend_test.go @@ -22,7 +22,6 @@ import ( "github.com/filecoin-project/lotus/api/test" "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/chain/wallet" "github.com/filecoin-project/lotus/node" builder "github.com/filecoin-project/lotus/node/test" ) @@ -53,7 +52,7 @@ func TestEndToEnd(t *testing.T) { fmt.Println(balance) // Create a wallet on the lite node - liteWalletAddr, err := lite.WalletNew(ctx, wallet.ActSigType("secp256k1")) + liteWalletAddr, err := lite.WalletNew(ctx, types.KTSecp256k1) require.NoError(t, err) // Send some funds from the full node to the lite node @@ -77,7 +76,7 @@ func TestEndToEnd(t *testing.T) { // Create some wallets on the lite node to use for testing multisig var walletAddrs []address.Address for i := 0; i < 4; i++ { - addr, err := lite.WalletNew(ctx, wallet.ActSigType("secp256k1")) + addr, err := lite.WalletNew(ctx, types.KTSecp256k1) require.NoError(t, err) walletAddrs = append(walletAddrs, addr) diff --git a/cmd/lotus-keygen/main.go b/cmd/lotus-keygen/main.go index 4b971cf48..d296cb5da 100644 --- a/cmd/lotus-keygen/main.go +++ b/cmd/lotus-keygen/main.go @@ -5,7 +5,7 @@ import ( "fmt" "os" - "github.com/filecoin-project/go-state-types/crypto" + "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/wallet" _ "github.com/filecoin-project/lotus/lib/sigs/bls" _ "github.com/filecoin-project/lotus/lib/sigs/secp" @@ -30,12 +30,12 @@ func main() { return err } - var kt crypto.SigType + var kt types.KeyType switch cctx.String("type") { case "bls": - kt = crypto.SigTypeBLS + kt = types.KTBLS case "secp256k1": - kt = crypto.SigTypeSecp256k1 + kt = types.KTSecp256k1 default: return fmt.Errorf("unrecognized key type: %q", cctx.String("type")) } diff --git a/cmd/lotus-seed/seed/seed.go b/cmd/lotus-seed/seed/seed.go index 5e911991d..cd0c4e7ab 100644 --- a/cmd/lotus-seed/seed/seed.go +++ b/cmd/lotus-seed/seed/seed.go @@ -21,7 +21,6 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" - "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/lotus/extern/sector-storage/zerocomm" "github.com/filecoin-project/specs-actors/actors/builtin/market" @@ -93,7 +92,7 @@ func PreSeal(maddr address.Address, spt abi.RegisteredSealProof, offset abi.Sect return nil, nil, err } } else { - minerAddr, err = wallet.GenerateKey(crypto.SigTypeBLS) + minerAddr, err = wallet.GenerateKey(types.KTBLS) if err != nil { return nil, nil, err } diff --git a/cmd/lotus-shed/keyinfo.go b/cmd/lotus-shed/keyinfo.go index fdd1fcb49..4dcd10cbf 100644 --- a/cmd/lotus-shed/keyinfo.go +++ b/cmd/lotus-shed/keyinfo.go @@ -32,10 +32,10 @@ import ( _ "github.com/filecoin-project/lotus/lib/sigs/secp" ) -var validTypes = []string{wallet.KTBLS, wallet.KTSecp256k1, lp2p.KTLibp2pHost} +var validTypes = []types.KeyType{types.KTBLS, types.KTSecp256k1, lp2p.KTLibp2pHost} type keyInfoOutput struct { - Type string + Type types.KeyType Address string PublicKey string } @@ -86,7 +86,7 @@ var keyinfoVerifyCmd = &cli.Command{ return xerrors.Errorf("decoding key: '%s': %w", fileName, err) } - if string(name) != keyInfo.Type { + if types.KeyType(name) != keyInfo.Type { return fmt.Errorf("%s of type %s is incorrect", fileName, keyInfo.Type) } case modules.KTJwtHmacSecret: @@ -98,7 +98,7 @@ var keyinfoVerifyCmd = &cli.Command{ if string(name) != modules.JWTSecretName { return fmt.Errorf("%s of type %s is incorrect", fileName, keyInfo.Type) } - case wallet.KTSecp256k1, wallet.KTBLS: + case types.KTSecp256k1, types.KTBLS: keystore := wallet.NewMemKeyStore() w, err := wallet.NewWallet(keystore) if err != nil { @@ -214,7 +214,7 @@ var keyinfoImportCmd = &cli.Command{ fmt.Printf("%s\n", peerid.String()) break - case wallet.KTSecp256k1, wallet.KTBLS: + case types.KTSecp256k1, types.KTBLS: w, err := wallet.NewWallet(keystore) if err != nil { return err @@ -317,7 +317,7 @@ var keyinfoInfoCmd = &cli.Command{ kio.PublicKey = base64.StdEncoding.EncodeToString(pkBytes) break - case wallet.KTSecp256k1, wallet.KTBLS: + case types.KTSecp256k1, types.KTBLS: kio.Type = keyInfo.Type key, err := wallet.NewKey(keyInfo) @@ -366,7 +366,7 @@ var keyinfoNewCmd = &cli.Command{ return fmt.Errorf("please specify a type to generate") } - keyType := cctx.Args().First() + keyType := types.KeyType(cctx.Args().First()) flagOutput := cctx.String("output") if i := SliceIndex(len(validTypes), func(i int) bool { @@ -404,8 +404,8 @@ var keyinfoNewCmd = &cli.Command{ keyInfo = ki break - case wallet.KTSecp256k1, wallet.KTBLS: - key, err := wallet.GenerateKey(wallet.ActSigType(keyType)) + case types.KTSecp256k1, types.KTBLS: + key, err := wallet.GenerateKey(keyType) if err != nil { return err } @@ -418,7 +418,7 @@ var keyinfoNewCmd = &cli.Command{ filename := flagOutput filename = strings.ReplaceAll(filename, "", keyAddr) - filename = strings.ReplaceAll(filename, "", keyType) + filename = strings.ReplaceAll(filename, "", string(keyType)) file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600) if err != nil { diff --git a/cmd/lotus-shed/ledger.go b/cmd/lotus-shed/ledger.go index f8267ce12..f64916534 100644 --- a/cmd/lotus-shed/ledger.go +++ b/cmd/lotus-shed/ledger.go @@ -13,7 +13,6 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/chain/wallet" ledgerwallet "github.com/filecoin-project/lotus/chain/wallet/ledger" lcli "github.com/filecoin-project/lotus/cli" ) @@ -191,7 +190,7 @@ var ledgerKeyInfoCmd = &cli.Command{ } var ki types.KeyInfo - ki.Type = wallet.KTSecp256k1 + ki.Type = types.KTSecp256k1Ledger ki.PrivateKey = b out, err := json.Marshal(ki) diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index 0f830023f..ccea707af 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -27,7 +27,6 @@ import ( cborutil "github.com/filecoin-project/go-cbor-util" paramfetch "github.com/filecoin-project/go-paramfetch" "github.com/filecoin-project/go-state-types/abi" - crypto2 "github.com/filecoin-project/go-state-types/crypto" sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/extern/sector-storage/stores" @@ -624,7 +623,7 @@ func createStorageMiner(ctx context.Context, api lapi.FullNode, peerid peer.ID, if cctx.String("worker") != "" { worker, err = address.NewFromString(cctx.String("worker")) } else if cctx.Bool("create-worker-key") { // TODO: Do we need to force this if owner is Secpk? - worker, err = api.WalletNew(ctx, crypto2.SigTypeBLS) + worker, err = api.WalletNew(ctx, types.KTBLS) } // TODO: Transfer some initial funds to worker if err != nil { diff --git a/cmd/lotus-wallet/logged.go b/cmd/lotus-wallet/logged.go index 3bcb3f867..272a8d10b 100644 --- a/cmd/lotus-wallet/logged.go +++ b/cmd/lotus-wallet/logged.go @@ -19,13 +19,8 @@ type LoggedWallet struct { under api.WalletAPI } -func (c *LoggedWallet) WalletNew(ctx context.Context, typ crypto.SigType) (address.Address, error) { - n, err := typ.Name() - if err != nil { - return address.Address{}, err - } - - log.Infow("WalletNew", "type", n) +func (c *LoggedWallet) WalletNew(ctx context.Context, typ types.KeyType) (address.Address, error) { + log.Infow("WalletNew", "type", typ) return c.under.WalletNew(ctx, typ) } diff --git a/cmd/lotus-wallet/main.go b/cmd/lotus-wallet/main.go index ffaddc7e3..3285b13e7 100644 --- a/cmd/lotus-wallet/main.go +++ b/cmd/lotus-wallet/main.go @@ -101,19 +101,22 @@ var runCmd = &cli.Command{ return err } - var w api.WalletAPI - if !cctx.Bool("ledger") { - w, err = wallet.NewWallet(ks) - if err != nil { - return err - } - } else { + lw, err := wallet.NewWallet(ks) + if err != nil { + return err + } + + var w api.WalletAPI = lw + if cctx.Bool("ledger") { ds, err := lr.Datastore("/metadata") if err != nil { return err } - w = ledgerwallet.NewWallet(ds) + w = wallet.MultiWallet{ + Local: lw, + Ledger: ledgerwallet.NewWallet(ds), + } } address := cctx.String("listen") diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index ec8071b57..6d201b435 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -4710,7 +4710,7 @@ Inputs: Response: ```json { - "Type": "string value", + "Type": "bls", "PrivateKey": "Ynl0ZSBhcnJheQ==" } ``` @@ -4740,7 +4740,7 @@ Inputs: ```json [ { - "Type": "string value", + "Type": "bls", "PrivateKey": "Ynl0ZSBhcnJheQ==" } ] @@ -4760,6 +4760,8 @@ Response: `null` ### WalletNew WalletNew creates a new address in the wallet with the given sigType. +Available key types: bls, secp256k1, secp256k1-ledger +Support for numerical types: 1 - secp256k1, 2 - BLS is deprecated Perms: write @@ -4767,7 +4769,7 @@ Perms: write Inputs: ```json [ - 2 + "bls" ] ``` diff --git a/node/modules/lp2p/libp2p.go b/node/modules/lp2p/libp2p.go index 5a1666cb6..51749c4d6 100644 --- a/node/modules/lp2p/libp2p.go +++ b/node/modules/lp2p/libp2p.go @@ -20,8 +20,8 @@ import ( var log = logging.Logger("p2pnode") const ( - KLibp2pHost = "libp2p-host" - KTLibp2pHost = KLibp2pHost + KLibp2pHost = "libp2p-host" + KTLibp2pHost types.KeyType = KLibp2pHost ) type Libp2pOpts struct { diff --git a/scripts/dev/api.bash b/scripts/dev/api.bash index 5539e4fef..ac9eb4e66 100644 --- a/scripts/dev/api.bash +++ b/scripts/dev/api.bash @@ -1,11 +1,11 @@ #!/bin/bash # vim: set expandtab ts=2 sw=2: -token=$(lotus auth create-token --perm admin) +_lotus_token=$(./lotus auth create-token --perm admin) runAPI() { curl -X POST \ -H "Content-Type: application/json" \ --data '{"jsonrpc":"2.0","id":2,"method":"Filecoin.'"$1"'","params":'"${2:-null}"'}' \ - 'http://127.0.0.1:1234/rpc/v0?token='"$token" + 'http://127.0.0.1:1234/rpc/v0?token='"$_lotus_token" } diff --git a/storage/mockstorage/preseal.go b/storage/mockstorage/preseal.go index 8ca789ba6..090aacc3c 100644 --- a/storage/mockstorage/preseal.go +++ b/storage/mockstorage/preseal.go @@ -7,7 +7,6 @@ import ( commcid "github.com/filecoin-project/go-fil-commcid" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" - "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/lotus/extern/sector-storage/mock" market0 "github.com/filecoin-project/specs-actors/actors/builtin/market" @@ -20,7 +19,7 @@ import ( ) func PreSeal(ssize abi.SectorSize, maddr address.Address, sectors int) (*genesis.Miner, *types.KeyInfo, error) { - k, err := wallet.GenerateKey(crypto.SigTypeBLS) + k, err := wallet.GenerateKey(types.KTBLS) if err != nil { return nil, nil, err } From d6c8276c7a3e76e32484aaae745a0ccb2e6fef53 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Mon, 12 Oct 2020 00:48:23 +0200 Subject: [PATCH 706/885] Add support for WalletNew to ledger Signed-off-by: Jakub Sztandera --- chain/wallet/key.go | 15 +------ chain/wallet/ledger/ledger.go | 76 +++++++++++++++++++++++++++++++++-- 2 files changed, 74 insertions(+), 17 deletions(-) diff --git a/chain/wallet/key.go b/chain/wallet/key.go index ac9fc436f..1b191cc4b 100644 --- a/chain/wallet/key.go +++ b/chain/wallet/key.go @@ -13,7 +13,7 @@ import ( func GenerateKey(typ types.KeyType) (*Key, error) { ctyp := ActSigType(typ) if ctyp == crypto.SigTypeUnknown { - return nil, xerrors.Errorf("unknown key type: %s", typ) + return nil, xerrors.Errorf("unknown sig type: %s", typ) } pk, err := sigs.Generate(ctyp) if err != nil { @@ -56,23 +56,12 @@ func NewKey(keyinfo types.KeyInfo) (*Key, error) { return nil, xerrors.Errorf("converting BLS to address: %w", err) } default: - return nil, xerrors.Errorf("unknown key type: %s", k.Type) + return nil, xerrors.Errorf("unsupported key type: %s", k.Type) } return k, nil } -func kstoreSigType(typ crypto.SigType) types.KeyType { - switch typ { - case crypto.SigTypeBLS: - return types.KTBLS - case crypto.SigTypeSecp256k1: - return types.KTSecp256k1 - default: - return "" - } -} - func ActSigType(typ types.KeyType) crypto.SigType { switch typ { case types.KTBLS: diff --git a/chain/wallet/ledger/ledger.go b/chain/wallet/ledger/ledger.go index b6b5683cd..9a0d13419 100644 --- a/chain/wallet/ledger/ledger.go +++ b/chain/wallet/ledger/ledger.go @@ -115,12 +115,22 @@ func (lw LedgerWallet) WalletImport(ctx context.Context, kinfo *types.KeyInfo) ( if err := json.Unmarshal(kinfo.PrivateKey, &ki); err != nil { return address.Undef, err } + return lw.importKey(ki) +} +func (lw LedgerWallet) importKey(ki LedgerKeyInfo) (address.Address, error) { if ki.Address == address.Undef { return address.Undef, fmt.Errorf("no address given in imported key info") } + if len(ki.Path) != filHdPathLen { + return address.Undef, fmt.Errorf("bad hd path len: %d, expected: %d", len(ki.Path), filHdPathLen) + } + bb, err := json.Marshal(ki) + if err != nil { + return address.Undef, xerrors.Errorf("marshaling key info: %w", err) + } - if err := lw.ds.Put(keyForAddr(ki.Address), kinfo.PrivateKey); err != nil { + if err := lw.ds.Put(keyForAddr(ki.Address), bb); err != nil { return address.Undef, err } @@ -128,10 +138,11 @@ func (lw LedgerWallet) WalletImport(ctx context.Context, kinfo *types.KeyInfo) ( } func (lw LedgerWallet) WalletList(ctx context.Context) ([]address.Address, error) { - res, err := lw.ds.Query(query.Query{Prefix: "/ledgerkey/"}) + res, err := lw.ds.Query(query.Query{Prefix: dsLedgerPrefix}) if err != nil { return nil, err } + defer res.Close() var out []address.Address for { @@ -150,8 +161,63 @@ func (lw LedgerWallet) WalletList(ctx context.Context) ([]address.Address, error return out, nil } +const hdHard = 0x80000000 + +var filHDBasePath = []uint32{hdHard | 44, hdHard | 461, hdHard, 0} +var filHdPathLen = 5 + func (lw LedgerWallet) WalletNew(ctx context.Context, t types.KeyType) (address.Address, error) { - return address.Undef, fmt.Errorf("cannot create new address on ledger") + if t != types.KTSecp256k1Ledger { + return address.Undef, fmt.Errorf("unsupported key type: '%s', only '%s' supported", + t, types.KTSecp256k1Ledger) + } + + res, err := lw.ds.Query(query.Query{Prefix: dsLedgerPrefix}) + if err != nil { + return address.Undef, err + } + defer res.Close() + + var maxi int64 = -1 + for { + res, ok := res.NextSync() + if !ok { + break + } + + var ki LedgerKeyInfo + if err := json.Unmarshal(res.Value, &ki); err != nil { + return address.Undef, err + } + if i := ki.Path[filHdPathLen-1]; maxi == -1 || maxi < int64(i) { + maxi = int64(i) + } + } + if maxi == -1 { + maxi = 0 + } + + fl, err := ledgerfil.FindLedgerFilecoinApp() + if err != nil { + return address.Undef, xerrors.Errorf("finding ledger: %w", err) + } + + path := append(append([]uint32(nil), filHDBasePath...), uint32(maxi)) + _, _, addr, err := fl.GetAddressPubKeySECP256K1(path) + if err != nil { + return address.Undef, xerrors.Errorf("getting public key from ledger: %w", err) + } + + a, err := address.NewFromString(addr) + if err != nil { + return address.Undef, fmt.Errorf("parsing address: %w", err) + } + + var lki LedgerKeyInfo + lki.Address = a + lki.Path = path + + return lw.importKey(lki) } func (lw *LedgerWallet) Get() api.WalletAPI { @@ -162,6 +228,8 @@ func (lw *LedgerWallet) Get() api.WalletAPI { return lw } +var dsLedgerPrefix = "/ledgerkey/" + func keyForAddr(addr address.Address) datastore.Key { - return datastore.NewKey("/ledgerkey/" + addr.String()) + return datastore.NewKey(dsLedgerPrefix + addr.String()) } From 7b4a5fc7ca0cd0fd5fd6068c93005adbec3167db Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Mon, 12 Oct 2020 00:50:29 +0200 Subject: [PATCH 707/885] Fixup off by one Signed-off-by: Jakub Sztandera --- chain/wallet/ledger/ledger.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/chain/wallet/ledger/ledger.go b/chain/wallet/ledger/ledger.go index 9a0d13419..14c75b356 100644 --- a/chain/wallet/ledger/ledger.go +++ b/chain/wallet/ledger/ledger.go @@ -193,16 +193,13 @@ func (lw LedgerWallet) WalletNew(ctx context.Context, t types.KeyType) (address. maxi = int64(i) } } - if maxi == -1 { - maxi = 0 - } fl, err := ledgerfil.FindLedgerFilecoinApp() if err != nil { return address.Undef, xerrors.Errorf("finding ledger: %w", err) } - path := append(append([]uint32(nil), filHDBasePath...), uint32(maxi)) + path := append(append([]uint32(nil), filHDBasePath...), uint32(maxi+1)) _, _, addr, err := fl.GetAddressPubKeySECP256K1(path) if err != nil { return address.Undef, xerrors.Errorf("getting public key from ledger: %w", err) From bae3b7d5940d758cdee0ff9f1daa0f03f6f8b76c Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Sun, 11 Oct 2020 18:55:50 -0400 Subject: [PATCH 708/885] add a shed util to determine % of power that has won a block --- chain/actors/builtin/power/power.go | 8 ++ cmd/lotus-shed/sync.go | 120 ++++++++++++++++++++++++++++ 2 files changed, 128 insertions(+) diff --git a/chain/actors/builtin/power/power.go b/chain/actors/builtin/power/power.go index e683cfd96..8faad718e 100644 --- a/chain/actors/builtin/power/power.go +++ b/chain/actors/builtin/power/power.go @@ -2,6 +2,7 @@ package power import ( "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/big" "github.com/ipfs/go-cid" "golang.org/x/xerrors" @@ -60,3 +61,10 @@ type Claim struct { // Sum of quality adjusted power for a miner's sectors. QualityAdjPower abi.StoragePower } + +func AddClaims(a Claim, b Claim) Claim { + return Claim{ + RawBytePower: big.Add(a.RawBytePower, b.RawBytePower), + QualityAdjPower: big.Add(a.QualityAdjPower, b.QualityAdjPower), + } +} diff --git a/cmd/lotus-shed/sync.go b/cmd/lotus-shed/sync.go index bfe7cc8b7..65d2b6d6f 100644 --- a/cmd/lotus-shed/sync.go +++ b/cmd/lotus-shed/sync.go @@ -2,6 +2,15 @@ package main import ( "fmt" + "strconv" + + "github.com/filecoin-project/go-state-types/big" + + "github.com/filecoin-project/lotus/chain/actors/builtin/power" + + "github.com/filecoin-project/go-address" + + "github.com/filecoin-project/go-state-types/abi" "github.com/ipfs/go-cid" @@ -16,6 +25,7 @@ var syncCmd = &cli.Command{ Flags: []cli.Flag{}, Subcommands: []*cli.Command{ syncValidateCmd, + syncScrapePowerCmd, }, } @@ -62,3 +72,113 @@ var syncValidateCmd = &cli.Command{ return nil }, } + +var syncScrapePowerCmd = &cli.Command{ + Name: "scrape-power", + Usage: "given a height and a tipset, reports what percentage of mining power had a winning ticket between the tipset and height", + ArgsUsage: "[height tipsetkey]", + Action: func(cctx *cli.Context) error { + if cctx.Args().Len() < 1 { + fmt.Println("usage: [blockCid1 blockCid2...]") + fmt.Println("Any CIDs passed after the height will be used as the tipset key") + fmt.Println("If no block CIDs are provided, chain head will be used") + return nil + } + + api, closer, err := lcli.GetFullNodeAPI(cctx) + if err != nil { + return err + } + + defer closer() + ctx := lcli.ReqContext(cctx) + + if cctx.Args().Len() < 1 { + fmt.Println("usage: ...") + fmt.Println("At least one block cid must be provided") + return nil + } + + h, err := strconv.ParseInt(cctx.Args().Get(0), 10, 0) + if err != nil { + return err + } + + height := abi.ChainEpoch(h) + + var ts *types.TipSet + var startTsk types.TipSetKey + if cctx.NArg() > 1 { + var tscids []cid.Cid + args := cctx.Args().Slice() + + for _, s := range args[1:] { + c, err := cid.Decode(s) + if err != nil { + return fmt.Errorf("block cid was invalid: %s", err) + } + tscids = append(tscids, c) + } + + startTsk = types.NewTipSetKey(tscids...) + ts, err = api.ChainGetTipSet(ctx, startTsk) + if err != nil { + return err + } + } else { + ts, err = api.ChainHead(ctx) + if err != nil { + return err + } + + startTsk = ts.Key() + } + + if ts.Height() < height { + return fmt.Errorf("start tipset's height < stop height: %d < %d", ts.Height(), height) + } + + miners := make(map[address.Address]struct{}) + for ts.Height() >= height { + for _, blk := range ts.Blocks() { + _, found := miners[blk.Miner] + if !found { + // do the thing + miners[blk.Miner] = struct{}{} + } + } + + ts, err = api.ChainGetTipSet(ctx, ts.Parents()) + if err != nil { + return err + } + } + + totalWonPower := power.Claim{ + RawBytePower: big.Zero(), + QualityAdjPower: big.Zero(), + } + for miner := range miners { + mp, err := api.StateMinerPower(ctx, miner, startTsk) + if err != nil { + return err + } + + totalWonPower = power.AddClaims(totalWonPower, mp.MinerPower) + } + + totalPower, err := api.StateMinerPower(ctx, address.Undef, startTsk) + if err != nil { + return err + } + + qpercI := types.BigDiv(types.BigMul(totalWonPower.QualityAdjPower, types.NewInt(1000000)), totalPower.TotalPower.QualityAdjPower) + + fmt.Println("Number of winning miners: ", len(miners)) + fmt.Println("QAdjPower of winning miners: ", totalWonPower.QualityAdjPower) + fmt.Println("QAdjPower of all miners: ", totalPower.TotalPower.QualityAdjPower) + fmt.Println("Percentage of winning QAdjPower: ", float64(qpercI.Int64())/10000) + + return nil + }, +} From fda8b932bab6899e28089b72ef18ea366408fbf0 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Mon, 12 Oct 2020 01:27:15 +0200 Subject: [PATCH 709/885] Ask for confirmation when creating a key Signed-off-by: Jakub Sztandera --- chain/wallet/ledger/ledger.go | 9 +++++++++ chain/wallet/multi.go | 15 ++++++++++----- go.mod | 3 ++- go.sum | 9 +++++++++ 4 files changed, 30 insertions(+), 6 deletions(-) diff --git a/chain/wallet/ledger/ledger.go b/chain/wallet/ledger/ledger.go index 14c75b356..34307d192 100644 --- a/chain/wallet/ledger/ledger.go +++ b/chain/wallet/ledger/ledger.go @@ -9,6 +9,7 @@ import ( "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/query" + logging "github.com/ipfs/go-log" ledgerfil "github.com/whyrusleeping/ledger-filecoin-go" "golang.org/x/xerrors" @@ -20,6 +21,8 @@ import ( "github.com/filecoin-project/lotus/node/modules/dtypes" ) +var log = logging.Logger("wallet-ledger") + type LedgerWallet struct { ds datastore.Datastore } @@ -205,6 +208,12 @@ func (lw LedgerWallet) WalletNew(ctx context.Context, t types.KeyType) (address. return address.Undef, xerrors.Errorf("getting public key from ledger: %w", err) } + log.Warnf("creating key: %s, accept the key in ledger device", addr) + _, _, addr, err = fl.ShowAddressPubKeySECP256K1(path) + if err != nil { + return address.Undef, xerrors.Errorf("verifying public key with ledger: %w", err) + } + a, err := address.NewFromString(addr) if err != nil { return address.Undef, fmt.Errorf("parsing address: %w", err) diff --git a/chain/wallet/multi.go b/chain/wallet/multi.go index 6eac03367..532ad217b 100644 --- a/chain/wallet/multi.go +++ b/chain/wallet/multi.go @@ -70,13 +70,18 @@ func (m MultiWallet) find(ctx context.Context, address address.Address, wallets return nil, nil } -func (m MultiWallet) WalletNew(ctx context.Context, sigType types.KeyType) (address.Address, error) { - w := firstNonNil(m.Remote, m.Local) - if w == nil { - return address.Undef, xerrors.Errorf("no wallet backends configured") +func (m MultiWallet) WalletNew(ctx context.Context, keyType types.KeyType) (address.Address, error) { + var local getif = m.Local + if keyType == types.KTSecp256k1Ledger { + local = m.Ledger } - return w.WalletNew(ctx, sigType) + w := firstNonNil(m.Remote, local) + if w == nil { + return address.Undef, xerrors.Errorf("no wallet backends supporting key type: %s", keyType) + } + + return w.WalletNew(ctx, keyType) } func (m MultiWallet) WalletHas(ctx context.Context, address address.Address) (bool, error) { diff --git a/go.mod b/go.mod index d6a4768b7..922e4b6b5 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/drand/kyber v1.1.2 github.com/dustin/go-humanize v1.0.0 github.com/elastic/go-sysinfo v1.3.0 - github.com/fatih/color v1.8.0 + github.com/fatih/color v1.9.0 github.com/filecoin-project/filecoin-ffi v0.30.4-0.20200716204036-cddc56607e1d github.com/filecoin-project/go-address v0.0.4 github.com/filecoin-project/go-amt-ipld/v2 v2.1.1-0.20201006184820-924ee87a1349 // indirect @@ -104,6 +104,7 @@ require ( github.com/libp2p/go-libp2p-tls v0.1.3 github.com/libp2p/go-libp2p-yamux v0.2.8 github.com/libp2p/go-maddr-filter v0.1.0 + github.com/mattn/go-colorable v0.1.6 // indirect github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 github.com/mitchellh/go-homedir v1.1.0 github.com/multiformats/go-base32 v0.0.3 diff --git a/go.sum b/go.sum index fe7aacbf9..6dfc9aac6 100644 --- a/go.sum +++ b/go.sum @@ -221,6 +221,8 @@ github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5/go.mod h1:Jp github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.8.0 h1:5bzFgL+oy7JITMTxUPJ00n7VxmYd/PdMp5mHFX40/RY= github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGjnw8= +github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E= github.com/filecoin-project/go-address v0.0.3 h1:eVfbdjEbpbzIrbiSa+PiGUY+oDK9HnUn+M1R/ggoHf8= github.com/filecoin-project/go-address v0.0.3/go.mod h1:jr8JxKsYx+lQlQZmF5i2U0Z+cGQ59wMIps/8YW/lDj8= @@ -1045,11 +1047,16 @@ github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcncea github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= @@ -1665,11 +1672,13 @@ golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191025021431-6c3a3bfe00ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191025090151-53bf42e6b339/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= From df7ee610cd426bbd20bdf6ccdeae4b9b4e89e3fe Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Mon, 12 Oct 2020 01:29:19 +0200 Subject: [PATCH 710/885] Re-enable repub Signed-off-by: Jakub Sztandera --- chain/messagepool/messagepool.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/chain/messagepool/messagepool.go b/chain/messagepool/messagepool.go index 2a4704c3b..d3c638b22 100644 --- a/chain/messagepool/messagepool.go +++ b/chain/messagepool/messagepool.go @@ -1249,9 +1249,6 @@ func (mp *MessagePool) Updates(ctx context.Context) (<-chan api.MpoolUpdate, err } func (mp *MessagePool) loadLocal() error { - return nil - // TODO: this causes super slow startup... - res, err := mp.localMsgs.Query(query.Query{}) if err != nil { return xerrors.Errorf("query local messages: %w", err) From b4215e5f46003284599f4736eb6f7003f9785ae5 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Sun, 11 Oct 2020 19:49:57 -0400 Subject: [PATCH 711/885] Add a CLI tool for miner's to repay debt --- chain/actors/builtin/miner/miner.go | 14 ++++ cmd/lotus-storage-miner/actor.go | 105 ++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+) diff --git a/chain/actors/builtin/miner/miner.go b/chain/actors/builtin/miner/miner.go index 8649d4351..281068827 100644 --- a/chain/actors/builtin/miner/miner.go +++ b/chain/actors/builtin/miner/miner.go @@ -148,6 +148,20 @@ type MinerInfo struct { ConsensusFaultElapsed abi.ChainEpoch } +func (mi MinerInfo) IsController(addr address.Address) bool { + if addr == mi.Owner || addr == mi.Worker { + return true + } + + for _, ca := range mi.ControlAddresses { + if addr == ca { + return true + } + } + + return false +} + type SectorExpiration struct { OnTime abi.ChainEpoch diff --git a/cmd/lotus-storage-miner/actor.go b/cmd/lotus-storage-miner/actor.go index fa320289e..35e2a268e 100644 --- a/cmd/lotus-storage-miner/actor.go +++ b/cmd/lotus-storage-miner/actor.go @@ -5,6 +5,11 @@ import ( "os" "strings" + "github.com/filecoin-project/lotus/api/apibstore" + "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + cbor "github.com/ipfs/go-ipld-cbor" + "github.com/filecoin-project/lotus/build" builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" @@ -34,6 +39,7 @@ var actorCmd = &cli.Command{ Subcommands: []*cli.Command{ actorSetAddrsCmd, actorWithdrawCmd, + actorRepayDebtCmd, actorSetPeeridCmd, actorSetOwnerCmd, actorControl, @@ -254,6 +260,105 @@ var actorWithdrawCmd = &cli.Command{ }, } +var actorRepayDebtCmd = &cli.Command{ + Name: "repay-debt", + Usage: "pay down a miner's debt", + ArgsUsage: "[amount (FIL)]", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "from", + Usage: "optionally specify the account to send funds from", + }, + }, + Action: func(cctx *cli.Context) error { + nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return err + } + defer closer() + + api, acloser, err := lcli.GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer acloser() + + ctx := lcli.ReqContext(cctx) + + maddr, err := nodeApi.ActorAddress(ctx) + if err != nil { + return err + } + + mi, err := api.StateMinerInfo(ctx, maddr, types.EmptyTSK) + if err != nil { + return err + } + + var amount abi.TokenAmount + if cctx.Args().Present() { + f, err := types.ParseFIL(cctx.Args().First()) + if err != nil { + return xerrors.Errorf("parsing 'amount' argument: %w", err) + } + + amount = abi.TokenAmount(f) + } else { + mact, err := api.StateGetActor(ctx, maddr, types.EmptyTSK) + if err != nil { + return err + } + + store := adt.WrapStore(ctx, cbor.NewCborStore(apibstore.NewAPIBlockstore(api))) + + mst, err := miner.Load(store, mact) + if err != nil { + return err + } + + amount, err = mst.FeeDebt() + if err != nil { + return err + } + + } + + fromAddr := mi.Worker + if from := cctx.String("from"); from != "" { + addr, err := address.NewFromString(from) + if err != nil { + return err + } + + fromAddr = addr + } + + fromId, err := api.StateLookupID(ctx, fromAddr, types.EmptyTSK) + if err != nil { + return err + } + + if !mi.IsController(fromId) { + return xerrors.Errorf("sender isn't a controller of miner: %s", fromId) + } + + smsg, err := api.MpoolPushMessage(ctx, &types.Message{ + To: maddr, + From: fromId, + Value: amount, + Method: builtin2.MethodsMiner.RepayDebt, + Params: nil, + }, nil) + if err != nil { + return err + } + + fmt.Printf("Sent repay debt message %s\n", smsg.Cid()) + + return nil + }, +} + var actorControl = &cli.Command{ Name: "control", Usage: "Manage control addresses", From ae366972a9e8170d616d8c1f36a819dd27d78c3f Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Mon, 12 Oct 2020 05:11:04 +0200 Subject: [PATCH 712/885] Chain is love Signed-off-by: Jakub Sztandera --- cli/chain.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cli/chain.go b/cli/chain.go index 763752f23..bc0d7b508 100644 --- a/cli/chain.go +++ b/cli/chain.go @@ -521,8 +521,9 @@ var chainInspectUsage = &cli.Command{ } var chainListCmd = &cli.Command{ - Name: "list", - Usage: "View a segment of the chain", + Name: "list", + Aliases: []string{"love"}, + Usage: "View a segment of the chain", Flags: []cli.Flag{ &cli.Uint64Flag{Name: "height"}, &cli.IntFlag{Name: "count", Value: 30}, From c296e1bbcac98225f1d5ef7b34e9790c3dbed31e Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Sun, 11 Oct 2020 20:31:20 -0700 Subject: [PATCH 713/885] use actor abstraction in metrics command --- chain/actors/builtin/power/power.go | 1 + chain/actors/builtin/power/v0.go | 19 +++++++++++++++++ chain/actors/builtin/power/v2.go | 19 +++++++++++++++++ tools/stats/metrics.go | 33 ++++------------------------- 4 files changed, 43 insertions(+), 29 deletions(-) diff --git a/chain/actors/builtin/power/power.go b/chain/actors/builtin/power/power.go index e683cfd96..bafb14de0 100644 --- a/chain/actors/builtin/power/power.go +++ b/chain/actors/builtin/power/power.go @@ -51,6 +51,7 @@ type State interface { MinerPower(address.Address) (Claim, bool, error) MinerNominalPowerMeetsConsensusMinimum(address.Address) (bool, error) ListAllMiners() ([]address.Address, error) + ForEachClaim(func(miner address.Address, claim Claim) error) error } type Claim struct { diff --git a/chain/actors/builtin/power/v0.go b/chain/actors/builtin/power/v0.go index e2a9cf382..3f9a65777 100644 --- a/chain/actors/builtin/power/v0.go +++ b/chain/actors/builtin/power/v0.go @@ -96,3 +96,22 @@ func (s *state0) ListAllMiners() ([]address.Address, error) { return miners, nil } + +func (s *state0) ForEachClaim(cb func(miner address.Address, claim Claim) error) error { + claims, err := adt0.AsMap(s.store, s.Claims) + if err != nil { + return err + } + + var claim power0.Claim + return claims.ForEach(&claim, func(k string) error { + a, err := address.NewFromBytes([]byte(k)) + if err != nil { + return err + } + return cb(a, Claim{ + RawBytePower: claim.RawBytePower, + QualityAdjPower: claim.QualityAdjPower, + }) + }) +} diff --git a/chain/actors/builtin/power/v2.go b/chain/actors/builtin/power/v2.go index 6346a09b6..0c15f0669 100644 --- a/chain/actors/builtin/power/v2.go +++ b/chain/actors/builtin/power/v2.go @@ -96,3 +96,22 @@ func (s *state2) ListAllMiners() ([]address.Address, error) { return miners, nil } + +func (s *state2) ForEachClaim(cb func(miner address.Address, claim Claim) error) error { + claims, err := adt2.AsMap(s.store, s.Claims) + if err != nil { + return err + } + + var claim power2.Claim + return claims.ForEach(&claim, func(k string) error { + a, err := address.NewFromBytes([]byte(k)) + if err != nil { + return err + } + return cb(a, Claim{ + RawBytePower: claim.RawBytePower, + QualityAdjPower: claim.QualityAdjPower, + }) + }) +} diff --git a/tools/stats/metrics.go b/tools/stats/metrics.go index 8dc79ea44..795203c40 100644 --- a/tools/stats/metrics.go +++ b/tools/stats/metrics.go @@ -13,12 +13,10 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors/builtin/power" "github.com/filecoin-project/lotus/chain/actors/builtin/reward" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/builtin/power" - "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/ipfs/go-cid" "github.com/multiformats/go-multihash" @@ -272,35 +270,17 @@ func RecordTipsetStatePoints(ctx context.Context, api api.FullNode, pl *PointLis p = NewPoint("chain.power", totalPower.TotalPower.QualityAdjPower.Int64()) pl.AddPoint(p) - powerActor, err := api.StateGetActor(ctx, builtin.StoragePowerActorAddr, tipset.Key()) + powerActor, err := api.StateGetActor(ctx, power.Address, tipset.Key()) if err != nil { return err } - powerRaw, err := api.ChainReadObj(ctx, powerActor.Head) + powerActorState, err := power.Load(&ApiIpldStore{ctx, api}, powerActor) if err != nil { return err } - var powerActorState power.State - - if err := powerActorState.UnmarshalCBOR(bytes.NewReader(powerRaw)); err != nil { - return fmt.Errorf("failed to unmarshal power actor state: %w", err) - } - - s := &ApiIpldStore{ctx, api} - mp, err := adt.AsMap(s, powerActorState.Claims) - if err != nil { - return err - } - - var claim power.Claim - err = mp.ForEach(&claim, func(key string) error { - addr, err := address.NewFromBytes([]byte(key)) - if err != nil { - return err - } - + return powerActorState.ForEachClaim(func(addr address.Address, claim power.Claim) error { if claim.QualityAdjPower.Int64() == 0 { return nil } @@ -311,11 +291,6 @@ func RecordTipsetStatePoints(ctx context.Context, api api.FullNode, pl *PointLis return nil }) - if err != nil { - return err - } - - return nil } type msgTag struct { From a2e0832a12f220678ac73bf96542c747814b02bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 12 Oct 2020 06:17:00 +0200 Subject: [PATCH 714/885] Fix lint --- chain/wallet/ledger/ledger.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chain/wallet/ledger/ledger.go b/chain/wallet/ledger/ledger.go index 34307d192..c1757fd0a 100644 --- a/chain/wallet/ledger/ledger.go +++ b/chain/wallet/ledger/ledger.go @@ -145,7 +145,7 @@ func (lw LedgerWallet) WalletList(ctx context.Context) ([]address.Address, error if err != nil { return nil, err } - defer res.Close() + defer res.Close() // nolint:errcheck var out []address.Address for { @@ -179,7 +179,7 @@ func (lw LedgerWallet) WalletNew(ctx context.Context, t types.KeyType) (address. if err != nil { return address.Undef, err } - defer res.Close() + defer res.Close() // nolint:errcheck var maxi int64 = -1 for { From 4ac2bee81ce92676e966959299a4cd1a18b3a6ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 12 Oct 2020 06:34:45 +0200 Subject: [PATCH 715/885] Update oni --- extern/oni | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/oni b/extern/oni index dbee44e4f..10ed9ef57 160000 --- a/extern/oni +++ b/extern/oni @@ -1 +1 @@ -Subproject commit dbee44e4f940a502971f17116ccbba61ceaf2674 +Subproject commit 10ed9ef576836186de3b8513c03cdc3fb18c44ed From 92177b573812cd19e56f5873921b7ed8ec685906 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Sun, 11 Oct 2020 22:31:46 -0700 Subject: [PATCH 716/885] implement tape upgrade Upgrade to specs-actors v2.1 and network version 5. This fixes the bug where prove commits were not accepted. --- build/params_2k.go | 3 ++- build/params_shared_vals.go | 2 +- build/params_testground.go | 5 +++-- build/params_testnet.go | 2 ++ chain/actors/version.go | 2 +- chain/stmgr/forks.go | 5 ++++- documentation/en/api-methods.md | 2 +- go.mod | 4 ++-- go.sum | 6 ++++-- 9 files changed, 20 insertions(+), 11 deletions(-) diff --git a/build/params_2k.go b/build/params_2k.go index c6538dc08..b09b60fae 100644 --- a/build/params_2k.go +++ b/build/params_2k.go @@ -17,9 +17,10 @@ const BreezeGasTampingDuration = 0 const UpgradeSmokeHeight = -1 const UpgradeIgnitionHeight = -2 const UpgradeRefuelHeight = -3 +const UpgradeTapeHeight = -4 var UpgradeActorsV2Height = abi.ChainEpoch(10) -var UpgradeLiftoffHeight = abi.ChainEpoch(-4) +var UpgradeLiftoffHeight = abi.ChainEpoch(-5) var DrandSchedule = map[abi.ChainEpoch]DrandEnum{ 0: DrandMainnet, diff --git a/build/params_shared_vals.go b/build/params_shared_vals.go index 722590575..ede40c0e3 100644 --- a/build/params_shared_vals.go +++ b/build/params_shared_vals.go @@ -25,7 +25,7 @@ const UnixfsLinksPerLevel = 1024 // Consensus / Network const AllowableClockDriftSecs = uint64(1) -const NewestNetworkVersion = network.Version4 +const NewestNetworkVersion = network.Version5 const ActorUpgradeNetworkVersion = network.Version4 // Epochs diff --git a/build/params_testground.go b/build/params_testground.go index 6109cbc04..7ef034234 100644 --- a/build/params_testground.go +++ b/build/params_testground.go @@ -83,14 +83,15 @@ var ( UpgradeSmokeHeight abi.ChainEpoch = -1 UpgradeIgnitionHeight abi.ChainEpoch = -2 UpgradeRefuelHeight abi.ChainEpoch = -3 + UpgradeTapeHeight abi.ChainEpoch = -4 UpgradeActorsV2Height abi.ChainEpoch = 10 - UpgradeLiftoffHeight abi.ChainEpoch = -4 + UpgradeLiftoffHeight abi.ChainEpoch = -5 DrandSchedule = map[abi.ChainEpoch]DrandEnum{ 0: DrandMainnet, } - NewestNetworkVersion = network.Version4 + NewestNetworkVersion = network.Version5 ActorUpgradeNetworkVersion = network.Version4 Devnet = true diff --git a/build/params_testnet.go b/build/params_testnet.go index fe70deaef..54f50ac6e 100644 --- a/build/params_testnet.go +++ b/build/params_testnet.go @@ -30,6 +30,8 @@ const UpgradeRefuelHeight = 130800 var UpgradeActorsV2Height = abi.ChainEpoch(138720) +const UpgradeTapeHeight = 140760 + // This signals our tentative epoch for mainnet launch. Can make it later, but not earlier. // Miners, clients, developers, custodians all need time to prepare. // We still have upgrades and state changes to do, but can happen after signaling timing here. diff --git a/chain/actors/version.go b/chain/actors/version.go index 17af8b08b..2efd903bb 100644 --- a/chain/actors/version.go +++ b/chain/actors/version.go @@ -18,7 +18,7 @@ func VersionForNetwork(version network.Version) Version { switch version { case network.Version0, network.Version1, network.Version2, network.Version3: return Version0 - case network.Version4: + case network.Version4, network.Version5: return Version2 default: panic(fmt.Sprintf("unsupported network version %d", version)) diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index fba92ee3f..6ebe8a5b9 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -79,9 +79,12 @@ func DefaultUpgradeSchedule() UpgradeSchedule { Network: network.Version4, Expensive: true, Migration: UpgradeActorsV2, + }, { + Height: build.UpgradeTapeHeight, + Network: network.Version5, }, { Height: build.UpgradeLiftoffHeight, - Network: network.Version4, + Network: network.Version5, Migration: UpgradeLiftoff, }} diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index ec8071b57..76dce56db 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -3975,7 +3975,7 @@ Inputs: ] ``` -Response: `4` +Response: `5` ### StateReadState StateReadState returns the indicated actor's state. diff --git a/go.mod b/go.mod index eb79a496c..ef43a5274 100644 --- a/go.mod +++ b/go.mod @@ -34,12 +34,12 @@ require ( github.com/filecoin-project/go-multistore v0.0.3 github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 github.com/filecoin-project/go-paramfetch v0.0.2-0.20200701152213-3e0f0afdc261 - github.com/filecoin-project/go-state-types v0.0.0-20200928172055-2df22083d8ab + github.com/filecoin-project/go-state-types v0.0.0-20201003010437-c33112184a2b github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b github.com/filecoin-project/specs-actors v0.9.12 - github.com/filecoin-project/specs-actors/v2 v2.0.3 + github.com/filecoin-project/specs-actors/v2 v2.1.0 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 github.com/filecoin-project/test-vectors/schema v0.0.4 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 diff --git a/go.sum b/go.sum index e13359f6c..a9ac12ff7 100644 --- a/go.sum +++ b/go.sum @@ -265,6 +265,8 @@ github.com/filecoin-project/go-state-types v0.0.0-20200904021452-1883f36ca2f4/go github.com/filecoin-project/go-state-types v0.0.0-20200905071437-95828685f9df/go.mod h1:IQ0MBPnonv35CJHtWSN3YY1Hz2gkPru1Q9qoaYLxx9I= github.com/filecoin-project/go-state-types v0.0.0-20200928172055-2df22083d8ab h1:cEDC5Ei8UuT99hPWhCjA72SM9AuRtnpvdSTIYbnzN8I= github.com/filecoin-project/go-state-types v0.0.0-20200928172055-2df22083d8ab/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= +github.com/filecoin-project/go-state-types v0.0.0-20201003010437-c33112184a2b h1:bMUfG6Sy6YSMbsjQAO1Q2vEZldbSdsbRy/FX3OlTck0= +github.com/filecoin-project/go-state-types v0.0.0-20201003010437-c33112184a2b/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-statemachine v0.0.0-20200714194326-a77c3ae20989/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe h1:dF8u+LEWeIcTcfUcCf3WFVlc81Fr2JKg8zPzIbBDKDw= github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= @@ -276,8 +278,8 @@ github.com/filecoin-project/specs-actors v0.9.4/go.mod h1:BStZQzx5x7TmCkLv0Bpa07 github.com/filecoin-project/specs-actors v0.9.7/go.mod h1:wM2z+kwqYgXn5Z7scV1YHLyd1Q1cy0R8HfTIWQ0BFGU= github.com/filecoin-project/specs-actors v0.9.12 h1:iIvk58tuMtmloFNHhAOQHG+4Gci6Lui0n7DYQGi3cJk= github.com/filecoin-project/specs-actors v0.9.12/go.mod h1:TS1AW/7LbG+615j4NsjMK1qlpAwaFsG9w0V2tg2gSao= -github.com/filecoin-project/specs-actors/v2 v2.0.3 h1:Niy6xncgi8bI8aBCt1McdZfATBfG4Uxytt8KW4s3bAc= -github.com/filecoin-project/specs-actors/v2 v2.0.3/go.mod h1:v2NZVYinNIKA9acEMBm5wWXxqv5+frFEbekBFemYghY= +github.com/filecoin-project/specs-actors/v2 v2.1.0 h1:ocEuGz8DG2cUWw32c/tvF8D6xT+dGVWJTr5yDevU00g= +github.com/filecoin-project/specs-actors/v2 v2.1.0/go.mod h1:E7fAX4CZkDVQvDNRCxfq+hc3nx56KcCKyuZf0hlQJ20= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= github.com/filecoin-project/test-vectors/schema v0.0.4 h1:QTRd0gb/NP4ZOTM7Dib5U3xE1/ToGDKnYLfxkC3t/m8= From 160f209f6a857eb4b16a8d96b48b0c0d1a0ac404 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Mon, 12 Oct 2020 02:08:38 -0400 Subject: [PATCH 717/885] Lotus version 0.10.0 --- CHANGELOG.md | 19 +++++++++++++++++++ build/version.go | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d397762a6..16d8d15b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,24 @@ # Lotus changelog +# 0.10.0 / 2020-10-12 + +This is a consensus-breaking hotfix that addresses an issue in specs-actors v2.0.3 that made it impossible to pledge new 32GiB sectors. The change in Lotus is to update to actors v2.1.0, behind the new network version 5. + +## Changes + +- make pledge test pass with the race detector (https://github.com/filecoin-project/lotus/pull/4291) +- fix a race in tipset cache usage (https://github.com/filecoin-project/lotus/pull/4282) +- add an api for removing multisig signers (https://github.com/filecoin-project/lotus/pull/4274) +- cli: Don't output errors to stdout (https://github.com/filecoin-project/lotus/pull/4298) +- Fix panic in wallet export when key is not found (https://github.com/filecoin-project/lotus/pull/4299) +- Dump the block validation cache whenever we perform an import (https://github.com/filecoin-project/lotus/pull/4287) +- Fix two races (https://github.com/filecoin-project/lotus/pull/4301) +- sync unmark-bad --all (https://github.com/filecoin-project/lotus/pull/4296) +- decode parameters for multisig transactions in inspect (https://github.com/filecoin-project/lotus/pull/4312) +- Chain is love (https://github.com/filecoin-project/lotus/pull/4321) +- lotus-stats: optmize getting miner power (https://github.com/filecoin-project/lotus/pull/4315) +- implement tape upgrade (https://github.com/filecoin-project/lotus/pull/4322) + # 0.9.1 / 2020-10-10 This release fixes an issue which may cause the actors v2 migration to compute the state incorrectly when more than one migration is running in parallel. diff --git a/build/version.go b/build/version.go index 5baed1fb7..baeafdfe7 100644 --- a/build/version.go +++ b/build/version.go @@ -29,7 +29,7 @@ func buildType() string { } // BuildVersion is the local build version, set by build system -const BuildVersion = "0.9.1" +const BuildVersion = "0.10.0" func UserVersion() string { return BuildVersion + buildType() + CurrentCommit From dc4e73c73724115f6fff6c17df608860c03de510 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Sun, 11 Oct 2020 23:31:23 -0700 Subject: [PATCH 718/885] Test the tape upgrade --- api/test/tape.go | 114 +++++++++++++++++++++++++++++ api/test/test.go | 4 +- extern/sector-storage/mock/mock.go | 9 ++- node/node_test.go | 10 +++ 4 files changed, 132 insertions(+), 5 deletions(-) create mode 100644 api/test/tape.go diff --git a/api/test/tape.go b/api/test/tape.go new file mode 100644 index 000000000..466bdd829 --- /dev/null +++ b/api/test/tape.go @@ -0,0 +1,114 @@ +package test + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/network" + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/stmgr" + sealing "github.com/filecoin-project/lotus/extern/storage-sealing" + "github.com/filecoin-project/lotus/node" + "github.com/filecoin-project/lotus/node/impl" + "github.com/stretchr/testify/require" +) + +func TestTapeFix(t *testing.T, b APIBuilder, blocktime time.Duration) { + t.Run("before", func(t *testing.T) { testTapeFix(t, b, blocktime, false) }) + t.Run("after", func(t *testing.T) { testTapeFix(t, b, blocktime, true) }) +} +func testTapeFix(t *testing.T, b APIBuilder, blocktime time.Duration, after bool) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + upgradeSchedule := stmgr.UpgradeSchedule{{ + Network: build.ActorUpgradeNetworkVersion, + Height: 1, + Migration: stmgr.UpgradeActorsV2, + }} + if after { + upgradeSchedule = append(upgradeSchedule, stmgr.Upgrade{ + Network: network.Version5, + Height: 2, + }) + } + + n, sn := b(t, []FullNodeOpts{{Opts: func(_ []TestNode) node.Option { + return node.Override(new(stmgr.UpgradeSchedule), upgradeSchedule) + }}}, OneMiner) + + client := n[0].FullNode.(*impl.FullNodeAPI) + miner := sn[0] + + addrinfo, err := client.NetAddrsListen(ctx) + if err != nil { + t.Fatal(err) + } + + if err := miner.NetConnect(ctx, addrinfo); err != nil { + t.Fatal(err) + } + build.Clock.Sleep(time.Second) + + done := make(chan struct{}) + go func() { + defer close(done) + for ctx.Err() == nil { + build.Clock.Sleep(blocktime) + if err := sn[0].MineOne(ctx, MineNext); err != nil { + if ctx.Err() != nil { + // context was canceled, ignore the error. + return + } + t.Error(err) + } + } + }() + defer func() { + cancel() + <-done + }() + + err = miner.PledgeSector(ctx) + require.NoError(t, err) + + // Wait till done. + var sectorNo abi.SectorNumber + for { + s, err := miner.SectorsList(ctx) // Note - the test builder doesn't import genesis sectors into FSM + require.NoError(t, err) + fmt.Printf("Sectors: %d\n", len(s)) + if len(s) == 1 { + sectorNo = s[0] + break + } + + build.Clock.Sleep(100 * time.Millisecond) + } + + fmt.Printf("All sectors is fsm\n") + + // If before, we expect the precommit to fail + successState := api.SectorState(sealing.CommitFailed) + failureState := api.SectorState(sealing.Proving) + if after { + // otherwise, it should succeed. + successState, failureState = failureState, successState + } + + for { + st, err := miner.SectorsStatus(ctx, sectorNo, false) + require.NoError(t, err) + if st.State == successState { + break + } + require.NotEqual(t, failureState, st.State) + build.Clock.Sleep(100 * time.Millisecond) + fmt.Println("WaitSeal") + } + +} diff --git a/api/test/test.go b/api/test/test.go index 35b397740..947f2bef4 100644 --- a/api/test/test.go +++ b/api/test/test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/require" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/miner" @@ -96,7 +97,8 @@ var FullNodeWithUpgradeAt = func(upgradeHeight abi.ChainEpoch) FullNodeOpts { return FullNodeOpts{ Opts: func(nodes []TestNode) node.Option { return node.Override(new(stmgr.UpgradeSchedule), stmgr.UpgradeSchedule{{ - Network: build.ActorUpgradeNetworkVersion, + // Skip directly to tape height so precommits work. + Network: network.Version5, Height: upgradeHeight, Migration: stmgr.UpgradeActorsV2, }}) diff --git a/extern/sector-storage/mock/mock.go b/extern/sector-storage/mock/mock.go index 64207e66d..001c7159c 100644 --- a/extern/sector-storage/mock/mock.go +++ b/extern/sector-storage/mock/mock.go @@ -230,8 +230,8 @@ func (mgr *SectorMgr) SealCommit1(ctx context.Context, sid abi.SectorID, ticket } func (mgr *SectorMgr) SealCommit2(ctx context.Context, sid abi.SectorID, phase1Out storage.Commit1Out) (proof storage.Proof, err error) { - var out [32]byte - for i := range out { + var out [1920]byte + for i := range out[:len(phase1Out)] { out[i] = phase1Out[i] ^ byte(sid.Number&0xff) } @@ -407,11 +407,12 @@ func (mgr *SectorMgr) CheckProvable(ctx context.Context, spt abi.RegisteredSealP } func (m mockVerif) VerifySeal(svi proof.SealVerifyInfo) (bool, error) { - if len(svi.Proof) != 32 { // Real ones are longer, but this should be fine + if len(svi.Proof) != 1920 { return false, nil } - for i, b := range svi.Proof { + // only the first 32 bytes, the rest are 0. + for i, b := range svi.Proof[:32] { if b != svi.UnsealedCID.Bytes()[i]+svi.SealedCID.Bytes()[31-i]-svi.InteractiveRandomness[i]*svi.Randomness[i] { return false, nil } diff --git a/node/node_test.go b/node/node_test.go index 001b99c04..e553e83b2 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -117,6 +117,16 @@ func TestPledgeSectors(t *testing.T) { }) } +func TestTapeFix(t *testing.T) { + logging.SetLogLevel("miner", "ERROR") + logging.SetLogLevel("chainstore", "ERROR") + logging.SetLogLevel("chain", "ERROR") + logging.SetLogLevel("sub", "ERROR") + logging.SetLogLevel("storageminer", "ERROR") + + test.TestTapeFix(t, builder.MockSbBuilder, 2*time.Millisecond) +} + func TestWindowedPost(t *testing.T) { if os.Getenv("LOTUS_TEST_WINDOW_POST") != "1" { t.Skip("this takes a few minutes, set LOTUS_TEST_WINDOW_POST=1 to run") From c0961fb63d7311d7c41f4c40340e0de013382703 Mon Sep 17 00:00:00 2001 From: Dan Shao Date: Mon, 12 Oct 2020 15:07:56 +0800 Subject: [PATCH 719/885] Improve the UX for `lotus-miner sealing workers`: add total number of cores --- cmd/lotus-storage-miner/sealing.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/lotus-storage-miner/sealing.go b/cmd/lotus-storage-miner/sealing.go index 5cc5c419a..507e842c3 100644 --- a/cmd/lotus-storage-miner/sealing.go +++ b/cmd/lotus-storage-miner/sealing.go @@ -80,7 +80,8 @@ var sealingWorkersCmd = &cli.Command{ cpuBars := int(stat.CpuUse * barCols / stat.Info.Resources.CPUs) cpuBar := strings.Repeat("|", cpuBars) + strings.Repeat(" ", int(barCols)-cpuBars) - fmt.Printf("\tCPU: [%s] %d core(s) in use\n", color.GreenString(cpuBar), stat.CpuUse) + fmt.Printf("\tCPU: [%s] %d/%d core(s) in use\n", + color.GreenString(cpuBar), stat.CpuUse, stat.Info.Resources.CPUs) ramBarsRes := int(stat.Info.Resources.MemReserved * barCols / stat.Info.Resources.MemPhysical) ramBarsUsed := int(stat.MemUsedMin * barCols / stat.Info.Resources.MemPhysical) From 8b648cf77cba845ae1bf0f0c46a906b51ffd1ec6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 12 Oct 2020 09:21:42 +0200 Subject: [PATCH 720/885] ledger: Close device in WalletNew --- chain/wallet/ledger/ledger.go | 1 + 1 file changed, 1 insertion(+) diff --git a/chain/wallet/ledger/ledger.go b/chain/wallet/ledger/ledger.go index c1757fd0a..07f92e7ff 100644 --- a/chain/wallet/ledger/ledger.go +++ b/chain/wallet/ledger/ledger.go @@ -201,6 +201,7 @@ func (lw LedgerWallet) WalletNew(ctx context.Context, t types.KeyType) (address. if err != nil { return address.Undef, xerrors.Errorf("finding ledger: %w", err) } + defer fl.Close() // nolint:errcheck path := append(append([]uint32(nil), filHDBasePath...), uint32(maxi+1)) _, _, addr, err := fl.GetAddressPubKeySECP256K1(path) From b7726c4ca83d632f15bc311d53281a5ad2a9929d Mon Sep 17 00:00:00 2001 From: zgfzgf <1901989065@qq.com> Date: Mon, 12 Oct 2020 16:44:39 +0800 Subject: [PATCH 721/885] optimize walkSnapshot recurseLinks --- chain/store/store.go | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/chain/store/store.go b/chain/store/store.go index c6fc0cbef..067629dc0 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -1312,11 +1312,13 @@ func (cs *ChainStore) WalkSnapshot(ctx context.Context, ts *types.TipSet, inclRe var cids []cid.Cid if !skipOldMsgs || b.Height > ts.Height()-inclRecentRoots { - mcids, err := recurseLinks(cs.bs, walked, b.Messages, []cid.Cid{b.Messages}) - if err != nil { - return xerrors.Errorf("recursing messages failed: %w", err) + if walked.Visit(b.Messages) { + mcids, err := recurseLinks(cs.bs, walked, b.Messages, []cid.Cid{b.Messages}) + if err != nil { + return xerrors.Errorf("recursing messages failed: %w", err) + } + cids = mcids } - cids = mcids } if b.Height > 0 { @@ -1331,12 +1333,14 @@ func (cs *ChainStore) WalkSnapshot(ctx context.Context, ts *types.TipSet, inclRe out := cids if b.Height == 0 || b.Height > ts.Height()-inclRecentRoots { - cids, err := recurseLinks(cs.bs, walked, b.ParentStateRoot, []cid.Cid{b.ParentStateRoot}) - if err != nil { - return xerrors.Errorf("recursing genesis state failed: %w", err) - } + if walked.Visit(b.ParentStateRoot) { + cids, err := recurseLinks(cs.bs, walked, b.ParentStateRoot, []cid.Cid{b.ParentStateRoot}) + if err != nil { + return xerrors.Errorf("recursing genesis state failed: %w", err) + } - out = append(out, cids...) + out = append(out, cids...) + } } for _, c := range out { From 8001ed30308f3e81bd695e8f329a45b162f4b46d Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 1 Oct 2020 11:02:57 +0200 Subject: [PATCH 722/885] Drand: add cloudflare. Use HTTP for PL nodes. Adding Cloudflare ensures additional redundancy. Switching to HTTP for PL nodes reduces latency and cost. Randomness is verified after being received. --- build/drand.go | 11 +---------- chain/beacon/drand/drand_test.go | 2 +- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/build/drand.go b/build/drand.go index 73299249a..3b976ac92 100644 --- a/build/drand.go +++ b/build/drand.go @@ -35,6 +35,7 @@ var DrandConfigs = map[DrandEnum]dtypes.DrandConfig{ "https://api.drand.sh", "https://api2.drand.sh", "https://api3.drand.sh", + "https://drand.cloudflare.com", }, Relays: []string{ "/dnsaddr/api.drand.sh/", @@ -68,16 +69,6 @@ var DrandConfigs = map[DrandEnum]dtypes.DrandConfig{ ChainInfoJSON: `{"public_key":"8cda589f88914aa728fd183f383980b35789ce81b274e5daee1f338b77d02566ef4d3fb0098af1f844f10f9c803c1827","period":25,"genesis_time":1595348225,"hash":"e73b7dc3c4f6a236378220c0dd6aa110eb16eed26c11259606e07ee122838d4f","groupHash":"567d4785122a5a3e75a9bc9911d7ea807dd85ff76b78dc4ff06b075712898607"}`, }, DrandIncentinet: { - Servers: []string{ - "https://pl-eu.incentinet.drand.sh", - "https://pl-us.incentinet.drand.sh", - "https://pl-sin.incentinet.drand.sh", - }, - Relays: []string{ - "/dnsaddr/pl-eu.incentinet.drand.sh/", - "/dnsaddr/pl-us.incentinet.drand.sh/", - "/dnsaddr/pl-sin.incentinet.drand.sh/", - }, ChainInfoJSON: `{"public_key":"8cad0c72c606ab27d36ee06de1d5b2db1faf92e447025ca37575ab3a8aac2eaae83192f846fc9e158bc738423753d000","period":30,"genesis_time":1595873820,"hash":"80c8b872c714f4c00fdd3daa465d5514049f457f01f85a4caf68cdcd394ba039","groupHash":"d9406aaed487f7af71851b4399448e311f2328923d454e971536c05398ce2d9b"}`, }, } diff --git a/chain/beacon/drand/drand_test.go b/chain/beacon/drand/drand_test.go index 0cb9c2ba8..d66ee7b54 100644 --- a/chain/beacon/drand/drand_test.go +++ b/chain/beacon/drand/drand_test.go @@ -12,7 +12,7 @@ import ( ) func TestPrintGroupInfo(t *testing.T) { - server := build.DrandConfigs[build.DrandIncentinet].Servers[0] + server := build.DrandConfigs[build.DrandDevnet].Servers[0] c, err := hclient.New(server, nil, nil) assert.NoError(t, err) cg := c.(interface { From 79c5d2c6d2524a064af1a715fd56bb10af000bb5 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 12 Oct 2020 11:04:46 -0700 Subject: [PATCH 723/885] Rename params_testnet to params_mainnet --- build/{params_testnet.go => params_mainnet.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename build/{params_testnet.go => params_mainnet.go} (100%) diff --git a/build/params_testnet.go b/build/params_mainnet.go similarity index 100% rename from build/params_testnet.go rename to build/params_mainnet.go From 6c0812062881bb26681bf84633af9e454d7824c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 12 Oct 2020 20:46:34 +0200 Subject: [PATCH 724/885] types: Add CID fields to messages in json marshalers --- chain/types/message.go | 15 ++++++++ chain/types/message_test.go | 66 ++++++++++++++++++++++++++++++++++++ chain/types/signedmessage.go | 15 ++++++++ 3 files changed, 96 insertions(+) diff --git a/chain/types/message.go b/chain/types/message.go index 4fead44bc..cbc7c1dad 100644 --- a/chain/types/message.go +++ b/chain/types/message.go @@ -2,6 +2,7 @@ package types import ( "bytes" + "encoding/json" "fmt" "github.com/filecoin-project/go-state-types/abi" @@ -106,6 +107,20 @@ func (m *Message) Cid() cid.Cid { return b.Cid() } +type mCid struct { + *RawMessage + CID cid.Cid +} + +type RawMessage Message + +func (m *Message) MarshalJSON() ([]byte, error) { + return json.Marshal(&mCid{ + RawMessage: (*RawMessage)(m), + CID: m.Cid(), + }) +} + func (m *Message) RequiredFunds() BigInt { return BigMul(m.GasFeeCap, NewInt(uint64(m.GasLimit))) } diff --git a/chain/types/message_test.go b/chain/types/message_test.go index f57385a09..dab8e0939 100644 --- a/chain/types/message_test.go +++ b/chain/types/message_test.go @@ -1,11 +1,14 @@ package types import ( + "encoding/json" + "fmt" "testing" "github.com/stretchr/testify/require" "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/specs-actors/actors/builtin" ) @@ -70,3 +73,66 @@ func TestEqualCall(t *testing.T) { require.True(t, m1.EqualCall(m3)) require.False(t, m1.EqualCall(m4)) } + +func TestMessageJson(t *testing.T) { + m := &Message{ + To: builtin.StoragePowerActorAddr, + From: builtin.SystemActorAddr, + Nonce: 34, + Value: big.Zero(), + + GasLimit: 123, + GasFeeCap: big.NewInt(234), + GasPremium: big.NewInt(234), + + Method: 6, + Params: []byte("hai"), + } + + b, err := json.Marshal(m) + require.NoError(t, err) + + exp := []byte("{\"Version\":0,\"To\":\"f04\",\"From\":\"f00\",\"Nonce\":34,\"Value\":\"0\",\"GasLimit\":123,\"GasFeeCap\":\"234\",\"GasPremium\":\"234\",\"Method\":6,\"Params\":\"aGFp\",\"CID\":{\"/\":\"bafy2bzaced5rdpz57e64sc7mdwjn3blicglhpialnrph2dlbufhf6iha63dmc\"}}") + fmt.Println(string(b)) + + require.Equal(t, exp, b) + + var um Message + require.NoError(t, json.Unmarshal(b, &um)) + + require.EqualValues(t, *m, um) +} + +func TestSignedMessageJson(t *testing.T) { + m := Message{ + To: builtin.StoragePowerActorAddr, + From: builtin.SystemActorAddr, + Nonce: 34, + Value: big.Zero(), + + GasLimit: 123, + GasFeeCap: big.NewInt(234), + GasPremium: big.NewInt(234), + + Method: 6, + Params: []byte("hai"), + } + + sm := &SignedMessage{ + Message: m, + Signature: crypto.Signature{}, + } + + b, err := json.Marshal(sm) + require.NoError(t, err) + + exp := []byte("{\"Message\":{\"Version\":0,\"To\":\"f04\",\"From\":\"f00\",\"Nonce\":34,\"Value\":\"0\",\"GasLimit\":123,\"GasFeeCap\":\"234\",\"GasPremium\":\"234\",\"Method\":6,\"Params\":\"aGFp\",\"CID\":{\"/\":\"bafy2bzaced5rdpz57e64sc7mdwjn3blicglhpialnrph2dlbufhf6iha63dmc\"}},\"Signature\":{\"Type\":0,\"Data\":null},\"CID\":{\"/\":\"bafy2bzacea5ainifngxj3rygaw2hppnyz2cw72x5pysqty2x6dxmjs5qg2uus\"}}") + fmt.Println(string(b)) + + require.Equal(t, exp, b) + + var um SignedMessage + require.NoError(t, json.Unmarshal(b, &um)) + + require.EqualValues(t, *sm, um) +} diff --git a/chain/types/signedmessage.go b/chain/types/signedmessage.go index 17d2f5d94..7532bea35 100644 --- a/chain/types/signedmessage.go +++ b/chain/types/signedmessage.go @@ -2,6 +2,7 @@ package types import ( "bytes" + "encoding/json" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/crypto" @@ -62,6 +63,20 @@ func (sm *SignedMessage) Serialize() ([]byte, error) { return buf.Bytes(), nil } +type smCid struct { + *RawSignedMessage + CID cid.Cid +} + +type RawSignedMessage SignedMessage + +func (sm *SignedMessage) MarshalJSON() ([]byte, error) { + return json.Marshal(&smCid{ + RawSignedMessage: (*RawSignedMessage)(sm), + CID: sm.Cid(), + }) +} + func (sm *SignedMessage) ChainLength() int { ser, err := sm.Serialize() if err != nil { From 61661fb1f06986d9fbfe25f31dfb15ea6743c9d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 12 Oct 2020 20:47:09 +0200 Subject: [PATCH 725/885] docsgen --- documentation/en/api-methods.md | 105 ++++++++++++++++++++++++++------ 1 file changed, 87 insertions(+), 18 deletions(-) diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index 50a6a160d..97d10b064 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -462,7 +462,10 @@ Response: "GasFeeCap": "0", "GasPremium": "0", "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" + "Params": "Ynl0ZSBhcnJheQ==", + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" + } } ``` @@ -1333,7 +1336,10 @@ Inputs: "GasFeeCap": "0", "GasPremium": "0", "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" + "Params": "Ynl0ZSBhcnJheQ==", + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" + } }, 9, [ @@ -1369,7 +1375,10 @@ Inputs: "GasFeeCap": "0", "GasPremium": "0", "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" + "Params": "Ynl0ZSBhcnJheQ==", + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" + } }, [ { @@ -1429,7 +1438,10 @@ Inputs: "GasFeeCap": "0", "GasPremium": "0", "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" + "Params": "Ynl0ZSBhcnJheQ==", + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" + } }, { "MaxFee": "0" @@ -1457,7 +1469,10 @@ Response: "GasFeeCap": "0", "GasPremium": "0", "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" + "Params": "Ynl0ZSBhcnJheQ==", + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" + } } ``` @@ -1741,11 +1756,17 @@ Inputs: "GasFeeCap": "0", "GasPremium": "0", "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" + "Params": "Ynl0ZSBhcnJheQ==", + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" + } }, "Signature": { "Type": 2, "Data": "Ynl0ZSBhcnJheQ==" + }, + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" } } ] @@ -1782,7 +1803,10 @@ Inputs: "GasFeeCap": "0", "GasPremium": "0", "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" + "Params": "Ynl0ZSBhcnJheQ==", + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" + } }, { "MaxFee": "0" @@ -1803,11 +1827,17 @@ Response: "GasFeeCap": "0", "GasPremium": "0", "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" + "Params": "Ynl0ZSBhcnJheQ==", + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" + } }, "Signature": { "Type": 2, "Data": "Ynl0ZSBhcnJheQ==" + }, + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" } } ``` @@ -1832,11 +1862,17 @@ Inputs: "GasFeeCap": "0", "GasPremium": "0", "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" + "Params": "Ynl0ZSBhcnJheQ==", + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" + } }, "Signature": { "Type": 2, "Data": "Ynl0ZSBhcnJheQ==" + }, + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" } } ] @@ -1916,11 +1952,17 @@ Response: "GasFeeCap": "0", "GasPremium": "0", "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" + "Params": "Ynl0ZSBhcnJheQ==", + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" + } }, "Signature": { "Type": 2, "Data": "Ynl0ZSBhcnJheQ==" + }, + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" } } } @@ -2997,7 +3039,10 @@ Inputs: "GasFeeCap": "0", "GasPremium": "0", "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" + "Params": "Ynl0ZSBhcnJheQ==", + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" + } }, [ { @@ -3023,7 +3068,10 @@ Response: "GasFeeCap": "0", "GasPremium": "0", "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" + "Params": "Ynl0ZSBhcnJheQ==", + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" + } }, "MsgRct": { "ExitCode": 0, @@ -3041,7 +3089,10 @@ Response: "GasFeeCap": "0", "GasPremium": "0", "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" + "Params": "Ynl0ZSBhcnJheQ==", + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" + } }, "MsgRct": { "ExitCode": 0, @@ -3296,7 +3347,10 @@ Inputs: "GasFeeCap": "0", "GasPremium": "0", "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" + "Params": "Ynl0ZSBhcnJheQ==", + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" + } }, [ { @@ -4042,7 +4096,10 @@ Response: "GasFeeCap": "0", "GasPremium": "0", "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" + "Params": "Ynl0ZSBhcnJheQ==", + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" + } }, "MsgRct": { "ExitCode": 0, @@ -4060,7 +4117,10 @@ Response: "GasFeeCap": "0", "GasPremium": "0", "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" + "Params": "Ynl0ZSBhcnJheQ==", + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" + } }, "MsgRct": { "ExitCode": 0, @@ -4832,7 +4892,10 @@ Inputs: "GasFeeCap": "0", "GasPremium": "0", "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" + "Params": "Ynl0ZSBhcnJheQ==", + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" + } } ] ``` @@ -4850,11 +4913,17 @@ Response: "GasFeeCap": "0", "GasPremium": "0", "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==" + "Params": "Ynl0ZSBhcnJheQ==", + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" + } }, "Signature": { "Type": 2, "Data": "Ynl0ZSBhcnJheQ==" + }, + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" } } ``` From 96e1dfd8d7f5778b5d534f3b29bac2c8a50f0118 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Sun, 11 Oct 2020 18:17:28 -0400 Subject: [PATCH 726/885] Add an endpoint for precise circulating supply --- api/api_full.go | 8 ++- api/apistruct/struct.go | 9 ++- chain/actors/builtin/builtin.go | 9 ++- chain/actors/builtin/miner/miner.go | 5 ++ chain/actors/builtin/miner/v0.go | 12 +++- chain/actors/builtin/miner/v2.go | 12 +++- chain/stmgr/call.go | 4 +- chain/stmgr/forks.go | 26 +++---- chain/stmgr/stmgr.go | 103 ++++++++++++++++++++++++++-- chain/stmgr/utils.go | 2 +- cli/state.go | 37 +++++++--- cmd/lotus-chainwatch/syncer/sync.go | 2 +- cmd/lotus-shed/balances.go | 2 +- cmd/tvx/extract.go | 2 +- documentation/en/api-methods.md | 47 ++++++++++--- node/impl/full/state.go | 21 ++++-- 16 files changed, 236 insertions(+), 65 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index f42c8d03f..b7cdf9524 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -416,8 +416,12 @@ type FullNode interface { // can issue. It takes the deal size and verified status as parameters. StateDealProviderCollateralBounds(context.Context, abi.PaddedPieceSize, bool, types.TipSetKey) (DealCollateralBounds, error) - // StateCirculatingSupply returns the circulating supply of Filecoin at the given tipset - StateCirculatingSupply(context.Context, types.TipSetKey) (CirculatingSupply, error) + // StateCirculatingSupply returns the exact circulating supply of Filecoin at the given tipset. + // This is not used anywhere in the protocol itself, and is only for external consumption. + StateCirculatingSupply(context.Context, types.TipSetKey) (abi.TokenAmount, error) + // StateVMCirculatingSupply returns an approximation of the circulating supply of Filecoin at the given tipset. + // This is the value reported by the runtime interface to actors code. + StateVMCirculatingSupply(context.Context, types.TipSetKey) (CirculatingSupply, error) // StateNetworkVersion returns the network version at the given tipset StateNetworkVersion(context.Context, types.TipSetKey) (network.Version, error) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index cef335939..a938653ce 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -210,7 +210,8 @@ type FullNodeStruct struct { StateVerifiedClientStatus func(context.Context, address.Address, types.TipSetKey) (*abi.StoragePower, error) `perm:"read"` StateVerifiedRegistryRootKey func(ctx context.Context, tsk types.TipSetKey) (address.Address, error) `perm:"read"` StateDealProviderCollateralBounds func(context.Context, abi.PaddedPieceSize, bool, types.TipSetKey) (api.DealCollateralBounds, error) `perm:"read"` - StateCirculatingSupply func(context.Context, types.TipSetKey) (api.CirculatingSupply, error) `perm:"read"` + StateCirculatingSupply func(context.Context, types.TipSetKey) (abi.TokenAmount, error) `perm:"read"` + StateVMCirculatingSupply func(context.Context, types.TipSetKey) (api.CirculatingSupply, error) `perm:"read"` StateNetworkVersion func(context.Context, types.TipSetKey) (stnetwork.Version, error) `perm:"read"` MsigGetAvailableBalance func(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) `perm:"read"` @@ -963,10 +964,14 @@ func (c *FullNodeStruct) StateDealProviderCollateralBounds(ctx context.Context, return c.Internal.StateDealProviderCollateralBounds(ctx, size, verified, tsk) } -func (c *FullNodeStruct) StateCirculatingSupply(ctx context.Context, tsk types.TipSetKey) (api.CirculatingSupply, error) { +func (c *FullNodeStruct) StateCirculatingSupply(ctx context.Context, tsk types.TipSetKey) (abi.TokenAmount, error) { return c.Internal.StateCirculatingSupply(ctx, tsk) } +func (c *FullNodeStruct) StateVMCirculatingSupply(ctx context.Context, tsk types.TipSetKey) (api.CirculatingSupply, error) { + return c.Internal.StateVMCirculatingSupply(ctx, tsk) +} + func (c *FullNodeStruct) StateNetworkVersion(ctx context.Context, tsk types.TipSetKey) (stnetwork.Version, error) { return c.Internal.StateNetworkVersion(ctx, tsk) } diff --git a/chain/actors/builtin/builtin.go b/chain/actors/builtin/builtin.go index cb24a2c33..350b3a9a6 100644 --- a/chain/actors/builtin/builtin.go +++ b/chain/actors/builtin/builtin.go @@ -2,25 +2,28 @@ package builtin import ( "github.com/filecoin-project/go-address" + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + smoothing2 "github.com/filecoin-project/specs-actors/v2/actors/util/smoothing" "github.com/ipfs/go-cid" "golang.org/x/xerrors" + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/cbor" "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/types" - builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" proof0 "github.com/filecoin-project/specs-actors/actors/runtime/proof" smoothing0 "github.com/filecoin-project/specs-actors/actors/util/smoothing" - builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" - smoothing2 "github.com/filecoin-project/specs-actors/v2/actors/util/smoothing" ) var SystemActorAddr = builtin0.SystemActorAddr var BurntFundsActorAddr = builtin0.BurntFundsActorAddr +var CronActorAddr = builtin0.CronActorAddr +var SaftAddress = makeAddress("t0122") var ReserveAddress = makeAddress("t090") var RootVerifierAddress = makeAddress("t080") diff --git a/chain/actors/builtin/miner/miner.go b/chain/actors/builtin/miner/miner.go index 281068827..3cca39326 100644 --- a/chain/actors/builtin/miner/miner.go +++ b/chain/actors/builtin/miner/miner.go @@ -1,6 +1,7 @@ package miner import ( + "github.com/filecoin-project/go-state-types/big" "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p-core/peer" cbg "github.com/whyrusleeping/cbor-gen" @@ -196,3 +197,7 @@ type LockedFunds struct { InitialPledgeRequirement abi.TokenAmount PreCommitDeposits abi.TokenAmount } + +func (lf LockedFunds) TotalLockedFunds() abi.TokenAmount { + return big.Add(lf.VestingFunds, big.Add(lf.InitialPledgeRequirement, lf.PreCommitDeposits)) +} diff --git a/chain/actors/builtin/miner/v0.go b/chain/actors/builtin/miner/v0.go index 7e71c7611..14718d002 100644 --- a/chain/actors/builtin/miner/v0.go +++ b/chain/actors/builtin/miner/v0.go @@ -47,8 +47,16 @@ type partition0 struct { store adt.Store } -func (s *state0) AvailableBalance(bal abi.TokenAmount) (abi.TokenAmount, error) { - return s.GetAvailableBalance(bal), nil +func (s *state0) AvailableBalance(bal abi.TokenAmount) (available abi.TokenAmount, err error) { + defer func() { + if r := recover(); r != nil { + err = xerrors.Errorf("failed to get available balance: %w", r) + available = abi.NewTokenAmount(0) + } + }() + // this panics if the miner doesnt have enough funds to cover their locked pledge + available = s.GetAvailableBalance(bal) + return available, err } func (s *state0) VestedFunds(epoch abi.ChainEpoch) (abi.TokenAmount, error) { diff --git a/chain/actors/builtin/miner/v2.go b/chain/actors/builtin/miner/v2.go index eed82257f..9e599f891 100644 --- a/chain/actors/builtin/miner/v2.go +++ b/chain/actors/builtin/miner/v2.go @@ -45,8 +45,16 @@ type partition2 struct { store adt.Store } -func (s *state2) AvailableBalance(bal abi.TokenAmount) (abi.TokenAmount, error) { - return s.GetAvailableBalance(bal) +func (s *state2) AvailableBalance(bal abi.TokenAmount) (available abi.TokenAmount, err error) { + defer func() { + if r := recover(); r != nil { + err = xerrors.Errorf("failed to get available balance: %w", r) + available = abi.NewTokenAmount(0) + } + }() + // this panics if the miner doesnt have enough funds to cover their locked pledge + available, err = s.GetAvailableBalance(bal) + return available, err } func (s *state2) VestedFunds(epoch abi.ChainEpoch) (abi.TokenAmount, error) { diff --git a/chain/stmgr/call.go b/chain/stmgr/call.go index df3bfa357..8d0f6f6c1 100644 --- a/chain/stmgr/call.go +++ b/chain/stmgr/call.go @@ -61,7 +61,7 @@ func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types. Rand: store.NewChainRand(sm.cs, ts.Cids()), Bstore: sm.cs.Blockstore(), Syscalls: sm.cs.VMSys(), - CircSupplyCalc: sm.GetCirculatingSupply, + CircSupplyCalc: sm.GetVMCirculatingSupply, NtwkVersion: sm.GetNtwkVersion, BaseFee: types.NewInt(0), } @@ -174,7 +174,7 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri Rand: r, Bstore: sm.cs.Blockstore(), Syscalls: sm.cs.VMSys(), - CircSupplyCalc: sm.GetCirculatingSupply, + CircSupplyCalc: sm.GetVMCirculatingSupply, NtwkVersion: sm.GetNtwkVersion, BaseFee: ts.Blocks()[0].ParentBaseFee, } diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index 6ebe8a5b9..7f2b57546 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -265,11 +265,6 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCal return cid.Undef, xerrors.Errorf("loading state tree failed: %w", err) } - ReserveAddress, err := address.NewFromString("t090") - if err != nil { - return cid.Undef, xerrors.Errorf("failed to parse reserve address: %w", err) - } - tree, err := sm.StateTree(root) if err != nil { return cid.Undef, xerrors.Errorf("getting state tree: %w", err) @@ -295,7 +290,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCal if !sysAcc { transfers = append(transfers, transfer{ From: addr, - To: ReserveAddress, + To: builtin.ReserveAddress, Amt: act.Balance, }) } @@ -319,7 +314,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCal transfers = append(transfers, transfer{ From: addr, - To: ReserveAddress, + To: builtin.ReserveAddress, Amt: available, }) } @@ -370,7 +365,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCal nbalance := big.Min(prevBalance, AccountCap) if nbalance.Sign() != 0 { transfersBack = append(transfersBack, transfer{ - From: ReserveAddress, + From: builtin.ReserveAddress, To: addr, Amt: nbalance, }) @@ -397,7 +392,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCal mfunds := minerFundsAlloc(power, totalPower) transfersBack = append(transfersBack, transfer{ - From: ReserveAddress, + From: builtin.ReserveAddress, To: minfo.Worker, Amt: mfunds, }) @@ -417,7 +412,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCal if lbsectors.Length() > 0 { transfersBack = append(transfersBack, transfer{ - From: ReserveAddress, + From: builtin.ReserveAddress, To: minfo.Worker, Amt: BaseMinerBalance, }) @@ -444,7 +439,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCal if err != nil { return cid.Undef, xerrors.Errorf("failed to load burnt funds actor: %w", err) } - if err := doTransfer(cb, tree, builtin0.BurntFundsActorAddr, ReserveAddress, burntAct.Balance); err != nil { + if err := doTransfer(cb, tree, builtin0.BurntFundsActorAddr, builtin.ReserveAddress, burntAct.Balance); err != nil { return cid.Undef, xerrors.Errorf("failed to unburn funds: %w", err) } @@ -460,7 +455,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCal } difference := types.BigSub(DesiredReimbursementBalance, reimb.Balance) - if err := doTransfer(cb, tree, ReserveAddress, reimbAddr, difference); err != nil { + if err := doTransfer(cb, tree, builtin.ReserveAddress, reimbAddr, difference); err != nil { return cid.Undef, xerrors.Errorf("failed to top up reimbursement account: %w", err) } @@ -545,12 +540,7 @@ func UpgradeRefuel(ctx context.Context, sm *StateManager, cb ExecCallback, root return cid.Undef, xerrors.Errorf("getting state tree: %w", err) } - addr, err := address.NewFromString("t0122") - if err != nil { - return cid.Undef, xerrors.Errorf("getting address: %w", err) - } - - err = resetMultisigVesting(ctx, store, tree, addr, 0, 0, big.Zero()) + err = resetMultisigVesting(ctx, store, tree, builtin.SaftAddress, 0, 0, big.Zero()) if err != nil { return cid.Undef, xerrors.Errorf("tweaking msig vesting: %w", err) } diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index d6b6f4360..b470cfe83 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -7,6 +7,11 @@ import ( "sync" "github.com/filecoin-project/lotus/chain/actors/builtin" + "github.com/filecoin-project/lotus/chain/actors/builtin/verifreg" + + _init "github.com/filecoin-project/lotus/chain/actors/builtin/init" + + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" msig0 "github.com/filecoin-project/specs-actors/actors/builtin/multisig" @@ -229,7 +234,7 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp Rand: r, Bstore: sm.cs.Blockstore(), Syscalls: sm.cs.VMSys(), - CircSupplyCalc: sm.GetCirculatingSupply, + CircSupplyCalc: sm.GetVMCirculatingSupply, NtwkVersion: sm.GetNtwkVersion, BaseFee: baseFee, } @@ -1294,7 +1299,16 @@ func GetFilBurnt(ctx context.Context, st *state.StateTree) (abi.TokenAmount, err return burnt.Balance, nil } -func (sm *StateManager) GetCirculatingSupplyDetailed(ctx context.Context, height abi.ChainEpoch, st *state.StateTree) (api.CirculatingSupply, error) { +func (sm *StateManager) GetVMCirculatingSupply(ctx context.Context, height abi.ChainEpoch, st *state.StateTree) (abi.TokenAmount, error) { + cs, err := sm.GetVMCirculatingSupplyDetailed(ctx, height, st) + if err != nil { + return types.EmptyInt, err + } + + return cs.FilCirculating, err +} + +func (sm *StateManager) GetVMCirculatingSupplyDetailed(ctx context.Context, height abi.ChainEpoch, st *state.StateTree) (api.CirculatingSupply, error) { sm.genesisMsigLk.Lock() defer sm.genesisMsigLk.Unlock() if sm.preIgnitionGenInfos == nil { @@ -1357,12 +1371,91 @@ func (sm *StateManager) GetCirculatingSupplyDetailed(ctx context.Context, height } func (sm *StateManager) GetCirculatingSupply(ctx context.Context, height abi.ChainEpoch, st *state.StateTree) (abi.TokenAmount, error) { - csi, err := sm.GetCirculatingSupplyDetailed(ctx, height, st) + circ := big.Zero() + unCirc := big.Zero() + err := st.ForEach(func(a address.Address, actor *types.Actor) error { + switch { + case actor.Balance.IsZero(): + // Do nothing for zero-balance actors + break + case a == _init.Address || + a == reward.Address || + a == verifreg.Address || + // The power actor itself should never receive funds + a == power.Address || + a == builtin.SystemActorAddr || + a == builtin.CronActorAddr || + a == builtin.BurntFundsActorAddr || + a == builtin.SaftAddress || + a == builtin.ReserveAddress: + + unCirc = big.Add(unCirc, actor.Balance) + + case a == market.Address: + mst, err := market.Load(sm.cs.Store(ctx), actor) + if err != nil { + return err + } + + lb, err := mst.TotalLocked() + if err != nil { + return err + } + + circ = big.Add(circ, big.Sub(actor.Balance, lb)) + unCirc = big.Add(unCirc, lb) + + case builtin.IsAccountActor(actor.Code) || builtin.IsPaymentChannelActor(actor.Code): + circ = big.Add(circ, actor.Balance) + + case builtin.IsStorageMinerActor(actor.Code): + mst, err := miner.Load(sm.cs.Store(ctx), actor) + if err != nil { + return err + } + + ab, err := mst.AvailableBalance(actor.Balance) + + if err == nil { + circ = big.Add(circ, ab) + unCirc = big.Add(unCirc, big.Sub(actor.Balance, ab)) + } else { + // Assume any error is because the miner state is "broken" (lower actor balance than locked funds) + // In this case, the actor's entire balance is considered "uncirculating" + unCirc = big.Add(unCirc, actor.Balance) + } + + case builtin.IsMultisigActor(actor.Code): + mst, err := multisig.Load(sm.cs.Store(ctx), actor) + if err != nil { + return err + } + + lb, err := mst.LockedBalance(height) + if err != nil { + return err + } + + ab := big.Sub(actor.Balance, lb) + circ = big.Add(circ, big.Max(ab, big.Zero())) + unCirc = big.Add(unCirc, big.Min(actor.Balance, lb)) + default: + return xerrors.Errorf("unexpected actor: %s", a) + } + + return nil + }) + if err != nil { - return big.Zero(), err + return types.EmptyInt, err } - return csi.FilCirculating, nil + total := big.Add(circ, unCirc) + if !total.Equals(types.TotalFilecoinInt) { + return types.EmptyInt, xerrors.Errorf("total filecoin didn't add to expected amount: %s != %s", total, types.TotalFilecoinInt) + } + + return circ, nil } func (sm *StateManager) GetNtwkVersion(ctx context.Context, height abi.ChainEpoch) network.Version { diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index c0f0c4d2f..cbc78ca5b 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -383,7 +383,7 @@ func ComputeState(ctx context.Context, sm *StateManager, height abi.ChainEpoch, Rand: r, Bstore: sm.cs.Blockstore(), Syscalls: sm.cs.VMSys(), - CircSupplyCalc: sm.GetCirculatingSupply, + CircSupplyCalc: sm.GetVMCirculatingSupply, NtwkVersion: sm.GetNtwkVersion, BaseFee: ts.Blocks()[0].ParentBaseFee, } diff --git a/cli/state.go b/cli/state.go index 453cde77f..5acdf940f 100644 --- a/cli/state.go +++ b/cli/state.go @@ -1688,7 +1688,14 @@ func parseParamsForMethod(act cid.Cid, method uint64, args []string) ([]byte, er var stateCircSupplyCmd = &cli.Command{ Name: "circulating-supply", - Usage: "Get the current circulating supply of filecoin", + Usage: "Get the exact current circulating supply of Filecoin", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "vm-supply", + Usage: "calculates the approximation of the circulating supply used internally by the VM (instead of the exact amount)", + Value: false, + }, + }, Action: func(cctx *cli.Context) error { api, closer, err := GetFullNodeAPI(cctx) if err != nil { @@ -1703,16 +1710,26 @@ var stateCircSupplyCmd = &cli.Command{ return err } - circ, err := api.StateCirculatingSupply(ctx, ts.Key()) - if err != nil { - return err - } + if cctx.IsSet("vm-supply") { + circ, err := api.StateVMCirculatingSupply(ctx, ts.Key()) + if err != nil { + return err + } - fmt.Println("Circulating supply: ", types.FIL(circ.FilCirculating)) - fmt.Println("Mined: ", types.FIL(circ.FilMined)) - fmt.Println("Vested: ", types.FIL(circ.FilVested)) - fmt.Println("Burnt: ", types.FIL(circ.FilBurnt)) - fmt.Println("Locked: ", types.FIL(circ.FilLocked)) + fmt.Println("Circulating supply: ", types.FIL(circ.FilCirculating)) + fmt.Println("Mined: ", types.FIL(circ.FilMined)) + fmt.Println("Vested: ", types.FIL(circ.FilVested)) + fmt.Println("Burnt: ", types.FIL(circ.FilBurnt)) + fmt.Println("Locked: ", types.FIL(circ.FilLocked)) + } else { + circ, err := api.StateCirculatingSupply(ctx, ts.Key()) + if err != nil { + return err + } + + fmt.Println("Exact circulating supply: ", types.FIL(circ)) + return nil + } return nil }, diff --git a/cmd/lotus-chainwatch/syncer/sync.go b/cmd/lotus-chainwatch/syncer/sync.go index 609b71088..d0931a978 100644 --- a/cmd/lotus-chainwatch/syncer/sync.go +++ b/cmd/lotus-chainwatch/syncer/sync.go @@ -316,7 +316,7 @@ limit 1 } func (s *Syncer) storeCirculatingSupply(ctx context.Context, tipset *types.TipSet) error { - supply, err := s.node.StateCirculatingSupply(ctx, tipset.Key()) + supply, err := s.node.StateVMCirculatingSupply(ctx, tipset.Key()) if err != nil { return err } diff --git a/cmd/lotus-shed/balances.go b/cmd/lotus-shed/balances.go index c2099bb2d..dfdaf9fa9 100644 --- a/cmd/lotus-shed/balances.go +++ b/cmd/lotus-shed/balances.go @@ -372,7 +372,7 @@ var chainPledgeCmd = &cli.Command{ pledgeCollateral = c } - circ, err := sm.GetCirculatingSupplyDetailed(ctx, abi.ChainEpoch(epoch), state) + circ, err := sm.GetVMCirculatingSupplyDetailed(ctx, abi.ChainEpoch(epoch), state) if err != nil { return err } diff --git a/cmd/tvx/extract.go b/cmd/tvx/extract.go index b0ed574df..cc4494d0f 100644 --- a/cmd/tvx/extract.go +++ b/cmd/tvx/extract.go @@ -135,7 +135,7 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { } // get the circulating supply before the message was executed. - circSupplyDetail, err := fapi.StateCirculatingSupply(ctx, incTs.Key()) + circSupplyDetail, err := fapi.StateVMCirculatingSupply(ctx, incTs.Key()) if err != nil { return fmt.Errorf("failed while fetching circulating supply: %w", err) } diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index 50a6a160d..f5dfcf815 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -165,6 +165,7 @@ * [StateSectorGetInfo](#StateSectorGetInfo) * [StateSectorPartition](#StateSectorPartition) * [StateSectorPreCommitInfo](#StateSectorPreCommitInfo) + * [StateVMCirculatingSupply](#StateVMCirculatingSupply) * [StateVerifiedClientStatus](#StateVerifiedClientStatus) * [StateVerifiedRegistryRootKey](#StateVerifiedRegistryRootKey) * [StateVerifierStatus](#StateVerifierStatus) @@ -3094,7 +3095,8 @@ Response: ``` ### StateCirculatingSupply -StateCirculatingSupply returns the circulating supply of Filecoin at the given tipset +StateCirculatingSupply returns the exact circulating supply of Filecoin at the given tipset. +This is not used anywhere in the protocol itself, and is only for external consumption. Perms: read @@ -3113,16 +3115,7 @@ Inputs: ] ``` -Response: -```json -{ - "FilVested": "0", - "FilMined": "0", - "FilBurnt": "0", - "FilLocked": "0", - "FilCirculating": "0" -} -``` +Response: `"0"` ### StateCompute StateCompute is a flexible command that applies the given messages on the given tipset. @@ -4265,6 +4258,38 @@ Response: } ``` +### StateVMCirculatingSupply +StateVMCirculatingSupply returns an approximation of the circulating supply of Filecoin at the given tipset. +This is the value reported by the runtime interface to actors code. + + +Perms: read + +Inputs: +```json +[ + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "FilVested": "0", + "FilMined": "0", + "FilBurnt": "0", + "FilLocked": "0", + "FilCirculating": "0" +} +``` + ### StateVerifiedClientStatus StateVerifiedClientStatus returns the data cap for the given address. Returns nil if there is no entry in the data cap table for the diff --git a/node/impl/full/state.go b/node/impl/full/state.go index db91433aa..62636e4fa 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -1049,7 +1049,7 @@ func (a *StateAPI) StateMinerInitialPledgeCollateral(ctx context.Context, maddr return types.EmptyInt, xerrors.Errorf("loading reward actor state: %w", err) } - circSupply, err := a.StateCirculatingSupply(ctx, ts.Key()) + circSupply, err := a.StateVMCirculatingSupply(ctx, ts.Key()) if err != nil { return big.Zero(), xerrors.Errorf("getting circulating supply: %w", err) } @@ -1203,7 +1203,7 @@ func (a *StateAPI) StateDealProviderCollateralBounds(ctx context.Context, size a return api.DealCollateralBounds{}, xerrors.Errorf("failed to load reward actor state: %w", err) } - circ, err := a.StateCirculatingSupply(ctx, ts.Key()) + circ, err := a.StateVMCirculatingSupply(ctx, ts.Key()) if err != nil { return api.DealCollateralBounds{}, xerrors.Errorf("getting total circulating supply: %w", err) } @@ -1231,7 +1231,20 @@ func (a *StateAPI) StateDealProviderCollateralBounds(ctx context.Context, size a }, nil } -func (a *StateAPI) StateCirculatingSupply(ctx context.Context, tsk types.TipSetKey) (api.CirculatingSupply, error) { +func (a *StateAPI) StateCirculatingSupply(ctx context.Context, tsk types.TipSetKey) (abi.TokenAmount, error) { + ts, err := a.Chain.GetTipSetFromKey(tsk) + if err != nil { + return types.EmptyInt, xerrors.Errorf("loading tipset %s: %w", tsk, err) + } + + sTree, err := a.stateForTs(ctx, ts) + if err != nil { + return types.EmptyInt, err + } + return a.StateManager.GetCirculatingSupply(ctx, ts.Height(), sTree) +} + +func (a *StateAPI) StateVMCirculatingSupply(ctx context.Context, tsk types.TipSetKey) (api.CirculatingSupply, error) { ts, err := a.Chain.GetTipSetFromKey(tsk) if err != nil { return api.CirculatingSupply{}, xerrors.Errorf("loading tipset %s: %w", tsk, err) @@ -1241,7 +1254,7 @@ func (a *StateAPI) StateCirculatingSupply(ctx context.Context, tsk types.TipSetK if err != nil { return api.CirculatingSupply{}, err } - return a.StateManager.GetCirculatingSupplyDetailed(ctx, ts.Height(), sTree) + return a.StateManager.GetVMCirculatingSupplyDetailed(ctx, ts.Height(), sTree) } func (a *StateAPI) StateNetworkVersion(ctx context.Context, tsk types.TipSetKey) (network.Version, error) { From 83624a8858d1983a28e0cd523c5ec353a5bbdaee Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Mon, 12 Oct 2020 16:41:27 -0400 Subject: [PATCH 727/885] Rename StateVMCirculatingSupply to StateVMCirculatingSupplyInternal --- api/api_full.go | 4 ++-- api/apistruct/struct.go | 6 +++--- cli/state.go | 2 +- cmd/lotus-chainwatch/syncer/sync.go | 2 +- cmd/tvx/extract.go | 2 +- documentation/en/api-methods.md | 6 +++--- node/impl/full/state.go | 6 +++--- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index b7cdf9524..657e945ca 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -419,9 +419,9 @@ type FullNode interface { // StateCirculatingSupply returns the exact circulating supply of Filecoin at the given tipset. // This is not used anywhere in the protocol itself, and is only for external consumption. StateCirculatingSupply(context.Context, types.TipSetKey) (abi.TokenAmount, error) - // StateVMCirculatingSupply returns an approximation of the circulating supply of Filecoin at the given tipset. + // StateVMCirculatingSupplyInternal returns an approximation of the circulating supply of Filecoin at the given tipset. // This is the value reported by the runtime interface to actors code. - StateVMCirculatingSupply(context.Context, types.TipSetKey) (CirculatingSupply, error) + StateVMCirculatingSupplyInternal(context.Context, types.TipSetKey) (CirculatingSupply, error) // StateNetworkVersion returns the network version at the given tipset StateNetworkVersion(context.Context, types.TipSetKey) (network.Version, error) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index a938653ce..7896d566e 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -211,7 +211,7 @@ type FullNodeStruct struct { StateVerifiedRegistryRootKey func(ctx context.Context, tsk types.TipSetKey) (address.Address, error) `perm:"read"` StateDealProviderCollateralBounds func(context.Context, abi.PaddedPieceSize, bool, types.TipSetKey) (api.DealCollateralBounds, error) `perm:"read"` StateCirculatingSupply func(context.Context, types.TipSetKey) (abi.TokenAmount, error) `perm:"read"` - StateVMCirculatingSupply func(context.Context, types.TipSetKey) (api.CirculatingSupply, error) `perm:"read"` + StateVMCirculatingSupplyInternal func(context.Context, types.TipSetKey) (api.CirculatingSupply, error) `perm:"read"` StateNetworkVersion func(context.Context, types.TipSetKey) (stnetwork.Version, error) `perm:"read"` MsigGetAvailableBalance func(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) `perm:"read"` @@ -968,8 +968,8 @@ func (c *FullNodeStruct) StateCirculatingSupply(ctx context.Context, tsk types.T return c.Internal.StateCirculatingSupply(ctx, tsk) } -func (c *FullNodeStruct) StateVMCirculatingSupply(ctx context.Context, tsk types.TipSetKey) (api.CirculatingSupply, error) { - return c.Internal.StateVMCirculatingSupply(ctx, tsk) +func (c *FullNodeStruct) StateVMCirculatingSupplyInternal(ctx context.Context, tsk types.TipSetKey) (api.CirculatingSupply, error) { + return c.Internal.StateVMCirculatingSupplyInternal(ctx, tsk) } func (c *FullNodeStruct) StateNetworkVersion(ctx context.Context, tsk types.TipSetKey) (stnetwork.Version, error) { diff --git a/cli/state.go b/cli/state.go index 5acdf940f..1e7e24ddb 100644 --- a/cli/state.go +++ b/cli/state.go @@ -1711,7 +1711,7 @@ var stateCircSupplyCmd = &cli.Command{ } if cctx.IsSet("vm-supply") { - circ, err := api.StateVMCirculatingSupply(ctx, ts.Key()) + circ, err := api.StateVMCirculatingSupplyInternal(ctx, ts.Key()) if err != nil { return err } diff --git a/cmd/lotus-chainwatch/syncer/sync.go b/cmd/lotus-chainwatch/syncer/sync.go index d0931a978..37af9cce0 100644 --- a/cmd/lotus-chainwatch/syncer/sync.go +++ b/cmd/lotus-chainwatch/syncer/sync.go @@ -316,7 +316,7 @@ limit 1 } func (s *Syncer) storeCirculatingSupply(ctx context.Context, tipset *types.TipSet) error { - supply, err := s.node.StateVMCirculatingSupply(ctx, tipset.Key()) + supply, err := s.node.StateVMCirculatingSupplyInternal(ctx, tipset.Key()) if err != nil { return err } diff --git a/cmd/tvx/extract.go b/cmd/tvx/extract.go index cc4494d0f..4bb391af6 100644 --- a/cmd/tvx/extract.go +++ b/cmd/tvx/extract.go @@ -135,7 +135,7 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { } // get the circulating supply before the message was executed. - circSupplyDetail, err := fapi.StateVMCirculatingSupply(ctx, incTs.Key()) + circSupplyDetail, err := fapi.StateVMCirculatingSupplyInternal(ctx, incTs.Key()) if err != nil { return fmt.Errorf("failed while fetching circulating supply: %w", err) } diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index f5dfcf815..cbec60c10 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -165,7 +165,7 @@ * [StateSectorGetInfo](#StateSectorGetInfo) * [StateSectorPartition](#StateSectorPartition) * [StateSectorPreCommitInfo](#StateSectorPreCommitInfo) - * [StateVMCirculatingSupply](#StateVMCirculatingSupply) + * [StateVMCirculatingSupplyInternal](#StateVMCirculatingSupplyInternal) * [StateVerifiedClientStatus](#StateVerifiedClientStatus) * [StateVerifiedRegistryRootKey](#StateVerifiedRegistryRootKey) * [StateVerifierStatus](#StateVerifierStatus) @@ -4258,8 +4258,8 @@ Response: } ``` -### StateVMCirculatingSupply -StateVMCirculatingSupply returns an approximation of the circulating supply of Filecoin at the given tipset. +### StateVMCirculatingSupplyInternal +StateVMCirculatingSupplyInternal returns an approximation of the circulating supply of Filecoin at the given tipset. This is the value reported by the runtime interface to actors code. diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 62636e4fa..1c39b59da 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -1049,7 +1049,7 @@ func (a *StateAPI) StateMinerInitialPledgeCollateral(ctx context.Context, maddr return types.EmptyInt, xerrors.Errorf("loading reward actor state: %w", err) } - circSupply, err := a.StateVMCirculatingSupply(ctx, ts.Key()) + circSupply, err := a.StateVMCirculatingSupplyInternal(ctx, ts.Key()) if err != nil { return big.Zero(), xerrors.Errorf("getting circulating supply: %w", err) } @@ -1203,7 +1203,7 @@ func (a *StateAPI) StateDealProviderCollateralBounds(ctx context.Context, size a return api.DealCollateralBounds{}, xerrors.Errorf("failed to load reward actor state: %w", err) } - circ, err := a.StateVMCirculatingSupply(ctx, ts.Key()) + circ, err := a.StateVMCirculatingSupplyInternal(ctx, ts.Key()) if err != nil { return api.DealCollateralBounds{}, xerrors.Errorf("getting total circulating supply: %w", err) } @@ -1244,7 +1244,7 @@ func (a *StateAPI) StateCirculatingSupply(ctx context.Context, tsk types.TipSetK return a.StateManager.GetCirculatingSupply(ctx, ts.Height(), sTree) } -func (a *StateAPI) StateVMCirculatingSupply(ctx context.Context, tsk types.TipSetKey) (api.CirculatingSupply, error) { +func (a *StateAPI) StateVMCirculatingSupplyInternal(ctx context.Context, tsk types.TipSetKey) (api.CirculatingSupply, error) { ts, err := a.Chain.GetTipSetFromKey(tsk) if err != nil { return api.CirculatingSupply{}, xerrors.Errorf("loading tipset %s: %w", tsk, err) From 49155ace62ab22ea13202ce60c755ad266d04581 Mon Sep 17 00:00:00 2001 From: Ingar Shu Date: Mon, 12 Oct 2020 13:45:37 -0700 Subject: [PATCH 728/885] Use seal-duration in calculating the earliest StartEpoch that we will consider (#4337) --- node/modules/storageminer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 9011c4821..4dbce4cdb 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -473,7 +473,7 @@ func BasicDealFilter(user dtypes.DealFilter) func(onlineOk dtypes.ConsiderOnline // Reject if it's more than 7 days in the future // TODO: read from cfg - maxStartEpoch := ht + abi.ChainEpoch(7*builtin.EpochsInDay) + maxStartEpoch := earliest + abi.ChainEpoch(7*builtin.EpochsInDay) if deal.Proposal.StartEpoch > maxStartEpoch { return false, fmt.Sprintf("deal start epoch is too far in the future: %s > %s", deal.Proposal.StartEpoch, maxStartEpoch), nil } From be74a137365bc250b592718aa2a7436e1c25eae0 Mon Sep 17 00:00:00 2001 From: Travis Person Date: Mon, 12 Oct 2020 21:41:48 +0000 Subject: [PATCH 729/885] lotus-pcr: ignore all other market messages --- cmd/lotus-pcr/main.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/lotus-pcr/main.go b/cmd/lotus-pcr/main.go index 5491e4af2..f88a0a9dc 100644 --- a/cmd/lotus-pcr/main.go +++ b/cmd/lotus-pcr/main.go @@ -884,6 +884,8 @@ func (r *refunder) processTipsetStorageMarketActor(ctx context.Context, tipset * } refundValue = types.BigMul(types.NewInt(uint64(recp.GasUsed)), tipset.Blocks()[0].ParentBaseFee) + default: + return false, messageMethod, types.NewInt(0), nil } return true, messageMethod, refundValue, nil From 9b4cac961229ca4a9396a10862c9a80470c67466 Mon Sep 17 00:00:00 2001 From: Lucas Molas Date: Mon, 12 Oct 2020 21:14:16 -0300 Subject: [PATCH 730/885] fix(sync state): set state height to actual tipset height --- chain/sync.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chain/sync.go b/chain/sync.go index 9040e3f05..dd83bf319 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -1478,7 +1478,7 @@ func (syncer *Syncer) syncFork(ctx context.Context, incoming *types.TipSet, know func (syncer *Syncer) syncMessagesAndCheckState(ctx context.Context, headers []*types.TipSet) error { ss := extractSyncState(ctx) - ss.SetHeight(0) + ss.SetHeight(headers[len(headers)-1].Height()) return syncer.iterFullTipsets(ctx, headers, func(ctx context.Context, fts *store.FullTipSet) error { log.Debugw("validating tipset", "height", fts.TipSet().Height(), "size", len(fts.TipSet().Cids())) From 07d0b6762812881fc85295aa5a1b45b14cf063ac Mon Sep 17 00:00:00 2001 From: zgfzgf <1901989065@qq.com> Date: Tue, 13 Oct 2020 10:46:41 +0800 Subject: [PATCH 731/885] optimize code use ts.Cids --- chain/stmgr/stmgr.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index b470cfe83..b381b2d59 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -435,12 +435,7 @@ func (sm *StateManager) computeTipSetState(ctx context.Context, ts *types.TipSet parentEpoch = parent.Height } - cids := make([]cid.Cid, len(blks)) - for i, v := range blks { - cids[i] = v.Cid() - } - - r := store.NewChainRand(sm.cs, cids) + r := store.NewChainRand(sm.cs, ts.Cids()) blkmsgs, err := sm.cs.BlockMsgsForTipset(ts) if err != nil { From 4edebcec2b9e5c13b69743d812db604bd05398da Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Tue, 13 Oct 2020 03:37:00 -0700 Subject: [PATCH 732/885] feat(markets): update markets 0.9.0 and add data transfer restart command --- api/api_full.go | 3 ++ api/apistruct/struct.go | 6 ++++ cli/client.go | 62 +++++++++++++++++++++++++++++++++ documentation/en/api-methods.md | 18 ++++++++++ go.mod | 6 ++-- go.sum | 16 ++++----- node/impl/client/client.go | 10 +++++- node/modules/client.go | 1 + node/modules/storageminer.go | 1 + 9 files changed, 110 insertions(+), 13 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 657e945ca..5b1316730 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -5,6 +5,7 @@ import ( "fmt" "time" + datatransfer "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/go-state-types/network" "github.com/ipfs/go-cid" @@ -298,6 +299,8 @@ type FullNode interface { // ClientListTransfers returns the status of all ongoing transfers of data ClientListDataTransfers(ctx context.Context) ([]DataTransferChannel, error) ClientDataTransferUpdates(ctx context.Context) (<-chan DataTransferChannel, error) + // ClientRestartDataTransfer attempts to restart a data transfer with the given transfer ID and other peer + ClientRestartDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error // ClientRetrieveTryRestartInsufficientFunds attempts to restart stalled retrievals on a given payment channel // which are stuck due to insufficient funds ClientRetrieveTryRestartInsufficientFunds(ctx context.Context, paymentChannel address.Address) error diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 7896d566e..7723e0b1f 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -15,6 +15,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-bitfield" + datatransfer "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/go-fil-markets/piecestore" "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/go-fil-markets/storagemarket" @@ -165,6 +166,7 @@ type FullNodeStruct struct { ClientDealSize func(ctx context.Context, root cid.Cid) (api.DataSize, error) `perm:"read"` ClientListDataTransfers func(ctx context.Context) ([]api.DataTransferChannel, error) `perm:"write"` ClientDataTransferUpdates func(ctx context.Context) (<-chan api.DataTransferChannel, error) `perm:"write"` + ClientRestartDataTransfer func(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error `perm:"write"` ClientRetrieveTryRestartInsufficientFunds func(ctx context.Context, paymentChannel address.Address) error `perm:"write"` StateNetworkName func(context.Context) (dtypes.NetworkName, error) `perm:"read"` @@ -552,6 +554,10 @@ func (c *FullNodeStruct) ClientDataTransferUpdates(ctx context.Context) (<-chan return c.Internal.ClientDataTransferUpdates(ctx) } +func (c *FullNodeStruct) ClientRestartDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error { + return c.Internal.ClientRestartDataTransfer(ctx, transferID, otherPeer, isInitiator) +} + func (c *FullNodeStruct) ClientRetrieveTryRestartInsufficientFunds(ctx context.Context, paymentChannel address.Address) error { return c.Internal.ClientRetrieveTryRestartInsufficientFunds(ctx, paymentChannel) } diff --git a/cli/client.go b/cli/client.go index 34d151ace..7035fb64b 100644 --- a/cli/client.go +++ b/cli/client.go @@ -3,6 +3,7 @@ package cli import ( "context" "encoding/json" + "errors" "fmt" "io" "os" @@ -82,6 +83,7 @@ var clientCmd = &cli.Command{ WithCategory("util", clientCarGenCmd), WithCategory("util", clientInfoCmd), WithCategory("util", clientListTransfers), + WithCategory("util", clientRestartTransfer), }, } @@ -1326,6 +1328,66 @@ var clientInfoCmd = &cli.Command{ }, } +var clientRestartTransfer = &cli.Command{ + Name: "restart-transfer", + Usage: "Force restart a stalled data transfer", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "peerid", + Usage: "narrow to transfer with specific peer", + }, + &cli.BoolFlag{ + Name: "initiator", + Usage: "specify only transfers where peer is/is not initiator", + Value: true, + }, + }, + Action: func(cctx *cli.Context) error { + if !cctx.Args().Present() { + return cli.ShowCommandHelp(cctx, cctx.Command.Name) + } + api, closer, err := GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + ctx := ReqContext(cctx) + + transferUint, err := strconv.ParseUint(cctx.Args().First(), 10, 64) + if err != nil { + return fmt.Errorf("Error reading transfer ID: %w", err) + } + transferID := datatransfer.TransferID(transferUint) + initiator := cctx.Bool("initiator") + var other peer.ID + if pidstr := cctx.String("peerid"); pidstr != "" { + p, err := peer.Decode(pidstr) + if err != nil { + return err + } + other = p + } else { + channels, err := api.ClientListDataTransfers(ctx) + if err != nil { + return err + } + found := false + for _, channel := range channels { + if channel.IsInitiator == initiator && channel.TransferID == transferID { + other = channel.OtherPeer + found = true + break + } + } + if !found { + return errors.New("unable to find matching data transfer") + } + } + + return api.ClientRestartDataTransfer(ctx, transferID, other, initiator) + }, +} + var clientListTransfers = &cli.Command{ Name: "list-transfers", Usage: "List ongoing data transfers for deals", diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index cbec60c10..c5d612fbe 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -46,6 +46,7 @@ * [ClientMinerQueryOffer](#ClientMinerQueryOffer) * [ClientQueryAsk](#ClientQueryAsk) * [ClientRemoveImport](#ClientRemoveImport) + * [ClientRestartDataTransfer](#ClientRestartDataTransfer) * [ClientRetrieve](#ClientRetrieve) * [ClientRetrieveTryRestartInsufficientFunds](#ClientRetrieveTryRestartInsufficientFunds) * [ClientRetrieveWithEvents](#ClientRetrieveWithEvents) @@ -1161,6 +1162,23 @@ Inputs: Response: `{}` +### ClientRestartDataTransfer +ClientRestartDataTransfer attempts to restart a data transfer with the given transfer ID and other peer + + +Perms: write + +Inputs: +```json +[ + 3, + "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf", + true +] +``` + +Response: `{}` + ### ClientRetrieve ClientRetrieve initiates the retrieval of a file, as specified in the order. diff --git a/go.mod b/go.mod index adff07d32..75926117a 100644 --- a/go.mod +++ b/go.mod @@ -27,9 +27,9 @@ require ( github.com/filecoin-project/go-bitfield v0.2.1 github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 - github.com/filecoin-project/go-data-transfer v0.6.7 + github.com/filecoin-project/go-data-transfer v0.6.8-0.20201013092017-eee8d1cde5d5 github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f - github.com/filecoin-project/go-fil-markets v0.7.1 + github.com/filecoin-project/go-fil-markets v0.7.2-0.20201013095312-bbda6c6fd17a github.com/filecoin-project/go-jsonrpc v0.1.2-0.20201008195726-68c6a2704e49 github.com/filecoin-project/go-multistore v0.0.3 github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 @@ -66,7 +66,7 @@ require ( github.com/ipfs/go-ds-pebble v0.0.2-0.20200921225637-ce220f8ac459 github.com/ipfs/go-filestore v1.0.0 github.com/ipfs/go-fs-lock v0.0.6 - github.com/ipfs/go-graphsync v0.2.1 + github.com/ipfs/go-graphsync v0.2.1-0.20201013053840-5d8ea8076a2c github.com/ipfs/go-ipfs-blockstore v1.0.1 github.com/ipfs/go-ipfs-chunker v0.0.5 github.com/ipfs/go-ipfs-ds-help v1.0.0 diff --git a/go.sum b/go.sum index 576690c10..50e85dfd5 100644 --- a/go.sum +++ b/go.sum @@ -240,14 +240,14 @@ github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:a github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= -github.com/filecoin-project/go-data-transfer v0.6.7 h1:Kacr5qz2YWtd3sensU6aXFtES7joeapVDeXApeUD35I= -github.com/filecoin-project/go-data-transfer v0.6.7/go.mod h1:C++k1U6+jMQODOaen5OPDo9XQbth9Yq3ie94vNjBJbk= +github.com/filecoin-project/go-data-transfer v0.6.8-0.20201013092017-eee8d1cde5d5 h1:kgmVHDP+rqcPqytpC215MzRRXrW9FqTfn0VFWhB2oog= +github.com/filecoin-project/go-data-transfer v0.6.8-0.20201013092017-eee8d1cde5d5/go.mod h1:1zaap881YZLMeIxhvLM0zA9qfUkh3bjOLSRaOoO9WRM= github.com/filecoin-project/go-ds-versioning v0.1.0 h1:y/X6UksYTsK8TLCI7rttCKEvl8btmWxyFMEeeWGUxIQ= github.com/filecoin-project/go-ds-versioning v0.1.0/go.mod h1:mp16rb4i2QPmxBnmanUx8i/XANp+PFCCJWiAb+VW4/s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f h1:GxJzR3oRIMTPtpZ0b7QF8FKPK6/iPAc7trhlL5k/g+s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= -github.com/filecoin-project/go-fil-markets v0.7.1 h1:e0NlpSnaeGyDUhCOzevjcxkSA54kt9BzlXpLRgduUFI= -github.com/filecoin-project/go-fil-markets v0.7.1/go.mod h1:5Pt4DXQqUoUrp9QzlSdlYTpItXxwAtqKrxRWQ6hAOqk= +github.com/filecoin-project/go-fil-markets v0.7.2-0.20201013095312-bbda6c6fd17a h1:jC3yO5R0+jGnVAHVqIHXb3BqCN/crslxTALz4I5gw1o= +github.com/filecoin-project/go-fil-markets v0.7.2-0.20201013095312-bbda6c6fd17a/go.mod h1:HqbtLIwJ4Lonp0BWq4ajmdXZ2UTh2+YK3wZSXXAE8yA= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24= @@ -264,12 +264,10 @@ github.com/filecoin-project/go-paramfetch v0.0.2-0.20200701152213-3e0f0afdc261 h github.com/filecoin-project/go-paramfetch v0.0.2-0.20200701152213-3e0f0afdc261/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc= github.com/filecoin-project/go-state-types v0.0.0-20200903145444-247639ffa6ad/go.mod h1:IQ0MBPnonv35CJHtWSN3YY1Hz2gkPru1Q9qoaYLxx9I= github.com/filecoin-project/go-state-types v0.0.0-20200904021452-1883f36ca2f4/go.mod h1:IQ0MBPnonv35CJHtWSN3YY1Hz2gkPru1Q9qoaYLxx9I= -github.com/filecoin-project/go-state-types v0.0.0-20200905071437-95828685f9df/go.mod h1:IQ0MBPnonv35CJHtWSN3YY1Hz2gkPru1Q9qoaYLxx9I= github.com/filecoin-project/go-state-types v0.0.0-20200928172055-2df22083d8ab h1:cEDC5Ei8UuT99hPWhCjA72SM9AuRtnpvdSTIYbnzN8I= github.com/filecoin-project/go-state-types v0.0.0-20200928172055-2df22083d8ab/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-state-types v0.0.0-20201003010437-c33112184a2b h1:bMUfG6Sy6YSMbsjQAO1Q2vEZldbSdsbRy/FX3OlTck0= github.com/filecoin-project/go-state-types v0.0.0-20201003010437-c33112184a2b/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= -github.com/filecoin-project/go-statemachine v0.0.0-20200714194326-a77c3ae20989/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe h1:dF8u+LEWeIcTcfUcCf3WFVlc81Fr2JKg8zPzIbBDKDw= github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIiWBRilQjQ+5IiwdQ= @@ -277,9 +275,9 @@ github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZO github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b h1:fkRZSPrYpk42PV3/lIXiL0LHetxde7vyYYvSsttQtfg= github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/go.mod h1:Q0GQOBtKf1oE10eSXSlhN45kDBdGvEcVOqMiffqX+N8= github.com/filecoin-project/specs-actors v0.9.4/go.mod h1:BStZQzx5x7TmCkLv0Bpa07U6cPKol6fd3w9KjMPZ6Z4= -github.com/filecoin-project/specs-actors v0.9.7/go.mod h1:wM2z+kwqYgXn5Z7scV1YHLyd1Q1cy0R8HfTIWQ0BFGU= github.com/filecoin-project/specs-actors v0.9.12 h1:iIvk58tuMtmloFNHhAOQHG+4Gci6Lui0n7DYQGi3cJk= github.com/filecoin-project/specs-actors v0.9.12/go.mod h1:TS1AW/7LbG+615j4NsjMK1qlpAwaFsG9w0V2tg2gSao= +github.com/filecoin-project/specs-actors/v2 v2.0.1/go.mod h1:v2NZVYinNIKA9acEMBm5wWXxqv5+frFEbekBFemYghY= github.com/filecoin-project/specs-actors/v2 v2.1.0 h1:ocEuGz8DG2cUWw32c/tvF8D6xT+dGVWJTr5yDevU00g= github.com/filecoin-project/specs-actors/v2 v2.1.0/go.mod h1:E7fAX4CZkDVQvDNRCxfq+hc3nx56KcCKyuZf0hlQJ20= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk= @@ -540,8 +538,8 @@ github.com/ipfs/go-filestore v1.0.0/go.mod h1:/XOCuNtIe2f1YPbiXdYvD0BKLA0JR1MgPi github.com/ipfs/go-fs-lock v0.0.6 h1:sn3TWwNVQqSeNjlWy6zQ1uUGAZrV3hPOyEA6y1/N2a0= github.com/ipfs/go-fs-lock v0.0.6/go.mod h1:OTR+Rj9sHiRubJh3dRhD15Juhd/+w6VPOY28L7zESmM= github.com/ipfs/go-graphsync v0.1.0/go.mod h1:jMXfqIEDFukLPZHqDPp8tJMbHO9Rmeb9CEGevngQbmE= -github.com/ipfs/go-graphsync v0.2.1 h1:MdehhqBSuTI2LARfKLkpYnt0mUrqHs/mtuDnESXHBfU= -github.com/ipfs/go-graphsync v0.2.1/go.mod h1:gEBvJUNelzMkaRPJTpg/jaKN4AQW/7wDWu0K92D8o10= +github.com/ipfs/go-graphsync v0.2.1-0.20201013053840-5d8ea8076a2c h1:De/AZGvRa3WMyw5zdMMhcvRcho46BVo+C0NRud+T4io= +github.com/ipfs/go-graphsync v0.2.1-0.20201013053840-5d8ea8076a2c/go.mod h1:gEBvJUNelzMkaRPJTpg/jaKN4AQW/7wDWu0K92D8o10= github.com/ipfs/go-hamt-ipld v0.1.1 h1:0IQdvwnAAUKmDE+PMJa5y1QiwOPHpI9+eAbQEEEYthk= github.com/ipfs/go-hamt-ipld v0.1.1/go.mod h1:1EZCr2v0jlCnhpa+aZ0JZYp8Tt2w16+JJOAVz17YcDk= github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= diff --git a/node/impl/client/client.go b/node/impl/client/client.go index fe83fda7b..f04d310ce 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -8,7 +8,6 @@ import ( "github.com/filecoin-project/go-state-types/dline" - datatransfer "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/go-state-types/big" "golang.org/x/xerrors" @@ -33,6 +32,7 @@ import ( "go.uber.org/fx" "github.com/filecoin-project/go-address" + datatransfer "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/go-fil-markets/discovery" "github.com/filecoin-project/go-fil-markets/pieceio" "github.com/filecoin-project/go-fil-markets/retrievalmarket" @@ -850,6 +850,14 @@ func (a *API) ClientDataTransferUpdates(ctx context.Context) (<-chan api.DataTra return channels, nil } +func (a *API) ClientRestartDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error { + selfPeer := a.Host.ID() + if isInitiator { + return a.DataTransfer.RestartDataTransferChannel(ctx, datatransfer.ChannelID{Initiator: selfPeer, Responder: otherPeer, ID: transferID}) + } + return a.DataTransfer.RestartDataTransferChannel(ctx, datatransfer.ChannelID{Initiator: otherPeer, Responder: selfPeer, ID: transferID}) +} + func newDealInfo(v storagemarket.ClientDeal) api.DealInfo { return api.DealInfo{ ProposalCid: v.ProposalCid, diff --git a/node/modules/client.go b/node/modules/client.go index d012e4539..eab309445 100644 --- a/node/modules/client.go +++ b/node/modules/client.go @@ -91,6 +91,7 @@ func NewClientGraphsyncDataTransfer(lc fx.Lifecycle, h host.Host, gs dtypes.Grap return nil, err } + dt.OnReady(marketevents.ReadyLogger("provider data transfer")) lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { return dt.Start(ctx) diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 4dbce4cdb..613203a60 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -267,6 +267,7 @@ func NewProviderDAGServiceDataTransfer(lc fx.Lifecycle, h host.Host, gs dtypes.S return nil, err } + dt.OnReady(marketevents.ReadyLogger("provider data transfer")) lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { return dt.Start(ctx) From 1eb2d0ae5059604be9d878cab370e71eeba288ea Mon Sep 17 00:00:00 2001 From: zgfzgf <1901989065@qq.com> Date: Tue, 13 Oct 2020 22:34:07 +0800 Subject: [PATCH 733/885] optimize map use struct{} --- chain/stmgr/stmgr.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index b470cfe83..e3bd5f89d 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -313,7 +313,7 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp } var receipts []cbg.CBORMarshaler - processedMsgs := map[cid.Cid]bool{} + processedMsgs := make(map[cid.Cid]struct{}) for _, b := range bms { penalty := types.NewInt(0) gasReward := big.Zero() @@ -337,7 +337,7 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp return cid.Undef, cid.Undef, err } } - processedMsgs[m.Cid()] = true + processedMsgs[m.Cid()] = struct{}{} } params, err := actors.SerializeParams(&reward.AwardBlockRewardParams{ From 87cd8c672571ec22ae36b9eef1a42557e8546436 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Tue, 13 Oct 2020 17:48:54 +0200 Subject: [PATCH 734/885] Fix off by one tipset in searchBackForMsg This way we will return the tipset the message was executed in Signed-off-by: Jakub Sztandera --- chain/stmgr/stmgr.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index b470cfe83..0cb17fcf3 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -738,7 +738,7 @@ func (sm *StateManager) searchBackForMsg(ctx context.Context, from *types.TipSet } if r != nil { - return pts, r, foundMsg, nil + return cur, r, foundMsg, nil } } From 8987defb9d8ba19ecc9bc6cb675bb3740cbb6923 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Tue, 13 Oct 2020 19:20:11 +0200 Subject: [PATCH 735/885] Fix BLS message ChainLength add more detail to ValidForBlockInclusion Signed-off-by: Jakub Sztandera --- chain/types/message.go | 2 +- chain/types/signedmessage.go | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/chain/types/message.go b/chain/types/message.go index cbc7c1dad..c53ecc7c1 100644 --- a/chain/types/message.go +++ b/chain/types/message.go @@ -195,7 +195,7 @@ func (m *Message) ValidForBlockInclusion(minGas int64) error { // since prices might vary with time, this is technically semantic validation if m.GasLimit < minGas { - return xerrors.New("'GasLimit' field cannot be less than the cost of storing a message on chain") + return xerrors.Errorf("'GasLimit' field cannot be less than the cost of storing a message on chain %d < %d", m.GasLimit, minGas) } return nil diff --git a/chain/types/signedmessage.go b/chain/types/signedmessage.go index 7532bea35..c539ac240 100644 --- a/chain/types/signedmessage.go +++ b/chain/types/signedmessage.go @@ -78,7 +78,14 @@ func (sm *SignedMessage) MarshalJSON() ([]byte, error) { } func (sm *SignedMessage) ChainLength() int { - ser, err := sm.Serialize() + var ser []byte + var err error + if sm.Signature.Type == crypto.SigTypeBLS { + // BLS chain message length doesn't include signature + ser, err = sm.Message.Serialize() + } else { + ser, err = sm.Serialize() + } if err != nil { panic(err) } From ce548c8f95aa7f3459ec58b8df27902579b6ef4e Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Tue, 13 Oct 2020 18:25:49 +0200 Subject: [PATCH 736/885] Add test Signed-off-by: Jakub Sztandera --- api/test/test.go | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/api/test/test.go b/api/test/test.go index 947f2bef4..4074ce4a6 100644 --- a/api/test/test.go +++ b/api/test/test.go @@ -3,8 +3,10 @@ package test import ( "context" "testing" + "time" "github.com/filecoin-project/lotus/chain/stmgr" + "github.com/filecoin-project/lotus/chain/types" "github.com/multiformats/go-multiaddr" @@ -12,6 +14,7 @@ import ( "github.com/stretchr/testify/require" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" @@ -75,6 +78,7 @@ func TestApis(t *testing.T, b APIBuilder) { t.Run("testConnectTwo", ts.testConnectTwo) t.Run("testMining", ts.testMining) t.Run("testMiningReal", ts.testMiningReal) + t.Run("testSearchMsg", ts.testSearchMsg) } func DefaultFullOpts(nFull int) []FullNodeOpts { @@ -120,6 +124,49 @@ func (ts *testSuite) testVersion(t *testing.T) { require.Equal(t, v.Version, build.BuildVersion) } +func (ts *testSuite) testSearchMsg(t *testing.T) { + apis, miners := ts.makeNodes(t, OneFull, OneMiner) + + api := apis[0] + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + senderAddr, err := api.WalletDefaultAddress(ctx) + if err != nil { + t.Fatal(err) + } + + msg := &types.Message{ + From: senderAddr, + To: senderAddr, + Value: big.Zero(), + } + bm := NewBlockMiner(ctx, t, miners[0], 100*time.Millisecond) + bm.MineBlocks() + defer bm.Stop() + + sm, err := api.MpoolPushMessage(ctx, msg, nil) + if err != nil { + t.Fatal(err) + } + res, err := api.StateWaitMsg(ctx, sm.Cid(), 1) + if err != nil { + t.Fatal(err) + } + if res.Receipt.ExitCode != 0 { + t.Fatal("did not successfully send message") + } + + searchRes, err := api.StateSearchMsg(ctx, sm.Cid()) + if err != nil { + t.Fatal(err) + } + + if searchRes.TipSet != res.TipSet { + t.Fatalf("search ts: %s, different from wait ts: %s", searchRes.TipSet, res.TipSet) + } + +} + func (ts *testSuite) testID(t *testing.T) { ctx := context.Background() apis, _ := ts.makeNodes(t, OneFull, OneMiner) From 98c1aa7988750b172bce86ce13554bc218af33ad Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 13 Oct 2020 13:31:20 -0700 Subject: [PATCH 737/885] add WalletVerify to lotus-gateway --- cmd/lotus-gateway/api.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cmd/lotus-gateway/api.go b/cmd/lotus-gateway/api.go index d5fac0a06..1e680daac 100644 --- a/cmd/lotus-gateway/api.go +++ b/cmd/lotus-gateway/api.go @@ -7,9 +7,13 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/sigs" + _ "github.com/filecoin-project/lotus/lib/sigs/bls" + _ "github.com/filecoin-project/lotus/lib/sigs/secp" "github.com/filecoin-project/lotus/node/impl/full" "github.com/ipfs/go-cid" ) @@ -172,6 +176,10 @@ func (a *GatewayAPI) StateWaitMsg(ctx context.Context, msg cid.Cid, confidence u return a.api.StateWaitMsgLimited(ctx, msg, confidence, stateWaitLookbackLimit) } +func (a *GatewayAPI) WalletVerify(ctx context.Context, k address.Address, msg []byte, sig *crypto.Signature) (bool, error) { + return sigs.Verify(sig, k, msg) == nil, nil +} + var _ api.GatewayAPI = (*GatewayAPI)(nil) var _ full.ChainModuleAPI = (*GatewayAPI)(nil) var _ full.GasModuleAPI = (*GatewayAPI)(nil) From 3d80c3806473c02984d51de61de903615f61a876 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 13 Oct 2020 14:23:08 -0700 Subject: [PATCH 738/885] support stateReadState in gateway --- cmd/lotus-gateway/api.go | 5 +++++ cmd/lotus-gateway/api_test.go | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/cmd/lotus-gateway/api.go b/cmd/lotus-gateway/api.go index 1e680daac..381dd931b 100644 --- a/cmd/lotus-gateway/api.go +++ b/cmd/lotus-gateway/api.go @@ -41,6 +41,7 @@ type gatewayDepsAPI interface { StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) StateWaitMsgLimited(ctx context.Context, msg cid.Cid, confidence uint64, h abi.ChainEpoch) (*api.MsgLookup, error) + StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*api.ActorState, error) } type GatewayAPI struct { @@ -180,6 +181,10 @@ func (a *GatewayAPI) WalletVerify(ctx context.Context, k address.Address, msg [] return sigs.Verify(sig, k, msg) == nil, nil } +func (a *GatewayAPI) StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*api.ActorState, error) { + return a.api.StateReadState(ctx, actor, tsk) +} + var _ api.GatewayAPI = (*GatewayAPI)(nil) var _ full.ChainModuleAPI = (*GatewayAPI)(nil) var _ full.GasModuleAPI = (*GatewayAPI)(nil) diff --git a/cmd/lotus-gateway/api_test.go b/cmd/lotus-gateway/api_test.go index f34f887f5..6719c57c5 100644 --- a/cmd/lotus-gateway/api_test.go +++ b/cmd/lotus-gateway/api_test.go @@ -189,3 +189,7 @@ func (m *mockGatewayDepsAPI) StateLookupID(ctx context.Context, addr address.Add func (m *mockGatewayDepsAPI) StateWaitMsgLimited(ctx context.Context, msg cid.Cid, confidence uint64, h abi.ChainEpoch) (*api.MsgLookup, error) { panic("implement me") } + +func (m *mockGatewayDepsAPI) StateReadState(ctx context.Context, act address.Address, ts types.TipSetKey) (*api.ActorState, error) { + panic("implement me") +} From b61281ba4bea27b334343e2a1ebc65663e736146 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 13 Oct 2020 15:06:47 -0700 Subject: [PATCH 739/885] fix a panic on startup when we fail to load the tipset There may be a deeper issue here, but we need to handle this more gracefully somehow. See #4358. --- chain/messagepool/messagepool.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/chain/messagepool/messagepool.go b/chain/messagepool/messagepool.go index d3c638b22..83b1a7f29 100644 --- a/chain/messagepool/messagepool.go +++ b/chain/messagepool/messagepool.go @@ -670,6 +670,10 @@ func (mp *MessagePool) addLoaded(m *types.SignedMessage) error { curTs := mp.curTs + if curTs == nil { + return xerrors.Errorf("current tipset not loaded") + } + snonce, err := mp.getStateNonce(m.Message.From, curTs) if err != nil { return xerrors.Errorf("failed to look up actor state nonce: %s: %w", err, ErrSoftValidationFailure) From 51aca86e98ccb23d1069a542e176bf1fe7ce26d6 Mon Sep 17 00:00:00 2001 From: zgfzgf <1901989065@qq.com> Date: Wed, 14 Oct 2020 08:44:42 +0800 Subject: [PATCH 740/885] optimize mining code --- chain/gen/mining.go | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/chain/gen/mining.go b/chain/gen/mining.go index 45a089452..f6d697ec0 100644 --- a/chain/gen/mining.go +++ b/chain/gen/mining.go @@ -6,15 +6,12 @@ import ( "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/specs-actors/actors/util/adt" cid "github.com/ipfs/go-cid" - cbor "github.com/ipfs/go-ipld-cbor" cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/chain/vm" "github.com/filecoin-project/lotus/lib/sigs/bls" ) @@ -114,23 +111,12 @@ func MinerCreateBlock(ctx context.Context, sm *stmgr.StateManager, w api.WalletA } next.ParentBaseFee = baseFee - cst := cbor.NewCborStore(sm.ChainStore().Blockstore()) - tree, err := state.LoadStateTree(cst, st) - if err != nil { - return nil, xerrors.Errorf("failed to load state tree: %w", err) - } - - waddr, err := vm.ResolveToKeyAddr(tree, cst, worker) - if err != nil { - return nil, xerrors.Errorf("failed to resolve miner address to key address: %w", err) - } - nosigbytes, err := next.SigningBytes() if err != nil { return nil, xerrors.Errorf("failed to get signing bytes for block: %w", err) } - sig, err := w.WalletSign(ctx, waddr, nosigbytes, api.MsgMeta{ + sig, err := w.WalletSign(ctx, worker, nosigbytes, api.MsgMeta{ Type: api.MTBlock, }) if err != nil { From 24267008212f8330bf67ba8d064d2784ae557e4a Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Mon, 12 Oct 2020 21:50:11 -0400 Subject: [PATCH 741/885] Don't transfer zero FIL during rebalancing fork --- chain/stmgr/forks.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index 7f2b57546..36b0da83c 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -312,11 +312,13 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCal available = st.GetAvailableBalance(act.Balance) } - transfers = append(transfers, transfer{ - From: addr, - To: builtin.ReserveAddress, - Amt: available, - }) + if !available.IsZero() { + transfers = append(transfers, transfer{ + From: addr, + To: builtin.ReserveAddress, + Amt: available, + }) + } } return nil }) From 7dbc8965bea3c17e22c643b34e4d91fc3091a38a Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Tue, 13 Oct 2020 01:05:28 -0400 Subject: [PATCH 742/885] Apply nonces to implicit messages --- chain/stmgr/stmgr.go | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index 0cb17fcf3..113304de1 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -247,17 +247,12 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp return cid.Undef, cid.Undef, xerrors.Errorf("making vm: %w", err) } - runCron := func() error { - // TODO: this nonce-getting is a tiny bit ugly - ca, err := vmi.StateTree().GetActor(builtin0.SystemActorAddr) - if err != nil { - return err - } + runCron := func(epoch abi.ChainEpoch) error { cronMsg := &types.Message{ To: builtin0.CronActorAddr, From: builtin0.SystemActorAddr, - Nonce: ca.Nonce, + Nonce: uint64(epoch), Value: types.NewInt(0), GasFeeCap: types.NewInt(0), GasPremium: types.NewInt(0), @@ -284,7 +279,7 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp for i := parentEpoch; i < epoch; i++ { if i > parentEpoch { // run cron for null rounds if any - if err := runCron(); err != nil { + if err := runCron(i); err != nil { return cid.Undef, cid.Undef, err } @@ -350,15 +345,10 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp return cid.Undef, cid.Undef, xerrors.Errorf("failed to serialize award params: %w", err) } - sysAct, actErr := vmi.StateTree().GetActor(builtin0.SystemActorAddr) - if actErr != nil { - return cid.Undef, cid.Undef, xerrors.Errorf("failed to get system actor: %w", actErr) - } - rwMsg := &types.Message{ From: builtin0.SystemActorAddr, To: reward.Address, - Nonce: sysAct.Nonce, + Nonce: uint64(epoch), Value: types.NewInt(0), GasFeeCap: types.NewInt(0), GasPremium: types.NewInt(0), @@ -381,7 +371,7 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp } } - if err := runCron(); err != nil { + if err := runCron(epoch); err != nil { return cid.Cid{}, cid.Cid{}, err } From 58662b79b38be538d76ec06c7ab06dcaef854ac7 Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Tue, 13 Oct 2020 19:50:13 -0700 Subject: [PATCH 743/885] fix(deps): use tagged go-fil-markets 0.9.0 --- go.mod | 6 +++--- go.sum | 12 ++++++------ node/modules/client.go | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 75926117a..0118baa88 100644 --- a/go.mod +++ b/go.mod @@ -27,9 +27,9 @@ require ( github.com/filecoin-project/go-bitfield v0.2.1 github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 - github.com/filecoin-project/go-data-transfer v0.6.8-0.20201013092017-eee8d1cde5d5 + github.com/filecoin-project/go-data-transfer v0.9.0 github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f - github.com/filecoin-project/go-fil-markets v0.7.2-0.20201013095312-bbda6c6fd17a + github.com/filecoin-project/go-fil-markets v0.9.0 github.com/filecoin-project/go-jsonrpc v0.1.2-0.20201008195726-68c6a2704e49 github.com/filecoin-project/go-multistore v0.0.3 github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 @@ -66,7 +66,7 @@ require ( github.com/ipfs/go-ds-pebble v0.0.2-0.20200921225637-ce220f8ac459 github.com/ipfs/go-filestore v1.0.0 github.com/ipfs/go-fs-lock v0.0.6 - github.com/ipfs/go-graphsync v0.2.1-0.20201013053840-5d8ea8076a2c + github.com/ipfs/go-graphsync v0.3.0 github.com/ipfs/go-ipfs-blockstore v1.0.1 github.com/ipfs/go-ipfs-chunker v0.0.5 github.com/ipfs/go-ipfs-ds-help v1.0.0 diff --git a/go.sum b/go.sum index 50e85dfd5..2b0859bfd 100644 --- a/go.sum +++ b/go.sum @@ -240,14 +240,14 @@ github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:a github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= -github.com/filecoin-project/go-data-transfer v0.6.8-0.20201013092017-eee8d1cde5d5 h1:kgmVHDP+rqcPqytpC215MzRRXrW9FqTfn0VFWhB2oog= -github.com/filecoin-project/go-data-transfer v0.6.8-0.20201013092017-eee8d1cde5d5/go.mod h1:1zaap881YZLMeIxhvLM0zA9qfUkh3bjOLSRaOoO9WRM= +github.com/filecoin-project/go-data-transfer v0.9.0 h1:nTT8j7Hu3TM0wRWrGy83/ctawG7sleJGdFWtIsUsKgY= +github.com/filecoin-project/go-data-transfer v0.9.0/go.mod h1:i2CqUy7TMQGKukj9BgqIxiP8nDHDXU2VLd771KVaCaQ= github.com/filecoin-project/go-ds-versioning v0.1.0 h1:y/X6UksYTsK8TLCI7rttCKEvl8btmWxyFMEeeWGUxIQ= github.com/filecoin-project/go-ds-versioning v0.1.0/go.mod h1:mp16rb4i2QPmxBnmanUx8i/XANp+PFCCJWiAb+VW4/s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f h1:GxJzR3oRIMTPtpZ0b7QF8FKPK6/iPAc7trhlL5k/g+s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= -github.com/filecoin-project/go-fil-markets v0.7.2-0.20201013095312-bbda6c6fd17a h1:jC3yO5R0+jGnVAHVqIHXb3BqCN/crslxTALz4I5gw1o= -github.com/filecoin-project/go-fil-markets v0.7.2-0.20201013095312-bbda6c6fd17a/go.mod h1:HqbtLIwJ4Lonp0BWq4ajmdXZ2UTh2+YK3wZSXXAE8yA= +github.com/filecoin-project/go-fil-markets v0.9.0 h1:RVdMKeOdRtpHawB14+uvJhvq243/UfZ4g3yFa/6qekk= +github.com/filecoin-project/go-fil-markets v0.9.0/go.mod h1:h+bJ/IUnYjnW5HMKyt9JQSnhslqetkpuzwwugc3K8vM= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24= @@ -538,8 +538,8 @@ github.com/ipfs/go-filestore v1.0.0/go.mod h1:/XOCuNtIe2f1YPbiXdYvD0BKLA0JR1MgPi github.com/ipfs/go-fs-lock v0.0.6 h1:sn3TWwNVQqSeNjlWy6zQ1uUGAZrV3hPOyEA6y1/N2a0= github.com/ipfs/go-fs-lock v0.0.6/go.mod h1:OTR+Rj9sHiRubJh3dRhD15Juhd/+w6VPOY28L7zESmM= github.com/ipfs/go-graphsync v0.1.0/go.mod h1:jMXfqIEDFukLPZHqDPp8tJMbHO9Rmeb9CEGevngQbmE= -github.com/ipfs/go-graphsync v0.2.1-0.20201013053840-5d8ea8076a2c h1:De/AZGvRa3WMyw5zdMMhcvRcho46BVo+C0NRud+T4io= -github.com/ipfs/go-graphsync v0.2.1-0.20201013053840-5d8ea8076a2c/go.mod h1:gEBvJUNelzMkaRPJTpg/jaKN4AQW/7wDWu0K92D8o10= +github.com/ipfs/go-graphsync v0.3.0 h1:I6Y20kSuCWkUvPoUWo4V3am704/9QjgDVVkf0zIV8+8= +github.com/ipfs/go-graphsync v0.3.0/go.mod h1:gEBvJUNelzMkaRPJTpg/jaKN4AQW/7wDWu0K92D8o10= github.com/ipfs/go-hamt-ipld v0.1.1 h1:0IQdvwnAAUKmDE+PMJa5y1QiwOPHpI9+eAbQEEEYthk= github.com/ipfs/go-hamt-ipld v0.1.1/go.mod h1:1EZCr2v0jlCnhpa+aZ0JZYp8Tt2w16+JJOAVz17YcDk= github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= diff --git a/node/modules/client.go b/node/modules/client.go index eab309445..f1380bc97 100644 --- a/node/modules/client.go +++ b/node/modules/client.go @@ -91,7 +91,7 @@ func NewClientGraphsyncDataTransfer(lc fx.Lifecycle, h host.Host, gs dtypes.Grap return nil, err } - dt.OnReady(marketevents.ReadyLogger("provider data transfer")) + dt.OnReady(marketevents.ReadyLogger("client data transfer")) lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { return dt.Start(ctx) From 458ea277fcb4cb9f0e52453da4b50399632040e6 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Tue, 13 Oct 2020 01:20:49 -0400 Subject: [PATCH 744/885] Aggregate irregular state transitions into a single EexcutionTrace --- chain/stmgr/forks.go | 118 +++++++++++++++++++++++++++++++++---------- 1 file changed, 92 insertions(+), 26 deletions(-) diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index 36b0da83c..156c68783 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -168,7 +168,7 @@ func (sm *StateManager) hasExpensiveFork(ctx context.Context, height abi.ChainEp return ok } -func doTransfer(cb ExecCallback, tree types.StateTree, from, to address.Address, amt abi.TokenAmount) error { +func doTransfer(tree types.StateTree, from, to address.Address, amt abi.TokenAmount, cb func(trace types.ExecutionTrace)) error { fromAct, err := tree.GetActor(from) if err != nil { return xerrors.Errorf("failed to get 'from' actor for transfer: %w", err) @@ -201,7 +201,6 @@ func doTransfer(cb ExecCallback, tree types.StateTree, from, to address.Address, From: from, To: to, Value: amt, - Nonce: math.MaxUint64, } fakeRct := &types.MessageReceipt{ ExitCode: 0, @@ -209,22 +208,14 @@ func doTransfer(cb ExecCallback, tree types.StateTree, from, to address.Address, GasUsed: 0, } - if err := cb(fakeMsg.Cid(), fakeMsg, &vm.ApplyRet{ - MessageReceipt: *fakeRct, - ActorErr: nil, - ExecutionTrace: types.ExecutionTrace{ - Msg: fakeMsg, - MsgRct: fakeRct, - Error: "", - Duration: 0, - GasCharges: nil, - Subcalls: nil, - }, - Duration: 0, - GasCosts: vm.ZeroGasOutputs(), - }); err != nil { - return xerrors.Errorf("recording transfer: %w", err) - } + cb(types.ExecutionTrace{ + Msg: fakeMsg, + MsgRct: fakeRct, + Error: "", + Duration: 0, + GasCharges: nil, + Subcalls: nil, + }) } return nil @@ -277,6 +268,10 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCal } var transfers []transfer + subcalls := make([]types.ExecutionTrace, 0) + transferCb := func(trace types.ExecutionTrace) { + subcalls = append(subcalls, trace) + } // Take all excess funds away, put them into the reserve account err = tree.ForEach(func(addr address.Address, act *types.Actor) error { @@ -328,7 +323,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCal // Execute transfers from previous step for _, t := range transfers { - if err := doTransfer(cb, tree, t.From, t.To, t.Amt); err != nil { + if err := doTransfer(tree, t.From, t.To, t.Amt, transferCb); err != nil { return cid.Undef, xerrors.Errorf("transfer %s %s->%s failed: %w", t.Amt, t.From, t.To, err) } } @@ -431,7 +426,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCal } for _, t := range transfersBack { - if err := doTransfer(cb, tree, t.From, t.To, t.Amt); err != nil { + if err := doTransfer(tree, t.From, t.To, t.Amt, transferCb); err != nil { return cid.Undef, xerrors.Errorf("transfer %s %s->%s failed: %w", t.Amt, t.From, t.To, err) } } @@ -441,7 +436,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCal if err != nil { return cid.Undef, xerrors.Errorf("failed to load burnt funds actor: %w", err) } - if err := doTransfer(cb, tree, builtin0.BurntFundsActorAddr, builtin.ReserveAddress, burntAct.Balance); err != nil { + if err := doTransfer(tree, builtin0.BurntFundsActorAddr, builtin.ReserveAddress, burntAct.Balance, transferCb); err != nil { return cid.Undef, xerrors.Errorf("failed to unburn funds: %w", err) } @@ -457,7 +452,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCal } difference := types.BigSub(DesiredReimbursementBalance, reimb.Balance) - if err := doTransfer(cb, tree, builtin.ReserveAddress, reimbAddr, difference); err != nil { + if err := doTransfer(tree, builtin.ReserveAddress, reimbAddr, difference, transferCb); err != nil { return cid.Undef, xerrors.Errorf("failed to top up reimbursement account: %w", err) } @@ -476,6 +471,39 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCal return cid.Undef, xerrors.Errorf("resultant state tree account balance was not correct: %s", total) } + if cb != nil { + // record the transfer in execution traces + + fakeMsg := &types.Message{ + From: builtin.SystemActorAddr, + To: builtin.SystemActorAddr, + Value: big.Zero(), + Nonce: uint64(epoch), + } + fakeRct := &types.MessageReceipt{ + ExitCode: 0, + Return: nil, + GasUsed: 0, + } + + if err := cb(fakeMsg.Cid(), fakeMsg, &vm.ApplyRet{ + MessageReceipt: *fakeRct, + ActorErr: nil, + ExecutionTrace: types.ExecutionTrace{ + Msg: fakeMsg, + MsgRct: fakeRct, + Error: "", + Duration: 0, + GasCharges: nil, + Subcalls: subcalls, + }, + Duration: 0, + GasCosts: vm.ZeroGasOutputs(), + }); err != nil { + return cid.Undef, xerrors.Errorf("recording transfers: %w", err) + } + } + return tree.Flush(ctx) } @@ -516,12 +544,12 @@ func UpgradeIgnition(ctx context.Context, sm *StateManager, cb ExecCallback, roo return cid.Undef, xerrors.Errorf("resetting genesis msig start epochs: %w", err) } - err = splitGenesisMultisig(ctx, cb, split1, store, tree, 50) + err = splitGenesisMultisig(ctx, cb, split1, store, tree, 50, epoch) if err != nil { return cid.Undef, xerrors.Errorf("splitting first msig: %w", err) } - err = splitGenesisMultisig(ctx, cb, split2, store, tree, 50) + err = splitGenesisMultisig(ctx, cb, split2, store, tree, 50, epoch) if err != nil { return cid.Undef, xerrors.Errorf("splitting second msig: %w", err) } @@ -647,7 +675,7 @@ func setNetworkName(ctx context.Context, store adt.Store, tree *state.StateTree, return nil } -func splitGenesisMultisig(ctx context.Context, cb ExecCallback, addr address.Address, store adt0.Store, tree *state.StateTree, portions uint64) error { +func splitGenesisMultisig(ctx context.Context, cb ExecCallback, addr address.Address, store adt0.Store, tree *state.StateTree, portions uint64, epoch abi.ChainEpoch) error { if portions < 1 { return xerrors.Errorf("cannot split into 0 portions") } @@ -716,6 +744,11 @@ func splitGenesisMultisig(ctx context.Context, cb ExecCallback, addr address.Add } i := uint64(0) + subcalls := make([]types.ExecutionTrace, 0, portions) + transferCb := func(trace types.ExecutionTrace) { + subcalls = append(subcalls, trace) + } + for i < portions { keyAddr, err := makeKeyAddr(addr, i) if err != nil { @@ -732,13 +765,46 @@ func splitGenesisMultisig(ctx context.Context, cb ExecCallback, addr address.Add return xerrors.Errorf("setting new msig actor state: %w", err) } - if err := doTransfer(cb, tree, addr, idAddr, newIbal); err != nil { + if err := doTransfer(tree, addr, idAddr, newIbal, transferCb); err != nil { return xerrors.Errorf("transferring split msig balance: %w", err) } i++ } + if cb != nil { + // record the transfer in execution traces + + fakeMsg := &types.Message{ + From: builtin.SystemActorAddr, + To: addr, + Value: big.Zero(), + Nonce: uint64(epoch), + } + fakeRct := &types.MessageReceipt{ + ExitCode: 0, + Return: nil, + GasUsed: 0, + } + + if err := cb(fakeMsg.Cid(), fakeMsg, &vm.ApplyRet{ + MessageReceipt: *fakeRct, + ActorErr: nil, + ExecutionTrace: types.ExecutionTrace{ + Msg: fakeMsg, + MsgRct: fakeRct, + Error: "", + Duration: 0, + GasCharges: nil, + Subcalls: subcalls, + }, + Duration: 0, + GasCosts: vm.ZeroGasOutputs(), + }); err != nil { + return xerrors.Errorf("recording transfers: %w", err) + } + } + return nil } From ce541102289ee8db4017c84c9fb88250e9ce4fbe Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Tue, 13 Oct 2020 23:45:47 -0400 Subject: [PATCH 745/885] Add message CID to InvocResult --- api/api_full.go | 1 + chain/stmgr/call.go | 2 ++ chain/stmgr/stmgr.go | 1 + documentation/en/api-methods.md | 6 ++++++ node/impl/full/state.go | 1 + 5 files changed, 11 insertions(+) diff --git a/api/api_full.go b/api/api_full.go index 657e945ca..22a7380ac 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -734,6 +734,7 @@ type RetrievalOrder struct { } type InvocResult struct { + MsgCid cid.Cid Msg *types.Message MsgRct *types.MessageReceipt ExecutionTrace types.ExecutionTrace diff --git a/chain/stmgr/call.go b/chain/stmgr/call.go index 8d0f6f6c1..86035140b 100644 --- a/chain/stmgr/call.go +++ b/chain/stmgr/call.go @@ -113,6 +113,7 @@ func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types. } return &api.InvocResult{ + MsgCid: msg.Cid(), Msg: msg, MsgRct: &ret.MessageReceipt, ExecutionTrace: ret.ExecutionTrace, @@ -228,6 +229,7 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri } return &api.InvocResult{ + MsgCid: msg.Cid(), Msg: msg, MsgRct: &ret.MessageReceipt, ExecutionTrace: ret.ExecutionTrace, diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index 0cb17fcf3..e5accc0c9 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -200,6 +200,7 @@ func (sm *StateManager) TipSetState(ctx context.Context, ts *types.TipSet) (st c func traceFunc(trace *[]*api.InvocResult) func(mcid cid.Cid, msg *types.Message, ret *vm.ApplyRet) error { return func(mcid cid.Cid, msg *types.Message, ret *vm.ApplyRet) error { ir := &api.InvocResult{ + MsgCid: mcid, Msg: msg, MsgRct: &ret.MessageReceipt, ExecutionTrace: ret.ExecutionTrace, diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index eefc53ab7..2ed000c59 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -3059,6 +3059,9 @@ Inputs: Response: ```json { + "MsgCid": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, "Msg": { "Version": 42, "To": "f01234", @@ -4079,6 +4082,9 @@ Inputs: Response: ```json { + "MsgCid": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, "Msg": { "Version": 42, "To": "f01234", diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 1c39b59da..08ac62f2e 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -362,6 +362,7 @@ func (a *StateAPI) StateReplay(ctx context.Context, tsk types.TipSetKey, mc cid. } return &api.InvocResult{ + MsgCid: mc, Msg: m, MsgRct: &r.MessageReceipt, ExecutionTrace: r.ExecutionTrace, From 2e7cd024df10d9c0ea9840dd17f4ec6b4c980698 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Wed, 14 Oct 2020 02:10:26 -0400 Subject: [PATCH 746/885] Lotus version 0.10.1 --- CHANGELOG.md | 23 +++++++++++++++++++++++ build/version.go | 4 ++-- documentation/en/api-methods.md | 2 +- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 16d8d15b8..7c94e387c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,28 @@ # Lotus changelog +# 0.10.1 / 2020-10-14 + +This is an optional release of Lotus that updates markets to 0.9.0, which adds the ability to restart data transfers. This release also introduces Ledger support, and various UX improvements. + +## Changes + +- Test the tape upgrade (https://github.com/filecoin-project/lotus/pull/4328) +- Adding in Ledger support (https://github.com/filecoin-project/lotus/pull/4290) +- Improve the UX for lotus-miner sealing workers (https://github.com/filecoin-project/lotus/pull/4329) +- Add a CLI tool for miner's to repay debt (https://github.com/filecoin-project/lotus/pull/4319) +- Rename params_testnet to params_mainnet (https://github.com/filecoin-project/lotus/pull/4336) +- Use seal-duration in calculating the earliest StartEpoch (https://github.com/filecoin-project/lotus/pull/4337) +- Reject deals that are > 7 days in the future in the BasicDealFilter (https://github.com/filecoin-project/lotus/pull/4173) +- Add an API endpoint to calculate the exact circulating supply (https://github.com/filecoin-project/lotus/pull/4148) +- lotus-pcr: ignore all other market messages (https://github.com/filecoin-project/lotus/pull/4341) +- Add message CID to InvocResult (https://github.com/filecoin-project/lotus/pull/4382) +- types: Add CID fields to messages in json marshalers (https://github.com/filecoin-project/lotus/pull/4338) +- fix(sync state): set state height to actual tipset height (https://github.com/filecoin-project/lotus/pull/4347) +- Fix off by one tipset in searchBackForMsg (https://github.com/filecoin-project/lotus/pull/4367) +- fix a panic on startup when we fail to load the tipset (https://github.com/filecoin-project/lotus/pull/4376) +- Avoid having the same message CID show up in execution traces (https://github.com/filecoin-project/lotus/pull/4350) +- feat(markets): update markets 0.9.0 and add data transfer restart (https://github.com/filecoin-project/lotus/pull/4363) + # 0.10.0 / 2020-10-12 This is a consensus-breaking hotfix that addresses an issue in specs-actors v2.0.3 that made it impossible to pledge new 32GiB sectors. The change in Lotus is to update to actors v2.1.0, behind the new network version 5. diff --git a/build/version.go b/build/version.go index baeafdfe7..39f9d48ff 100644 --- a/build/version.go +++ b/build/version.go @@ -29,7 +29,7 @@ func buildType() string { } // BuildVersion is the local build version, set by build system -const BuildVersion = "0.10.0" +const BuildVersion = "0.10.1" func UserVersion() string { return BuildVersion + buildType() + CurrentCommit @@ -83,7 +83,7 @@ func VersionForType(nodeType NodeType) (Version, error) { // semver versions of the rpc api exposed var ( - FullAPIVersion = newVer(0, 16, 0) + FullAPIVersion = newVer(0, 17, 0) MinerAPIVersion = newVer(0, 15, 0) WorkerAPIVersion = newVer(0, 15, 0) ) diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index 1c8a3a24f..a905ef2e3 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -228,7 +228,7 @@ Response: ```json { "Version": "string value", - "APIVersion": 4096, + "APIVersion": 4352, "BlockDelay": 42 } ``` From 30dd5d9644d88c157b0f025607df84a50e7ad388 Mon Sep 17 00:00:00 2001 From: Aarsh Shah Date: Wed, 14 Oct 2020 11:17:44 +0530 Subject: [PATCH 747/885] use updated stored ask api --- node/modules/storageminer.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 613203a60..adacf5405 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -393,13 +393,13 @@ func NewStorageAsk(ctx helpers.MetricsCtx, fapi lapi.FullNode, ds dtypes.Metadat if err != nil { return nil, err } - storedAsk, err := storedask.NewStoredAsk(namespace.Wrap(providerDs, datastore.NewKey("/storage-ask")), datastore.NewKey("latest"), spn, address.Address(minerAddress)) + storedAsk, err := storedask.NewStoredAsk(namespace.Wrap(providerDs, datastore.NewKey("/storage-ask")), datastore.NewKey("latest"), spn, address.Address(minerAddress), + storagemarket.MaxPieceSize(abi.PaddedPieceSize(mi.SectorSize))) if err != nil { return nil, err } - // Hacky way to set max piece size to the sector size a := storedAsk.GetAsk().Ask - err = storedAsk.SetAsk(a.Price, a.VerifiedPrice, a.Expiry-a.Timestamp, storagemarket.MaxPieceSize(abi.PaddedPieceSize(mi.SectorSize))) + err = storedAsk.SetAsk(a.Price, a.VerifiedPrice, a.Expiry-a.Timestamp) if err != nil { return storedAsk, err } From c6ed6a0d65bbf294c1d634161b2ea4288630bddf Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Wed, 14 Oct 2020 14:53:11 +0200 Subject: [PATCH 748/885] Add support for /https, /http, /wss API multiaddresses. Lotus API endpoint are by default expressed as multiaddresses (i.e. lotus auth api-info) which end in /http, although they can be provided as standard URLs too. There is an inconsistency here because despite the "http" part, Lotus will use "ws" protocol. This lets lotus default to "ws" but honor whatever the user puts in the multiaddress (i.e. /dns4/my.lotus.node/tcp/443/https) would work now using https, where before it uses "ws". --- cli/util/apiinfo.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/cli/util/apiinfo.go b/cli/util/apiinfo.go index 1f9a83769..dec4c2e19 100644 --- a/cli/util/apiinfo.go +++ b/cli/util/apiinfo.go @@ -44,7 +44,22 @@ func (a APIInfo) DialArgs() (string, error) { return "", err } - return "ws://" + addr + "/rpc/v0", nil + protocol := "ws" + + // If the user specifies the multiaddress as + // /something/tcp/1234/http or/something/tcp/1234/https + // or /something/tcp/1234/wss then honor that. + for _, p := range []int{ + multiaddr.P_HTTP, + multiaddr.P_HTTPS, + multiaddr.P_WSS, + } { + if _, err := ma.ValueForProtocol(p); err == nil { + protocol = multiaddr.ProtocolWithCode(p).Name + break + } + } + return protocol + "://" + addr + "/rpc/v0", nil } _, err = url.Parse(a.Addr) From 00a50831460c41727cb9d1bc67a1ef4f78b949a0 Mon Sep 17 00:00:00 2001 From: David Dias Date: Wed, 14 Oct 2020 15:11:30 +0200 Subject: [PATCH 749/885] Update SECURITY.md --- SECURITY.md | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index ecb600deb..592206bc5 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -2,9 +2,7 @@ ## Reporting a Vulnerability -For *critical* bugs, please send an email to security@filecoin.org. - -The bug reporting process differs between bugs that are critical and may crash the network, and others that are unlikely to cause problems if malicious parties know about it. For non-critical bugs, please simply file a GitHub [issue](https://github.com/filecoin-project/lotus/issues/new?template=bug_report.md). +For *critical* bugs, please consult our Security Policy and Responsible Disclosure Program information at https://github.com/filecoin-project/community/blob/master/SECURITY.md Please try to provide a clear description of any bugs reported, along with how to reproduce the bug if possible. More detailed bug reports (especially those with a PoC included) will help us move forward much faster. Additionally, please avoid reporting bugs that already have open issues. Take a moment to search the issue list of the related GitHub repositories before writing up a new report. @@ -20,10 +18,6 @@ Here are some examples of bugs we would consider 'critical': This is not an exhaustive list, but should provide some idea of what we consider 'critical'. -## Supported Versions +## Reporting a non security bug -* TODO: This should be defined and set up by Mainnet launch. - -| Version | Supported | -| ------- | ------------------ | -| Testnet | :white_check_mark: | +For non-critical bugs, please simply file a GitHub [issue](https://github.com/filecoin-project/lotus/issues/new?template=bug_report.md). From 45cd510da11807351ef87cc47c4169581eb2779b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Tue, 13 Oct 2020 23:00:01 +0100 Subject: [PATCH 750/885] conformance: support multiple protocol versions. This PR introduces support for running multiple variants of a vector, each of which targets a unique protocol version. tvx tooling has been adapted to produce and parse the new version of the schema. --- cmd/tvx/codenames.go | 40 ++++++++++++++++++++++++++++++++++++++ cmd/tvx/codenames_test.go | 28 ++++++++++++++++++++++++++ cmd/tvx/exec.go | 30 +++++++++++++++------------- cmd/tvx/extract.go | 14 ++++++++++++- conformance/corpus_test.go | 22 ++++++++++++++------- conformance/driver.go | 5 ++--- conformance/runner.go | 26 +++++++++++++------------ go.mod | 2 +- go.sum | 4 ++-- 9 files changed, 132 insertions(+), 39 deletions(-) create mode 100644 cmd/tvx/codenames.go create mode 100644 cmd/tvx/codenames_test.go diff --git a/cmd/tvx/codenames.go b/cmd/tvx/codenames.go new file mode 100644 index 000000000..1cb1b32fb --- /dev/null +++ b/cmd/tvx/codenames.go @@ -0,0 +1,40 @@ +package main + +import ( + "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/lotus/build" +) + +// ProtocolCodenames is a table that summarises the protocol codenames that +// will be set on extracted vectors, depending on the original execution height. +// +// Implementers rely on these names to filter the vectors they can run through +// their implementations, based on their support level +var ProtocolCodenames = []struct { + firstEpoch abi.ChainEpoch + name string +}{ + {0, "genesis"}, + // TODO there is some off-by-one trickery in GetNtwkVersion. Not sure if the + // protocol version really kicks in at the designated height, or at the + // following epoch. + {build.UpgradeBreezeHeight + 1, "breeze"}, + {build.UpgradeSmokeHeight + 1, "smoke"}, + {build.UpgradeIgnitionHeight + 1, "ignition"}, + {build.UpgradeRefuelHeight + 1, "refuel"}, + {build.UpgradeActorsV2Height + 1, "actorsv2"}, + {build.UpgradeTapeHeight + 1, "tape"}, + {build.UpgradeLiftoffHeight + 1, "liftoff"}, +} + +// GetProtocolCodename gets the protocol codename associated with a height. +func GetProtocolCodename(height abi.ChainEpoch) string { + for i, v := range ProtocolCodenames { + if height < v.firstEpoch { + // found the cutoff, return previous. + return ProtocolCodenames[i-1].name + } + } + return ProtocolCodenames[len(ProtocolCodenames)-1].name +} diff --git a/cmd/tvx/codenames_test.go b/cmd/tvx/codenames_test.go new file mode 100644 index 000000000..0b1de9fc2 --- /dev/null +++ b/cmd/tvx/codenames_test.go @@ -0,0 +1,28 @@ +package main + +import ( + "math" + "testing" + + "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/lotus/build" +) + +func TestProtocolCodenames(t *testing.T) { + if height := abi.ChainEpoch(100); GetProtocolCodename(height) != "genesis" { + t.Fatal("expected genesis codename") + } + + if height := abi.ChainEpoch(build.UpgradeBreezeHeight + 1); GetProtocolCodename(height) != "breeze" { + t.Fatal("expected breeze codename") + } + + if height := abi.ChainEpoch(build.UpgradeActorsV2Height + 1); GetProtocolCodename(height) != "actorsv2" { + t.Fatal("expected actorsv2 codename") + } + + if height := abi.ChainEpoch(math.MaxInt64); GetProtocolCodename(height) != ProtocolCodenames[len(ProtocolCodenames)-1].name { + t.Fatal("expected last codename") + } +} diff --git a/cmd/tvx/exec.go b/cmd/tvx/exec.go index 9ec6f9e2b..de7d2a398 100644 --- a/cmd/tvx/exec.go +++ b/cmd/tvx/exec.go @@ -72,20 +72,24 @@ func runExecLotus(_ *cli.Context) error { func executeTestVector(tv schema.TestVector) error { log.Println("executing test vector:", tv.Meta.ID) - r := new(conformance.LogReporter) - switch class := tv.Class; class { - case "message": - conformance.ExecuteMessageVector(r, &tv) - case "tipset": - conformance.ExecuteTipsetVector(r, &tv) - default: - return fmt.Errorf("test vector class %s not supported", class) - } - if r.Failed() { - log.Println(color.HiRedString("❌ test vector failed")) - } else { - log.Println(color.GreenString("✅ test vector succeeded")) + for _, v := range tv.Pre.Variants { + r := new(conformance.LogReporter) + + switch class := tv.Class; class { + case "message": + conformance.ExecuteMessageVector(r, &tv, &v) + case "tipset": + conformance.ExecuteTipsetVector(r, &tv, &v) + default: + return fmt.Errorf("test vector class %s not supported", class) + } + + if r.Failed() { + log.Println(color.HiRedString("❌ test vector failed for variant %s", v.ID)) + } else { + log.Println(color.GreenString("✅ test vector succeeded for variant %s", v.ID)) + } } return nil diff --git a/cmd/tvx/extract.go b/cmd/tvx/extract.go index 4bb391af6..4a8dd162c 100644 --- a/cmd/tvx/extract.go +++ b/cmd/tvx/extract.go @@ -347,6 +347,13 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { return err } + nv, err := fapi.StateNetworkVersion(ctx, execTs.Key()) + if err != nil { + return err + } + + codename := GetProtocolCodename(execTs.Height()) + // Write out the test vector. vector := schema.TestVector{ Class: schema.ClassMessage, @@ -363,10 +370,15 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { {Source: fmt.Sprintf("execution_tipset:%s", execTs.Key().String())}, {Source: "github.com/filecoin-project/lotus", Version: version.String()}}, }, + Selector: schema.Selector{ + schema.SelectorMinProtocolVersion: codename, + }, Randomness: recordingRand.Recorded(), CAR: out.Bytes(), Pre: &schema.Preconditions{ - Epoch: int64(execTs.Height()), + Variants: []schema.Variant{ + {ID: codename, Epoch: int64(execTs.Height()), NetworkVersion: uint(nv)}, + }, CircSupply: circSupply.Int, BaseFee: basefee.Int, StateTree: &schema.StateTree{ diff --git a/conformance/corpus_test.go b/conformance/corpus_test.go index 3d447570d..a09f9a8d3 100644 --- a/conformance/corpus_test.go +++ b/conformance/corpus_test.go @@ -11,6 +11,11 @@ import ( "github.com/filecoin-project/test-vectors/schema" ) +var invokees = map[schema.Class]func(Reporter, *schema.TestVector, *schema.Variant){ + schema.ClassMessage: ExecuteMessageVector, + schema.ClassTipset: ExecuteTipsetVector, +} + const ( // EnvSkipConformance, if 1, skips the conformance test suite. EnvSkipConformance = "SKIP_CONFORMANCE" @@ -120,13 +125,16 @@ func TestConformance(t *testing.T) { } // dispatch the execution depending on the vector class. - switch vector.Class { - case "message": - ExecuteMessageVector(t, &vector) - case "tipset": - ExecuteTipsetVector(t, &vector) - default: - t.Fatalf("test vector class not supported: %s", vector.Class) + invokee, ok := invokees[vector.Class] + if !ok { + t.Fatalf("unsupported test vector class: %s", vector.Class) + } + + for _, variant := range vector.Pre.Variants { + variant := variant + t.Run(variant.ID, func(t *testing.T) { + invokee(t, &vector, &variant) + }) } }) } diff --git a/conformance/driver.go b/conformance/driver.go index f49022b9c..fbb7dda6d 100644 --- a/conformance/driver.go +++ b/conformance/driver.go @@ -82,7 +82,7 @@ type ExecuteTipsetResult struct { // This method returns the the receipts root, the poststate root, and the VM // message results. The latter _include_ implicit messages, such as cron ticks // and reward withdrawal per miner. -func (d *Driver) ExecuteTipset(bs blockstore.Blockstore, ds ds.Batching, preroot cid.Cid, parentEpoch abi.ChainEpoch, tipset *schema.Tipset) (*ExecuteTipsetResult, error) { +func (d *Driver) ExecuteTipset(bs blockstore.Blockstore, ds ds.Batching, preroot cid.Cid, parentEpoch abi.ChainEpoch, tipset *schema.Tipset, execEpoch abi.ChainEpoch) (*ExecuteTipsetResult, error) { var ( syscalls = vm.Syscalls(ffiwrapper.ProofVerifier) vmRand = NewFixedRand() @@ -121,11 +121,10 @@ func (d *Driver) ExecuteTipset(bs blockstore.Blockstore, ds ds.Batching, preroot messages []*types.Message results []*vm.ApplyRet - epoch = abi.ChainEpoch(tipset.Epoch) basefee = abi.NewTokenAmount(tipset.BaseFee.Int64()) ) - postcid, receiptsroot, err := sm.ApplyBlocks(context.Background(), parentEpoch, preroot, blocks, epoch, vmRand, func(_ cid.Cid, msg *types.Message, ret *vm.ApplyRet) error { + postcid, receiptsroot, err := sm.ApplyBlocks(context.Background(), parentEpoch, preroot, blocks, execEpoch, vmRand, func(_ cid.Cid, msg *types.Message, ret *vm.ApplyRet) error { messages = append(messages, msg) results = append(results, ret) return nil diff --git a/conformance/runner.go b/conformance/runner.go index d489ac288..6f9d73305 100644 --- a/conformance/runner.go +++ b/conformance/runner.go @@ -30,11 +30,11 @@ import ( ) // ExecuteMessageVector executes a message-class test vector. -func ExecuteMessageVector(r Reporter, vector *schema.TestVector) { +func ExecuteMessageVector(r Reporter, vector *schema.TestVector, variant *schema.Variant) { var ( - ctx = context.Background() - epoch = vector.Pre.Epoch - root = vector.Pre.StateTree.RootCID + ctx = context.Background() + baseEpoch = variant.Epoch + root = vector.Pre.StateTree.RootCID ) // Load the CAR into a new temporary Blockstore. @@ -53,16 +53,16 @@ func ExecuteMessageVector(r Reporter, vector *schema.TestVector) { r.Fatalf("failed to deserialize message: %s", err) } - // add an epoch if one's set. - if m.Epoch != nil { - epoch = *m.Epoch + // add the epoch offset if one is set. + if m.EpochOffset != nil { + baseEpoch += *m.EpochOffset } // Execute the message. var ret *vm.ApplyRet ret, root, err = driver.ExecuteMessage(bs, ExecuteMessageParams{ Preroot: root, - Epoch: abi.ChainEpoch(epoch), + Epoch: abi.ChainEpoch(baseEpoch), Message: msg, BaseFee: BaseFeeOrDefault(vector.Pre.BaseFee), CircSupply: CircSupplyOrDefault(vector.Pre.CircSupply), @@ -86,10 +86,10 @@ func ExecuteMessageVector(r Reporter, vector *schema.TestVector) { } // ExecuteTipsetVector executes a tipset-class test vector. -func ExecuteTipsetVector(r Reporter, vector *schema.TestVector) { +func ExecuteTipsetVector(r Reporter, vector *schema.TestVector, variant *schema.Variant) { var ( ctx = context.Background() - prevEpoch = vector.Pre.Epoch + baseEpoch = abi.ChainEpoch(variant.Epoch) root = vector.Pre.StateTree.RootCID tmpds = ds.NewMapDatastore() ) @@ -105,9 +105,11 @@ func ExecuteTipsetVector(r Reporter, vector *schema.TestVector) { // Apply every tipset. var receiptsIdx int + var prevEpoch = baseEpoch for i, ts := range vector.ApplyTipsets { ts := ts // capture - ret, err := driver.ExecuteTipset(bs, tmpds, root, abi.ChainEpoch(prevEpoch), &ts) + execEpoch := baseEpoch + abi.ChainEpoch(ts.EpochOffset) + ret, err := driver.ExecuteTipset(bs, tmpds, root, prevEpoch, &ts, execEpoch) if err != nil { r.Fatalf("failed to apply tipset %d message: %s", i, err) } @@ -122,7 +124,7 @@ func ExecuteTipsetVector(r Reporter, vector *schema.TestVector) { r.Errorf("post receipts root doesn't match; expected: %s, was: %s", expected, actual) } - prevEpoch = ts.Epoch + prevEpoch = execEpoch root = ret.PostStateRoot } diff --git a/go.mod b/go.mod index 0118baa88..1516cba81 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/filecoin-project/specs-actors v0.9.12 github.com/filecoin-project/specs-actors/v2 v2.1.0 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 - github.com/filecoin-project/test-vectors/schema v0.0.4 + github.com/filecoin-project/test-vectors/schema v0.0.5-0.20201014133607-1352e6bb4e71 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 github.com/go-kit/kit v0.10.0 github.com/go-ole/go-ole v1.2.4 // indirect diff --git a/go.sum b/go.sum index 2b0859bfd..47301c784 100644 --- a/go.sum +++ b/go.sum @@ -282,8 +282,8 @@ github.com/filecoin-project/specs-actors/v2 v2.1.0 h1:ocEuGz8DG2cUWw32c/tvF8D6xT github.com/filecoin-project/specs-actors/v2 v2.1.0/go.mod h1:E7fAX4CZkDVQvDNRCxfq+hc3nx56KcCKyuZf0hlQJ20= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= -github.com/filecoin-project/test-vectors/schema v0.0.4 h1:QTRd0gb/NP4ZOTM7Dib5U3xE1/ToGDKnYLfxkC3t/m8= -github.com/filecoin-project/test-vectors/schema v0.0.4/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E= +github.com/filecoin-project/test-vectors/schema v0.0.5-0.20201014133607-1352e6bb4e71 h1:qnleaW7X8Gi2e3QtqPMFdqVn/DUaNI5tCnq7wNVMnio= +github.com/filecoin-project/test-vectors/schema v0.0.5-0.20201014133607-1352e6bb4e71/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6 h1:u/UEqS66A5ckRmS4yNpjmVH56sVtS/RfclBAYocb4as= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ= From 12f36c5bda4c8c49b42bbaf19d0964bccf448987 Mon Sep 17 00:00:00 2001 From: Whyrusleeping Date: Wed, 14 Oct 2020 09:22:25 -0500 Subject: [PATCH 751/885] Update cmd/lotus-gateway/api.go Co-authored-by: dirkmc --- cmd/lotus-gateway/api.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/lotus-gateway/api.go b/cmd/lotus-gateway/api.go index 381dd931b..8f1d69c18 100644 --- a/cmd/lotus-gateway/api.go +++ b/cmd/lotus-gateway/api.go @@ -182,6 +182,9 @@ func (a *GatewayAPI) WalletVerify(ctx context.Context, k address.Address, msg [] } func (a *GatewayAPI) StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*api.ActorState, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return nil, err + } return a.api.StateReadState(ctx, actor, tsk) } From ab4f051b837646cd2fe7efeef98513604e2abf82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 14 Oct 2020 18:09:31 +0200 Subject: [PATCH 752/885] sync wait --watch --- cli/sync.go | 12 +++++++++--- cmd/lotus-storage-miner/init.go | 2 +- cmd/lotus-storage-miner/init_restore.go | 2 +- cmd/lotus-storage-miner/run.go | 2 +- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/cli/sync.go b/cli/sync.go index ea066cbda..c3f25eb1d 100644 --- a/cli/sync.go +++ b/cli/sync.go @@ -84,6 +84,12 @@ var syncStatusCmd = &cli.Command{ var syncWaitCmd = &cli.Command{ Name: "wait", Usage: "Wait for sync to be complete", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "watch", + Usage: "don't exit after node is synced", + }, + }, Action: func(cctx *cli.Context) error { napi, closer, err := GetFullNodeAPI(cctx) if err != nil { @@ -92,7 +98,7 @@ var syncWaitCmd = &cli.Command{ defer closer() ctx := ReqContext(cctx) - return SyncWait(ctx, napi) + return SyncWait(ctx, napi, cctx.Bool("watch")) }, } @@ -234,7 +240,7 @@ var syncCheckpointCmd = &cli.Command{ }, } -func SyncWait(ctx context.Context, napi api.FullNode) error { +func SyncWait(ctx context.Context, napi api.FullNode, watch bool) error { tick := time.Second / 4 lastLines := 0 @@ -311,7 +317,7 @@ func SyncWait(ctx context.Context, napi api.FullNode) error { _ = target // todo: maybe print? (creates a bunch of line wrapping issues with most tipsets) - if time.Now().Unix()-int64(head.MinTimestamp()) < int64(build.BlockDelaySecs) { + if !watch && time.Now().Unix()-int64(head.MinTimestamp()) < int64(build.BlockDelaySecs) { fmt.Println("\nDone!") return nil } diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index ccea707af..326a39e5e 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -155,7 +155,7 @@ var initCmd = &cli.Command{ log.Info("Checking full node sync status") if !cctx.Bool("genesis-miner") && !cctx.Bool("nosync") { - if err := lcli.SyncWait(ctx, api); err != nil { + if err := lcli.SyncWait(ctx, api, false); err != nil { return xerrors.Errorf("sync wait: %w", err) } } diff --git a/cmd/lotus-storage-miner/init_restore.go b/cmd/lotus-storage-miner/init_restore.go index bdbb99fe0..83a9ad87c 100644 --- a/cmd/lotus-storage-miner/init_restore.go +++ b/cmd/lotus-storage-miner/init_restore.go @@ -72,7 +72,7 @@ var initRestoreCmd = &cli.Command{ } if !cctx.Bool("nosync") { - if err := lcli.SyncWait(ctx, api); err != nil { + if err := lcli.SyncWait(ctx, api, false); err != nil { return xerrors.Errorf("sync wait: %w", err) } } diff --git a/cmd/lotus-storage-miner/run.go b/cmd/lotus-storage-miner/run.go index 98a9cfaba..c043bd903 100644 --- a/cmd/lotus-storage-miner/run.go +++ b/cmd/lotus-storage-miner/run.go @@ -84,7 +84,7 @@ var runCmd = &cli.Command{ log.Info("Checking full node sync status") if !cctx.Bool("nosync") { - if err := lcli.SyncWait(ctx, nodeApi); err != nil { + if err := lcli.SyncWait(ctx, nodeApi, false); err != nil { return xerrors.Errorf("sync wait: %w", err) } } From b060569fe92cb374ef79c22df2dab40e93c43477 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 14 Oct 2020 19:55:36 +0200 Subject: [PATCH 753/885] Revert "Add support for /https, /http, /wss API multiaddresses." --- cli/util/apiinfo.go | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/cli/util/apiinfo.go b/cli/util/apiinfo.go index dec4c2e19..1f9a83769 100644 --- a/cli/util/apiinfo.go +++ b/cli/util/apiinfo.go @@ -44,22 +44,7 @@ func (a APIInfo) DialArgs() (string, error) { return "", err } - protocol := "ws" - - // If the user specifies the multiaddress as - // /something/tcp/1234/http or/something/tcp/1234/https - // or /something/tcp/1234/wss then honor that. - for _, p := range []int{ - multiaddr.P_HTTP, - multiaddr.P_HTTPS, - multiaddr.P_WSS, - } { - if _, err := ma.ValueForProtocol(p); err == nil { - protocol = multiaddr.ProtocolWithCode(p).Name - break - } - } - return protocol + "://" + addr + "/rpc/v0", nil + return "ws://" + addr + "/rpc/v0", nil } _, err = url.Parse(a.Addr) From 8df58064e3da1a19cc9ae1e869443143b1636be6 Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Wed, 14 Oct 2020 13:20:18 -0700 Subject: [PATCH 754/885] feat(markets): update to 0.9.1 bugfix release --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 0118baa88..c3935be19 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 github.com/filecoin-project/go-data-transfer v0.9.0 github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f - github.com/filecoin-project/go-fil-markets v0.9.0 + github.com/filecoin-project/go-fil-markets v0.9.1 github.com/filecoin-project/go-jsonrpc v0.1.2-0.20201008195726-68c6a2704e49 github.com/filecoin-project/go-multistore v0.0.3 github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 diff --git a/go.sum b/go.sum index 2b0859bfd..feff6abbd 100644 --- a/go.sum +++ b/go.sum @@ -246,8 +246,8 @@ github.com/filecoin-project/go-ds-versioning v0.1.0 h1:y/X6UksYTsK8TLCI7rttCKEvl github.com/filecoin-project/go-ds-versioning v0.1.0/go.mod h1:mp16rb4i2QPmxBnmanUx8i/XANp+PFCCJWiAb+VW4/s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f h1:GxJzR3oRIMTPtpZ0b7QF8FKPK6/iPAc7trhlL5k/g+s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= -github.com/filecoin-project/go-fil-markets v0.9.0 h1:RVdMKeOdRtpHawB14+uvJhvq243/UfZ4g3yFa/6qekk= -github.com/filecoin-project/go-fil-markets v0.9.0/go.mod h1:h+bJ/IUnYjnW5HMKyt9JQSnhslqetkpuzwwugc3K8vM= +github.com/filecoin-project/go-fil-markets v0.9.1 h1:MgO+UkpreD6x8DV2Zkw2xlBogixfpw9/wf4+nBii7bU= +github.com/filecoin-project/go-fil-markets v0.9.1/go.mod h1:h+bJ/IUnYjnW5HMKyt9JQSnhslqetkpuzwwugc3K8vM= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24= From 470a565c349cd150675b01607ea08cd995b407ee Mon Sep 17 00:00:00 2001 From: frrist Date: Wed, 14 Oct 2020 14:27:35 -0700 Subject: [PATCH 755/885] fix: return true when deadlines changed --- chain/actors/builtin/miner/v0.go | 2 +- chain/actors/builtin/miner/v2.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/chain/actors/builtin/miner/v0.go b/chain/actors/builtin/miner/v0.go index 14718d002..1b1c817b6 100644 --- a/chain/actors/builtin/miner/v0.go +++ b/chain/actors/builtin/miner/v0.go @@ -274,7 +274,7 @@ func (s *state0) DeadlinesChanged(other State) (bool, error) { return true, nil } - return s.State.Deadlines.Equals(other0.Deadlines), nil + return !s.State.Deadlines.Equals(other0.Deadlines), nil } func (s *state0) Info() (MinerInfo, error) { diff --git a/chain/actors/builtin/miner/v2.go b/chain/actors/builtin/miner/v2.go index 9e599f891..7217f6b58 100644 --- a/chain/actors/builtin/miner/v2.go +++ b/chain/actors/builtin/miner/v2.go @@ -273,7 +273,7 @@ func (s *state2) DeadlinesChanged(other State) (bool, error) { return true, nil } - return s.State.Deadlines.Equals(other2.Deadlines), nil + return !s.State.Deadlines.Equals(other2.Deadlines), nil } func (s *state2) Info() (MinerInfo, error) { From ea1349205b69c548b35b774830c592ba8bdd6997 Mon Sep 17 00:00:00 2001 From: frrist Date: Wed, 14 Oct 2020 14:37:33 -0700 Subject: [PATCH 756/885] fix: return true when partitions changed --- chain/actors/builtin/miner/v0.go | 2 +- chain/actors/builtin/miner/v2.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/chain/actors/builtin/miner/v0.go b/chain/actors/builtin/miner/v0.go index 1b1c817b6..c79f4a2f7 100644 --- a/chain/actors/builtin/miner/v0.go +++ b/chain/actors/builtin/miner/v0.go @@ -370,7 +370,7 @@ func (d *deadline0) PartitionsChanged(other Deadline) (bool, error) { return true, nil } - return d.Deadline.Partitions.Equals(other0.Deadline.Partitions), nil + return !d.Deadline.Partitions.Equals(other0.Deadline.Partitions), nil } func (d *deadline0) PostSubmissions() (bitfield.BitField, error) { diff --git a/chain/actors/builtin/miner/v2.go b/chain/actors/builtin/miner/v2.go index 7217f6b58..c686fa483 100644 --- a/chain/actors/builtin/miner/v2.go +++ b/chain/actors/builtin/miner/v2.go @@ -369,7 +369,7 @@ func (d *deadline2) PartitionsChanged(other Deadline) (bool, error) { return true, nil } - return d.Deadline.Partitions.Equals(other2.Deadline.Partitions), nil + return !d.Deadline.Partitions.Equals(other2.Deadline.Partitions), nil } func (d *deadline2) PostSubmissions() (bitfield.BitField, error) { From bbc6de94fc5ce48aa18928827ff6485f96c7f27d Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 14 Oct 2020 12:23:11 -0700 Subject: [PATCH 757/885] write messages to a temp blockstore when validating --- chain/sync.go | 86 ++++++++++++++++++++++++++++----------------------- 1 file changed, 47 insertions(+), 39 deletions(-) diff --git a/chain/sync.go b/chain/sync.go index dd83bf319..97915d8df 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -323,25 +323,35 @@ func (syncer *Syncer) ValidateMsgMeta(fblk *types.FullBlock) error { return xerrors.Errorf("block %s has too many messages (%d)", fblk.Header.Cid(), msgc) } - // Collect the CIDs of both types of messages separately: BLS and Secpk. - var bcids, scids []cid.Cid - for _, m := range fblk.BlsMessages { - bcids = append(bcids, m.Cid()) - } - - for _, m := range fblk.SecpkMessages { - scids = append(scids, m.Cid()) - } - // TODO: IMPORTANT(GARBAGE). These message puts and the msgmeta // computation need to go into the 'temporary' side of the blockstore when // we implement that - blockstore := syncer.store.Blockstore() - bs := cbor.NewCborStore(blockstore) + // We use a temporary bstore here to avoid writing intermediate pieces + // into the blockstore. + blockstore := bstore.NewTemporary() + cst := cbor.NewCborStore(blockstore) + + var bcids, scids []cid.Cid + + for _, m := range fblk.BlsMessages { + c, err := store.PutMessage(blockstore, m) + if err != nil { + return xerrors.Errorf("putting bls message to blockstore after msgmeta computation: %w", err) + } + bcids = append(bcids, c) + } + + for _, m := range fblk.SecpkMessages { + c, err := store.PutMessage(blockstore, m) + if err != nil { + return xerrors.Errorf("putting bls message to blockstore after msgmeta computation: %w", err) + } + scids = append(scids, c) + } // Compute the root CID of the combined message trie. - smroot, err := computeMsgMeta(bs, bcids, scids) + smroot, err := computeMsgMeta(cst, bcids, scids) if err != nil { return xerrors.Errorf("validating msgmeta, compute failed: %w", err) } @@ -351,21 +361,8 @@ func (syncer *Syncer) ValidateMsgMeta(fblk *types.FullBlock) error { return xerrors.Errorf("messages in full block did not match msgmeta root in header (%s != %s)", fblk.Header.Messages, smroot) } - for _, m := range fblk.BlsMessages { - _, err := store.PutMessage(blockstore, m) - if err != nil { - return xerrors.Errorf("putting bls message to blockstore after msgmeta computation: %w", err) - } - } - - for _, m := range fblk.SecpkMessages { - _, err := store.PutMessage(blockstore, m) - if err != nil { - return xerrors.Errorf("putting bls message to blockstore after msgmeta computation: %w", err) - } - } - - return nil + // Finally, flush. + return vm.Copy(context.TODO(), blockstore, syncer.store.Blockstore(), smroot) } func (syncer *Syncer) LocalPeer() peer.ID { @@ -1064,8 +1061,7 @@ func (syncer *Syncer) checkBlockMessages(ctx context.Context, b *types.FullBlock return err } - cst := cbor.NewCborStore(syncer.store.Blockstore()) - st, err := state.LoadStateTree(cst, stateroot) + st, err := state.LoadStateTree(syncer.store.Store(ctx), stateroot) if err != nil { return xerrors.Errorf("failed to load base state tree: %w", err) } @@ -1111,21 +1107,28 @@ func (syncer *Syncer) checkBlockMessages(ctx context.Context, b *types.FullBlock return nil } - store := adt0.WrapStore(ctx, cst) + // Validate message arrays in a temporary blockstore. + tmpbs := bstore.NewTemporary() + tmpstore := adt0.WrapStore(ctx, cbor.NewCborStore(tmpbs)) - bmArr := adt0.MakeEmptyArray(store) + bmArr := adt0.MakeEmptyArray(tmpstore) for i, m := range b.BlsMessages { if err := checkMsg(m); err != nil { return xerrors.Errorf("block had invalid bls message at index %d: %w", i, err) } - c := cbg.CborCid(m.Cid()) - if err := bmArr.Set(uint64(i), &c); err != nil { + c, err := store.PutMessage(tmpbs, m) + if err != nil { + return xerrors.Errorf("failed to store message %s: %w", m.Cid(), err) + } + + k := cbg.CborCid(c) + if err := bmArr.Set(uint64(i), &k); err != nil { return xerrors.Errorf("failed to put bls message at index %d: %w", i, err) } } - smArr := adt0.MakeEmptyArray(store) + smArr := adt0.MakeEmptyArray(tmpstore) for i, m := range b.SecpkMessages { if err := checkMsg(m); err != nil { return xerrors.Errorf("block had invalid secpk message at index %d: %w", i, err) @@ -1142,8 +1145,12 @@ func (syncer *Syncer) checkBlockMessages(ctx context.Context, b *types.FullBlock return xerrors.Errorf("secpk message %s has invalid signature: %w", m.Cid(), err) } - c := cbg.CborCid(m.Cid()) - if err := smArr.Set(uint64(i), &c); err != nil { + c, err := store.PutMessage(tmpbs, m) + if err != nil { + return xerrors.Errorf("failed to store message %s: %w", m.Cid(), err) + } + k := cbg.CborCid(c) + if err := smArr.Set(uint64(i), &k); err != nil { return xerrors.Errorf("failed to put secpk message at index %d: %w", i, err) } } @@ -1158,7 +1165,7 @@ func (syncer *Syncer) checkBlockMessages(ctx context.Context, b *types.FullBlock return err } - mrcid, err := cst.Put(ctx, &types.MsgMeta{ + mrcid, err := tmpstore.Put(ctx, &types.MsgMeta{ BlsMessages: bmroot, SecpkMessages: smroot, }) @@ -1170,7 +1177,8 @@ func (syncer *Syncer) checkBlockMessages(ctx context.Context, b *types.FullBlock return fmt.Errorf("messages didnt match message root in header") } - return nil + // Finally, flush. + return vm.Copy(ctx, tmpbs, syncer.store.Blockstore(), mrcid) } func (syncer *Syncer) verifyBlsAggregate(ctx context.Context, sig *crypto.Signature, msgs []cid.Cid, pubks [][]byte) error { From e2fbbdcb15d30c79e02810a5bca1d8a5467e5c4b Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 14 Oct 2020 12:23:30 -0700 Subject: [PATCH 758/885] reject messages with invalid CIDs These can't possibly be valid messages, so we should just drop the block. --- chain/sub/incoming.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/chain/sub/incoming.go b/chain/sub/incoming.go index 68ee5e20c..99ca2fc65 100644 --- a/chain/sub/incoming.go +++ b/chain/sub/incoming.go @@ -37,6 +37,7 @@ import ( "github.com/filecoin-project/lotus/lib/bufbstore" "github.com/filecoin-project/lotus/lib/sigs" "github.com/filecoin-project/lotus/metrics" + "github.com/filecoin-project/lotus/node/impl/client" ) var log = logging.Logger("sub") @@ -44,6 +45,13 @@ var log = logging.Logger("sub") var ErrSoftFailure = errors.New("soft validation failure") var ErrInsufficientPower = errors.New("incoming block's miner does not have minimum power") +var msgCidPrefix = cid.Prefix{ + Version: 1, + Codec: cid.DagCBOR, + MhType: client.DefaultHashFunction, + MhLength: 32, +} + func HandleIncomingBlocks(ctx context.Context, bsub *pubsub.Subscription, s *chain.Syncer, bs bserv.BlockService, cmgr connmgr.ConnManager) { // Timeout after (block time + propagation delay). This is useless at // this point. @@ -168,6 +176,9 @@ func fetchCids( cidIndex := make(map[cid.Cid]int) for i, c := range cids { + if c.Prefix() != msgCidPrefix { + return fmt.Errorf("invalid msg CID: %s", c) + } cidIndex[c] = i } if len(cids) != len(cidIndex) { From 4b38809c0b4cab14fef028c24502c096937c1c7b Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 12 Oct 2020 15:52:38 -0700 Subject: [PATCH 759/885] in-memory blockstore Instead of using an in-memory datastore and dealing with the overhead of computing datastore keys, creating new blocks, etc, use an in-memory blockstore. --- lib/blockstore/blockstore.go | 9 ++-- lib/blockstore/memstore.go | 80 ++++++++++++++++++++++++++++++++++++ lib/blockstore/syncstore.go | 68 ++++++++++++++++++++++++++++++ lib/bufbstore/buf_bstore.go | 4 +- 4 files changed, 155 insertions(+), 6 deletions(-) create mode 100644 lib/blockstore/memstore.go create mode 100644 lib/blockstore/syncstore.go diff --git a/lib/blockstore/blockstore.go b/lib/blockstore/blockstore.go index 9e74f4373..f274cd708 100644 --- a/lib/blockstore/blockstore.go +++ b/lib/blockstore/blockstore.go @@ -18,19 +18,18 @@ import ( "context" ds "github.com/ipfs/go-datastore" - dssync "github.com/ipfs/go-datastore/sync" blockstore "github.com/ipfs/go-ipfs-blockstore" ) // NewTemporary returns a temporary blockstore. -func NewTemporary() blockstore.Blockstore { - return NewBlockstore(ds.NewMapDatastore()) +func NewTemporary() MemStore { + return make(MemStore) } // NewTemporarySync returns a thread-safe temporary blockstore. -func NewTemporarySync() blockstore.Blockstore { - return NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore())) +func NewTemporarySync() *SyncStore { + return &SyncStore{bs: make(MemStore)} } // WrapIDStore wraps the underlying blockstore in an "identity" blockstore. diff --git a/lib/blockstore/memstore.go b/lib/blockstore/memstore.go new file mode 100644 index 000000000..9745d6f03 --- /dev/null +++ b/lib/blockstore/memstore.go @@ -0,0 +1,80 @@ +package blockstore + +import ( + "context" + + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + blockstore "github.com/ipfs/go-ipfs-blockstore" +) + +type MemStore map[cid.Cid]blocks.Block + +func (m MemStore) DeleteBlock(k cid.Cid) error { + delete(m, k) + return nil +} +func (m MemStore) Has(k cid.Cid) (bool, error) { + _, ok := m[k] + return ok, nil +} +func (m MemStore) Get(k cid.Cid) (blocks.Block, error) { + b, ok := m[k] + if !ok { + return nil, blockstore.ErrNotFound + } + return b, nil +} + +// GetSize returns the CIDs mapped BlockSize +func (m MemStore) GetSize(k cid.Cid) (int, error) { + b, ok := m[k] + if !ok { + return 0, blockstore.ErrNotFound + } + return len(b.RawData()), nil +} + +// Put puts a given block to the underlying datastore +func (m MemStore) Put(b blocks.Block) error { + // Convert to a basic block for safety, but try to reuse the existing + // block if it's already a basic block. + k := b.Cid() + if _, ok := b.(*blocks.BasicBlock); !ok { + // If we already have the block, abort. + if _, ok := m[k]; ok { + return nil + } + // the error is only for debugging. + b, _ = blocks.NewBlockWithCid(b.RawData(), b.Cid()) + } + m[b.Cid()] = b + return nil +} + +// PutMany puts a slice of blocks at the same time using batching +// capabilities of the underlying datastore whenever possible. +func (m MemStore) PutMany(bs []blocks.Block) error { + for _, b := range bs { + _ = m.Put(b) // can't fail + } + return nil +} + +// AllKeysChan returns a channel from which +// the CIDs in the Blockstore can be read. It should respect +// the given context, closing the channel if it becomes Done. +func (m MemStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { + ch := make(chan cid.Cid, len(m)) + for k := range m { + ch <- k + } + close(ch) + return ch, nil +} + +// HashOnRead specifies if every read block should be +// rehashed to make sure it matches its CID. +func (m MemStore) HashOnRead(enabled bool) { + // no-op +} diff --git a/lib/blockstore/syncstore.go b/lib/blockstore/syncstore.go new file mode 100644 index 000000000..be9f6b5c4 --- /dev/null +++ b/lib/blockstore/syncstore.go @@ -0,0 +1,68 @@ +package blockstore + +import ( + "context" + "sync" + + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" +) + +type SyncStore struct { + mu sync.RWMutex + bs MemStore // specifically use a memStore to save indirection overhead. +} + +func (m *SyncStore) DeleteBlock(k cid.Cid) error { + m.mu.Lock() + defer m.mu.Unlock() + return m.bs.DeleteBlock(k) +} +func (m *SyncStore) Has(k cid.Cid) (bool, error) { + m.mu.RLock() + defer m.mu.RUnlock() + return m.bs.Has(k) +} +func (m *SyncStore) Get(k cid.Cid) (blocks.Block, error) { + m.mu.RLock() + defer m.mu.RUnlock() + return m.bs.Get(k) +} + +// GetSize returns the CIDs mapped BlockSize +func (m *SyncStore) GetSize(k cid.Cid) (int, error) { + m.mu.RLock() + defer m.mu.RUnlock() + return m.bs.GetSize(k) +} + +// Put puts a given block to the underlying datastore +func (m *SyncStore) Put(b blocks.Block) error { + m.mu.Lock() + defer m.mu.Unlock() + return m.bs.Put(b) +} + +// PutMany puts a slice of blocks at the same time using batching +// capabilities of the underlying datastore whenever possible. +func (m *SyncStore) PutMany(bs []blocks.Block) error { + m.mu.Lock() + defer m.mu.Unlock() + return m.bs.PutMany(bs) +} + +// AllKeysChan returns a channel from which +// the CIDs in the Blockstore can be read. It should respect +// the given context, closing the channel if it becomes Done. +func (m *SyncStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { + m.mu.RLock() + defer m.mu.RUnlock() + // this blockstore implementation doesn't do any async work. + return m.bs.AllKeysChan(ctx) +} + +// HashOnRead specifies if every read block should be +// rehashed to make sure it matches its CID. +func (m *SyncStore) HashOnRead(enabled bool) { + // noop +} diff --git a/lib/bufbstore/buf_bstore.go b/lib/bufbstore/buf_bstore.go index a766c2b52..4ea746444 100644 --- a/lib/bufbstore/buf_bstore.go +++ b/lib/bufbstore/buf_bstore.go @@ -19,10 +19,12 @@ type BufferedBS struct { } func NewBufferedBstore(base bstore.Blockstore) *BufferedBS { - buf := bstore.NewTemporary() + var buf bstore.Blockstore if os.Getenv("LOTUS_DISABLE_VM_BUF") == "iknowitsabadidea" { log.Warn("VM BLOCKSTORE BUFFERING IS DISABLED") buf = base + } else { + buf = bstore.NewTemporary() } return &BufferedBS{ From 15fe998c68b3980cc064feabfe770eded2e77fde Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 14 Oct 2020 14:02:19 -0700 Subject: [PATCH 760/885] add an timed-cache blockstore This blockstore lets us write to a temporary scratch location where blocks older than the specified cache time are automatically cleared. --- lib/timedbs/timedbs.go | 156 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 lib/timedbs/timedbs.go diff --git a/lib/timedbs/timedbs.go b/lib/timedbs/timedbs.go new file mode 100644 index 000000000..bb03a59e9 --- /dev/null +++ b/lib/timedbs/timedbs.go @@ -0,0 +1,156 @@ +package timedbs + +import ( + "context" + "fmt" + "sync" + "time" + + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + "go.uber.org/multierr" + + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/lib/blockstore" +) + +// TimedCacheBS is a blockstore that keeps blocks for at least the specified +// caching interval before discarding them. Garbage collection must be started +// and stopped by calling Start/Stop. +// +// Under the covers, it's implemented with an active and an inactive blockstore +// that are rotated every cache time interval. This means all blocks will be +// stored at most 2x the cache interval. +type TimedCacheBS struct { + mu sync.RWMutex + active, inactive blockstore.MemStore + + interval time.Duration + closeCh chan struct{} +} + +func NewTimedCacheBS(cacheTime time.Duration) *TimedCacheBS { + return &TimedCacheBS{ + active: blockstore.NewTemporary(), + inactive: blockstore.NewTemporary(), + interval: cacheTime, + } +} + +func (t *TimedCacheBS) Start(ctx context.Context) error { + t.mu.Lock() + defer t.mu.Unlock() + if t.closeCh != nil { + return fmt.Errorf("already started") + } + t.closeCh = make(chan struct{}) + go func() { + ticker := build.Clock.Ticker(t.interval) + defer ticker.Stop() + for { + select { + case <-ticker.C: + t.rotate() + case <-t.closeCh: + return + } + } + }() + return nil +} + +func (t *TimedCacheBS) Stop(ctx context.Context) error { + t.mu.Lock() + defer t.mu.Unlock() + if t.closeCh == nil { + return fmt.Errorf("not started started") + } + select { + case <-t.closeCh: + // already closed + default: + close(t.closeCh) + } + return nil +} + +func (t *TimedCacheBS) rotate() { + newBs := blockstore.NewTemporary() + + t.mu.Lock() + t.inactive, t.active = t.active, newBs + t.mu.Unlock() +} + +func (t *TimedCacheBS) Put(b blocks.Block) error { + // Don't check the inactive set here. We want to keep this block for at + // least one interval. + t.mu.Lock() + defer t.mu.Unlock() + return t.active.Put(b) +} + +func (t *TimedCacheBS) PutMany(bs []blocks.Block) error { + t.mu.Lock() + defer t.mu.Unlock() + return t.active.PutMany(bs) +} + +func (t *TimedCacheBS) Get(k cid.Cid) (blocks.Block, error) { + t.mu.RLock() + defer t.mu.RUnlock() + b, err := t.active.Get(k) + if err == blockstore.ErrNotFound { + b, err = t.inactive.Get(k) + } + return b, err +} + +func (t *TimedCacheBS) GetSize(k cid.Cid) (int, error) { + t.mu.RLock() + defer t.mu.RUnlock() + size, err := t.active.GetSize(k) + if err == blockstore.ErrNotFound { + size, err = t.inactive.GetSize(k) + } + return size, err +} + +func (t *TimedCacheBS) Has(k cid.Cid) (bool, error) { + t.mu.RLock() + defer t.mu.RUnlock() + if has, err := t.active.Has(k); err != nil { + return false, err + } else if has { + return true, nil + } + return t.inactive.Has(k) +} + +func (t *TimedCacheBS) HashOnRead(_ bool) { + // no-op +} + +func (t *TimedCacheBS) DeleteBlock(k cid.Cid) error { + t.mu.Lock() + defer t.mu.Unlock() + return multierr.Combine(t.active.DeleteBlock(k), t.inactive.DeleteBlock(k)) +} + +func (t *TimedCacheBS) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { + t.mu.RLock() + defer t.mu.RUnlock() + + ch := make(chan cid.Cid, len(t.active)+len(t.inactive)) + for c := range t.active { + ch <- c + } + for c := range t.inactive { + if _, ok := t.active[c]; ok { + continue + } + ch <- c + } + close(ch) + return ch, nil +} From ddade32bd37654f343e13ba4ad0983d03f1f05e9 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 14 Oct 2020 13:49:43 -0700 Subject: [PATCH 761/885] write bitswap blocks into a temporary, in-memory block cache If they end up validating, we'll write them back to the underlying blockstore. Otherwise, there's no reason to keep them. --- node/modules/chain.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/node/modules/chain.go b/node/modules/chain.go index f563b4cdd..ce3e9f749 100644 --- a/node/modules/chain.go +++ b/node/modules/chain.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "os" + "time" "github.com/ipfs/go-bitswap" "github.com/ipfs/go-bitswap/network" @@ -30,6 +31,8 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" "github.com/filecoin-project/lotus/lib/blockstore" + "github.com/filecoin-project/lotus/lib/bufbstore" + "github.com/filecoin-project/lotus/lib/timedbs" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/helpers" "github.com/filecoin-project/lotus/node/repo" @@ -41,8 +44,15 @@ func ChainBitswap(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, rt r bitswapNetwork := network.NewFromIpfsHost(host, rt, network.Prefix("/chain")) bitswapOptions := []bitswap.Option{bitswap.ProvideEnabled(false)} + // Write all incoming bitswap blocks into a temporary blockstore for two + // block times. If they validate, they'll be persisted later. + cache := timedbs.NewTimedCacheBS(2 * time.Duration(build.BlockDelaySecs) * time.Second) + lc.Append(fx.Hook{OnStop: cache.Stop, OnStart: cache.Start}) + + bitswapBs := bufbstore.NewTieredBstore(bs, cache) + // Use just exch.Close(), closing the context is not needed - exch := bitswap.New(mctx, bitswapNetwork, bs, bitswapOptions...) + exch := bitswap.New(mctx, bitswapNetwork, bitswapBs, bitswapOptions...) lc.Append(fx.Hook{ OnStop: func(ctx context.Context) error { return exch.Close() From 28823fb8e9f7f276e915f0935e424c404287e957 Mon Sep 17 00:00:00 2001 From: frrist Date: Wed, 14 Oct 2020 14:50:59 -0700 Subject: [PATCH 762/885] fix: bad indirection of deadline diffing --- chain/actors/builtin/miner/diff_deadlines.go | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/chain/actors/builtin/miner/diff_deadlines.go b/chain/actors/builtin/miner/diff_deadlines.go index e1e839960..7d686ece5 100644 --- a/chain/actors/builtin/miner/diff_deadlines.go +++ b/chain/actors/builtin/miner/diff_deadlines.go @@ -7,9 +7,9 @@ import ( "github.com/filecoin-project/go-state-types/exitcode" ) -type DeadlinesDiff map[uint64]*DeadlineDiff +type DeadlinesDiff map[uint64]DeadlineDiff -func DiffDeadlines(pre, cur State) (*DeadlinesDiff, error) { +func DiffDeadlines(pre, cur State) (DeadlinesDiff, error) { changed, err := pre.DeadlinesChanged(cur) if err != nil { return nil, err @@ -18,11 +18,7 @@ func DiffDeadlines(pre, cur State) (*DeadlinesDiff, error) { return nil, nil } - numDl, err := pre.NumDeadlines() - if err != nil { - return nil, err - } - dlDiff := make(DeadlinesDiff, numDl) + dlDiff := make(DeadlinesDiff) if err := pre.ForEachDeadline(func(idx uint64, preDl Deadline) error { curDl, err := cur.LoadDeadline(idx) if err != nil { @@ -39,12 +35,12 @@ func DiffDeadlines(pre, cur State) (*DeadlinesDiff, error) { }); err != nil { return nil, err } - return &dlDiff, nil + return dlDiff, nil } type DeadlineDiff map[uint64]*PartitionDiff -func DiffDeadline(pre, cur Deadline) (*DeadlineDiff, error) { +func DiffDeadline(pre, cur Deadline) (DeadlineDiff, error) { changed, err := pre.PartitionsChanged(cur) if err != nil { return nil, err @@ -104,7 +100,7 @@ func DiffDeadline(pre, cur Deadline) (*DeadlineDiff, error) { return nil, err } - return &partDiff, nil + return partDiff, nil } type PartitionDiff struct { From 811f1304e673c2208a8902c761da9d3548473d3d Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 14 Oct 2020 16:17:55 -0700 Subject: [PATCH 763/885] test timed cache blockstore --- lib/timedbs/timedbs_test.go | 78 +++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 lib/timedbs/timedbs_test.go diff --git a/lib/timedbs/timedbs_test.go b/lib/timedbs/timedbs_test.go new file mode 100644 index 000000000..75f298a1d --- /dev/null +++ b/lib/timedbs/timedbs_test.go @@ -0,0 +1,78 @@ +package timedbs_test + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lotus/lib/timedbs" +) + +func TestTimedBSSimple(t *testing.T) { + tc := timedbs.NewTimedCacheBS(3 * time.Millisecond) + _ = tc.Start(context.Background()) + defer func() { + _ = tc.Stop(context.Background()) + }() + + b1 := blocks.NewBlock([]byte("foo")) + require.NoError(t, tc.Put(b1)) + + b2 := blocks.NewBlock([]byte("bar")) + require.NoError(t, tc.Put(b2)) + + b3 := blocks.NewBlock([]byte("baz")) + + b1out, err := tc.Get(b1.Cid()) + require.NoError(t, err) + require.Equal(t, b1.RawData(), b1out.RawData()) + + has, err := tc.Has(b1.Cid()) + require.NoError(t, err) + require.True(t, has) + + time.Sleep(4 * time.Millisecond) + + // We should still have everything. + has, err = tc.Has(b1.Cid()) + require.NoError(t, err) + require.True(t, has) + + has, err = tc.Has(b2.Cid()) + require.NoError(t, err) + require.True(t, has) + + // extend b2, add b3. + require.NoError(t, tc.Put(b2)) + require.NoError(t, tc.Put(b3)) + + // all keys once. + allKeys, err := tc.AllKeysChan(context.Background()) + var ks []cid.Cid + for k := range allKeys { + ks = append(ks, k) + } + require.NoError(t, err) + require.ElementsMatch(t, ks, []cid.Cid{b1.Cid(), b2.Cid(), b3.Cid()}) + + time.Sleep(4 * time.Millisecond) + + // should still have b2, and b3, but not b1 + + has, err = tc.Has(b1.Cid()) + require.NoError(t, err) + require.False(t, has) + + has, err = tc.Has(b2.Cid()) + require.NoError(t, err) + require.True(t, has) + + has, err = tc.Has(b3.Cid()) + require.NoError(t, err) + require.True(t, has) +} From 86e30729bf142a2c8db12b9b26ee63c83de279c6 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 14 Oct 2020 16:51:26 -0700 Subject: [PATCH 764/885] give the TimeCacheBS tests a bit more time Otherwise, they flake once in a while. --- lib/timedbs/timedbs_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/timedbs/timedbs_test.go b/lib/timedbs/timedbs_test.go index 75f298a1d..2126b9287 100644 --- a/lib/timedbs/timedbs_test.go +++ b/lib/timedbs/timedbs_test.go @@ -14,7 +14,7 @@ import ( ) func TestTimedBSSimple(t *testing.T) { - tc := timedbs.NewTimedCacheBS(3 * time.Millisecond) + tc := timedbs.NewTimedCacheBS(10 * time.Millisecond) _ = tc.Start(context.Background()) defer func() { _ = tc.Stop(context.Background()) @@ -36,7 +36,7 @@ func TestTimedBSSimple(t *testing.T) { require.NoError(t, err) require.True(t, has) - time.Sleep(4 * time.Millisecond) + time.Sleep(15 * time.Millisecond) // We should still have everything. has, err = tc.Has(b1.Cid()) @@ -60,7 +60,7 @@ func TestTimedBSSimple(t *testing.T) { require.NoError(t, err) require.ElementsMatch(t, ks, []cid.Cid{b1.Cid(), b2.Cid(), b3.Cid()}) - time.Sleep(4 * time.Millisecond) + time.Sleep(10 * time.Millisecond) // should still have b2, and b3, but not b1 From 698b49b7ef5dfcce4a6eb277b8d38ad9c094887c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 15 Oct 2020 02:46:47 +0200 Subject: [PATCH 765/885] Improve gas defaults --- chain/messagepool/messagepool.go | 4 +++- chain/types/fil.go | 9 +++++++++ node/config/def.go | 10 +++++----- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/chain/messagepool/messagepool.go b/chain/messagepool/messagepool.go index 83b1a7f29..88e721ef4 100644 --- a/chain/messagepool/messagepool.go +++ b/chain/messagepool/messagepool.go @@ -59,6 +59,8 @@ var MaxUntrustedActorPendingMessages = 10 var MaxNonceGap = uint64(4) +var DefaultMaxFee = abi.TokenAmount(types.MustParseFIL("0.007")) + var ( ErrMessageTooBig = errors.New("message too big") @@ -183,7 +185,7 @@ func ComputeMinRBF(curPrem abi.TokenAmount) abi.TokenAmount { func CapGasFee(msg *types.Message, maxFee abi.TokenAmount) { if maxFee.Equals(big.Zero()) { - maxFee = types.NewInt(build.FilecoinPrecision / 10) + maxFee = DefaultMaxFee } gl := types.NewInt(uint64(msg.GasLimit)) diff --git a/chain/types/fil.go b/chain/types/fil.go index 7eac8ce93..0ea77660c 100644 --- a/chain/types/fil.go +++ b/chain/types/fil.go @@ -81,5 +81,14 @@ func ParseFIL(s string) (FIL, error) { return FIL{r.Num()}, nil } +func MustParseFIL(s string) FIL { + n, err := ParseFIL(s) + if err != nil { + panic(err) + } + + return n +} + var _ encoding.TextMarshaler = (*FIL)(nil) var _ encoding.TextUnmarshaler = (*FIL)(nil) diff --git a/node/config/def.go b/node/config/def.go index 7eb7e19f0..f1bb1edef 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -182,11 +182,11 @@ func DefaultStorageMiner() *StorageMiner { }, Fees: MinerFeeConfig{ - MaxPreCommitGasFee: types.FIL(types.BigDiv(types.FromFil(1), types.NewInt(20))), // 0.05 - MaxCommitGasFee: types.FIL(types.BigDiv(types.FromFil(1), types.NewInt(20))), - MaxWindowPoStGasFee: types.FIL(types.FromFil(50)), - MaxPublishDealsFee: types.FIL(types.BigDiv(types.FromFil(1), types.NewInt(33))), // 0.03ish - MaxMarketBalanceAddFee: types.FIL(types.BigDiv(types.FromFil(1), types.NewInt(100))), // 0.01 + MaxPreCommitGasFee: types.MustParseFIL("0.025"), + MaxCommitGasFee: types.MustParseFIL("0.05"), + MaxWindowPoStGasFee: types.MustParseFIL("5"), + MaxPublishDealsFee: types.MustParseFIL("0.05"), + MaxMarketBalanceAddFee: types.MustParseFIL("0.007"), }, } cfg.Common.API.ListenAddress = "/ip4/127.0.0.1/tcp/2345/http" From df6f4a572cc0d45e5695a1135c72e3963fecc23c Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 7 Oct 2020 21:00:24 +0200 Subject: [PATCH 766/885] Change default gas premium to for 6 block inclusion Signed-off-by: Jakub Sztandera --- node/impl/full/gas.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/impl/full/gas.go b/node/impl/full/gas.go index 0cb1eb084..2e5063ccb 100644 --- a/node/impl/full/gas.go +++ b/node/impl/full/gas.go @@ -277,7 +277,7 @@ func (m *GasModule) GasEstimateMessageGas(ctx context.Context, msg *types.Messag } if msg.GasPremium == types.EmptyInt || types.BigCmp(msg.GasPremium, types.NewInt(0)) == 0 { - gasPremium, err := m.GasEstimateGasPremium(ctx, 2, msg.From, msg.GasLimit, types.TipSetKey{}) + gasPremium, err := m.GasEstimateGasPremium(ctx, 10, msg.From, msg.GasLimit, types.TipSetKey{}) if err != nil { return nil, xerrors.Errorf("estimating gas price: %w", err) } From 251374eca723959cf836e2f5cc48d585b742fe6b Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Wed, 14 Oct 2020 21:01:20 -0400 Subject: [PATCH 767/885] Lotus version 0.10.2 --- CHANGELOG.md | 18 ++++++++++++++++++ build/version.go | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c94e387c..701f221b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ # Lotus changelog +# 0.10.2 / 2020-10-14 + +This is an optional release of Lotus that updates markets to 0.9.1, which fixes an issue affecting deals that were mid-transfer when the node was upgraded to 0.9.0. This release also includes some tweaks to default gas values and minor performance improvements. + +## Changes + +- Use updated stored ask API (https://github.com/filecoin-project/lotus/pull/4384) +- tvx: trace puts to blockstore for inclusion in CAR. (https://github.com/filecoin-project/lotus/pull/4278) +- Add propose remove (https://github.com/filecoin-project/lotus/pull/4311) +- Update to 0.9.1 bugfix release (https://github.com/filecoin-project/lotus/pull/4402) +- Update drand endpoints (https://github.com/filecoin-project/lotus/pull/4125) +- fix: return true when deadlines changed (https://github.com/filecoin-project/lotus/pull/4403) +- sync wait --watch (https://github.com/filecoin-project/lotus/pull/4396) +- reduce garbage in blockstore (https://github.com/filecoin-project/lotus/pull/4406) +- give the TimeCacheBS tests a bit more time (https://github.com/filecoin-project/lotus/pull/4407) +- Improve gas defaults (https://github.com/filecoin-project/lotus/pull/4408) +- Change default gas premium to for 10 block inclusion (https://github.com/filecoin-project/lotus/pull/4222) + # 0.10.1 / 2020-10-14 This is an optional release of Lotus that updates markets to 0.9.0, which adds the ability to restart data transfers. This release also introduces Ledger support, and various UX improvements. diff --git a/build/version.go b/build/version.go index 39f9d48ff..df1fe28de 100644 --- a/build/version.go +++ b/build/version.go @@ -29,7 +29,7 @@ func buildType() string { } // BuildVersion is the local build version, set by build system -const BuildVersion = "0.10.1" +const BuildVersion = "0.10.2" func UserVersion() string { return BuildVersion + buildType() + CurrentCommit From a40fc98e5635545cb70f4fd675971fdb1df5de11 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 14 Oct 2020 17:22:39 -0700 Subject: [PATCH 768/885] fix code coverage ignore It looks like we need to use globs. --- .codecov.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.codecov.yml b/.codecov.yml index 1551f2276..a53081be7 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -1,5 +1,9 @@ comment: off ignore: - - "cbor_gen.go" + - "**/cbor_gen.go" + - "api/test/**/*" + - "api/test/*" + - "gen/**/*" + - "gen/*" github_checks: annotations: false From 3fb5334959eab2a2f8aaf5493dcef2b6ad581f1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 15 Oct 2020 03:36:55 +0200 Subject: [PATCH 769/885] mpool: Don't block node startup loading messages --- chain/messagepool/messagepool.go | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/chain/messagepool/messagepool.go b/chain/messagepool/messagepool.go index 88e721ef4..cc26be0b5 100644 --- a/chain/messagepool/messagepool.go +++ b/chain/messagepool/messagepool.go @@ -370,11 +370,23 @@ func New(api Provider, ds dtypes.MetadataDS, netName dtypes.NetworkName, j journ return err }) - if err := mp.loadLocal(); err != nil { - log.Errorf("loading local messages: %+v", err) - } + mp.curTsLk.Lock() + mp.lk.Lock() - go mp.runLoop() + go func() { + err := mp.loadLocal() + + mp.lk.Unlock() + mp.curTsLk.Unlock() + + if err != nil { + log.Errorf("loading local messages: %+v", err) + } + + log.Info("mpool ready") + + mp.runLoop() + }() return mp, nil } @@ -667,9 +679,6 @@ func (mp *MessagePool) addLoaded(m *types.SignedMessage) error { return err } - mp.curTsLk.Lock() - defer mp.curTsLk.Unlock() - curTs := mp.curTs if curTs == nil { @@ -685,9 +694,6 @@ func (mp *MessagePool) addLoaded(m *types.SignedMessage) error { return xerrors.Errorf("minimum expected nonce is %d: %w", snonce, ErrNonceTooLow) } - mp.lk.Lock() - defer mp.lk.Unlock() - _, err = mp.verifyMsgBeforeAdd(m, curTs, true) if err != nil { return err From b29e3b5242704d4a8a82f07abcc195016417a1b3 Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Wed, 14 Oct 2020 21:28:36 -0700 Subject: [PATCH 770/885] feat(markets): use build.BlockDelaySecs for start buffer instead of constant from spec-actors which uses hardcoded start epoch, use one that factors in possibility of different block time --- node/modules/storageminer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index adacf5405..cd5517a1c 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -474,7 +474,7 @@ func BasicDealFilter(user dtypes.DealFilter) func(onlineOk dtypes.ConsiderOnline // Reject if it's more than 7 days in the future // TODO: read from cfg - maxStartEpoch := earliest + abi.ChainEpoch(7*builtin.EpochsInDay) + maxStartEpoch := earliest + abi.ChainEpoch(7*builtin.SecondsInDay/build.BlockDelaySecs) if deal.Proposal.StartEpoch > maxStartEpoch { return false, fmt.Sprintf("deal start epoch is too far in the future: %s > %s", deal.Proposal.StartEpoch, maxStartEpoch), nil } From 6d771d29c7c5420d13f00de4785e689a97166ee8 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Wed, 14 Oct 2020 10:56:51 -0700 Subject: [PATCH 771/885] improve the UX of multisig approves --- cli/multisig.go | 132 ++++++++++++++++++++++++++++-------------------- 1 file changed, 76 insertions(+), 56 deletions(-) diff --git a/cli/multisig.go b/cli/multisig.go index 831b1bbe4..19a80e66f 100644 --- a/cli/multisig.go +++ b/cli/multisig.go @@ -24,6 +24,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-address" + cid "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" "github.com/urfave/cli/v2" "golang.org/x/xerrors" @@ -41,6 +42,13 @@ import ( var multisigCmd = &cli.Command{ Name: "msig", Usage: "Interact with a multisig wallet", + Flags: []cli.Flag{ + &cli.IntFlag{ + Name: "confirmations", + Usage: "number of block confirmations to wait for", + Value: int(build.MessageConfidence), + }, + }, Subcommands: []*cli.Command{ msigCreateCmd, msigInspectCmd, @@ -146,7 +154,7 @@ var msigCreateCmd = &cli.Command{ } // wait for it to get mined into a block - wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) + wait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence"))) if err != nil { return err } @@ -410,7 +418,7 @@ var msigProposeCmd = &cli.Command{ fmt.Println("send proposal in message: ", msgCid) - wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) + wait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence"))) if err != nil { return err } @@ -446,14 +454,18 @@ var msigApproveCmd = &cli.Command{ }, }, Action: func(cctx *cli.Context) error { - if cctx.Args().Len() < 5 { - return ShowHelp(cctx, fmt.Errorf("must pass multisig address, message ID, proposer address, destination, and value")) + if cctx.Args().Len() < 2 { + return ShowHelp(cctx, fmt.Errorf("must pass at least multisig address and message ID")) } if cctx.Args().Len() > 5 && cctx.Args().Len() != 7 { return ShowHelp(cctx, fmt.Errorf("usage: msig approve [ ]")) } + if cctx.Args().Len() > 2 && cctx.Args().Len() != 5 { + return ShowHelp(cctx, fmt.Errorf("usage: msig approve ")) + } + api, closer, err := GetFullNodeAPI(cctx) if err != nil { return err @@ -471,44 +483,6 @@ var msigApproveCmd = &cli.Command{ return err } - proposer, err := address.NewFromString(cctx.Args().Get(2)) - if err != nil { - return err - } - - if proposer.Protocol() != address.ID { - proposer, err = api.StateLookupID(ctx, proposer, types.EmptyTSK) - if err != nil { - return err - } - } - - dest, err := address.NewFromString(cctx.Args().Get(3)) - if err != nil { - return err - } - - value, err := types.ParseFIL(cctx.Args().Get(4)) - if err != nil { - return err - } - - var method uint64 - var params []byte - if cctx.Args().Len() == 7 { - m, err := strconv.ParseUint(cctx.Args().Get(5), 10, 64) - if err != nil { - return err - } - method = m - - p, err := hex.DecodeString(cctx.Args().Get(6)) - if err != nil { - return err - } - params = p - } - var from address.Address if cctx.IsSet("from") { f, err := address.NewFromString(cctx.String("from")) @@ -524,14 +498,60 @@ var msigApproveCmd = &cli.Command{ from = defaddr } - msgCid, err := api.MsigApproveTxnHash(ctx, msig, txid, proposer, dest, types.BigInt(value), from, method, params) - if err != nil { - return err + var msgCid cid.Cid + if cctx.Args().Len() == 2 { + msgCid, err = api.MsigApprove(ctx, msig, txid, from) + if err != nil { + return err + } + } else { + proposer, err := address.NewFromString(cctx.Args().Get(2)) + if err != nil { + return err + } + + if proposer.Protocol() != address.ID { + proposer, err = api.StateLookupID(ctx, proposer, types.EmptyTSK) + if err != nil { + return err + } + } + + dest, err := address.NewFromString(cctx.Args().Get(3)) + if err != nil { + return err + } + + value, err := types.ParseFIL(cctx.Args().Get(4)) + if err != nil { + return err + } + + var method uint64 + var params []byte + if cctx.Args().Len() == 7 { + m, err := strconv.ParseUint(cctx.Args().Get(5), 10, 64) + if err != nil { + return err + } + method = m + + p, err := hex.DecodeString(cctx.Args().Get(6)) + if err != nil { + return err + } + params = p + } + + msgCid, err = api.MsigApproveTxnHash(ctx, msig, txid, proposer, dest, types.BigInt(value), from, method, params) + if err != nil { + return err + } } fmt.Println("sent approval in message: ", msgCid) - wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) + wait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence"))) if err != nil { return err } @@ -602,7 +622,7 @@ var msigRemoveProposeCmd = &cli.Command{ fmt.Println("sent remove proposal in message: ", msgCid) - wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) + wait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence"))) if err != nil { return err } @@ -680,7 +700,7 @@ var msigAddProposeCmd = &cli.Command{ fmt.Println("sent add proposal in message: ", msgCid) - wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) + wait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence"))) if err != nil { return err } @@ -762,7 +782,7 @@ var msigAddApproveCmd = &cli.Command{ fmt.Println("sent add approval in message: ", msgCid) - wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) + wait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence"))) if err != nil { return err } @@ -839,7 +859,7 @@ var msigAddCancelCmd = &cli.Command{ fmt.Println("sent add cancellation in message: ", msgCid) - wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) + wait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence"))) if err != nil { return err } @@ -911,7 +931,7 @@ var msigSwapProposeCmd = &cli.Command{ fmt.Println("sent swap proposal in message: ", msgCid) - wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) + wait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence"))) if err != nil { return err } @@ -993,7 +1013,7 @@ var msigSwapApproveCmd = &cli.Command{ fmt.Println("sent swap approval in message: ", msgCid) - wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) + wait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence"))) if err != nil { return err } @@ -1070,7 +1090,7 @@ var msigSwapCancelCmd = &cli.Command{ fmt.Println("sent swap cancellation in message: ", msgCid) - wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) + wait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence"))) if err != nil { return err } @@ -1157,7 +1177,7 @@ var msigLockProposeCmd = &cli.Command{ fmt.Println("sent lock proposal in message: ", msgCid) - wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) + wait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence"))) if err != nil { return err } @@ -1254,7 +1274,7 @@ var msigLockApproveCmd = &cli.Command{ fmt.Println("sent lock approval in message: ", msgCid) - wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) + wait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence"))) if err != nil { return err } @@ -1346,7 +1366,7 @@ var msigLockCancelCmd = &cli.Command{ fmt.Println("sent lock cancellation in message: ", msgCid) - wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) + wait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence"))) if err != nil { return err } From 3bd2e2f7771b068f7d32e37e2813c38f58149d41 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Wed, 14 Oct 2020 22:55:23 -0700 Subject: [PATCH 772/885] fix typo --- cli/multisig.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/multisig.go b/cli/multisig.go index 19a80e66f..d606d2acb 100644 --- a/cli/multisig.go +++ b/cli/multisig.go @@ -44,7 +44,7 @@ var multisigCmd = &cli.Command{ Usage: "Interact with a multisig wallet", Flags: []cli.Flag{ &cli.IntFlag{ - Name: "confirmations", + Name: "confidence", Usage: "number of block confirmations to wait for", Value: int(build.MessageConfidence), }, From fdb115e3692493d5e74f7224aee7a0781d540838 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Wed, 14 Oct 2020 23:10:00 -0700 Subject: [PATCH 773/885] add command to propose changing multisig threshold --- cli/multisig.go | 76 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/cli/multisig.go b/cli/multisig.go index d606d2acb..73589086a 100644 --- a/cli/multisig.go +++ b/cli/multisig.go @@ -65,6 +65,7 @@ var multisigCmd = &cli.Command{ msigLockApproveCmd, msigLockCancelCmd, msigVestedCmd, + msigProposeThresholdCmd, }, } @@ -1440,3 +1441,78 @@ var msigVestedCmd = &cli.Command{ return nil }, } + +var msigProposeThresholdCmd = &cli.Command{ + Name: "propose-threshold", + Usage: "Propose setting a different signing threshold on the account", + ArgsUsage: "", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "from", + Usage: "account to send the proposal from", + }, + }, + Action: func(cctx *cli.Context) error { + if cctx.Args().Len() != 2 { + return ShowHelp(cctx, fmt.Errorf("must pass multisig address and new threshold value")) + } + + api, closer, err := GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + ctx := ReqContext(cctx) + + msig, err := address.NewFromString(cctx.Args().Get(0)) + if err != nil { + return err + } + + newM, err := strconv.ParseUint(cctx.Args().Get(2), 10, 64) + if err != nil { + return err + } + + var from address.Address + if cctx.IsSet("from") { + f, err := address.NewFromString(cctx.String("from")) + if err != nil { + return err + } + from = f + } else { + defaddr, err := api.WalletDefaultAddress(ctx) + if err != nil { + return err + } + from = defaddr + } + + params, actErr := actors.SerializeParams(&msig0.ChangeNumApprovalsThresholdParams{ + NewThreshold: newM, + }) + + if actErr != nil { + return actErr + } + + msgCid, err := api.MsigPropose(ctx, msig, msig, types.NewInt(0), from, uint64(builtin2.MethodsMultisig.ChangeNumApprovalsThreshold), params) + if err != nil { + return fmt.Errorf("failed to propose change of threshold: %w", err) + } + + fmt.Println("sent change threshold proposal in message: ", msgCid) + + wait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence"))) + if err != nil { + return err + } + + if wait.Receipt.ExitCode != 0 { + return fmt.Errorf("change threshold proposal returned exit %d", wait.Receipt.ExitCode) + } + + return nil + }, +} From ccc780f35ac27eb7136efed3b55851e3076ed68b Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Wed, 14 Oct 2020 23:16:31 -0700 Subject: [PATCH 774/885] fix wrong index --- cli/multisig.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/multisig.go b/cli/multisig.go index 73589086a..e352ed64b 100644 --- a/cli/multisig.go +++ b/cli/multisig.go @@ -1469,7 +1469,7 @@ var msigProposeThresholdCmd = &cli.Command{ return err } - newM, err := strconv.ParseUint(cctx.Args().Get(2), 10, 64) + newM, err := strconv.ParseUint(cctx.Args().Get(1), 10, 64) if err != nil { return err } From 925606ff26c3e3856caf2c3a725da46faf7b9905 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 15 Oct 2020 12:02:12 +0200 Subject: [PATCH 775/885] Chore: update drand to v1.2.0 --- go.mod | 6 +++--- go.sum | 23 +++++++++++++++++------ 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index c3935be19..e36f821d3 100644 --- a/go.mod +++ b/go.mod @@ -16,8 +16,8 @@ require ( github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e github.com/dgraph-io/badger/v2 v2.2007.2 github.com/docker/go-units v0.4.0 - github.com/drand/drand v1.1.2-0.20200905144319-79c957281b32 - github.com/drand/kyber v1.1.2 + github.com/drand/drand v1.2.0 + github.com/drand/kyber v1.1.4 github.com/dustin/go-humanize v1.0.0 github.com/elastic/go-sysinfo v1.3.0 github.com/fatih/color v1.9.0 @@ -131,7 +131,7 @@ require ( go.uber.org/multierr v1.5.0 go.uber.org/zap v1.15.0 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 - golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 + golang.org/x/sys v0.0.0-20200926100807-9d91bd62050c golang.org/x/time v0.0.0-20191024005414-555d28b269f0 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 gopkg.in/cheggaaa/pb.v1 v1.0.28 diff --git a/go.sum b/go.sum index feff6abbd..6737eff06 100644 --- a/go.sum +++ b/go.sum @@ -91,6 +91,7 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= +github.com/briandowns/spinner v1.11.1 h1:OixPqDEcX3juo5AjQZAnFPbeUA0jvkp2qzB5gOZJ/L0= github.com/briandowns/spinner v1.11.1/go.mod h1:QOuQk7x+EaDASo80FEXwlwiA+j/PPIcX3FScO+3/ZPQ= github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= github.com/btcsuite/btcd v0.0.0-20190523000118-16327141da8c/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= @@ -191,14 +192,15 @@ github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/drand/bls12-381 v0.3.2 h1:RImU8Wckmx8XQx1tp1q04OV73J9Tj6mmpQLYDP7V1XE= github.com/drand/bls12-381 v0.3.2/go.mod h1:dtcLgPtYT38L3NO6mPDYH0nbpc5tjPassDqiniuAt4Y= -github.com/drand/drand v1.1.2-0.20200905144319-79c957281b32 h1:sU+51aQRaDxg0KnjQg19KuYRIxDBEUHffBAICSnBys8= -github.com/drand/drand v1.1.2-0.20200905144319-79c957281b32/go.mod h1:0sQEVg+ngs1jaDPVIiEgY0lbENWJPaUlWxGHEaSmKVM= +github.com/drand/drand v1.2.0 h1:EWeIxKFxUR9A+9xZ2oV09sqlxxWhDhgBbgCaCVwnvok= +github.com/drand/drand v1.2.0/go.mod h1:j0P7RGmVaY7E/OuO2yQOcQj7OgeZCuhgu2gdv0JAm+g= github.com/drand/kyber v1.0.1-0.20200110225416-8de27ed8c0e2/go.mod h1:UpXoA0Upd1N9l4TvRPHr1qAUBBERj6JQ/mnKI3BPEmw= github.com/drand/kyber v1.0.2/go.mod h1:x6KOpK7avKj0GJ4emhXFP5n7M7W7ChAPmnQh/OL6vRw= -github.com/drand/kyber v1.1.2 h1:faemqlaFyLrbBSjZGRzzu5SG/do+uTYpHlnrJIHbAhQ= -github.com/drand/kyber v1.1.2/go.mod h1:x6KOpK7avKj0GJ4emhXFP5n7M7W7ChAPmnQh/OL6vRw= -github.com/drand/kyber-bls12381 v0.1.0 h1:/P4C65VnyEwxzR5ZYYVMNzY1If+aYBrdUU5ukwh7LQw= -github.com/drand/kyber-bls12381 v0.1.0/go.mod h1:N1emiHpm+jj7kMlxEbu3MUyOiooTgNySln564cgD9mk= +github.com/drand/kyber v1.1.4 h1:YvKM03QWGvLrdTnYmxxP5iURAX+Gdb6qRDUOgg8i60Q= +github.com/drand/kyber v1.1.4/go.mod h1:9+IgTq7kadePhZg7eRwSD7+bA+bmvqRK+8DtmoV5a3U= +github.com/drand/kyber-bls12381 v0.2.0/go.mod h1:zQip/bHdeEB6HFZSU3v+d3cQE0GaBVQw9aR2E7AdoeI= +github.com/drand/kyber-bls12381 v0.2.1 h1:/d5/YAdaCmHpYjF1NZevOEcKGaq6LBbyvkCTIdGqDjs= +github.com/drand/kyber-bls12381 v0.2.1/go.mod h1:JwWn4nHO9Mp4F5qCie5sVIPQZ0X6cw8XAeMRvc/GXBE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -705,6 +707,9 @@ github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dv github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/kilic/bls12-381 v0.0.0-20200607163746-32e1441c8a9f h1:qET3Wx0v8tMtoTOQnsJXVvqvCopSf48qobR6tcJuDHo= github.com/kilic/bls12-381 v0.0.0-20200607163746-32e1441c8a9f/go.mod h1:XXfR6YFCRSrkEXbNlIyDsgXVNJWVUV30m/ebkVy9n6s= +github.com/kilic/bls12-381 v0.0.0-20200731194930-64c428e1bff5/go.mod h1:XXfR6YFCRSrkEXbNlIyDsgXVNJWVUV30m/ebkVy9n6s= +github.com/kilic/bls12-381 v0.0.0-20200820230200-6b2c19996391 h1:51kHw7l/dUDdOdW06AlUGT5jnpj6nqQSILebcsikSjA= +github.com/kilic/bls12-381 v0.0.0-20200820230200-6b2c19996391/go.mod h1:XXfR6YFCRSrkEXbNlIyDsgXVNJWVUV30m/ebkVy9n6s= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= @@ -1537,6 +1542,9 @@ golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM= +golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1691,6 +1699,9 @@ golang.org/x/sys v0.0.0-20200509044756-6aff5f38e54f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 h1:OjiUf46hAmXblsZdnoSXsEUSKU8r1UEzcL5RVZ4gO9Y= golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200812155832-6a926be9bd1d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200926100807-9d91bd62050c h1:38q6VNPWR010vN82/SB121GujZNIfAUb4YttE2rhGuc= +golang.org/x/sys v0.0.0-20200926100807-9d91bd62050c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= From 9c99171cb830b811daf23be21bf64394c700bf63 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Thu, 15 Oct 2020 12:15:21 +0200 Subject: [PATCH 776/885] fix: ensure msig inspect cli works with lotus-lite --- api/api_gateway.go | 2 + api/apistruct/struct.go | 10 +++ cli/multisig.go | 25 +++--- cli/multisig_test.go | 55 +++++++++++++ cli/paych_test.go | 1 + cli/test/mockcli.go | 124 +++++++++++++++++++++++++++++ cli/test/multisig.go | 77 ++++++++++++++++++ cmd/lotus-gateway/api.go | 25 +++++- cmd/lotus-gateway/api_test.go | 8 ++ cmd/lotus-gateway/endtoend_test.go | 52 +++++++++--- cmd/lotus-gateway/main.go | 2 +- node/impl/full/chain.go | 10 ++- 12 files changed, 361 insertions(+), 30 deletions(-) create mode 100644 cli/multisig_test.go create mode 100644 cli/test/mockcli.go create mode 100644 cli/test/multisig.go diff --git a/api/api_gateway.go b/api/api_gateway.go index 95d28887d..c99ff8408 100644 --- a/api/api_gateway.go +++ b/api/api_gateway.go @@ -10,9 +10,11 @@ import ( ) type GatewayAPI interface { + ChainHasObj(context.Context, cid.Cid) (bool, error) ChainHead(ctx context.Context) (*types.TipSet, error) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) + ChainReadObj(context.Context, cid.Cid) ([]byte, error) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 7723e0b1f..3854e1dd6 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -371,9 +371,11 @@ type WorkerStruct struct { type GatewayStruct struct { Internal struct { // TODO: does the gateway need perms? + ChainHasObj func(context.Context, cid.Cid) (bool, error) ChainGetTipSet func(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) ChainGetTipSetByHeight func(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) ChainHead func(ctx context.Context) (*types.TipSet, error) + ChainReadObj func(context.Context, cid.Cid) ([]byte, error) GasEstimateMessageGas func(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) MpoolPush func(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) MsigGetAvailableBalance func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) @@ -1432,6 +1434,10 @@ func (w *WorkerStruct) Closing(ctx context.Context) (<-chan struct{}, error) { return w.Internal.Closing(ctx) } +func (g GatewayStruct) ChainHasObj(ctx context.Context, c cid.Cid) (bool, error) { + return g.Internal.ChainHasObj(ctx, c) +} + func (g GatewayStruct) ChainHead(ctx context.Context) (*types.TipSet, error) { return g.Internal.ChainHead(ctx) } @@ -1444,6 +1450,10 @@ func (g GatewayStruct) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEp return g.Internal.ChainGetTipSetByHeight(ctx, h, tsk) } +func (g GatewayStruct) ChainReadObj(ctx context.Context, c cid.Cid) ([]byte, error) { + return g.Internal.ChainReadObj(ctx, c) +} + func (g GatewayStruct) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) { return g.Internal.GasEstimateMessageGas(ctx, msg, spec, tsk) } diff --git a/cli/multisig.go b/cli/multisig.go index 831b1bbe4..97a77a75b 100644 --- a/cli/multisig.go +++ b/cli/multisig.go @@ -5,7 +5,6 @@ import ( "encoding/hex" "encoding/json" "fmt" - "os" "reflect" "sort" "strconv" @@ -153,7 +152,7 @@ var msigCreateCmd = &cli.Command{ // check it executed successfully if wait.Receipt.ExitCode != 0 { - fmt.Println("actor creation failed!") + fmt.Fprintln(cctx.App.Writer, "actor creation failed!") return err } @@ -163,7 +162,7 @@ var msigCreateCmd = &cli.Command{ if err := execreturn.UnmarshalCBOR(bytes.NewReader(wait.Receipt.Return)); err != nil { return err } - fmt.Println("Created new multisig: ", execreturn.IDAddress, execreturn.RobustAddress) + fmt.Fprintln(cctx.App.Writer, "Created new multisig: ", execreturn.IDAddress, execreturn.RobustAddress) // TODO: maybe register this somewhere return nil @@ -227,25 +226,25 @@ var msigInspectCmd = &cli.Command{ return err } - fmt.Printf("Balance: %s\n", types.FIL(act.Balance)) - fmt.Printf("Spendable: %s\n", types.FIL(types.BigSub(act.Balance, locked))) + fmt.Fprintf(cctx.App.Writer, "Balance: %s\n", types.FIL(act.Balance)) + fmt.Fprintf(cctx.App.Writer, "Spendable: %s\n", types.FIL(types.BigSub(act.Balance, locked))) if cctx.Bool("vesting") { ib, err := mstate.InitialBalance() if err != nil { return err } - fmt.Printf("InitialBalance: %s\n", types.FIL(ib)) + fmt.Fprintf(cctx.App.Writer, "InitialBalance: %s\n", types.FIL(ib)) se, err := mstate.StartEpoch() if err != nil { return err } - fmt.Printf("StartEpoch: %d\n", se) + fmt.Fprintf(cctx.App.Writer, "StartEpoch: %d\n", se) ud, err := mstate.UnlockDuration() if err != nil { return err } - fmt.Printf("UnlockDuration: %d\n", ud) + fmt.Fprintf(cctx.App.Writer, "UnlockDuration: %d\n", ud) } signers, err := mstate.Signers() @@ -256,10 +255,10 @@ var msigInspectCmd = &cli.Command{ if err != nil { return err } - fmt.Printf("Threshold: %d / %d\n", threshold, len(signers)) - fmt.Println("Signers:") + fmt.Fprintf(cctx.App.Writer, "Threshold: %d / %d\n", threshold, len(signers)) + fmt.Fprintln(cctx.App.Writer, "Signers:") for _, s := range signers { - fmt.Printf("\t%s\n", s) + fmt.Fprintf(cctx.App.Writer, "\t%s\n", s) } pending := make(map[int64]multisig.Transaction) @@ -271,7 +270,7 @@ var msigInspectCmd = &cli.Command{ } decParams := cctx.Bool("decode-params") - fmt.Println("Transactions: ", len(pending)) + fmt.Fprintln(cctx.App.Writer, "Transactions: ", len(pending)) if len(pending) > 0 { var txids []int64 for txid := range pending { @@ -281,7 +280,7 @@ var msigInspectCmd = &cli.Command{ return txids[i] < txids[j] }) - w := tabwriter.NewWriter(os.Stdout, 8, 4, 2, ' ', 0) + w := tabwriter.NewWriter(cctx.App.Writer, 8, 4, 2, ' ', 0) fmt.Fprintf(w, "ID\tState\tApprovals\tTo\tValue\tMethod\tParams\n") for _, txid := range txids { tx := pending[txid] diff --git a/cli/multisig_test.go b/cli/multisig_test.go new file mode 100644 index 000000000..09cdbe056 --- /dev/null +++ b/cli/multisig_test.go @@ -0,0 +1,55 @@ +package cli + +import ( + "context" + "os" + "testing" + "time" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/api/test" + clitest "github.com/filecoin-project/lotus/cli/test" + builder "github.com/filecoin-project/lotus/node/test" +) + +// TestMultisig does a basic test to exercise the multisig CLI +// commands +func TestMultisig(t *testing.T) { + _ = os.Setenv("BELLMAN_NO_GPU", "1") + + blocktime := 5 * time.Millisecond + ctx := context.Background() + nodes, _ := startNodes(ctx, t, blocktime) + clientNode := nodes[0] + clitest.RunMultisigTest(t, Commands, clientNode) +} + +func startNodes(ctx context.Context, t *testing.T, blocktime time.Duration) ([]test.TestNode, []address.Address) { + n, sn := builder.RPCMockSbBuilder(t, test.OneFull, test.OneMiner) + + full := n[0] + miner := sn[0] + + // Get everyone connected + addrs, err := full.NetAddrsListen(ctx) + if err != nil { + t.Fatal(err) + } + + if err := miner.NetConnect(ctx, addrs); err != nil { + t.Fatal(err) + } + + // Start mining blocks + bm := test.NewBlockMiner(ctx, t, miner, blocktime) + bm.MineBlocks() + + // Get the creator's address + creatorAddr, err := full.WalletDefaultAddress(ctx) + if err != nil { + t.Fatal(err) + } + + // Create mock CLI + return n, []address.Address{creatorAddr} +} diff --git a/cli/paych_test.go b/cli/paych_test.go index b94ba27e7..77a6e61eb 100644 --- a/cli/paych_test.go +++ b/cli/paych_test.go @@ -437,6 +437,7 @@ type mockCLI struct { out *bytes.Buffer } +// TODO: refactor to use the methods in cli/test/mockcli.go func newMockCLI(t *testing.T) *mockCLI { // Create a CLI App with an --api-url flag so that we can specify which node // the command should be executed against diff --git a/cli/test/mockcli.go b/cli/test/mockcli.go new file mode 100644 index 000000000..f5d5cfcba --- /dev/null +++ b/cli/test/mockcli.go @@ -0,0 +1,124 @@ +package test + +import ( + "bytes" + "flag" + "strings" + "testing" + + "github.com/multiformats/go-multiaddr" + "github.com/stretchr/testify/require" + lcli "github.com/urfave/cli/v2" +) + +type mockCLI struct { + t *testing.T + cmds []*lcli.Command + cctx *lcli.Context + out *bytes.Buffer +} + +func newMockCLI(t *testing.T, cmds []*lcli.Command) *mockCLI { + // Create a CLI App with an --api-url flag so that we can specify which node + // the command should be executed against + app := &lcli.App{ + Flags: []lcli.Flag{ + &lcli.StringFlag{ + Name: "api-url", + Hidden: true, + }, + }, + Commands: cmds, + } + + var out bytes.Buffer + app.Writer = &out + app.Setup() + + cctx := lcli.NewContext(app, &flag.FlagSet{}, nil) + return &mockCLI{t: t, cmds: cmds, cctx: cctx, out: &out} +} + +func (c *mockCLI) client(addr multiaddr.Multiaddr) *mockCLIClient { + return &mockCLIClient{t: c.t, cmds: c.cmds, addr: addr, cctx: c.cctx, out: c.out} +} + +// mockCLIClient runs commands against a particular node +type mockCLIClient struct { + t *testing.T + cmds []*lcli.Command + addr multiaddr.Multiaddr + cctx *lcli.Context + out *bytes.Buffer +} + +func (c *mockCLIClient) run(cmd []string, params []string, args []string) string { + // Add parameter --api-url= + apiFlag := "--api-url=" + c.addr.String() + params = append([]string{apiFlag}, params...) + + err := c.cctx.App.Run(append(append(cmd, params...), args...)) + require.NoError(c.t, err) + + // Get the output + str := strings.TrimSpace(c.out.String()) + c.out.Reset() + return str +} + +func (c *mockCLIClient) runCmd(input []string) string { + cmd := c.cmdByNameSub(input[0], input[1]) + out, err := c.runCmdRaw(cmd, input[2:]) + require.NoError(c.t, err) + + return out +} + +func (c *mockCLIClient) cmdByNameSub(name string, sub string) *lcli.Command { + for _, c := range c.cmds { + if c.Name == name { + for _, s := range c.Subcommands { + if s.Name == sub { + return s + } + } + } + } + return nil +} + +func (c *mockCLIClient) runCmdRaw(cmd *lcli.Command, input []string) (string, error) { + // prepend --api-url= + apiFlag := "--api-url=" + c.addr.String() + input = append([]string{apiFlag}, input...) + + fs := c.flagSet(cmd) + err := fs.Parse(input) + require.NoError(c.t, err) + + err = cmd.Action(lcli.NewContext(c.cctx.App, fs, c.cctx)) + + // Get the output + str := strings.TrimSpace(c.out.String()) + c.out.Reset() + return str, err +} + +func (c *mockCLIClient) flagSet(cmd *lcli.Command) *flag.FlagSet { + // Apply app level flags (so we can process --api-url flag) + fs := &flag.FlagSet{} + for _, f := range c.cctx.App.Flags { + err := f.Apply(fs) + if err != nil { + c.t.Fatal(err) + } + } + // Apply command level flags + for _, f := range cmd.Flags { + err := f.Apply(fs) + if err != nil { + c.t.Fatal(err) + } + } + return fs +} diff --git a/cli/test/multisig.go b/cli/test/multisig.go new file mode 100644 index 000000000..96025aa8c --- /dev/null +++ b/cli/test/multisig.go @@ -0,0 +1,77 @@ +package test + +import ( + "context" + "fmt" + "regexp" + "strings" + "testing" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/api/test" + "github.com/filecoin-project/lotus/chain/types" + logging "github.com/ipfs/go-log/v2" + "github.com/stretchr/testify/require" + lcli "github.com/urfave/cli/v2" +) + +func QuietMiningLogs() { + logging.SetLogLevel("miner", "ERROR") + logging.SetLogLevel("chainstore", "ERROR") + logging.SetLogLevel("chain", "ERROR") + logging.SetLogLevel("sub", "ERROR") + logging.SetLogLevel("storageminer", "ERROR") +} + +func RunMultisigTest(t *testing.T, cmds []*lcli.Command, clientNode test.TestNode) { + ctx := context.Background() + + // Create mock CLI + mockCLI := newMockCLI(t, cmds) + clientCLI := mockCLI.client(clientNode.ListenAddr) + + // Create some wallets on the node to use for testing multisig + var walletAddrs []address.Address + for i := 0; i < 4; i++ { + addr, err := clientNode.WalletNew(ctx, types.KTSecp256k1) + require.NoError(t, err) + + walletAddrs = append(walletAddrs, addr) + + test.SendFunds(ctx, t, clientNode, addr, types.NewInt(1e15)) + } + + // Create an msig with three of the addresses and threshold of two sigs + // msig create --required=2 --duration=50 --value=1000attofil + amtAtto := types.NewInt(1000) + threshold := 2 + paramDuration := "--duration=50" + paramRequired := fmt.Sprintf("--required=%d", threshold) + paramValue := fmt.Sprintf("--value=%dattofil", amtAtto) + cmd := []string{ + "msig", "create", + paramRequired, + paramDuration, + paramValue, + walletAddrs[0].String(), + walletAddrs[1].String(), + walletAddrs[2].String(), + } + out := clientCLI.runCmd(cmd) + fmt.Println(out) + + // Extract msig robust address from output + expCreateOutPrefix := "Created new multisig:" + require.Regexp(t, regexp.MustCompile(expCreateOutPrefix), out) + parts := strings.Split(strings.TrimSpace(strings.Replace(out, expCreateOutPrefix, "", -1)), " ") + require.Len(t, parts, 2) + msigRobustAddr := parts[1] + fmt.Println("msig robust address:", msigRobustAddr) + + // msig inspect + cmd = []string{"msig", "inspect", msigRobustAddr} + out = clientCLI.runCmd(cmd) + fmt.Println(out) + + require.Regexp(t, regexp.MustCompile("Balance: 0.000000000000001 FIL"), out) +} diff --git a/cmd/lotus-gateway/api.go b/cmd/lotus-gateway/api.go index d5fac0a06..6d47ff214 100644 --- a/cmd/lotus-gateway/api.go +++ b/cmd/lotus-gateway/api.go @@ -26,9 +26,11 @@ var ( // gatewayDepsAPI defines the API methods that the GatewayAPI depends on // (to make it easy to mock for tests) type gatewayDepsAPI interface { + ChainHasObj(context.Context, cid.Cid) (bool, error) ChainHead(ctx context.Context) (*types.TipSet, error) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) + ChainReadObj(context.Context, cid.Cid) ([]byte, error) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) MpoolPushUntrusted(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) @@ -40,7 +42,18 @@ type gatewayDepsAPI interface { } type GatewayAPI struct { - api gatewayDepsAPI + api gatewayDepsAPI + lookbackCap time.Duration +} + +// NewGatewayAPI creates a new GatewayAPI with the default lookback cap +func NewGatewayAPI(api gatewayDepsAPI) *GatewayAPI { + return newGatewayAPI(api, LookbackCap) +} + +// used by the tests +func newGatewayAPI(api gatewayDepsAPI, lookbackCap time.Duration) *GatewayAPI { + return &GatewayAPI{api: api, lookbackCap: lookbackCap} } func (a *GatewayAPI) checkTipsetKey(ctx context.Context, tsk types.TipSetKey) error { @@ -76,13 +89,17 @@ func (a *GatewayAPI) checkTipsetHeight(ts *types.TipSet, h abi.ChainEpoch) error } func (a *GatewayAPI) checkTimestamp(at time.Time) error { - if time.Since(at) > LookbackCap { + if time.Since(at) > a.lookbackCap { return ErrLookbackTooLong } return nil } +func (a *GatewayAPI) ChainHasObj(ctx context.Context, c cid.Cid) (bool, error) { + return a.api.ChainHasObj(ctx, c) +} + func (a *GatewayAPI) ChainHead(ctx context.Context) (*types.TipSet, error) { // TODO: cache and invalidate cache when timestamp is up (or have internal ChainNotify) @@ -112,6 +129,10 @@ func (a *GatewayAPI) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoc return a.api.ChainGetTipSetByHeight(ctx, h, tsk) } +func (a *GatewayAPI) ChainReadObj(ctx context.Context, c cid.Cid) ([]byte, error) { + return a.api.ChainReadObj(ctx, c) +} + func (a *GatewayAPI) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) { if err := a.checkTipsetKey(ctx, tsk); err != nil { return nil, err diff --git a/cmd/lotus-gateway/api_test.go b/cmd/lotus-gateway/api_test.go index f34f887f5..4946e2a26 100644 --- a/cmd/lotus-gateway/api_test.go +++ b/cmd/lotus-gateway/api_test.go @@ -109,6 +109,10 @@ type mockGatewayDepsAPI struct { tipsets []*types.TipSet } +func (m *mockGatewayDepsAPI) ChainHasObj(context.Context, cid.Cid) (bool, error) { + panic("implement me") +} + func (m *mockGatewayDepsAPI) ChainHead(ctx context.Context) (*types.TipSet, error) { m.lk.RLock() defer m.lk.RUnlock() @@ -158,6 +162,10 @@ func (m *mockGatewayDepsAPI) ChainGetTipSetByHeight(ctx context.Context, h abi.C return m.tipsets[h], nil } +func (m *mockGatewayDepsAPI) ChainReadObj(ctx context.Context, c cid.Cid) ([]byte, error) { + panic("implement me") +} + func (m *mockGatewayDepsAPI) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) { panic("implement me") } diff --git a/cmd/lotus-gateway/endtoend_test.go b/cmd/lotus-gateway/endtoend_test.go index eebf56a9f..379c96053 100644 --- a/cmd/lotus-gateway/endtoend_test.go +++ b/cmd/lotus-gateway/endtoend_test.go @@ -4,10 +4,14 @@ import ( "bytes" "context" "fmt" + "math" "os" "testing" "time" + "github.com/filecoin-project/lotus/cli" + clitest "github.com/filecoin-project/lotus/cli/test" + init0 "github.com/filecoin-project/specs-actors/actors/builtin/init" "github.com/filecoin-project/specs-actors/actors/builtin/multisig" @@ -26,20 +30,22 @@ import ( builder "github.com/filecoin-project/lotus/node/test" ) +const maxLookbackCap = time.Duration(math.MaxInt64) + func init() { policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg2KiBV1) policy.SetConsensusMinerMinPower(abi.NewStoragePower(2048)) policy.SetMinVerifiedDealSize(abi.NewStoragePower(256)) } -// TestEndToEnd tests that API calls can be made on a lite node that is -// connected through a gateway to a full API node -func TestEndToEnd(t *testing.T) { +// TestEndToEndWalletMsig tests that wallet and msig API calls can be made +// on a lite node that is connected through a gateway to a full API node +func TestEndToEndWalletMsig(t *testing.T) { _ = os.Setenv("BELLMAN_NO_GPU", "1") blocktime := 5 * time.Millisecond ctx := context.Background() - full, lite, closer := startNodes(ctx, t, blocktime) + full, lite, closer := startNodes(ctx, t, blocktime, maxLookbackCap) defer closer() // The full node starts with a wallet @@ -56,11 +62,11 @@ func TestEndToEnd(t *testing.T) { require.NoError(t, err) // Send some funds from the full node to the lite node - err = sendFunds(ctx, t, full, fullWalletAddr, liteWalletAddr, types.NewInt(1e18)) + err = sendFunds(ctx, full, fullWalletAddr, liteWalletAddr, types.NewInt(1e18)) require.NoError(t, err) // Send some funds from the lite node back to the full node - err = sendFunds(ctx, t, lite, liteWalletAddr, fullWalletAddr, types.NewInt(100)) + err = sendFunds(ctx, lite, liteWalletAddr, fullWalletAddr, types.NewInt(100)) require.NoError(t, err) // Sign some data with the lite node wallet address @@ -81,7 +87,7 @@ func TestEndToEnd(t *testing.T) { walletAddrs = append(walletAddrs, addr) - err = sendFunds(ctx, t, lite, liteWalletAddr, addr, types.NewInt(1e15)) + err = sendFunds(ctx, lite, liteWalletAddr, addr, types.NewInt(1e15)) require.NoError(t, err) } @@ -135,7 +141,33 @@ func TestEndToEnd(t *testing.T) { require.True(t, approveReturn.Applied) } -func sendFunds(ctx context.Context, t *testing.T, fromNode test.TestNode, fromAddr address.Address, toAddr address.Address, amt types.BigInt) error { +// TestEndToEndMsigCLI tests that msig CLI calls can be made +// on a lite node that is connected through a gateway to a full API node +func TestEndToEndMsigCLI(t *testing.T) { + _ = os.Setenv("BELLMAN_NO_GPU", "1") + clitest.QuietMiningLogs() + + blocktime := 5 * time.Millisecond + ctx := context.Background() + full, lite, closer := startNodes(ctx, t, blocktime, maxLookbackCap) + defer closer() + + // The full node starts with a wallet + fullWalletAddr, err := full.WalletDefaultAddress(ctx) + require.NoError(t, err) + + // Create a wallet on the lite node + liteWalletAddr, err := lite.WalletNew(ctx, types.KTSecp256k1) + require.NoError(t, err) + + // Send some funds from the full node to the lite node + err = sendFunds(ctx, full, fullWalletAddr, liteWalletAddr, types.NewInt(1e18)) + require.NoError(t, err) + + clitest.RunMultisigTest(t, cli.Commands, lite) +} + +func sendFunds(ctx context.Context, fromNode test.TestNode, fromAddr address.Address, toAddr address.Address, amt types.BigInt) error { msg := &types.Message{ From: fromAddr, To: toAddr, @@ -158,7 +190,7 @@ func sendFunds(ctx context.Context, t *testing.T, fromNode test.TestNode, fromAd return nil } -func startNodes(ctx context.Context, t *testing.T, blocktime time.Duration) (test.TestNode, test.TestNode, jsonrpc.ClientCloser) { +func startNodes(ctx context.Context, t *testing.T, blocktime time.Duration, lookbackCap time.Duration) (test.TestNode, test.TestNode, jsonrpc.ClientCloser) { var closer jsonrpc.ClientCloser // Create one miner and two full nodes. @@ -175,7 +207,7 @@ func startNodes(ctx context.Context, t *testing.T, blocktime time.Duration) (tes fullNode := nodes[0] // Create a gateway server in front of the full node - _, addr, err := builder.CreateRPCServer(&GatewayAPI{api: fullNode}) + _, addr, err := builder.CreateRPCServer(newGatewayAPI(fullNode, lookbackCap)) require.NoError(t, err) // Create a gateway client API that connects to the gateway server diff --git a/cmd/lotus-gateway/main.go b/cmd/lotus-gateway/main.go index c19599084..34e068ffd 100644 --- a/cmd/lotus-gateway/main.go +++ b/cmd/lotus-gateway/main.go @@ -76,7 +76,7 @@ var runCmd = &cli.Command{ log.Info("Setting up API endpoint at " + address) rpcServer := jsonrpc.NewServer() - rpcServer.Register("Filecoin", &GatewayAPI{api: api}) + rpcServer.Register("Filecoin", NewGatewayAPI(api)) mux.Handle("/rpc/v0", rpcServer) mux.PathPrefix("/").Handler(http.DefaultServeMux) diff --git a/node/impl/full/chain.go b/node/impl/full/chain.go index 5b4f41114..095b51ed8 100644 --- a/node/impl/full/chain.go +++ b/node/impl/full/chain.go @@ -40,9 +40,11 @@ import ( var log = logging.Logger("fullnode") type ChainModuleAPI interface { + ChainHasObj(context.Context, cid.Cid) (bool, error) ChainHead(context.Context) (*types.TipSet, error) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) + ChainReadObj(context.Context, cid.Cid) ([]byte, error) } // ChainModule provides a default implementation of ChainModuleAPI. @@ -206,8 +208,8 @@ func (m *ChainModule) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpo return m.Chain.GetTipsetByHeight(ctx, h, ts, true) } -func (a *ChainAPI) ChainReadObj(ctx context.Context, obj cid.Cid) ([]byte, error) { - blk, err := a.Chain.Blockstore().Get(obj) +func (m *ChainModule) ChainReadObj(ctx context.Context, obj cid.Cid) ([]byte, error) { + blk, err := m.Chain.Blockstore().Get(obj) if err != nil { return nil, xerrors.Errorf("blockstore get: %w", err) } @@ -219,8 +221,8 @@ func (a *ChainAPI) ChainDeleteObj(ctx context.Context, obj cid.Cid) error { return a.Chain.Blockstore().DeleteBlock(obj) } -func (a *ChainAPI) ChainHasObj(ctx context.Context, obj cid.Cid) (bool, error) { - return a.Chain.Blockstore().Has(obj) +func (m *ChainModule) ChainHasObj(ctx context.Context, obj cid.Cid) (bool, error) { + return m.Chain.Blockstore().Has(obj) } func (a *ChainAPI) ChainStatObj(ctx context.Context, obj cid.Cid, base cid.Cid) (api.ObjStat, error) { From f74852bd73ab9beed57500f879d5e07b0abf3dcf Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Thu, 15 Oct 2020 12:45:02 +0200 Subject: [PATCH 777/885] test: add more msig cli tests --- cli/multisig.go | 2 +- cli/test/multisig.go | 37 +++++++++++++++++++++++++++++++++++-- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/cli/multisig.go b/cli/multisig.go index 97a77a75b..90b074421 100644 --- a/cli/multisig.go +++ b/cli/multisig.go @@ -677,7 +677,7 @@ var msigAddProposeCmd = &cli.Command{ return err } - fmt.Println("sent add proposal in message: ", msgCid) + fmt.Fprintln(cctx.App.Writer, "sent add proposal in message: ", msgCid) wait, err := api.StateWaitMsg(ctx, msgCid, build.MessageConfidence) if err != nil { diff --git a/cli/test/multisig.go b/cli/test/multisig.go index 96025aa8c..e77879346 100644 --- a/cli/test/multisig.go +++ b/cli/test/multisig.go @@ -68,10 +68,43 @@ func RunMultisigTest(t *testing.T, cmds []*lcli.Command, clientNode test.TestNod msigRobustAddr := parts[1] fmt.Println("msig robust address:", msigRobustAddr) - // msig inspect - cmd = []string{"msig", "inspect", msigRobustAddr} + // Propose to add a new address to the msig + // msig add-propose --from= + paramFrom := fmt.Sprintf("--from=%s", walletAddrs[0]) + cmd = []string{ + "msig", "add-propose", + paramFrom, + msigRobustAddr, + walletAddrs[3].String(), + } out = clientCLI.runCmd(cmd) fmt.Println(out) + // msig inspect + cmd = []string{"msig", "inspect", "--vesting", "--decode-params", msigRobustAddr} + out = clientCLI.runCmd(cmd) + fmt.Println(out) + + // Expect correct balance require.Regexp(t, regexp.MustCompile("Balance: 0.000000000000001 FIL"), out) + // Expect 1 transaction + require.Regexp(t, regexp.MustCompile(`Transactions:\s*1`), out) + // Expect transaction to be "AddSigner" + require.Regexp(t, regexp.MustCompile(`AddSigner`), out) + + // Approve adding the new address + // msig add-approve --from= 0 false + txnID := "0" + paramFrom = fmt.Sprintf("--from=%s", walletAddrs[1]) + cmd = []string{ + "msig", "add-approve", + paramFrom, + msigRobustAddr, + walletAddrs[0].String(), + txnID, + walletAddrs[3].String(), + "false", + } + out = clientCLI.runCmd(cmd) + fmt.Println(out) } From f5d48b0e86a3771a9aa36232c21e749aca188491 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Thu, 15 Oct 2020 12:47:24 +0200 Subject: [PATCH 778/885] fix: lotus-gateway api tests --- cmd/lotus-gateway/api_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/lotus-gateway/api_test.go b/cmd/lotus-gateway/api_test.go index 4946e2a26..5b8261974 100644 --- a/cmd/lotus-gateway/api_test.go +++ b/cmd/lotus-gateway/api_test.go @@ -88,7 +88,7 @@ func TestGatewayAPIChainGetTipSetByHeight(t *testing.T) { tt := tt t.Run(tt.name, func(t *testing.T) { mock := &mockGatewayDepsAPI{} - a := &GatewayAPI{api: mock} + a := NewGatewayAPI(mock) // Create tipsets from genesis up to tskh and return the highest ts := mock.createTipSets(tt.args.tskh, tt.args.genesisTS) From 4e659e30c54e58af2a59be858aa5f3d14cafb6df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Thu, 15 Oct 2020 12:14:16 +0100 Subject: [PATCH 779/885] fix lint errors. --- cmd/tvx/codenames_test.go | 2 +- cmd/tvx/exec.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/tvx/codenames_test.go b/cmd/tvx/codenames_test.go index 0b1de9fc2..00d107707 100644 --- a/cmd/tvx/codenames_test.go +++ b/cmd/tvx/codenames_test.go @@ -18,7 +18,7 @@ func TestProtocolCodenames(t *testing.T) { t.Fatal("expected breeze codename") } - if height := abi.ChainEpoch(build.UpgradeActorsV2Height + 1); GetProtocolCodename(height) != "actorsv2" { + if height := build.UpgradeActorsV2Height + 1; GetProtocolCodename(height) != "actorsv2" { t.Fatal("expected actorsv2 codename") } diff --git a/cmd/tvx/exec.go b/cmd/tvx/exec.go index de7d2a398..89ad23913 100644 --- a/cmd/tvx/exec.go +++ b/cmd/tvx/exec.go @@ -76,7 +76,7 @@ func executeTestVector(tv schema.TestVector) error { for _, v := range tv.Pre.Variants { r := new(conformance.LogReporter) - switch class := tv.Class; class { + switch class, v := tv.Class, v; class { case "message": conformance.ExecuteMessageVector(r, &tv, &v) case "tipset": From 7e6ede7563fc859b909fa5f54660bb5e5127a4e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Thu, 15 Oct 2020 12:49:23 +0100 Subject: [PATCH 780/885] update schema; test-vectors submodule. --- extern/test-vectors | 2 +- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/extern/test-vectors b/extern/test-vectors index a8f968ade..d9a75a787 160000 --- a/extern/test-vectors +++ b/extern/test-vectors @@ -1 +1 @@ -Subproject commit a8f968adeba1995f161f7be0048188affc425079 +Subproject commit d9a75a7873aee0db28b87e3970d2ea16a2f37c6a diff --git a/go.mod b/go.mod index 1516cba81..a810e118c 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/filecoin-project/specs-actors v0.9.12 github.com/filecoin-project/specs-actors/v2 v2.1.0 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 - github.com/filecoin-project/test-vectors/schema v0.0.5-0.20201014133607-1352e6bb4e71 + github.com/filecoin-project/test-vectors/schema v0.0.5 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 github.com/go-kit/kit v0.10.0 github.com/go-ole/go-ole v1.2.4 // indirect diff --git a/go.sum b/go.sum index 47301c784..3d543fab0 100644 --- a/go.sum +++ b/go.sum @@ -282,8 +282,8 @@ github.com/filecoin-project/specs-actors/v2 v2.1.0 h1:ocEuGz8DG2cUWw32c/tvF8D6xT github.com/filecoin-project/specs-actors/v2 v2.1.0/go.mod h1:E7fAX4CZkDVQvDNRCxfq+hc3nx56KcCKyuZf0hlQJ20= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= -github.com/filecoin-project/test-vectors/schema v0.0.5-0.20201014133607-1352e6bb4e71 h1:qnleaW7X8Gi2e3QtqPMFdqVn/DUaNI5tCnq7wNVMnio= -github.com/filecoin-project/test-vectors/schema v0.0.5-0.20201014133607-1352e6bb4e71/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E= +github.com/filecoin-project/test-vectors/schema v0.0.5 h1:w3zHQhzM4pYxJDl21avXjOKBLF8egrvwUwjpT8TquDg= +github.com/filecoin-project/test-vectors/schema v0.0.5/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6 h1:u/UEqS66A5ckRmS4yNpjmVH56sVtS/RfclBAYocb4as= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ= From 1fb9ed1c9a53fc304151f65d8409ccb4ae072d71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Thu, 15 Oct 2020 12:58:43 +0100 Subject: [PATCH 781/885] remove TODO. --- cmd/tvx/codenames.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/cmd/tvx/codenames.go b/cmd/tvx/codenames.go index 1cb1b32fb..851e9d841 100644 --- a/cmd/tvx/codenames.go +++ b/cmd/tvx/codenames.go @@ -16,9 +16,6 @@ var ProtocolCodenames = []struct { name string }{ {0, "genesis"}, - // TODO there is some off-by-one trickery in GetNtwkVersion. Not sure if the - // protocol version really kicks in at the designated height, or at the - // following epoch. {build.UpgradeBreezeHeight + 1, "breeze"}, {build.UpgradeSmokeHeight + 1, "smoke"}, {build.UpgradeIgnitionHeight + 1, "ignition"}, From a931ff94e908a6467f35aaedf183f10f21dc5d9c Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 15 Oct 2020 08:54:36 -0700 Subject: [PATCH 782/885] fix message list api --- api/api_full.go | 7 ++++++- api/apistruct/struct.go | 4 ++-- cli/state.go | 2 +- node/impl/full/state.go | 2 +- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 65182078a..89a41dede 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -327,7 +327,7 @@ type FullNode interface { // StateReadState returns the indicated actor's state. StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*ActorState, error) // StateListMessages looks back and returns all messages with a matching to or from address, stopping at the given height. - StateListMessages(ctx context.Context, match *types.Message, tsk types.TipSetKey, toht abi.ChainEpoch) ([]cid.Cid, error) + StateListMessages(ctx context.Context, match *MessageMatch, tsk types.TipSetKey, toht abi.ChainEpoch) ([]cid.Cid, error) // StateNetworkName returns the name of the network the node is synced to StateNetworkName(context.Context) (dtypes.NetworkName, error) @@ -918,3 +918,8 @@ type MsigVesting struct { StartEpoch abi.ChainEpoch UnlockDuration abi.ChainEpoch } + +type MessageMatch struct { + To address.Address + From address.Address +} diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 3854e1dd6..ba3a39516 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -206,7 +206,7 @@ type FullNodeStruct struct { StateChangedActors func(context.Context, cid.Cid, cid.Cid) (map[string]types.Actor, error) `perm:"read"` StateGetReceipt func(context.Context, cid.Cid, types.TipSetKey) (*types.MessageReceipt, error) `perm:"read"` StateMinerSectorCount func(context.Context, address.Address, types.TipSetKey) (api.MinerSectors, error) `perm:"read"` - StateListMessages func(ctx context.Context, match *types.Message, tsk types.TipSetKey, toht abi.ChainEpoch) ([]cid.Cid, error) `perm:"read"` + StateListMessages func(ctx context.Context, match *api.MessageMatch, tsk types.TipSetKey, toht abi.ChainEpoch) ([]cid.Cid, error) `perm:"read"` StateCompute func(context.Context, abi.ChainEpoch, []*types.Message, types.TipSetKey) (*api.ComputeStateOutput, error) `perm:"read"` StateVerifierStatus func(context.Context, address.Address, types.TipSetKey) (*abi.StoragePower, error) `perm:"read"` StateVerifiedClientStatus func(context.Context, address.Address, types.TipSetKey) (*abi.StoragePower, error) `perm:"read"` @@ -948,7 +948,7 @@ func (c *FullNodeStruct) StateGetReceipt(ctx context.Context, msg cid.Cid, tsk t return c.Internal.StateGetReceipt(ctx, msg, tsk) } -func (c *FullNodeStruct) StateListMessages(ctx context.Context, match *types.Message, tsk types.TipSetKey, toht abi.ChainEpoch) ([]cid.Cid, error) { +func (c *FullNodeStruct) StateListMessages(ctx context.Context, match *api.MessageMatch, tsk types.TipSetKey, toht abi.ChainEpoch) ([]cid.Cid, error) { return c.Internal.StateListMessages(ctx, match, tsk, toht) } diff --git a/cli/state.go b/cli/state.go index 1e7e24ddb..a02a70505 100644 --- a/cli/state.go +++ b/cli/state.go @@ -844,7 +844,7 @@ var stateListMessagesCmd = &cli.Command{ return err } - msgs, err := api.StateListMessages(ctx, &types.Message{To: toa, From: froma}, ts.Key(), abi.ChainEpoch(toh)) + msgs, err := api.StateListMessages(ctx, &lapi.MessageMatch{To: toa, From: froma}, ts.Key(), abi.ChainEpoch(toh)) if err != nil { return err } diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 08ac62f2e..81947cbe3 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -760,7 +760,7 @@ func (a *StateAPI) StateSectorPartition(ctx context.Context, maddr address.Addre return mas.FindSector(sectorNumber) } -func (a *StateAPI) StateListMessages(ctx context.Context, match *types.Message, tsk types.TipSetKey, toheight abi.ChainEpoch) ([]cid.Cid, error) { +func (a *StateAPI) StateListMessages(ctx context.Context, match *api.MessageMatch, tsk types.TipSetKey, toheight abi.ChainEpoch) ([]cid.Cid, error) { ts, err := a.Chain.GetTipSetFromKey(tsk) if err != nil { return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) From 8c7d2efe72f864c7ebb483d047aee12138df7359 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Thu, 15 Oct 2020 11:51:40 -0400 Subject: [PATCH 783/885] Make audit balances capable of printing robust addresses --- cmd/lotus-shed/balances.go | 51 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/cmd/lotus-shed/balances.go b/cmd/lotus-shed/balances.go index dfdaf9fa9..b12c069f5 100644 --- a/cmd/lotus-shed/balances.go +++ b/cmd/lotus-shed/balances.go @@ -5,6 +5,10 @@ import ( "fmt" "strconv" + "github.com/filecoin-project/lotus/chain/gen/genesis" + + _init "github.com/filecoin-project/lotus/chain/actors/builtin/init" + "github.com/docker/go-units" "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/builtin/multisig" @@ -136,6 +140,9 @@ var chainBalanceStateCmd = &cli.Command{ &cli.BoolFlag{ Name: "miner-info", }, + &cli.BoolFlag{ + Name: "robust-addresses", + }, }, Action: func(cctx *cli.Context) error { ctx := context.TODO() @@ -187,6 +194,33 @@ var chainBalanceStateCmd = &cli.Command{ minerInfo := cctx.Bool("miner-info") + robustMap := make(map[address.Address]address.Address) + if cctx.Bool("robust-addresses") { + iact, err := tree.GetActor(_init.Address) + if err != nil { + return xerrors.Errorf("failed to load init actor: %w", err) + } + + ist, err := _init.Load(store, iact) + if err != nil { + return xerrors.Errorf("failed to load init actor state: %w", err) + } + + err = ist.ForEachActor(func(id abi.ActorID, addr address.Address) error { + idAddr, err := address.NewIDAddress(uint64(id)) + if err != nil { + return xerrors.Errorf("failed to write to addr map: %w", err) + } + + robustMap[idAddr] = addr + + return nil + }) + if err != nil { + return xerrors.Errorf("failed to invert init address map: %w", err) + } + } + var infos []accountInfo err = tree.ForEach(func(addr address.Address, act *types.Actor) error { @@ -201,6 +235,23 @@ var chainBalanceStateCmd = &cli.Command{ VestingAmount: types.FIL(big.NewInt(0)), } + if cctx.Bool("robust-addresses") { + robust, found := robustMap[addr] + if found { + ai.Address = robust + } else { + id, err := address.IDFromAddress(addr) + if err != nil { + return xerrors.Errorf("failed to get ID address: %w", err) + } + + // TODO: This is not the correctest way to determine whether a robust address should exist + if id >= genesis.MinerStart { + return xerrors.Errorf("address doesn't have a robust address: %s", addr) + } + } + } + if minerInfo && builtin.IsStorageMinerActor(act.Code) { pow, _, _, err := stmgr.GetPowerRaw(ctx, sm, sroot, addr) if err != nil { From 0b7dc6971dbe1a1e5caae2678d5ed6606bb7dcdc Mon Sep 17 00:00:00 2001 From: Ingar Shu Date: Thu, 15 Oct 2020 09:24:48 -0700 Subject: [PATCH 784/885] Rebasing --- markets/dealfilter/cli.go | 60 ++++++++++++++++++--------- node/builder.go | 6 ++- node/modules/dtypes/miner.go | 4 +- node/modules/storageminer.go | 78 +++++++++++++++++++++++------------- 4 files changed, 98 insertions(+), 50 deletions(-) diff --git a/markets/dealfilter/cli.go b/markets/dealfilter/cli.go index 2cb9d6c4f..af832bfa0 100644 --- a/markets/dealfilter/cli.go +++ b/markets/dealfilter/cli.go @@ -6,35 +6,57 @@ import ( "encoding/json" "os/exec" + "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/lotus/node/modules/dtypes" ) -func CliDealFilter(cmd string) dtypes.DealFilter { - // TODO: run some checks on the cmd string - +func CliStorageDealFilter(cmd string) dtypes.StorageDealFilter { return func(ctx context.Context, deal storagemarket.MinerDeal) (bool, string, error) { - j, err := json.MarshalIndent(deal, "", " ") - if err != nil { - return false, "", err + d := struct { + storagemarket.MinerDeal + DealType string + }{ + MinerDeal: deal, + DealType: "storage", } + return runDealFilter(ctx, cmd, d) + } +} - var out bytes.Buffer +func CliRetrievalDealFilter(cmd string) dtypes.RetrievalDealFilter { + return func(ctx context.Context, deal retrievalmarket.ProviderDealState) (bool, string, error) { + d := struct { + retrievalmarket.ProviderDealState + DealType string + }{ + ProviderDealState: deal, + DealType: "retrieval", + } + return runDealFilter(ctx, cmd, d) + } +} + +func runDealFilter(ctx context.Context, cmd string, deal interface{}) (bool, string, error) { + j, err := json.MarshalIndent(deal, "", " ") + if err != nil { + return false, "", err + } - c := exec.Command("sh", "-c", cmd) - c.Stdin = bytes.NewReader(j) - c.Stdout = &out - c.Stderr = &out + var out bytes.Buffer - switch err := c.Run().(type) { - case nil: - return true, "", nil - case *exec.ExitError: - return false, out.String(), nil - default: - return false, "filter cmd run error", err - } + c := exec.Command("sh", "-c", cmd) + c.Stdin = bytes.NewReader(j) + c.Stdout = &out + c.Stderr = &out + switch err := c.Run().(type) { + case nil: + return true, "", nil + case *exec.ExitError: + return false, out.String(), nil + default: + return false, "filter cmd run error", err } } diff --git a/node/builder.go b/node/builder.go index f96505eec..b6b75ecc6 100644 --- a/node/builder.go +++ b/node/builder.go @@ -354,7 +354,8 @@ func Online() Option { Override(new(dtypes.ProviderDataTransfer), modules.NewProviderDAGServiceDataTransfer), Override(new(dtypes.ProviderPieceStore), modules.NewProviderPieceStore), Override(new(*storedask.StoredAsk), modules.NewStorageAsk), - Override(new(dtypes.DealFilter), modules.BasicDealFilter(nil)), + Override(new(dtypes.StorageDealFilter), modules.BasicDealFilter(nil)), + Override(new(dtypes.RetrievalDealFilter), modules.RetrievalDealFilter(nil)), Override(new(modules.ProviderDealFunds), modules.NewProviderDealFunds), Override(new(storagemarket.StorageProvider), modules.StorageProvider), Override(new(storagemarket.StorageProviderNode), storageadapter.NewProviderNodeAdapter(nil)), @@ -484,7 +485,8 @@ func ConfigStorageMiner(c interface{}) Option { ConfigCommon(&cfg.Common), If(cfg.Dealmaking.Filter != "", - Override(new(dtypes.DealFilter), modules.BasicDealFilter(dealfilter.CliDealFilter(cfg.Dealmaking.Filter))), + Override(new(dtypes.StorageDealFilter), modules.BasicDealFilter(dealfilter.CliStorageDealFilter(cfg.Dealmaking.Filter))), + Override(new(dtypes.RetrievalDealFilter), modules.RetrievalDealFilter(dealfilter.CliRetrievalDealFilter(cfg.Dealmaking.Filter))), ), Override(new(storagemarket.StorageProviderNode), storageadapter.NewProviderNodeAdapter(&cfg.Fees)), diff --git a/node/modules/dtypes/miner.go b/node/modules/dtypes/miner.go index 5bb439b4d..1ef157b7e 100644 --- a/node/modules/dtypes/miner.go +++ b/node/modules/dtypes/miner.go @@ -7,6 +7,7 @@ import ( "github.com/ipfs/go-cid" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-state-types/abi" @@ -71,4 +72,5 @@ type SetExpectedSealDurationFunc func(time.Duration) error // too determine how long sealing is expected to take type GetExpectedSealDurationFunc func() (time.Duration, error) -type DealFilter func(ctx context.Context, deal storagemarket.MinerDeal) (bool, string, error) +type StorageDealFilter func(ctx context.Context, deal storagemarket.MinerDeal) (bool, string, error) +type RetrievalDealFilter func(ctx context.Context, deal retrievalmarket.ProviderDealState) (bool, string, error) diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index adacf5405..74dfff5e1 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -412,16 +412,16 @@ func NewProviderDealFunds(ds dtypes.MetadataDS) (ProviderDealFunds, error) { return funds.NewDealFunds(ds, datastore.NewKey("/marketfunds/provider")) } -func BasicDealFilter(user dtypes.DealFilter) func(onlineOk dtypes.ConsiderOnlineStorageDealsConfigFunc, +func BasicDealFilter(user dtypes.StorageDealFilter) func(onlineOk dtypes.ConsiderOnlineStorageDealsConfigFunc, offlineOk dtypes.ConsiderOfflineStorageDealsConfigFunc, blocklistFunc dtypes.StorageDealPieceCidBlocklistConfigFunc, expectedSealTimeFunc dtypes.GetExpectedSealDurationFunc, - spn storagemarket.StorageProviderNode) dtypes.DealFilter { + spn storagemarket.StorageProviderNode) dtypes.StorageDealFilter { return func(onlineOk dtypes.ConsiderOnlineStorageDealsConfigFunc, offlineOk dtypes.ConsiderOfflineStorageDealsConfigFunc, blocklistFunc dtypes.StorageDealPieceCidBlocklistConfigFunc, expectedSealTimeFunc dtypes.GetExpectedSealDurationFunc, - spn storagemarket.StorageProviderNode) dtypes.DealFilter { + spn storagemarket.StorageProviderNode) dtypes.StorageDealFilter { return func(ctx context.Context, deal storagemarket.MinerDeal) (bool, string, error) { b, err := onlineOk() @@ -497,7 +497,7 @@ func StorageProvider(minerAddress dtypes.MinerAddress, pieceStore dtypes.ProviderPieceStore, dataTransfer dtypes.ProviderDataTransfer, spn storagemarket.StorageProviderNode, - df dtypes.DealFilter, + df dtypes.StorageDealFilter, funds ProviderDealFunds, ) (storagemarket.StorageProvider, error) { net := smnet.NewFromLibp2pHost(h) @@ -511,8 +511,52 @@ func StorageProvider(minerAddress dtypes.MinerAddress, return storageimpl.NewProvider(net, namespace.Wrap(ds, datastore.NewKey("/deals/provider")), store, mds, pieceStore, dataTransfer, spn, address.Address(minerAddress), ffiConfig.SealProofType, storedAsk, funds, opt) } +func RetrievalDealFilter(userFilter dtypes.RetrievalDealFilter) func(onlineOk dtypes.ConsiderOnlineRetrievalDealsConfigFunc, + offlineOk dtypes.ConsiderOfflineRetrievalDealsConfigFunc) dtypes.RetrievalDealFilter { + return func(onlineOk dtypes.ConsiderOnlineRetrievalDealsConfigFunc, + offlineOk dtypes.ConsiderOfflineRetrievalDealsConfigFunc) dtypes.RetrievalDealFilter { + return func(ctx context.Context, state retrievalmarket.ProviderDealState) (bool, string, error) { + b, err := onlineOk() + if err != nil { + return false, "miner error", err + } + + if !b { + log.Warn("online retrieval deal consideration disabled; rejecting retrieval deal proposal from client") + return false, "miner is not accepting online retrieval deals", nil + } + + b, err = offlineOk() + if err != nil { + return false, "miner error", err + } + + if !b { + log.Info("offline retrieval has not been implemented yet") + } + + if userFilter != nil { + return userFilter(ctx, state) + } + + return true, "", nil + } + } +} + // RetrievalProvider creates a new retrieval provider attached to the provider blockstore -func RetrievalProvider(h host.Host, miner *storage.Miner, sealer sectorstorage.SectorManager, full lapi.FullNode, ds dtypes.MetadataDS, pieceStore dtypes.ProviderPieceStore, mds dtypes.StagingMultiDstore, dt dtypes.ProviderDataTransfer, onlineOk dtypes.ConsiderOnlineRetrievalDealsConfigFunc, offlineOk dtypes.ConsiderOfflineRetrievalDealsConfigFunc) (retrievalmarket.RetrievalProvider, error) { +func RetrievalProvider(h host.Host, + miner *storage.Miner, + sealer sectorstorage.SectorManager, + full lapi.FullNode, + ds dtypes.MetadataDS, + pieceStore dtypes.ProviderPieceStore, + mds dtypes.StagingMultiDstore, + dt dtypes.ProviderDataTransfer, + onlineOk dtypes.ConsiderOnlineRetrievalDealsConfigFunc, + offlineOk dtypes.ConsiderOfflineRetrievalDealsConfigFunc, + userFilter dtypes.RetrievalDealFilter, +) (retrievalmarket.RetrievalProvider, error) { adapter := retrievaladapter.NewRetrievalProviderNode(miner, sealer, full) maddr, err := minerAddrFromDS(ds) @@ -521,29 +565,7 @@ func RetrievalProvider(h host.Host, miner *storage.Miner, sealer sectorstorage.S } netwk := rmnet.NewFromLibp2pHost(h) - - opt := retrievalimpl.DealDeciderOpt(func(ctx context.Context, state retrievalmarket.ProviderDealState) (bool, string, error) { - b, err := onlineOk() - if err != nil { - return false, "miner error", err - } - - if !b { - log.Warn("online retrieval deal consideration disabled; rejecting retrieval deal proposal from client") - return false, "miner is not accepting online retrieval deals", nil - } - - b, err = offlineOk() - if err != nil { - return false, "miner error", err - } - - if !b { - log.Info("offline retrieval has not been implemented yet") - } - - return true, "", nil - }) + opt := retrievalimpl.DealDeciderOpt(retrievalimpl.DealDecider(userFilter)) return retrievalimpl.NewProvider(maddr, adapter, netwk, pieceStore, mds, dt, namespace.Wrap(ds, datastore.NewKey("/retrievals/provider")), opt) } From 5bee85d57a43d45d77acc910800b1514b8e4517d Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 15 Oct 2020 09:38:27 -0700 Subject: [PATCH 785/885] incremental output for list-messages --- cli/state.go | 55 +++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 44 insertions(+), 11 deletions(-) diff --git a/cli/state.go b/cli/state.go index a02a70505..99ace833a 100644 --- a/cli/state.go +++ b/cli/state.go @@ -837,33 +837,66 @@ var stateListMessagesCmd = &cli.Command{ froma = a } - toh := cctx.Uint64("toheight") + toh := abi.ChainEpoch(cctx.Uint64("toheight")) ts, err := LoadTipSet(ctx, cctx, api) if err != nil { return err } - msgs, err := api.StateListMessages(ctx, &lapi.MessageMatch{To: toa, From: froma}, ts.Key(), abi.ChainEpoch(toh)) - if err != nil { - return err + if ts == nil { + head, err := api.ChainHead(ctx) + if err != nil { + return err + } + ts = head } - for _, c := range msgs { - if cctx.Bool("cids") { - fmt.Println(c.String()) - continue + windowSize := abi.ChainEpoch(100) + + cur := ts + for cur.Height() > toh { + if ctx.Err() != nil { + return ctx.Err() } - m, err := api.ChainGetMessage(ctx, c) + end := toh + if cur.Height()-windowSize > end { + end = cur.Height() - windowSize + } + + msgs, err := api.StateListMessages(ctx, &lapi.MessageMatch{To: toa, From: froma}, cur.Key(), end) if err != nil { return err } - b, err := json.MarshalIndent(m, "", " ") + + for _, c := range msgs { + if cctx.Bool("cids") { + fmt.Println(c.String()) + continue + } + + m, err := api.ChainGetMessage(ctx, c) + if err != nil { + return err + } + b, err := json.MarshalIndent(m, "", " ") + if err != nil { + return err + } + fmt.Println(string(b)) + } + + if end <= 0 { + break + } + + next, err := api.ChainGetTipSetByHeight(ctx, end-1, cur.Key()) if err != nil { return err } - fmt.Println(string(b)) + + cur = next } return nil From eae7d62a2b416c7e51d0884a0a875c644a72431f Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 15 Oct 2020 12:07:41 -0700 Subject: [PATCH 786/885] docsgen --- documentation/en/api-methods.md | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index a905ef2e3..4c3bd44ff 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -3352,19 +3352,8 @@ Inputs: ```json [ { - "Version": 42, "To": "f01234", - "From": "f01234", - "Nonce": 42, - "Value": "0", - "GasLimit": 9, - "GasFeeCap": "0", - "GasPremium": "0", - "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==", - "CID": { - "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" - } + "From": "f01234" }, [ { From 726eacbe037c02b5d5cd9648f0171e968dedf63e Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 15 Oct 2020 13:54:58 -0700 Subject: [PATCH 787/885] add endpoints needed by spacegap --- cmd/lotus-gateway/api.go | 57 ++++++++++++++++++++++++++++++++--- cmd/lotus-gateway/api_test.go | 3 ++ 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/cmd/lotus-gateway/api.go b/cmd/lotus-gateway/api.go index 1cdd37915..fe68018d8 100644 --- a/cmd/lotus-gateway/api.go +++ b/cmd/lotus-gateway/api.go @@ -6,10 +6,12 @@ import ( "time" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-bitfield" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/sigs" _ "github.com/filecoin-project/lotus/lib/sigs/bls" @@ -44,6 +46,12 @@ type gatewayDepsAPI interface { StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) StateWaitMsgLimited(ctx context.Context, msg cid.Cid, confidence uint64, h abi.ChainEpoch) (*api.MsgLookup, error) StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*api.ActorState, error) + StateMinerPower(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error) + StateMinerFaults(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error) + StateMinerRecoveries(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error) + StateMinerInfo(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error) + StateMinerDeadlines(context.Context, address.Address, types.TipSetKey) ([]api.Deadline, error) + StateMinerAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) } type GatewayAPI struct { @@ -198,10 +206,6 @@ func (a *GatewayAPI) StateWaitMsg(ctx context.Context, msg cid.Cid, confidence u return a.api.StateWaitMsgLimited(ctx, msg, confidence, stateWaitLookbackLimit) } -func (a *GatewayAPI) WalletVerify(ctx context.Context, k address.Address, msg []byte, sig *crypto.Signature) (bool, error) { - return sigs.Verify(sig, k, msg) == nil, nil -} - func (a *GatewayAPI) StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*api.ActorState, error) { if err := a.checkTipsetKey(ctx, tsk); err != nil { return nil, err @@ -209,6 +213,51 @@ func (a *GatewayAPI) StateReadState(ctx context.Context, actor address.Address, return a.api.StateReadState(ctx, actor, tsk) } +func (a *GatewayAPI) StateMinerPower(ctx context.Context, m address.Address, tsk types.TipSetKey) (*api.MinerPower, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return nil, err + } + return a.api.StateMinerPower(ctx, m, tsk) +} + +func (a *GatewayAPI) StateMinerFaults(ctx context.Context, m address.Address, tsk types.TipSetKey) (bitfield.BitField, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return bitfield.BitField{}, err + } + return a.api.StateMinerFaults(ctx, m, tsk) +} +func (a *GatewayAPI) StateMinerRecoveries(ctx context.Context, m address.Address, tsk types.TipSetKey) (bitfield.BitField, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return bitfield.BitField{}, err + } + return a.api.StateMinerRecoveries(ctx, m, tsk) +} + +func (a *GatewayAPI) StateMinerInfo(ctx context.Context, m address.Address, tsk types.TipSetKey) (miner.MinerInfo, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return miner.MinerInfo{}, err + } + return a.api.StateMinerInfo(ctx, m, tsk) +} + +func (a *GatewayAPI) StateMinerDeadlines(ctx context.Context, m address.Address, tsk types.TipSetKey) ([]api.Deadline, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return nil, err + } + return a.api.StateMinerDeadlines(ctx, m, tsk) +} + +func (a *GatewayAPI) StateMinerAvailableBalance(ctx context.Context, m address.Address, tsk types.TipSetKey) (types.BigInt, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return types.BigInt{}, err + } + return a.api.StateMinerAvailableBalance(ctx, m, tsk) +} + +func (a *GatewayAPI) WalletVerify(ctx context.Context, k address.Address, msg []byte, sig *crypto.Signature) (bool, error) { + return sigs.Verify(sig, k, msg) == nil, nil +} + var _ api.GatewayAPI = (*GatewayAPI)(nil) var _ full.ChainModuleAPI = (*GatewayAPI)(nil) var _ full.GasModuleAPI = (*GatewayAPI)(nil) diff --git a/cmd/lotus-gateway/api_test.go b/cmd/lotus-gateway/api_test.go index 4f10fe987..ae161945b 100644 --- a/cmd/lotus-gateway/api_test.go +++ b/cmd/lotus-gateway/api_test.go @@ -107,6 +107,9 @@ func TestGatewayAPIChainGetTipSetByHeight(t *testing.T) { type mockGatewayDepsAPI struct { lk sync.RWMutex tipsets []*types.TipSet + + gatewayDepsAPI // satisfies all interface requirements but will panic if + // methods are called. easier than filling out with panic stubs IMO } func (m *mockGatewayDepsAPI) ChainHasObj(context.Context, cid.Cid) (bool, error) { From cb801d47c7712898aa25a757ce225916f605a354 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Thu, 15 Oct 2020 18:48:51 -0400 Subject: [PATCH 788/885] Add GasCost to InvocResult --- api/api_full.go | 1 + chain/stmgr/call.go | 2 ++ chain/stmgr/stmgr.go | 1 + chain/stmgr/utils.go | 13 +++++++++++++ node/impl/full/state.go | 13 +++---------- 5 files changed, 20 insertions(+), 10 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 65182078a..130cba957 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -740,6 +740,7 @@ type InvocResult struct { MsgCid cid.Cid Msg *types.Message MsgRct *types.MessageReceipt + GasCost MsgGasCost ExecutionTrace types.ExecutionTrace Error string Duration time.Duration diff --git a/chain/stmgr/call.go b/chain/stmgr/call.go index 86035140b..5eaa2e16a 100644 --- a/chain/stmgr/call.go +++ b/chain/stmgr/call.go @@ -116,6 +116,7 @@ func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types. MsgCid: msg.Cid(), Msg: msg, MsgRct: &ret.MessageReceipt, + GasCost: MakeMsgGasCost(msg, ret), ExecutionTrace: ret.ExecutionTrace, Error: errs, Duration: ret.Duration, @@ -232,6 +233,7 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri MsgCid: msg.Cid(), Msg: msg, MsgRct: &ret.MessageReceipt, + GasCost: MakeMsgGasCost(msg, ret), ExecutionTrace: ret.ExecutionTrace, Error: errs, Duration: ret.Duration, diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index b7d5050e1..d5cb1958d 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -203,6 +203,7 @@ func traceFunc(trace *[]*api.InvocResult) func(mcid cid.Cid, msg *types.Message, MsgCid: mcid, Msg: msg, MsgRct: &ret.MessageReceipt, + GasCost: MakeMsgGasCost(msg, ret), ExecutionTrace: ret.ExecutionTrace, Duration: ret.Duration, } diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index cbc78ca5b..de4f947df 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -701,3 +701,16 @@ func CheckTotalFIL(ctx context.Context, sm *StateManager, ts *types.TipSet) (abi return sum, nil } + +func MakeMsgGasCost(msg *types.Message, ret *vm.ApplyRet) api.MsgGasCost { + return api.MsgGasCost{ + Message: msg.Cid(), + GasUsed: big.NewInt(ret.GasUsed), + BaseFeeBurn: ret.GasCosts.BaseFeeBurn, + OverEstimationBurn: ret.GasCosts.OverEstimationBurn, + MinerPenalty: ret.GasCosts.MinerPenalty, + MinerTip: ret.GasCosts.MinerTip, + Refund: ret.GasCosts.Refund, + TotalCost: big.Sub(msg.RequiredFunds(), ret.GasCosts.Refund), + } +} diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 08ac62f2e..9558219c7 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -365,6 +365,7 @@ func (a *StateAPI) StateReplay(ctx context.Context, tsk types.TipSetKey, mc cid. MsgCid: mc, Msg: m, MsgRct: &r.MessageReceipt, + GasCost: stmgr.MakeMsgGasCost(m, r), ExecutionTrace: r.ExecutionTrace, Error: errstr, Duration: r.Duration, @@ -1304,14 +1305,6 @@ func (a *StateAPI) StateMsgGasCost(ctx context.Context, inputMsg cid.Cid, tsk ty return nil, err } - return &api.MsgGasCost{ - Message: msg, - GasUsed: big.NewInt(r.GasUsed), - BaseFeeBurn: r.GasCosts.BaseFeeBurn, - OverEstimationBurn: r.GasCosts.OverEstimationBurn, - MinerPenalty: r.GasCosts.MinerPenalty, - MinerTip: r.GasCosts.MinerTip, - Refund: r.GasCosts.Refund, - TotalCost: big.Sub(m.RequiredFunds(), r.GasCosts.Refund), - }, nil + gc := stmgr.MakeMsgGasCost(m, r) + return &gc, nil } From 7826cc0babf005dfb84f5c4ab7614ab3d1fa201b Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Thu, 15 Oct 2020 19:14:53 -0400 Subject: [PATCH 789/885] Rename StateReplay to StateTransplant --- api/api_full.go | 4 ++-- api/apistruct/struct.go | 6 +++--- cli/state.go | 27 ++++++++------------------- lotuspond/front/src/Block.js | 2 +- node/impl/full/state.go | 2 +- 5 files changed, 15 insertions(+), 26 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 130cba957..9eaca53f9 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -320,8 +320,8 @@ type FullNode interface { // StateCall runs the given message and returns its result without any persisted changes. StateCall(context.Context, *types.Message, types.TipSetKey) (*InvocResult, error) - // StateReplay returns the result of executing the indicated message, assuming it was executed in the indicated tipset. - StateReplay(context.Context, types.TipSetKey, cid.Cid) (*InvocResult, error) + // StateTransplant returns the result of executing the indicated message, assuming it was executed in the indicated tipset. + StateTransplant(context.Context, types.TipSetKey, cid.Cid) (*InvocResult, error) // StateGetActor returns the indicated actor's nonce and balance. StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) // StateReadState returns the indicated actor's state. diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 3854e1dd6..3e474f596 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -188,7 +188,7 @@ type FullNodeStruct struct { StateSectorExpiration func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorExpiration, error) `perm:"read"` StateSectorPartition func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorLocation, error) `perm:"read"` StateCall func(context.Context, *types.Message, types.TipSetKey) (*api.InvocResult, error) `perm:"read"` - StateReplay func(context.Context, types.TipSetKey, cid.Cid) (*api.InvocResult, error) `perm:"read"` + StateTransplant func(context.Context, types.TipSetKey, cid.Cid) (*api.InvocResult, error) `perm:"read"` StateGetActor func(context.Context, address.Address, types.TipSetKey) (*types.Actor, error) `perm:"read"` StateReadState func(context.Context, address.Address, types.TipSetKey) (*api.ActorState, error) `perm:"read"` StateMsgGasCost func(context.Context, cid.Cid, types.TipSetKey) (*api.MsgGasCost, error) `perm:"read"` @@ -880,8 +880,8 @@ func (c *FullNodeStruct) StateCall(ctx context.Context, msg *types.Message, tsk return c.Internal.StateCall(ctx, msg, tsk) } -func (c *FullNodeStruct) StateReplay(ctx context.Context, tsk types.TipSetKey, mc cid.Cid) (*api.InvocResult, error) { - return c.Internal.StateReplay(ctx, tsk, mc) +func (c *FullNodeStruct) StateTransplant(ctx context.Context, tsk types.TipSetKey, mc cid.Cid) (*api.InvocResult, error) { + return c.Internal.StateTransplant(ctx, tsk, mc) } func (c *FullNodeStruct) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) { diff --git a/cli/state.go b/cli/state.go index 1e7e24ddb..362e29636 100644 --- a/cli/state.go +++ b/cli/state.go @@ -60,7 +60,7 @@ var stateCmd = &cli.Command{ stateSectorCmd, stateGetActorCmd, stateLookupIDCmd, - stateReplaySetCmd, + stateTransplantCmd, stateSectorSizeCmd, stateReadStateCmd, stateListMessagesCmd, @@ -384,9 +384,9 @@ var stateExecTraceCmd = &cli.Command{ }, } -var stateReplaySetCmd = &cli.Command{ - Name: "replay", - Usage: "Replay a particular message within a tipset", +var stateTransplantCmd = &cli.Command{ + Name: "transplant", + Usage: "Play a particular message within a tipset", ArgsUsage: "[tipsetKey messageCid]", Action: func(cctx *cli.Context) error { if cctx.Args().Len() < 1 { @@ -437,30 +437,19 @@ var stateReplaySetCmd = &cli.Command{ return err } } else { - var r *api.MsgLookup - r, err = fapi.StateWaitMsg(ctx, mcid, build.MessageConfidence) - if err != nil { - return xerrors.Errorf("finding message in chain: %w", err) - } - - childTs, err := fapi.ChainGetTipSet(ctx, r.TipSet) - if err != nil { - return xerrors.Errorf("loading tipset: %w", err) - } - ts, err = fapi.ChainGetTipSet(ctx, childTs.Parents()) + ts, err = fapi.ChainHead(ctx) if err != nil { return err } } - } - res, err := fapi.StateReplay(ctx, ts.Key(), mcid) + res, err := fapi.StateTransplant(ctx, ts.Key(), mcid) if err != nil { - return xerrors.Errorf("replay call failed: %w", err) + return xerrors.Errorf("transplant call failed: %w", err) } - fmt.Println("Replay receipt:") + fmt.Println("Transplant receipt:") fmt.Printf("Exit code: %d\n", res.MsgRct.ExitCode) fmt.Printf("Return: %x\n", res.MsgRct.Return) fmt.Printf("Gas Used: %d\n", res.MsgRct.GasUsed) diff --git a/lotuspond/front/src/Block.js b/lotuspond/front/src/Block.js index 4bae57c81..cb8d25c9d 100644 --- a/lotuspond/front/src/Block.js +++ b/lotuspond/front/src/Block.js @@ -26,7 +26,7 @@ class Block extends React.Component { messages = await Promise.all(messages.map(async (msg, i) => { if (msg.receipt.ExitCode !== 0) { - let reply = await this.props.conn.call('Filecoin.StateReplay', [{Cids: [this.props.cid], Blocks: [header], Height: header.Height}, msg.Cid]) + let reply = await this.props.conn.call('Filecoin.StateTransplant', [{Cids: [this.props.cid], Blocks: [header], Height: header.Height}, msg.Cid]) if(!reply.Error) { reply.Error = "reply: no error" } diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 9558219c7..349a4918b 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -346,7 +346,7 @@ func (a *StateAPI) StateCall(ctx context.Context, msg *types.Message, tsk types. return res, err } -func (a *StateAPI) StateReplay(ctx context.Context, tsk types.TipSetKey, mc cid.Cid) (*api.InvocResult, error) { +func (a *StateAPI) StateTransplant(ctx context.Context, tsk types.TipSetKey, mc cid.Cid) (*api.InvocResult, error) { ts, err := a.Chain.GetTipSetFromKey(tsk) if err != nil { return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) From 8351420404bc97e3e4eef0ad85db4e43eed11a11 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 15 Oct 2020 16:27:49 -0700 Subject: [PATCH 790/885] Add more endpoints for spacegap --- cmd/lotus-gateway/api.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/cmd/lotus-gateway/api.go b/cmd/lotus-gateway/api.go index fe68018d8..855dbe1e8 100644 --- a/cmd/lotus-gateway/api.go +++ b/cmd/lotus-gateway/api.go @@ -9,6 +9,7 @@ import ( "github.com/filecoin-project/go-bitfield" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/crypto" + "github.com/filecoin-project/go-state-types/dline" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" @@ -37,6 +38,7 @@ type gatewayDepsAPI interface { ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) ChainReadObj(context.Context, cid.Cid) ([]byte, error) + ChainGetNode(ctx context.Context, p string) (*api.IpldObject, error) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) MpoolPushUntrusted(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) @@ -52,6 +54,8 @@ type gatewayDepsAPI interface { StateMinerInfo(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error) StateMinerDeadlines(context.Context, address.Address, types.TipSetKey) ([]api.Deadline, error) StateMinerAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) + StateMinerProvingDeadline(context.Context, address.Address, types.TipSetKey) (*dline.Info, error) + StateCirculatingSupply(context.Context, types.TipSetKey) (abi.TokenAmount, error) } type GatewayAPI struct { @@ -146,6 +150,10 @@ func (a *GatewayAPI) ChainReadObj(ctx context.Context, c cid.Cid) ([]byte, error return a.api.ChainReadObj(ctx, c) } +func (a *GatewayAPI) ChainGetNode(ctx context.Context, p string) (*api.IpldObject, error) { + return a.api.ChainGetNode(ctx, p) +} + func (a *GatewayAPI) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) { if err := a.checkTipsetKey(ctx, tsk); err != nil { return nil, err @@ -254,6 +262,21 @@ func (a *GatewayAPI) StateMinerAvailableBalance(ctx context.Context, m address.A return a.api.StateMinerAvailableBalance(ctx, m, tsk) } +func (a *GatewayAPI) StateMinerProvingDeadline(ctx context.Context, m address.Address, tsk types.TipSetKey) (*dline.Info, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return nil, err + } + return a.api.StateMinerProvingDeadline(ctx, m, tsk) +} + +func (a *GatewayAPI) StateCirculatingSupply(ctx context.Context, tsk types.TipSetKey) (abi.TokenAmount, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return types.BigInt{}, err + } + return a.api.StateCirculatingSupply(ctx, tsk) + +} + func (a *GatewayAPI) WalletVerify(ctx context.Context, k address.Address, msg []byte, sig *crypto.Signature) (bool, error) { return sigs.Verify(sig, k, msg) == nil, nil } From af1a422ddd0773c9de68b7b9489d56ff456846a3 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 15 Oct 2020 16:36:45 -0700 Subject: [PATCH 791/885] statevminternal circ supply --- cmd/lotus-gateway/api.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cmd/lotus-gateway/api.go b/cmd/lotus-gateway/api.go index 855dbe1e8..86e49e62f 100644 --- a/cmd/lotus-gateway/api.go +++ b/cmd/lotus-gateway/api.go @@ -56,6 +56,7 @@ type gatewayDepsAPI interface { StateMinerAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) StateMinerProvingDeadline(context.Context, address.Address, types.TipSetKey) (*dline.Info, error) StateCirculatingSupply(context.Context, types.TipSetKey) (abi.TokenAmount, error) + StateVMCirculatingSupplyInternal(context.Context, types.TipSetKey) (api.CirculatingSupply, error) } type GatewayAPI struct { @@ -277,6 +278,13 @@ func (a *GatewayAPI) StateCirculatingSupply(ctx context.Context, tsk types.TipSe } +func (a *GatewayAPI) StateVMCirculatingSupplyInternal(ctx context.Context, tsk types.TipSetKey) (api.CirculatingSupply, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return api.CirculatingSupply{}, err + } + return a.api.StateVMCirculatingSupplyInternal(ctx, tsk) +} + func (a *GatewayAPI) WalletVerify(ctx context.Context, k address.Address, msg []byte, sig *crypto.Signature) (bool, error) { return sigs.Verify(sig, k, msg) == nil, nil } From 99c07703f8f60799e9e2637bffc30613267d14de Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Thu, 15 Oct 2020 19:47:02 -0400 Subject: [PATCH 792/885] Add a new StateReplay CLI/API endpoint --- api/api_full.go | 4 +- api/apistruct/struct.go | 5 +++ cli/state.go | 91 +++++++++++++++++++++++++++++++++++++++++ node/impl/full/state.go | 40 ++++++++++++++++++ 4 files changed, 139 insertions(+), 1 deletion(-) diff --git a/api/api_full.go b/api/api_full.go index 9eaca53f9..29688a5e6 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -315,13 +315,15 @@ type FullNode interface { // MethodGroup: State // The State methods are used to query, inspect, and interact with chain state. - // All methods take a TipSetKey as a parameter. The state looked up is the state at that tipset. + // Most methods take a TipSetKey as a parameter. The state looked up is the state at that tipset. // A nil TipSetKey can be provided as a param, this will cause the heaviest tipset in the chain to be used. // StateCall runs the given message and returns its result without any persisted changes. StateCall(context.Context, *types.Message, types.TipSetKey) (*InvocResult, error) // StateTransplant returns the result of executing the indicated message, assuming it was executed in the indicated tipset. StateTransplant(context.Context, types.TipSetKey, cid.Cid) (*InvocResult, error) + // StateReplay searches for where the given message was executed, and replays it in that tipset. + StateReplay(context.Context, cid.Cid) (*InvocResult, error) // StateGetActor returns the indicated actor's nonce and balance. StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) // StateReadState returns the indicated actor's state. diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 3e474f596..33670c276 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -189,6 +189,7 @@ type FullNodeStruct struct { StateSectorPartition func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorLocation, error) `perm:"read"` StateCall func(context.Context, *types.Message, types.TipSetKey) (*api.InvocResult, error) `perm:"read"` StateTransplant func(context.Context, types.TipSetKey, cid.Cid) (*api.InvocResult, error) `perm:"read"` + StateReplay func(context.Context, cid.Cid) (*api.InvocResult, error) `perm:"read"` StateGetActor func(context.Context, address.Address, types.TipSetKey) (*types.Actor, error) `perm:"read"` StateReadState func(context.Context, address.Address, types.TipSetKey) (*api.ActorState, error) `perm:"read"` StateMsgGasCost func(context.Context, cid.Cid, types.TipSetKey) (*api.MsgGasCost, error) `perm:"read"` @@ -884,6 +885,10 @@ func (c *FullNodeStruct) StateTransplant(ctx context.Context, tsk types.TipSetKe return c.Internal.StateTransplant(ctx, tsk, mc) } +func (c *FullNodeStruct) StateReplay(ctx context.Context, mc cid.Cid) (*api.InvocResult, error) { + return c.Internal.StateReplay(ctx, mc) +} + func (c *FullNodeStruct) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) { return c.Internal.StateGetActor(ctx, actor, tsk) } diff --git a/cli/state.go b/cli/state.go index 362e29636..bff41e268 100644 --- a/cli/state.go +++ b/cli/state.go @@ -61,6 +61,7 @@ var stateCmd = &cli.Command{ stateGetActorCmd, stateLookupIDCmd, stateTransplantCmd, + stateReplayCmd, stateSectorSizeCmd, stateReadStateCmd, stateListMessagesCmd, @@ -388,6 +389,16 @@ var stateTransplantCmd = &cli.Command{ Name: "transplant", Usage: "Play a particular message within a tipset", ArgsUsage: "[tipsetKey messageCid]", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "show-trace", + Usage: "print out full execution trace for given message", + }, + &cli.BoolFlag{ + Name: "detailed-gas", + Usage: "print out detailed gas costs for given message", + }, + }, Action: func(cctx *cli.Context) error { if cctx.Args().Len() < 1 { fmt.Println("usage: [tipset] ") @@ -453,10 +464,90 @@ var stateTransplantCmd = &cli.Command{ fmt.Printf("Exit code: %d\n", res.MsgRct.ExitCode) fmt.Printf("Return: %x\n", res.MsgRct.Return) fmt.Printf("Gas Used: %d\n", res.MsgRct.GasUsed) + + if cctx.Bool("detailed-gas") { + fmt.Printf("Base Fee Burn: %d\n", res.GasCost.BaseFeeBurn) + fmt.Printf("Overestimaton Burn: %d\n", res.GasCost.OverEstimationBurn) + fmt.Printf("Miner Penalty: %d\n", res.GasCost.MinerPenalty) + fmt.Printf("Miner Tip: %d\n", res.GasCost.MinerTip) + fmt.Printf("Refund: %d\n", res.GasCost.Refund) + } + fmt.Printf("Total Message Cost: %d\n", res.GasCost.TotalCost) + if res.MsgRct.ExitCode != 0 { fmt.Printf("Error message: %q\n", res.Error) } + if cctx.Bool("show-trace") { + fmt.Printf("%s\t%s\t%s\t%d\t%x\t%d\t%x\n", res.Msg.From, res.Msg.To, res.Msg.Value, res.Msg.Method, res.Msg.Params, res.MsgRct.ExitCode, res.MsgRct.Return) + printInternalExecutions("\t", res.ExecutionTrace.Subcalls) + } + + return nil + }, +} + +var stateReplayCmd = &cli.Command{ + Name: "replay", + Usage: "Replay a particular message", + ArgsUsage: "", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "show-trace", + Usage: "print out full execution trace for given message", + }, + &cli.BoolFlag{ + Name: "detailed-gas", + Usage: "print out detailed gas costs for given message", + }, + }, + Action: func(cctx *cli.Context) error { + if cctx.Args().Len() != 1 { + fmt.Println("must provide cid of message to replay") + return nil + } + + mcid, err := cid.Decode(cctx.Args().First()) + if err != nil { + return fmt.Errorf("message cid was invalid: %s", err) + } + + fapi, closer, err := GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + + ctx := ReqContext(cctx) + + res, err := fapi.StateReplay(ctx, mcid) + if err != nil { + return xerrors.Errorf("replay call failed: %w", err) + } + + fmt.Println("Replay receipt:") + fmt.Printf("Exit code: %d\n", res.MsgRct.ExitCode) + fmt.Printf("Return: %x\n", res.MsgRct.Return) + fmt.Printf("Gas Used: %d\n", res.MsgRct.GasUsed) + + if cctx.Bool("detailed-gas") { + fmt.Printf("Base Fee Burn: %d\n", res.GasCost.BaseFeeBurn) + fmt.Printf("Overestimaton Burn: %d\n", res.GasCost.OverEstimationBurn) + fmt.Printf("Miner Penalty: %d\n", res.GasCost.MinerPenalty) + fmt.Printf("Miner Tip: %d\n", res.GasCost.MinerTip) + fmt.Printf("Refund: %d\n", res.GasCost.Refund) + } + fmt.Printf("Total Message Cost: %d\n", res.GasCost.TotalCost) + + if res.MsgRct.ExitCode != 0 { + fmt.Printf("Error message: %q\n", res.Error) + } + + if cctx.Bool("show-trace") { + fmt.Printf("%s\t%s\t%s\t%d\t%x\t%d\t%x\n", res.Msg.From, res.Msg.To, res.Msg.Value, res.Msg.Method, res.Msg.Params, res.MsgRct.ExitCode, res.MsgRct.Return) + printInternalExecutions("\t", res.ExecutionTrace.Subcalls) + } + return nil }, } diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 349a4918b..959660085 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -372,6 +372,46 @@ func (a *StateAPI) StateTransplant(ctx context.Context, tsk types.TipSetKey, mc }, nil } +func (a *StateAPI) StateReplay(ctx context.Context, mc cid.Cid) (*api.InvocResult, error) { + mlkp, err := a.StateSearchMsg(ctx, mc) + if err != nil { + return nil, xerrors.Errorf("searching for msg %s: %w", mc, err) + } + if mlkp == nil { + return nil, xerrors.Errorf("didn't find msg %s", mc) + } + + executionTs, err := a.Chain.GetTipSetFromKey(mlkp.TipSet) + if err != nil { + return nil, xerrors.Errorf("loading tipset %s: %w", mlkp.TipSet, err) + } + + ts, err := a.Chain.LoadTipSet(executionTs.Parents()) + if err != nil { + return nil, xerrors.Errorf("loading parent tipset %s: %w", mlkp.TipSet, err) + } + + m, r, err := a.StateManager.Replay(ctx, ts, mlkp.Message) + if err != nil { + return nil, err + } + + var errstr string + if r.ActorErr != nil { + errstr = r.ActorErr.Error() + } + + return &api.InvocResult{ + MsgCid: mlkp.Message, + Msg: m, + MsgRct: &r.MessageReceipt, + GasCost: stmgr.MakeMsgGasCost(m, r), + ExecutionTrace: r.ExecutionTrace, + Error: errstr, + Duration: r.Duration, + }, nil +} + func stateForTs(ctx context.Context, ts *types.TipSet, cstore *store.ChainStore, smgr *stmgr.StateManager) (*state.StateTree, error) { if ts == nil { ts = cstore.GetHeaviestTipSet() From a71fbee2d13e83c9fbe49a3dd59d27fa39a711c7 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 15 Oct 2020 16:49:25 -0700 Subject: [PATCH 793/885] bump lookback limit to 12 hours --- cmd/lotus-gateway/api.go | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/cmd/lotus-gateway/api.go b/cmd/lotus-gateway/api.go index 86e49e62f..105f8881e 100644 --- a/cmd/lotus-gateway/api.go +++ b/cmd/lotus-gateway/api.go @@ -22,7 +22,7 @@ import ( ) const ( - LookbackCap = time.Hour + LookbackCap = time.Hour * 12 stateWaitLookbackLimit = abi.ChainEpoch(20) ) @@ -129,9 +129,19 @@ func (a *GatewayAPI) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (* } func (a *GatewayAPI) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) { - ts, err := a.api.ChainGetTipSet(ctx, tsk) - if err != nil { - return nil, err + var ts *types.TipSet + if tsk.IsEmpty() { + head, err := a.api.ChainHead(ctx) + if err != nil { + return nil, err + } + ts = head + } else { + gts, err := a.api.ChainGetTipSet(ctx, tsk) + if err != nil { + return nil, err + } + ts = gts } // Check if the tipset key refers to a tipset that's too far in the past From 6f86b95f62bd59b38f1499ee381b93643247500b Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Thu, 15 Oct 2020 21:00:47 -0400 Subject: [PATCH 794/885] Remove StateMsgGasCost --- api/api_full.go | 2 - api/apistruct/struct.go | 5 - cli/state.go | 55 ----------- documentation/en/api-methods.md | 169 ++++++++++++++++++++++---------- node/impl/full/state.go | 41 -------- 5 files changed, 119 insertions(+), 153 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 29688a5e6..88bcb3cba 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -372,8 +372,6 @@ type FullNode interface { StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*miner.SectorLocation, error) // StateSearchMsg searches for a message in the chain, and returns its receipt and the tipset where it was executed StateSearchMsg(context.Context, cid.Cid) (*MsgLookup, error) - // StateMsgGasCost searches for a message in the chain, and returns details of the messages gas costs, including the penalty and miner tip - StateMsgGasCost(context.Context, cid.Cid, types.TipSetKey) (*MsgGasCost, error) // StateWaitMsg looks back in the chain for a message. If not found, it blocks until the // message arrives on chain, and gets to the indicated confidence depth. StateWaitMsg(ctx context.Context, cid cid.Cid, confidence uint64) (*MsgLookup, error) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 33670c276..0279e8c8a 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -192,7 +192,6 @@ type FullNodeStruct struct { StateReplay func(context.Context, cid.Cid) (*api.InvocResult, error) `perm:"read"` StateGetActor func(context.Context, address.Address, types.TipSetKey) (*types.Actor, error) `perm:"read"` StateReadState func(context.Context, address.Address, types.TipSetKey) (*api.ActorState, error) `perm:"read"` - StateMsgGasCost func(context.Context, cid.Cid, types.TipSetKey) (*api.MsgGasCost, error) `perm:"read"` StateWaitMsg func(ctx context.Context, cid cid.Cid, confidence uint64) (*api.MsgLookup, error) `perm:"read"` StateWaitMsgLimited func(context.Context, cid.Cid, uint64, abi.ChainEpoch) (*api.MsgLookup, error) `perm:"read"` StateSearchMsg func(context.Context, cid.Cid) (*api.MsgLookup, error) `perm:"read"` @@ -897,10 +896,6 @@ func (c *FullNodeStruct) StateReadState(ctx context.Context, addr address.Addres return c.Internal.StateReadState(ctx, addr, tsk) } -func (c *FullNodeStruct) StateMsgGasCost(ctx context.Context, msgc cid.Cid, tsk types.TipSetKey) (*api.MsgGasCost, error) { - return c.Internal.StateMsgGasCost(ctx, msgc, tsk) -} - func (c *FullNodeStruct) StateWaitMsg(ctx context.Context, msgc cid.Cid, confidence uint64) (*api.MsgLookup, error) { return c.Internal.StateWaitMsg(ctx, msgc, confidence) } diff --git a/cli/state.go b/cli/state.go index bff41e268..38454ec27 100644 --- a/cli/state.go +++ b/cli/state.go @@ -70,7 +70,6 @@ var stateCmd = &cli.Command{ stateGetDealSetCmd, stateWaitMsgCmd, stateSearchMsgCmd, - stateMsgCostCmd, stateMinerInfo, stateMarketCmd, stateExecTraceCmd, @@ -1504,60 +1503,6 @@ var stateSearchMsgCmd = &cli.Command{ }, } -var stateMsgCostCmd = &cli.Command{ - Name: "msg-cost", - Usage: "Get the detailed gas costs of a message", - ArgsUsage: "[messageCid]", - Action: func(cctx *cli.Context) error { - if !cctx.Args().Present() { - return fmt.Errorf("must specify message cid to get gas costs for") - } - - api, closer, err := GetFullNodeAPI(cctx) - if err != nil { - return err - } - defer closer() - - ctx := ReqContext(cctx) - - msg, err := cid.Decode(cctx.Args().First()) - if err != nil { - return err - } - - tsk := types.EmptyTSK - - ts, err := LoadTipSet(ctx, cctx, api) - if err != nil { - return err - } - - if ts != nil { - tsk = ts.Key() - } - - mgc, err := api.StateMsgGasCost(ctx, msg, tsk) - if err != nil { - return err - } - - if mgc != nil { - fmt.Printf("Message CID: %s", mgc.Message) - fmt.Printf("\nGas Used: %d", mgc.GasUsed) - fmt.Printf("\nBase Fee Burn: %d", mgc.BaseFeeBurn) - fmt.Printf("\nOverestimation Burn: %d", mgc.OverEstimationBurn) - fmt.Printf("\nMiner Tip: %d", mgc.MinerTip) - fmt.Printf("\nRefund: %d", mgc.Refund) - fmt.Printf("\nTotal Cost: %d", mgc.TotalCost) - fmt.Printf("\nMiner Penalty: %d", mgc.MinerPenalty) - } else { - fmt.Print("message was not found on chain") - } - return nil - }, -} - var stateCallCmd = &cli.Command{ Name: "call", Usage: "Invoke a method on an actor locally", diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index a905ef2e3..9be12722d 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -156,7 +156,6 @@ * [StateMinerRecoveries](#StateMinerRecoveries) * [StateMinerSectorCount](#StateMinerSectorCount) * [StateMinerSectors](#StateMinerSectors) - * [StateMsgGasCost](#StateMsgGasCost) * [StateNetworkName](#StateNetworkName) * [StateNetworkVersion](#StateNetworkVersion) * [StateReadState](#StateReadState) @@ -166,6 +165,7 @@ * [StateSectorGetInfo](#StateSectorGetInfo) * [StateSectorPartition](#StateSectorPartition) * [StateSectorPreCommitInfo](#StateSectorPreCommitInfo) + * [StateTransplant](#StateTransplant) * [StateVMCirculatingSupplyInternal](#StateVMCirculatingSupplyInternal) * [StateVerifiedClientStatus](#StateVerifiedClientStatus) * [StateVerifiedRegistryRootKey](#StateVerifiedRegistryRootKey) @@ -2989,7 +2989,7 @@ Response: ## State The State methods are used to query, inspect, and interact with chain state. -All methods take a TipSetKey as a parameter. The state looked up is the state at that tipset. +Most methods take a TipSetKey as a parameter. The state looked up is the state at that tipset. A nil TipSetKey can be provided as a param, this will cause the heaviest tipset in the chain to be used. @@ -3100,6 +3100,18 @@ Response: "Return": "Ynl0ZSBhcnJheQ==", "GasUsed": 9 }, + "GasCost": { + "Message": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "GasUsed": "0", + "BaseFeeBurn": "0", + "OverEstimationBurn": "0", + "MinerPenalty": "0", + "MinerTip": "0", + "Refund": "0", + "TotalCost": "0" + }, "ExecutionTrace": { "Msg": { "Version": 42, @@ -3974,45 +3986,6 @@ Inputs: Response: `null` -### StateMsgGasCost -StateMsgGasCost searches for a message in the chain, and returns details of the messages gas costs, including the penalty and miner tip - - -Perms: read - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ] -] -``` - -Response: -```json -{ - "Message": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "GasUsed": "0", - "BaseFeeBurn": "0", - "OverEstimationBurn": "0", - "MinerPenalty": "0", - "MinerTip": "0", - "Refund": "0", - "TotalCost": "0" -} -``` - ### StateNetworkName StateNetworkName returns the name of the network the node is synced to @@ -4075,7 +4048,7 @@ Response: ``` ### StateReplay -StateReplay returns the result of executing the indicated message, assuming it was executed in the indicated tipset. +StateReplay searches for where the given message was executed, and replays it in that tipset. Perms: read @@ -4083,14 +4056,6 @@ Perms: read Inputs: ```json [ - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ], { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" } @@ -4123,6 +4088,18 @@ Response: "Return": "Ynl0ZSBhcnJheQ==", "GasUsed": 9 }, + "GasCost": { + "Message": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "GasUsed": "0", + "BaseFeeBurn": "0", + "OverEstimationBurn": "0", + "MinerPenalty": "0", + "MinerTip": "0", + "Refund": "0", + "TotalCost": "0" + }, "ExecutionTrace": { "Msg": { "Version": 42, @@ -4342,6 +4319,98 @@ Response: } ``` +### StateTransplant +StateTransplant returns the result of executing the indicated message, assuming it was executed in the indicated tipset. + + +Perms: read + +Inputs: +```json +[ + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ], + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: +```json +{ + "MsgCid": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Msg": { + "Version": 42, + "To": "f01234", + "From": "f01234", + "Nonce": 42, + "Value": "0", + "GasLimit": 9, + "GasFeeCap": "0", + "GasPremium": "0", + "Method": 1, + "Params": "Ynl0ZSBhcnJheQ==", + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" + } + }, + "MsgRct": { + "ExitCode": 0, + "Return": "Ynl0ZSBhcnJheQ==", + "GasUsed": 9 + }, + "GasCost": { + "Message": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "GasUsed": "0", + "BaseFeeBurn": "0", + "OverEstimationBurn": "0", + "MinerPenalty": "0", + "MinerTip": "0", + "Refund": "0", + "TotalCost": "0" + }, + "ExecutionTrace": { + "Msg": { + "Version": 42, + "To": "f01234", + "From": "f01234", + "Nonce": 42, + "Value": "0", + "GasLimit": 9, + "GasFeeCap": "0", + "GasPremium": "0", + "Method": 1, + "Params": "Ynl0ZSBhcnJheQ==", + "CID": { + "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" + } + }, + "MsgRct": { + "ExitCode": 0, + "Return": "Ynl0ZSBhcnJheQ==", + "GasUsed": 9 + }, + "Error": "string value", + "Duration": 60000000000, + "GasCharges": null, + "Subcalls": null + }, + "Error": "string value", + "Duration": 60000000000 +} +``` + ### StateVMCirculatingSupplyInternal StateVMCirculatingSupplyInternal returns an approximation of the circulating supply of Filecoin at the given tipset. This is the value reported by the runtime interface to actors code. diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 959660085..a63ac8c67 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -1307,44 +1307,3 @@ func (a *StateAPI) StateNetworkVersion(ctx context.Context, tsk types.TipSetKey) return a.StateManager.GetNtwkVersion(ctx, ts.Height()), nil } - -func (a *StateAPI) StateMsgGasCost(ctx context.Context, inputMsg cid.Cid, tsk types.TipSetKey) (*api.MsgGasCost, error) { - var msg cid.Cid - var ts *types.TipSet - var err error - if tsk != types.EmptyTSK { - msg = inputMsg - ts, err = a.Chain.LoadTipSet(tsk) - if err != nil { - return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) - } - } else { - mlkp, err := a.StateSearchMsg(ctx, inputMsg) - if err != nil { - return nil, xerrors.Errorf("searching for msg %s: %w", inputMsg, err) - } - if mlkp == nil { - return nil, xerrors.Errorf("didn't find msg %s", inputMsg) - } - - executionTs, err := a.Chain.GetTipSetFromKey(mlkp.TipSet) - if err != nil { - return nil, xerrors.Errorf("loading tipset %s: %w", mlkp.TipSet, err) - } - - ts, err = a.Chain.LoadTipSet(executionTs.Parents()) - if err != nil { - return nil, xerrors.Errorf("loading parent tipset %s: %w", mlkp.TipSet, err) - } - - msg = mlkp.Message - } - - m, r, err := a.StateManager.Replay(ctx, ts, msg) - if err != nil { - return nil, err - } - - gc := stmgr.MakeMsgGasCost(m, r) - return &gc, nil -} From 89f46cb5ccd7a19b72c7432404f658eb1d7ecfb5 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Thu, 15 Oct 2020 21:12:38 -0400 Subject: [PATCH 795/885] Make ApplyRet's GasOutputs a pointer, return nil when unused --- chain/stmgr/call.go | 1 - chain/stmgr/forks.go | 4 ++-- chain/stmgr/stmgr.go | 4 +++- chain/vm/vm.go | 16 ++++++++-------- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/chain/stmgr/call.go b/chain/stmgr/call.go index 5eaa2e16a..3d7d284bc 100644 --- a/chain/stmgr/call.go +++ b/chain/stmgr/call.go @@ -116,7 +116,6 @@ func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types. MsgCid: msg.Cid(), Msg: msg, MsgRct: &ret.MessageReceipt, - GasCost: MakeMsgGasCost(msg, ret), ExecutionTrace: ret.ExecutionTrace, Error: errs, Duration: ret.Duration, diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index 156c68783..488f84167 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -498,7 +498,7 @@ func UpgradeFaucetBurnRecovery(ctx context.Context, sm *StateManager, cb ExecCal Subcalls: subcalls, }, Duration: 0, - GasCosts: vm.ZeroGasOutputs(), + GasCosts: nil, }); err != nil { return cid.Undef, xerrors.Errorf("recording transfers: %w", err) } @@ -799,7 +799,7 @@ func splitGenesisMultisig(ctx context.Context, cb ExecCallback, addr address.Add Subcalls: subcalls, }, Duration: 0, - GasCosts: vm.ZeroGasOutputs(), + GasCosts: nil, }); err != nil { return xerrors.Errorf("recording transfers: %w", err) } diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index d5cb1958d..be14797d9 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -203,13 +203,15 @@ func traceFunc(trace *[]*api.InvocResult) func(mcid cid.Cid, msg *types.Message, MsgCid: mcid, Msg: msg, MsgRct: &ret.MessageReceipt, - GasCost: MakeMsgGasCost(msg, ret), ExecutionTrace: ret.ExecutionTrace, Duration: ret.Duration, } if ret.ActorErr != nil { ir.Error = ret.ActorErr.Error() } + if ret.GasCosts != nil { + ir.GasCost = MakeMsgGasCost(msg, ret) + } *trace = append(*trace, ir) return nil } diff --git a/chain/vm/vm.go b/chain/vm/vm.go index a4efccb29..72ad731aa 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -214,7 +214,7 @@ type ApplyRet struct { ActorErr aerrors.ActorError ExecutionTrace types.ExecutionTrace Duration time.Duration - GasCosts GasOutputs + GasCosts *GasOutputs } func (vm *VM) send(ctx context.Context, msg *types.Message, parent *Runtime, @@ -361,7 +361,7 @@ func (vm *VM) ApplyImplicitMessage(ctx context.Context, msg *types.Message) (*Ap }, ActorErr: actorErr, ExecutionTrace: rt.executionTrace, - GasCosts: GasOutputs{}, + GasCosts: nil, Duration: time.Since(start), }, actorErr } @@ -397,7 +397,7 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, ExitCode: exitcode.SysErrOutOfGas, GasUsed: 0, }, - GasCosts: gasOutputs, + GasCosts: &gasOutputs, Duration: time.Since(start), }, nil } @@ -417,7 +417,7 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, GasUsed: 0, }, ActorErr: aerrors.Newf(exitcode.SysErrSenderInvalid, "actor not found: %s", msg.From), - GasCosts: gasOutputs, + GasCosts: &gasOutputs, Duration: time.Since(start), }, nil } @@ -434,7 +434,7 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, GasUsed: 0, }, ActorErr: aerrors.Newf(exitcode.SysErrSenderInvalid, "send from not account actor: %s", fromActor.Code), - GasCosts: gasOutputs, + GasCosts: &gasOutputs, Duration: time.Since(start), }, nil } @@ -450,7 +450,7 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, ActorErr: aerrors.Newf(exitcode.SysErrSenderStateInvalid, "actor nonce invalid: msg:%d != state:%d", msg.Nonce, fromActor.Nonce), - GasCosts: gasOutputs, + GasCosts: &gasOutputs, Duration: time.Since(start), }, nil } @@ -466,7 +466,7 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, }, ActorErr: aerrors.Newf(exitcode.SysErrSenderStateInvalid, "actor balance less than needed: %s < %s", types.FIL(fromActor.Balance), types.FIL(gascost)), - GasCosts: gasOutputs, + GasCosts: &gasOutputs, Duration: time.Since(start), }, nil } @@ -560,7 +560,7 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, }, ActorErr: actorErr, ExecutionTrace: rt.executionTrace, - GasCosts: gasOutputs, + GasCosts: &gasOutputs, Duration: time.Since(start), }, nil } From 2fd4a976984bb3b51e8b227b3175ed0c5eb61af3 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Thu, 15 Oct 2020 22:45:11 -0400 Subject: [PATCH 796/885] Remove StateTransplant, modify StateReplay --- api/api_full.go | 7 +-- api/apistruct/struct.go | 11 +--- cli/state.go | 105 +------------------------------- documentation/en/api-methods.md | 104 +++---------------------------- lotuspond/front/src/Block.js | 2 +- node/impl/full/state.go | 74 +++++++++------------- 6 files changed, 46 insertions(+), 257 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 88bcb3cba..1946ca28f 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -320,10 +320,9 @@ type FullNode interface { // StateCall runs the given message and returns its result without any persisted changes. StateCall(context.Context, *types.Message, types.TipSetKey) (*InvocResult, error) - // StateTransplant returns the result of executing the indicated message, assuming it was executed in the indicated tipset. - StateTransplant(context.Context, types.TipSetKey, cid.Cid) (*InvocResult, error) - // StateReplay searches for where the given message was executed, and replays it in that tipset. - StateReplay(context.Context, cid.Cid) (*InvocResult, error) + // StateReplay replays a given message, assuming it was included in a block in the specified tipset. + // If no tipset key is provided, the appropriate tipset is looked up. + StateReplay(context.Context, types.TipSetKey, cid.Cid) (*InvocResult, error) // StateGetActor returns the indicated actor's nonce and balance. StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) // StateReadState returns the indicated actor's state. diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 0279e8c8a..6effcf727 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -188,8 +188,7 @@ type FullNodeStruct struct { StateSectorExpiration func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorExpiration, error) `perm:"read"` StateSectorPartition func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorLocation, error) `perm:"read"` StateCall func(context.Context, *types.Message, types.TipSetKey) (*api.InvocResult, error) `perm:"read"` - StateTransplant func(context.Context, types.TipSetKey, cid.Cid) (*api.InvocResult, error) `perm:"read"` - StateReplay func(context.Context, cid.Cid) (*api.InvocResult, error) `perm:"read"` + StateReplay func(context.Context, types.TipSetKey, cid.Cid) (*api.InvocResult, error) `perm:"read"` StateGetActor func(context.Context, address.Address, types.TipSetKey) (*types.Actor, error) `perm:"read"` StateReadState func(context.Context, address.Address, types.TipSetKey) (*api.ActorState, error) `perm:"read"` StateWaitMsg func(ctx context.Context, cid cid.Cid, confidence uint64) (*api.MsgLookup, error) `perm:"read"` @@ -880,12 +879,8 @@ func (c *FullNodeStruct) StateCall(ctx context.Context, msg *types.Message, tsk return c.Internal.StateCall(ctx, msg, tsk) } -func (c *FullNodeStruct) StateTransplant(ctx context.Context, tsk types.TipSetKey, mc cid.Cid) (*api.InvocResult, error) { - return c.Internal.StateTransplant(ctx, tsk, mc) -} - -func (c *FullNodeStruct) StateReplay(ctx context.Context, mc cid.Cid) (*api.InvocResult, error) { - return c.Internal.StateReplay(ctx, mc) +func (c *FullNodeStruct) StateReplay(ctx context.Context, tsk types.TipSetKey, mc cid.Cid) (*api.InvocResult, error) { + return c.Internal.StateReplay(ctx, tsk, mc) } func (c *FullNodeStruct) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) { diff --git a/cli/state.go b/cli/state.go index 38454ec27..0fb3ecde6 100644 --- a/cli/state.go +++ b/cli/state.go @@ -60,7 +60,6 @@ var stateCmd = &cli.Command{ stateSectorCmd, stateGetActorCmd, stateLookupIDCmd, - stateTransplantCmd, stateReplayCmd, stateSectorSizeCmd, stateReadStateCmd, @@ -384,108 +383,6 @@ var stateExecTraceCmd = &cli.Command{ }, } -var stateTransplantCmd = &cli.Command{ - Name: "transplant", - Usage: "Play a particular message within a tipset", - ArgsUsage: "[tipsetKey messageCid]", - Flags: []cli.Flag{ - &cli.BoolFlag{ - Name: "show-trace", - Usage: "print out full execution trace for given message", - }, - &cli.BoolFlag{ - Name: "detailed-gas", - Usage: "print out detailed gas costs for given message", - }, - }, - Action: func(cctx *cli.Context) error { - if cctx.Args().Len() < 1 { - fmt.Println("usage: [tipset] ") - fmt.Println("The last cid passed will be used as the message CID") - fmt.Println("All preceding ones will be used as the tipset") - return nil - } - - args := cctx.Args().Slice() - mcid, err := cid.Decode(args[len(args)-1]) - if err != nil { - return fmt.Errorf("message cid was invalid: %s", err) - } - - fapi, closer, err := GetFullNodeAPI(cctx) - if err != nil { - return err - } - defer closer() - - ctx := ReqContext(cctx) - - var ts *types.TipSet - { - var tscids []cid.Cid - for _, s := range args[:len(args)-1] { - c, err := cid.Decode(s) - if err != nil { - return fmt.Errorf("tipset cid was invalid: %s", err) - } - tscids = append(tscids, c) - } - - if len(tscids) > 0 { - var headers []*types.BlockHeader - for _, c := range tscids { - h, err := fapi.ChainGetBlock(ctx, c) - if err != nil { - return err - } - - headers = append(headers, h) - } - - ts, err = types.NewTipSet(headers) - if err != nil { - return err - } - } else { - ts, err = fapi.ChainHead(ctx) - if err != nil { - return err - } - } - } - - res, err := fapi.StateTransplant(ctx, ts.Key(), mcid) - if err != nil { - return xerrors.Errorf("transplant call failed: %w", err) - } - - fmt.Println("Transplant receipt:") - fmt.Printf("Exit code: %d\n", res.MsgRct.ExitCode) - fmt.Printf("Return: %x\n", res.MsgRct.Return) - fmt.Printf("Gas Used: %d\n", res.MsgRct.GasUsed) - - if cctx.Bool("detailed-gas") { - fmt.Printf("Base Fee Burn: %d\n", res.GasCost.BaseFeeBurn) - fmt.Printf("Overestimaton Burn: %d\n", res.GasCost.OverEstimationBurn) - fmt.Printf("Miner Penalty: %d\n", res.GasCost.MinerPenalty) - fmt.Printf("Miner Tip: %d\n", res.GasCost.MinerTip) - fmt.Printf("Refund: %d\n", res.GasCost.Refund) - } - fmt.Printf("Total Message Cost: %d\n", res.GasCost.TotalCost) - - if res.MsgRct.ExitCode != 0 { - fmt.Printf("Error message: %q\n", res.Error) - } - - if cctx.Bool("show-trace") { - fmt.Printf("%s\t%s\t%s\t%d\t%x\t%d\t%x\n", res.Msg.From, res.Msg.To, res.Msg.Value, res.Msg.Method, res.Msg.Params, res.MsgRct.ExitCode, res.MsgRct.Return) - printInternalExecutions("\t", res.ExecutionTrace.Subcalls) - } - - return nil - }, -} - var stateReplayCmd = &cli.Command{ Name: "replay", Usage: "Replay a particular message", @@ -519,7 +416,7 @@ var stateReplayCmd = &cli.Command{ ctx := ReqContext(cctx) - res, err := fapi.StateReplay(ctx, mcid) + res, err := fapi.StateReplay(ctx, types.EmptyTSK, mcid) if err != nil { return xerrors.Errorf("replay call failed: %w", err) } diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index 9be12722d..9c2a82cad 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -165,7 +165,6 @@ * [StateSectorGetInfo](#StateSectorGetInfo) * [StateSectorPartition](#StateSectorPartition) * [StateSectorPreCommitInfo](#StateSectorPreCommitInfo) - * [StateTransplant](#StateTransplant) * [StateVMCirculatingSupplyInternal](#StateVMCirculatingSupplyInternal) * [StateVerifiedClientStatus](#StateVerifiedClientStatus) * [StateVerifiedRegistryRootKey](#StateVerifiedRegistryRootKey) @@ -4048,7 +4047,8 @@ Response: ``` ### StateReplay -StateReplay searches for where the given message was executed, and replays it in that tipset. +StateReplay replays a given message, assuming it was included in a block in the specified tipset. +If no tipset key is provided, the appropriate tipset is looked up. Perms: read @@ -4056,6 +4056,14 @@ Perms: read Inputs: ```json [ + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ], { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" } @@ -4319,98 +4327,6 @@ Response: } ``` -### StateTransplant -StateTransplant returns the result of executing the indicated message, assuming it was executed in the indicated tipset. - - -Perms: read - -Inputs: -```json -[ - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ], - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } -] -``` - -Response: -```json -{ - "MsgCid": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "Msg": { - "Version": 42, - "To": "f01234", - "From": "f01234", - "Nonce": 42, - "Value": "0", - "GasLimit": 9, - "GasFeeCap": "0", - "GasPremium": "0", - "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==", - "CID": { - "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" - } - }, - "MsgRct": { - "ExitCode": 0, - "Return": "Ynl0ZSBhcnJheQ==", - "GasUsed": 9 - }, - "GasCost": { - "Message": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "GasUsed": "0", - "BaseFeeBurn": "0", - "OverEstimationBurn": "0", - "MinerPenalty": "0", - "MinerTip": "0", - "Refund": "0", - "TotalCost": "0" - }, - "ExecutionTrace": { - "Msg": { - "Version": 42, - "To": "f01234", - "From": "f01234", - "Nonce": 42, - "Value": "0", - "GasLimit": 9, - "GasFeeCap": "0", - "GasPremium": "0", - "Method": 1, - "Params": "Ynl0ZSBhcnJheQ==", - "CID": { - "/": "bafy2bzacebbpdegvr3i4cosewthysg5xkxpqfn2wfcz6mv2hmoktwbdxkax4s" - } - }, - "MsgRct": { - "ExitCode": 0, - "Return": "Ynl0ZSBhcnJheQ==", - "GasUsed": 9 - }, - "Error": "string value", - "Duration": 60000000000, - "GasCharges": null, - "Subcalls": null - }, - "Error": "string value", - "Duration": 60000000000 -} -``` - ### StateVMCirculatingSupplyInternal StateVMCirculatingSupplyInternal returns an approximation of the circulating supply of Filecoin at the given tipset. This is the value reported by the runtime interface to actors code. diff --git a/lotuspond/front/src/Block.js b/lotuspond/front/src/Block.js index cb8d25c9d..4bae57c81 100644 --- a/lotuspond/front/src/Block.js +++ b/lotuspond/front/src/Block.js @@ -26,7 +26,7 @@ class Block extends React.Component { messages = await Promise.all(messages.map(async (msg, i) => { if (msg.receipt.ExitCode !== 0) { - let reply = await this.props.conn.call('Filecoin.StateTransplant', [{Cids: [this.props.cid], Blocks: [header], Height: header.Height}, msg.Cid]) + let reply = await this.props.conn.call('Filecoin.StateReplay', [{Cids: [this.props.cid], Blocks: [header], Height: header.Height}, msg.Cid]) if(!reply.Error) { reply.Error = "reply: no error" } diff --git a/node/impl/full/state.go b/node/impl/full/state.go index a63ac8c67..221b83e3a 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -346,12 +346,34 @@ func (a *StateAPI) StateCall(ctx context.Context, msg *types.Message, tsk types. return res, err } -func (a *StateAPI) StateTransplant(ctx context.Context, tsk types.TipSetKey, mc cid.Cid) (*api.InvocResult, error) { - ts, err := a.Chain.GetTipSetFromKey(tsk) - if err != nil { - return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) +func (a *StateAPI) StateReplay(ctx context.Context, tsk types.TipSetKey, mc cid.Cid) (*api.InvocResult, error) { + msgToReplay := mc + var ts *types.TipSet + if tsk == types.EmptyTSK { + mlkp, err := a.StateSearchMsg(ctx, mc) + if err != nil { + return nil, xerrors.Errorf("searching for msg %s: %w", mc, err) + } + if mlkp == nil { + return nil, xerrors.Errorf("didn't find msg %s", mc) + } + + msgToReplay = mlkp.Message + + executionTs, err := a.Chain.GetTipSetFromKey(mlkp.TipSet) + if err != nil { + return nil, xerrors.Errorf("loading tipset %s: %w", mlkp.TipSet, err) + } + + ts, err = a.Chain.LoadTipSet(executionTs.Parents()) + if err != nil { + return nil, xerrors.Errorf("loading parent tipset %s: %w", mlkp.TipSet, err) + } + } else { + ts = a.Chain.GetHeaviestTipSet() } - m, r, err := a.StateManager.Replay(ctx, ts, mc) + + m, r, err := a.StateManager.Replay(ctx, ts, msgToReplay) if err != nil { return nil, err } @@ -362,47 +384,7 @@ func (a *StateAPI) StateTransplant(ctx context.Context, tsk types.TipSetKey, mc } return &api.InvocResult{ - MsgCid: mc, - Msg: m, - MsgRct: &r.MessageReceipt, - GasCost: stmgr.MakeMsgGasCost(m, r), - ExecutionTrace: r.ExecutionTrace, - Error: errstr, - Duration: r.Duration, - }, nil -} - -func (a *StateAPI) StateReplay(ctx context.Context, mc cid.Cid) (*api.InvocResult, error) { - mlkp, err := a.StateSearchMsg(ctx, mc) - if err != nil { - return nil, xerrors.Errorf("searching for msg %s: %w", mc, err) - } - if mlkp == nil { - return nil, xerrors.Errorf("didn't find msg %s", mc) - } - - executionTs, err := a.Chain.GetTipSetFromKey(mlkp.TipSet) - if err != nil { - return nil, xerrors.Errorf("loading tipset %s: %w", mlkp.TipSet, err) - } - - ts, err := a.Chain.LoadTipSet(executionTs.Parents()) - if err != nil { - return nil, xerrors.Errorf("loading parent tipset %s: %w", mlkp.TipSet, err) - } - - m, r, err := a.StateManager.Replay(ctx, ts, mlkp.Message) - if err != nil { - return nil, err - } - - var errstr string - if r.ActorErr != nil { - errstr = r.ActorErr.Error() - } - - return &api.InvocResult{ - MsgCid: mlkp.Message, + MsgCid: msgToReplay, Msg: m, MsgRct: &r.MessageReceipt, GasCost: stmgr.MakeMsgGasCost(m, r), From d0e4150ceaf036f4af367261b8b983a3e0c51e68 Mon Sep 17 00:00:00 2001 From: Ingar Shu Date: Fri, 16 Oct 2020 08:30:55 -0700 Subject: [PATCH 797/885] Use separate config for Retrieval Filter --- node/builder.go | 5 ++++- node/config/def.go | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/node/builder.go b/node/builder.go index b6b75ecc6..1dc12e80a 100644 --- a/node/builder.go +++ b/node/builder.go @@ -486,7 +486,10 @@ func ConfigStorageMiner(c interface{}) Option { If(cfg.Dealmaking.Filter != "", Override(new(dtypes.StorageDealFilter), modules.BasicDealFilter(dealfilter.CliStorageDealFilter(cfg.Dealmaking.Filter))), - Override(new(dtypes.RetrievalDealFilter), modules.RetrievalDealFilter(dealfilter.CliRetrievalDealFilter(cfg.Dealmaking.Filter))), + ), + + If(cfg.Dealmaking.RetrievalFilter != "", + Override(new(dtypes.RetrievalDealFilter), modules.RetrievalDealFilter(dealfilter.CliRetrievalDealFilter(cfg.Dealmaking.RetrievalFilter))), ), Override(new(storagemarket.StorageProviderNode), storageadapter.NewProviderNodeAdapter(&cfg.Fees)), diff --git a/node/config/def.go b/node/config/def.go index 7eb7e19f0..a1b57286c 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -45,7 +45,8 @@ type DealmakingConfig struct { PieceCidBlocklist []cid.Cid ExpectedSealDuration Duration - Filter string + Filter string + RetrievalFilter string } type SealingConfig struct { From 4de8f506424ad9664b15459f0e96a834e4727e69 Mon Sep 17 00:00:00 2001 From: Travis Person Date: Fri, 16 Oct 2020 19:13:33 +0000 Subject: [PATCH 798/885] replace bootstrap peers --- build/bootstrap/bootstrappers.pi | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/build/bootstrap/bootstrappers.pi b/build/bootstrap/bootstrappers.pi index f3f8b8a55..886ac8e99 100644 --- a/build/bootstrap/bootstrappers.pi +++ b/build/bootstrap/bootstrappers.pi @@ -1,9 +1,12 @@ -/dns4/bootstrap-0.testnet.fildev.network/tcp/1347/p2p/12D3KooWJTUBUjtzWJGWU1XSiY21CwmHaCNLNYn2E7jqHEHyZaP7 -/dns4/bootstrap-1.testnet.fildev.network/tcp/1347/p2p/12D3KooW9yeKXha4hdrJKq74zEo99T8DhriQdWNoojWnnQbsgB3v -/dns4/bootstrap-2.testnet.fildev.network/tcp/1347/p2p/12D3KooWCrx8yVG9U9Kf7w8KLN3Edkj5ZKDhgCaeMqQbcQUoB6CT -/dns4/bootstrap-4.testnet.fildev.network/tcp/1347/p2p/12D3KooWPkL9LrKRQgHtq7kn9ecNhGU9QaziG8R5tX8v9v7t3h34 -/dns4/bootstrap-3.testnet.fildev.network/tcp/1347/p2p/12D3KooWKYSsbpgZ3HAjax5M1BXCwXLa6gVkUARciz7uN3FNtr7T -/dns4/bootstrap-5.testnet.fildev.network/tcp/1347/p2p/12D3KooWQYzqnLASJAabyMpPb1GcWZvNSe7JDcRuhdRqonFoiK9W +/dns4/bootstrap-0.mainnet.filops.net/tcp/1347/p2p/12D3KooWCVe8MmsEMes2FzgTpt9fXtmCY7wrq91GRiaC8PHSCCBj +/dns4/bootstrap-1.mainnet.filops.net/tcp/1347/p2p/12D3KooWCwevHg1yLCvktf2nvLu7L9894mcrJR4MsBCcm4syShVc +/dns4/bootstrap-2.mainnet.filops.net/tcp/1347/p2p/12D3KooWEWVwHGn2yR36gKLozmb4YjDJGerotAPGxmdWZx2nxMC4 +/dns4/bootstrap-3.mainnet.filops.net/tcp/1347/p2p/12D3KooWKhgq8c7NQ9iGjbyK7v7phXvG6492HQfiDaGHLHLQjk7R +/dns4/bootstrap-4.mainnet.filops.net/tcp/1347/p2p/12D3KooWL6PsFNPhYftrJzGgF5U18hFoaVhfGk7xwzD8yVrHJ3Uc +/dns4/bootstrap-5.mainnet.filops.net/tcp/1347/p2p/12D3KooWLFynvDQiUpXoHroV1YxKHhPJgysQGH2k3ZGwtWzR4dFH +/dns4/bootstrap-6.mainnet.filops.net/tcp/1347/p2p/12D3KooWP5MwCiqdMETF9ub1P3MbCvQCcfconnYHbWg6sUJcDRQQ +/dns4/bootstrap-7.mainnet.filops.net/tcp/1347/p2p/12D3KooWRs3aY1p3juFjPy8gPN95PEQChm2QKGUCAdcDCC4EBMKf +/dns4/bootstrap-8.mainnet.filops.net/tcp/1347/p2p/12D3KooWScFR7385LTyR4zU1bYdzSiiAb5rnNABfVahPvVSzyTkR /dns4/lotus-bootstrap.forceup.cn/tcp/41778/p2p/12D3KooWFQsv3nRMUevZNWWsY1Wu6NUzUbawnWU5NcRhgKuJA37C /dns4/bootstrap-0.starpool.in/tcp/12757/p2p/12D3KooWGHpBMeZbestVEWkfdnC9u7p6uFHXL1n7m1ZBqsEmiUzz /dns4/bootstrap-1.starpool.in/tcp/12757/p2p/12D3KooWQZrGH1PxSNZPum99M1zNvjNFM33d1AAu5DcvdHptuU7u From 70e2170e3e225d79b64f8942eae5834848931ffa Mon Sep 17 00:00:00 2001 From: austinabell Date: Fri, 16 Oct 2020 21:14:36 -0400 Subject: [PATCH 799/885] Fix circ supply default in conformance --- conformance/driver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conformance/driver.go b/conformance/driver.go index fbb7dda6d..95b6f2659 100644 --- a/conformance/driver.go +++ b/conformance/driver.go @@ -248,7 +248,7 @@ func BaseFeeOrDefault(basefee *gobig.Int) abi.TokenAmount { // DefaultCirculatingSupply. func CircSupplyOrDefault(circSupply *gobig.Int) abi.TokenAmount { if circSupply == nil { - return DefaultBaseFee + return DefaultCirculatingSupply } return big.NewFromGo(circSupply) } From 869d71446d7fe8d5bb31ae587070ed76155db33c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 19 Oct 2020 16:08:33 +0200 Subject: [PATCH 800/885] miner: fix init --create-worker-key --- cmd/lotus-storage-miner/init.go | 35 ++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index 326a39e5e..baf71a2c7 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -601,8 +601,6 @@ func configureStorageMiner(ctx context.Context, api lapi.FullNode, addr address. } func createStorageMiner(ctx context.Context, api lapi.FullNode, peerid peer.ID, gasPrice types.BigInt, cctx *cli.Context) (address.Address, error) { - log.Info("Creating StorageMarket.CreateStorageMiner message") - var err error var owner address.Address if cctx.String("owner") != "" { @@ -625,9 +623,32 @@ func createStorageMiner(ctx context.Context, api lapi.FullNode, peerid peer.ID, } else if cctx.Bool("create-worker-key") { // TODO: Do we need to force this if owner is Secpk? worker, err = api.WalletNew(ctx, types.KTBLS) } - // TODO: Transfer some initial funds to worker if err != nil { - return address.Undef, err + return address.Address{}, err + } + + // make sure the worker account exists on chain + _, err = api.StateLookupID(ctx, worker, types.EmptyTSK) + if err != nil { + signed, err := api.MpoolPushMessage(ctx, &types.Message{ + From: owner, + To: worker, + Value: types.NewInt(0), + }, nil) + if err != nil { + return address.Undef, xerrors.Errorf("push worker init: %w", err) + } + + log.Infof("Initializing worker account %s, message: %s", worker, signed.Cid()) + log.Infof("Waiting for confirmation") + + mw, err := api.StateWaitMsg(ctx, signed.Cid(), build.MessageConfidence) + if err != nil { + return address.Undef, xerrors.Errorf("waiting for worker init: %w", err) + } + if mw.Receipt.ExitCode != 0 { + return address.Undef, xerrors.Errorf("initializing worker account failed: exit code %d", mw.Receipt.ExitCode) + } } spt, err := ffiwrapper.SealProofTypeFromSectorSize(abi.SectorSize(ssize)) @@ -668,15 +689,15 @@ func createStorageMiner(ctx context.Context, api lapi.FullNode, peerid peer.ID, signed, err := api.MpoolPushMessage(ctx, createStorageMinerMsg, nil) if err != nil { - return address.Undef, err + return address.Undef, xerrors.Errorf("pushing createMiner message: %w", err) } - log.Infof("Pushed StorageMarket.CreateStorageMiner, %s to Mpool", signed.Cid()) + log.Infof("Pushed CreateMiner message: %s", signed.Cid()) log.Infof("Waiting for confirmation") mw, err := api.StateWaitMsg(ctx, signed.Cid(), build.MessageConfidence) if err != nil { - return address.Undef, err + return address.Undef, xerrors.Errorf("waiting for createMiner message: %w", err) } if mw.Receipt.ExitCode != 0 { From d24d3e420d66d4ef2a3f0cd530bbd6ffa21511a1 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Mon, 19 Oct 2020 14:27:04 -0400 Subject: [PATCH 801/885] Fix StateReplay to use provided tipset --- node/impl/full/state.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/node/impl/full/state.go b/node/impl/full/state.go index b2e9c3e51..a6564296b 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -349,6 +349,7 @@ func (a *StateAPI) StateCall(ctx context.Context, msg *types.Message, tsk types. func (a *StateAPI) StateReplay(ctx context.Context, tsk types.TipSetKey, mc cid.Cid) (*api.InvocResult, error) { msgToReplay := mc var ts *types.TipSet + var err error if tsk == types.EmptyTSK { mlkp, err := a.StateSearchMsg(ctx, mc) if err != nil { @@ -370,7 +371,10 @@ func (a *StateAPI) StateReplay(ctx context.Context, tsk types.TipSetKey, mc cid. return nil, xerrors.Errorf("loading parent tipset %s: %w", mlkp.TipSet, err) } } else { - ts = a.Chain.GetHeaviestTipSet() + ts, err = a.Chain.LoadTipSet(tsk) + if err != nil { + return nil, xerrors.Errorf("loading specified tipset %s: %w", tsk, err) + } } m, r, err := a.StateManager.Replay(ctx, ts, msgToReplay) From 1974490ab05fd005619ea0f1b8eac9802d31ed53 Mon Sep 17 00:00:00 2001 From: Ingar Shu Date: Mon, 19 Oct 2020 13:12:28 -0700 Subject: [PATCH 802/885] Don't overwrite previously-configured maxPieceSize for a persisted ask --- node/modules/storageminer.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index bcc38898e..7f510aa70 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -398,8 +398,6 @@ func NewStorageAsk(ctx helpers.MetricsCtx, fapi lapi.FullNode, ds dtypes.Metadat if err != nil { return nil, err } - a := storedAsk.GetAsk().Ask - err = storedAsk.SetAsk(a.Price, a.VerifiedPrice, a.Expiry-a.Timestamp) if err != nil { return storedAsk, err } From be42dd824b696a1195290348359577753ac7fb15 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Mon, 19 Oct 2020 22:32:05 +0200 Subject: [PATCH 803/885] state: optimize state snapshot address cache Signed-off-by: Jakub Sztandera --- chain/state/statetree.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/chain/state/statetree.go b/chain/state/statetree.go index e9b76ea77..35760fdf9 100644 --- a/chain/state/statetree.go +++ b/chain/state/statetree.go @@ -35,7 +35,8 @@ type StateTree struct { } type stateSnaps struct { - layers []*stateSnapLayer + layers []*stateSnapLayer + lastMaybeNonEmptyResolveCache int } type stateSnapLayer struct { @@ -67,7 +68,12 @@ func (ss *stateSnaps) addLayer() { func (ss *stateSnaps) dropLayer() { ss.layers[len(ss.layers)-1] = nil // allow it to be GCed + ss.layers = ss.layers[:len(ss.layers)-1] + + if ss.lastMaybeNonEmptyResolveCache == len(ss.layers) { + ss.lastMaybeNonEmptyResolveCache = len(ss.layers) - 1 + } } func (ss *stateSnaps) mergeLastLayer() { @@ -86,7 +92,13 @@ func (ss *stateSnaps) mergeLastLayer() { } func (ss *stateSnaps) resolveAddress(addr address.Address) (address.Address, bool) { - for i := len(ss.layers) - 1; i >= 0; i-- { + for i := ss.lastMaybeNonEmptyResolveCache; i >= 0; i-- { + if len(ss.layers[i].resolveCache) == 0 { + if ss.lastMaybeNonEmptyResolveCache == i { + ss.lastMaybeNonEmptyResolveCache = i - 1 + } + continue + } resa, ok := ss.layers[i].resolveCache[addr] if ok { return resa, true @@ -97,6 +109,7 @@ func (ss *stateSnaps) resolveAddress(addr address.Address) (address.Address, boo func (ss *stateSnaps) cacheResolveAddress(addr, resa address.Address) { ss.layers[len(ss.layers)-1].resolveCache[addr] = resa + ss.lastMaybeNonEmptyResolveCache = len(ss.layers) - 1 } func (ss *stateSnaps) getActor(addr address.Address) (*types.Actor, error) { From cb0513f4c1a978f18391dc6a28e0f4bb9051bb67 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Tue, 20 Oct 2020 01:25:41 +0200 Subject: [PATCH 804/885] Add test Signed-off-by: Jakub Sztandera --- chain/state/statetree.go | 50 +++++++++++------- chain/state/statetree_test.go | 98 +++++++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+), 19 deletions(-) diff --git a/chain/state/statetree.go b/chain/state/statetree.go index 35760fdf9..7fa55b31c 100644 --- a/chain/state/statetree.go +++ b/chain/state/statetree.go @@ -26,10 +26,11 @@ var log = logging.Logger("statetree") // StateTree stores actors state by their ID. type StateTree struct { - root adt.Map - version types.StateTreeVersion - info cid.Cid - Store cbor.IpldStore + root adt.Map + version types.StateTreeVersion + info cid.Cid + Store cbor.IpldStore + lookupIDFun func(address.Address) (address.Address, error) snaps *stateSnaps } @@ -173,13 +174,15 @@ func NewStateTree(cst cbor.IpldStore, ver types.StateTreeVersion) (*StateTree, e return nil, err } - return &StateTree{ + s := &StateTree{ root: root, info: info, version: ver, Store: cst, snaps: newStateSnaps(), - }, nil + } + s.lookupIDFun = s.lookupIDinternal + return s, nil } func LoadStateTree(cst cbor.IpldStore, c cid.Cid) (*StateTree, error) { @@ -203,13 +206,15 @@ func LoadStateTree(cst cbor.IpldStore, c cid.Cid) (*StateTree, error) { return nil, err } - return &StateTree{ + s := &StateTree{ root: nd, info: root.Info, version: root.Version, Store: cst, snaps: newStateSnaps(), - }, nil + } + s.lookupIDFun = s.lookupIDinternal + return s, nil default: return nil, xerrors.Errorf("unsupported state tree version: %d", root.Version) } @@ -226,17 +231,7 @@ func (st *StateTree) SetActor(addr address.Address, act *types.Actor) error { return nil } -// LookupID gets the ID address of this actor's `addr` stored in the `InitActor`. -func (st *StateTree) LookupID(addr address.Address) (address.Address, error) { - if addr.Protocol() == address.ID { - return addr, nil - } - - resa, ok := st.snaps.resolveAddress(addr) - if ok { - return resa, nil - } - +func (st *StateTree) lookupIDinternal(addr address.Address) (address.Address, error) { act, err := st.GetActor(init_.Address) if err != nil { return address.Undef, xerrors.Errorf("getting init actor: %w", err) @@ -254,6 +249,23 @@ func (st *StateTree) LookupID(addr address.Address) (address.Address, error) { if err != nil { return address.Undef, xerrors.Errorf("resolve address %s: %w", addr, err) } + return a, err +} + +// LookupID gets the ID address of this actor's `addr` stored in the `InitActor`. +func (st *StateTree) LookupID(addr address.Address) (address.Address, error) { + if addr.Protocol() == address.ID { + return addr, nil + } + + resa, ok := st.snaps.resolveAddress(addr) + if ok { + return resa, nil + } + a, err := st.lookupIDFun(addr) + if err != nil { + return a, err + } st.snaps.cacheResolveAddress(addr, a) diff --git a/chain/state/statetree_test.go b/chain/state/statetree_test.go index ed1fb1889..584d9ba7e 100644 --- a/chain/state/statetree_test.go +++ b/chain/state/statetree_test.go @@ -73,6 +73,104 @@ func BenchmarkStateTreeSetFlush(b *testing.B) { } } +func TestResolveCache(t *testing.T) { + cst := cbor.NewMemCborStore() + st, err := NewStateTree(cst, VersionForNetwork(build.NewestNetworkVersion)) + if err != nil { + t.Fatal(err) + } + nonId := address.NewForTestGetter()() + id, _ := address.NewIDAddress(1000) + + st.lookupIDFun = func(a address.Address) (address.Address, error) { + if a == nonId { + return id, nil + } else { + return address.Undef, types.ErrActorNotFound + } + } + + err = st.SetActor(nonId, &types.Actor{Nonce: 1}) + if err != nil { + t.Fatal(err) + } + + { + err = st.Snapshot(context.TODO()) + if err != nil { + t.Fatal(err) + } + act, err := st.GetActor(nonId) + if err != nil { + t.Fatal(err) + } + if act.Nonce != 1 { + t.Fatalf("expected nonce 1, got %d", act.Nonce) + } + err = st.SetActor(nonId, &types.Actor{Nonce: 2}) + if err != nil { + t.Fatal(err) + } + + act, err = st.GetActor(nonId) + if err != nil { + t.Fatal(err) + } + if act.Nonce != 2 { + t.Fatalf("expected nonce 2, got %d", act.Nonce) + } + + if err := st.Revert(); err != nil { + t.Fatal(err) + } + st.ClearSnapshot() + } + + act, err := st.GetActor(nonId) + if err != nil { + t.Fatal(err) + } + if act.Nonce != 1 { + t.Fatalf("expected nonce 1, got %d", act.Nonce) + } + + { + err = st.Snapshot(context.TODO()) + if err != nil { + t.Fatal(err) + } + act, err := st.GetActor(nonId) + if err != nil { + t.Fatal(err) + } + if act.Nonce != 1 { + t.Fatalf("expected nonce 1, got %d", act.Nonce) + } + err = st.SetActor(nonId, &types.Actor{Nonce: 2}) + if err != nil { + t.Fatal(err) + } + + act, err = st.GetActor(nonId) + if err != nil { + t.Fatal(err) + } + if act.Nonce != 2 { + t.Fatalf("expected nonce 2, got %d", act.Nonce) + } + st.ClearSnapshot() + } + + act, err = st.GetActor(nonId) + if err != nil { + t.Fatal(err) + } + if act.Nonce != 2 { + t.Fatalf("expected nonce 2, got %d", act.Nonce) + } + +} + func BenchmarkStateTree10kGetActor(b *testing.B) { cst := cbor.NewMemCborStore() st, err := NewStateTree(cst, VersionForNetwork(build.NewestNetworkVersion)) From a055540c254d13648031d5f873acb653d71c06f5 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Tue, 20 Oct 2020 01:32:51 +0200 Subject: [PATCH 805/885] Fix lint Signed-off-by: Jakub Sztandera --- chain/state/statetree_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/chain/state/statetree_test.go b/chain/state/statetree_test.go index 584d9ba7e..61e7d8c22 100644 --- a/chain/state/statetree_test.go +++ b/chain/state/statetree_test.go @@ -85,9 +85,8 @@ func TestResolveCache(t *testing.T) { st.lookupIDFun = func(a address.Address) (address.Address, error) { if a == nonId { return id, nil - } else { - return address.Undef, types.ErrActorNotFound } + return address.Undef, types.ErrActorNotFound } err = st.SetActor(nonId, &types.Actor{Nonce: 1}) From 1ee9fb44e237da3490115eba8aea97a7a03f4eb7 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Sat, 17 Oct 2020 19:06:12 -0400 Subject: [PATCH 806/885] Lotus version 1.0.0 --- CHANGELOG.md | 31 +++++++++++++++++++++++++++++++ build/version.go | 2 +- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 701f221b0..76aa51de2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,36 @@ # Lotus changelog +# 1.0.0 / 2020-10-19 + +It's 1.0.0! This is an optional release of Lotus that introduces some UX improvements to the 0.10 series. + +This very small release is largely cosmetic, and intended to flag the code that the Filecoin mainnet was launched with. + +## API changes + +- `StateMsgGasCost` has been removed. The equivalent information can be gained by calling `StateReplay`. +- A `GasCost` field has been added to the `InvocResult` type, meaning detailed gas costs will be returned when calling `StateReplay`, `StateCompute`, and `StateCall`. +- The behaviour of `StateReplay` in response to an empty tipset key has been changed. Instead of simply using the heaviest tipset (which is almost guaranteed to be an unsuccessful replay), we search now search the chain for the tipset that included the message, and replay the message in that tipset (we fail if no such tipset is found). + +## Changes + +- Increase code coverage! (https://github.com/filecoin-project/lotus/pull/4410) +- Mpool: Don't block node startup loading messages (https://github.com/filecoin-project/lotus/pull/4411) +- Improve the UX of multisig approves (https://github.com/filecoin-project/lotus/pull/4398) +- Use build.BlockDelaySecs for deal start buffer (https://github.com/filecoin-project/lotus/pull/4415) +- Conformance: support multiple protocol versions (https://github.com/filecoin-project/lotus/pull/4393) +- Ensure msig inspect cli works with lotus-lite (https://github.com/filecoin-project/lotus/pull/4421) +- Add command to (slowly) prune lotus chain datastore (https://github.com/filecoin-project/lotus/pull/3876) +- Add WalletVerify to lotus-gateway (https://github.com/filecoin-project/lotus/pull/4373) +- Improve StateMsg APIs (https://github.com/filecoin-project/lotus/pull/4429) +- Add endpoints needed by spacegap (https://github.com/filecoin-project/lotus/pull/4426) +- Make audit balances capable of printing robust addresses (https://github.com/filecoin-project/lotus/pull/4423) +- Custom filters for retrieval deals (https://github.com/filecoin-project/lotus/pull/4424) +- Fix message list api (https://github.com/filecoin-project/lotus/pull/4422) +- Replace bootstrap peers (https://github.com/filecoin-project/lotus/pull/4447) +- Don't overwrite previously-configured maxPieceSize for a persisted ask (https://github.com/filecoin-project/lotus/pull/4480) +- State: optimize state snapshot address cache (https://github.com/filecoin-project/lotus/pull/4481) + # 0.10.2 / 2020-10-14 This is an optional release of Lotus that updates markets to 0.9.1, which fixes an issue affecting deals that were mid-transfer when the node was upgraded to 0.9.0. This release also includes some tweaks to default gas values and minor performance improvements. diff --git a/build/version.go b/build/version.go index df1fe28de..352fa9b0e 100644 --- a/build/version.go +++ b/build/version.go @@ -29,7 +29,7 @@ func buildType() string { } // BuildVersion is the local build version, set by build system -const BuildVersion = "0.10.2" +const BuildVersion = "1.0.0" func UserVersion() string { return BuildVersion + buildType() + CurrentCommit From 608010c35deffb4e82b1928c8b4e1c70fe2273be Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Mon, 19 Oct 2020 19:09:49 -0700 Subject: [PATCH 807/885] add some extra logging to try and debug sync issues --- chain/sync.go | 5 +++-- chain/sync_manager.go | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/chain/sync.go b/chain/sync.go index 97915d8df..2c7408113 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -268,14 +268,15 @@ func (syncer *Syncer) InformNewHead(from peer.ID, fts *store.FullTipSet) bool { syncer.Exchange.AddPeer(from) - bestPweight := syncer.store.GetHeaviestTipSet().ParentWeight() + hts := syncer.store.GetHeaviestTipSet() + bestPweight := hts.ParentWeight() targetWeight := fts.TipSet().ParentWeight() if targetWeight.LessThan(bestPweight) { var miners []string for _, blk := range fts.TipSet().Blocks() { miners = append(miners, blk.Miner.String()) } - log.Infof("incoming tipset from %s does not appear to be better than our best chain, ignoring for now", miners) + log.Infow("incoming tipset does not appear to be better than our best chain, ignoring for now", "miners", miners, "bestPweight", bestPweight, "bestTS", hts.Cids(), "incomingWeight", targetWeight, "incomingTS", fts.TipSet().Cids()) return false } diff --git a/chain/sync_manager.go b/chain/sync_manager.go index c7fdea726..17cec78d7 100644 --- a/chain/sync_manager.go +++ b/chain/sync_manager.go @@ -257,6 +257,7 @@ func (stb *syncTargetBucket) add(ts *types.TipSet) { func (stb *syncTargetBucket) heaviestTipSet() *types.TipSet { if stb == nil { + log.Warn("sync target bucket was nil when heaviestTipSet got called on it") return nil } From e7a68dba58d7583c0d3e337a8e0794d5d29773c4 Mon Sep 17 00:00:00 2001 From: jennijuju Date: Tue, 20 Oct 2020 02:28:43 -0400 Subject: [PATCH 808/885] Add a comment to BlockMessages to address #4446. --- api/api_full.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/api_full.go b/api/api_full.go index 772a0d709..57ca5bcfd 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -591,6 +591,8 @@ type MsgGasCost struct { TotalCost abi.TokenAmount } +// BlsMessages[x].cid = Cids[x] +// SecpkMessages[y].cid = Cids[BlsMessages.length + y] type BlockMessages struct { BlsMessages []*types.Message SecpkMessages []*types.SignedMessage From 09e92f98fff1651b70c00854f9b9adf09b010c0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 20 Oct 2020 12:52:52 +0200 Subject: [PATCH 809/885] circle: Run tests for some subsystems separately --- .circleci/config.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index acd447f69..f82008e9b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -188,6 +188,14 @@ jobs: command: | bash <(curl -s https://codecov.io/bash) + test-chain: + <<: *test + test-node: + <<: *test + test-storage: + <<: *test + test-cli: + <<: *test test-short: <<: *test test-window-post: @@ -428,6 +436,22 @@ workflows: - test: codecov-upload: true test-suite-name: full + - test-chain: + codecov-upload: true + test-suite-name: chain + packages: "./chain/..." + - test-node: + codecov-upload: true + test-suite-name: node + packages: "./node/..." + - test-storage: + codecov-upload: true + test-suite-name: storage + packages: "./storage/... ./extern/..." + - test-cli: + codecov-upload: true + test-suite-name: cli + packages: "./cli/... ./cmd/... ./api/..." - test-window-post: go-test-flags: "-run=TestWindowedPost" winpost-test: "1" From ee0078f7c16110c2211a823827fae1792005da1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 20 Oct 2020 13:21:42 +0200 Subject: [PATCH 810/885] make: Drop npm calls from buildall --- Makefile | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 79f7fa81e..093f62ef6 100644 --- a/Makefile +++ b/Makefile @@ -133,18 +133,30 @@ benchmarks: lotus-pond: 2k go build -o lotus-pond ./lotuspond - (cd lotuspond/front && npm i && CI=false npm run build) .PHONY: lotus-pond BINS+=lotus-pond +lotus-pond-front: + (cd lotuspond/front && npm i && CI=false npm run build) +.PHONY: lotus-pond-front + +lotus-pond-app: lotus-pond-front lotus-pond +.PHONY: lotus-pond-app + lotus-townhall: rm -f lotus-townhall go build -o lotus-townhall ./cmd/lotus-townhall - (cd ./cmd/lotus-townhall/townhall && npm i && npm run build) - go run github.com/GeertJohan/go.rice/rice append --exec lotus-townhall -i ./cmd/lotus-townhall -i ./build .PHONY: lotus-townhall BINS+=lotus-townhall +lotus-townhall-front: + (cd ./cmd/lotus-townhall/townhall && npm i && npm run build) +.PHONY: lotus-townhall-front + +lotus-townhall-app: lotus-touch lotus-townhall-front + go run github.com/GeertJohan/go.rice/rice append --exec lotus-townhall -i ./cmd/lotus-townhall -i ./build +.PHONY: lotus-townhall-app + lotus-fountain: rm -f lotus-fountain go build -o lotus-fountain ./cmd/lotus-fountain From 580980d149906644606dc54c2f10fc0ee23477ae Mon Sep 17 00:00:00 2001 From: vyzo Date: Tue, 20 Oct 2020 19:13:17 +0300 Subject: [PATCH 811/885] make push and addLocal atomic --- chain/messagepool/messagepool.go | 65 ++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 28 deletions(-) diff --git a/chain/messagepool/messagepool.go b/chain/messagepool/messagepool.go index cc26be0b5..79ab572ba 100644 --- a/chain/messagepool/messagepool.go +++ b/chain/messagepool/messagepool.go @@ -432,9 +432,14 @@ func (mp *MessagePool) runLoop() { } } -func (mp *MessagePool) addLocal(m *types.SignedMessage, msgb []byte) error { +func (mp *MessagePool) addLocal(m *types.SignedMessage) error { mp.localAddrs[m.Message.From] = struct{}{} + msgb, err := m.Serialize() + if err != nil { + return xerrors.Errorf("error serializing message: %w", err) + } + if err := mp.localMsgs.Put(datastore.NewKey(string(m.Cid().Bytes())), msgb); err != nil { return xerrors.Errorf("persisting local message: %w", err) } @@ -507,11 +512,6 @@ func (mp *MessagePool) Push(m *types.SignedMessage) (cid.Cid, error) { <-mp.addSema }() - msgb, err := m.Serialize() - if err != nil { - return cid.Undef, err - } - mp.curTsLk.Lock() publish, err := mp.addTs(m, mp.curTs, true, false) if err != nil { @@ -520,18 +520,19 @@ func (mp *MessagePool) Push(m *types.SignedMessage) (cid.Cid, error) { } mp.curTsLk.Unlock() - mp.lk.Lock() - if err := mp.addLocal(m, msgb); err != nil { - mp.lk.Unlock() - return cid.Undef, err - } - mp.lk.Unlock() - if publish { + msgb, err := m.Serialize() + if err != nil { + return cid.Undef, xerrors.Errorf("error serializing message: %w", err) + } + err = mp.api.PubSubPublish(build.MessagesTopic(mp.netName), msgb) + if err != nil { + return cid.Undef, xerrors.Errorf("error publishing message: %w", err) + } } - return m.Cid(), err + return m.Cid(), nil } func (mp *MessagePool) checkMessage(m *types.SignedMessage) error { @@ -670,7 +671,19 @@ func (mp *MessagePool) addTs(m *types.SignedMessage, curTs *types.TipSet, local, return false, err } - return publish, mp.addLocked(m, !local, untrusted) + err = mp.addLocked(m, !local, untrusted) + if err != nil { + return false, err + } + + if local { + err = mp.addLocal(m) + if err != nil { + return false, xerrors.Errorf("error persisting local message: %w", err) + } + } + + return publish, nil } func (mp *MessagePool) addLoaded(m *types.SignedMessage) error { @@ -837,11 +850,6 @@ func (mp *MessagePool) PushUntrusted(m *types.SignedMessage) (cid.Cid, error) { <-mp.addSema }() - msgb, err := m.Serialize() - if err != nil { - return cid.Undef, err - } - mp.curTsLk.Lock() publish, err := mp.addTs(m, mp.curTs, false, true) if err != nil { @@ -850,18 +858,19 @@ func (mp *MessagePool) PushUntrusted(m *types.SignedMessage) (cid.Cid, error) { } mp.curTsLk.Unlock() - mp.lk.Lock() - if err := mp.addLocal(m, msgb); err != nil { - mp.lk.Unlock() - return cid.Undef, err - } - mp.lk.Unlock() - if publish { + msgb, err := m.Serialize() + if err != nil { + return cid.Undef, xerrors.Errorf("error serializing message: %w", err) + } + err = mp.api.PubSubPublish(build.MessagesTopic(mp.netName), msgb) + if err != nil { + return cid.Undef, xerrors.Errorf("error publishing message: %w", err) + } } - return m.Cid(), err + return m.Cid(), nil } func (mp *MessagePool) Remove(from address.Address, nonce uint64, applied bool) { From 6bc56c549a16b470f2019c743393fdf818cd2e47 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 20 Oct 2020 10:04:51 -0700 Subject: [PATCH 812/885] add some methods that oni needs --- cmd/lotus-gateway/api.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/cmd/lotus-gateway/api.go b/cmd/lotus-gateway/api.go index 105f8881e..a0febef2c 100644 --- a/cmd/lotus-gateway/api.go +++ b/cmd/lotus-gateway/api.go @@ -10,6 +10,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/go-state-types/dline" + "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" @@ -33,6 +34,8 @@ var ( // gatewayDepsAPI defines the API methods that the GatewayAPI depends on // (to make it easy to mock for tests) type gatewayDepsAPI interface { + Version(context.Context) (api.Version, error) + ChainHasObj(context.Context, cid.Cid) (bool, error) ChainHead(ctx context.Context) (*types.TipSet, error) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) @@ -57,6 +60,7 @@ type gatewayDepsAPI interface { StateMinerProvingDeadline(context.Context, address.Address, types.TipSetKey) (*dline.Info, error) StateCirculatingSupply(context.Context, types.TipSetKey) (abi.TokenAmount, error) StateVMCirculatingSupplyInternal(context.Context, types.TipSetKey) (api.CirculatingSupply, error) + StateNetworkVersion(context.Context, types.TipSetKey) (network.Version, error) } type GatewayAPI struct { @@ -114,6 +118,10 @@ func (a *GatewayAPI) checkTimestamp(at time.Time) error { return nil } +func (a *GatewayAPI) Version(ctx context.Context) (api.Version, error) { + return a.api.Version(ctx) +} + func (a *GatewayAPI) ChainHasObj(ctx context.Context, c cid.Cid) (bool, error) { return a.api.ChainHasObj(ctx, c) } @@ -295,6 +303,13 @@ func (a *GatewayAPI) StateVMCirculatingSupplyInternal(ctx context.Context, tsk t return a.api.StateVMCirculatingSupplyInternal(ctx, tsk) } +func (a *GatewayAPI) StateNetworkVersion(ctx context.Context, tsk types.TipSetKey) (network.Version, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return 0, err + } + return a.api.StateNetworkVersion(ctx, tsk) +} + func (a *GatewayAPI) WalletVerify(ctx context.Context, k address.Address, msg []byte, sig *crypto.Signature) (bool, error) { return sigs.Verify(sig, k, msg) == nil, nil } From abc3e783ffc3cdf04dc9c2b4748930829e31e898 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 20 Oct 2020 10:09:21 -0700 Subject: [PATCH 813/885] bump lookback cap to 24 hours --- cmd/lotus-gateway/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/lotus-gateway/api.go b/cmd/lotus-gateway/api.go index a0febef2c..7fc004327 100644 --- a/cmd/lotus-gateway/api.go +++ b/cmd/lotus-gateway/api.go @@ -23,7 +23,7 @@ import ( ) const ( - LookbackCap = time.Hour * 12 + LookbackCap = time.Hour * 24 stateWaitLookbackLimit = abi.ChainEpoch(20) ) From dc40469cb3edbb32f9d7815a3fb5cbd676feed48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 20 Oct 2020 21:54:19 +0200 Subject: [PATCH 814/885] Miner finder for interactive client deal CLI --- cli/client.go | 498 ++++++++++++++++++++++++++++++++++++++++---------- go.mod | 1 + go.sum | 3 + 3 files changed, 406 insertions(+), 96 deletions(-) diff --git a/cli/client.go b/cli/client.go index 7035fb64b..01de8801c 100644 --- a/cli/client.go +++ b/cli/client.go @@ -1,19 +1,25 @@ package cli import ( + "bufio" "context" "encoding/json" "errors" "fmt" "io" + "math/rand" "os" "path/filepath" "sort" "strconv" + "strings" + "sync" + "sync/atomic" "text/tabwriter" "time" tm "github.com/buger/goterm" + "github.com/chzyer/readline" "github.com/docker/go-units" "github.com/fatih/color" datatransfer "github.com/filecoin-project/go-data-transfer" @@ -74,6 +80,7 @@ var clientCmd = &cli.Command{ WithCategory("storage", clientQueryAskCmd), WithCategory("storage", clientListDeals), WithCategory("storage", clientGetDealCmd), + WithCategory("storage", clientListAsksCmd), WithCategory("data", clientImportCmd), WithCategory("data", clientDropCmd), WithCategory("data", clientLocalCmd), @@ -468,16 +475,26 @@ func interactiveDeal(cctx *cli.Context) error { } defer closer() ctx := ReqContext(cctx) + ctx, cancel := context.WithCancel(ctx) + defer cancel() state := "import" + gib := types.NewInt(1 << 30) var data cid.Cid var days int - var maddr address.Address - var ask storagemarket.StorageAsk - var epochPrice big.Int + var maddrs []address.Address + var ask []storagemarket.StorageAsk + var epochPrices []big.Int + var dur time.Duration var epochs abi.ChainEpoch var verified bool + var ds lapi.DataSize + + // find + var candidateAsks []*storagemarket.StorageAsk + var budget types.FIL + var dealCount int64 var a address.Address if from := cctx.String("from"); from != "" { @@ -494,10 +511,24 @@ func interactiveDeal(cctx *cli.Context) error { a = def } + fromBal, err := api.WalletBalance(ctx, a) + if err != nil { + return xerrors.Errorf("checking from address balance: %w", err) + } + printErr := func(err error) { fmt.Printf("%s %s\n", color.RedString("Error:"), err.Error()) } + cs := readline.NewCancelableStdin(os.Stdin) + go func() { + <-ctx.Done() + cs.Close() // nolint:errcheck + }() + + rl := bufio.NewReader(cs) + +uiLoop: for { // TODO: better exit handling if err := ctx.Err(); err != nil { @@ -508,8 +539,8 @@ func interactiveDeal(cctx *cli.Context) error { case "import": fmt.Print("Data CID (from " + color.YellowString("lotus client import") + "): ") - var cidStr string - _, err := fmt.Scan(&cidStr) + _cidStr, _, err := rl.ReadLine() + cidStr := string(_cidStr) if err != nil { printErr(xerrors.Errorf("reading cid string: %w", err)) continue @@ -521,11 +552,23 @@ func interactiveDeal(cctx *cli.Context) error { continue } + color.Blue(".. calculating data size\n") + ds, err = api.ClientDealSize(ctx, data) + if err != nil { + return err + } + state = "duration" case "duration": fmt.Print("Deal duration (days): ") - _, err := fmt.Scan(&days) + _daystr, _, err := rl.ReadLine() + daystr := string(_daystr) + if err != nil { + return err + } + + _, err = fmt.Sscan(daystr, &days) if err != nil { printErr(xerrors.Errorf("parsing duration: %w", err)) continue @@ -536,44 +579,9 @@ func interactiveDeal(cctx *cli.Context) error { continue } - state = "miner" - case "miner": - fmt.Print("Miner Address (f0..): ") - var maddrStr string + dur = 24 * time.Hour * time.Duration(days) + epochs = abi.ChainEpoch(dur / (time.Duration(build.BlockDelaySecs) * time.Second)) - _, err := fmt.Scan(&maddrStr) - if err != nil { - printErr(xerrors.Errorf("reading miner address: %w", err)) - continue - } - - maddr, err = address.NewFromString(maddrStr) - if err != nil { - printErr(xerrors.Errorf("parsing miner address: %w", err)) - continue - } - - state = "query" - case "query": - color.Blue(".. querying miner ask") - - mi, err := api.StateMinerInfo(ctx, maddr, types.EmptyTSK) - if err != nil { - printErr(xerrors.Errorf("failed to get peerID for miner: %w", err)) - state = "miner" - continue - } - - a, err := api.ClientQueryAsk(ctx, *mi.PeerId, maddr) - if err != nil { - printErr(xerrors.Errorf("failed to query ask: %w", err)) - state = "miner" - continue - } - - ask = *a - - // TODO: run more validation state = "verified" case "verified": ts, err := api.ChainHead(ctx) @@ -587,26 +595,20 @@ func interactiveDeal(cctx *cli.Context) error { } if dcap == nil { - state = "confirm" + state = "miner" continue } - color.Blue(".. checking verified deal eligibility\n") - ds, err := api.ClientDealSize(ctx, data) - if err != nil { - return err - } - if dcap.Uint64() < uint64(ds.PieceSize) { color.Yellow(".. not enough DataCap available for a verified deal\n") - state = "confirm" + state = "miner" continue } fmt.Print("\nMake this a verified deal? (yes/no): ") - var yn string - _, err = fmt.Scan(&yn) + _yn, _, err := rl.ReadLine() + yn := string(_yn) if err != nil { return err } @@ -621,34 +623,162 @@ func interactiveDeal(cctx *cli.Context) error { continue } - state = "confirm" - case "confirm": - fromBal, err := api.WalletBalance(ctx, a) + state = "miner" + case "miner": + fmt.Print("Miner Addresses (f0.. f0..), none to find: ") + + _maddrsStr, _, err := rl.ReadLine() + maddrsStr := string(_maddrsStr) if err != nil { - return xerrors.Errorf("checking from address balance: %w", err) + printErr(xerrors.Errorf("reading miner address: %w", err)) + continue } - color.Blue(".. calculating data size\n") - ds, err := api.ClientDealSize(ctx, data) + for _, s := range strings.Fields(maddrsStr) { + maddr, err := address.NewFromString(strings.TrimSpace(s)) + if err != nil { + printErr(xerrors.Errorf("parsing miner address: %w", err)) + continue uiLoop + } + + maddrs = append(maddrs, maddr) + } + + state = "query" + if len(maddrs) == 0 { + state = "find" + } + case "find": + asks, err := getAsks(ctx, api) if err != nil { return err } - dur := 24 * time.Hour * time.Duration(days) - - epochs = abi.ChainEpoch(dur / (time.Duration(build.BlockDelaySecs) * time.Second)) - // TODO: do some more or epochs math (round to miner PP, deal start buffer) - - pricePerGib := ask.Price - if verified { - pricePerGib = ask.VerifiedPrice + for _, ask := range asks { + if ask.MinPieceSize > ds.PieceSize { + continue + } + if ask.MaxPieceSize < ds.PieceSize { + continue + } + candidateAsks = append(candidateAsks, ask) } - gib := types.NewInt(1 << 30) + fmt.Printf("Found %d candidate asks\n", len(candidateAsks)) + state = "find-budget" + case "find-budget": + fmt.Printf("Proposing from %s, Current Balance: %s\n", a, types.FIL(fromBal)) + fmt.Print("Maximum budget (FIL): ") // TODO: Propose some default somehow? - // TODO: price is based on PaddedPieceSize, right? - epochPrice = types.BigDiv(types.BigMul(pricePerGib, types.NewInt(uint64(ds.PieceSize))), gib) - totalPrice := types.BigMul(epochPrice, types.NewInt(uint64(epochs))) + _budgetStr, _, err := rl.ReadLine() + budgetStr := string(_budgetStr) + if err != nil { + printErr(xerrors.Errorf("reading miner address: %w", err)) + continue + } + + budget, err = types.ParseFIL(budgetStr) + if err != nil { + printErr(xerrors.Errorf("parsing FIL: %w", err)) + continue uiLoop + } + + var goodAsks []*storagemarket.StorageAsk + for _, ask := range candidateAsks { + p := ask.Price + if verified { + p = ask.VerifiedPrice + } + + epochPrice := types.BigDiv(types.BigMul(p, types.NewInt(uint64(ds.PieceSize))), gib) + totalPrice := types.BigMul(epochPrice, types.NewInt(uint64(epochs))) + + if totalPrice.LessThan(abi.TokenAmount(budget)) { + goodAsks = append(goodAsks, ask) + } + } + candidateAsks = goodAsks + fmt.Printf("%d asks within budget\n", len(candidateAsks)) + state = "find-count" + case "find-count": + fmt.Print("Deals to make (1): ") + dealcStr, _, err := rl.ReadLine() + if err != nil { + printErr(xerrors.Errorf("reading deal count: %w", err)) + continue + } + + dealCount, err = strconv.ParseInt(string(dealcStr), 10, 64) + if err != nil { + return err + } + + color.Blue(".. Picking miners") + + // TODO: some better strategy (this tries to pick randomly) + var pickedAsks []*storagemarket.StorageAsk + pickLoop: + for i := 0; i < 64; i++ { + rand.Shuffle(len(candidateAsks), func(i, j int) { + candidateAsks[i], candidateAsks[j] = candidateAsks[j], candidateAsks[i] + }) + + remainingBudget := abi.TokenAmount(budget) + pickedAsks = []*storagemarket.StorageAsk{} + + for _, ask := range candidateAsks { + p := ask.Price + if verified { + p = ask.VerifiedPrice + } + + epochPrice := types.BigDiv(types.BigMul(p, types.NewInt(uint64(ds.PieceSize))), gib) + totalPrice := types.BigMul(epochPrice, types.NewInt(uint64(epochs))) + + if totalPrice.GreaterThan(remainingBudget) { + continue + } + + pickedAsks = append(pickedAsks, ask) + remainingBudget = big.Sub(remainingBudget, totalPrice) + + if len(pickedAsks) == int(dealCount) { + break pickLoop + } + } + } + + for _, pickedAsk := range pickedAsks { + maddrs = append(maddrs, pickedAsk.Miner) + ask = append(ask, *pickedAsk) + } + + state = "confirm" + case "query": + color.Blue(".. querying miner asks") + + for _, maddr := range maddrs { + mi, err := api.StateMinerInfo(ctx, maddr, types.EmptyTSK) + if err != nil { + printErr(xerrors.Errorf("failed to get peerID for miner: %w", err)) + state = "miner" + continue uiLoop + } + + a, err := api.ClientQueryAsk(ctx, *mi.PeerId, maddr) + if err != nil { + printErr(xerrors.Errorf("failed to query ask: %w", err)) + state = "miner" + continue uiLoop + } + + ask = append(ask, *a) + } + + // TODO: run more validation + state = "confirm" + case "confirm": + // TODO: do some more or epochs math (round to miner PP, deal start buffer) fmt.Printf("-----\n") fmt.Printf("Proposing from %s\n", a) @@ -656,15 +786,41 @@ func interactiveDeal(cctx *cli.Context) error { fmt.Printf("\n") fmt.Printf("Piece size: %s (Payload size: %s)\n", units.BytesSize(float64(ds.PieceSize)), units.BytesSize(float64(ds.PayloadSize))) fmt.Printf("Duration: %s\n", dur) - fmt.Printf("Total price: ~%s (%s per epoch)\n", types.FIL(totalPrice), types.FIL(epochPrice)) + + pricePerGib := big.Zero() + for _, a := range ask { + p := a.Price + if verified { + p = a.VerifiedPrice + } + pricePerGib = big.Add(pricePerGib, p) + epochPrice := types.BigDiv(types.BigMul(p, types.NewInt(uint64(ds.PieceSize))), gib) + epochPrices = append(epochPrices, epochPrice) + + mpow, err := api.StateMinerPower(ctx, a.Miner, types.EmptyTSK) + if err != nil { + return xerrors.Errorf("getting power (%s): %w", a.Miner, err) + } + + if len(ask) > 1 { + totalPrice := types.BigMul(epochPrice, types.NewInt(uint64(epochs))) + fmt.Printf("Miner %s (Power:%s) price: ~%s (%s per epoch)\n", color.YellowString(a.Miner.String()), color.GreenString(types.SizeStr(mpow.MinerPower.QualityAdjPower)), color.BlueString(types.FIL(totalPrice).String()), types.FIL(epochPrice)) + } + } + + // TODO: price is based on PaddedPieceSize, right? + epochPrice := types.BigDiv(types.BigMul(pricePerGib, types.NewInt(uint64(ds.PieceSize))), gib) + totalPrice := types.BigMul(epochPrice, types.NewInt(uint64(epochs))) + + fmt.Printf("Total price: ~%s (%s per epoch)\n", color.CyanString(types.FIL(totalPrice).String()), types.FIL(epochPrice)) fmt.Printf("Verified: %v\n", verified) state = "accept" case "accept": fmt.Print("\nAccept (yes/no): ") - var yn string - _, err := fmt.Scan(&yn) + _yn, _, err := rl.ReadLine() + yn := string(_yn) if err != nil { return err } @@ -680,30 +836,34 @@ func interactiveDeal(cctx *cli.Context) error { state = "execute" case "execute": - color.Blue(".. executing") - proposal, err := api.ClientStartDeal(ctx, &lapi.StartDealParams{ - Data: &storagemarket.DataRef{ - TransferType: storagemarket.TTGraphsync, - Root: data, - }, - Wallet: a, - Miner: maddr, - EpochPrice: epochPrice, - MinBlocksDuration: uint64(epochs), - DealStartEpoch: abi.ChainEpoch(cctx.Int64("start-epoch")), - FastRetrieval: cctx.Bool("fast-retrieval"), - VerifiedDeal: verified, - }) - if err != nil { - return err + color.Blue(".. executing\n") + + for i, maddr := range maddrs { + proposal, err := api.ClientStartDeal(ctx, &lapi.StartDealParams{ + Data: &storagemarket.DataRef{ + TransferType: storagemarket.TTGraphsync, + Root: data, + }, + Wallet: a, + Miner: maddr, + EpochPrice: epochPrices[i], + MinBlocksDuration: uint64(epochs), + DealStartEpoch: abi.ChainEpoch(cctx.Int64("start-epoch")), + FastRetrieval: cctx.Bool("fast-retrieval"), + VerifiedDeal: verified, + }) + if err != nil { + return err + } + + encoder, err := GetCidEncoder(cctx) + if err != nil { + return err + } + + fmt.Printf("Deal (%s) CID: %s\n", maddr, color.GreenString(encoder.Encode(*proposal))) } - encoder, err := GetCidEncoder(cctx) - if err != nil { - return err - } - - fmt.Println("\nDeal CID:", color.GreenString(encoder.Encode(*proposal))) return nil default: return xerrors.Errorf("unknown state: %s", state) @@ -944,6 +1104,152 @@ var clientRetrieveCmd = &cli.Command{ }, } +var clientListAsksCmd = &cli.Command{ + Name: "list-asks", + Usage: "List asks for top miners", + Action: func(cctx *cli.Context) error { + api, closer, err := GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + ctx := ReqContext(cctx) + + asks, err := getAsks(ctx, api) + if err != nil { + return err + } + + for _, ask := range asks { + fmt.Printf("%s: min:%s max:%s price:%s/GiB/Epoch verifiedPrice:%s/GiB/Epoch\n", ask.Miner, + types.SizeStr(types.NewInt(uint64(ask.MinPieceSize))), + types.SizeStr(types.NewInt(uint64(ask.MaxPieceSize))), + types.FIL(ask.Price), + types.FIL(ask.VerifiedPrice), + ) + } + + return nil + }, +} + +func getAsks(ctx context.Context, api lapi.FullNode) ([]*storagemarket.StorageAsk, error) { + color.Blue(".. getting miner list") + miners, err := api.StateListMiners(ctx, types.EmptyTSK) + if err != nil { + return nil, xerrors.Errorf("getting miner list: %w", err) + } + + var lk sync.Mutex + var found int64 + var withMinPower []address.Address + done := make(chan struct{}) + + go func() { + defer close(done) + + var wg sync.WaitGroup + wg.Add(len(miners)) + + throttle := make(chan struct{}, 50) + for _, miner := range miners { + throttle <- struct{}{} + go func(miner address.Address) { + defer wg.Done() + defer func() { + <-throttle + }() + + power, err := api.StateMinerPower(ctx, miner, types.EmptyTSK) + if err != nil { + return + } + + if power.HasMinPower { // TODO: Lower threshold + atomic.AddInt64(&found, 1) + lk.Lock() + withMinPower = append(withMinPower, miner) + lk.Unlock() + } + }(miner) + } + }() + +loop: + for { + select { + case <-time.After(150 * time.Millisecond): + fmt.Printf("\r* Found %d miners with power", atomic.LoadInt64(&found)) + case <-done: + break loop + } + } + fmt.Printf("\r* Found %d miners with power\n", atomic.LoadInt64(&found)) + + color.Blue(".. querying asks") + + var asks []*storagemarket.StorageAsk + var queried, got int64 + + done = make(chan struct{}) + go func() { + defer close(done) + + var wg sync.WaitGroup + wg.Add(len(withMinPower)) + + throttle := make(chan struct{}, 50) + for _, miner := range withMinPower { + throttle <- struct{}{} + go func(miner address.Address) { + defer wg.Done() + defer func() { + <-throttle + atomic.AddInt64(&queried, 1) + }() + + ctx, cancel := context.WithTimeout(ctx, 4*time.Second) + defer cancel() + + mi, err := api.StateMinerInfo(ctx, miner, types.EmptyTSK) + if err != nil { + return + } + if mi.PeerId == nil { + return + } + + ask, err := api.ClientQueryAsk(ctx, *mi.PeerId, miner) + if err != nil { + return + } + + atomic.AddInt64(&got, 1) + lk.Lock() + asks = append(asks, ask) + lk.Unlock() + }(miner) + } + }() + +loop2: + for { + select { + case <-time.After(150 * time.Millisecond): + fmt.Printf("\r* Queried %d asks, got %d responses", atomic.LoadInt64(&queried), atomic.LoadInt64(&got)) + case <-done: + break loop2 + } + } + fmt.Printf("\r* Queried %d asks, got %d responses\n", atomic.LoadInt64(&queried), atomic.LoadInt64(&got)) + + sort.Slice(asks, func(i, j int) bool { + return asks[i].Price.LessThan(asks[j].Price) + }) + + return asks, nil +} + var clientQueryAskCmd = &cli.Command{ Name: "query-ask", Usage: "Find a miners ask", diff --git a/go.mod b/go.mod index a6a074e49..b648523d1 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d github.com/buger/goterm v0.0.0-20200322175922-2f3e71b85129 + github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e github.com/cockroachdb/pebble v0.0.0-20200916222308-4e219a90ba5b github.com/coreos/go-systemd/v22 v22.0.0 github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e diff --git a/go.sum b/go.sum index 4cf7b3628..b7a717c18 100644 --- a/go.sum +++ b/go.sum @@ -120,8 +120,11 @@ github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE= github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= +github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= From e08032da93b9cc4efa3eb139ec0443fe99bdb53f Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Tue, 20 Oct 2020 13:56:13 -0700 Subject: [PATCH 815/885] feat(markets): update markets v1.0.0 --- go.mod | 4 ++-- go.sum | 10 ++++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index a6a074e49..35b627415 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 github.com/filecoin-project/go-data-transfer v0.9.0 github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f - github.com/filecoin-project/go-fil-markets v0.9.1 + github.com/filecoin-project/go-fil-markets v1.0.0 github.com/filecoin-project/go-jsonrpc v0.1.2-0.20201008195726-68c6a2704e49 github.com/filecoin-project/go-multistore v0.0.3 github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 @@ -66,7 +66,7 @@ require ( github.com/ipfs/go-ds-pebble v0.0.2-0.20200921225637-ce220f8ac459 github.com/ipfs/go-filestore v1.0.0 github.com/ipfs/go-fs-lock v0.0.6 - github.com/ipfs/go-graphsync v0.3.0 + github.com/ipfs/go-graphsync v0.3.1 github.com/ipfs/go-ipfs-blockstore v1.0.1 github.com/ipfs/go-ipfs-chunker v0.0.5 github.com/ipfs/go-ipfs-ds-help v1.0.0 diff --git a/go.sum b/go.sum index 4cf7b3628..991a6c5a2 100644 --- a/go.sum +++ b/go.sum @@ -246,8 +246,8 @@ github.com/filecoin-project/go-ds-versioning v0.1.0 h1:y/X6UksYTsK8TLCI7rttCKEvl github.com/filecoin-project/go-ds-versioning v0.1.0/go.mod h1:mp16rb4i2QPmxBnmanUx8i/XANp+PFCCJWiAb+VW4/s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f h1:GxJzR3oRIMTPtpZ0b7QF8FKPK6/iPAc7trhlL5k/g+s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= -github.com/filecoin-project/go-fil-markets v0.9.1 h1:MgO+UkpreD6x8DV2Zkw2xlBogixfpw9/wf4+nBii7bU= -github.com/filecoin-project/go-fil-markets v0.9.1/go.mod h1:h+bJ/IUnYjnW5HMKyt9JQSnhslqetkpuzwwugc3K8vM= +github.com/filecoin-project/go-fil-markets v1.0.0 h1:np9+tlnWXh9xYG4oZfha6HZFLYOaAZoMGR3V4w6DM48= +github.com/filecoin-project/go-fil-markets v1.0.0/go.mod h1:lXExJyYHwpMMddCqhEdNrc7euYJKNkp04K76NZqJLGg= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24= @@ -376,6 +376,8 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -540,6 +542,8 @@ github.com/ipfs/go-fs-lock v0.0.6/go.mod h1:OTR+Rj9sHiRubJh3dRhD15Juhd/+w6VPOY28 github.com/ipfs/go-graphsync v0.1.0/go.mod h1:jMXfqIEDFukLPZHqDPp8tJMbHO9Rmeb9CEGevngQbmE= github.com/ipfs/go-graphsync v0.3.0 h1:I6Y20kSuCWkUvPoUWo4V3am704/9QjgDVVkf0zIV8+8= github.com/ipfs/go-graphsync v0.3.0/go.mod h1:gEBvJUNelzMkaRPJTpg/jaKN4AQW/7wDWu0K92D8o10= +github.com/ipfs/go-graphsync v0.3.1 h1:dJLYrck4oyJDfMVhGEKiWHxaY8oYMWko4m2Fi+4bofo= +github.com/ipfs/go-graphsync v0.3.1/go.mod h1:bw4LiLM5Oq/uLdzEtih9LK8GrwSijv+XqYiWCTxHMqs= github.com/ipfs/go-hamt-ipld v0.1.1 h1:0IQdvwnAAUKmDE+PMJa5y1QiwOPHpI9+eAbQEEEYthk= github.com/ipfs/go-hamt-ipld v0.1.1/go.mod h1:1EZCr2v0jlCnhpa+aZ0JZYp8Tt2w16+JJOAVz17YcDk= github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= @@ -1818,6 +1822,8 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 556f92a8235e1b608a5842324693856d08deb26a Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Fri, 16 Oct 2020 20:35:46 -0400 Subject: [PATCH 816/885] Update to actors v2.2.0 at PostLiftoff epoch --- build/params_2k.go | 2 ++ build/params_mainnet.go | 3 +++ build/params_shared_vals.go | 2 +- build/params_testground.go | 1 + chain/stmgr/forks.go | 4 ++++ cmd/tvx/codenames.go | 1 + documentation/en/api-methods.md | 2 +- go.mod | 4 ++-- go.sum | 8 ++++---- 9 files changed, 19 insertions(+), 8 deletions(-) diff --git a/build/params_2k.go b/build/params_2k.go index b09b60fae..5a0e8fd61 100644 --- a/build/params_2k.go +++ b/build/params_2k.go @@ -22,6 +22,8 @@ const UpgradeTapeHeight = -4 var UpgradeActorsV2Height = abi.ChainEpoch(10) var UpgradeLiftoffHeight = abi.ChainEpoch(-5) +const UpgradeKumquatHeight = -6 + var DrandSchedule = map[abi.ChainEpoch]DrandEnum{ 0: DrandMainnet, } diff --git a/build/params_mainnet.go b/build/params_mainnet.go index 54f50ac6e..9d5c48ff6 100644 --- a/build/params_mainnet.go +++ b/build/params_mainnet.go @@ -37,6 +37,9 @@ const UpgradeTapeHeight = 140760 // We still have upgrades and state changes to do, but can happen after signaling timing here. const UpgradeLiftoffHeight = 148888 +// TODO: Confirm epoch +const UpgradeKumquatHeight = 170000 + func init() { policy.SetConsensusMinerMinPower(abi.NewStoragePower(10 << 40)) policy.SetSupportedProofTypes( diff --git a/build/params_shared_vals.go b/build/params_shared_vals.go index ede40c0e3..b80472594 100644 --- a/build/params_shared_vals.go +++ b/build/params_shared_vals.go @@ -25,7 +25,7 @@ const UnixfsLinksPerLevel = 1024 // Consensus / Network const AllowableClockDriftSecs = uint64(1) -const NewestNetworkVersion = network.Version5 +const NewestNetworkVersion = network.Version6 const ActorUpgradeNetworkVersion = network.Version4 // Epochs diff --git a/build/params_testground.go b/build/params_testground.go index 7ef034234..beee1c727 100644 --- a/build/params_testground.go +++ b/build/params_testground.go @@ -86,6 +86,7 @@ var ( UpgradeTapeHeight abi.ChainEpoch = -4 UpgradeActorsV2Height abi.ChainEpoch = 10 UpgradeLiftoffHeight abi.ChainEpoch = -5 + UpgradeKumquatHeight abi.ChainEpoch = -6 DrandSchedule = map[abi.ChainEpoch]DrandEnum{ 0: DrandMainnet, diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index 488f84167..5d6cac51f 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -86,6 +86,10 @@ func DefaultUpgradeSchedule() UpgradeSchedule { Height: build.UpgradeLiftoffHeight, Network: network.Version5, Migration: UpgradeLiftoff, + }, { + Height: build.UpgradeKumquatHeight, + Network: network.Version6, + Migration: nil, }} if build.UpgradeActorsV2Height == math.MaxInt64 { // disable actors upgrade diff --git a/cmd/tvx/codenames.go b/cmd/tvx/codenames.go index 851e9d841..b9f590914 100644 --- a/cmd/tvx/codenames.go +++ b/cmd/tvx/codenames.go @@ -23,6 +23,7 @@ var ProtocolCodenames = []struct { {build.UpgradeActorsV2Height + 1, "actorsv2"}, {build.UpgradeTapeHeight + 1, "tape"}, {build.UpgradeLiftoffHeight + 1, "liftoff"}, + {build.UpgradeKumquatHeight + 1, "postliftoff"}, } // GetProtocolCodename gets the protocol codename associated with a height. diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index c5bc24b04..3aeb6c096 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -4004,7 +4004,7 @@ Inputs: ] ``` -Response: `5` +Response: `6` ### StateReadState StateReadState returns the indicated actor's state. diff --git a/go.mod b/go.mod index 35b627415..cb8d5d3f3 100644 --- a/go.mod +++ b/go.mod @@ -34,12 +34,12 @@ require ( github.com/filecoin-project/go-multistore v0.0.3 github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 github.com/filecoin-project/go-paramfetch v0.0.2-0.20200701152213-3e0f0afdc261 - github.com/filecoin-project/go-state-types v0.0.0-20201003010437-c33112184a2b + github.com/filecoin-project/go-state-types v0.0.0-20201013222834-41ea465f274f github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b github.com/filecoin-project/specs-actors v0.9.12 - github.com/filecoin-project/specs-actors/v2 v2.1.0 + github.com/filecoin-project/specs-actors/v2 v2.2.0 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 github.com/filecoin-project/test-vectors/schema v0.0.5 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 diff --git a/go.sum b/go.sum index 991a6c5a2..37694572c 100644 --- a/go.sum +++ b/go.sum @@ -266,8 +266,8 @@ github.com/filecoin-project/go-state-types v0.0.0-20200903145444-247639ffa6ad/go github.com/filecoin-project/go-state-types v0.0.0-20200904021452-1883f36ca2f4/go.mod h1:IQ0MBPnonv35CJHtWSN3YY1Hz2gkPru1Q9qoaYLxx9I= github.com/filecoin-project/go-state-types v0.0.0-20200928172055-2df22083d8ab h1:cEDC5Ei8UuT99hPWhCjA72SM9AuRtnpvdSTIYbnzN8I= github.com/filecoin-project/go-state-types v0.0.0-20200928172055-2df22083d8ab/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= -github.com/filecoin-project/go-state-types v0.0.0-20201003010437-c33112184a2b h1:bMUfG6Sy6YSMbsjQAO1Q2vEZldbSdsbRy/FX3OlTck0= -github.com/filecoin-project/go-state-types v0.0.0-20201003010437-c33112184a2b/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= +github.com/filecoin-project/go-state-types v0.0.0-20201013222834-41ea465f274f h1:TZDTu4MtBKSFLXWGKLy+cvC3nHfMFIrVgWLAz/+GgZQ= +github.com/filecoin-project/go-state-types v0.0.0-20201013222834-41ea465f274f/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe h1:dF8u+LEWeIcTcfUcCf3WFVlc81Fr2JKg8zPzIbBDKDw= github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIiWBRilQjQ+5IiwdQ= @@ -278,8 +278,8 @@ github.com/filecoin-project/specs-actors v0.9.4/go.mod h1:BStZQzx5x7TmCkLv0Bpa07 github.com/filecoin-project/specs-actors v0.9.12 h1:iIvk58tuMtmloFNHhAOQHG+4Gci6Lui0n7DYQGi3cJk= github.com/filecoin-project/specs-actors v0.9.12/go.mod h1:TS1AW/7LbG+615j4NsjMK1qlpAwaFsG9w0V2tg2gSao= github.com/filecoin-project/specs-actors/v2 v2.0.1/go.mod h1:v2NZVYinNIKA9acEMBm5wWXxqv5+frFEbekBFemYghY= -github.com/filecoin-project/specs-actors/v2 v2.1.0 h1:ocEuGz8DG2cUWw32c/tvF8D6xT+dGVWJTr5yDevU00g= -github.com/filecoin-project/specs-actors/v2 v2.1.0/go.mod h1:E7fAX4CZkDVQvDNRCxfq+hc3nx56KcCKyuZf0hlQJ20= +github.com/filecoin-project/specs-actors/v2 v2.2.0 h1:IyCICb0NHYeD0sdSqjVGwWydn/7r7xXuxdpvGAcRCGY= +github.com/filecoin-project/specs-actors/v2 v2.2.0/go.mod h1:rlv5Mx9wUhV8Qsz+vUezZNm+zL4tK08O0HreKKPB2Wc= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk= github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= github.com/filecoin-project/test-vectors/schema v0.0.5 h1:w3zHQhzM4pYxJDl21avXjOKBLF8egrvwUwjpT8TquDg= From 277d3e9ab43daf23d5c833761472997df57387a9 Mon Sep 17 00:00:00 2001 From: jennijuju Date: Tue, 20 Oct 2020 17:19:42 -0400 Subject: [PATCH 817/885] Add terminate sectors based on 1475's #4433 implementation in lotus-shed: - Moving the cmd to lotus-shed so that miners can use this right away without waiting for fsm changes.- Note: this should only be used when the miner is 100% sure that they want to terminate the sector on chain and they will be lose the power and pay a big one-time termination penalty for the sectors. - Only live sectors can be terminated. - To use, run `./lotus-shed sectors terminate --really-do-it=true sectorNum1 sectorNum2`. - A message will be sent and after the message is landed on chain, miner can run `lotus-miner sectors status --on-chain-info `, and check Expiration Info at the bottom. Both `OnTime` and `Early` are 0 indicates the sector is terminated. --- cmd/lotus-shed/main.go | 1 + cmd/lotus-shed/sectors.go | 121 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+) create mode 100644 cmd/lotus-shed/sectors.go diff --git a/cmd/lotus-shed/main.go b/cmd/lotus-shed/main.go index 29a105355..987118f46 100644 --- a/cmd/lotus-shed/main.go +++ b/cmd/lotus-shed/main.go @@ -42,6 +42,7 @@ func main() { stateTreePruneCmd, datastoreCmd, ledgerCmd, + sectorsCmd, } app := &cli.App{ diff --git a/cmd/lotus-shed/sectors.go b/cmd/lotus-shed/sectors.go new file mode 100644 index 000000000..45af36118 --- /dev/null +++ b/cmd/lotus-shed/sectors.go @@ -0,0 +1,121 @@ +package main + +import ( + "fmt" + "github.com/filecoin-project/go-bitfield" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/lotus/chain/actors" + "golang.org/x/xerrors" + "strconv" + + "github.com/filecoin-project/lotus/chain/types" + lcli "github.com/filecoin-project/lotus/cli" + "github.com/filecoin-project/specs-actors/actors/builtin" + miner0 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" + "github.com/urfave/cli/v2" +) + +var sectorsCmd = &cli.Command{ + Name: "sectors", + Usage: "Tools for interacting with sectors", + Flags: []cli.Flag{}, + Subcommands: []*cli.Command{ + terminateSectorCmd, + }, +} + +var terminateSectorCmd = &cli.Command{ + Name: "terminate", + Usage: "Forcefully terminate a sector (WARNING: This means losing power and pay a one-time termination penalty(including collateral) for the terminated sector)", + ArgsUsage: "[sectorNum1 sectorNum2 ...]", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "really-do-it", + Usage: "pass this flag if you know what you are doing", + }, + }, + Action: func(cctx *cli.Context) error { + if cctx.Args().Len() < 1 { + return fmt.Errorf("at least one sector must be specified") + } + + if !cctx.Bool("really-do-it") { + return fmt.Errorf("this is a command for advanced users, only use it if you are sure of what you are doing") + } + + nodeApi, closer, err := lcli.GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + + api, acloser, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return err + } + defer acloser() + + ctx := lcli.ReqContext(cctx) + + maddr, err := api.ActorAddress(ctx) + if err != nil { + return err + } + + mi, err := nodeApi.StateMinerInfo(ctx, maddr, types.EmptyTSK) + if err != nil { + return err + } + + terminationDeclarationParams := []miner0.TerminationDeclaration{} + + for _, sn := range cctx.Args().Slice() { + sectorNum, err := strconv.ParseUint(sn, 10, 64) + if err != nil { + return fmt.Errorf("could not parse sector number: %w", err) + } + + sectorbit := bitfield.New() + sectorbit.Set(sectorNum) + + loca, err := nodeApi.StateSectorPartition(ctx, maddr, abi.SectorNumber(sectorNum), types.EmptyTSK) + if err != nil { + return fmt.Errorf("get state sector partition %s", err) + } + + para := miner0.TerminationDeclaration{ + Deadline: loca.Deadline, + Partition: loca.Partition, + Sectors: sectorbit, + } + + terminationDeclarationParams = append(terminationDeclarationParams, para) + } + + terminateSectorParams := &miner0.TerminateSectorsParams{ + Terminations: terminationDeclarationParams, + } + + sp, err := actors.SerializeParams(terminateSectorParams) + if err != nil { + return xerrors.Errorf("serializing params: %w", err) + } + + smsg, err := nodeApi.MpoolPushMessage(ctx, &types.Message{ + From: mi.Owner, + To: maddr, + Method: builtin.MethodsMiner.TerminateSectors, + + Value: big.Zero(), + Params: sp, + }, nil) + if err != nil { + return xerrors.Errorf("mpool push message: %w", err) + } + + fmt.Println("Message CID:", smsg.Cid()) + + return nil + }, +} From ed1645109af83e2b6492a8435e955d468c8217b4 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 20 Oct 2020 14:16:32 -0700 Subject: [PATCH 818/885] Add lotus storage miner API support to lotus-shed --- cmd/lotus-shed/main.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cmd/lotus-shed/main.go b/cmd/lotus-shed/main.go index 987118f46..5acaa3218 100644 --- a/cmd/lotus-shed/main.go +++ b/cmd/lotus-shed/main.go @@ -1,6 +1,7 @@ package main import ( + "fmt" "os" logging "github.com/ipfs/go-log/v2" @@ -57,6 +58,13 @@ func main() { Hidden: true, Value: "~/.lotus", // TODO: Consider XDG_DATA_HOME }, + &cli.StringFlag{ + Name: "miner-repo", + Aliases: []string{"storagerepo"}, + EnvVars: []string{"LOTUS_MINER_PATH", "LOTUS_STORAGE_PATH"}, + Value: "~/.lotusminer", // TODO: Consider XDG_DATA_HOME + Usage: fmt.Sprintf("Specify miner repo path. flag storagerepo and env LOTUS_STORAGE_PATH are DEPRECATION, will REMOVE SOON"), + }, &cli.StringFlag{ Name: "log-level", Value: "info", From 65304319957d7ede184336cffdcc3c8eeb50bce8 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Tue, 20 Oct 2020 23:54:35 +0200 Subject: [PATCH 819/885] MinerGetBaseInfo: if miner is not found in lookback, check current If miner is found in current state, just return nil base info as miner cannot mine if he is not found in the lookback. Signed-off-by: Jakub Sztandera --- chain/stmgr/utils.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index de4f947df..d573f385b 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -461,12 +461,21 @@ func MinerGetBaseInfo(ctx context.Context, sm *StateManager, bcs beacon.Schedule return nil, xerrors.Errorf("getting lookback miner actor state: %w", err) } + // TODO: load the state instead of computing it? lbst, _, err := sm.TipSetState(ctx, lbts) if err != nil { return nil, err } act, err := sm.LoadActorRaw(ctx, maddr, lbst) + if xerrors.Is(err, types.ErrActorNotFound) { + _, err := sm.LoadActor(ctx, maddr, ts) + if err != nil { + return nil, xerrors.Errorf("loading miner in current state: %w", err) + } + + return nil, nil + } if err != nil { return nil, xerrors.Errorf("failed to load miner actor: %w", err) } From 7f46ad19ac0b2b471362b30d18230452d7e674f4 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Mon, 19 Oct 2020 13:26:07 -0400 Subject: [PATCH 820/885] VM: Enforce a call depth limit --- build/params_mainnet.go | 1 - chain/actors/version.go | 2 +- chain/gen/genesis/miners.go | 2 +- chain/vm/runtime.go | 1 + chain/vm/vm.go | 42 ++++++++++++++++++++----------------- 5 files changed, 26 insertions(+), 22 deletions(-) diff --git a/build/params_mainnet.go b/build/params_mainnet.go index 9d5c48ff6..71e96e614 100644 --- a/build/params_mainnet.go +++ b/build/params_mainnet.go @@ -37,7 +37,6 @@ const UpgradeTapeHeight = 140760 // We still have upgrades and state changes to do, but can happen after signaling timing here. const UpgradeLiftoffHeight = 148888 -// TODO: Confirm epoch const UpgradeKumquatHeight = 170000 func init() { diff --git a/chain/actors/version.go b/chain/actors/version.go index 2efd903bb..fe16d521e 100644 --- a/chain/actors/version.go +++ b/chain/actors/version.go @@ -18,7 +18,7 @@ func VersionForNetwork(version network.Version) Version { switch version { case network.Version0, network.Version1, network.Version2, network.Version3: return Version0 - case network.Version4, network.Version5: + case network.Version4, network.Version5, network.Version6: return Version2 default: panic(fmt.Sprintf("unsupported network version %d", version)) diff --git a/chain/gen/genesis/miners.go b/chain/gen/genesis/miners.go index 1023e5efa..6f0c136a3 100644 --- a/chain/gen/genesis/miners.go +++ b/chain/gen/genesis/miners.go @@ -405,7 +405,7 @@ func circSupply(ctx context.Context, vmi *vm.VM, maddr address.Address) abi.Toke rt := unsafeVM.MakeRuntime(ctx, &types.Message{ GasLimit: 1_000_000_000, From: maddr, - }, maddr, 0, 0, 0) + }) return rt.TotalFilCircSupply() } diff --git a/chain/vm/runtime.go b/chain/vm/runtime.go index 8f124247c..ed5501fe9 100644 --- a/chain/vm/runtime.go +++ b/chain/vm/runtime.go @@ -72,6 +72,7 @@ type Runtime struct { originNonce uint64 executionTrace types.ExecutionTrace + depth uint64 numActorsCreated uint64 allowInternal bool callerValidated bool diff --git a/chain/vm/vm.go b/chain/vm/vm.go index 72ad731aa..a7aa05719 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -38,6 +38,8 @@ import ( "github.com/filecoin-project/lotus/lib/bufbstore" ) +const MaxCallDepth = 4096 + var log = logging.Logger("vm") var actorLog = logging.Logger("actors") var gasOnActorExec = newGasCharge("OnActorExec", 0, 0) @@ -97,24 +99,37 @@ func (bs *gasChargingBlocks) Put(blk block.Block) error { return nil } -func (vm *VM) makeRuntime(ctx context.Context, msg *types.Message, origin address.Address, originNonce uint64, usedGas int64, nac uint64) *Runtime { +func (vm *VM) makeRuntime(ctx context.Context, msg *types.Message, parent *Runtime) *Runtime { rt := &Runtime{ ctx: ctx, vm: vm, state: vm.cstate, - origin: origin, - originNonce: originNonce, + origin: msg.From, + originNonce: msg.Nonce, height: vm.blockHeight, - gasUsed: usedGas, + gasUsed: 0, gasAvailable: msg.GasLimit, - numActorsCreated: nac, + depth: 0, + numActorsCreated: 0, pricelist: PricelistByEpoch(vm.blockHeight), allowInternal: true, callerValidated: false, executionTrace: types.ExecutionTrace{Msg: msg}, } + if parent != nil { + rt.gasUsed = parent.gasUsed + rt.origin = parent.origin + rt.originNonce = parent.originNonce + rt.numActorsCreated = parent.numActorsCreated + rt.depth = parent.depth + 1 + } + + if rt.depth > MaxCallDepth && rt.NetworkVersion() >= network.Version6 { + rt.Abortf(exitcode.SysErrForbidden, "message execution exceeds call depth") + } + rt.cst = &cbor.BasicIpldStore{ Blocks: &gasChargingBlocks{rt.chargeGasFunc(2), rt.pricelist, vm.cst.Blocks}, Atlas: vm.cst.Atlas, @@ -148,8 +163,8 @@ type UnsafeVM struct { VM *VM } -func (vm *UnsafeVM) MakeRuntime(ctx context.Context, msg *types.Message, origin address.Address, originNonce uint64, usedGas int64, nac uint64) *Runtime { - return vm.VM.makeRuntime(ctx, msg, origin, originNonce, usedGas, nac) +func (vm *UnsafeVM) MakeRuntime(ctx context.Context, msg *types.Message) *Runtime { + return vm.VM.makeRuntime(ctx, msg, nil) } type CircSupplyCalculator func(context.Context, abi.ChainEpoch, *state.StateTree) (abi.TokenAmount, error) @@ -224,18 +239,7 @@ func (vm *VM) send(ctx context.Context, msg *types.Message, parent *Runtime, st := vm.cstate - origin := msg.From - on := msg.Nonce - var nac uint64 = 0 - var gasUsed int64 - if parent != nil { - gasUsed = parent.gasUsed - origin = parent.origin - on = parent.originNonce - nac = parent.numActorsCreated - } - - rt := vm.makeRuntime(ctx, msg, origin, on, gasUsed, nac) + rt := vm.makeRuntime(ctx, msg, parent) if EnableGasTracing { rt.lastGasChargeTime = start if parent != nil { From ee5dcbc1d825081558f2f89841fe7ef7a5615da4 Mon Sep 17 00:00:00 2001 From: jennijuju Date: Tue, 20 Oct 2020 18:26:17 -0400 Subject: [PATCH 821/885] Add wait message --- cmd/lotus-shed/sectors.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/cmd/lotus-shed/sectors.go b/cmd/lotus-shed/sectors.go index 45af36118..11e678642 100644 --- a/cmd/lotus-shed/sectors.go +++ b/cmd/lotus-shed/sectors.go @@ -2,12 +2,13 @@ package main import ( "fmt" + "strconv" + "github.com/filecoin-project/go-bitfield" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/lotus/chain/actors" "golang.org/x/xerrors" - "strconv" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" @@ -114,7 +115,16 @@ var terminateSectorCmd = &cli.Command{ return xerrors.Errorf("mpool push message: %w", err) } - fmt.Println("Message CID:", smsg.Cid()) + fmt.Println("sent termination message:", smsg.Cid()) + + wait, err := nodeApi.StateWaitMsg(ctx, smsg.Cid(), uint64(cctx.Int("confidence"))) + if err != nil { + return err + } + + if wait.Receipt.ExitCode != 0 { + return fmt.Errorf("terminate sectors message returned exit %d", wait.Receipt.ExitCode) + } return nil }, From d78dd42ed471bc7f1fe1dff3a51d13de115a5020 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Tue, 20 Oct 2020 17:29:17 -0400 Subject: [PATCH 822/885] Lotus version 1.1.0 --- CHANGELOG.md | 13 +++++++++++++ build/version.go | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 76aa51de2..c21ffb685 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Lotus changelog +# 1.1.0 / 2020-10-20 + +This is a mandatory release that introduces the first post-liftoff upgrade to the Filecoin network. The changes that break consensus are an upgrade to specs-actors v2.2.0 at epoch 170000. + +## Changes + +- Introduce Network version 6 (https://github.com/filecoin-project/lotus/pull/4506) +- Update markets v1.0.0 (https://github.com/filecoin-project/lotus/pull/4505) +- Add some extra logging to try and debug sync issues (https://github.com/filecoin-project/lotus/pull/4486) +- Circle: Run tests for some subsystems separately (https://github.com/filecoin-project/lotus/pull/4496) +- Add a terminate sectors command to lotus-shed (https://github.com/filecoin-project/lotus/pull/4507) +- Add a comment to BlockMessages to address #4446 (https://github.com/filecoin-project/lotus/pull/4491) + # 1.0.0 / 2020-10-19 It's 1.0.0! This is an optional release of Lotus that introduces some UX improvements to the 0.10 series. diff --git a/build/version.go b/build/version.go index 352fa9b0e..2d01526b9 100644 --- a/build/version.go +++ b/build/version.go @@ -29,7 +29,7 @@ func buildType() string { } // BuildVersion is the local build version, set by build system -const BuildVersion = "1.0.0" +const BuildVersion = "1.1.0" func UserVersion() string { return BuildVersion + buildType() + CurrentCommit From 00dcb1bce94e1bef65608525457adf656e56cf1d Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 20 Oct 2020 18:30:56 -0700 Subject: [PATCH 823/885] Manage sectors by size instead of proof type. * We may have multiple sectors with the same size and different proof types, but all these management functions stay the same. * This simplifies PoSt logic. --- api/apistruct/struct.go | 32 +++++++++++----------- build/version.go | 2 +- extern/sector-storage/faults.go | 8 +++--- extern/sector-storage/localworker.go | 14 ++++++++-- extern/sector-storage/manager.go | 2 +- extern/sector-storage/manager_test.go | 2 +- extern/sector-storage/mock/mock.go | 2 +- extern/sector-storage/roprov.go | 7 ++++- extern/sector-storage/selector_alloc.go | 7 ++++- extern/sector-storage/selector_existing.go | 7 ++++- extern/sector-storage/stores/filetype.go | 7 +---- extern/sector-storage/stores/index.go | 12 ++++---- extern/sector-storage/stores/interface.go | 4 +-- extern/sector-storage/stores/local.go | 19 +++++-------- extern/sector-storage/stores/remote.go | 14 +++++----- storage/wdpost_run.go | 7 +---- storage/wdpost_run_test.go | 2 +- 17 files changed, 78 insertions(+), 70 deletions(-) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index e302aa8dc..547373e71 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -300,18 +300,18 @@ type StorageMinerStruct struct { SealingSchedDiag func(context.Context) (interface{}, error) `perm:"admin"` - StorageList func(context.Context) (map[stores.ID][]stores.Decl, error) `perm:"admin"` - StorageLocal func(context.Context) (map[stores.ID]string, error) `perm:"admin"` - StorageStat func(context.Context, stores.ID) (fsutil.FsStat, error) `perm:"admin"` - StorageAttach func(context.Context, stores.StorageInfo, fsutil.FsStat) error `perm:"admin"` - StorageDeclareSector func(context.Context, stores.ID, abi.SectorID, stores.SectorFileType, bool) error `perm:"admin"` - StorageDropSector func(context.Context, stores.ID, abi.SectorID, stores.SectorFileType) error `perm:"admin"` - StorageFindSector func(context.Context, abi.SectorID, stores.SectorFileType, abi.RegisteredSealProof, bool) ([]stores.SectorStorageInfo, error) `perm:"admin"` - StorageInfo func(context.Context, stores.ID) (stores.StorageInfo, error) `perm:"admin"` - StorageBestAlloc func(ctx context.Context, allocate stores.SectorFileType, spt abi.RegisteredSealProof, sealing stores.PathType) ([]stores.StorageInfo, error) `perm:"admin"` - StorageReportHealth func(ctx context.Context, id stores.ID, report stores.HealthReport) error `perm:"admin"` - StorageLock func(ctx context.Context, sector abi.SectorID, read stores.SectorFileType, write stores.SectorFileType) error `perm:"admin"` - StorageTryLock func(ctx context.Context, sector abi.SectorID, read stores.SectorFileType, write stores.SectorFileType) (bool, error) `perm:"admin"` + StorageList func(context.Context) (map[stores.ID][]stores.Decl, error) `perm:"admin"` + StorageLocal func(context.Context) (map[stores.ID]string, error) `perm:"admin"` + StorageStat func(context.Context, stores.ID) (fsutil.FsStat, error) `perm:"admin"` + StorageAttach func(context.Context, stores.StorageInfo, fsutil.FsStat) error `perm:"admin"` + StorageDeclareSector func(context.Context, stores.ID, abi.SectorID, stores.SectorFileType, bool) error `perm:"admin"` + StorageDropSector func(context.Context, stores.ID, abi.SectorID, stores.SectorFileType) error `perm:"admin"` + StorageFindSector func(context.Context, abi.SectorID, stores.SectorFileType, abi.SectorSize, bool) ([]stores.SectorStorageInfo, error) `perm:"admin"` + StorageInfo func(context.Context, stores.ID) (stores.StorageInfo, error) `perm:"admin"` + StorageBestAlloc func(ctx context.Context, allocate stores.SectorFileType, ssize abi.SectorSize, sealing stores.PathType) ([]stores.StorageInfo, error) `perm:"admin"` + StorageReportHealth func(ctx context.Context, id stores.ID, report stores.HealthReport) error `perm:"admin"` + StorageLock func(ctx context.Context, sector abi.SectorID, read stores.SectorFileType, write stores.SectorFileType) error `perm:"admin"` + StorageTryLock func(ctx context.Context, sector abi.SectorID, read stores.SectorFileType, write stores.SectorFileType) (bool, error) `perm:"admin"` DealsImportData func(ctx context.Context, dealPropCid cid.Cid, file string) error `perm:"write"` DealsList func(ctx context.Context) ([]api.MarketDeal, error) `perm:"read"` @@ -1203,8 +1203,8 @@ func (c *StorageMinerStruct) StorageDropSector(ctx context.Context, storageId st return c.Internal.StorageDropSector(ctx, storageId, s, ft) } -func (c *StorageMinerStruct) StorageFindSector(ctx context.Context, si abi.SectorID, types stores.SectorFileType, spt abi.RegisteredSealProof, allowFetch bool) ([]stores.SectorStorageInfo, error) { - return c.Internal.StorageFindSector(ctx, si, types, spt, allowFetch) +func (c *StorageMinerStruct) StorageFindSector(ctx context.Context, si abi.SectorID, types stores.SectorFileType, ssize abi.SectorSize, allowFetch bool) ([]stores.SectorStorageInfo, error) { + return c.Internal.StorageFindSector(ctx, si, types, ssize, allowFetch) } func (c *StorageMinerStruct) StorageList(ctx context.Context) (map[stores.ID][]stores.Decl, error) { @@ -1223,8 +1223,8 @@ func (c *StorageMinerStruct) StorageInfo(ctx context.Context, id stores.ID) (sto return c.Internal.StorageInfo(ctx, id) } -func (c *StorageMinerStruct) StorageBestAlloc(ctx context.Context, allocate stores.SectorFileType, spt abi.RegisteredSealProof, pt stores.PathType) ([]stores.StorageInfo, error) { - return c.Internal.StorageBestAlloc(ctx, allocate, spt, pt) +func (c *StorageMinerStruct) StorageBestAlloc(ctx context.Context, allocate stores.SectorFileType, ssize abi.SectorSize, pt stores.PathType) ([]stores.StorageInfo, error) { + return c.Internal.StorageBestAlloc(ctx, allocate, ssize, pt) } func (c *StorageMinerStruct) StorageReportHealth(ctx context.Context, id stores.ID, report stores.HealthReport) error { diff --git a/build/version.go b/build/version.go index 2d01526b9..b8941cb7a 100644 --- a/build/version.go +++ b/build/version.go @@ -84,7 +84,7 @@ func VersionForType(nodeType NodeType) (Version, error) { // semver versions of the rpc api exposed var ( FullAPIVersion = newVer(0, 17, 0) - MinerAPIVersion = newVer(0, 15, 0) + MinerAPIVersion = newVer(0, 16, 0) WorkerAPIVersion = newVer(0, 15, 0) ) diff --git a/extern/sector-storage/faults.go b/extern/sector-storage/faults.go index 31a1a3690..6a4be200f 100644 --- a/extern/sector-storage/faults.go +++ b/extern/sector-storage/faults.go @@ -14,14 +14,14 @@ import ( // FaultTracker TODO: Track things more actively type FaultTracker interface { - CheckProvable(ctx context.Context, spt abi.RegisteredSealProof, sectors []abi.SectorID) ([]abi.SectorID, error) + CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []abi.SectorID) ([]abi.SectorID, error) } // CheckProvable returns unprovable sectors -func (m *Manager) CheckProvable(ctx context.Context, spt abi.RegisteredSealProof, sectors []abi.SectorID) ([]abi.SectorID, error) { +func (m *Manager) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []abi.SectorID) ([]abi.SectorID, error) { var bad []abi.SectorID - ssize, err := spt.SectorSize() + ssize, err := pp.SectorSize() if err != nil { return nil, err } @@ -43,7 +43,7 @@ func (m *Manager) CheckProvable(ctx context.Context, spt abi.RegisteredSealProof return nil } - lp, _, err := m.localStore.AcquireSector(ctx, sector, spt, stores.FTSealed|stores.FTCache, stores.FTNone, stores.PathStorage, stores.AcquireMove) + lp, _, err := m.localStore.AcquireSector(ctx, sector, ssize, stores.FTSealed|stores.FTCache, stores.FTNone, stores.PathStorage, stores.AcquireMove) if err != nil { log.Warnw("CheckProvable Sector FAULT: acquire sector in checkProvable", "sector", sector, "error", err) bad = append(bad, sector) diff --git a/extern/sector-storage/localworker.go b/extern/sector-storage/localworker.go index b1193a2e2..0033b5d59 100644 --- a/extern/sector-storage/localworker.go +++ b/extern/sector-storage/localworker.go @@ -65,12 +65,16 @@ type localWorkerPathProvider struct { func (l *localWorkerPathProvider) AcquireSector(ctx context.Context, sector abi.SectorID, existing stores.SectorFileType, allocate stores.SectorFileType, sealing stores.PathType) (stores.SectorPaths, func(), error) { - paths, storageIDs, err := l.w.storage.AcquireSector(ctx, sector, l.w.scfg.SealProofType, existing, allocate, sealing, l.op) + ssize, err := l.w.scfg.SealProofType.SectorSize() + if err != nil { + return stores.SectorPaths{}, nil, err + } + paths, storageIDs, err := l.w.storage.AcquireSector(ctx, sector, ssize, existing, allocate, sealing, l.op) if err != nil { return stores.SectorPaths{}, nil, err } - releaseStorage, err := l.w.localStore.Reserve(ctx, sector, l.w.scfg.SealProofType, allocate, storageIDs, stores.FSOverheadSeal) + releaseStorage, err := l.w.localStore.Reserve(ctx, sector, ssize, allocate, storageIDs, stores.FSOverheadSeal) if err != nil { return stores.SectorPaths{}, nil, xerrors.Errorf("reserving storage space: %w", err) } @@ -212,7 +216,11 @@ func (l *LocalWorker) Remove(ctx context.Context, sector abi.SectorID) error { } func (l *LocalWorker) MoveStorage(ctx context.Context, sector abi.SectorID, types stores.SectorFileType) error { - if err := l.storage.MoveStorage(ctx, sector, l.scfg.SealProofType, types); err != nil { + ssize, err := l.scfg.SealProofType.SectorSize() + if err != nil { + return err + } + if err := l.storage.MoveStorage(ctx, sector, ssize, types); err != nil { return xerrors.Errorf("moving sealed data to storage: %w", err) } diff --git a/extern/sector-storage/manager.go b/extern/sector-storage/manager.go index 73a5eb51e..a17a5ba99 100644 --- a/extern/sector-storage/manager.go +++ b/extern/sector-storage/manager.go @@ -95,7 +95,7 @@ func New(ctx context.Context, ls stores.LocalStorage, si stores.SectorIndex, cfg return nil, err } - prover, err := ffiwrapper.New(&readonlyProvider{stor: lstor, index: si}, cfg) + prover, err := ffiwrapper.New(&readonlyProvider{stor: lstor, index: si, spt: cfg.SealProofType}, cfg) if err != nil { return nil, xerrors.Errorf("creating prover instance: %w", err) } diff --git a/extern/sector-storage/manager_test.go b/extern/sector-storage/manager_test.go index ee704cb5a..fbd712be4 100644 --- a/extern/sector-storage/manager_test.go +++ b/extern/sector-storage/manager_test.go @@ -92,7 +92,7 @@ func newTestMgr(ctx context.Context, t *testing.T) (*Manager, *stores.Local, *st lstor, err := stores.NewLocal(ctx, st, si, nil) require.NoError(t, err) - prover, err := ffiwrapper.New(&readonlyProvider{stor: lstor}, cfg) + prover, err := ffiwrapper.New(&readonlyProvider{stor: lstor, spt: cfg.SealProofType}, cfg) require.NoError(t, err) stor := stores.NewRemote(lstor, si, nil, 6000) diff --git a/extern/sector-storage/mock/mock.go b/extern/sector-storage/mock/mock.go index 001c7159c..3791cc6e2 100644 --- a/extern/sector-storage/mock/mock.go +++ b/extern/sector-storage/mock/mock.go @@ -392,7 +392,7 @@ func (mgr *SectorMgr) Remove(ctx context.Context, sector abi.SectorID) error { return nil } -func (mgr *SectorMgr) CheckProvable(ctx context.Context, spt abi.RegisteredSealProof, ids []abi.SectorID) ([]abi.SectorID, error) { +func (mgr *SectorMgr) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, ids []abi.SectorID) ([]abi.SectorID, error) { var bad []abi.SectorID for _, sid := range ids { diff --git a/extern/sector-storage/roprov.go b/extern/sector-storage/roprov.go index 2b009c63b..e8e2bf4f4 100644 --- a/extern/sector-storage/roprov.go +++ b/extern/sector-storage/roprov.go @@ -21,6 +21,11 @@ func (l *readonlyProvider) AcquireSector(ctx context.Context, id abi.SectorID, e return stores.SectorPaths{}, nil, xerrors.New("read-only storage") } + ssize, err := l.spt.SectorSize() + if err != nil { + return stores.SectorPaths{}, nil, xerrors.Errorf("failed to determine sector size: %w", err) + } + ctx, cancel := context.WithCancel(ctx) // use TryLock to avoid blocking @@ -34,7 +39,7 @@ func (l *readonlyProvider) AcquireSector(ctx context.Context, id abi.SectorID, e return stores.SectorPaths{}, nil, xerrors.Errorf("failed to acquire sector lock") } - p, _, err := l.stor.AcquireSector(ctx, id, l.spt, existing, allocate, sealing, stores.AcquireMove) + p, _, err := l.stor.AcquireSector(ctx, id, ssize, existing, allocate, sealing, stores.AcquireMove) return p, cancel, err } diff --git a/extern/sector-storage/selector_alloc.go b/extern/sector-storage/selector_alloc.go index b891383fb..15de1142f 100644 --- a/extern/sector-storage/selector_alloc.go +++ b/extern/sector-storage/selector_alloc.go @@ -44,7 +44,12 @@ func (s *allocSelector) Ok(ctx context.Context, task sealtasks.TaskType, spt abi have[path.ID] = struct{}{} } - best, err := s.index.StorageBestAlloc(ctx, s.alloc, spt, s.ptype) + ssize, err := spt.SectorSize() + if err != nil { + return false, xerrors.Errorf("getting sector size: %w", err) + } + + best, err := s.index.StorageBestAlloc(ctx, s.alloc, ssize, s.ptype) if err != nil { return false, xerrors.Errorf("finding best alloc storage: %w", err) } diff --git a/extern/sector-storage/selector_existing.go b/extern/sector-storage/selector_existing.go index fb161f085..1afdddf7b 100644 --- a/extern/sector-storage/selector_existing.go +++ b/extern/sector-storage/selector_existing.go @@ -46,7 +46,12 @@ func (s *existingSelector) Ok(ctx context.Context, task sealtasks.TaskType, spt have[path.ID] = struct{}{} } - best, err := s.index.StorageFindSector(ctx, s.sector, s.alloc, spt, s.allowFetch) + ssize, err := spt.SectorSize() + if err != nil { + return false, xerrors.Errorf("getting sector size: %w", err) + } + + best, err := s.index.StorageFindSector(ctx, s.sector, s.alloc, ssize, s.allowFetch) if err != nil { return false, xerrors.Errorf("finding best storage: %w", err) } diff --git a/extern/sector-storage/stores/filetype.go b/extern/sector-storage/stores/filetype.go index 90cc1d160..0b2d02e80 100644 --- a/extern/sector-storage/stores/filetype.go +++ b/extern/sector-storage/stores/filetype.go @@ -53,12 +53,7 @@ func (t SectorFileType) Has(singleType SectorFileType) bool { return t&singleType == singleType } -func (t SectorFileType) SealSpaceUse(spt abi.RegisteredSealProof) (uint64, error) { - ssize, err := spt.SectorSize() - if err != nil { - return 0, xerrors.Errorf("getting sector size: %w", err) - } - +func (t SectorFileType) SealSpaceUse(ssize abi.SectorSize) (uint64, error) { var need uint64 for _, pathType := range PathTypes { if !t.Has(pathType) { diff --git a/extern/sector-storage/stores/index.go b/extern/sector-storage/stores/index.go index e2bd7e4ee..198dc0b84 100644 --- a/extern/sector-storage/stores/index.go +++ b/extern/sector-storage/stores/index.go @@ -55,9 +55,9 @@ type SectorIndex interface { // part of storage-miner api StorageDeclareSector(ctx context.Context, storageID ID, s abi.SectorID, ft SectorFileType, primary bool) error StorageDropSector(ctx context.Context, storageID ID, s abi.SectorID, ft SectorFileType) error - StorageFindSector(ctx context.Context, sector abi.SectorID, ft SectorFileType, spt abi.RegisteredSealProof, allowFetch bool) ([]SectorStorageInfo, error) + StorageFindSector(ctx context.Context, sector abi.SectorID, ft SectorFileType, ssize abi.SectorSize, allowFetch bool) ([]SectorStorageInfo, error) - StorageBestAlloc(ctx context.Context, allocate SectorFileType, spt abi.RegisteredSealProof, pathType PathType) ([]StorageInfo, error) + StorageBestAlloc(ctx context.Context, allocate SectorFileType, ssize abi.SectorSize, pathType PathType) ([]StorageInfo, error) // atomically acquire locks on all sector file types. close ctx to unlock StorageLock(ctx context.Context, sector abi.SectorID, read SectorFileType, write SectorFileType) error @@ -246,7 +246,7 @@ func (i *Index) StorageDropSector(ctx context.Context, storageID ID, s abi.Secto return nil } -func (i *Index) StorageFindSector(ctx context.Context, s abi.SectorID, ft SectorFileType, spt abi.RegisteredSealProof, allowFetch bool) ([]SectorStorageInfo, error) { +func (i *Index) StorageFindSector(ctx context.Context, s abi.SectorID, ft SectorFileType, ssize abi.SectorSize, allowFetch bool) ([]SectorStorageInfo, error) { i.lk.RLock() defer i.lk.RUnlock() @@ -297,7 +297,7 @@ func (i *Index) StorageFindSector(ctx context.Context, s abi.SectorID, ft Sector } if allowFetch { - spaceReq, err := ft.SealSpaceUse(spt) + spaceReq, err := ft.SealSpaceUse(ssize) if err != nil { return nil, xerrors.Errorf("estimating required space: %w", err) } @@ -365,13 +365,13 @@ func (i *Index) StorageInfo(ctx context.Context, id ID) (StorageInfo, error) { return *si.info, nil } -func (i *Index) StorageBestAlloc(ctx context.Context, allocate SectorFileType, spt abi.RegisteredSealProof, pathType PathType) ([]StorageInfo, error) { +func (i *Index) StorageBestAlloc(ctx context.Context, allocate SectorFileType, ssize abi.SectorSize, pathType PathType) ([]StorageInfo, error) { i.lk.RLock() defer i.lk.RUnlock() var candidates []storageEntry - spaceReq, err := allocate.SealSpaceUse(spt) + spaceReq, err := allocate.SealSpaceUse(ssize) if err != nil { return nil, xerrors.Errorf("estimating required space: %w", err) } diff --git a/extern/sector-storage/stores/interface.go b/extern/sector-storage/stores/interface.go index 875754fc5..6a8691ff7 100644 --- a/extern/sector-storage/stores/interface.go +++ b/extern/sector-storage/stores/interface.go @@ -22,7 +22,7 @@ const ( ) type Store interface { - AcquireSector(ctx context.Context, s abi.SectorID, spt abi.RegisteredSealProof, existing SectorFileType, allocate SectorFileType, sealing PathType, op AcquireMode) (paths SectorPaths, stores SectorPaths, err error) + AcquireSector(ctx context.Context, s abi.SectorID, ssize abi.SectorSize, existing SectorFileType, allocate SectorFileType, sealing PathType, op AcquireMode) (paths SectorPaths, stores SectorPaths, err error) Remove(ctx context.Context, s abi.SectorID, types SectorFileType, force bool) error // like remove, but doesn't remove the primary sector copy, nor the last @@ -30,7 +30,7 @@ type Store interface { RemoveCopies(ctx context.Context, s abi.SectorID, types SectorFileType) error // move sectors into storage - MoveStorage(ctx context.Context, s abi.SectorID, spt abi.RegisteredSealProof, types SectorFileType) error + MoveStorage(ctx context.Context, s abi.SectorID, ssize abi.SectorSize, types SectorFileType) error FsStat(ctx context.Context, id ID) (fsutil.FsStat, error) } diff --git a/extern/sector-storage/stores/local.go b/extern/sector-storage/stores/local.go index 50968e7bd..b2431dbee 100644 --- a/extern/sector-storage/stores/local.go +++ b/extern/sector-storage/stores/local.go @@ -269,12 +269,7 @@ func (st *Local) reportHealth(ctx context.Context) { } } -func (st *Local) Reserve(ctx context.Context, sid abi.SectorID, spt abi.RegisteredSealProof, ft SectorFileType, storageIDs SectorPaths, overheadTab map[SectorFileType]int) (func(), error) { - ssize, err := spt.SectorSize() - if err != nil { - return nil, xerrors.Errorf("getting sector size: %w", err) - } - +func (st *Local) Reserve(ctx context.Context, sid abi.SectorID, ssize abi.SectorSize, ft SectorFileType, storageIDs SectorPaths, overheadTab map[SectorFileType]int) (func(), error) { st.localLk.Lock() done := func() {} @@ -324,7 +319,7 @@ func (st *Local) Reserve(ctx context.Context, sid abi.SectorID, spt abi.Register return done, nil } -func (st *Local) AcquireSector(ctx context.Context, sid abi.SectorID, spt abi.RegisteredSealProof, existing SectorFileType, allocate SectorFileType, pathType PathType, op AcquireMode) (SectorPaths, SectorPaths, error) { +func (st *Local) AcquireSector(ctx context.Context, sid abi.SectorID, ssize abi.SectorSize, existing SectorFileType, allocate SectorFileType, pathType PathType, op AcquireMode) (SectorPaths, SectorPaths, error) { if existing|allocate != existing^allocate { return SectorPaths{}, SectorPaths{}, xerrors.New("can't both find and allocate a sector") } @@ -340,7 +335,7 @@ func (st *Local) AcquireSector(ctx context.Context, sid abi.SectorID, spt abi.Re continue } - si, err := st.index.StorageFindSector(ctx, sid, fileType, spt, false) + si, err := st.index.StorageFindSector(ctx, sid, fileType, ssize, false) if err != nil { log.Warnf("finding existing sector %d(t:%d) failed: %+v", sid, fileType, err) continue @@ -370,7 +365,7 @@ func (st *Local) AcquireSector(ctx context.Context, sid abi.SectorID, spt abi.Re continue } - sis, err := st.index.StorageBestAlloc(ctx, fileType, spt, pathType) + sis, err := st.index.StorageBestAlloc(ctx, fileType, ssize, pathType) if err != nil { return SectorPaths{}, SectorPaths{}, xerrors.Errorf("finding best storage for allocating : %w", err) } @@ -525,13 +520,13 @@ func (st *Local) removeSector(ctx context.Context, sid abi.SectorID, typ SectorF return nil } -func (st *Local) MoveStorage(ctx context.Context, s abi.SectorID, spt abi.RegisteredSealProof, types SectorFileType) error { - dest, destIds, err := st.AcquireSector(ctx, s, spt, FTNone, types, PathStorage, AcquireMove) +func (st *Local) MoveStorage(ctx context.Context, s abi.SectorID, ssize abi.SectorSize, types SectorFileType) error { + dest, destIds, err := st.AcquireSector(ctx, s, ssize, FTNone, types, PathStorage, AcquireMove) if err != nil { return xerrors.Errorf("acquire dest storage: %w", err) } - src, srcIds, err := st.AcquireSector(ctx, s, spt, types, FTNone, PathStorage, AcquireMove) + src, srcIds, err := st.AcquireSector(ctx, s, ssize, types, FTNone, PathStorage, AcquireMove) if err != nil { return xerrors.Errorf("acquire src storage: %w", err) } diff --git a/extern/sector-storage/stores/remote.go b/extern/sector-storage/stores/remote.go index b9d241b5f..188fd3be2 100644 --- a/extern/sector-storage/stores/remote.go +++ b/extern/sector-storage/stores/remote.go @@ -58,7 +58,7 @@ func NewRemote(local *Local, index SectorIndex, auth http.Header, fetchLimit int } } -func (r *Remote) AcquireSector(ctx context.Context, s abi.SectorID, spt abi.RegisteredSealProof, existing SectorFileType, allocate SectorFileType, pathType PathType, op AcquireMode) (SectorPaths, SectorPaths, error) { +func (r *Remote) AcquireSector(ctx context.Context, s abi.SectorID, ssize abi.SectorSize, existing SectorFileType, allocate SectorFileType, pathType PathType, op AcquireMode) (SectorPaths, SectorPaths, error) { if existing|allocate != existing^allocate { return SectorPaths{}, SectorPaths{}, xerrors.New("can't both find and allocate a sector") } @@ -90,7 +90,7 @@ func (r *Remote) AcquireSector(ctx context.Context, s abi.SectorID, spt abi.Regi r.fetchLk.Unlock() }() - paths, stores, err := r.local.AcquireSector(ctx, s, spt, existing, allocate, pathType, op) + paths, stores, err := r.local.AcquireSector(ctx, s, ssize, existing, allocate, pathType, op) if err != nil { return SectorPaths{}, SectorPaths{}, xerrors.Errorf("local acquire error: %w", err) } @@ -106,7 +106,7 @@ func (r *Remote) AcquireSector(ctx context.Context, s abi.SectorID, spt abi.Regi } } - apaths, ids, err := r.local.AcquireSector(ctx, s, spt, FTNone, toFetch, pathType, op) + apaths, ids, err := r.local.AcquireSector(ctx, s, ssize, FTNone, toFetch, pathType, op) if err != nil { return SectorPaths{}, SectorPaths{}, xerrors.Errorf("allocate local sector for fetching: %w", err) } @@ -116,7 +116,7 @@ func (r *Remote) AcquireSector(ctx context.Context, s abi.SectorID, spt abi.Regi odt = FsOverheadFinalized } - releaseStorage, err := r.local.Reserve(ctx, s, spt, toFetch, ids, odt) + releaseStorage, err := r.local.Reserve(ctx, s, ssize, toFetch, ids, odt) if err != nil { return SectorPaths{}, SectorPaths{}, xerrors.Errorf("reserving storage space: %w", err) } @@ -281,14 +281,14 @@ func (r *Remote) fetch(ctx context.Context, url, outname string) error { } } -func (r *Remote) MoveStorage(ctx context.Context, s abi.SectorID, spt abi.RegisteredSealProof, types SectorFileType) error { +func (r *Remote) MoveStorage(ctx context.Context, s abi.SectorID, ssize abi.SectorSize, types SectorFileType) error { // Make sure we have the data local - _, _, err := r.AcquireSector(ctx, s, spt, types, FTNone, PathStorage, AcquireMove) + _, _, err := r.AcquireSector(ctx, s, ssize, types, FTNone, PathStorage, AcquireMove) if err != nil { return xerrors.Errorf("acquire src storage (remote): %w", err) } - return r.local.MoveStorage(ctx, s, spt, types) + return r.local.MoveStorage(ctx, s, ssize, types) } func (r *Remote) Remove(ctx context.Context, sid abi.SectorID, typ SectorFileType, force bool) error { diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index d4ed4d64c..80873d70b 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -181,11 +181,6 @@ func (s *WindowPoStScheduler) runSubmitPoST( } func (s *WindowPoStScheduler) checkSectors(ctx context.Context, check bitfield.BitField) (bitfield.BitField, error) { - spt, err := s.proofType.RegisteredSealProof() - if err != nil { - return bitfield.BitField{}, xerrors.Errorf("getting seal proof type: %w", err) - } - mid, err := address.IDFromAddress(s.actor) if err != nil { return bitfield.BitField{}, err @@ -207,7 +202,7 @@ func (s *WindowPoStScheduler) checkSectors(ctx context.Context, check bitfield.B return bitfield.BitField{}, xerrors.Errorf("iterating over bitfield: %w", err) } - bad, err := s.faultTracker.CheckProvable(ctx, spt, tocheck) + bad, err := s.faultTracker.CheckProvable(ctx, s.proofType, tocheck) if err != nil { return bitfield.BitField{}, xerrors.Errorf("checking provable sectors: %w", err) } diff --git a/storage/wdpost_run_test.go b/storage/wdpost_run_test.go index dd7ac4c24..8c145498c 100644 --- a/storage/wdpost_run_test.go +++ b/storage/wdpost_run_test.go @@ -114,7 +114,7 @@ func (m *mockProver) GenerateWindowPoSt(ctx context.Context, aid abi.ActorID, si type mockFaultTracker struct { } -func (m mockFaultTracker) CheckProvable(ctx context.Context, spt abi.RegisteredSealProof, sectors []abi.SectorID) ([]abi.SectorID, error) { +func (m mockFaultTracker) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []abi.SectorID) ([]abi.SectorID, error) { // Returns "bad" sectors so just return nil meaning all sectors are good return nil, nil } From 3a46c15c4ac31a5098c1f01cf752630fcef1b819 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 21 Oct 2020 04:22:49 +0200 Subject: [PATCH 824/885] Disable blockstore bloom filter Signed-off-by: Jakub Sztandera --- lib/blockstore/blockstore.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/blockstore/blockstore.go b/lib/blockstore/blockstore.go index f274cd708..99d849188 100644 --- a/lib/blockstore/blockstore.go +++ b/lib/blockstore/blockstore.go @@ -50,9 +50,16 @@ type GCLocker = blockstore.GCLocker var NewGCLocker = blockstore.NewGCLocker var NewGCBlockstore = blockstore.NewGCBlockstore -var DefaultCacheOpts = blockstore.DefaultCacheOpts var ErrNotFound = blockstore.ErrNotFound +func DefaultCacheOpts() CacheOpts { + return CacheOpts{ + HasBloomFilterSize: 0, + HasBloomFilterHashes: 0, + HasARCCacheSize: 512 << 10, + } +} + func CachedBlockstore(ctx context.Context, bs Blockstore, opts CacheOpts) (Blockstore, error) { bs, err := blockstore.CachedBlockstore(ctx, bs, opts) if err != nil { From 3e767ca5c8357c430f246433cdf2f89008e74554 Mon Sep 17 00:00:00 2001 From: lanzafame Date: Wed, 21 Oct 2020 18:10:27 +1000 Subject: [PATCH 825/885] wrap api in metrics proxy --- cmd/lotus/rpc.go | 3 ++- metrics/metrics.go | 19 ++++++++++++++ metrics/proxy.go | 65 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 metrics/proxy.go diff --git a/cmd/lotus/rpc.go b/cmd/lotus/rpc.go index 9718deb3a..28af47ad7 100644 --- a/cmd/lotus/rpc.go +++ b/cmd/lotus/rpc.go @@ -22,6 +22,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/apistruct" + "github.com/filecoin-project/lotus/metrics" "github.com/filecoin-project/lotus/node" "github.com/filecoin-project/lotus/node/impl" ) @@ -30,7 +31,7 @@ var log = logging.Logger("main") func serveRPC(a api.FullNode, stop node.StopFunc, addr multiaddr.Multiaddr, shutdownCh <-chan struct{}) error { rpcServer := jsonrpc.NewServer() - rpcServer.Register("Filecoin", apistruct.PermissionedFullAPI(a)) + rpcServer.Register("Filecoin", apistruct.PermissionedFullAPI(metrics.MetricedFullAPI(a))) ah := &auth.Handler{ Verify: a.AuthVerify, diff --git a/metrics/metrics.go b/metrics/metrics.go index 5dd865263..8de6afc1e 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -1,6 +1,7 @@ package metrics import ( + "context" "time" "go.opencensus.io/stats" @@ -24,6 +25,8 @@ var ( MessageTo, _ = tag.NewKey("message_to") MessageNonce, _ = tag.NewKey("message_nonce") ReceivedFrom, _ = tag.NewKey("received_from") + Endpoint, _ = tag.NewKey("endpoint") + APIInterface, _ = tag.NewKey("api") // to distinguish between gateway api and full node api endpoint calls ) // Measures @@ -49,6 +52,7 @@ var ( PubsubRecvRPC = stats.Int64("pubsub/recv_rpc", "Counter for total received RPCs", stats.UnitDimensionless) PubsubSendRPC = stats.Int64("pubsub/send_rpc", "Counter for total sent RPCs", stats.UnitDimensionless) PubsubDropRPC = stats.Int64("pubsub/drop_rpc", "Counter for total dropped RPCs", stats.UnitDimensionless) + APIRequestDuration = stats.Float64("api/request_duration_ms", "Duration of API requests", stats.UnitMilliseconds) ) var ( @@ -137,6 +141,11 @@ var ( Measure: PubsubDropRPC, Aggregation: view.Count(), } + APIRequestDurationView = &view.View{ + Measure: APIRequestDuration, + Aggregation: defaultMillisecondsDistribution, + TagKeys: []tag.Key{APIInterface, Endpoint}, + } ) // DefaultViews is an array of OpenCensus views for metric gathering purposes @@ -161,6 +170,7 @@ var DefaultViews = append([]*view.View{ PubsubRecvRPCView, PubsubSendRPCView, PubsubDropRPCView, + APIRequestDurationView, }, rpcmetrics.DefaultViews...) @@ -168,3 +178,12 @@ var DefaultViews = append([]*view.View{ func SinceInMilliseconds(startTime time.Time) float64 { return float64(time.Since(startTime).Nanoseconds()) / 1e6 } + +// Timer is a function stopwatch, calling it starts the timer, +// calling the returned function will record the duration. +func Timer(ctx context.Context, m *stats.Float64Measure) func() { + start := time.Now() + return func() { + stats.Record(ctx, m.M(SinceInMilliseconds(start))) + } +} \ No newline at end of file diff --git a/metrics/proxy.go b/metrics/proxy.go new file mode 100644 index 000000000..cafcaad89 --- /dev/null +++ b/metrics/proxy.go @@ -0,0 +1,65 @@ +package metrics + +import ( + "context" + "reflect" + + "go.opencensus.io/tag" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/api/apistruct" +) + +func MetricedStorMinerAPI(a api.StorageMiner) api.StorageMiner { + var out apistruct.StorageMinerStruct + proxy(a, &out.Internal) + proxy(a, &out.CommonStruct.Internal) + return &out +} + +func MetricedFullAPI(a api.FullNode) api.FullNode { + var out apistruct.FullNodeStruct + proxy(a, &out.Internal) + proxy(a, &out.CommonStruct.Internal) + return &out +} + +func MetricedWorkerAPI(a api.WorkerAPI) api.WorkerAPI { + var out apistruct.WorkerStruct + proxy(a, &out.Internal) + return &out +} + +func MetricedWalletAPI(a api.WalletAPI) api.WalletAPI { + var out apistruct.WalletStruct + proxy(a, &out.Internal) + return &out +} + +func MetricedGatewayAPI(a api.GatewayAPI) api.GatewayAPI { + var out apistruct.GatewayStruct + proxy(a, &out.Internal) + return &out +} + +func proxy(in interface{}, out interface{}) { + rint := reflect.ValueOf(out).Elem() + ra := reflect.ValueOf(in) + + for f := 0; f < rint.NumField(); f++ { + field := rint.Type().Field(f) + fn := ra.MethodByName(field.Name) + + rint.Field(f).Set(reflect.MakeFunc(field.Type, func(args []reflect.Value) (results []reflect.Value) { + ctx := args[0].Interface().(context.Context) + // upsert function name into context + ctx, _ = tag.New(ctx, tag.Upsert(Endpoint, field.Name)) + stop := Timer(ctx, APIRequestDuration) + defer stop() + // pass tagged ctx back into function call + args[0] = reflect.ValueOf(ctx) + return fn.Call(args) + })) + + } +} \ No newline at end of file From 7a33d5bb3424041aa50bb914a16708010fe2e1bc Mon Sep 17 00:00:00 2001 From: lanzafame Date: Wed, 21 Oct 2020 18:37:50 +1000 Subject: [PATCH 826/885] register all apis with metric wrapper --- cmd/lotus-gateway/main.go | 15 ++++++++++++++- cmd/lotus-seal-worker/main.go | 13 ++++++++++++- cmd/lotus-storage-miner/run.go | 21 +++++++++++++++++++-- cmd/lotus-wallet/main.go | 13 ++++++++++++- cmd/lotus/rpc.go | 10 +++++++++- 5 files changed, 66 insertions(+), 6 deletions(-) diff --git a/cmd/lotus-gateway/main.go b/cmd/lotus-gateway/main.go index 34e068ffd..3fed88468 100644 --- a/cmd/lotus-gateway/main.go +++ b/cmd/lotus-gateway/main.go @@ -7,10 +7,15 @@ import ( "os" "github.com/filecoin-project/go-jsonrpc" + "go.opencensus.io/tag" + "github.com/filecoin-project/lotus/build" lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/lib/lotuslog" + "github.com/filecoin-project/lotus/metrics" + logging "github.com/ipfs/go-log" + "go.opencensus.io/stats/view" "github.com/gorilla/mux" "github.com/urfave/cli/v2" @@ -64,6 +69,13 @@ var runCmd = &cli.Command{ ctx, cancel := context.WithCancel(ctx) defer cancel() + // Register all metric views + if err := view.Register( + metrics.DefaultViews..., + ); err != nil { + log.Fatalf("Cannot register the view: %v", err) + } + api, closer, err := lcli.GetFullNodeAPI(cctx) if err != nil { return err @@ -76,7 +88,7 @@ var runCmd = &cli.Command{ log.Info("Setting up API endpoint at " + address) rpcServer := jsonrpc.NewServer() - rpcServer.Register("Filecoin", NewGatewayAPI(api)) + rpcServer.Register("Filecoin", metrics.MetricedGatewayAPI(NewGatewayAPI(api))) mux.Handle("/rpc/v0", rpcServer) mux.PathPrefix("/").Handler(http.DefaultServeMux) @@ -89,6 +101,7 @@ var runCmd = &cli.Command{ srv := &http.Server{ Handler: mux, BaseContext: func(listener net.Listener) context.Context { + ctx, _ := tag.New(context.Background(), tag.Upsert(metrics.APIInterface, "lotus-gateway")) return ctx }, } diff --git a/cmd/lotus-seal-worker/main.go b/cmd/lotus-seal-worker/main.go index d2c57e680..9c2ad6319 100644 --- a/cmd/lotus-seal-worker/main.go +++ b/cmd/lotus-seal-worker/main.go @@ -18,6 +18,8 @@ import ( logging "github.com/ipfs/go-log/v2" manet "github.com/multiformats/go-multiaddr/net" "github.com/urfave/cli/v2" + "go.opencensus.io/stats/view" + "go.opencensus.io/tag" "golang.org/x/xerrors" "github.com/filecoin-project/go-jsonrpc" @@ -34,6 +36,7 @@ import ( "github.com/filecoin-project/lotus/extern/sector-storage/stores" "github.com/filecoin-project/lotus/lib/lotuslog" "github.com/filecoin-project/lotus/lib/rpcenc" + "github.com/filecoin-project/lotus/metrics" "github.com/filecoin-project/lotus/node/repo" ) @@ -190,6 +193,13 @@ var runCmd = &cli.Command{ ctx, cancel := context.WithCancel(ctx) defer cancel() + // Register all metric views + if err := view.Register( + metrics.DefaultViews..., + ); err != nil { + log.Fatalf("Cannot register the view: %v", err) + } + v, err := nodeApi.Version(ctx) if err != nil { return err @@ -363,7 +373,7 @@ var runCmd = &cli.Command{ readerHandler, readerServerOpt := rpcenc.ReaderParamDecoder() rpcServer := jsonrpc.NewServer(readerServerOpt) - rpcServer.Register("Filecoin", apistruct.PermissionedWorkerAPI(workerApi)) + rpcServer.Register("Filecoin", apistruct.PermissionedWorkerAPI(metrics.MetricedWorkerAPI(workerApi))) mux.Handle("/rpc/v0", rpcServer) mux.Handle("/rpc/streams/v0/push/{uuid}", readerHandler) @@ -378,6 +388,7 @@ var runCmd = &cli.Command{ srv := &http.Server{ Handler: ah, BaseContext: func(listener net.Listener) context.Context { + ctx, _ := tag.New(context.Background(), tag.Upsert(metrics.APIInterface, "lotus-worker")) return ctx }, } diff --git a/cmd/lotus-storage-miner/run.go b/cmd/lotus-storage-miner/run.go index c043bd903..0c2fba8b3 100644 --- a/cmd/lotus-storage-miner/run.go +++ b/cmd/lotus-storage-miner/run.go @@ -2,6 +2,7 @@ package main import ( "context" + "net" "net/http" _ "net/http/pprof" "os" @@ -12,6 +13,8 @@ import ( "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr/net" "github.com/urfave/cli/v2" + "go.opencensus.io/stats/view" + "go.opencensus.io/tag" "golang.org/x/xerrors" "github.com/filecoin-project/go-jsonrpc" @@ -22,6 +25,7 @@ import ( "github.com/filecoin-project/lotus/build" lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/lib/ulimit" + "github.com/filecoin-project/lotus/metrics" "github.com/filecoin-project/lotus/node" "github.com/filecoin-project/lotus/node/impl" "github.com/filecoin-project/lotus/node/modules/dtypes" @@ -66,6 +70,13 @@ var runCmd = &cli.Command{ defer ncloser() ctx := lcli.DaemonContext(cctx) + // Register all metric views + if err := view.Register( + metrics.DefaultViews..., + ); err != nil { + log.Fatalf("Cannot register the view: %v", err) + } + v, err := nodeApi.Version(ctx) if err != nil { return err @@ -147,7 +158,7 @@ var runCmd = &cli.Command{ mux := mux.NewRouter() rpcServer := jsonrpc.NewServer() - rpcServer.Register("Filecoin", apistruct.PermissionedStorMinerAPI(minerapi)) + rpcServer.Register("Filecoin", apistruct.PermissionedStorMinerAPI(metrics.MetricedStorMinerAPI(minerapi))) mux.Handle("/rpc/v0", rpcServer) mux.PathPrefix("/remote").HandlerFunc(minerapi.(*impl.StorageMinerAPI).ServeRemote) @@ -158,7 +169,13 @@ var runCmd = &cli.Command{ Next: mux.ServeHTTP, } - srv := &http.Server{Handler: ah} + srv := &http.Server{ + Handler: ah, + BaseContext: func(listener net.Listener) context.Context { + ctx, _ := tag.New(context.Background(), tag.Upsert(metrics.APIInterface, "lotus-miner")) + return ctx + }, + } sigChan := make(chan os.Signal, 2) go func() { diff --git a/cmd/lotus-wallet/main.go b/cmd/lotus-wallet/main.go index 3285b13e7..25b89eb9d 100644 --- a/cmd/lotus-wallet/main.go +++ b/cmd/lotus-wallet/main.go @@ -9,6 +9,8 @@ import ( "github.com/gorilla/mux" logging "github.com/ipfs/go-log/v2" "github.com/urfave/cli/v2" + "go.opencensus.io/stats/view" + "go.opencensus.io/tag" "github.com/filecoin-project/go-jsonrpc" @@ -18,6 +20,7 @@ import ( ledgerwallet "github.com/filecoin-project/lotus/chain/wallet/ledger" lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/lib/lotuslog" + "github.com/filecoin-project/lotus/metrics" "github.com/filecoin-project/lotus/node/repo" ) @@ -75,6 +78,13 @@ var runCmd = &cli.Command{ ctx, cancel := context.WithCancel(ctx) defer cancel() + // Register all metric views + if err := view.Register( + metrics.DefaultViews..., + ); err != nil { + log.Fatalf("Cannot register the view: %v", err) + } + repoPath := cctx.String(FlagWalletRepo) r, err := repo.NewFS(repoPath) if err != nil { @@ -125,7 +135,7 @@ var runCmd = &cli.Command{ log.Info("Setting up API endpoint at " + address) rpcServer := jsonrpc.NewServer() - rpcServer.Register("Filecoin", &LoggedWallet{under: w}) + rpcServer.Register("Filecoin", &LoggedWallet{under: metrics.MetricedWalletAPI(w)}) mux.Handle("/rpc/v0", rpcServer) mux.PathPrefix("/").Handler(http.DefaultServeMux) // pprof @@ -138,6 +148,7 @@ var runCmd = &cli.Command{ srv := &http.Server{ Handler: mux, BaseContext: func(listener net.Listener) context.Context { + ctx, _ := tag.New(context.Background(), tag.Upsert(metrics.APIInterface, "lotus-wallet")) return ctx }, } diff --git a/cmd/lotus/rpc.go b/cmd/lotus/rpc.go index 28af47ad7..4f68ac85a 100644 --- a/cmd/lotus/rpc.go +++ b/cmd/lotus/rpc.go @@ -3,6 +3,7 @@ package main import ( "context" "encoding/json" + "net" "net/http" _ "net/http/pprof" "os" @@ -13,6 +14,7 @@ import ( logging "github.com/ipfs/go-log/v2" "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr/net" + "go.opencensus.io/tag" "golang.org/x/xerrors" "contrib.go.opencensus.io/exporter/prometheus" @@ -61,7 +63,13 @@ func serveRPC(a api.FullNode, stop node.StopFunc, addr multiaddr.Multiaddr, shut return xerrors.Errorf("could not listen: %w", err) } - srv := &http.Server{Handler: http.DefaultServeMux} + srv := &http.Server{ + Handler: http.DefaultServeMux, + BaseContext: func(listener net.Listener) context.Context { + ctx, _ := tag.New(context.Background(), tag.Upsert(metrics.APIInterface, "lotus-daemon")) + return ctx + }, + } sigCh := make(chan os.Signal, 2) shutdownDone := make(chan struct{}) From 7cbd4d49c379a1c30cf6222ab8bd2ceaeb65b406 Mon Sep 17 00:00:00 2001 From: lanzafame Date: Wed, 21 Oct 2020 18:39:57 +1000 Subject: [PATCH 827/885] gofmt --- metrics/metrics.go | 2 +- metrics/proxy.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/metrics/metrics.go b/metrics/metrics.go index 8de6afc1e..33d9e9174 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -186,4 +186,4 @@ func Timer(ctx context.Context, m *stats.Float64Measure) func() { return func() { stats.Record(ctx, m.M(SinceInMilliseconds(start))) } -} \ No newline at end of file +} diff --git a/metrics/proxy.go b/metrics/proxy.go index cafcaad89..f3714ec2e 100644 --- a/metrics/proxy.go +++ b/metrics/proxy.go @@ -62,4 +62,4 @@ func proxy(in interface{}, out interface{}) { })) } -} \ No newline at end of file +} From f0d966a5954158baa53a6666e9ebcb192f3621e1 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 21 Oct 2020 19:40:51 +0200 Subject: [PATCH 828/885] Fix lotus-shed ledger list Signed-off-by: Jakub Sztandera --- cmd/lotus-shed/ledger.go | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/cmd/lotus-shed/ledger.go b/cmd/lotus-shed/ledger.go index f64916534..ecb13ec64 100644 --- a/cmd/lotus-shed/ledger.go +++ b/cmd/lotus-shed/ledger.go @@ -7,9 +7,9 @@ import ( "strings" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/big" "github.com/urfave/cli/v2" ledgerfil "github.com/whyrusleeping/ledger-filecoin-go" - "golang.org/x/xerrors" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" @@ -76,15 +76,22 @@ var ledgerListAddressesCmd = &cli.Command{ } if cctx.Bool("print-balances") && api != nil { // api check makes linter happier - b, err := api.WalletBalance(ctx, addr) + a, err := api.StateGetActor(ctx, addr, types.EmptyTSK) if err != nil { - return xerrors.Errorf("getting balance: %w", err) - } - if !b.IsZero() { - end = i + 21 // BIP32 spec, stop after 20 empty addresses + if strings.Contains(err.Error(), "actor not found") { + a = nil + } else { + return err + } } - fmt.Printf("%s %s %s\n", addr, printHDPath(p), types.FIL(b)) + balance := big.Zero() + if a != nil { + balance = a.Balance + end = i + 20 + 1 + } + + fmt.Printf("%s %s %s\n", addr, printHDPath(p), types.FIL(balance)) } else { fmt.Printf("%s %s\n", addr, printHDPath(p)) } From 4e730b5ec89c068cacf9bb1f613dd2d38609c545 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 7 Oct 2020 18:09:33 -0700 Subject: [PATCH 829/885] port to v2 imports --- api/test/paych.go | 9 +- build/params_mainnet.go | 4 +- build/params_shared_funcs.go | 23 ---- build/params_shared_vals.go | 12 +- build/params_testground.go | 9 +- chain/actors/adt/diff_adt_test.go | 73 +++++----- chain/actors/builtin/builtin.go | 10 ++ chain/actors/policy/policy.go | 36 +++++ chain/actors/policy/policy_test.go | 20 +++ chain/events/state/predicates_test.go | 130 ++++++++++-------- chain/gen/gen.go | 19 ++- chain/gen/genesis/miners.go | 27 ++-- chain/gen/mining.go | 6 +- chain/market/fundmgr.go | 4 +- chain/market/fundmgr_test.go | 6 +- chain/messagepool/messagepool_test.go | 20 +-- chain/messagepool/repub_test.go | 8 +- chain/messagepool/selection_test.go | 27 ++-- chain/state/statetree_test.go | 28 ++-- chain/stmgr/forks.go | 18 +-- chain/stmgr/forks_test.go | 21 +-- chain/stmgr/stmgr.go | 53 ++++--- chain/stmgr/utils.go | 8 +- chain/store/store.go | 8 +- chain/sub/incoming.go | 8 +- chain/sync.go | 22 +-- chain/sync_test.go | 8 +- chain/types/blockheader.go | 4 +- chain/types/blockheader_test.go | 4 +- chain/types/message_test.go | 28 ++-- chain/vm/gas.go | 18 +-- chain/vm/gas_v0.go | 12 +- chain/vm/invoker_test.go | 13 +- chain/vm/mkactor.go | 3 +- chain/vm/runtime.go | 4 +- chain/vm/syscalls.go | 34 ++--- cli/client.go | 2 +- cli/multisig.go | 16 +-- cmd/lotus-bench/caching_verifier.go | 9 +- cmd/lotus-bench/main.go | 24 ++-- cmd/lotus-chainwatch/processor/processor.go | 18 +-- cmd/lotus-gateway/endtoend_test.go | 10 +- cmd/lotus-pcr/main.go | 15 +- cmd/lotus-seed/seed/seed.go | 5 +- cmd/lotus-shed/mempool-stats.go | 4 +- cmd/lotus-shed/proofs.go | 4 +- cmd/lotus-shed/sectors.go | 4 +- cmd/lotus-shed/verifreg.go | 12 +- cmd/lotus-storage-miner/actor.go | 21 ++- cmd/lotus-storage-miner/init.go | 26 ++-- cmd/lotus-storage-miner/sectors.go | 5 +- cmd/tvx/extract.go | 3 +- conformance/chaos/actor.go | 49 +++---- conformance/chaos/actor_test.go | 90 ++++++------ conformance/chaos/cbor_gen.go | 4 +- .../sector-storage/ffiwrapper/sealer_test.go | 10 +- extern/sector-storage/ffiwrapper/types.go | 8 +- .../sector-storage/ffiwrapper/verifier_cgo.go | 14 +- extern/sector-storage/mock/mock.go | 20 +-- extern/storage-sealing/checks.go | 4 +- extern/storage-sealing/states_sealing.go | 7 +- extern/storage-sealing/types.go | 4 +- extern/storage-sealing/types_test.go | 4 +- genesis/types.go | 5 +- markets/storageadapter/client.go | 32 ++--- markets/storageadapter/provider.go | 16 +-- miner/miner.go | 4 +- node/impl/full/gas.go | 4 +- node/impl/full/multisig.go | 24 ++-- node/modules/storageminer.go | 2 +- node/test/builder.go | 19 +-- paychmgr/paych_test.go | 12 +- paychmgr/paychvoucherfunds_test.go | 26 ++-- paychmgr/settler/settler.go | 4 +- storage/adapter_storage_miner.go | 8 +- storage/miner.go | 11 +- storage/mockstorage/preseal.go | 4 +- storage/wdpost_run.go | 39 +++--- storage/wdpost_run_test.go | 42 +++--- 79 files changed, 724 insertions(+), 657 deletions(-) diff --git a/api/test/paych.go b/api/test/paych.go index 43401554d..69def18bb 100644 --- a/api/test/paych.go +++ b/api/test/paych.go @@ -8,9 +8,6 @@ import ( "testing" "time" - builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" - paych0 "github.com/filecoin-project/specs-actors/actors/builtin/paych" - "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/ipfs/go-cid" @@ -22,7 +19,9 @@ import ( "github.com/filecoin-project/lotus/api/apibstore" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/builtin/paych" + "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/events" "github.com/filecoin-project/lotus/chain/events/state" "github.com/filecoin-project/lotus/chain/types" @@ -227,7 +226,7 @@ func TestPaymentChannels(t *testing.T, b APIBuilder, blocktime time.Duration) { } // wait for the settlement period to pass before collecting - waitForBlocks(ctx, t, bm, paymentReceiver, receiverAddr, paych0.SettleDelay) + waitForBlocks(ctx, t, bm, paymentReceiver, receiverAddr, policy.PaychSettleDelay) creatorPreCollectBalance, err := paymentCreator.WalletBalance(ctx, createrAddr) if err != nil { @@ -283,7 +282,7 @@ func waitForBlocks(ctx context.Context, t *testing.T, bm *BlockMiner, paymentRec // Add a real block m, err := paymentReceiver.MpoolPushMessage(ctx, &types.Message{ - To: builtin0.BurntFundsActorAddr, + To: builtin.BurntFundsActorAddr, From: receiverAddr, Value: types.NewInt(0), }, nil) diff --git a/build/params_mainnet.go b/build/params_mainnet.go index 71e96e614..94deedfec 100644 --- a/build/params_mainnet.go +++ b/build/params_mainnet.go @@ -12,7 +12,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/actors/policy" - builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" ) var DrandSchedule = map[abi.ChainEpoch]DrandEnum{ @@ -57,6 +57,6 @@ func init() { Devnet = false } -const BlockDelaySecs = uint64(builtin0.EpochDurationSeconds) +const BlockDelaySecs = uint64(builtin2.EpochDurationSeconds) const PropagationDelaySecs = uint64(6) diff --git a/build/params_shared_funcs.go b/build/params_shared_funcs.go index 40ccca50b..c48b3c507 100644 --- a/build/params_shared_funcs.go +++ b/build/params_shared_funcs.go @@ -1,36 +1,13 @@ package build import ( - "sort" - "github.com/filecoin-project/go-address" "github.com/libp2p/go-libp2p-core/protocol" - "github.com/filecoin-project/go-state-types/abi" - miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" - "github.com/filecoin-project/lotus/node/modules/dtypes" ) -func DefaultSectorSize() abi.SectorSize { - szs := make([]abi.SectorSize, 0, len(miner0.SupportedProofTypes)) - for spt := range miner0.SupportedProofTypes { - ss, err := spt.SectorSize() - if err != nil { - panic(err) - } - - szs = append(szs, ss) - } - - sort.Slice(szs, func(i, j int) bool { - return szs[i] < szs[j] - }) - - return szs[0] -} - // Core network constants func BlocksTopic(netName dtypes.NetworkName) string { return "/fil/blocks/" + string(netName) } diff --git a/build/params_shared_vals.go b/build/params_shared_vals.go index b80472594..5070777bd 100644 --- a/build/params_shared_vals.go +++ b/build/params_shared_vals.go @@ -7,12 +7,12 @@ import ( "os" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/actors/policy" - + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/network" - "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/specs-actors/actors/builtin" + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + + "github.com/filecoin-project/lotus/chain/actors/policy" ) // ///// @@ -32,7 +32,7 @@ const ActorUpgradeNetworkVersion = network.Version4 const ForkLengthThreshold = Finality // Blocks (e) -var BlocksPerEpoch = uint64(builtin.ExpectedLeadersPerEpoch) +var BlocksPerEpoch = uint64(builtin2.ExpectedLeadersPerEpoch) // Epochs const Finality = policy.ChainFinality @@ -116,4 +116,4 @@ const PackingEfficiencyDenom = 5 // Actor consts // TODO: Pull from actors when its made not private -var MinDealDuration = abi.ChainEpoch(180 * builtin.EpochsInDay) +var MinDealDuration = abi.ChainEpoch(180 * builtin2.EpochsInDay) diff --git a/build/params_testground.go b/build/params_testground.go index beee1c727..d9893a5f5 100644 --- a/build/params_testground.go +++ b/build/params_testground.go @@ -12,7 +12,8 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/network" - "github.com/filecoin-project/specs-actors/actors/builtin" + + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/policy" ) @@ -21,14 +22,14 @@ var ( UnixfsChunkSize = uint64(1 << 20) UnixfsLinksPerLevel = 1024 - BlocksPerEpoch = uint64(builtin.ExpectedLeadersPerEpoch) + BlocksPerEpoch = uint64(builtin2.ExpectedLeadersPerEpoch) BlockMessageLimit = 512 BlockGasLimit = int64(100_000_000_000) BlockGasTarget = int64(BlockGasLimit / 2) BaseFeeMaxChangeDenom = int64(8) // 12.5% InitialBaseFee = int64(100e6) MinimumBaseFee = int64(100) - BlockDelaySecs = uint64(builtin.EpochDurationSeconds) + BlockDelaySecs = uint64(builtin2.EpochDurationSeconds) PropagationDelaySecs = uint64(6) AllowableClockDriftSecs = uint64(1) @@ -72,7 +73,7 @@ var ( // Actor consts // TODO: Pull from actors when its made not private - MinDealDuration = abi.ChainEpoch(180 * builtin.EpochsInDay) + MinDealDuration = abi.ChainEpoch(180 * builtin2.EpochsInDay) PackingEfficiencyNum int64 = 4 PackingEfficiencyDenom int64 = 5 diff --git a/chain/actors/adt/diff_adt_test.go b/chain/actors/adt/diff_adt_test.go index 1c0726003..a187c9f35 100644 --- a/chain/actors/adt/diff_adt_test.go +++ b/chain/actors/adt/diff_adt_test.go @@ -12,8 +12,9 @@ import ( typegen "github.com/whyrusleeping/cbor-gen" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/specs-actors/actors/runtime" - adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" + + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + adt2 "github.com/filecoin-project/specs-actors/v2/actors/util/adt" bstore "github.com/filecoin-project/lotus/lib/blockstore" ) @@ -22,24 +23,24 @@ func TestDiffAdtArray(t *testing.T) { ctxstoreA := newContextStore() ctxstoreB := newContextStore() - arrA := adt0.MakeEmptyArray(ctxstoreA) - arrB := adt0.MakeEmptyArray(ctxstoreB) + arrA := adt2.MakeEmptyArray(ctxstoreA) + arrB := adt2.MakeEmptyArray(ctxstoreB) - require.NoError(t, arrA.Set(0, runtime.CBORBytes([]byte{0}))) // delete + require.NoError(t, arrA.Set(0, builtin2.CBORBytes([]byte{0}))) // delete - require.NoError(t, arrA.Set(1, runtime.CBORBytes([]byte{0}))) // modify - require.NoError(t, arrB.Set(1, runtime.CBORBytes([]byte{1}))) + require.NoError(t, arrA.Set(1, builtin2.CBORBytes([]byte{0}))) // modify + require.NoError(t, arrB.Set(1, builtin2.CBORBytes([]byte{1}))) - require.NoError(t, arrA.Set(2, runtime.CBORBytes([]byte{1}))) // delete + require.NoError(t, arrA.Set(2, builtin2.CBORBytes([]byte{1}))) // delete - require.NoError(t, arrA.Set(3, runtime.CBORBytes([]byte{0}))) // noop - require.NoError(t, arrB.Set(3, runtime.CBORBytes([]byte{0}))) + require.NoError(t, arrA.Set(3, builtin2.CBORBytes([]byte{0}))) // noop + require.NoError(t, arrB.Set(3, builtin2.CBORBytes([]byte{0}))) - require.NoError(t, arrA.Set(4, runtime.CBORBytes([]byte{0}))) // modify - require.NoError(t, arrB.Set(4, runtime.CBORBytes([]byte{6}))) + require.NoError(t, arrA.Set(4, builtin2.CBORBytes([]byte{0}))) // modify + require.NoError(t, arrB.Set(4, builtin2.CBORBytes([]byte{6}))) - require.NoError(t, arrB.Set(5, runtime.CBORBytes{8})) // add - require.NoError(t, arrB.Set(6, runtime.CBORBytes{9})) // add + require.NoError(t, arrB.Set(5, builtin2.CBORBytes{8})) // add + require.NoError(t, arrB.Set(6, builtin2.CBORBytes{9})) // add changes := new(TestDiffArray) @@ -76,24 +77,24 @@ func TestDiffAdtMap(t *testing.T) { ctxstoreA := newContextStore() ctxstoreB := newContextStore() - mapA := adt0.MakeEmptyMap(ctxstoreA) - mapB := adt0.MakeEmptyMap(ctxstoreB) + mapA := adt2.MakeEmptyMap(ctxstoreA) + mapB := adt2.MakeEmptyMap(ctxstoreB) - require.NoError(t, mapA.Put(abi.UIntKey(0), runtime.CBORBytes([]byte{0}))) // delete + require.NoError(t, mapA.Put(abi.UIntKey(0), builtin2.CBORBytes([]byte{0}))) // delete - require.NoError(t, mapA.Put(abi.UIntKey(1), runtime.CBORBytes([]byte{0}))) // modify - require.NoError(t, mapB.Put(abi.UIntKey(1), runtime.CBORBytes([]byte{1}))) + require.NoError(t, mapA.Put(abi.UIntKey(1), builtin2.CBORBytes([]byte{0}))) // modify + require.NoError(t, mapB.Put(abi.UIntKey(1), builtin2.CBORBytes([]byte{1}))) - require.NoError(t, mapA.Put(abi.UIntKey(2), runtime.CBORBytes([]byte{1}))) // delete + require.NoError(t, mapA.Put(abi.UIntKey(2), builtin2.CBORBytes([]byte{1}))) // delete - require.NoError(t, mapA.Put(abi.UIntKey(3), runtime.CBORBytes([]byte{0}))) // noop - require.NoError(t, mapB.Put(abi.UIntKey(3), runtime.CBORBytes([]byte{0}))) + require.NoError(t, mapA.Put(abi.UIntKey(3), builtin2.CBORBytes([]byte{0}))) // noop + require.NoError(t, mapB.Put(abi.UIntKey(3), builtin2.CBORBytes([]byte{0}))) - require.NoError(t, mapA.Put(abi.UIntKey(4), runtime.CBORBytes([]byte{0}))) // modify - require.NoError(t, mapB.Put(abi.UIntKey(4), runtime.CBORBytes([]byte{6}))) + require.NoError(t, mapA.Put(abi.UIntKey(4), builtin2.CBORBytes([]byte{0}))) // modify + require.NoError(t, mapB.Put(abi.UIntKey(4), builtin2.CBORBytes([]byte{6}))) - require.NoError(t, mapB.Put(abi.UIntKey(5), runtime.CBORBytes{8})) // add - require.NoError(t, mapB.Put(abi.UIntKey(6), runtime.CBORBytes{9})) // add + require.NoError(t, mapB.Put(abi.UIntKey(5), builtin2.CBORBytes{8})) // add + require.NoError(t, mapB.Put(abi.UIntKey(6), builtin2.CBORBytes{9})) // add changes := new(TestDiffMap) @@ -144,7 +145,7 @@ func (t *TestDiffMap) AsKey(key string) (abi.Keyer, error) { } func (t *TestDiffMap) Add(key string, val *typegen.Deferred) error { - v := new(runtime.CBORBytes) + v := new(builtin2.CBORBytes) err := v.UnmarshalCBOR(bytes.NewReader(val.Raw)) if err != nil { return err @@ -161,13 +162,13 @@ func (t *TestDiffMap) Add(key string, val *typegen.Deferred) error { } func (t *TestDiffMap) Modify(key string, from, to *typegen.Deferred) error { - vFrom := new(runtime.CBORBytes) + vFrom := new(builtin2.CBORBytes) err := vFrom.UnmarshalCBOR(bytes.NewReader(from.Raw)) if err != nil { return err } - vTo := new(runtime.CBORBytes) + vTo := new(builtin2.CBORBytes) err = vTo.UnmarshalCBOR(bytes.NewReader(to.Raw)) if err != nil { return err @@ -194,7 +195,7 @@ func (t *TestDiffMap) Modify(key string, from, to *typegen.Deferred) error { } func (t *TestDiffMap) Remove(key string, val *typegen.Deferred) error { - v := new(runtime.CBORBytes) + v := new(builtin2.CBORBytes) err := v.UnmarshalCBOR(bytes.NewReader(val.Raw)) if err != nil { return err @@ -212,7 +213,7 @@ func (t *TestDiffMap) Remove(key string, val *typegen.Deferred) error { type adtMapDiffResult struct { key uint64 - val runtime.CBORBytes + val builtin2.CBORBytes } type TestAdtMapDiffModified struct { @@ -222,7 +223,7 @@ type TestAdtMapDiffModified struct { type adtArrayDiffResult struct { key uint64 - val runtime.CBORBytes + val builtin2.CBORBytes } type TestDiffArray struct { @@ -239,7 +240,7 @@ type TestAdtArrayDiffModified struct { } func (t *TestDiffArray) Add(key uint64, val *typegen.Deferred) error { - v := new(runtime.CBORBytes) + v := new(builtin2.CBORBytes) err := v.UnmarshalCBOR(bytes.NewReader(val.Raw)) if err != nil { return err @@ -252,13 +253,13 @@ func (t *TestDiffArray) Add(key uint64, val *typegen.Deferred) error { } func (t *TestDiffArray) Modify(key uint64, from, to *typegen.Deferred) error { - vFrom := new(runtime.CBORBytes) + vFrom := new(builtin2.CBORBytes) err := vFrom.UnmarshalCBOR(bytes.NewReader(from.Raw)) if err != nil { return err } - vTo := new(runtime.CBORBytes) + vTo := new(builtin2.CBORBytes) err = vTo.UnmarshalCBOR(bytes.NewReader(to.Raw)) if err != nil { return err @@ -280,7 +281,7 @@ func (t *TestDiffArray) Modify(key uint64, from, to *typegen.Deferred) error { } func (t *TestDiffArray) Remove(key uint64, val *typegen.Deferred) error { - v := new(runtime.CBORBytes) + v := new(builtin2.CBORBytes) err := v.UnmarshalCBOR(bytes.NewReader(val.Raw)) if err != nil { return err diff --git a/chain/actors/builtin/builtin.go b/chain/actors/builtin/builtin.go index 350b3a9a6..52d1375b4 100644 --- a/chain/actors/builtin/builtin.go +++ b/chain/actors/builtin/builtin.go @@ -27,6 +27,16 @@ var SaftAddress = makeAddress("t0122") var ReserveAddress = makeAddress("t090") var RootVerifierAddress = makeAddress("t080") +var ( + ExpectedLeadersPerEpoch = builtin0.ExpectedLeadersPerEpoch +) + +const ( + EpochDurationSeconds = builtin0.EpochDurationSeconds + EpochsInDay = builtin0.EpochsInDay + SecondsInDay = builtin0.SecondsInDay +) + // TODO: Why does actors have 2 different versions of this? type SectorInfo = proof0.SectorInfo type PoStProof = proof0.PoStProof diff --git a/chain/actors/policy/policy.go b/chain/actors/policy/policy.go index ba09e4424..c1a971db5 100644 --- a/chain/actors/policy/policy.go +++ b/chain/actors/policy/policy.go @@ -1,6 +1,8 @@ package policy import ( + "sort" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/chain/actors" @@ -11,12 +13,14 @@ import ( builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" market2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/market" miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" + paych2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/paych" verifreg2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/verifreg" ) const ( ChainFinality = miner0.ChainFinality SealRandomnessLookback = ChainFinality + PaychSettleDelay = paych2.SettleDelay ) // SetSupportedProofTypes sets supported proof types, across all actor versions. @@ -114,3 +118,35 @@ func GetWinningPoStSectorSetLookback(nwVer network.Version) abi.ChainEpoch { return ChainFinality } + +func GetMaxSectorExpirationExtension() abi.ChainEpoch { + return miner0.MaxSectorExpirationExtension +} + +// TODO: we'll probably need to abstract over this better in the future. +func GetMaxPoStPartitions(p abi.RegisteredPoStProof) (int, error) { + sectorsPerPart, err := builtin2.PoStProofWindowPoStPartitionSectors(p) + if err != nil { + return 0, err + } + return int(miner2.AddressedSectorsMax / sectorsPerPart), nil +} + +func GetDefaultSectorSize() abi.SectorSize { + // supported proof types are the same across versions. + szs := make([]abi.SectorSize, 0, len(miner2.SupportedProofTypes)) + for spt := range miner2.SupportedProofTypes { + ss, err := spt.SectorSize() + if err != nil { + panic(err) + } + + szs = append(szs, ss) + } + + sort.Slice(szs, func(i, j int) bool { + return szs[i] < szs[j] + }) + + return szs[0] +} diff --git a/chain/actors/policy/policy_test.go b/chain/actors/policy/policy_test.go index 62e7f8964..af600cc75 100644 --- a/chain/actors/policy/policy_test.go +++ b/chain/actors/policy/policy_test.go @@ -6,9 +6,13 @@ import ( "github.com/stretchr/testify/require" "github.com/filecoin-project/go-state-types/abi" + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" + paych0 "github.com/filecoin-project/specs-actors/actors/builtin/paych" verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" + paych2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/paych" verifreg2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/verifreg" ) @@ -42,9 +46,25 @@ func TestSupportedProofTypes(t *testing.T) { func TestAssumptions(t *testing.T) { require.EqualValues(t, miner0.SupportedProofTypes, miner2.SupportedProofTypes) require.Equal(t, miner0.PreCommitChallengeDelay, miner2.PreCommitChallengeDelay) + require.Equal(t, miner0.MaxSectorExpirationExtension, miner2.MaxSectorExpirationExtension) require.Equal(t, miner0.ChainFinality, miner2.ChainFinality) require.Equal(t, miner0.WPoStChallengeWindow, miner2.WPoStChallengeWindow) require.Equal(t, miner0.WPoStProvingPeriod, miner2.WPoStProvingPeriod) require.Equal(t, miner0.WPoStPeriodDeadlines, miner2.WPoStPeriodDeadlines) + require.Equal(t, miner0.AddressedSectorsMax, miner2.AddressedSectorsMax) + require.Equal(t, paych0.SettleDelay, paych2.SettleDelay) require.True(t, verifreg0.MinVerifiedDealSize.Equals(verifreg2.MinVerifiedDealSize)) } + +func TestPartitionSizes(t *testing.T) { + for p := range abi.PoStSealProofTypes { + sizeNew, err := builtin2.PoStProofWindowPoStPartitionSectors(p) + require.NoError(t, err) + sizeOld, err := builtin0.PoStProofWindowPoStPartitionSectors(p) + if err != nil { + // new proof type. + continue + } + require.Equal(t, sizeOld, sizeNew) + } +} diff --git a/chain/events/state/predicates_test.go b/chain/events/state/predicates_test.go index 461ac4997..9b393f6e4 100644 --- a/chain/events/state/predicates_test.go +++ b/chain/events/state/predicates_test.go @@ -18,14 +18,14 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" + + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + market2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/market" + miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" + adt2 "github.com/filecoin-project/specs-actors/v2/actors/util/adt" + tutils "github.com/filecoin-project/specs-actors/v2/support/testing" + "github.com/filecoin-project/lotus/chain/actors/builtin/market" - builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" - market0 "github.com/filecoin-project/specs-actors/actors/builtin/market" - - miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" - "github.com/filecoin-project/specs-actors/actors/util/adt" - tutils "github.com/filecoin-project/specs-actors/support/testing" - "github.com/filecoin-project/lotus/chain/types" bstore "github.com/filecoin-project/lotus/lib/blockstore" ) @@ -72,24 +72,24 @@ func (m mockAPI) setActor(tsk types.TipSetKey, act *types.Actor) { func TestMarketPredicates(t *testing.T) { ctx := context.Background() bs := bstore.NewTemporarySync() - store := adt.WrapStore(ctx, cbornode.NewCborStore(bs)) + store := adt2.WrapStore(ctx, cbornode.NewCborStore(bs)) - oldDeal1 := &market0.DealState{ + oldDeal1 := &market2.DealState{ SectorStartEpoch: 1, LastUpdatedEpoch: 2, SlashEpoch: 0, } - oldDeal2 := &market0.DealState{ + oldDeal2 := &market2.DealState{ SectorStartEpoch: 4, LastUpdatedEpoch: 5, SlashEpoch: 0, } - oldDeals := map[abi.DealID]*market0.DealState{ + oldDeals := map[abi.DealID]*market2.DealState{ abi.DealID(1): oldDeal1, abi.DealID(2): oldDeal2, } - oldProp1 := &market0.DealProposal{ + oldProp1 := &market2.DealProposal{ PieceCID: dummyCid, PieceSize: 0, VerifiedDeal: false, @@ -101,7 +101,7 @@ func TestMarketPredicates(t *testing.T) { ProviderCollateral: big.Zero(), ClientCollateral: big.Zero(), } - oldProp2 := &market0.DealProposal{ + oldProp2 := &market2.DealProposal{ PieceCID: dummyCid, PieceSize: 0, VerifiedDeal: false, @@ -113,7 +113,7 @@ func TestMarketPredicates(t *testing.T) { ProviderCollateral: big.Zero(), ClientCollateral: big.Zero(), } - oldProps := map[abi.DealID]*market0.DealProposal{ + oldProps := map[abi.DealID]*market2.DealProposal{ abi.DealID(1): oldProp1, abi.DealID(2): oldProp2, } @@ -127,7 +127,7 @@ func TestMarketPredicates(t *testing.T) { oldStateC := createMarketState(ctx, t, store, oldDeals, oldProps, oldBalances) - newDeal1 := &market0.DealState{ + newDeal1 := &market2.DealState{ SectorStartEpoch: 1, LastUpdatedEpoch: 3, SlashEpoch: 0, @@ -136,19 +136,19 @@ func TestMarketPredicates(t *testing.T) { // deal 2 removed // added - newDeal3 := &market0.DealState{ + newDeal3 := &market2.DealState{ SectorStartEpoch: 1, LastUpdatedEpoch: 2, SlashEpoch: 3, } - newDeals := map[abi.DealID]*market0.DealState{ + newDeals := map[abi.DealID]*market2.DealState{ abi.DealID(1): newDeal1, // deal 2 was removed abi.DealID(3): newDeal3, } // added - newProp3 := &market0.DealProposal{ + newProp3 := &market2.DealProposal{ PieceCID: dummyCid, PieceSize: 0, VerifiedDeal: false, @@ -160,7 +160,7 @@ func TestMarketPredicates(t *testing.T) { ProviderCollateral: big.Zero(), ClientCollateral: big.Zero(), } - newProps := map[abi.DealID]*market0.DealProposal{ + newProps := map[abi.DealID]*market2.DealProposal{ abi.DealID(1): oldProp1, // 1 was persisted // prop 2 was removed abi.DealID(3): newProp3, // new @@ -183,8 +183,8 @@ func TestMarketPredicates(t *testing.T) { require.NoError(t, err) api := newMockAPI(bs) - api.setActor(oldState.Key(), &types.Actor{Code: builtin0.StorageMarketActorCodeID, Head: oldStateC}) - api.setActor(newState.Key(), &types.Actor{Code: builtin0.StorageMarketActorCodeID, Head: newStateC}) + api.setActor(oldState.Key(), &types.Actor{Code: builtin2.StorageMarketActorCodeID, Head: oldStateC}) + api.setActor(newState.Key(), &types.Actor{Code: builtin2.StorageMarketActorCodeID, Head: newStateC}) t.Run("deal ID predicate", func(t *testing.T) { preds := NewStatePredicates(api) @@ -243,7 +243,7 @@ func TestMarketPredicates(t *testing.T) { marketCid, err := store.Put(ctx, marketState0) require.NoError(t, err) marketState, err := market.Load(store, &types.Actor{ - Code: builtin0.StorageMarketActorCodeID, + Code: builtin2.StorageMarketActorCodeID, Head: marketCid, }) require.NoError(t, err) @@ -356,7 +356,7 @@ func TestMarketPredicates(t *testing.T) { marketCid, err := store.Put(ctx, marketState0) require.NoError(t, err) marketState, err := market.Load(store, &types.Actor{ - Code: builtin0.StorageMarketActorCodeID, + Code: builtin2.StorageMarketActorCodeID, Head: marketCid, }) require.NoError(t, err) @@ -370,7 +370,7 @@ func TestMarketPredicates(t *testing.T) { func TestMinerSectorChange(t *testing.T) { ctx := context.Background() bs := bstore.NewTemporarySync() - store := adt.WrapStore(ctx, cbornode.NewCborStore(bs)) + store := adt2.WrapStore(ctx, cbornode.NewCborStore(bs)) nextID := uint64(0) nextIDAddrF := func() address.Address { @@ -379,12 +379,12 @@ func TestMinerSectorChange(t *testing.T) { } owner, worker := nextIDAddrF(), nextIDAddrF() - si0 := newSectorOnChainInfo(0, tutils.MakeCID("0", &miner0.SealedCIDPrefix), big.NewInt(0), abi.ChainEpoch(0), abi.ChainEpoch(10)) - si1 := newSectorOnChainInfo(1, tutils.MakeCID("1", &miner0.SealedCIDPrefix), big.NewInt(1), abi.ChainEpoch(1), abi.ChainEpoch(11)) - si2 := newSectorOnChainInfo(2, tutils.MakeCID("2", &miner0.SealedCIDPrefix), big.NewInt(2), abi.ChainEpoch(2), abi.ChainEpoch(11)) + si0 := newSectorOnChainInfo(0, tutils.MakeCID("0", &miner2.SealedCIDPrefix), big.NewInt(0), abi.ChainEpoch(0), abi.ChainEpoch(10)) + si1 := newSectorOnChainInfo(1, tutils.MakeCID("1", &miner2.SealedCIDPrefix), big.NewInt(1), abi.ChainEpoch(1), abi.ChainEpoch(11)) + si2 := newSectorOnChainInfo(2, tutils.MakeCID("2", &miner2.SealedCIDPrefix), big.NewInt(2), abi.ChainEpoch(2), abi.ChainEpoch(11)) oldMinerC := createMinerState(ctx, t, store, owner, worker, []miner.SectorOnChainInfo{si0, si1, si2}) - si3 := newSectorOnChainInfo(3, tutils.MakeCID("3", &miner0.SealedCIDPrefix), big.NewInt(3), abi.ChainEpoch(3), abi.ChainEpoch(12)) + si3 := newSectorOnChainInfo(3, tutils.MakeCID("3", &miner2.SealedCIDPrefix), big.NewInt(3), abi.ChainEpoch(3), abi.ChainEpoch(12)) // 0 delete // 1 extend // 2 same @@ -400,8 +400,8 @@ func TestMinerSectorChange(t *testing.T) { require.NoError(t, err) api := newMockAPI(bs) - api.setActor(oldState.Key(), &types.Actor{Head: oldMinerC, Code: builtin0.StorageMinerActorCodeID}) - api.setActor(newState.Key(), &types.Actor{Head: newMinerC, Code: builtin0.StorageMinerActorCodeID}) + api.setActor(oldState.Key(), &types.Actor{Head: oldMinerC, Code: builtin2.StorageMinerActorCodeID}) + api.setActor(newState.Key(), &types.Actor{Head: newMinerC, Code: builtin2.StorageMinerActorCodeID}) preds := NewStatePredicates(api) @@ -467,7 +467,7 @@ type balance struct { locked abi.TokenAmount } -func createMarketState(ctx context.Context, t *testing.T, store adt.Store, deals map[abi.DealID]*market0.DealState, props map[abi.DealID]*market0.DealProposal, balances map[address.Address]balance) cid.Cid { +func createMarketState(ctx context.Context, t *testing.T, store adt2.Store, deals map[abi.DealID]*market2.DealState, props map[abi.DealID]*market2.DealProposal, balances map[address.Address]balance) cid.Cid { dealRootCid := createDealAMT(ctx, t, store, deals) propRootCid := createProposalAMT(ctx, t, store, props) balancesCids := createBalanceTable(ctx, t, store, balances) @@ -482,16 +482,16 @@ func createMarketState(ctx context.Context, t *testing.T, store adt.Store, deals return stateC } -func createEmptyMarketState(t *testing.T, store adt.Store) *market0.State { - emptyArrayCid, err := adt.MakeEmptyArray(store).Root() +func createEmptyMarketState(t *testing.T, store adt2.Store) *market2.State { + emptyArrayCid, err := adt2.MakeEmptyArray(store).Root() require.NoError(t, err) - emptyMap, err := adt.MakeEmptyMap(store).Root() + emptyMap, err := adt2.MakeEmptyMap(store).Root() require.NoError(t, err) - return market0.ConstructState(emptyArrayCid, emptyMap, emptyMap) + return market2.ConstructState(emptyArrayCid, emptyMap, emptyMap) } -func createDealAMT(ctx context.Context, t *testing.T, store adt.Store, deals map[abi.DealID]*market0.DealState) cid.Cid { - root := adt.MakeEmptyArray(store) +func createDealAMT(ctx context.Context, t *testing.T, store adt2.Store, deals map[abi.DealID]*market2.DealState) cid.Cid { + root := adt2.MakeEmptyArray(store) for dealID, dealState := range deals { err := root.Set(uint64(dealID), dealState) require.NoError(t, err) @@ -501,8 +501,8 @@ func createDealAMT(ctx context.Context, t *testing.T, store adt.Store, deals map return rootCid } -func createProposalAMT(ctx context.Context, t *testing.T, store adt.Store, props map[abi.DealID]*market0.DealProposal) cid.Cid { - root := adt.MakeEmptyArray(store) +func createProposalAMT(ctx context.Context, t *testing.T, store adt2.Store, props map[abi.DealID]*market2.DealProposal) cid.Cid { + root := adt2.MakeEmptyArray(store) for dealID, prop := range props { err := root.Set(uint64(dealID), prop) require.NoError(t, err) @@ -512,16 +512,16 @@ func createProposalAMT(ctx context.Context, t *testing.T, store adt.Store, props return rootCid } -func createBalanceTable(ctx context.Context, t *testing.T, store adt.Store, balances map[address.Address]balance) [2]cid.Cid { - escrowMapRoot := adt.MakeEmptyMap(store) +func createBalanceTable(ctx context.Context, t *testing.T, store adt2.Store, balances map[address.Address]balance) [2]cid.Cid { + escrowMapRoot := adt2.MakeEmptyMap(store) escrowMapRootCid, err := escrowMapRoot.Root() require.NoError(t, err) - escrowRoot, err := adt.AsBalanceTable(store, escrowMapRootCid) + escrowRoot, err := adt2.AsBalanceTable(store, escrowMapRootCid) require.NoError(t, err) - lockedMapRoot := adt.MakeEmptyMap(store) + lockedMapRoot := adt2.MakeEmptyMap(store) lockedMapRootCid, err := lockedMapRoot.Root() require.NoError(t, err) - lockedRoot, err := adt.AsBalanceTable(store, lockedMapRootCid) + lockedRoot, err := adt2.AsBalanceTable(store, lockedMapRootCid) require.NoError(t, err) for addr, balance := range balances { @@ -538,7 +538,7 @@ func createBalanceTable(ctx context.Context, t *testing.T, store adt.Store, bala return [2]cid.Cid{escrowRootCid, lockedRootCid} } -func createMinerState(ctx context.Context, t *testing.T, store adt.Store, owner, worker address.Address, sectors []miner.SectorOnChainInfo) cid.Cid { +func createMinerState(ctx context.Context, t *testing.T, store adt2.Store, owner, worker address.Address, sectors []miner.SectorOnChainInfo) cid.Cid { rootCid := createSectorsAMT(ctx, t, store, sectors) state := createEmptyMinerState(ctx, t, store, owner, worker) @@ -549,20 +549,20 @@ func createMinerState(ctx context.Context, t *testing.T, store adt.Store, owner, return stateC } -func createEmptyMinerState(ctx context.Context, t *testing.T, store adt.Store, owner, worker address.Address) *miner0.State { - emptyArrayCid, err := adt.MakeEmptyArray(store).Root() +func createEmptyMinerState(ctx context.Context, t *testing.T, store adt2.Store, owner, worker address.Address) *miner2.State { + emptyArrayCid, err := adt2.MakeEmptyArray(store).Root() require.NoError(t, err) - emptyMap, err := adt.MakeEmptyMap(store).Root() + emptyMap, err := adt2.MakeEmptyMap(store).Root() require.NoError(t, err) - emptyDeadline, err := store.Put(store.Context(), miner0.ConstructDeadline(emptyArrayCid)) + emptyDeadline, err := store.Put(store.Context(), miner2.ConstructDeadline(emptyArrayCid)) require.NoError(t, err) - emptyVestingFunds := miner0.ConstructVestingFunds() + emptyVestingFunds := miner2.ConstructVestingFunds() emptyVestingFundsCid, err := store.Put(store.Context(), emptyVestingFunds) require.NoError(t, err) - emptyDeadlines := miner0.ConstructDeadlines(emptyDeadline) + emptyDeadlines := miner2.ConstructDeadlines(emptyDeadline) emptyDeadlinesCid, err := store.Put(store.Context(), emptyDeadlines) require.NoError(t, err) @@ -572,16 +572,30 @@ func createEmptyMinerState(ctx context.Context, t *testing.T, store adt.Store, o emptyBitfieldCid, err := store.Put(store.Context(), emptyBitfield) require.NoError(t, err) - state, err := miner0.ConstructState(minerInfo, 123, emptyBitfieldCid, emptyArrayCid, emptyMap, emptyDeadlinesCid, emptyVestingFundsCid) + state, err := miner2.ConstructState(minerInfo, 123, 4, emptyBitfieldCid, emptyArrayCid, emptyMap, emptyDeadlinesCid, emptyVestingFundsCid) require.NoError(t, err) return state } -func createSectorsAMT(ctx context.Context, t *testing.T, store adt.Store, sectors []miner.SectorOnChainInfo) cid.Cid { - root := adt.MakeEmptyArray(store) +func createSectorsAMT(ctx context.Context, t *testing.T, store adt2.Store, sectors []miner.SectorOnChainInfo) cid.Cid { + root := adt2.MakeEmptyArray(store) for _, sector := range sectors { - sector := (miner0.SectorOnChainInfo)(sector) + sector := miner2.SectorOnChainInfo{ + SectorNumber: sector.SectorNumber, + SealProof: sector.SealProof, + SealedCID: sector.SealedCID, + DealIDs: sector.DealIDs, + Activation: sector.Activation, + Expiration: sector.Expiration, + DealWeight: sector.DealWeight, + VerifiedDealWeight: sector.VerifiedDealWeight, + InitialPledge: sector.InitialPledge, + ExpectedDayReward: sector.ExpectedDayReward, + ExpectedStoragePledge: sector.ExpectedStoragePledge, + ReplacedSectorAge: 0, + ReplacedDayReward: big.NewInt(0), + } err := root.Set(uint64(sector.SectorNumber), §or) require.NoError(t, err) } @@ -614,8 +628,8 @@ const ( ) // returns a unique SectorPreCommitInfo with each invocation with SectorNumber set to `sectorNo`. -func newSectorPreCommitInfo(sectorNo abi.SectorNumber, sealed cid.Cid, expiration abi.ChainEpoch) *miner0.SectorPreCommitInfo { - return &miner0.SectorPreCommitInfo{ +func newSectorPreCommitInfo(sectorNo abi.SectorNumber, sealed cid.Cid, expiration abi.ChainEpoch) *miner2.SectorPreCommitInfo { + return &miner2.SectorPreCommitInfo{ SealProof: abi.RegisteredSealProof_StackedDrg32GiBV1, SectorNumber: sectorNo, SealedCID: sealed, @@ -625,7 +639,7 @@ func newSectorPreCommitInfo(sectorNo abi.SectorNumber, sealed cid.Cid, expiratio } } -func dealEquality(expected market0.DealState, actual market.DealState) bool { +func dealEquality(expected market2.DealState, actual market.DealState) bool { return expected.LastUpdatedEpoch == actual.LastUpdatedEpoch && expected.SectorStartEpoch == actual.SectorStartEpoch && expected.SlashEpoch == actual.SlashEpoch diff --git a/chain/gen/gen.go b/chain/gen/gen.go index 288c84219..f2c3832c1 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -8,8 +8,7 @@ import ( "sync/atomic" "time" - "github.com/filecoin-project/specs-actors/actors/runtime/proof" - "github.com/google/uuid" + proof2 "github.com/filecoin-project/specs-actors/v2/actors/runtime/proof" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" @@ -50,7 +49,7 @@ const msgsPerBlock = 20 //nolint:deadcode,varcheck var log = logging.Logger("gen") -var ValidWpostForTesting = []proof.PoStProof{{ +var ValidWpostForTesting = []proof2.PoStProof{{ ProofBytes: []byte("valid proof"), }} @@ -227,7 +226,7 @@ func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) { }, VerifregRootKey: DefaultVerifregRootkeyActor, RemainderAccount: DefaultRemainderAccountActor, - NetworkName: uuid.New().String(), + NetworkName: "", Timestamp: uint64(build.Clock.Now().Add(-500 * time.Duration(build.BlockDelaySecs) * time.Second).Unix()), } @@ -467,7 +466,7 @@ func (cg *ChainGen) NextTipSetFromMinersWithMessages(base *types.TipSet, miners func (cg *ChainGen) makeBlock(parents *types.TipSet, m address.Address, vrfticket *types.Ticket, eticket *types.ElectionProof, bvals []types.BeaconEntry, height abi.ChainEpoch, - wpost []proof.PoStProof, msgs []*types.SignedMessage) (*types.FullBlock, error) { + wpost []proof2.PoStProof, msgs []*types.SignedMessage) (*types.FullBlock, error) { var ts uint64 if cg.Timestamper != nil { @@ -605,7 +604,7 @@ func (mca mca) WalletSign(ctx context.Context, a address.Address, v []byte) (*cr type WinningPoStProver interface { GenerateCandidates(context.Context, abi.PoStRandomness, uint64) ([]uint64, error) - ComputeProof(context.Context, []proof.SectorInfo, abi.PoStRandomness) ([]proof.PoStProof, error) + ComputeProof(context.Context, []proof2.SectorInfo, abi.PoStRandomness) ([]proof2.PoStProof, error) } type wppProvider struct{} @@ -614,7 +613,7 @@ func (wpp *wppProvider) GenerateCandidates(ctx context.Context, _ abi.PoStRandom return []uint64{0}, nil } -func (wpp *wppProvider) ComputeProof(context.Context, []proof.SectorInfo, abi.PoStRandomness) ([]proof.PoStProof, error) { +func (wpp *wppProvider) ComputeProof(context.Context, []proof2.SectorInfo, abi.PoStRandomness) ([]proof2.PoStProof, error) { return ValidWpostForTesting, nil } @@ -681,15 +680,15 @@ type genFakeVerifier struct{} var _ ffiwrapper.Verifier = (*genFakeVerifier)(nil) -func (m genFakeVerifier) VerifySeal(svi proof.SealVerifyInfo) (bool, error) { +func (m genFakeVerifier) VerifySeal(svi proof2.SealVerifyInfo) (bool, error) { return true, nil } -func (m genFakeVerifier) VerifyWinningPoSt(ctx context.Context, info proof.WinningPoStVerifyInfo) (bool, error) { +func (m genFakeVerifier) VerifyWinningPoSt(ctx context.Context, info proof2.WinningPoStVerifyInfo) (bool, error) { panic("not supported") } -func (m genFakeVerifier) VerifyWindowPoSt(ctx context.Context, info proof.WindowPoStVerifyInfo) (bool, error) { +func (m genFakeVerifier) VerifyWindowPoSt(ctx context.Context, info proof2.WindowPoStVerifyInfo) (bool, error) { panic("not supported") } diff --git a/chain/gen/genesis/miners.go b/chain/gen/genesis/miners.go index 6f0c136a3..49e09d792 100644 --- a/chain/gen/genesis/miners.go +++ b/chain/gen/genesis/miners.go @@ -24,11 +24,12 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" - "github.com/filecoin-project/specs-actors/actors/builtin" + + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" power0 "github.com/filecoin-project/specs-actors/actors/builtin/power" reward0 "github.com/filecoin-project/specs-actors/actors/builtin/reward" - "github.com/filecoin-project/specs-actors/actors/runtime" + runtime2 "github.com/filecoin-project/specs-actors/v2/actors/runtime" "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/store" @@ -47,7 +48,7 @@ func MinerAddress(genesisIndex uint64) address.Address { } type fakedSigSyscalls struct { - runtime.Syscalls + runtime2.Syscalls } func (fss *fakedSigSyscalls) VerifySignature(signature crypto.Signature, signer address.Address, plaintext []byte) error { @@ -55,7 +56,7 @@ func (fss *fakedSigSyscalls) VerifySignature(signature crypto.Signature, signer } func mkFakedSigSyscalls(base vm.SyscallBuilder) vm.SyscallBuilder { - return func(ctx context.Context, cstate *state.StateTree, cst cbor.IpldStore) runtime.Syscalls { + return func(ctx context.Context, cstate *state.StateTree, cst cbor.IpldStore) runtime2.Syscalls { return &fakedSigSyscalls{ base(ctx, cstate, cst), } @@ -114,7 +115,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid } params := mustEnc(constructorParams) - rval, err := doExecValue(ctx, vm, power.Address, m.Owner, m.PowerBalance, builtin.MethodsPower.CreateMiner, params) + rval, err := doExecValue(ctx, vm, power.Address, m.Owner, m.PowerBalance, builtin0.MethodsPower.CreateMiner, params) if err != nil { return cid.Undef, xerrors.Errorf("failed to create genesis miner: %w", err) } @@ -146,7 +147,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid if m.MarketBalance.GreaterThan(big.Zero()) { params := mustEnc(&minerInfos[i].maddr) - _, err := doExecValue(ctx, vm, market.Address, m.Worker, m.MarketBalance, builtin.MethodsMarket.AddBalance, params) + _, err := doExecValue(ctx, vm, market.Address, m.Worker, m.MarketBalance, builtin0.MethodsMarket.AddBalance, params) if err != nil { return cid.Undef, xerrors.Errorf("failed to create genesis miner (add balance): %w", err) } @@ -158,7 +159,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid publish := func(params *market.PublishStorageDealsParams) error { fmt.Printf("publishing %d storage deals on miner %s with worker %s\n", len(params.Deals), params.Deals[0].Proposal.Provider, m.Worker) - ret, err := doExecValue(ctx, vm, market.Address, m.Worker, big.Zero(), builtin.MethodsMarket.PublishStorageDeals, mustEnc(params)) + ret, err := doExecValue(ctx, vm, market.Address, m.Worker, big.Zero(), builtin0.MethodsMarket.PublishStorageDeals, mustEnc(params)) if err != nil { return xerrors.Errorf("failed to create genesis miner (publish deals): %w", err) } @@ -290,17 +291,17 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid pledge = big.Add(pcd, pledge) fmt.Println(types.FIL(pledge)) - _, err = doExecValue(ctx, vm, minerInfos[i].maddr, m.Worker, pledge, builtin.MethodsMiner.PreCommitSector, mustEnc(params)) + _, err = doExecValue(ctx, vm, minerInfos[i].maddr, m.Worker, pledge, builtin0.MethodsMiner.PreCommitSector, mustEnc(params)) if err != nil { return cid.Undef, xerrors.Errorf("failed to confirm presealed sectors: %w", err) } // Commit one-by-one, otherwise pledge math tends to explode - confirmParams := &builtin.ConfirmSectorProofsParams{ + confirmParams := &builtin0.ConfirmSectorProofsParams{ Sectors: []abi.SectorNumber{preseal.SectorID}, } - _, err = doExecValue(ctx, vm, minerInfos[i].maddr, power.Address, big.Zero(), builtin.MethodsMiner.ConfirmSectorProofsValid, mustEnc(confirmParams)) + _, err = doExecValue(ctx, vm, minerInfos[i].maddr, power.Address, big.Zero(), builtin0.MethodsMiner.ConfirmSectorProofsValid, mustEnc(confirmParams)) if err != nil { return cid.Undef, xerrors.Errorf("failed to confirm presealed sectors: %w", err) } @@ -349,7 +350,7 @@ func (fr *fakeRand) GetBeaconRandomness(ctx context.Context, personalization cry } func currentTotalPower(ctx context.Context, vm *vm.VM, maddr address.Address) (*power0.CurrentTotalPowerReturn, error) { - pwret, err := doExecValue(ctx, vm, power.Address, maddr, big.Zero(), builtin.MethodsPower.CurrentTotalPower, nil) + pwret, err := doExecValue(ctx, vm, power.Address, maddr, big.Zero(), builtin0.MethodsPower.CurrentTotalPower, nil) if err != nil { return nil, err } @@ -373,7 +374,7 @@ func dealWeight(ctx context.Context, vm *vm.VM, maddr address.Address, dealIDs [ market.Address, maddr, abi.NewTokenAmount(0), - builtin.MethodsMarket.VerifyDealsForActivation, + builtin0.MethodsMarket.VerifyDealsForActivation, mustEnc(params), ) if err != nil { @@ -387,7 +388,7 @@ func dealWeight(ctx context.Context, vm *vm.VM, maddr address.Address, dealIDs [ } func currentEpochBlockReward(ctx context.Context, vm *vm.VM, maddr address.Address) (*reward0.ThisEpochRewardReturn, error) { - rwret, err := doExecValue(ctx, vm, reward.Address, maddr, big.Zero(), builtin.MethodsReward.ThisEpochReward, nil) + rwret, err := doExecValue(ctx, vm, reward.Address, maddr, big.Zero(), builtin0.MethodsReward.ThisEpochReward, nil) if err != nil { return nil, err } diff --git a/chain/gen/mining.go b/chain/gen/mining.go index 45a089452..94776c5f4 100644 --- a/chain/gen/mining.go +++ b/chain/gen/mining.go @@ -4,7 +4,7 @@ import ( "context" "github.com/filecoin-project/go-state-types/crypto" - "github.com/filecoin-project/specs-actors/actors/util/adt" + blockadt "github.com/filecoin-project/specs-actors/actors/util/adt" cid "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" cbg "github.com/whyrusleeping/cbor-gen" @@ -181,8 +181,8 @@ func aggregateSignatures(sigs []crypto.Signature) (*crypto.Signature, error) { }, nil } -func toArray(store adt.Store, cids []cid.Cid) (cid.Cid, error) { - arr := adt.MakeEmptyArray(store) +func toArray(store blockadt.Store, cids []cid.Cid) (cid.Cid, error) { + arr := blockadt.MakeEmptyArray(store) for i, c := range cids { oc := cbg.CborCid(c) if err := arr.Set(uint64(i), &oc); err != nil { diff --git a/chain/market/fundmgr.go b/chain/market/fundmgr.go index 8b5f85a12..c65547285 100644 --- a/chain/market/fundmgr.go +++ b/chain/market/fundmgr.go @@ -8,7 +8,7 @@ import ( "github.com/filecoin-project/go-state-types/big" "go.uber.org/fx" - "github.com/filecoin-project/specs-actors/actors/builtin" + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log" @@ -156,7 +156,7 @@ func (fm *FundMgr) EnsureAvailable(ctx context.Context, addr, wallet address.Add To: market.Address, From: wallet, Value: toAdd, - Method: builtin.MethodsMarket.AddBalance, + Method: builtin2.MethodsMarket.AddBalance, Params: params, }, nil) if err != nil { diff --git a/chain/market/fundmgr_test.go b/chain/market/fundmgr_test.go index f5936f73d..7f163201e 100644 --- a/chain/market/fundmgr_test.go +++ b/chain/market/fundmgr_test.go @@ -12,8 +12,8 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/crypto" - "github.com/filecoin-project/specs-actors/actors/builtin" - tutils "github.com/filecoin-project/specs-actors/support/testing" + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + tutils "github.com/filecoin-project/specs-actors/v2/support/testing" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" @@ -51,7 +51,7 @@ func addFundsMsg(toAdd abi.TokenAmount, addr address.Address, wallet address.Add To: market.Address, From: wallet, Value: toAdd, - Method: builtin.MethodsMarket.AddBalance, + Method: builtin2.MethodsMarket.AddBalance, Params: params, } } diff --git a/chain/messagepool/messagepool_test.go b/chain/messagepool/messagepool_test.go index 8f6613cad..e31df936c 100644 --- a/chain/messagepool/messagepool_test.go +++ b/chain/messagepool/messagepool_test.go @@ -8,16 +8,18 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" + logging "github.com/ipfs/go-log/v2" + + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + "github.com/filecoin-project/lotus/chain/messagepool/gasguess" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/types/mock" "github.com/filecoin-project/lotus/chain/wallet" _ "github.com/filecoin-project/lotus/lib/sigs/bls" _ "github.com/filecoin-project/lotus/lib/sigs/secp" - "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/ipfs/go-cid" - "github.com/ipfs/go-datastore" - logging "github.com/ipfs/go-log/v2" ) func init() { @@ -142,7 +144,7 @@ func (tma *testMpoolAPI) GetActorAfter(addr address.Address, ts *types.TipSet) ( } return &types.Actor{ - Code: builtin.StorageMarketActorCodeID, + Code: builtin2.StorageMarketActorCodeID, Nonce: nonce, Balance: balance, }, nil @@ -449,7 +451,7 @@ func TestLoadLocal(t *testing.T) { tma.setBalance(a1, 1) // in FIL tma.setBalance(a2, 1) // in FIL - gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin.StorageMarketActorCodeID, M: 2}] + gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin2.StorageMarketActorCodeID, M: 2}] msgs := make(map[cid.Cid]struct{}) for i := 0; i < 10; i++ { m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(i+1)) @@ -521,7 +523,7 @@ func TestClearAll(t *testing.T) { tma.setBalance(a1, 1) // in FIL tma.setBalance(a2, 1) // in FIL - gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin.StorageMarketActorCodeID, M: 2}] + gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin2.StorageMarketActorCodeID, M: 2}] for i := 0; i < 10; i++ { m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(i+1)) _, err := mp.Push(m) @@ -576,7 +578,7 @@ func TestClearNonLocal(t *testing.T) { tma.setBalance(a1, 1) // in FIL tma.setBalance(a2, 1) // in FIL - gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin.StorageMarketActorCodeID, M: 2}] + gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin2.StorageMarketActorCodeID, M: 2}] for i := 0; i < 10; i++ { m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(i+1)) _, err := mp.Push(m) @@ -642,7 +644,7 @@ func TestUpdates(t *testing.T) { t.Fatal(err) } - gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin.StorageMarketActorCodeID, M: 2}] + gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin2.StorageMarketActorCodeID, M: 2}] tma.setBalance(a1, 1) // in FIL tma.setBalance(a2, 1) // in FIL diff --git a/chain/messagepool/repub_test.go b/chain/messagepool/repub_test.go index 9cadf24c7..8da64f974 100644 --- a/chain/messagepool/repub_test.go +++ b/chain/messagepool/repub_test.go @@ -5,11 +5,13 @@ import ( "testing" "time" + "github.com/ipfs/go-datastore" + + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + "github.com/filecoin-project/lotus/chain/messagepool/gasguess" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/wallet" - "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/ipfs/go-datastore" ) func TestRepubMessages(t *testing.T) { @@ -48,7 +50,7 @@ func TestRepubMessages(t *testing.T) { t.Fatal(err) } - gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin.StorageMarketActorCodeID, M: 2}] + gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin2.StorageMarketActorCodeID, M: 2}] tma.setBalance(a1, 1) // in FIL diff --git a/chain/messagepool/selection_test.go b/chain/messagepool/selection_test.go index 37c2f2dc0..08cf286c8 100644 --- a/chain/messagepool/selection_test.go +++ b/chain/messagepool/selection_test.go @@ -13,17 +13,18 @@ import ( "sort" "testing" + "github.com/filecoin-project/go-address" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" logging "github.com/ipfs/go-log" - "github.com/filecoin-project/go-address" + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/messagepool/gasguess" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/types/mock" "github.com/filecoin-project/lotus/chain/wallet" - "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/lotus/api" _ "github.com/filecoin-project/lotus/lib/sigs/bls" @@ -94,7 +95,7 @@ func TestMessageChains(t *testing.T) { block := tma.nextBlock() ts := mock.TipSet(block) - gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin.StorageMarketActorCodeID, M: 2}] + gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin2.StorageMarketActorCodeID, M: 2}] tma.setBalance(a1, 1) // in FIL @@ -332,7 +333,7 @@ func TestMessageChainSkipping(t *testing.T) { block := tma.nextBlock() ts := mock.TipSet(block) - gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin.StorageMarketActorCodeID, M: 2}] + gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin2.StorageMarketActorCodeID, M: 2}] baseFee := types.NewInt(0) tma.setBalance(a1, 1) // in FIL @@ -409,7 +410,7 @@ func TestBasicMessageSelection(t *testing.T) { ts := mock.TipSet(block) tma.applyBlock(t, block) - gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin.StorageMarketActorCodeID, M: 2}] + gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin2.StorageMarketActorCodeID, M: 2}] tma.setBalance(a1, 1) // in FIL tma.setBalance(a2, 1) // in FIL @@ -553,7 +554,7 @@ func TestMessageSelectionTrimming(t *testing.T) { ts := mock.TipSet(block) tma.applyBlock(t, block) - gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin.StorageMarketActorCodeID, M: 2}] + gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin2.StorageMarketActorCodeID, M: 2}] tma.setBalance(a1, 1) // in FIL tma.setBalance(a2, 1) // in FIL @@ -616,7 +617,7 @@ func TestPriorityMessageSelection(t *testing.T) { ts := mock.TipSet(block) tma.applyBlock(t, block) - gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin.StorageMarketActorCodeID, M: 2}] + gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin2.StorageMarketActorCodeID, M: 2}] tma.setBalance(a1, 1) // in FIL tma.setBalance(a2, 1) // in FIL @@ -695,7 +696,7 @@ func TestPriorityMessageSelection2(t *testing.T) { ts := mock.TipSet(block) tma.applyBlock(t, block) - gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin.StorageMarketActorCodeID, M: 2}] + gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin2.StorageMarketActorCodeID, M: 2}] tma.setBalance(a1, 1) // in FIL tma.setBalance(a2, 1) // in FIL @@ -764,7 +765,7 @@ func TestPriorityMessageSelection3(t *testing.T) { ts := mock.TipSet(block) tma.applyBlock(t, block) - gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin.StorageMarketActorCodeID, M: 2}] + gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin2.StorageMarketActorCodeID, M: 2}] tma.setBalance(a1, 1) // in FIL tma.setBalance(a2, 1) // in FIL @@ -861,7 +862,7 @@ func TestOptimalMessageSelection1(t *testing.T) { ts := mock.TipSet(block) tma.applyBlock(t, block) - gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin.StorageMarketActorCodeID, M: 2}] + gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin2.StorageMarketActorCodeID, M: 2}] tma.setBalance(a1, 1) // in FIL tma.setBalance(a2, 1) // in FIL @@ -928,7 +929,7 @@ func TestOptimalMessageSelection2(t *testing.T) { ts := mock.TipSet(block) tma.applyBlock(t, block) - gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin.StorageMarketActorCodeID, M: 2}] + gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin2.StorageMarketActorCodeID, M: 2}] tma.setBalance(a1, 1) // in FIL tma.setBalance(a2, 1) // in FIL @@ -1006,7 +1007,7 @@ func TestOptimalMessageSelection3(t *testing.T) { ts := mock.TipSet(block) tma.applyBlock(t, block) - gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin.StorageMarketActorCodeID, M: 2}] + gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin2.StorageMarketActorCodeID, M: 2}] for _, a := range actors { tma.setBalance(a, 1) // in FIL @@ -1086,7 +1087,7 @@ func testCompetitiveMessageSelection(t *testing.T, rng *rand.Rand, getPremium fu ts := mock.TipSet(block) tma.applyBlock(t, block) - gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin.StorageMarketActorCodeID, M: 2}] + gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin2.StorageMarketActorCodeID, M: 2}] baseFee := types.NewInt(0) for _, a := range actors { diff --git a/chain/state/statetree_test.go b/chain/state/statetree_test.go index 61e7d8c22..91674337b 100644 --- a/chain/state/statetree_test.go +++ b/chain/state/statetree_test.go @@ -10,7 +10,7 @@ import ( address "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/network" - "github.com/filecoin-project/specs-actors/actors/builtin" + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" @@ -18,7 +18,7 @@ import ( func BenchmarkStateTreeSet(b *testing.B) { cst := cbor.NewMemCborStore() - st, err := NewStateTree(cst, VersionForNetwork(build.NewestNetworkVersion)) + st, err := NewStateTree(cst, types.StateTreeVersion1) if err != nil { b.Fatal(err) } @@ -33,8 +33,8 @@ func BenchmarkStateTreeSet(b *testing.B) { } err = st.SetActor(a, &types.Actor{ Balance: types.NewInt(1258812523), - Code: builtin.StorageMinerActorCodeID, - Head: builtin.AccountActorCodeID, + Code: builtin2.StorageMinerActorCodeID, + Head: builtin2.AccountActorCodeID, Nonce: uint64(i), }) if err != nil { @@ -60,8 +60,8 @@ func BenchmarkStateTreeSetFlush(b *testing.B) { } err = st.SetActor(a, &types.Actor{ Balance: types.NewInt(1258812523), - Code: builtin.StorageMinerActorCodeID, - Head: builtin.AccountActorCodeID, + Code: builtin2.StorageMinerActorCodeID, + Head: builtin2.AccountActorCodeID, Nonce: uint64(i), }) if err != nil { @@ -183,8 +183,8 @@ func BenchmarkStateTree10kGetActor(b *testing.B) { } err = st.SetActor(a, &types.Actor{ Balance: types.NewInt(1258812523 + uint64(i)), - Code: builtin.StorageMinerActorCodeID, - Head: builtin.AccountActorCodeID, + Code: builtin2.StorageMinerActorCodeID, + Head: builtin2.AccountActorCodeID, Nonce: uint64(i), }) if err != nil { @@ -226,8 +226,8 @@ func TestSetCache(t *testing.T) { act := &types.Actor{ Balance: types.NewInt(0), - Code: builtin.StorageMinerActorCodeID, - Head: builtin.AccountActorCodeID, + Code: builtin2.StorageMinerActorCodeID, + Head: builtin2.AccountActorCodeID, Nonce: 0, } @@ -270,7 +270,7 @@ func TestSnapshots(t *testing.T) { t.Fatal(err) } - if err := st.SetActor(addrs[0], &types.Actor{Code: builtin.AccountActorCodeID, Head: builtin.AccountActorCodeID, Balance: types.NewInt(55)}); err != nil { + if err := st.SetActor(addrs[0], &types.Actor{Code: builtin2.AccountActorCodeID, Head: builtin2.AccountActorCodeID, Balance: types.NewInt(55)}); err != nil { t.Fatal(err) } @@ -279,7 +279,7 @@ func TestSnapshots(t *testing.T) { t.Fatal(err) } - if err := st.SetActor(addrs[1], &types.Actor{Code: builtin.AccountActorCodeID, Head: builtin.AccountActorCodeID, Balance: types.NewInt(77)}); err != nil { + if err := st.SetActor(addrs[1], &types.Actor{Code: builtin2.AccountActorCodeID, Head: builtin2.AccountActorCodeID, Balance: types.NewInt(77)}); err != nil { t.Fatal(err) } @@ -290,7 +290,7 @@ func TestSnapshots(t *testing.T) { } // more operations in top level call... - if err := st.SetActor(addrs[2], &types.Actor{Code: builtin.AccountActorCodeID, Head: builtin.AccountActorCodeID, Balance: types.NewInt(123)}); err != nil { + if err := st.SetActor(addrs[2], &types.Actor{Code: builtin2.AccountActorCodeID, Head: builtin2.AccountActorCodeID, Balance: types.NewInt(123)}); err != nil { t.Fatal(err) } @@ -299,7 +299,7 @@ func TestSnapshots(t *testing.T) { t.Fatal(err) } - if err := st.SetActor(addrs[3], &types.Actor{Code: builtin.AccountActorCodeID, Head: builtin.AccountActorCodeID, Balance: types.NewInt(5)}); err != nil { + if err := st.SetActor(addrs[3], &types.Actor{Code: builtin2.AccountActorCodeID, Head: builtin2.AccountActorCodeID, Balance: types.NewInt(5)}); err != nil { t.Fatal(err) } diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index 5d6cac51f..e089a1084 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -543,17 +543,17 @@ func UpgradeIgnition(ctx context.Context, sm *StateManager, cb ExecCallback, roo return cid.Undef, xerrors.Errorf("second split address: %w", err) } - err = resetGenesisMsigs(ctx, sm, store, tree, build.UpgradeLiftoffHeight) + err = resetGenesisMsigs0(ctx, sm, store, tree, build.UpgradeLiftoffHeight) if err != nil { return cid.Undef, xerrors.Errorf("resetting genesis msig start epochs: %w", err) } - err = splitGenesisMultisig(ctx, cb, split1, store, tree, 50, epoch) + err = splitGenesisMultisig0(ctx, cb, split1, store, tree, 50, epoch) if err != nil { return cid.Undef, xerrors.Errorf("splitting first msig: %w", err) } - err = splitGenesisMultisig(ctx, cb, split2, store, tree, 50, epoch) + err = splitGenesisMultisig0(ctx, cb, split2, store, tree, 50, epoch) if err != nil { return cid.Undef, xerrors.Errorf("splitting second msig: %w", err) } @@ -574,17 +574,17 @@ func UpgradeRefuel(ctx context.Context, sm *StateManager, cb ExecCallback, root return cid.Undef, xerrors.Errorf("getting state tree: %w", err) } - err = resetMultisigVesting(ctx, store, tree, builtin.SaftAddress, 0, 0, big.Zero()) + err = resetMultisigVesting0(ctx, store, tree, builtin.SaftAddress, 0, 0, big.Zero()) if err != nil { return cid.Undef, xerrors.Errorf("tweaking msig vesting: %w", err) } - err = resetMultisigVesting(ctx, store, tree, builtin.ReserveAddress, 0, 0, big.Zero()) + err = resetMultisigVesting0(ctx, store, tree, builtin.ReserveAddress, 0, 0, big.Zero()) if err != nil { return cid.Undef, xerrors.Errorf("tweaking msig vesting: %w", err) } - err = resetMultisigVesting(ctx, store, tree, builtin.RootVerifierAddress, 0, 0, big.Zero()) + err = resetMultisigVesting0(ctx, store, tree, builtin.RootVerifierAddress, 0, 0, big.Zero()) if err != nil { return cid.Undef, xerrors.Errorf("tweaking msig vesting: %w", err) } @@ -679,7 +679,7 @@ func setNetworkName(ctx context.Context, store adt.Store, tree *state.StateTree, return nil } -func splitGenesisMultisig(ctx context.Context, cb ExecCallback, addr address.Address, store adt0.Store, tree *state.StateTree, portions uint64, epoch abi.ChainEpoch) error { +func splitGenesisMultisig0(ctx context.Context, cb ExecCallback, addr address.Address, store adt0.Store, tree *state.StateTree, portions uint64, epoch abi.ChainEpoch) error { if portions < 1 { return xerrors.Errorf("cannot split into 0 portions") } @@ -835,7 +835,7 @@ func makeKeyAddr(splitAddr address.Address, count uint64) (address.Address, erro } // TODO: After the Liftoff epoch, refactor this to use resetMultisigVesting -func resetGenesisMsigs(ctx context.Context, sm *StateManager, store adt0.Store, tree *state.StateTree, startEpoch abi.ChainEpoch) error { +func resetGenesisMsigs0(ctx context.Context, sm *StateManager, store adt0.Store, tree *state.StateTree, startEpoch abi.ChainEpoch) error { gb, err := sm.cs.GetGenesis() if err != nil { return xerrors.Errorf("getting genesis block: %w", err) @@ -885,7 +885,7 @@ func resetGenesisMsigs(ctx context.Context, sm *StateManager, store adt0.Store, return nil } -func resetMultisigVesting(ctx context.Context, store adt0.Store, tree *state.StateTree, addr address.Address, startEpoch abi.ChainEpoch, duration abi.ChainEpoch, balance abi.TokenAmount) error { +func resetMultisigVesting0(ctx context.Context, store adt0.Store, tree *state.StateTree, addr address.Address, startEpoch abi.ChainEpoch, duration abi.ChainEpoch, balance abi.TokenAmount) error { act, err := tree.GetActor(addr) if err != nil { return xerrors.Errorf("getting actor: %w", err) diff --git a/chain/stmgr/forks_test.go b/chain/stmgr/forks_test.go index 0388af6ad..3e9e9e8c0 100644 --- a/chain/stmgr/forks_test.go +++ b/chain/stmgr/forks_test.go @@ -16,9 +16,10 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/cbor" - "github.com/filecoin-project/specs-actors/actors/builtin" - init0 "github.com/filecoin-project/specs-actors/actors/builtin/init" - "github.com/filecoin-project/specs-actors/actors/runtime" + + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + init2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/init" + rt2 "github.com/filecoin-project/specs-actors/v2/actors/runtime" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" @@ -45,7 +46,7 @@ type testActor struct { } // must use existing actor that an account is allowed to exec. -func (testActor) Code() cid.Cid { return builtin.PaymentChannelActorCodeID } +func (testActor) Code() cid.Cid { return builtin2.PaymentChannelActorCodeID } func (testActor) State() cbor.Er { return new(testActorState) } type testActorState struct { @@ -75,7 +76,7 @@ func (ta testActor) Exports() []interface{} { } } -func (ta *testActor) Constructor(rt runtime.Runtime, params *abi.EmptyValue) *abi.EmptyValue { +func (ta *testActor) Constructor(rt rt2.Runtime, params *abi.EmptyValue) *abi.EmptyValue { rt.ValidateImmediateCallerAcceptAny() rt.StateCreate(&testActorState{11}) //fmt.Println("NEW ACTOR ADDRESS IS: ", rt.Receiver()) @@ -83,7 +84,7 @@ func (ta *testActor) Constructor(rt runtime.Runtime, params *abi.EmptyValue) *ab return abi.Empty } -func (ta *testActor) TestMethod(rt runtime.Runtime, params *abi.EmptyValue) *abi.EmptyValue { +func (ta *testActor) TestMethod(rt rt2.Runtime, params *abi.EmptyValue) *abi.EmptyValue { rt.ValidateImmediateCallerAcceptAny() var st testActorState rt.StateReadonly(&st) @@ -175,7 +176,7 @@ func TestForkHeightTriggers(t *testing.T) { var msgs []*types.SignedMessage - enc, err := actors.SerializeParams(&init0.ExecParams{CodeCID: (testActor{}).Code()}) + enc, err := actors.SerializeParams(&init2.ExecParams{CodeCID: (testActor{}).Code()}) if err != nil { t.Fatal(err) } @@ -183,7 +184,7 @@ func TestForkHeightTriggers(t *testing.T) { m := &types.Message{ From: cg.Banker(), To: lotusinit.Address, - Method: builtin.MethodsInit.Exec, + Method: builtin2.MethodsInit.Exec, Params: enc, GasLimit: types.TestGasLimit, } @@ -273,7 +274,7 @@ func TestForkRefuseCall(t *testing.T) { cg.SetStateManager(sm) - enc, err := actors.SerializeParams(&init0.ExecParams{CodeCID: (testActor{}).Code()}) + enc, err := actors.SerializeParams(&init2.ExecParams{CodeCID: (testActor{}).Code()}) if err != nil { t.Fatal(err) } @@ -281,7 +282,7 @@ func TestForkRefuseCall(t *testing.T) { m := &types.Message{ From: cg.Banker(), To: lotusinit.Address, - Method: builtin.MethodsInit.Exec, + Method: builtin2.MethodsInit.Exec, Params: enc, GasLimit: types.TestGasLimit, Value: types.NewInt(0), diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index be14797d9..6558b1ea8 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -6,16 +6,6 @@ import ( "fmt" "sync" - "github.com/filecoin-project/lotus/chain/actors/builtin" - "github.com/filecoin-project/lotus/chain/actors/builtin/verifreg" - - _init "github.com/filecoin-project/lotus/chain/actors/builtin/init" - - "github.com/filecoin-project/lotus/chain/actors/builtin/miner" - - builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" - msig0 "github.com/filecoin-project/specs-actors/actors/builtin/multisig" - "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" logging "github.com/ipfs/go-log/v2" @@ -28,15 +18,24 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/network" + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + + // Used for genesis. + msig0 "github.com/filecoin-project/specs-actors/actors/builtin/multisig" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/chain/actors/builtin" + _init "github.com/filecoin-project/lotus/chain/actors/builtin/init" "github.com/filecoin-project/lotus/chain/actors/builtin/market" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/actors/builtin/multisig" "github.com/filecoin-project/lotus/chain/actors/builtin/paych" "github.com/filecoin-project/lotus/chain/actors/builtin/power" "github.com/filecoin-project/lotus/chain/actors/builtin/reward" + "github.com/filecoin-project/lotus/chain/actors/builtin/verifreg" "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" @@ -254,14 +253,14 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp runCron := func(epoch abi.ChainEpoch) error { cronMsg := &types.Message{ - To: builtin0.CronActorAddr, - From: builtin0.SystemActorAddr, + To: builtin2.CronActorAddr, + From: builtin2.SystemActorAddr, Nonce: uint64(epoch), Value: types.NewInt(0), GasFeeCap: types.NewInt(0), GasPremium: types.NewInt(0), GasLimit: build.BlockGasLimit * 10000, // Make super sure this is never too little - Method: builtin0.MethodsCron.EpochTick, + Method: builtin2.MethodsCron.EpochTick, Params: nil, } ret, err := vmi.ApplyImplicitMessage(ctx, cronMsg) @@ -350,14 +349,14 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp } rwMsg := &types.Message{ - From: builtin0.SystemActorAddr, + From: builtin2.SystemActorAddr, To: reward.Address, Nonce: uint64(epoch), Value: types.NewInt(0), GasFeeCap: types.NewInt(0), GasPremium: types.NewInt(0), GasLimit: 1 << 30, - Method: builtin0.MethodsReward.AwardBlockReward, + Method: builtin2.MethodsReward.AwardBlockReward, Params: params, } ret, actErr := vmi.ApplyImplicitMessage(ctx, rwMsg) @@ -974,7 +973,7 @@ func (sm *StateManager) setupGenesisActors(ctx context.Context) error { } else if builtin.IsAccountActor(act.Code) { // should exclude burnt funds actor and "remainder account actor" // should only ever be "faucet" accounts in testnets - if kaddr == builtin0.BurntFundsActorAddr { + if kaddr == builtin2.BurntFundsActorAddr { return nil } @@ -1052,24 +1051,24 @@ func (sm *StateManager) setupPreIgnitionGenesisActorsTestnet(ctx context.Context totalsByEpoch := make(map[abi.ChainEpoch]abi.TokenAmount) // 6 months - sixMonths := abi.ChainEpoch(183 * builtin0.EpochsInDay) + sixMonths := abi.ChainEpoch(183 * builtin2.EpochsInDay) totalsByEpoch[sixMonths] = big.NewInt(49_929_341) totalsByEpoch[sixMonths] = big.Add(totalsByEpoch[sixMonths], big.NewInt(32_787_700)) // 1 year - oneYear := abi.ChainEpoch(365 * builtin0.EpochsInDay) + oneYear := abi.ChainEpoch(365 * builtin2.EpochsInDay) totalsByEpoch[oneYear] = big.NewInt(22_421_712) // 2 years - twoYears := abi.ChainEpoch(2 * 365 * builtin0.EpochsInDay) + twoYears := abi.ChainEpoch(2 * 365 * builtin2.EpochsInDay) totalsByEpoch[twoYears] = big.NewInt(7_223_364) // 3 years - threeYears := abi.ChainEpoch(3 * 365 * builtin0.EpochsInDay) + threeYears := abi.ChainEpoch(3 * 365 * builtin2.EpochsInDay) totalsByEpoch[threeYears] = big.NewInt(87_637_883) // 6 years - sixYears := abi.ChainEpoch(6 * 365 * builtin0.EpochsInDay) + sixYears := abi.ChainEpoch(6 * 365 * builtin2.EpochsInDay) totalsByEpoch[sixYears] = big.NewInt(100_000_000) totalsByEpoch[sixYears] = big.Add(totalsByEpoch[sixYears], big.NewInt(300_000_000)) @@ -1129,24 +1128,24 @@ func (sm *StateManager) setupPostIgnitionGenesisActors(ctx context.Context) erro totalsByEpoch := make(map[abi.ChainEpoch]abi.TokenAmount) // 6 months - sixMonths := abi.ChainEpoch(183 * builtin0.EpochsInDay) + sixMonths := abi.ChainEpoch(183 * builtin2.EpochsInDay) totalsByEpoch[sixMonths] = big.NewInt(49_929_341) totalsByEpoch[sixMonths] = big.Add(totalsByEpoch[sixMonths], big.NewInt(32_787_700)) // 1 year - oneYear := abi.ChainEpoch(365 * builtin0.EpochsInDay) + oneYear := abi.ChainEpoch(365 * builtin2.EpochsInDay) totalsByEpoch[oneYear] = big.NewInt(22_421_712) // 2 years - twoYears := abi.ChainEpoch(2 * 365 * builtin0.EpochsInDay) + twoYears := abi.ChainEpoch(2 * 365 * builtin2.EpochsInDay) totalsByEpoch[twoYears] = big.NewInt(7_223_364) // 3 years - threeYears := abi.ChainEpoch(3 * 365 * builtin0.EpochsInDay) + threeYears := abi.ChainEpoch(3 * 365 * builtin2.EpochsInDay) totalsByEpoch[threeYears] = big.NewInt(87_637_883) // 6 years - sixYears := abi.ChainEpoch(6 * 365 * builtin0.EpochsInDay) + sixYears := abi.ChainEpoch(6 * 365 * builtin2.EpochsInDay) totalsByEpoch[sixYears] = big.NewInt(100_000_000) totalsByEpoch[sixYears] = big.Add(totalsByEpoch[sixYears], big.NewInt(300_000_000)) @@ -1280,7 +1279,7 @@ func (sm *StateManager) GetFilLocked(ctx context.Context, st *state.StateTree) ( } func GetFilBurnt(ctx context.Context, st *state.StateTree) (abi.TokenAmount, error) { - burnt, err := st.GetActor(builtin0.BurntFundsActorAddr) + burnt, err := st.GetActor(builtin2.BurntFundsActorAddr) if err != nil { return big.Zero(), xerrors.Errorf("failed to load burnt actor: %w", err) } diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index de4f947df..2f5acf0db 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -12,6 +12,7 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/network" + "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/policy" cid "github.com/ipfs/go-cid" @@ -26,7 +27,6 @@ import ( builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" exported0 "github.com/filecoin-project/specs-actors/actors/builtin/exported" - proof0 "github.com/filecoin-project/specs-actors/actors/runtime/proof" exported2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/exported" "github.com/filecoin-project/lotus/api" @@ -159,7 +159,7 @@ func GetMinerSectorSet(ctx context.Context, sm *StateManager, ts *types.TipSet, return mas.LoadSectors(snos) } -func GetSectorsForWinningPoSt(ctx context.Context, pv ffiwrapper.Verifier, sm *StateManager, st cid.Cid, maddr address.Address, rand abi.PoStRandomness) ([]proof0.SectorInfo, error) { +func GetSectorsForWinningPoSt(ctx context.Context, pv ffiwrapper.Verifier, sm *StateManager, st cid.Cid, maddr address.Address, rand abi.PoStRandomness) ([]builtin.SectorInfo, error) { act, err := sm.LoadActorRaw(ctx, maddr, st) if err != nil { return nil, xerrors.Errorf("failed to load miner actor: %w", err) @@ -244,9 +244,9 @@ func GetSectorsForWinningPoSt(ctx context.Context, pv ffiwrapper.Verifier, sm *S return nil, xerrors.Errorf("loading proving sectors: %w", err) } - out := make([]proof0.SectorInfo, len(sectors)) + out := make([]builtin.SectorInfo, len(sectors)) for i, sinfo := range sectors { - out[i] = proof0.SectorInfo{ + out[i] = builtin.SectorInfo{ SealProof: spt, SectorNumber: sinfo.SectorNumber, SealedCID: sinfo.SealedCID, diff --git a/chain/store/store.go b/chain/store/store.go index c6fc0cbef..092e2d308 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -19,12 +19,12 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/specs-actors/actors/builtin" - adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" + blockadt "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/vm" "github.com/filecoin-project/lotus/journal" bstore "github.com/filecoin-project/lotus/lib/blockstore" @@ -815,7 +815,7 @@ func (cs *ChainStore) GetSignedMessage(c cid.Cid) (*types.SignedMessage, error) func (cs *ChainStore) readAMTCids(root cid.Cid) ([]cid.Cid, error) { ctx := context.TODO() // block headers use adt0, for now. - a, err := adt0.AsArray(cs.Store(ctx), root) + a, err := blockadt.AsArray(cs.Store(ctx), root) if err != nil { return nil, xerrors.Errorf("amt load: %w", err) } @@ -1009,7 +1009,7 @@ func (cs *ChainStore) MessagesForBlock(b *types.BlockHeader) ([]*types.Message, func (cs *ChainStore) GetParentReceipt(b *types.BlockHeader, i int) (*types.MessageReceipt, error) { ctx := context.TODO() // block headers use adt0, for now. - a, err := adt0.AsArray(cs.Store(ctx), b.ParentMessageReceipts) + a, err := blockadt.AsArray(cs.Store(ctx), b.ParentMessageReceipts) if err != nil { return nil, xerrors.Errorf("amt load: %w", err) } diff --git a/chain/sub/incoming.go b/chain/sub/incoming.go index 99ca2fc65..625c8d1e2 100644 --- a/chain/sub/incoming.go +++ b/chain/sub/incoming.go @@ -23,7 +23,7 @@ import ( "go.opencensus.io/stats" "go.opencensus.io/tag" - adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" + blockadt "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain" @@ -391,9 +391,9 @@ func (bv *BlockValidator) isChainNearSynced() bool { func (bv *BlockValidator) validateMsgMeta(ctx context.Context, msg *types.BlockMsg) error { // TODO there has to be a simpler way to do this without the blockstore dance // block headers use adt0 - store := adt0.WrapStore(ctx, cbor.NewCborStore(blockstore.NewTemporary())) - bmArr := adt0.MakeEmptyArray(store) - smArr := adt0.MakeEmptyArray(store) + store := blockadt.WrapStore(ctx, cbor.NewCborStore(blockstore.NewTemporary())) + bmArr := blockadt.MakeEmptyArray(store) + smArr := blockadt.MakeEmptyArray(store) for i, m := range msg.BlsMessages { c := cbg.CborCid(m) diff --git a/chain/sync.go b/chain/sync.go index 2c7408113..dda6f88d8 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -15,8 +15,6 @@ import ( "github.com/filecoin-project/lotus/node/modules/dtypes" - "github.com/filecoin-project/specs-actors/actors/runtime/proof" - "github.com/Gurpartap/async" "github.com/hashicorp/go-multierror" blocks "github.com/ipfs/go-block-format" @@ -37,7 +35,11 @@ import ( "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" blst "github.com/supranational/blst/bindings/go" - adt0 "github.com/filecoin-project/specs-actors/actors/util/adt" + // named msgarray here to make it clear that these are the types used by + // messages, regardless of specs-actors version. + blockadt "github.com/filecoin-project/specs-actors/actors/util/adt" + + proof2 "github.com/filecoin-project/specs-actors/v2/actors/runtime/proof" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" @@ -463,9 +465,9 @@ func zipTipSetAndMessages(bs cbor.IpldStore, ts *types.TipSet, allbmsgs []*types // of both types (BLS and Secpk). func computeMsgMeta(bs cbor.IpldStore, bmsgCids, smsgCids []cid.Cid) (cid.Cid, error) { // block headers use adt0 - store := adt0.WrapStore(context.TODO(), bs) - bmArr := adt0.MakeEmptyArray(store) - smArr := adt0.MakeEmptyArray(store) + store := blockadt.WrapStore(context.TODO(), bs) + bmArr := blockadt.MakeEmptyArray(store) + smArr := blockadt.MakeEmptyArray(store) for i, m := range bmsgCids { c := cbg.CborCid(m) @@ -1015,7 +1017,7 @@ func (syncer *Syncer) VerifyWinningPoStProof(ctx context.Context, h *types.Block return xerrors.Errorf("getting winning post sector set: %w", err) } - ok, err := ffiwrapper.ProofVerifier.VerifyWinningPoSt(ctx, proof.WinningPoStVerifyInfo{ + ok, err := ffiwrapper.ProofVerifier.VerifyWinningPoSt(ctx, proof2.WinningPoStVerifyInfo{ Randomness: rand, Proofs: h.WinPoStProof, ChallengedSectors: sectors, @@ -1110,9 +1112,9 @@ func (syncer *Syncer) checkBlockMessages(ctx context.Context, b *types.FullBlock // Validate message arrays in a temporary blockstore. tmpbs := bstore.NewTemporary() - tmpstore := adt0.WrapStore(ctx, cbor.NewCborStore(tmpbs)) + tmpstore := blockadt.WrapStore(ctx, cbor.NewCborStore(tmpbs)) - bmArr := adt0.MakeEmptyArray(tmpstore) + bmArr := blockadt.MakeEmptyArray(tmpstore) for i, m := range b.BlsMessages { if err := checkMsg(m); err != nil { return xerrors.Errorf("block had invalid bls message at index %d: %w", i, err) @@ -1129,7 +1131,7 @@ func (syncer *Syncer) checkBlockMessages(ctx context.Context, b *types.FullBlock } } - smArr := adt0.MakeEmptyArray(tmpstore) + smArr := blockadt.MakeEmptyArray(tmpstore) for i, m := range b.SecpkMessages { if err := checkMsg(m); err != nil { return xerrors.Errorf("block had invalid secpk message at index %d: %w", i, err) diff --git a/chain/sync_test.go b/chain/sync_test.go index 0a8174c41..559a73bf5 100644 --- a/chain/sync_test.go +++ b/chain/sync_test.go @@ -7,8 +7,6 @@ import ( "testing" "time" - "github.com/filecoin-project/specs-actors/actors/runtime/proof" - "github.com/ipfs/go-cid" ds "github.com/ipfs/go-datastore" @@ -20,6 +18,8 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + proof2 "github.com/filecoin-project/specs-actors/v2/actors/runtime/proof" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors/policy" @@ -469,8 +469,8 @@ func (wpp badWpp) GenerateCandidates(context.Context, abi.PoStRandomness, uint64 return []uint64{1}, nil } -func (wpp badWpp) ComputeProof(context.Context, []proof.SectorInfo, abi.PoStRandomness) ([]proof.PoStProof, error) { - return []proof.PoStProof{ +func (wpp badWpp) ComputeProof(context.Context, []proof2.SectorInfo, abi.PoStRandomness) ([]proof2.PoStProof, error) { + return []proof2.PoStProof{ { PoStProof: abi.RegisteredPoStProof_StackedDrgWinning2KiBV1, ProofBytes: []byte("evil"), diff --git a/chain/types/blockheader.go b/chain/types/blockheader.go index 0ec33fe42..4db6788e1 100644 --- a/chain/types/blockheader.go +++ b/chain/types/blockheader.go @@ -4,7 +4,7 @@ import ( "bytes" "math/big" - "github.com/filecoin-project/specs-actors/actors/runtime/proof" + proof2 "github.com/filecoin-project/specs-actors/v2/actors/runtime/proof" "github.com/minio/blake2b-simd" @@ -55,7 +55,7 @@ type BlockHeader struct { BeaconEntries []BeaconEntry // 3 - WinPoStProof []proof.PoStProof // 4 + WinPoStProof []proof2.PoStProof // 4 Parents []cid.Cid // 5 diff --git a/chain/types/blockheader_test.go b/chain/types/blockheader_test.go index f5faac3b3..6674f1205 100644 --- a/chain/types/blockheader_test.go +++ b/chain/types/blockheader_test.go @@ -7,7 +7,7 @@ import ( "reflect" "testing" - "github.com/filecoin-project/specs-actors/actors/runtime/proof" + proof2 "github.com/filecoin-project/specs-actors/v2/actors/runtime/proof" cid "github.com/ipfs/go-cid" "github.com/stretchr/testify/require" @@ -82,7 +82,7 @@ func TestInteropBH(t *testing.T) { t.Fatal(err) } - posts := []proof.PoStProof{ + posts := []proof2.PoStProof{ {PoStProof: abi.RegisteredPoStProof_StackedDrgWinning2KiBV1, ProofBytes: []byte{0x07}}, } diff --git a/chain/types/message_test.go b/chain/types/message_test.go index dab8e0939..a5a00f66b 100644 --- a/chain/types/message_test.go +++ b/chain/types/message_test.go @@ -9,13 +9,15 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" - "github.com/filecoin-project/specs-actors/actors/builtin" + + // we can't import the actors shims from this package due to cyclic imports. + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" ) func TestEqualCall(t *testing.T) { m1 := &Message{ - To: builtin.StoragePowerActorAddr, - From: builtin.SystemActorAddr, + To: builtin2.StoragePowerActorAddr, + From: builtin2.SystemActorAddr, Nonce: 34, Value: big.Zero(), @@ -28,8 +30,8 @@ func TestEqualCall(t *testing.T) { } m2 := &Message{ - To: builtin.StoragePowerActorAddr, - From: builtin.SystemActorAddr, + To: builtin2.StoragePowerActorAddr, + From: builtin2.SystemActorAddr, Nonce: 34, Value: big.Zero(), @@ -42,8 +44,8 @@ func TestEqualCall(t *testing.T) { } m3 := &Message{ - To: builtin.StoragePowerActorAddr, - From: builtin.SystemActorAddr, + To: builtin2.StoragePowerActorAddr, + From: builtin2.SystemActorAddr, Nonce: 34, Value: big.Zero(), @@ -56,8 +58,8 @@ func TestEqualCall(t *testing.T) { } m4 := &Message{ - To: builtin.StoragePowerActorAddr, - From: builtin.SystemActorAddr, + To: builtin2.StoragePowerActorAddr, + From: builtin2.SystemActorAddr, Nonce: 34, Value: big.Zero(), @@ -76,8 +78,8 @@ func TestEqualCall(t *testing.T) { func TestMessageJson(t *testing.T) { m := &Message{ - To: builtin.StoragePowerActorAddr, - From: builtin.SystemActorAddr, + To: builtin2.StoragePowerActorAddr, + From: builtin2.SystemActorAddr, Nonce: 34, Value: big.Zero(), @@ -105,8 +107,8 @@ func TestMessageJson(t *testing.T) { func TestSignedMessageJson(t *testing.T) { m := Message{ - To: builtin.StoragePowerActorAddr, - From: builtin.SystemActorAddr, + To: builtin2.StoragePowerActorAddr, + From: builtin2.SystemActorAddr, Nonce: 34, Value: big.Zero(), diff --git a/chain/vm/gas.go b/chain/vm/gas.go index 6802013e5..cbe5bab13 100644 --- a/chain/vm/gas.go +++ b/chain/vm/gas.go @@ -3,13 +3,13 @@ package vm import ( "fmt" - "github.com/filecoin-project/specs-actors/actors/runtime/proof" + vmr2 "github.com/filecoin-project/specs-actors/v2/actors/runtime" + proof2 "github.com/filecoin-project/specs-actors/v2/actors/runtime/proof" "github.com/filecoin-project/go-address" addr "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/crypto" - vmr "github.com/filecoin-project/specs-actors/actors/runtime" "github.com/ipfs/go-cid" ) @@ -78,8 +78,8 @@ type Pricelist interface { OnVerifySignature(sigType crypto.SigType, planTextSize int) (GasCharge, error) OnHashing(dataSize int) GasCharge OnComputeUnsealedSectorCid(proofType abi.RegisteredSealProof, pieces []abi.PieceInfo) GasCharge - OnVerifySeal(info proof.SealVerifyInfo) GasCharge - OnVerifyPost(info proof.WindowPoStVerifyInfo) GasCharge + OnVerifySeal(info proof2.SealVerifyInfo) GasCharge + OnVerifyPost(info proof2.WindowPoStVerifyInfo) GasCharge OnVerifyConsensusFault() GasCharge } @@ -150,7 +150,7 @@ func PricelistByEpoch(epoch abi.ChainEpoch) Pricelist { } type pricedSyscalls struct { - under vmr.Syscalls + under vmr2.Syscalls pl Pricelist chargeGas func(GasCharge) } @@ -184,7 +184,7 @@ func (ps pricedSyscalls) ComputeUnsealedSectorCID(reg abi.RegisteredSealProof, p } // Verifies a sector seal proof. -func (ps pricedSyscalls) VerifySeal(vi proof.SealVerifyInfo) error { +func (ps pricedSyscalls) VerifySeal(vi proof2.SealVerifyInfo) error { ps.chargeGas(ps.pl.OnVerifySeal(vi)) defer ps.chargeGas(gasOnActorExec) @@ -192,7 +192,7 @@ func (ps pricedSyscalls) VerifySeal(vi proof.SealVerifyInfo) error { } // Verifies a proof of spacetime. -func (ps pricedSyscalls) VerifyPoSt(vi proof.WindowPoStVerifyInfo) error { +func (ps pricedSyscalls) VerifyPoSt(vi proof2.WindowPoStVerifyInfo) error { ps.chargeGas(ps.pl.OnVerifyPost(vi)) defer ps.chargeGas(gasOnActorExec) @@ -209,14 +209,14 @@ func (ps pricedSyscalls) VerifyPoSt(vi proof.WindowPoStVerifyInfo) error { // the "parent grinding fault", in which case it must be the sibling of h1 (same parent tipset) and one of the // blocks in the parent of h2 (i.e. h2's grandparent). // Returns nil and an error if the headers don't prove a fault. -func (ps pricedSyscalls) VerifyConsensusFault(h1 []byte, h2 []byte, extra []byte) (*vmr.ConsensusFault, error) { +func (ps pricedSyscalls) VerifyConsensusFault(h1 []byte, h2 []byte, extra []byte) (*vmr2.ConsensusFault, error) { ps.chargeGas(ps.pl.OnVerifyConsensusFault()) defer ps.chargeGas(gasOnActorExec) return ps.under.VerifyConsensusFault(h1, h2, extra) } -func (ps pricedSyscalls) BatchVerifySeals(inp map[address.Address][]proof.SealVerifyInfo) (map[address.Address][]bool, error) { +func (ps pricedSyscalls) BatchVerifySeals(inp map[address.Address][]proof2.SealVerifyInfo) (map[address.Address][]bool, error) { count := int64(0) for _, svis := range inp { count += int64(len(svis)) diff --git a/chain/vm/gas_v0.go b/chain/vm/gas_v0.go index bfb49c345..7e879b8c3 100644 --- a/chain/vm/gas_v0.go +++ b/chain/vm/gas_v0.go @@ -3,12 +3,12 @@ package vm import ( "fmt" - "github.com/filecoin-project/specs-actors/actors/runtime/proof" + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + proof2 "github.com/filecoin-project/specs-actors/v2/actors/runtime/proof" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" - builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" ) type scalingCost struct { @@ -112,14 +112,14 @@ func (pl *pricelistV0) OnMethodInvocation(value abi.TokenAmount, methodNum abi.M if big.Cmp(value, abi.NewTokenAmount(0)) != 0 { ret += pl.sendTransferFunds - if methodNum == builtin0.MethodSend { + if methodNum == builtin2.MethodSend { // transfer only ret += pl.sendTransferOnlyPremium } extra += "t" } - if methodNum != builtin0.MethodSend { + if methodNum != builtin2.MethodSend { extra += "i" // running actors is cheaper becase we hand over to actors ret += pl.sendInvokeMethod @@ -175,14 +175,14 @@ func (pl *pricelistV0) OnComputeUnsealedSectorCid(proofType abi.RegisteredSealPr } // OnVerifySeal -func (pl *pricelistV0) OnVerifySeal(info proof.SealVerifyInfo) GasCharge { +func (pl *pricelistV0) OnVerifySeal(info proof2.SealVerifyInfo) GasCharge { // TODO: this needs more cost tunning, check with @lotus // this is not used return newGasCharge("OnVerifySeal", pl.verifySealBase, 0) } // OnVerifyPost -func (pl *pricelistV0) OnVerifyPost(info proof.WindowPoStVerifyInfo) GasCharge { +func (pl *pricelistV0) OnVerifyPost(info proof2.WindowPoStVerifyInfo) GasCharge { sectorSize := "unknown" var proofType abi.RegisteredPoStProof diff --git a/chain/vm/invoker_test.go b/chain/vm/invoker_test.go index 4005dd42f..bce385b02 100644 --- a/chain/vm/invoker_test.go +++ b/chain/vm/invoker_test.go @@ -5,16 +5,17 @@ import ( "io" "testing" - "github.com/filecoin-project/go-state-types/abi" - cbor "github.com/ipfs/go-ipld-cbor" "github.com/stretchr/testify/assert" cbg "github.com/whyrusleeping/cbor-gen" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/exitcode" + + runtime2 "github.com/filecoin-project/specs-actors/v2/actors/runtime" + "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/aerrors" - "github.com/filecoin-project/specs-actors/actors/runtime" ) type basicContract struct{} @@ -61,17 +62,17 @@ func (b basicContract) Exports() []interface{} { } } -func (basicContract) InvokeSomething0(rt runtime.Runtime, params *basicParams) *abi.EmptyValue { +func (basicContract) InvokeSomething0(rt runtime2.Runtime, params *basicParams) *abi.EmptyValue { rt.Abortf(exitcode.ExitCode(params.B), "params.B") return nil } -func (basicContract) BadParam(rt runtime.Runtime, params *basicParams) *abi.EmptyValue { +func (basicContract) BadParam(rt runtime2.Runtime, params *basicParams) *abi.EmptyValue { rt.Abortf(255, "bad params") return nil } -func (basicContract) InvokeSomething10(rt runtime.Runtime, params *basicParams) *abi.EmptyValue { +func (basicContract) InvokeSomething10(rt runtime2.Runtime, params *basicParams) *abi.EmptyValue { rt.Abortf(exitcode.ExitCode(params.B+10), "params.B") return nil } diff --git a/chain/vm/mkactor.go b/chain/vm/mkactor.go index 22a2acb8b..56eac1e0c 100644 --- a/chain/vm/mkactor.go +++ b/chain/vm/mkactor.go @@ -15,6 +15,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/actors/aerrors" + "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/types" ) @@ -56,7 +57,7 @@ func TryCreateAccountActor(rt *Runtime, addr address.Address) (*types.Actor, add } // call constructor on account - _, aerr = rt.internalSend(builtin0.SystemActorAddr, addrID, builtin0.MethodsAccount.Constructor, big.Zero(), p) + _, aerr = rt.internalSend(builtin.SystemActorAddr, addrID, builtin2.MethodsAccount.Constructor, big.Zero(), p) if aerr != nil { return nil, address.Undef, aerrors.Wrap(aerr, "failed to invoke account constructor") } diff --git a/chain/vm/runtime.go b/chain/vm/runtime.go index ed5501fe9..6e36e8e87 100644 --- a/chain/vm/runtime.go +++ b/chain/vm/runtime.go @@ -53,8 +53,8 @@ func (m *Message) ValueReceived() abi.TokenAmount { var EnableGasTracing = false type Runtime struct { - rt0.Message - rt0.Syscalls + rt2.Message + rt2.Syscalls ctx context.Context diff --git a/chain/vm/syscalls.go b/chain/vm/syscalls.go index a7f5dab0c..347c3409c 100644 --- a/chain/vm/syscalls.go +++ b/chain/vm/syscalls.go @@ -7,8 +7,6 @@ import ( goruntime "runtime" "sync" - "github.com/filecoin-project/specs-actors/actors/runtime/proof" - "github.com/filecoin-project/go-address" "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" @@ -23,7 +21,9 @@ import ( "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/sigs" - "github.com/filecoin-project/specs-actors/actors/runtime" + + runtime2 "github.com/filecoin-project/specs-actors/v2/actors/runtime" + proof2 "github.com/filecoin-project/specs-actors/v2/actors/runtime/proof" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" ) @@ -34,10 +34,10 @@ func init() { // Actual type is defined in chain/types/vmcontext.go because the VMContext interface is there -type SyscallBuilder func(ctx context.Context, cstate *state.StateTree, cst cbor.IpldStore) runtime.Syscalls +type SyscallBuilder func(ctx context.Context, cstate *state.StateTree, cst cbor.IpldStore) runtime2.Syscalls func Syscalls(verifier ffiwrapper.Verifier) SyscallBuilder { - return func(ctx context.Context, cstate *state.StateTree, cst cbor.IpldStore) runtime.Syscalls { + return func(ctx context.Context, cstate *state.StateTree, cst cbor.IpldStore) runtime2.Syscalls { return &syscallShim{ ctx: ctx, @@ -79,7 +79,7 @@ func (ss *syscallShim) HashBlake2b(data []byte) [32]byte { // Checks validity of the submitted consensus fault with the two block headers needed to prove the fault // and an optional extra one to check common ancestry (as needed). // Note that the blocks are ordered: the method requires a.Epoch() <= b.Epoch(). -func (ss *syscallShim) VerifyConsensusFault(a, b, extra []byte) (*runtime.ConsensusFault, error) { +func (ss *syscallShim) VerifyConsensusFault(a, b, extra []byte) (*runtime2.ConsensusFault, error) { // Note that block syntax is not validated. Any validly signed block will be accepted pursuant to the below conditions. // Whether or not it could ever have been accepted in a chain is not checked/does not matter here. // for that reason when checking block parent relationships, rather than instantiating a Tipset to do so @@ -115,14 +115,14 @@ func (ss *syscallShim) VerifyConsensusFault(a, b, extra []byte) (*runtime.Consen } // (2) check for the consensus faults themselves - var consensusFault *runtime.ConsensusFault + var consensusFault *runtime2.ConsensusFault // (a) double-fork mining fault if blockA.Height == blockB.Height { - consensusFault = &runtime.ConsensusFault{ + consensusFault = &runtime2.ConsensusFault{ Target: blockA.Miner, Epoch: blockB.Height, - Type: runtime.ConsensusFaultDoubleForkMining, + Type: runtime2.ConsensusFaultDoubleForkMining, } } @@ -130,10 +130,10 @@ func (ss *syscallShim) VerifyConsensusFault(a, b, extra []byte) (*runtime.Consen // strictly speaking no need to compare heights based on double fork mining check above, // but at same height this would be a different fault. if types.CidArrsEqual(blockA.Parents, blockB.Parents) && blockA.Height != blockB.Height { - consensusFault = &runtime.ConsensusFault{ + consensusFault = &runtime2.ConsensusFault{ Target: blockA.Miner, Epoch: blockB.Height, - Type: runtime.ConsensusFaultTimeOffsetMining, + Type: runtime2.ConsensusFaultTimeOffsetMining, } } @@ -153,10 +153,10 @@ func (ss *syscallShim) VerifyConsensusFault(a, b, extra []byte) (*runtime.Consen if types.CidArrsEqual(blockA.Parents, blockC.Parents) && blockA.Height == blockC.Height && types.CidArrsContains(blockB.Parents, blockC.Cid()) && !types.CidArrsContains(blockB.Parents, blockA.Cid()) { - consensusFault = &runtime.ConsensusFault{ + consensusFault = &runtime2.ConsensusFault{ Target: blockA.Miner, Epoch: blockB.Height, - Type: runtime.ConsensusFaultParentGrinding, + Type: runtime2.ConsensusFaultParentGrinding, } } } @@ -215,7 +215,7 @@ func (ss *syscallShim) VerifyBlockSig(blk *types.BlockHeader) error { return nil } -func (ss *syscallShim) VerifyPoSt(proof proof.WindowPoStVerifyInfo) error { +func (ss *syscallShim) VerifyPoSt(proof proof2.WindowPoStVerifyInfo) error { ok, err := ss.verifier.VerifyWindowPoSt(context.TODO(), proof) if err != nil { return err @@ -226,7 +226,7 @@ func (ss *syscallShim) VerifyPoSt(proof proof.WindowPoStVerifyInfo) error { return nil } -func (ss *syscallShim) VerifySeal(info proof.SealVerifyInfo) error { +func (ss *syscallShim) VerifySeal(info proof2.SealVerifyInfo) error { //_, span := trace.StartSpan(ctx, "ValidatePoRep") //defer span.End() @@ -266,7 +266,7 @@ func (ss *syscallShim) VerifySignature(sig crypto.Signature, addr address.Addres var BatchSealVerifyParallelism = goruntime.NumCPU() -func (ss *syscallShim) BatchVerifySeals(inp map[address.Address][]proof.SealVerifyInfo) (map[address.Address][]bool, error) { +func (ss *syscallShim) BatchVerifySeals(inp map[address.Address][]proof2.SealVerifyInfo) (map[address.Address][]bool, error) { out := make(map[address.Address][]bool) sema := make(chan struct{}, BatchSealVerifyParallelism) @@ -278,7 +278,7 @@ func (ss *syscallShim) BatchVerifySeals(inp map[address.Address][]proof.SealVeri for i, s := range seals { wg.Add(1) - go func(ma address.Address, ix int, svi proof.SealVerifyInfo, res []bool) { + go func(ma address.Address, ix int, svi proof2.SealVerifyInfo, res []bool) { defer wg.Done() sema <- struct{}{} diff --git a/cli/client.go b/cli/client.go index 01de8801c..83c452555 100644 --- a/cli/client.go +++ b/cli/client.go @@ -24,7 +24,6 @@ import ( "github.com/fatih/color" datatransfer "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/go-fil-markets/retrievalmarket" - "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/ipfs/go-cid" "github.com/ipfs/go-cidutil/cidenc" "github.com/libp2p/go-libp2p-core/peer" @@ -41,6 +40,7 @@ import ( "github.com/filecoin-project/lotus/api" lapi "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/builtin/market" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/tablewriter" diff --git a/cli/multisig.go b/cli/multisig.go index 7cf7ed750..ebdc8ef7d 100644 --- a/cli/multisig.go +++ b/cli/multisig.go @@ -28,8 +28,8 @@ import ( "github.com/urfave/cli/v2" "golang.org/x/xerrors" - init0 "github.com/filecoin-project/specs-actors/actors/builtin/init" - msig0 "github.com/filecoin-project/specs-actors/actors/builtin/multisig" + init2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/init" + msig2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/multisig" "github.com/filecoin-project/lotus/api/apibstore" "github.com/filecoin-project/lotus/build" @@ -167,7 +167,7 @@ var msigCreateCmd = &cli.Command{ // get address of newly created miner - var execreturn init0.ExecReturn + var execreturn init2.ExecReturn if err := execreturn.UnmarshalCBOR(bytes.NewReader(wait.Receipt.Return)); err != nil { return err } @@ -427,7 +427,7 @@ var msigProposeCmd = &cli.Command{ return fmt.Errorf("proposal returned exit %d", wait.Receipt.ExitCode) } - var retval msig0.ProposeReturn + var retval msig2.ProposeReturn if err := retval.UnmarshalCBOR(bytes.NewReader(wait.Receipt.Return)); err != nil { return fmt.Errorf("failed to unmarshal propose return value: %w", err) } @@ -1160,7 +1160,7 @@ var msigLockProposeCmd = &cli.Command{ from = defaddr } - params, actErr := actors.SerializeParams(&msig0.LockBalanceParams{ + params, actErr := actors.SerializeParams(&msig2.LockBalanceParams{ StartEpoch: abi.ChainEpoch(start), UnlockDuration: abi.ChainEpoch(duration), Amount: abi.NewTokenAmount(amount.Int64()), @@ -1257,7 +1257,7 @@ var msigLockApproveCmd = &cli.Command{ from = defaddr } - params, actErr := actors.SerializeParams(&msig0.LockBalanceParams{ + params, actErr := actors.SerializeParams(&msig2.LockBalanceParams{ StartEpoch: abi.ChainEpoch(start), UnlockDuration: abi.ChainEpoch(duration), Amount: abi.NewTokenAmount(amount.Int64()), @@ -1349,7 +1349,7 @@ var msigLockCancelCmd = &cli.Command{ from = defaddr } - params, actErr := actors.SerializeParams(&msig0.LockBalanceParams{ + params, actErr := actors.SerializeParams(&msig2.LockBalanceParams{ StartEpoch: abi.ChainEpoch(start), UnlockDuration: abi.ChainEpoch(duration), Amount: abi.NewTokenAmount(amount.Int64()), @@ -1488,7 +1488,7 @@ var msigProposeThresholdCmd = &cli.Command{ from = defaddr } - params, actErr := actors.SerializeParams(&msig0.ChangeNumApprovalsThresholdParams{ + params, actErr := actors.SerializeParams(&msig2.ChangeNumApprovalsThresholdParams{ NewThreshold: newM, }) diff --git a/cmd/lotus-bench/caching_verifier.go b/cmd/lotus-bench/caching_verifier.go index 51ab696f7..5b434c762 100644 --- a/cmd/lotus-bench/caching_verifier.go +++ b/cmd/lotus-bench/caching_verifier.go @@ -7,7 +7,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" - "github.com/filecoin-project/specs-actors/actors/runtime/proof" + proof2 "github.com/filecoin-project/specs-actors/v2/actors/runtime/proof" "github.com/ipfs/go-datastore" "github.com/minio/blake2b-simd" cbg "github.com/whyrusleeping/cbor-gen" @@ -78,15 +78,16 @@ func (cv cachingVerifier) withCache(execute func() (bool, error), param cbg.CBOR } } -func (cv *cachingVerifier) VerifySeal(svi proof.SealVerifyInfo) (bool, error) { +func (cv *cachingVerifier) VerifySeal(svi proof2.SealVerifyInfo) (bool, error) { return cv.withCache(func() (bool, error) { return cv.backend.VerifySeal(svi) }, &svi) } -func (cv *cachingVerifier) VerifyWinningPoSt(ctx context.Context, info proof.WinningPoStVerifyInfo) (bool, error) { + +func (cv *cachingVerifier) VerifyWinningPoSt(ctx context.Context, info proof2.WinningPoStVerifyInfo) (bool, error) { return cv.backend.VerifyWinningPoSt(ctx, info) } -func (cv *cachingVerifier) VerifyWindowPoSt(ctx context.Context, info proof.WindowPoStVerifyInfo) (bool, error) { +func (cv *cachingVerifier) VerifyWindowPoSt(ctx context.Context, info proof2.WindowPoStVerifyInfo) (bool, error) { return cv.withCache(func() (bool, error) { return cv.backend.VerifyWindowPoSt(ctx, info) }, &info) diff --git a/cmd/lotus-bench/main.go b/cmd/lotus-bench/main.go index e409dfe5a..ac39a9714 100644 --- a/cmd/lotus-bench/main.go +++ b/cmd/lotus-bench/main.go @@ -11,7 +11,7 @@ import ( "path/filepath" "time" - saproof "github.com/filecoin-project/specs-actors/actors/runtime/proof" + saproof2 "github.com/filecoin-project/specs-actors/v2/actors/runtime/proof" "github.com/docker/go-units" logging "github.com/ipfs/go-log/v2" @@ -237,7 +237,7 @@ var sealBenchCmd = &cli.Command{ } var sealTimings []SealingResult - var sealedSectors []saproof.SectorInfo + var sealedSectors []saproof2.SectorInfo if robench == "" { var err error @@ -280,7 +280,7 @@ var sealBenchCmd = &cli.Command{ } for _, s := range genm.Sectors { - sealedSectors = append(sealedSectors, saproof.SectorInfo{ + sealedSectors = append(sealedSectors, saproof2.SectorInfo{ SealedCID: s.CommR, SectorNumber: s.SectorID, SealProof: s.ProofType, @@ -305,7 +305,7 @@ var sealBenchCmd = &cli.Command{ return err } - candidates := make([]saproof.SectorInfo, len(fcandidates)) + candidates := make([]saproof2.SectorInfo, len(fcandidates)) for i, fcandidate := range fcandidates { candidates[i] = sealedSectors[fcandidate] } @@ -328,7 +328,7 @@ var sealBenchCmd = &cli.Command{ winnningpost2 := time.Now() - pvi1 := saproof.WinningPoStVerifyInfo{ + pvi1 := saproof2.WinningPoStVerifyInfo{ Randomness: abi.PoStRandomness(challenge[:]), Proofs: proof1, ChallengedSectors: candidates, @@ -344,7 +344,7 @@ var sealBenchCmd = &cli.Command{ verifyWinningPost1 := time.Now() - pvi2 := saproof.WinningPoStVerifyInfo{ + pvi2 := saproof2.WinningPoStVerifyInfo{ Randomness: abi.PoStRandomness(challenge[:]), Proofs: proof2, ChallengedSectors: candidates, @@ -376,7 +376,7 @@ var sealBenchCmd = &cli.Command{ windowpost2 := time.Now() - wpvi1 := saproof.WindowPoStVerifyInfo{ + wpvi1 := saproof2.WindowPoStVerifyInfo{ Randomness: challenge[:], Proofs: wproof1, ChallengedSectors: sealedSectors, @@ -392,7 +392,7 @@ var sealBenchCmd = &cli.Command{ verifyWindowpost1 := time.Now() - wpvi2 := saproof.WindowPoStVerifyInfo{ + wpvi2 := saproof2.WindowPoStVerifyInfo{ Randomness: challenge[:], Proofs: wproof2, ChallengedSectors: sealedSectors, @@ -464,10 +464,10 @@ type ParCfg struct { Commit int } -func runSeals(sb *ffiwrapper.Sealer, sbfs *basicfs.Provider, numSectors int, par ParCfg, mid abi.ActorID, sectorSize abi.SectorSize, ticketPreimage []byte, saveC2inp string, skipc2, skipunseal bool) ([]SealingResult, []saproof.SectorInfo, error) { +func runSeals(sb *ffiwrapper.Sealer, sbfs *basicfs.Provider, numSectors int, par ParCfg, mid abi.ActorID, sectorSize abi.SectorSize, ticketPreimage []byte, saveC2inp string, skipc2, skipunseal bool) ([]SealingResult, []saproof2.SectorInfo, error) { var pieces []abi.PieceInfo sealTimings := make([]SealingResult, numSectors) - sealedSectors := make([]saproof.SectorInfo, numSectors) + sealedSectors := make([]saproof2.SectorInfo, numSectors) preCommit2Sema := make(chan struct{}, par.PreCommit2) commitSema := make(chan struct{}, par.Commit) @@ -537,7 +537,7 @@ func runSeals(sb *ffiwrapper.Sealer, sbfs *basicfs.Provider, numSectors int, par precommit2 := time.Now() <-preCommit2Sema - sealedSectors[ix] = saproof.SectorInfo{ + sealedSectors[ix] = saproof2.SectorInfo{ SealProof: sb.SealProofType(), SectorNumber: i, SealedCID: cids.Sealed, @@ -589,7 +589,7 @@ func runSeals(sb *ffiwrapper.Sealer, sbfs *basicfs.Provider, numSectors int, par <-commitSema if !skipc2 { - svi := saproof.SealVerifyInfo{ + svi := saproof2.SealVerifyInfo{ SectorID: abi.SectorID{Miner: mid, Number: i}, SealedCID: cids.Sealed, SealProof: sb.SealProofType(), diff --git a/cmd/lotus-chainwatch/processor/processor.go b/cmd/lotus-chainwatch/processor/processor.go index bce2b9fb7..1f8b246ed 100644 --- a/cmd/lotus-chainwatch/processor/processor.go +++ b/cmd/lotus-chainwatch/processor/processor.go @@ -15,7 +15,7 @@ import ( logging "github.com/ipfs/go-log/v2" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/specs-actors/actors/builtin" + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" @@ -134,10 +134,10 @@ func (p *Processor) Start(ctx context.Context) { log.Fatalw("Failed to collect actor changes", "error", err) } log.Infow("Collected Actor Changes", - "MarketChanges", len(actorChanges[builtin.StorageMarketActorCodeID]), - "MinerChanges", len(actorChanges[builtin.StorageMinerActorCodeID]), - "RewardChanges", len(actorChanges[builtin.RewardActorCodeID]), - "AccountChanges", len(actorChanges[builtin.AccountActorCodeID]), + "MarketChanges", len(actorChanges[builtin2.StorageMarketActorCodeID]), + "MinerChanges", len(actorChanges[builtin2.StorageMinerActorCodeID]), + "RewardChanges", len(actorChanges[builtin2.RewardActorCodeID]), + "AccountChanges", len(actorChanges[builtin2.AccountActorCodeID]), "nullRounds", len(nullRounds)) grp := sync.WaitGroup{} @@ -145,7 +145,7 @@ func (p *Processor) Start(ctx context.Context) { grp.Add(1) go func() { defer grp.Done() - if err := p.HandleMarketChanges(ctx, actorChanges[builtin.StorageMarketActorCodeID]); err != nil { + if err := p.HandleMarketChanges(ctx, actorChanges[builtin2.StorageMarketActorCodeID]); err != nil { log.Errorf("Failed to handle market changes: %w", err) return } @@ -154,7 +154,7 @@ func (p *Processor) Start(ctx context.Context) { grp.Add(1) go func() { defer grp.Done() - if err := p.HandleMinerChanges(ctx, actorChanges[builtin.StorageMinerActorCodeID]); err != nil { + if err := p.HandleMinerChanges(ctx, actorChanges[builtin2.StorageMinerActorCodeID]); err != nil { log.Errorf("Failed to handle miner changes: %w", err) return } @@ -163,7 +163,7 @@ func (p *Processor) Start(ctx context.Context) { grp.Add(1) go func() { defer grp.Done() - if err := p.HandleRewardChanges(ctx, actorChanges[builtin.RewardActorCodeID], nullRounds); err != nil { + if err := p.HandleRewardChanges(ctx, actorChanges[builtin2.RewardActorCodeID], nullRounds); err != nil { log.Errorf("Failed to handle reward changes: %w", err) return } @@ -172,7 +172,7 @@ func (p *Processor) Start(ctx context.Context) { grp.Add(1) go func() { defer grp.Done() - if err := p.HandlePowerChanges(ctx, actorChanges[builtin.StoragePowerActorCodeID]); err != nil { + if err := p.HandlePowerChanges(ctx, actorChanges[builtin2.StoragePowerActorCodeID]); err != nil { log.Errorf("Failed to handle power actor changes: %w", err) return } diff --git a/cmd/lotus-gateway/endtoend_test.go b/cmd/lotus-gateway/endtoend_test.go index 379c96053..b043ce28c 100644 --- a/cmd/lotus-gateway/endtoend_test.go +++ b/cmd/lotus-gateway/endtoend_test.go @@ -12,8 +12,8 @@ import ( "github.com/filecoin-project/lotus/cli" clitest "github.com/filecoin-project/lotus/cli/test" - init0 "github.com/filecoin-project/specs-actors/actors/builtin/init" - "github.com/filecoin-project/specs-actors/actors/builtin/multisig" + init2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/init" + multisig2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/multisig" "github.com/stretchr/testify/require" "golang.org/x/xerrors" @@ -101,7 +101,7 @@ func TestEndToEndWalletMsig(t *testing.T) { require.NoError(t, err) require.EqualValues(t, 0, res.Receipt.ExitCode) - var execReturn init0.ExecReturn + var execReturn init2.ExecReturn err = execReturn.UnmarshalCBOR(bytes.NewReader(res.Receipt.Return)) require.NoError(t, err) @@ -121,7 +121,7 @@ func TestEndToEndWalletMsig(t *testing.T) { require.NoError(t, err) require.EqualValues(t, 0, res.Receipt.ExitCode) - var proposeReturn multisig.ProposeReturn + var proposeReturn multisig2.ProposeReturn err = proposeReturn.UnmarshalCBOR(bytes.NewReader(res.Receipt.Return)) require.NoError(t, err) @@ -135,7 +135,7 @@ func TestEndToEndWalletMsig(t *testing.T) { require.NoError(t, err) require.EqualValues(t, 0, res.Receipt.ExitCode) - var approveReturn multisig.ApproveReturn + var approveReturn multisig2.ApproveReturn err = approveReturn.UnmarshalCBOR(bytes.NewReader(res.Receipt.Return)) require.NoError(t, err) require.True(t, approveReturn.Applied) diff --git a/cmd/lotus-pcr/main.go b/cmd/lotus-pcr/main.go index f88a0a9dc..02df4a192 100644 --- a/cmd/lotus-pcr/main.go +++ b/cmd/lotus-pcr/main.go @@ -16,9 +16,8 @@ import ( "github.com/filecoin-project/lotus/chain/actors/builtin" - builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" - - miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" "github.com/filecoin-project/go-state-types/network" @@ -871,7 +870,7 @@ func (r *refunder) processTipsetStorageMarketActor(ctx context.Context, tipset * var messageMethod string switch m.Method { - case builtin0.MethodsMarket.PublishStorageDeals: + case builtin2.MethodsMarket.PublishStorageDeals: if !r.publishStorageDealsEnabled { return false, messageMethod, types.NewInt(0), nil } @@ -898,7 +897,7 @@ func (r *refunder) processTipsetStorageMinerActor(ctx context.Context, tipset *t var messageMethod string switch m.Method { - case builtin0.MethodsMiner.SubmitWindowedPoSt: + case builtin2.MethodsMiner.SubmitWindowedPoSt: if !r.windowedPoStEnabled { return false, messageMethod, types.NewInt(0), nil } @@ -911,7 +910,7 @@ func (r *refunder) processTipsetStorageMinerActor(ctx context.Context, tipset *t } refundValue = types.BigMul(types.NewInt(uint64(recp.GasUsed)), tipset.Blocks()[0].ParentBaseFee) - case builtin0.MethodsMiner.ProveCommitSector: + case builtin2.MethodsMiner.ProveCommitSector: if !r.proveCommitEnabled { return false, messageMethod, types.NewInt(0), nil } @@ -930,7 +929,7 @@ func (r *refunder) processTipsetStorageMinerActor(ctx context.Context, tipset *t var sn abi.SectorNumber - var proveCommitSector miner0.ProveCommitSectorParams + var proveCommitSector miner2.ProveCommitSectorParams if err := proveCommitSector.UnmarshalCBOR(bytes.NewBuffer(m.Params)); err != nil { log.Warnw("failed to decode provecommit params", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To) return false, messageMethod, types.NewInt(0), nil @@ -967,7 +966,7 @@ func (r *refunder) processTipsetStorageMinerActor(ctx context.Context, tipset *t if r.refundPercent > 0 { refundValue = types.BigMul(types.BigDiv(refundValue, types.NewInt(100)), types.NewInt(uint64(r.refundPercent))) } - case builtin0.MethodsMiner.PreCommitSector: + case builtin2.MethodsMiner.PreCommitSector: if !r.preCommitEnabled { return false, messageMethod, types.NewInt(0), nil } diff --git a/cmd/lotus-seed/seed/seed.go b/cmd/lotus-seed/seed/seed.go index cd0c4e7ab..49c259084 100644 --- a/cmd/lotus-seed/seed/seed.go +++ b/cmd/lotus-seed/seed/seed.go @@ -22,7 +22,8 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/lotus/extern/sector-storage/zerocomm" - "github.com/filecoin-project/specs-actors/actors/builtin/market" + + market2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/market" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/wallet" @@ -252,7 +253,7 @@ func WriteGenesisMiner(maddr address.Address, sbroot string, gm *genesis.Miner, func createDeals(m *genesis.Miner, k *wallet.Key, maddr address.Address, ssize abi.SectorSize) error { for i, sector := range m.Sectors { - proposal := &market.DealProposal{ + proposal := &market2.DealProposal{ PieceCID: sector.CommD, PieceSize: abi.PaddedPieceSize(ssize), Client: k.Address, diff --git a/cmd/lotus-shed/mempool-stats.go b/cmd/lotus-shed/mempool-stats.go index d70cd4b71..408e06f71 100644 --- a/cmd/lotus-shed/mempool-stats.go +++ b/cmd/lotus-shed/mempool-stats.go @@ -14,7 +14,7 @@ import ( "go.opencensus.io/stats/view" "go.opencensus.io/tag" - builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" "github.com/filecoin-project/go-address" lapi "github.com/filecoin-project/lotus/api" @@ -145,7 +145,7 @@ var mpoolStatsCmd = &cli.Command{ seen: time.Now(), } - if u.Message.Message.Method == builtin0.MethodsMiner.SubmitWindowedPoSt { + if u.Message.Message.Method == builtin2.MethodsMiner.SubmitWindowedPoSt { miner, err := isMiner(u.Message.Message.To) if err != nil { diff --git a/cmd/lotus-shed/proofs.go b/cmd/lotus-shed/proofs.go index 2379d8599..e75aeed14 100644 --- a/cmd/lotus-shed/proofs.go +++ b/cmd/lotus-shed/proofs.go @@ -4,7 +4,7 @@ import ( "encoding/hex" "fmt" - saproof "github.com/filecoin-project/specs-actors/actors/runtime/proof" + proof2 "github.com/filecoin-project/specs-actors/v2/actors/runtime/proof" "github.com/urfave/cli/v2" @@ -84,7 +84,7 @@ var verifySealProofCmd = &cli.Command{ snum := abi.SectorNumber(cctx.Uint64("sector-id")) - ok, err := ffi.VerifySeal(saproof.SealVerifyInfo{ + ok, err := ffi.VerifySeal(proof2.SealVerifyInfo{ SectorID: abi.SectorID{ Miner: abi.ActorID(mid), Number: snum, diff --git a/cmd/lotus-shed/sectors.go b/cmd/lotus-shed/sectors.go index 11e678642..cf0d5fcdb 100644 --- a/cmd/lotus-shed/sectors.go +++ b/cmd/lotus-shed/sectors.go @@ -12,7 +12,7 @@ import ( "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" - "github.com/filecoin-project/specs-actors/actors/builtin" + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" miner0 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" "github.com/urfave/cli/v2" ) @@ -106,7 +106,7 @@ var terminateSectorCmd = &cli.Command{ smsg, err := nodeApi.MpoolPushMessage(ctx, &types.Message{ From: mi.Owner, To: maddr, - Method: builtin.MethodsMiner.TerminateSectors, + Method: builtin2.MethodsMiner.TerminateSectors, Value: big.Zero(), Params: sp, diff --git a/cmd/lotus-shed/verifreg.go b/cmd/lotus-shed/verifreg.go index 860498302..c4eada68c 100644 --- a/cmd/lotus-shed/verifreg.go +++ b/cmd/lotus-shed/verifreg.go @@ -11,8 +11,8 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" - builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" - verifreg0 "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + verifreg2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/verifreg" "github.com/filecoin-project/lotus/api/apibstore" "github.com/filecoin-project/lotus/build" @@ -63,7 +63,7 @@ var verifRegAddVerifierCmd = &cli.Command{ } // TODO: ActorUpgrade: Abstract - params, err := actors.SerializeParams(&verifreg0.AddVerifierParams{Address: verifier, Allowance: allowance}) + params, err := actors.SerializeParams(&verifreg2.AddVerifierParams{Address: verifier, Allowance: allowance}) if err != nil { return err } @@ -80,7 +80,7 @@ var verifRegAddVerifierCmd = &cli.Command{ return err } - smsg, err := api.MsigPropose(ctx, vrk, verifreg.Address, big.Zero(), sender, uint64(builtin0.MethodsVerifiedRegistry.AddVerifier), params) + smsg, err := api.MsigPropose(ctx, vrk, verifreg.Address, big.Zero(), sender, uint64(builtin2.MethodsVerifiedRegistry.AddVerifier), params) if err != nil { return err } @@ -136,7 +136,7 @@ var verifRegVerifyClientCmd = &cli.Command{ return err } - params, err := actors.SerializeParams(&verifreg0.AddVerifiedClientParams{Address: target, Allowance: allowance}) + params, err := actors.SerializeParams(&verifreg2.AddVerifiedClientParams{Address: target, Allowance: allowance}) if err != nil { return err } @@ -151,7 +151,7 @@ var verifRegVerifyClientCmd = &cli.Command{ msg := &types.Message{ To: verifreg.Address, From: fromk, - Method: builtin0.MethodsVerifiedRegistry.AddVerifiedClient, + Method: builtin2.MethodsVerifiedRegistry.AddVerifiedClient, Params: params, } diff --git a/cmd/lotus-storage-miner/actor.go b/cmd/lotus-storage-miner/actor.go index 35e2a268e..ad2232e1c 100644 --- a/cmd/lotus-storage-miner/actor.go +++ b/cmd/lotus-storage-miner/actor.go @@ -11,7 +11,6 @@ import ( cbor "github.com/ipfs/go-ipld-cbor" "github.com/filecoin-project/lotus/build" - builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" "github.com/fatih/color" "github.com/libp2p/go-libp2p-core/peer" @@ -23,8 +22,8 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" - "github.com/filecoin-project/specs-actors/actors/builtin" - miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/types" @@ -98,7 +97,7 @@ var actorSetAddrsCmd = &cli.Command{ return err } - params, err := actors.SerializeParams(&miner0.ChangeMultiaddrsParams{NewMultiaddrs: addrs}) + params, err := actors.SerializeParams(&miner2.ChangeMultiaddrsParams{NewMultiaddrs: addrs}) if err != nil { return err } @@ -110,7 +109,7 @@ var actorSetAddrsCmd = &cli.Command{ From: minfo.Worker, Value: types.NewInt(0), GasLimit: gasLimit, - Method: builtin.MethodsMiner.ChangeMultiaddrs, + Method: builtin2.MethodsMiner.ChangeMultiaddrs, Params: params, }, nil) if err != nil { @@ -163,7 +162,7 @@ var actorSetPeeridCmd = &cli.Command{ return err } - params, err := actors.SerializeParams(&miner0.ChangePeerIDParams{NewID: abi.PeerID(pid)}) + params, err := actors.SerializeParams(&miner2.ChangePeerIDParams{NewID: abi.PeerID(pid)}) if err != nil { return err } @@ -175,7 +174,7 @@ var actorSetPeeridCmd = &cli.Command{ From: minfo.Worker, Value: types.NewInt(0), GasLimit: gasLimit, - Method: builtin.MethodsMiner.ChangePeerID, + Method: builtin2.MethodsMiner.ChangePeerID, Params: params, }, nil) if err != nil { @@ -236,7 +235,7 @@ var actorWithdrawCmd = &cli.Command{ } } - params, err := actors.SerializeParams(&miner0.WithdrawBalanceParams{ + params, err := actors.SerializeParams(&miner2.WithdrawBalanceParams{ AmountRequested: amount, // Default to attempting to withdraw all the extra funds in the miner actor }) if err != nil { @@ -247,7 +246,7 @@ var actorWithdrawCmd = &cli.Command{ To: maddr, From: mi.Owner, Value: types.NewInt(0), - Method: builtin.MethodsMiner.WithdrawBalance, + Method: builtin2.MethodsMiner.WithdrawBalance, Params: params, }, nil) if err != nil { @@ -560,7 +559,7 @@ var actorControlSet = &cli.Command{ return nil } - cwp := &miner0.ChangeWorkerAddressParams{ + cwp := &miner2.ChangeWorkerAddressParams{ NewWorker: mi.Worker, NewControlAddrs: toSet, } @@ -573,7 +572,7 @@ var actorControlSet = &cli.Command{ smsg, err := api.MpoolPushMessage(ctx, &types.Message{ From: mi.Owner, To: maddr, - Method: builtin.MethodsMiner.ChangeWorkerAddress, + Method: builtin2.MethodsMiner.ChangeWorkerAddress, Value: big.Zero(), Params: sp, diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index 326a39e5e..b39ede752 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -31,14 +31,16 @@ import ( "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/extern/sector-storage/stores" - builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" - market0 "github.com/filecoin-project/specs-actors/actors/builtin/market" - miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" - power0 "github.com/filecoin-project/specs-actors/actors/builtin/power" + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + market2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/market" + miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" + power2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/power" lapi "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/actors/builtin/power" + "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/gen/slashfilter" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" @@ -82,7 +84,7 @@ var initCmd = &cli.Command{ &cli.StringFlag{ Name: "sector-size", Usage: "specify sector size to use", - Value: units.BytesSize(float64(build.DefaultSectorSize())), + Value: units.BytesSize(float64(policy.GetDefaultSectorSize())), }, &cli.StringSliceFlag{ Name: "pre-sealed-sectors", @@ -375,7 +377,7 @@ func migratePreSealMeta(ctx context.Context, api lapi.FullNode, metadata string, return mds.Put(datastore.NewKey(modules.StorageCounterDSPrefix), buf[:size]) } -func findMarketDealID(ctx context.Context, api lapi.FullNode, deal market0.DealProposal) (abi.DealID, error) { +func findMarketDealID(ctx context.Context, api lapi.FullNode, deal market2.DealProposal) (abi.DealID, error) { // TODO: find a better way // (this is only used by genesis miners) @@ -568,7 +570,7 @@ func configureStorageMiner(ctx context.Context, api lapi.FullNode, addr address. return xerrors.Errorf("getWorkerAddr returned bad address: %w", err) } - enc, err := actors.SerializeParams(&miner0.ChangePeerIDParams{NewID: abi.PeerID(peerid)}) + enc, err := actors.SerializeParams(&miner2.ChangePeerIDParams{NewID: abi.PeerID(peerid)}) if err != nil { return err } @@ -576,7 +578,7 @@ func configureStorageMiner(ctx context.Context, api lapi.FullNode, addr address. msg := &types.Message{ To: addr, From: mi.Worker, - Method: builtin0.MethodsMiner.ChangePeerID, + Method: builtin2.MethodsMiner.ChangePeerID, Params: enc, Value: types.NewInt(0), GasPremium: gasPrice, @@ -635,7 +637,7 @@ func createStorageMiner(ctx context.Context, api lapi.FullNode, peerid peer.ID, return address.Undef, err } - params, err := actors.SerializeParams(&power0.CreateMinerParams{ + params, err := actors.SerializeParams(&power2.CreateMinerParams{ Owner: owner, Worker: worker, SealProofType: spt, @@ -655,11 +657,11 @@ func createStorageMiner(ctx context.Context, api lapi.FullNode, peerid peer.ID, } createStorageMinerMsg := &types.Message{ - To: builtin0.StoragePowerActorAddr, + To: power.Address, From: sender, Value: big.Zero(), - Method: builtin0.MethodsPower.CreateMiner, + Method: builtin2.MethodsPower.CreateMiner, Params: params, GasLimit: 0, @@ -683,7 +685,7 @@ func createStorageMiner(ctx context.Context, api lapi.FullNode, peerid peer.ID, return address.Undef, xerrors.Errorf("create miner failed: exit code %d", mw.Receipt.ExitCode) } - var retval power0.CreateMinerReturn + var retval power2.CreateMinerReturn if err := retval.UnmarshalCBOR(bytes.NewReader(mw.Receipt.Return)); err != nil { return address.Undef, err } diff --git a/cmd/lotus-storage-miner/sectors.go b/cmd/lotus-storage-miner/sectors.go index b50f4a86d..0c3ef58d6 100644 --- a/cmd/lotus-storage-miner/sectors.go +++ b/cmd/lotus-storage-miner/sectors.go @@ -15,10 +15,9 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" - miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" - "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/tablewriter" @@ -463,7 +462,7 @@ var sectorsCapacityCollateralCmd = &cli.Command{ Expiration: abi.ChainEpoch(cctx.Uint64("expiration")), } if pci.Expiration == 0 { - pci.Expiration = miner0.MaxSectorExpirationExtension + pci.Expiration = policy.GetMaxSectorExpirationExtension() } pc, err := nApi.StateMinerInitialPledgeCollateral(ctx, maddr, pci, types.EmptyTSK) if err != nil { diff --git a/cmd/tvx/extract.go b/cmd/tvx/extract.go index 4a8dd162c..276c38cd4 100644 --- a/cmd/tvx/extract.go +++ b/cmd/tvx/extract.go @@ -14,6 +14,7 @@ import ( "github.com/fatih/color" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/actors/builtin" init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init" "github.com/filecoin-project/lotus/chain/actors/builtin/reward" "github.com/filecoin-project/lotus/chain/types" @@ -21,8 +22,6 @@ import ( lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/conformance" - "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/test-vectors/schema" "github.com/ipfs/go-cid" diff --git a/conformance/chaos/actor.go b/conformance/chaos/actor.go index d5e0b4352..3c280e27e 100644 --- a/conformance/chaos/actor.go +++ b/conformance/chaos/actor.go @@ -6,9 +6,10 @@ import ( "github.com/filecoin-project/go-state-types/cbor" "github.com/filecoin-project/go-state-types/exitcode" "github.com/filecoin-project/go-state-types/rt" - "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/runtime" "github.com/ipfs/go-cid" + + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + runtime2 "github.com/filecoin-project/specs-actors/v2/actors/runtime" ) //go:generate go run ./gen @@ -55,7 +56,7 @@ const ( const ( _ = 0 // skip zero iota value; first usage of iota gets 1. - MethodCallerValidation = builtin.MethodConstructor + iota + MethodCallerValidation = builtin2.MethodConstructor + iota MethodCreateActor MethodResolveAddress // MethodDeleteActor is the identifier for the method that deletes this actor. @@ -76,15 +77,15 @@ const ( // Exports defines the methods this actor exposes publicly. func (a Actor) Exports() []interface{} { return []interface{}{ - builtin.MethodConstructor: a.Constructor, - MethodCallerValidation: a.CallerValidation, - MethodCreateActor: a.CreateActor, - MethodResolveAddress: a.ResolveAddress, - MethodDeleteActor: a.DeleteActor, - MethodSend: a.Send, - MethodMutateState: a.MutateState, - MethodAbortWith: a.AbortWith, - MethodInspectRuntime: a.InspectRuntime, + builtin2.MethodConstructor: a.Constructor, + MethodCallerValidation: a.CallerValidation, + MethodCreateActor: a.CreateActor, + MethodResolveAddress: a.ResolveAddress, + MethodDeleteActor: a.DeleteActor, + MethodSend: a.Send, + MethodMutateState: a.MutateState, + MethodAbortWith: a.AbortWith, + MethodInspectRuntime: a.InspectRuntime, } } @@ -104,19 +105,19 @@ type SendArgs struct { // SendReturn is the return values for the Send method. type SendReturn struct { - Return runtime.CBORBytes + Return builtin2.CBORBytes Code exitcode.ExitCode } // Send requests for this actor to send a message to an actor with the // passed parameters. -func (a Actor) Send(rt runtime.Runtime, args *SendArgs) *SendReturn { +func (a Actor) Send(rt runtime2.Runtime, args *SendArgs) *SendReturn { rt.ValidateImmediateCallerAcceptAny() - var out runtime.CBORBytes + var out builtin2.CBORBytes code := rt.Send( args.To, args.Method, - runtime.CBORBytes(args.Params), + builtin2.CBORBytes(args.Params), args.Value, &out, ) @@ -127,7 +128,7 @@ func (a Actor) Send(rt runtime.Runtime, args *SendArgs) *SendReturn { } // Constructor will panic because the Chaos actor is a singleton. -func (a Actor) Constructor(_ runtime.Runtime, _ *abi.EmptyValue) *abi.EmptyValue { +func (a Actor) Constructor(_ runtime2.Runtime, _ *abi.EmptyValue) *abi.EmptyValue { panic("constructor should not be called; the Chaos actor is a singleton actor") } @@ -144,7 +145,7 @@ type CallerValidationArgs struct { // CallerValidationBranchTwice validates twice. // CallerValidationBranchIsAddress validates caller against CallerValidationArgs.Addrs. // CallerValidationBranchIsType validates caller against CallerValidationArgs.Types. -func (a Actor) CallerValidation(rt runtime.Runtime, args *CallerValidationArgs) *abi.EmptyValue { +func (a Actor) CallerValidation(rt runtime2.Runtime, args *CallerValidationArgs) *abi.EmptyValue { switch args.Branch { case CallerValidationBranchNone: case CallerValidationBranchTwice: @@ -174,7 +175,7 @@ type CreateActorArgs struct { } // CreateActor creates an actor with the supplied CID and Address. -func (a Actor) CreateActor(rt runtime.Runtime, args *CreateActorArgs) *abi.EmptyValue { +func (a Actor) CreateActor(rt runtime2.Runtime, args *CreateActorArgs) *abi.EmptyValue { rt.ValidateImmediateCallerAcceptAny() var ( @@ -199,7 +200,7 @@ type ResolveAddressResponse struct { Success bool } -func (a Actor) ResolveAddress(rt runtime.Runtime, args *address.Address) *ResolveAddressResponse { +func (a Actor) ResolveAddress(rt runtime2.Runtime, args *address.Address) *ResolveAddressResponse { rt.ValidateImmediateCallerAcceptAny() resolvedAddr, ok := rt.ResolveAddress(*args) @@ -212,7 +213,7 @@ func (a Actor) ResolveAddress(rt runtime.Runtime, args *address.Address) *Resolv // DeleteActor deletes the executing actor from the state tree, transferring any // balance to beneficiary. -func (a Actor) DeleteActor(rt runtime.Runtime, beneficiary *address.Address) *abi.EmptyValue { +func (a Actor) DeleteActor(rt runtime2.Runtime, beneficiary *address.Address) *abi.EmptyValue { rt.ValidateImmediateCallerAcceptAny() rt.DeleteActor(*beneficiary) return nil @@ -226,7 +227,7 @@ type MutateStateArgs struct { } // MutateState attempts to mutate a state value in the actor. -func (a Actor) MutateState(rt runtime.Runtime, args *MutateStateArgs) *abi.EmptyValue { +func (a Actor) MutateState(rt runtime2.Runtime, args *MutateStateArgs) *abi.EmptyValue { rt.ValidateImmediateCallerAcceptAny() var st State switch args.Branch { @@ -257,7 +258,7 @@ type AbortWithArgs struct { } // AbortWith simply causes a panic with the passed exit code. -func (a Actor) AbortWith(rt runtime.Runtime, args *AbortWithArgs) *abi.EmptyValue { +func (a Actor) AbortWith(rt runtime2.Runtime, args *AbortWithArgs) *abi.EmptyValue { if args.Uncontrolled { // uncontrolled abort: directly panic panic(args.Message) } else { @@ -277,7 +278,7 @@ type InspectRuntimeReturn struct { } // InspectRuntime returns a copy of the serializable values available in the Runtime. -func (a Actor) InspectRuntime(rt runtime.Runtime, _ *abi.EmptyValue) *InspectRuntimeReturn { +func (a Actor) InspectRuntime(rt runtime2.Runtime, _ *abi.EmptyValue) *InspectRuntimeReturn { rt.ValidateImmediateCallerAcceptAny() var st State rt.StateReadonly(&st) diff --git a/conformance/chaos/actor_test.go b/conformance/chaos/actor_test.go index 2061efb82..dbce4f4c5 100644 --- a/conformance/chaos/actor_test.go +++ b/conformance/chaos/actor_test.go @@ -7,15 +7,16 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/exitcode" - "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/support/mock" - atesting "github.com/filecoin-project/specs-actors/support/testing" "github.com/ipfs/go-cid" + + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + mock2 "github.com/filecoin-project/specs-actors/v2/support/mock" + atesting2 "github.com/filecoin-project/specs-actors/v2/support/testing" ) func TestSingleton(t *testing.T) { - receiver := atesting.NewIDAddr(t, 100) - builder := mock.NewBuilder(context.Background(), receiver) + receiver := atesting2.NewIDAddr(t, 100) + builder := mock2.NewBuilder(context.Background(), receiver) rt := builder.Build(t) var a Actor @@ -28,8 +29,8 @@ func TestSingleton(t *testing.T) { } func TestCallerValidationNone(t *testing.T) { - receiver := atesting.NewIDAddr(t, 100) - builder := mock.NewBuilder(context.Background(), receiver) + receiver := atesting2.NewIDAddr(t, 100) + builder := mock2.NewBuilder(context.Background(), receiver) rt := builder.Build(t) var a Actor @@ -39,19 +40,19 @@ func TestCallerValidationNone(t *testing.T) { } func TestCallerValidationIs(t *testing.T) { - caller := atesting.NewIDAddr(t, 100) - receiver := atesting.NewIDAddr(t, 101) - builder := mock.NewBuilder(context.Background(), receiver) + caller := atesting2.NewIDAddr(t, 100) + receiver := atesting2.NewIDAddr(t, 101) + builder := mock2.NewBuilder(context.Background(), receiver) rt := builder.Build(t) - rt.SetCaller(caller, builtin.AccountActorCodeID) + rt.SetCaller(caller, builtin2.AccountActorCodeID) var a Actor - caddrs := []address.Address{atesting.NewIDAddr(t, 101)} + caddrs := []address.Address{atesting2.NewIDAddr(t, 101)} rt.ExpectValidateCallerAddr(caddrs...) - // FIXME: https://github.com/filecoin-project/specs-actors/pull/1155 - rt.ExpectAbort(exitcode.ErrForbidden, func() { + // fixed in: https://github.com/filecoin-project/specs-actors/pull/1155 + rt.ExpectAbort(exitcode.SysErrForbidden, func() { rt.Call(a.CallerValidation, &CallerValidationArgs{ Branch: CallerValidationBranchIsAddress, Addrs: caddrs, @@ -68,35 +69,34 @@ func TestCallerValidationIs(t *testing.T) { } func TestCallerValidationType(t *testing.T) { - caller := atesting.NewIDAddr(t, 100) - receiver := atesting.NewIDAddr(t, 101) - builder := mock.NewBuilder(context.Background(), receiver) + caller := atesting2.NewIDAddr(t, 100) + receiver := atesting2.NewIDAddr(t, 101) + builder := mock2.NewBuilder(context.Background(), receiver) rt := builder.Build(t) - rt.SetCaller(caller, builtin.AccountActorCodeID) + rt.SetCaller(caller, builtin2.AccountActorCodeID) var a Actor - rt.ExpectValidateCallerType(builtin.CronActorCodeID) - // FIXME: https://github.com/filecoin-project/specs-actors/pull/1155 - rt.ExpectAbort(exitcode.ErrForbidden, func() { + rt.ExpectValidateCallerType(builtin2.CronActorCodeID) + rt.ExpectAbort(exitcode.SysErrForbidden, func() { rt.Call(a.CallerValidation, &CallerValidationArgs{ Branch: CallerValidationBranchIsType, - Types: []cid.Cid{builtin.CronActorCodeID}, + Types: []cid.Cid{builtin2.CronActorCodeID}, }) }) rt.Verify() - rt.ExpectValidateCallerType(builtin.AccountActorCodeID) + rt.ExpectValidateCallerType(builtin2.AccountActorCodeID) rt.Call(a.CallerValidation, &CallerValidationArgs{ Branch: CallerValidationBranchIsType, - Types: []cid.Cid{builtin.AccountActorCodeID}, + Types: []cid.Cid{builtin2.AccountActorCodeID}, }) rt.Verify() } func TestCallerValidationInvalidBranch(t *testing.T) { - receiver := atesting.NewIDAddr(t, 100) - builder := mock.NewBuilder(context.Background(), receiver) + receiver := atesting2.NewIDAddr(t, 100) + builder := mock2.NewBuilder(context.Background(), receiver) rt := builder.Build(t) var a Actor @@ -108,9 +108,9 @@ func TestCallerValidationInvalidBranch(t *testing.T) { } func TestDeleteActor(t *testing.T) { - receiver := atesting.NewIDAddr(t, 100) - beneficiary := atesting.NewIDAddr(t, 101) - builder := mock.NewBuilder(context.Background(), receiver) + receiver := atesting2.NewIDAddr(t, 100) + beneficiary := atesting2.NewIDAddr(t, 101) + builder := mock2.NewBuilder(context.Background(), receiver) rt := builder.Build(t) var a Actor @@ -122,8 +122,8 @@ func TestDeleteActor(t *testing.T) { } func TestMutateStateInTransaction(t *testing.T) { - receiver := atesting.NewIDAddr(t, 100) - builder := mock.NewBuilder(context.Background(), receiver) + receiver := atesting2.NewIDAddr(t, 100) + builder := mock2.NewBuilder(context.Background(), receiver) rt := builder.Build(t) var a Actor @@ -148,8 +148,8 @@ func TestMutateStateInTransaction(t *testing.T) { } func TestMutateStateAfterTransaction(t *testing.T) { - receiver := atesting.NewIDAddr(t, 100) - builder := mock.NewBuilder(context.Background(), receiver) + receiver := atesting2.NewIDAddr(t, 100) + builder := mock2.NewBuilder(context.Background(), receiver) rt := builder.Build(t) var a Actor @@ -175,8 +175,8 @@ func TestMutateStateAfterTransaction(t *testing.T) { } func TestMutateStateReadonly(t *testing.T) { - receiver := atesting.NewIDAddr(t, 100) - builder := mock.NewBuilder(context.Background(), receiver) + receiver := atesting2.NewIDAddr(t, 100) + builder := mock2.NewBuilder(context.Background(), receiver) rt := builder.Build(t) var a Actor @@ -201,8 +201,8 @@ func TestMutateStateReadonly(t *testing.T) { } func TestMutateStateInvalidBranch(t *testing.T) { - receiver := atesting.NewIDAddr(t, 100) - builder := mock.NewBuilder(context.Background(), receiver) + receiver := atesting2.NewIDAddr(t, 100) + builder := mock2.NewBuilder(context.Background(), receiver) rt := builder.Build(t) var a Actor @@ -215,8 +215,8 @@ func TestMutateStateInvalidBranch(t *testing.T) { } func TestAbortWith(t *testing.T) { - receiver := atesting.NewIDAddr(t, 100) - builder := mock.NewBuilder(context.Background(), receiver) + receiver := atesting2.NewIDAddr(t, 100) + builder := mock2.NewBuilder(context.Background(), receiver) rt := builder.Build(t) var a Actor @@ -233,8 +233,8 @@ func TestAbortWith(t *testing.T) { } func TestAbortWithUncontrolled(t *testing.T) { - receiver := atesting.NewIDAddr(t, 100) - builder := mock.NewBuilder(context.Background(), receiver) + receiver := atesting2.NewIDAddr(t, 100) + builder := mock2.NewBuilder(context.Background(), receiver) rt := builder.Build(t) var a Actor @@ -250,12 +250,12 @@ func TestAbortWithUncontrolled(t *testing.T) { } func TestInspectRuntime(t *testing.T) { - caller := atesting.NewIDAddr(t, 100) - receiver := atesting.NewIDAddr(t, 101) - builder := mock.NewBuilder(context.Background(), receiver) + caller := atesting2.NewIDAddr(t, 100) + receiver := atesting2.NewIDAddr(t, 101) + builder := mock2.NewBuilder(context.Background(), receiver) rt := builder.Build(t) - rt.SetCaller(caller, builtin.AccountActorCodeID) + rt.SetCaller(caller, builtin2.AccountActorCodeID) rt.StateCreate(&State{}) var a Actor diff --git a/conformance/chaos/cbor_gen.go b/conformance/chaos/cbor_gen.go index 882af7026..876d6a893 100644 --- a/conformance/chaos/cbor_gen.go +++ b/conformance/chaos/cbor_gen.go @@ -587,7 +587,7 @@ func (t *SendReturn) MarshalCBOR(w io.Writer) error { scratch := make([]byte, 9) - // t.Return (runtime.CBORBytes) (slice) + // t.Return (builtin.CBORBytes) (slice) if len(t.Return) > cbg.ByteArrayMaxLen { return xerrors.Errorf("Byte array in field t.Return was too long") } @@ -631,7 +631,7 @@ func (t *SendReturn) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.Return (runtime.CBORBytes) (slice) + // t.Return (builtin.CBORBytes) (slice) maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) if err != nil { diff --git a/extern/sector-storage/ffiwrapper/sealer_test.go b/extern/sector-storage/ffiwrapper/sealer_test.go index bb26adb77..15a98667f 100644 --- a/extern/sector-storage/ffiwrapper/sealer_test.go +++ b/extern/sector-storage/ffiwrapper/sealer_test.go @@ -15,7 +15,7 @@ import ( "testing" "time" - saproof "github.com/filecoin-project/specs-actors/actors/runtime/proof" + proof2 "github.com/filecoin-project/specs-actors/v2/actors/runtime/proof" "github.com/ipfs/go-cid" @@ -93,7 +93,7 @@ func (s *seal) commit(t *testing.T, sb *Sealer, done func()) { t.Fatalf("%+v", err) } - ok, err := ProofVerifier.VerifySeal(saproof.SealVerifyInfo{ + ok, err := ProofVerifier.VerifySeal(proof2.SealVerifyInfo{ SectorID: s.id, SealedCID: s.cids.Sealed, SealProof: sealProofType, @@ -171,9 +171,9 @@ func (s *seal) unseal(t *testing.T, sb *Sealer, sp *basicfs.Provider, si abi.Sec func post(t *testing.T, sealer *Sealer, skipped []abi.SectorID, seals ...seal) { randomness := abi.PoStRandomness{0, 9, 2, 7, 6, 5, 4, 3, 2, 1, 0, 9, 8, 7, 6, 45, 3, 2, 1, 0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9, 7} - sis := make([]saproof.SectorInfo, len(seals)) + sis := make([]proof2.SectorInfo, len(seals)) for i, s := range seals { - sis[i] = saproof.SectorInfo{ + sis[i] = proof2.SectorInfo{ SealProof: sealProofType, SectorNumber: s.id.Number, SealedCID: s.cids.Sealed, @@ -191,7 +191,7 @@ func post(t *testing.T, sealer *Sealer, skipped []abi.SectorID, seals ...seal) { t.Fatalf("%+v", err) } - ok, err := ProofVerifier.VerifyWindowPoSt(context.TODO(), saproof.WindowPoStVerifyInfo{ + ok, err := ProofVerifier.VerifyWindowPoSt(context.TODO(), proof2.WindowPoStVerifyInfo{ Randomness: randomness, Proofs: proofs, ChallengedSectors: sis, diff --git a/extern/sector-storage/ffiwrapper/types.go b/extern/sector-storage/ffiwrapper/types.go index 318dbd2b0..c8f263b29 100644 --- a/extern/sector-storage/ffiwrapper/types.go +++ b/extern/sector-storage/ffiwrapper/types.go @@ -4,7 +4,7 @@ import ( "context" "io" - "github.com/filecoin-project/specs-actors/actors/runtime/proof" + proof2 "github.com/filecoin-project/specs-actors/v2/actors/runtime/proof" "github.com/ipfs/go-cid" @@ -35,9 +35,9 @@ type Storage interface { } type Verifier interface { - VerifySeal(proof.SealVerifyInfo) (bool, error) - VerifyWinningPoSt(ctx context.Context, info proof.WinningPoStVerifyInfo) (bool, error) - VerifyWindowPoSt(ctx context.Context, info proof.WindowPoStVerifyInfo) (bool, error) + VerifySeal(proof2.SealVerifyInfo) (bool, error) + VerifyWinningPoSt(ctx context.Context, info proof2.WinningPoStVerifyInfo) (bool, error) + VerifyWindowPoSt(ctx context.Context, info proof2.WindowPoStVerifyInfo) (bool, error) GenerateWinningPoStSectorChallenge(context.Context, abi.RegisteredPoStProof, abi.ActorID, abi.PoStRandomness, uint64) ([]uint64, error) } diff --git a/extern/sector-storage/ffiwrapper/verifier_cgo.go b/extern/sector-storage/ffiwrapper/verifier_cgo.go index d6c0ae35f..bfa77345a 100644 --- a/extern/sector-storage/ffiwrapper/verifier_cgo.go +++ b/extern/sector-storage/ffiwrapper/verifier_cgo.go @@ -5,7 +5,7 @@ package ffiwrapper import ( "context" - "github.com/filecoin-project/specs-actors/actors/runtime/proof" + proof2 "github.com/filecoin-project/specs-actors/v2/actors/runtime/proof" "golang.org/x/xerrors" @@ -18,7 +18,7 @@ import ( "go.opencensus.io/trace" ) -func (sb *Sealer) GenerateWinningPoSt(ctx context.Context, minerID abi.ActorID, sectorInfo []proof.SectorInfo, randomness abi.PoStRandomness) ([]proof.PoStProof, error) { +func (sb *Sealer) GenerateWinningPoSt(ctx context.Context, minerID abi.ActorID, sectorInfo []proof2.SectorInfo, randomness abi.PoStRandomness) ([]proof2.PoStProof, error) { randomness[31] &= 0x3f privsectors, skipped, done, err := sb.pubSectorToPriv(ctx, minerID, sectorInfo, nil, abi.RegisteredSealProof.RegisteredWinningPoStProof) // TODO: FAULTS? if err != nil { @@ -32,7 +32,7 @@ func (sb *Sealer) GenerateWinningPoSt(ctx context.Context, minerID abi.ActorID, return ffi.GenerateWinningPoSt(minerID, privsectors, randomness) } -func (sb *Sealer) GenerateWindowPoSt(ctx context.Context, minerID abi.ActorID, sectorInfo []proof.SectorInfo, randomness abi.PoStRandomness) ([]proof.PoStProof, []abi.SectorID, error) { +func (sb *Sealer) GenerateWindowPoSt(ctx context.Context, minerID abi.ActorID, sectorInfo []proof2.SectorInfo, randomness abi.PoStRandomness) ([]proof2.PoStProof, []abi.SectorID, error) { randomness[31] &= 0x3f privsectors, skipped, done, err := sb.pubSectorToPriv(ctx, minerID, sectorInfo, nil, abi.RegisteredSealProof.RegisteredWindowPoStProof) if err != nil { @@ -57,7 +57,7 @@ func (sb *Sealer) GenerateWindowPoSt(ctx context.Context, minerID abi.ActorID, s return proof, faultyIDs, err } -func (sb *Sealer) pubSectorToPriv(ctx context.Context, mid abi.ActorID, sectorInfo []proof.SectorInfo, faults []abi.SectorNumber, rpt func(abi.RegisteredSealProof) (abi.RegisteredPoStProof, error)) (ffi.SortedPrivateSectorInfo, []abi.SectorID, func(), error) { +func (sb *Sealer) pubSectorToPriv(ctx context.Context, mid abi.ActorID, sectorInfo []proof2.SectorInfo, faults []abi.SectorNumber, rpt func(abi.RegisteredSealProof) (abi.RegisteredPoStProof, error)) (ffi.SortedPrivateSectorInfo, []abi.SectorID, func(), error) { fmap := map[abi.SectorNumber]struct{}{} for _, fault := range faults { fmap[fault] = struct{}{} @@ -110,11 +110,11 @@ type proofVerifier struct{} var ProofVerifier = proofVerifier{} -func (proofVerifier) VerifySeal(info proof.SealVerifyInfo) (bool, error) { +func (proofVerifier) VerifySeal(info proof2.SealVerifyInfo) (bool, error) { return ffi.VerifySeal(info) } -func (proofVerifier) VerifyWinningPoSt(ctx context.Context, info proof.WinningPoStVerifyInfo) (bool, error) { +func (proofVerifier) VerifyWinningPoSt(ctx context.Context, info proof2.WinningPoStVerifyInfo) (bool, error) { info.Randomness[31] &= 0x3f _, span := trace.StartSpan(ctx, "VerifyWinningPoSt") defer span.End() @@ -122,7 +122,7 @@ func (proofVerifier) VerifyWinningPoSt(ctx context.Context, info proof.WinningPo return ffi.VerifyWinningPoSt(info) } -func (proofVerifier) VerifyWindowPoSt(ctx context.Context, info proof.WindowPoStVerifyInfo) (bool, error) { +func (proofVerifier) VerifyWindowPoSt(ctx context.Context, info proof2.WindowPoStVerifyInfo) (bool, error) { info.Randomness[31] &= 0x3f _, span := trace.StartSpan(ctx, "VerifyWindowPoSt") defer span.End() diff --git a/extern/sector-storage/mock/mock.go b/extern/sector-storage/mock/mock.go index 001c7159c..aa3f1cc55 100644 --- a/extern/sector-storage/mock/mock.go +++ b/extern/sector-storage/mock/mock.go @@ -9,7 +9,7 @@ import ( "math/rand" "sync" - "github.com/filecoin-project/specs-actors/actors/runtime/proof" + proof2 "github.com/filecoin-project/specs-actors/v2/actors/runtime/proof" commcid "github.com/filecoin-project/go-fil-commcid" "github.com/filecoin-project/go-state-types/abi" @@ -280,12 +280,12 @@ func AddOpFinish(ctx context.Context) (context.Context, func()) { } } -func (mgr *SectorMgr) GenerateWinningPoSt(ctx context.Context, minerID abi.ActorID, sectorInfo []proof.SectorInfo, randomness abi.PoStRandomness) ([]proof.PoStProof, error) { +func (mgr *SectorMgr) GenerateWinningPoSt(ctx context.Context, minerID abi.ActorID, sectorInfo []proof2.SectorInfo, randomness abi.PoStRandomness) ([]proof2.PoStProof, error) { return generateFakePoSt(sectorInfo, abi.RegisteredSealProof.RegisteredWinningPoStProof, randomness), nil } -func (mgr *SectorMgr) GenerateWindowPoSt(ctx context.Context, minerID abi.ActorID, sectorInfo []proof.SectorInfo, randomness abi.PoStRandomness) ([]proof.PoStProof, []abi.SectorID, error) { - si := make([]proof.SectorInfo, 0, len(sectorInfo)) +func (mgr *SectorMgr) GenerateWindowPoSt(ctx context.Context, minerID abi.ActorID, sectorInfo []proof2.SectorInfo, randomness abi.PoStRandomness) ([]proof2.PoStProof, []abi.SectorID, error) { + si := make([]proof2.SectorInfo, 0, len(sectorInfo)) var skipped []abi.SectorID var err error @@ -313,7 +313,7 @@ func (mgr *SectorMgr) GenerateWindowPoSt(ctx context.Context, minerID abi.ActorI return generateFakePoSt(si, abi.RegisteredSealProof.RegisteredWindowPoStProof, randomness), skipped, nil } -func generateFakePoStProof(sectorInfo []proof.SectorInfo, randomness abi.PoStRandomness) []byte { +func generateFakePoStProof(sectorInfo []proof2.SectorInfo, randomness abi.PoStRandomness) []byte { hasher := sha256.New() _, _ = hasher.Write(randomness) for _, info := range sectorInfo { @@ -326,13 +326,13 @@ func generateFakePoStProof(sectorInfo []proof.SectorInfo, randomness abi.PoStRan } -func generateFakePoSt(sectorInfo []proof.SectorInfo, rpt func(abi.RegisteredSealProof) (abi.RegisteredPoStProof, error), randomness abi.PoStRandomness) []proof.PoStProof { +func generateFakePoSt(sectorInfo []proof2.SectorInfo, rpt func(abi.RegisteredSealProof) (abi.RegisteredPoStProof, error), randomness abi.PoStRandomness) []proof2.PoStProof { wp, err := rpt(sectorInfo[0].SealProof) if err != nil { panic(err) } - return []proof.PoStProof{ + return []proof2.PoStProof{ { PoStProof: wp, ProofBytes: generateFakePoStProof(sectorInfo, randomness), @@ -406,7 +406,7 @@ func (mgr *SectorMgr) CheckProvable(ctx context.Context, spt abi.RegisteredSealP return bad, nil } -func (m mockVerif) VerifySeal(svi proof.SealVerifyInfo) (bool, error) { +func (m mockVerif) VerifySeal(svi proof2.SealVerifyInfo) (bool, error) { if len(svi.Proof) != 1920 { return false, nil } @@ -421,11 +421,11 @@ func (m mockVerif) VerifySeal(svi proof.SealVerifyInfo) (bool, error) { return true, nil } -func (m mockVerif) VerifyWinningPoSt(ctx context.Context, info proof.WinningPoStVerifyInfo) (bool, error) { +func (m mockVerif) VerifyWinningPoSt(ctx context.Context, info proof2.WinningPoStVerifyInfo) (bool, error) { return true, nil } -func (m mockVerif) VerifyWindowPoSt(ctx context.Context, info proof.WindowPoStVerifyInfo) (bool, error) { +func (m mockVerif) VerifyWindowPoSt(ctx context.Context, info proof2.WindowPoStVerifyInfo) (bool, error) { if len(info.Proofs) != 1 { return false, xerrors.Errorf("expected 1 proof entry") } diff --git a/extern/storage-sealing/checks.go b/extern/storage-sealing/checks.go index 1010d31b2..ed7a691ef 100644 --- a/extern/storage-sealing/checks.go +++ b/extern/storage-sealing/checks.go @@ -7,7 +7,7 @@ import ( "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/policy" - proof0 "github.com/filecoin-project/specs-actors/actors/runtime/proof" + proof2 "github.com/filecoin-project/specs-actors/v2/actors/runtime/proof" "golang.org/x/xerrors" @@ -179,7 +179,7 @@ func (m *Sealing) checkCommit(ctx context.Context, si SectorInfo, proof []byte, log.Warn("on-chain sealed CID doesn't match!") } - ok, err := m.verif.VerifySeal(proof0.SealVerifyInfo{ + ok, err := m.verif.VerifySeal(proof2.SealVerifyInfo{ SectorID: m.minerSector(si.SectorNumber), SealedCID: pci.Info.SealedCID, SealProof: spt, diff --git a/extern/storage-sealing/states_sealing.go b/extern/storage-sealing/states_sealing.go index 96589bcd2..1e67a0a03 100644 --- a/extern/storage-sealing/states_sealing.go +++ b/extern/storage-sealing/states_sealing.go @@ -15,8 +15,9 @@ import ( "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/go-state-types/exitcode" "github.com/filecoin-project/go-statemachine" - "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-storage/storage" + + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" ) var DealSectorPriority = 1024 @@ -220,7 +221,7 @@ func (m *Sealing) handlePreCommitting(ctx statemachine.Context, sector SectorInf deposit := big.Max(depositMinimum, collateral) log.Infof("submitting precommit for sector %d (deposit: %s): ", sector.SectorNumber, deposit) - mcid, err := m.api.SendMsg(ctx.Context(), waddr, m.maddr, builtin.MethodsMiner.PreCommitSector, deposit, m.feeCfg.MaxPreCommitGasFee, enc.Bytes()) + mcid, err := m.api.SendMsg(ctx.Context(), waddr, m.maddr, builtin2.MethodsMiner.PreCommitSector, deposit, m.feeCfg.MaxPreCommitGasFee, enc.Bytes()) if err != nil { if params.ReplaceCapacity { m.remarkForUpgrade(params.ReplaceSectorNumber) @@ -401,7 +402,7 @@ func (m *Sealing) handleSubmitCommit(ctx statemachine.Context, sector SectorInfo } // TODO: check seed / ticket / deals are up to date - mcid, err := m.api.SendMsg(ctx.Context(), waddr, m.maddr, builtin.MethodsMiner.ProveCommitSector, collateral, m.feeCfg.MaxCommitGasFee, enc.Bytes()) + mcid, err := m.api.SendMsg(ctx.Context(), waddr, m.maddr, builtin2.MethodsMiner.ProveCommitSector, collateral, m.feeCfg.MaxCommitGasFee, enc.Bytes()) if err != nil { return ctx.Send(SectorCommitFailed{xerrors.Errorf("pushing message to mpool: %w", err)}) } diff --git a/extern/storage-sealing/types.go b/extern/storage-sealing/types.go index 046271a7f..8f3e82a0b 100644 --- a/extern/storage-sealing/types.go +++ b/extern/storage-sealing/types.go @@ -9,9 +9,9 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/exitcode" - miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/specs-storage/storage" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" "github.com/filecoin-project/lotus/extern/storage-sealing/sealiface" ) @@ -82,7 +82,7 @@ type SectorInfo struct { CommR *cid.Cid Proof []byte - PreCommitInfo *miner0.SectorPreCommitInfo + PreCommitInfo *miner.SectorPreCommitInfo PreCommitDeposit big.Int PreCommitMessage *cid.Cid PreCommitTipSet TipSetToken diff --git a/extern/storage-sealing/types_test.go b/extern/storage-sealing/types_test.go index fc56620dc..0b3c97032 100644 --- a/extern/storage-sealing/types_test.go +++ b/extern/storage-sealing/types_test.go @@ -8,7 +8,7 @@ import ( cborutil "github.com/filecoin-project/go-cbor-util" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/specs-actors/actors/builtin" + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" ) func TestSectorInfoSelialization(t *testing.T) { @@ -22,7 +22,7 @@ func TestSectorInfoSelialization(t *testing.T) { }, } - dummyCid := builtin.AccountActorCodeID + dummyCid := builtin2.AccountActorCodeID si := &SectorInfo{ State: "stateful", diff --git a/genesis/types.go b/genesis/types.go index 79656feac..db8d32a3b 100644 --- a/genesis/types.go +++ b/genesis/types.go @@ -5,9 +5,10 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/specs-actors/actors/builtin/market" "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p-core/peer" + + market2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/market" ) type ActorType string @@ -21,7 +22,7 @@ type PreSeal struct { CommR cid.Cid CommD cid.Cid SectorID abi.SectorNumber - Deal market.DealProposal + Deal market2.DealProposal ProofType abi.RegisteredSealProof } diff --git a/markets/storageadapter/client.go b/markets/storageadapter/client.go index 6496fffad..ad2b2df8a 100644 --- a/markets/storageadapter/client.go +++ b/markets/storageadapter/client.go @@ -6,20 +6,20 @@ import ( "bytes" "context" - "golang.org/x/xerrors" - - "github.com/ipfs/go-cid" - "github.com/filecoin-project/go-address" cborutil "github.com/filecoin-project/go-cbor-util" + "github.com/ipfs/go-cid" + "golang.org/x/xerrors" + "github.com/filecoin-project/go-fil-markets/shared" "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/go-state-types/exitcode" - miner0 "github.com/filecoin-project/specs-actors/actors/builtin" - market0 "github.com/filecoin-project/specs-actors/actors/builtin/market" + + miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + market2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/market" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" @@ -103,10 +103,10 @@ func (c *ClientNodeAdapter) VerifySignature(ctx context.Context, sig crypto.Sign func (c *ClientNodeAdapter) AddFunds(ctx context.Context, addr address.Address, amount abi.TokenAmount) (cid.Cid, error) { // (Provider Node API) smsg, err := c.MpoolPushMessage(ctx, &types.Message{ - To: miner0.StorageMarketActorAddr, + To: miner2.StorageMarketActorAddr, From: addr, Value: amount, - Method: miner0.MethodsMarket.AddBalance, + Method: miner2.MethodsMarket.AddBalance, }, nil) if err != nil { return cid.Undef, err @@ -157,15 +157,15 @@ func (c *ClientNodeAdapter) ValidatePublishedDeal(ctx context.Context, deal stor return 0, xerrors.Errorf("deal wasn't published by storage provider: from=%s, provider=%s", pubmsg.From, deal.Proposal.Provider) } - if pubmsg.To != miner0.StorageMarketActorAddr { + if pubmsg.To != miner2.StorageMarketActorAddr { return 0, xerrors.Errorf("deal publish message wasn't set to StorageMarket actor (to=%s)", pubmsg.To) } - if pubmsg.Method != miner0.MethodsMarket.PublishStorageDeals { + if pubmsg.Method != miner2.MethodsMarket.PublishStorageDeals { return 0, xerrors.Errorf("deal publish message called incorrect method (method=%s)", pubmsg.Method) } - var params market0.PublishStorageDealsParams + var params market2.PublishStorageDealsParams if err := params.UnmarshalCBOR(bytes.NewReader(pubmsg.Params)); err != nil { return 0, err } @@ -197,7 +197,7 @@ func (c *ClientNodeAdapter) ValidatePublishedDeal(ctx context.Context, deal stor return 0, xerrors.Errorf("deal publish failed: exit=%d", ret.ExitCode) } - var res market0.PublishStorageDealsReturn + var res market2.PublishStorageDealsReturn if err := res.UnmarshalCBOR(bytes.NewReader(ret.Return)); err != nil { return 0, err } @@ -275,7 +275,7 @@ func (c *ClientNodeAdapter) OnDealSectorCommitted(ctx context.Context, provider } switch msg.Method { - case miner0.MethodsMiner.PreCommitSector: + case miner2.MethodsMiner.PreCommitSector: var params miner.SectorPreCommitInfo if err := params.UnmarshalCBOR(bytes.NewReader(msg.Params)); err != nil { return true, false, xerrors.Errorf("unmarshal pre commit: %w", err) @@ -290,7 +290,7 @@ func (c *ClientNodeAdapter) OnDealSectorCommitted(ctx context.Context, provider } return true, false, nil - case miner0.MethodsMiner.ProveCommitSector: + case miner2.MethodsMiner.ProveCommitSector: var params miner.ProveCommitSectorParams if err := params.UnmarshalCBOR(bytes.NewReader(msg.Params)); err != nil { return true, false, xerrors.Errorf("failed to unmarshal prove commit sector params: %w", err) @@ -412,7 +412,7 @@ func (c *ClientNodeAdapter) OnDealExpiredOrSlashed(ctx context.Context, dealID a return nil } -func (c *ClientNodeAdapter) SignProposal(ctx context.Context, signer address.Address, proposal market0.DealProposal) (*market0.ClientDealProposal, error) { +func (c *ClientNodeAdapter) SignProposal(ctx context.Context, signer address.Address, proposal market2.DealProposal) (*market2.ClientDealProposal, error) { // TODO: output spec signed proposal buf, err := cborutil.Dump(&proposal) if err != nil { @@ -431,7 +431,7 @@ func (c *ClientNodeAdapter) SignProposal(ctx context.Context, signer address.Add return nil, err } - return &market0.ClientDealProposal{ + return &market2.ClientDealProposal{ Proposal: proposal, ClientSignature: *sig, }, nil diff --git a/markets/storageadapter/provider.go b/markets/storageadapter/provider.go index b1071adcd..9c088e4de 100644 --- a/markets/storageadapter/provider.go +++ b/markets/storageadapter/provider.go @@ -12,8 +12,8 @@ import ( logging "github.com/ipfs/go-log/v2" "golang.org/x/xerrors" - builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" - market0 "github.com/filecoin-project/specs-actors/actors/builtin/market" + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + market2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/market" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-fil-markets/shared" @@ -79,8 +79,8 @@ func (n *ProviderNodeAdapter) PublishDeals(ctx context.Context, deal storagemark return cid.Undef, err } - params, err := actors.SerializeParams(&market0.PublishStorageDealsParams{ - Deals: []market0.ClientDealProposal{deal.ClientDealProposal}, + params, err := actors.SerializeParams(&market2.PublishStorageDealsParams{ + Deals: []market2.ClientDealProposal{deal.ClientDealProposal}, }) if err != nil { @@ -92,7 +92,7 @@ func (n *ProviderNodeAdapter) PublishDeals(ctx context.Context, deal storagemark To: market.Address, From: mi.Worker, Value: types.NewInt(0), - Method: builtin0.MethodsMarket.PublishStorageDeals, + Method: builtin2.MethodsMarket.PublishStorageDeals, Params: params, }, n.publishSpec) if err != nil { @@ -192,7 +192,7 @@ func (n *ProviderNodeAdapter) AddFunds(ctx context.Context, addr address.Address To: market.Address, From: addr, Value: amount, - Method: builtin0.MethodsMarket.AddBalance, + Method: builtin2.MethodsMarket.AddBalance, }, n.addBalanceSpec) if err != nil { return cid.Undef, err @@ -314,7 +314,7 @@ func (n *ProviderNodeAdapter) OnDealSectorCommitted(ctx context.Context, provide } switch msg.Method { - case builtin0.MethodsMiner.PreCommitSector: + case builtin2.MethodsMiner.PreCommitSector: var params miner.SectorPreCommitInfo if err := params.UnmarshalCBOR(bytes.NewReader(msg.Params)); err != nil { return true, false, xerrors.Errorf("unmarshal pre commit: %w", err) @@ -329,7 +329,7 @@ func (n *ProviderNodeAdapter) OnDealSectorCommitted(ctx context.Context, provide } return true, false, nil - case builtin0.MethodsMiner.ProveCommitSector: + case builtin2.MethodsMiner.ProveCommitSector: var params miner.ProveCommitSectorParams if err := params.UnmarshalCBOR(bytes.NewReader(msg.Params)); err != nil { return true, false, xerrors.Errorf("failed to unmarshal prove commit sector params: %w", err) diff --git a/miner/miner.go b/miner/miner.go index 73985a649..7451d617a 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -9,7 +9,7 @@ import ( "sync" "time" - proof0 "github.com/filecoin-project/specs-actors/actors/runtime/proof" + proof2 "github.com/filecoin-project/specs-actors/v2/actors/runtime/proof" "github.com/filecoin-project/lotus/chain/gen/slashfilter" @@ -492,7 +492,7 @@ func (m *Miner) computeTicket(ctx context.Context, brand *types.BeaconEntry, bas } func (m *Miner) createBlock(base *MiningBase, addr address.Address, ticket *types.Ticket, - eproof *types.ElectionProof, bvals []types.BeaconEntry, wpostProof []proof0.PoStProof, msgs []*types.SignedMessage) (*types.BlockMsg, error) { + eproof *types.ElectionProof, bvals []types.BeaconEntry, wpostProof []proof2.PoStProof, msgs []*types.SignedMessage) (*types.BlockMsg, error) { uts := base.TipSet.MinTimestamp() + build.BlockDelaySecs*(uint64(base.NullRounds)+1) nheight := base.TipSet.Height() + base.NullRounds + 1 diff --git a/node/impl/full/gas.go b/node/impl/full/gas.go index 2e5063ccb..4e4fc99d2 100644 --- a/node/impl/full/gas.go +++ b/node/impl/full/gas.go @@ -16,7 +16,7 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/exitcode" - builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" @@ -259,7 +259,7 @@ func gasEstimateGasLimit( if !builtin.IsPaymentChannelActor(act.Code) { return res.MsgRct.GasUsed, nil } - if msgIn.Method != builtin0.MethodsPaych.Collect { + if msgIn.Method != builtin2.MethodsPaych.Collect { return res.MsgRct.GasUsed, nil } diff --git a/node/impl/full/multisig.go b/node/impl/full/multisig.go index d840620eb..3b13839dd 100644 --- a/node/impl/full/multisig.go +++ b/node/impl/full/multisig.go @@ -12,8 +12,8 @@ import ( "github.com/filecoin-project/lotus/chain/actors/builtin/multisig" "github.com/filecoin-project/lotus/chain/types" - builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" - multisig0 "github.com/filecoin-project/specs-actors/actors/builtin/multisig" + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + multisig2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/multisig" "github.com/ipfs/go-cid" "go.uber.org/fx" @@ -85,7 +85,7 @@ func (a *MsigAPI) MsigAddPropose(ctx context.Context, msig address.Address, src return cid.Undef, actErr } - return a.MsigPropose(ctx, msig, msig, big.Zero(), src, uint64(builtin0.MethodsMultisig.AddSigner), enc) + return a.MsigPropose(ctx, msig, msig, big.Zero(), src, uint64(builtin2.MethodsMultisig.AddSigner), enc) } func (a *MsigAPI) MsigAddApprove(ctx context.Context, msig address.Address, src address.Address, txID uint64, proposer address.Address, newAdd address.Address, inc bool) (cid.Cid, error) { @@ -94,7 +94,7 @@ func (a *MsigAPI) MsigAddApprove(ctx context.Context, msig address.Address, src return cid.Undef, actErr } - return a.MsigApproveTxnHash(ctx, msig, txID, proposer, msig, big.Zero(), src, uint64(builtin0.MethodsMultisig.AddSigner), enc) + return a.MsigApproveTxnHash(ctx, msig, txID, proposer, msig, big.Zero(), src, uint64(builtin2.MethodsMultisig.AddSigner), enc) } func (a *MsigAPI) MsigAddCancel(ctx context.Context, msig address.Address, src address.Address, txID uint64, newAdd address.Address, inc bool) (cid.Cid, error) { @@ -103,7 +103,7 @@ func (a *MsigAPI) MsigAddCancel(ctx context.Context, msig address.Address, src a return cid.Undef, actErr } - return a.MsigCancel(ctx, msig, txID, msig, big.Zero(), src, uint64(builtin0.MethodsMultisig.AddSigner), enc) + return a.MsigCancel(ctx, msig, txID, msig, big.Zero(), src, uint64(builtin2.MethodsMultisig.AddSigner), enc) } func (a *MsigAPI) MsigSwapPropose(ctx context.Context, msig address.Address, src address.Address, oldAdd address.Address, newAdd address.Address) (cid.Cid, error) { @@ -112,7 +112,7 @@ func (a *MsigAPI) MsigSwapPropose(ctx context.Context, msig address.Address, src return cid.Undef, actErr } - return a.MsigPropose(ctx, msig, msig, big.Zero(), src, uint64(builtin0.MethodsMultisig.SwapSigner), enc) + return a.MsigPropose(ctx, msig, msig, big.Zero(), src, uint64(builtin2.MethodsMultisig.SwapSigner), enc) } func (a *MsigAPI) MsigSwapApprove(ctx context.Context, msig address.Address, src address.Address, txID uint64, proposer address.Address, oldAdd address.Address, newAdd address.Address) (cid.Cid, error) { @@ -121,7 +121,7 @@ func (a *MsigAPI) MsigSwapApprove(ctx context.Context, msig address.Address, src return cid.Undef, actErr } - return a.MsigApproveTxnHash(ctx, msig, txID, proposer, msig, big.Zero(), src, uint64(builtin0.MethodsMultisig.SwapSigner), enc) + return a.MsigApproveTxnHash(ctx, msig, txID, proposer, msig, big.Zero(), src, uint64(builtin2.MethodsMultisig.SwapSigner), enc) } func (a *MsigAPI) MsigSwapCancel(ctx context.Context, msig address.Address, src address.Address, txID uint64, oldAdd address.Address, newAdd address.Address) (cid.Cid, error) { @@ -130,7 +130,7 @@ func (a *MsigAPI) MsigSwapCancel(ctx context.Context, msig address.Address, src return cid.Undef, actErr } - return a.MsigCancel(ctx, msig, txID, msig, big.Zero(), src, uint64(builtin0.MethodsMultisig.SwapSigner), enc) + return a.MsigCancel(ctx, msig, txID, msig, big.Zero(), src, uint64(builtin2.MethodsMultisig.SwapSigner), enc) } func (a *MsigAPI) MsigApprove(ctx context.Context, msig address.Address, txID uint64, src address.Address) (cid.Cid, error) { @@ -151,7 +151,7 @@ func (a *MsigAPI) MsigRemoveSigner(ctx context.Context, msig address.Address, pr return cid.Undef, actErr } - return a.MsigPropose(ctx, msig, msig, types.NewInt(0), proposer, uint64(builtin0.MethodsMultisig.RemoveSigner), enc) + return a.MsigPropose(ctx, msig, msig, types.NewInt(0), proposer, uint64(builtin2.MethodsMultisig.RemoveSigner), enc) } func (a *MsigAPI) msigApproveOrCancelSimple(ctx context.Context, operation api.MsigProposeResponse, msig address.Address, txID uint64, src address.Address) (cid.Cid, error) { @@ -242,7 +242,7 @@ func (a *MsigAPI) msigApproveOrCancelTxnHash(ctx context.Context, operation api. } func serializeAddParams(new address.Address, inc bool) ([]byte, error) { - enc, actErr := actors.SerializeParams(&multisig0.AddSignerParams{ + enc, actErr := actors.SerializeParams(&multisig2.AddSignerParams{ Signer: new, Increase: inc, }) @@ -254,7 +254,7 @@ func serializeAddParams(new address.Address, inc bool) ([]byte, error) { } func serializeSwapParams(old address.Address, new address.Address) ([]byte, error) { - enc, actErr := actors.SerializeParams(&multisig0.SwapSignerParams{ + enc, actErr := actors.SerializeParams(&multisig2.SwapSignerParams{ From: old, To: new, }) @@ -266,7 +266,7 @@ func serializeSwapParams(old address.Address, new address.Address) ([]byte, erro } func serializeRemoveParams(rem address.Address, dec bool) ([]byte, error) { - enc, actErr := actors.SerializeParams(&multisig0.RemoveSignerParams{ + enc, actErr := actors.SerializeParams(&multisig2.RemoveSignerParams{ Signer: rem, Decrease: dec, }) diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 7f510aa70..66b16e9f3 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -44,7 +44,6 @@ import ( paramfetch "github.com/filecoin-project/go-paramfetch" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-storedcounter" - "github.com/filecoin-project/specs-actors/actors/builtin" sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" @@ -56,6 +55,7 @@ import ( lapi "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/gen/slashfilter" "github.com/filecoin-project/lotus/chain/types" diff --git a/node/test/builder.go b/node/test/builder.go index 4aa8a55ea..785e07bbe 100644 --- a/node/test/builder.go +++ b/node/test/builder.go @@ -24,6 +24,7 @@ import ( "github.com/filecoin-project/lotus/api/test" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/gen" genesis2 "github.com/filecoin-project/lotus/chain/gen/genesis" "github.com/filecoin-project/lotus/chain/types" @@ -33,14 +34,14 @@ import ( "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/extern/sector-storage/mock" "github.com/filecoin-project/lotus/genesis" - miner2 "github.com/filecoin-project/lotus/miner" + lotusminer "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node" "github.com/filecoin-project/lotus/node/modules" testing2 "github.com/filecoin-project/lotus/node/modules/testing" "github.com/filecoin-project/lotus/node/repo" "github.com/filecoin-project/lotus/storage/mockstorage" - "github.com/filecoin-project/specs-actors/actors/builtin" - miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" "github.com/ipfs/go-datastore" "github.com/libp2p/go-libp2p-core/crypto" "github.com/libp2p/go-libp2p-core/peer" @@ -86,13 +87,13 @@ func CreateTestStorageNode(ctx context.Context, t *testing.T, waddr address.Addr peerid, err := peer.IDFromPrivateKey(pk) require.NoError(t, err) - enc, err := actors.SerializeParams(&miner0.ChangePeerIDParams{NewID: abi.PeerID(peerid)}) + enc, err := actors.SerializeParams(&miner2.ChangePeerIDParams{NewID: abi.PeerID(peerid)}) require.NoError(t, err) msg := &types.Message{ To: act, From: waddr, - Method: builtin.MethodsMiner.ChangePeerID, + Method: builtin2.MethodsMiner.ChangePeerID, Params: enc, Value: types.NewInt(0), } @@ -103,7 +104,7 @@ func CreateTestStorageNode(ctx context.Context, t *testing.T, waddr address.Addr // start node var minerapi api.StorageMiner - mineBlock := make(chan miner2.MineReq) + mineBlock := make(chan lotusminer.MineReq) stop, err := node.New(ctx, node.StorageMiner(&minerapi), node.Online(), @@ -113,7 +114,7 @@ func CreateTestStorageNode(ctx context.Context, t *testing.T, waddr address.Addr node.MockHost(mn), node.Override(new(api.FullNode), tnd), - node.Override(new(*miner2.Miner), miner2.NewTestMiner(mineBlock, act)), + node.Override(new(*lotusminer.Miner), lotusminer.NewTestMiner(mineBlock, act)), opts, ) @@ -129,7 +130,7 @@ func CreateTestStorageNode(ctx context.Context, t *testing.T, waddr address.Addr err = minerapi.NetConnect(ctx, remoteAddrs) require.NoError(t, err)*/ - mineOne := func(ctx context.Context, req miner2.MineReq) error { + mineOne := func(ctx context.Context, req lotusminer.MineReq) error { select { case mineBlock <- req: return nil @@ -444,7 +445,7 @@ func mockSbBuilderOpts(t *testing.T, fullOpts []test.FullNodeOpts, storage []tes storers[i] = CreateTestStorageNode(ctx, t, genms[i].Worker, maddrs[i], pidKeys[i], f, mn, node.Options( node.Override(new(sectorstorage.SectorManager), func() (sectorstorage.SectorManager, error) { - return mock.NewMockSectorMgr(build.DefaultSectorSize(), sectors), nil + return mock.NewMockSectorMgr(policy.GetDefaultSectorSize(), sectors), nil }), node.Override(new(ffiwrapper.Verifier), mock.MockVerifier), node.Unset(new(*sectorstorage.Manager)), diff --git a/paychmgr/paych_test.go b/paychmgr/paych_test.go index fcd3d50a8..8557dfb63 100644 --- a/paychmgr/paych_test.go +++ b/paychmgr/paych_test.go @@ -14,9 +14,9 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" - "github.com/filecoin-project/specs-actors/actors/builtin" - tutils "github.com/filecoin-project/specs-actors/support/testing" + "github.com/filecoin-project/specs-actors/v2/actors/builtin" paych2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/paych" + tutils "github.com/filecoin-project/specs-actors/v2/support/testing" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors/builtin/paych" @@ -237,7 +237,7 @@ func TestCreateVoucher(t *testing.T) { // Create a voucher in lane 1 voucherLane1Amt := big.NewInt(5) - voucher := paych2.SignedVoucher{ + voucher := paych.SignedVoucher{ Lane: 1, Amount: voucherLane1Amt, } @@ -252,7 +252,7 @@ func TestCreateVoucher(t *testing.T) { // Create a voucher in lane 1 again, with a higher amount voucherLane1Amt = big.NewInt(8) - voucher = paych2.SignedVoucher{ + voucher = paych.SignedVoucher{ Lane: 1, Amount: voucherLane1Amt, } @@ -267,7 +267,7 @@ func TestCreateVoucher(t *testing.T) { // Create a voucher in lane 2 that covers all the remaining funds // in the channel voucherLane2Amt := big.Sub(s.amt, voucherLane1Amt) - voucher = paych2.SignedVoucher{ + voucher = paych.SignedVoucher{ Lane: 2, Amount: voucherLane2Amt, } @@ -281,7 +281,7 @@ func TestCreateVoucher(t *testing.T) { // Create a voucher in lane 2 that exceeds the remaining funds in the // channel voucherLane2Amt = big.Add(voucherLane2Amt, big.NewInt(1)) - voucher = paych2.SignedVoucher{ + voucher = paych.SignedVoucher{ Lane: 2, Amount: voucherLane2Amt, } diff --git a/paychmgr/paychvoucherfunds_test.go b/paychmgr/paychvoucherfunds_test.go index dcbb4acc9..f83a7cd62 100644 --- a/paychmgr/paychvoucherfunds_test.go +++ b/paychmgr/paychvoucherfunds_test.go @@ -4,19 +4,19 @@ import ( "context" "testing" - "github.com/filecoin-project/lotus/chain/actors/builtin/paych" - paychmock "github.com/filecoin-project/lotus/chain/actors/builtin/paych/mock" - - "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" - "github.com/filecoin-project/specs-actors/actors/builtin" - tutils "github.com/filecoin-project/specs-actors/support/testing" "github.com/ipfs/go-cid" ds "github.com/ipfs/go-datastore" ds_sync "github.com/ipfs/go-datastore/sync" "github.com/stretchr/testify/require" + + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + tutils2 "github.com/filecoin-project/specs-actors/v2/support/testing" + + "github.com/filecoin-project/lotus/chain/actors/builtin/paych" + paychmock "github.com/filecoin-project/lotus/chain/actors/builtin/paych/mock" + "github.com/filecoin-project/lotus/chain/types" ) // TestPaychAddVoucherAfterAddFunds tests adding a voucher to a channel with @@ -27,11 +27,11 @@ func TestPaychAddVoucherAfterAddFunds(t *testing.T) { store := NewStore(ds_sync.MutexWrap(ds.NewMapDatastore())) fromKeyPrivate, fromKeyPublic := testGenerateKeyPair(t) - ch := tutils.NewIDAddr(t, 100) - from := tutils.NewSECP256K1Addr(t, string(fromKeyPublic)) - to := tutils.NewSECP256K1Addr(t, "secpTo") - fromAcct := tutils.NewActorAddr(t, "fromAct") - toAcct := tutils.NewActorAddr(t, "toAct") + ch := tutils2.NewIDAddr(t, 100) + from := tutils2.NewSECP256K1Addr(t, string(fromKeyPublic)) + to := tutils2.NewSECP256K1Addr(t, "secpTo") + fromAcct := tutils2.NewActorAddr(t, "fromAct") + toAcct := tutils2.NewActorAddr(t, "toAct") mock := newMockManagerAPI() defer mock.close() @@ -55,7 +55,7 @@ func TestPaychAddVoucherAfterAddFunds(t *testing.T) { // Create an actor in state for the channel with the initial channel balance act := &types.Actor{ - Code: builtin.AccountActorCodeID, + Code: builtin2.AccountActorCodeID, Head: cid.Cid{}, Nonce: 0, Balance: createAmt, diff --git a/paychmgr/settler/settler.go b/paychmgr/settler/settler.go index 02fe9256e..8074e8a87 100644 --- a/paychmgr/settler/settler.go +++ b/paychmgr/settler/settler.go @@ -14,7 +14,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" - builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" @@ -107,7 +107,7 @@ func (pcs *paymentChannelSettler) revertHandler(ctx context.Context, ts *types.T func (pcs *paymentChannelSettler) matcher(msg *types.Message) (matchOnce bool, matched bool, err error) { // Check if this is a settle payment channel message - if msg.Method != builtin0.MethodsPaych.Settle { + if msg.Method != builtin2.MethodsPaych.Settle { return false, false, nil } // Check if this payment channel is of concern to this node (i.e. tracked in payment channel store), diff --git a/storage/adapter_storage_miner.go b/storage/adapter_storage_miner.go index 380fb4471..f645c301a 100644 --- a/storage/adapter_storage_miner.go +++ b/storage/adapter_storage_miner.go @@ -15,8 +15,8 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" - builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" - market0 "github.com/filecoin-project/specs-actors/actors/builtin/market" + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + market2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/market" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/apibstore" @@ -138,7 +138,7 @@ func (s SealingAPIAdapter) StateComputeDataCommitment(ctx context.Context, maddr return cid.Undef, xerrors.Errorf("failed to unmarshal TipSetToken to TipSetKey: %w", err) } - ccparams, err := actors.SerializeParams(&market0.ComputeDataCommitmentParams{ + ccparams, err := actors.SerializeParams(&market2.ComputeDataCommitmentParams{ DealIDs: deals, SectorType: sectorType, }) @@ -150,7 +150,7 @@ func (s SealingAPIAdapter) StateComputeDataCommitment(ctx context.Context, maddr To: market.Address, From: maddr, Value: types.NewInt(0), - Method: builtin0.MethodsMarket.ComputeDataCommitment, + Method: builtin2.MethodsMarket.ComputeDataCommitment, Params: ccparams, } r, err := s.delegate.StateCall(ctx, ccmt, tsk) diff --git a/storage/miner.go b/storage/miner.go index 74a048c8e..b5a2fd6a5 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -5,9 +5,6 @@ import ( "errors" "time" - miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" - proof0 "github.com/filecoin-project/specs-actors/actors/runtime/proof" - "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/go-state-types/dline" @@ -29,7 +26,9 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/events" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/types" @@ -150,7 +149,7 @@ func (m *Miner) Run(ctx context.Context) error { evts := events.NewEvents(ctx, m.api) adaptedAPI := NewSealingAPIAdapter(m.api) // TODO: Maybe we update this policy after actor upgrades? - pcp := sealing.NewBasicPreCommitPolicy(adaptedAPI, miner0.MaxSectorExpirationExtension-(miner0.WPoStProvingPeriod*2), md.PeriodStart%miner0.WPoStProvingPeriod) + pcp := sealing.NewBasicPreCommitPolicy(adaptedAPI, policy.GetMaxSectorExpirationExtension()-(md.WPoStProvingPeriod*2), md.PeriodStart%md.WPoStProvingPeriod) m.sealing = sealing.New(adaptedAPI, fc, NewEventsAdapter(evts), m.maddr, m.ds, m.sealer, m.sc, m.verif, &pcp, sealing.GetSealingConfigFunc(m.getSealConfig), m.handleSealingNotifications) go m.sealing.Run(ctx) //nolint:errcheck // logged intside the function @@ -238,9 +237,9 @@ func (wpp *StorageWpp) GenerateCandidates(ctx context.Context, randomness abi.Po return cds, nil } -func (wpp *StorageWpp) ComputeProof(ctx context.Context, ssi []proof0.SectorInfo, rand abi.PoStRandomness) ([]proof0.PoStProof, error) { +func (wpp *StorageWpp) ComputeProof(ctx context.Context, ssi []builtin.SectorInfo, rand abi.PoStRandomness) ([]builtin.PoStProof, error) { if build.InsecurePoStValidation { - return []proof0.PoStProof{{ProofBytes: []byte("valid proof")}}, nil + return []builtin.PoStProof{{ProofBytes: []byte("valid proof")}}, nil } log.Infof("Computing WinningPoSt ;%+v; %v", ssi, rand) diff --git a/storage/mockstorage/preseal.go b/storage/mockstorage/preseal.go index 090aacc3c..0417405c8 100644 --- a/storage/mockstorage/preseal.go +++ b/storage/mockstorage/preseal.go @@ -9,7 +9,7 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/lotus/extern/sector-storage/mock" - market0 "github.com/filecoin-project/specs-actors/actors/builtin/market" + market2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/market" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/wallet" @@ -48,7 +48,7 @@ func PreSeal(ssize abi.SectorSize, maddr address.Address, sectors int) (*genesis r := mock.CommDR(d) preseal.CommR, _ = commcid.ReplicaCommitmentV1ToCID(r[:]) preseal.SectorID = abi.SectorNumber(i + 1) - preseal.Deal = market0.DealProposal{ + preseal.Deal = market2.DealProposal{ PieceCID: preseal.CommD, PieceSize: abi.PaddedPieceSize(ssize), Client: k.Address, diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index d4ed4d64c..c11e5dca0 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -17,14 +17,14 @@ import ( "go.opencensus.io/trace" "golang.org/x/xerrors" - builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" - miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" - "github.com/filecoin-project/specs-actors/actors/runtime/proof" + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + proof2 "github.com/filecoin-project/specs-actors/v2/actors/runtime/proof" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/types" ) @@ -290,7 +290,7 @@ func (s *WindowPoStScheduler) checkNextRecoveries(ctx context.Context, dlIdx uin msg := &types.Message{ To: s.actor, From: s.worker, - Method: builtin0.MethodsMiner.DeclareFaultsRecovered, + Method: builtin2.MethodsMiner.DeclareFaultsRecovered, Params: enc, Value: types.NewInt(0), } @@ -374,7 +374,7 @@ func (s *WindowPoStScheduler) checkNextFaults(ctx context.Context, dlIdx uint64, msg := &types.Message{ To: s.actor, From: s.worker, - Method: builtin0.MethodsMiner.DeclareFaults, + Method: builtin2.MethodsMiner.DeclareFaults, Params: enc, Value: types.NewInt(0), // TODO: Is there a fee? } @@ -505,12 +505,12 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty skipCount := uint64(0) postSkipped := bitfield.New() - var postOut []proof.PoStProof + var postOut []proof2.PoStProof somethingToProve := true for retries := 0; retries < 5; retries++ { var partitions []miner.PoStPartition - var sinfos []proof.SectorInfo + var sinfos []proof2.SectorInfo for partIdx, partition := range batch { // TODO: Can do this in parallel toProve, err := bitfield.SubtractBitField(partition.LiveSectors, partition.FaultySectors) @@ -625,12 +625,6 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty } func (s *WindowPoStScheduler) batchPartitions(partitions []api.Partition) ([][]api.Partition, error) { - // Get the number of sectors allowed in a partition, for this proof size - sectorsPerPartition, err := builtin0.PoStProofWindowPoStPartitionSectors(s.proofType) - if err != nil { - return nil, xerrors.Errorf("getting sectors per partition: %w", err) - } - // We don't want to exceed the number of sectors allowed in a message. // So given the number of sectors in a partition, work out the number of // partitions that can be in a message without exceeding sectors per @@ -641,9 +635,10 @@ func (s *WindowPoStScheduler) batchPartitions(partitions []api.Partition) ([][]a // sectors per partition 3: ooo // partitions per message 2: oooOOO // <1><2> (3rd doesn't fit) - // TODO(NETUPGRADE): we're going to need some form of policy abstraction - // where we can get policy from the future. Unfortunately, we can't just get this from the state. - partitionsPerMsg := int(miner0.AddressedSectorsMax / sectorsPerPartition) + partitionsPerMsg, err := policy.GetMaxPoStPartitions(s.proofType) + if err != nil { + return nil, xerrors.Errorf("getting sectors per partition: %w", err) + } // The number of messages will be: // ceiling(number of partitions / partitions per message) @@ -665,7 +660,7 @@ func (s *WindowPoStScheduler) batchPartitions(partitions []api.Partition) ([][]a return batches, nil } -func (s *WindowPoStScheduler) sectorsForProof(ctx context.Context, goodSectors, allSectors bitfield.BitField, ts *types.TipSet) ([]proof.SectorInfo, error) { +func (s *WindowPoStScheduler) sectorsForProof(ctx context.Context, goodSectors, allSectors bitfield.BitField, ts *types.TipSet) ([]proof2.SectorInfo, error) { sset, err := s.api.StateMinerSectors(ctx, s.actor, &goodSectors, ts.Key()) if err != nil { return nil, err @@ -675,22 +670,22 @@ func (s *WindowPoStScheduler) sectorsForProof(ctx context.Context, goodSectors, return nil, nil } - substitute := proof.SectorInfo{ + substitute := proof2.SectorInfo{ SectorNumber: sset[0].SectorNumber, SealedCID: sset[0].SealedCID, SealProof: sset[0].SealProof, } - sectorByID := make(map[uint64]proof.SectorInfo, len(sset)) + sectorByID := make(map[uint64]proof2.SectorInfo, len(sset)) for _, sector := range sset { - sectorByID[uint64(sector.SectorNumber)] = proof.SectorInfo{ + sectorByID[uint64(sector.SectorNumber)] = proof2.SectorInfo{ SectorNumber: sector.SectorNumber, SealedCID: sector.SealedCID, SealProof: sector.SealProof, } } - proofSectors := make([]proof.SectorInfo, 0, len(sset)) + proofSectors := make([]proof2.SectorInfo, 0, len(sset)) if err := allSectors.ForEach(func(sectorNo uint64) error { if info, found := sectorByID[sectorNo]; found { proofSectors = append(proofSectors, info) @@ -719,7 +714,7 @@ func (s *WindowPoStScheduler) submitPost(ctx context.Context, proof *miner.Submi msg := &types.Message{ To: s.actor, From: s.worker, - Method: builtin0.MethodsMiner.SubmitWindowedPoSt, + Method: builtin2.MethodsMiner.SubmitWindowedPoSt, Params: enc, Value: types.NewInt(0), } diff --git a/storage/wdpost_run_test.go b/storage/wdpost_run_test.go index dd7ac4c24..75588d7f8 100644 --- a/storage/wdpost_run_test.go +++ b/storage/wdpost_run_test.go @@ -17,10 +17,10 @@ import ( "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/go-state-types/dline" "github.com/filecoin-project/go-state-types/network" - builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" - miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" - proof0 "github.com/filecoin-project/specs-actors/actors/runtime/proof" - tutils "github.com/filecoin-project/specs-actors/support/testing" + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" + proof2 "github.com/filecoin-project/specs-actors/v2/actors/runtime/proof" + tutils "github.com/filecoin-project/specs-actors/v2/support/testing" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" @@ -98,12 +98,12 @@ func (m *mockStorageMinerAPI) StateWaitMsg(ctx context.Context, cid cid.Cid, con type mockProver struct { } -func (m *mockProver) GenerateWinningPoSt(context.Context, abi.ActorID, []proof0.SectorInfo, abi.PoStRandomness) ([]proof0.PoStProof, error) { +func (m *mockProver) GenerateWinningPoSt(context.Context, abi.ActorID, []proof2.SectorInfo, abi.PoStRandomness) ([]proof2.PoStProof, error) { panic("implement me") } -func (m *mockProver) GenerateWindowPoSt(ctx context.Context, aid abi.ActorID, sis []proof0.SectorInfo, pr abi.PoStRandomness) ([]proof0.PoStProof, []abi.SectorID, error) { - return []proof0.PoStProof{ +func (m *mockProver) GenerateWindowPoSt(ctx context.Context, aid abi.ActorID, sis []proof2.SectorInfo, pr abi.PoStRandomness) ([]proof2.PoStProof, []abi.SectorID, error) { + return []proof2.PoStProof{ { PoStProof: abi.RegisteredPoStProof_StackedDrgWindow2KiBV1, ProofBytes: []byte("post-proof"), @@ -132,13 +132,13 @@ func TestWDPostDoPost(t *testing.T) { mockStgMinerAPI := newMockStorageMinerAPI() // Get the number of sectors allowed in a partition for this proof type - sectorsPerPartition, err := builtin0.PoStProofWindowPoStPartitionSectors(proofType) + sectorsPerPartition, err := builtin2.PoStProofWindowPoStPartitionSectors(proofType) require.NoError(t, err) // Work out the number of partitions that can be included in a message // without exceeding the message sector limit require.NoError(t, err) - partitionsPerMsg := int(miner0.AddressedSectorsMax / sectorsPerPartition) + partitionsPerMsg := int(miner2.AddressedSectorsMax / sectorsPerPartition) // Enough partitions to fill expectedMsgCount-1 messages partitionCount := (expectedMsgCount - 1) * partitionsPerMsg @@ -173,11 +173,11 @@ func TestWDPostDoPost(t *testing.T) { } di := &dline.Info{ - WPoStPeriodDeadlines: miner0.WPoStPeriodDeadlines, - WPoStProvingPeriod: miner0.WPoStProvingPeriod, - WPoStChallengeWindow: miner0.WPoStChallengeWindow, - WPoStChallengeLookback: miner0.WPoStChallengeLookback, - FaultDeclarationCutoff: miner0.FaultDeclarationCutoff, + WPoStPeriodDeadlines: miner2.WPoStPeriodDeadlines, + WPoStProvingPeriod: miner2.WPoStProvingPeriod, + WPoStChallengeWindow: miner2.WPoStChallengeWindow, + WPoStChallengeLookback: miner2.WPoStChallengeLookback, + FaultDeclarationCutoff: miner2.FaultDeclarationCutoff, } ts := mockTipSet(t) @@ -188,7 +188,7 @@ func TestWDPostDoPost(t *testing.T) { // Read the window PoST messages for i := 0; i < expectedMsgCount; i++ { msg := <-mockStgMinerAPI.pushedMessages - require.Equal(t, builtin0.MethodsMiner.SubmitWindowedPoSt, msg.Method) + require.Equal(t, builtin2.MethodsMiner.SubmitWindowedPoSt, msg.Method) var params miner.SubmitWindowedPoStParams err := params.UnmarshalCBOR(bytes.NewReader(msg.Params)) require.NoError(t, err) @@ -254,11 +254,11 @@ func (m *mockStorageMinerAPI) StateMinerProvingDeadline(ctx context.Context, add Close: 0, Challenge: 0, FaultCutoff: 0, - WPoStPeriodDeadlines: miner0.WPoStPeriodDeadlines, - WPoStProvingPeriod: miner0.WPoStProvingPeriod, - WPoStChallengeWindow: miner0.WPoStChallengeWindow, - WPoStChallengeLookback: miner0.WPoStChallengeLookback, - FaultDeclarationCutoff: miner0.FaultDeclarationCutoff, + WPoStPeriodDeadlines: miner2.WPoStPeriodDeadlines, + WPoStProvingPeriod: miner2.WPoStProvingPeriod, + WPoStChallengeWindow: miner2.WPoStChallengeWindow, + WPoStChallengeLookback: miner2.WPoStChallengeLookback, + FaultDeclarationCutoff: miner2.FaultDeclarationCutoff, }, nil } @@ -276,7 +276,7 @@ func (m *mockStorageMinerAPI) StateSearchMsg(ctx context.Context, cid cid.Cid) ( func (m *mockStorageMinerAPI) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) { return &types.Actor{ - Code: builtin0.StorageMinerActorCodeID, + Code: builtin2.StorageMinerActorCodeID, }, nil } From bcabe7b3b59fed67c823d90fc9ef3b4125ec65aa Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 8 Oct 2020 13:32:54 -0700 Subject: [PATCH 830/885] migrate methods to abstracted methods Method numbers never change anyways. At worst, we'll deprecate old methods and have to explicitly import them from the correct actors version to use them. --- chain/actors/builtin/account/account.go | 2 ++ chain/actors/builtin/builtin.go | 5 +++ chain/actors/builtin/cron/cron.go | 10 ++++++ chain/actors/builtin/init/init.go | 5 ++- chain/actors/builtin/market/market.go | 5 ++- chain/actors/builtin/miner/miner.go | 2 ++ chain/actors/builtin/multisig/message.go | 3 ++ chain/actors/builtin/paych/message.go | 4 +++ chain/actors/builtin/power/power.go | 5 ++- chain/actors/builtin/reward/reward.go | 5 ++- chain/actors/builtin/verifreg/verifreg.go | 5 ++- chain/market/fundmgr.go | 8 ++--- chain/market/fundmgr_test.go | 4 +-- chain/messagepool/gasguess/guessgas.go | 3 +- chain/stmgr/forks_test.go | 10 +++--- chain/stmgr/stmgr.go | 37 +++++++++++------------ chain/stmgr/utils.go | 14 +++------ chain/vm/gas_v0.go | 7 +++-- chain/vm/mkactor.go | 3 +- cli/multisig.go | 10 +++--- cmd/lotus-pcr/main.go | 9 +++--- cmd/lotus-shed/mempool-stats.go | 5 ++- cmd/lotus-shed/sectors.go | 20 ++++++------ cmd/lotus-shed/verifreg.go | 5 ++- cmd/lotus-storage-miner/actor.go | 24 +++++++-------- cmd/lotus-storage-miner/init.go | 10 +++--- conformance/chaos/actor.go | 21 +++++++------ extern/storage-sealing/states_sealing.go | 6 ++-- markets/storageadapter/provider.go | 9 +++--- node/impl/full/gas.go | 5 ++- node/impl/full/multisig.go | 15 +++++---- node/test/builder.go | 4 +-- paychmgr/settler/settler.go | 4 +-- storage/adapter_storage_miner.go | 3 +- storage/wdpost_run.go | 7 ++--- storage/wdpost_run_test.go | 2 +- 36 files changed, 160 insertions(+), 136 deletions(-) create mode 100644 chain/actors/builtin/cron/cron.go diff --git a/chain/actors/builtin/account/account.go b/chain/actors/builtin/account/account.go index 7b1b2a792..38ed2654b 100644 --- a/chain/actors/builtin/account/account.go +++ b/chain/actors/builtin/account/account.go @@ -24,6 +24,8 @@ func init() { }) } +var Methods = builtin2.MethodsAccount + func Load(store adt.Store, act *types.Actor) (State, error) { switch act.Code { case builtin0.AccountActorCodeID: diff --git a/chain/actors/builtin/builtin.go b/chain/actors/builtin/builtin.go index 52d1375b4..afba8efe8 100644 --- a/chain/actors/builtin/builtin.go +++ b/chain/actors/builtin/builtin.go @@ -37,6 +37,11 @@ const ( SecondsInDay = builtin0.SecondsInDay ) +const ( + MethodSend = builtin2.MethodSend + MethodConstructor = builtin2.MethodConstructor +) + // TODO: Why does actors have 2 different versions of this? type SectorInfo = proof0.SectorInfo type PoStProof = proof0.PoStProof diff --git a/chain/actors/builtin/cron/cron.go b/chain/actors/builtin/cron/cron.go new file mode 100644 index 000000000..65bfd992f --- /dev/null +++ b/chain/actors/builtin/cron/cron.go @@ -0,0 +1,10 @@ +package cron + +import ( + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" +) + +var ( + Address = builtin2.CronActorAddr + Methods = builtin2.MethodsCron +) diff --git a/chain/actors/builtin/init/init.go b/chain/actors/builtin/init/init.go index 5777bb890..60dbdf4fe 100644 --- a/chain/actors/builtin/init/init.go +++ b/chain/actors/builtin/init/init.go @@ -26,7 +26,10 @@ func init() { }) } -var Address = builtin0.InitActorAddr +var ( + Address = builtin2.InitActorAddr + Methods = builtin2.MethodsInit +) func Load(store adt.Store, act *types.Actor) (State, error) { switch act.Code { diff --git a/chain/actors/builtin/market/market.go b/chain/actors/builtin/market/market.go index fd08a0119..195ca40b9 100644 --- a/chain/actors/builtin/market/market.go +++ b/chain/actors/builtin/market/market.go @@ -27,7 +27,10 @@ func init() { }) } -var Address = builtin0.StorageMarketActorAddr +var ( + Address = builtin2.StorageMarketActorAddr + Methods = builtin2.MethodsMarket +) func Load(store adt.Store, act *types.Actor) (st State, err error) { switch act.Code { diff --git a/chain/actors/builtin/miner/miner.go b/chain/actors/builtin/miner/miner.go index 3cca39326..aad41e8a9 100644 --- a/chain/actors/builtin/miner/miner.go +++ b/chain/actors/builtin/miner/miner.go @@ -31,6 +31,8 @@ func init() { }) } +var Methods = builtin2.MethodsMiner + // Unchanged between v0 and v2 actors var WPoStProvingPeriod = miner0.WPoStProvingPeriod var WPoStPeriodDeadlines = miner0.WPoStPeriodDeadlines diff --git a/chain/actors/builtin/multisig/message.go b/chain/actors/builtin/multisig/message.go index 5beb5319d..3d2c66e6b 100644 --- a/chain/actors/builtin/multisig/message.go +++ b/chain/actors/builtin/multisig/message.go @@ -9,12 +9,15 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" multisig2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/multisig" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/types" ) +var Methods = builtin2.MethodsMultisig + func Message(version actors.Version, from address.Address) MessageBuilder { switch version { case actors.Version0: diff --git a/chain/actors/builtin/paych/message.go b/chain/actors/builtin/paych/message.go index 23b360394..5709d4b23 100644 --- a/chain/actors/builtin/paych/message.go +++ b/chain/actors/builtin/paych/message.go @@ -7,8 +7,12 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/types" + + builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" ) +var Methods = builtin2.MethodsPaych + func Message(version actors.Version, from address.Address) MessageBuilder { switch version { case actors.Version0: diff --git a/chain/actors/builtin/power/power.go b/chain/actors/builtin/power/power.go index bafb14de0..c9182b7cf 100644 --- a/chain/actors/builtin/power/power.go +++ b/chain/actors/builtin/power/power.go @@ -25,7 +25,10 @@ func init() { }) } -var Address = builtin0.StoragePowerActorAddr +var ( + Address = builtin2.StoragePowerActorAddr + Methods = builtin2.MethodsPower +) func Load(store adt.Store, act *types.Actor) (st State, err error) { switch act.Code { diff --git a/chain/actors/builtin/reward/reward.go b/chain/actors/builtin/reward/reward.go index 065f242e2..952ca270b 100644 --- a/chain/actors/builtin/reward/reward.go +++ b/chain/actors/builtin/reward/reward.go @@ -24,7 +24,10 @@ func init() { }) } -var Address = builtin0.RewardActorAddr +var ( + Address = builtin2.RewardActorAddr + Methods = builtin2.MethodsReward +) func Load(store adt.Store, act *types.Actor) (st State, err error) { switch act.Code { diff --git a/chain/actors/builtin/verifreg/verifreg.go b/chain/actors/builtin/verifreg/verifreg.go index 204cdae95..a4468d8a0 100644 --- a/chain/actors/builtin/verifreg/verifreg.go +++ b/chain/actors/builtin/verifreg/verifreg.go @@ -24,7 +24,10 @@ func init() { }) } -var Address = builtin0.VerifiedRegistryActorAddr +var ( + Address = builtin2.VerifiedRegistryActorAddr + Methods = builtin2.MethodsVerifiedRegistry +) func Load(store adt.Store, act *types.Actor) (State, error) { switch act.Code { diff --git a/chain/market/fundmgr.go b/chain/market/fundmgr.go index c65547285..50467a6e1 100644 --- a/chain/market/fundmgr.go +++ b/chain/market/fundmgr.go @@ -4,15 +4,13 @@ import ( "context" "sync" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" - "go.uber.org/fx" - - builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log" + "go.uber.org/fx" - "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/builtin/market" @@ -156,7 +154,7 @@ func (fm *FundMgr) EnsureAvailable(ctx context.Context, addr, wallet address.Add To: market.Address, From: wallet, Value: toAdd, - Method: builtin2.MethodsMarket.AddBalance, + Method: market.Methods.AddBalance, Params: params, }, nil) if err != nil { diff --git a/chain/market/fundmgr_test.go b/chain/market/fundmgr_test.go index 7f163201e..88ca2e16f 100644 --- a/chain/market/fundmgr_test.go +++ b/chain/market/fundmgr_test.go @@ -12,7 +12,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/crypto" - builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + tutils "github.com/filecoin-project/specs-actors/v2/support/testing" "github.com/filecoin-project/lotus/api" @@ -51,7 +51,7 @@ func addFundsMsg(toAdd abi.TokenAmount, addr address.Address, wallet address.Add To: market.Address, From: wallet, Value: toAdd, - Method: builtin2.MethodsMarket.AddBalance, + Method: market.Methods.AddBalance, Params: params, } } diff --git a/chain/messagepool/gasguess/guessgas.go b/chain/messagepool/gasguess/guessgas.go index 607c7824a..76ab39078 100644 --- a/chain/messagepool/gasguess/guessgas.go +++ b/chain/messagepool/gasguess/guessgas.go @@ -6,6 +6,7 @@ import ( "github.com/ipfs/go-cid" "golang.org/x/xerrors" + "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/go-address" @@ -68,7 +69,7 @@ func failedGuess(msg *types.SignedMessage) int64 { func GuessGasUsed(ctx context.Context, tsk types.TipSetKey, msg *types.SignedMessage, al ActorLookup) (int64, error) { // MethodSend is the same in all versions. - if msg.Message.Method == builtin0.MethodSend { + if msg.Message.Method == builtin.MethodSend { switch msg.Message.From.Protocol() { case address.BLS: return 1298450, nil diff --git a/chain/stmgr/forks_test.go b/chain/stmgr/forks_test.go index 3e9e9e8c0..89e051088 100644 --- a/chain/stmgr/forks_test.go +++ b/chain/stmgr/forks_test.go @@ -24,7 +24,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/aerrors" - lotusinit "github.com/filecoin-project/lotus/chain/actors/builtin/init" + _init "github.com/filecoin-project/lotus/chain/actors/builtin/init" "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/gen" . "github.com/filecoin-project/lotus/chain/stmgr" @@ -183,8 +183,8 @@ func TestForkHeightTriggers(t *testing.T) { m := &types.Message{ From: cg.Banker(), - To: lotusinit.Address, - Method: builtin2.MethodsInit.Exec, + To: _init.Address, + Method: _init.Methods.Exec, Params: enc, GasLimit: types.TestGasLimit, } @@ -281,8 +281,8 @@ func TestForkRefuseCall(t *testing.T) { m := &types.Message{ From: cg.Banker(), - To: lotusinit.Address, - Method: builtin2.MethodsInit.Exec, + To: _init.Address, + Method: _init.Methods.Exec, Params: enc, GasLimit: types.TestGasLimit, Value: types.NewInt(0), diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index 6558b1ea8..4f0c26b7c 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -18,8 +18,6 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/network" - builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" - // Used for genesis. msig0 "github.com/filecoin-project/specs-actors/actors/builtin/multisig" @@ -28,6 +26,7 @@ import ( "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/actors/builtin" + "github.com/filecoin-project/lotus/chain/actors/builtin/cron" _init "github.com/filecoin-project/lotus/chain/actors/builtin/init" "github.com/filecoin-project/lotus/chain/actors/builtin/market" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" @@ -253,14 +252,14 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp runCron := func(epoch abi.ChainEpoch) error { cronMsg := &types.Message{ - To: builtin2.CronActorAddr, - From: builtin2.SystemActorAddr, + To: cron.Address, + From: builtin.SystemActorAddr, Nonce: uint64(epoch), Value: types.NewInt(0), GasFeeCap: types.NewInt(0), GasPremium: types.NewInt(0), GasLimit: build.BlockGasLimit * 10000, // Make super sure this is never too little - Method: builtin2.MethodsCron.EpochTick, + Method: cron.Methods.EpochTick, Params: nil, } ret, err := vmi.ApplyImplicitMessage(ctx, cronMsg) @@ -349,14 +348,14 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp } rwMsg := &types.Message{ - From: builtin2.SystemActorAddr, + From: builtin.SystemActorAddr, To: reward.Address, Nonce: uint64(epoch), Value: types.NewInt(0), GasFeeCap: types.NewInt(0), GasPremium: types.NewInt(0), GasLimit: 1 << 30, - Method: builtin2.MethodsReward.AwardBlockReward, + Method: reward.Methods.AwardBlockReward, Params: params, } ret, actErr := vmi.ApplyImplicitMessage(ctx, rwMsg) @@ -973,7 +972,7 @@ func (sm *StateManager) setupGenesisActors(ctx context.Context) error { } else if builtin.IsAccountActor(act.Code) { // should exclude burnt funds actor and "remainder account actor" // should only ever be "faucet" accounts in testnets - if kaddr == builtin2.BurntFundsActorAddr { + if kaddr == builtin.BurntFundsActorAddr { return nil } @@ -1051,24 +1050,24 @@ func (sm *StateManager) setupPreIgnitionGenesisActorsTestnet(ctx context.Context totalsByEpoch := make(map[abi.ChainEpoch]abi.TokenAmount) // 6 months - sixMonths := abi.ChainEpoch(183 * builtin2.EpochsInDay) + sixMonths := abi.ChainEpoch(183 * builtin.EpochsInDay) totalsByEpoch[sixMonths] = big.NewInt(49_929_341) totalsByEpoch[sixMonths] = big.Add(totalsByEpoch[sixMonths], big.NewInt(32_787_700)) // 1 year - oneYear := abi.ChainEpoch(365 * builtin2.EpochsInDay) + oneYear := abi.ChainEpoch(365 * builtin.EpochsInDay) totalsByEpoch[oneYear] = big.NewInt(22_421_712) // 2 years - twoYears := abi.ChainEpoch(2 * 365 * builtin2.EpochsInDay) + twoYears := abi.ChainEpoch(2 * 365 * builtin.EpochsInDay) totalsByEpoch[twoYears] = big.NewInt(7_223_364) // 3 years - threeYears := abi.ChainEpoch(3 * 365 * builtin2.EpochsInDay) + threeYears := abi.ChainEpoch(3 * 365 * builtin.EpochsInDay) totalsByEpoch[threeYears] = big.NewInt(87_637_883) // 6 years - sixYears := abi.ChainEpoch(6 * 365 * builtin2.EpochsInDay) + sixYears := abi.ChainEpoch(6 * 365 * builtin.EpochsInDay) totalsByEpoch[sixYears] = big.NewInt(100_000_000) totalsByEpoch[sixYears] = big.Add(totalsByEpoch[sixYears], big.NewInt(300_000_000)) @@ -1128,24 +1127,24 @@ func (sm *StateManager) setupPostIgnitionGenesisActors(ctx context.Context) erro totalsByEpoch := make(map[abi.ChainEpoch]abi.TokenAmount) // 6 months - sixMonths := abi.ChainEpoch(183 * builtin2.EpochsInDay) + sixMonths := abi.ChainEpoch(183 * builtin.EpochsInDay) totalsByEpoch[sixMonths] = big.NewInt(49_929_341) totalsByEpoch[sixMonths] = big.Add(totalsByEpoch[sixMonths], big.NewInt(32_787_700)) // 1 year - oneYear := abi.ChainEpoch(365 * builtin2.EpochsInDay) + oneYear := abi.ChainEpoch(365 * builtin.EpochsInDay) totalsByEpoch[oneYear] = big.NewInt(22_421_712) // 2 years - twoYears := abi.ChainEpoch(2 * 365 * builtin2.EpochsInDay) + twoYears := abi.ChainEpoch(2 * 365 * builtin.EpochsInDay) totalsByEpoch[twoYears] = big.NewInt(7_223_364) // 3 years - threeYears := abi.ChainEpoch(3 * 365 * builtin2.EpochsInDay) + threeYears := abi.ChainEpoch(3 * 365 * builtin.EpochsInDay) totalsByEpoch[threeYears] = big.NewInt(87_637_883) // 6 years - sixYears := abi.ChainEpoch(6 * 365 * builtin2.EpochsInDay) + sixYears := abi.ChainEpoch(6 * 365 * builtin.EpochsInDay) totalsByEpoch[sixYears] = big.NewInt(100_000_000) totalsByEpoch[sixYears] = big.Add(totalsByEpoch[sixYears], big.NewInt(300_000_000)) @@ -1279,7 +1278,7 @@ func (sm *StateManager) GetFilLocked(ctx context.Context, st *state.StateTree) ( } func GetFilBurnt(ctx context.Context, st *state.StateTree) (abi.TokenAmount, error) { - burnt, err := st.GetActor(builtin2.BurntFundsActorAddr) + burnt, err := st.GetActor(builtin.BurntFundsActorAddr) if err != nil { return big.Zero(), xerrors.Errorf("failed to load burnt actor: %w", err) } diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index 2f5acf0db..eec7c3a15 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -12,8 +12,6 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/network" - "github.com/filecoin-project/lotus/chain/actors/builtin" - "github.com/filecoin-project/lotus/chain/actors/policy" cid "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" @@ -25,15 +23,16 @@ import ( "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/go-state-types/rt" - builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" exported0 "github.com/filecoin-project/specs-actors/actors/builtin/exported" exported2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/exported" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/actors/builtin" init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init" "github.com/filecoin-project/lotus/chain/actors/builtin/market" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/actors/builtin/power" + "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/beacon" "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/store" @@ -548,8 +547,7 @@ func init() { methods := make(map[abi.MethodNum]MethodMeta, len(exports)) // Explicitly add send, it's special. - // Note that builtin2.MethodSend = builtin0.MethodSend = 0. - methods[builtin0.MethodSend] = MethodMeta{ + methods[builtin.MethodSend] = MethodMeta{ Name: "Send", Params: reflect.TypeOf(new(abi.EmptyValue)), Ret: reflect.TypeOf(new(abi.EmptyValue)), @@ -573,11 +571,9 @@ func init() { fnName = strings.TrimSuffix(fnName[strings.LastIndexByte(fnName, '.')+1:], "-fm") switch abi.MethodNum(number) { - case builtin0.MethodSend: - // Note that builtin2.MethodSend = builtin0.MethodSend = 0. + case builtin.MethodSend: panic("method 0 is reserved for Send") - case builtin0.MethodConstructor: - // Note that builtin2.MethodConstructor = builtin0.MethodConstructor = 1. + case builtin.MethodConstructor: if fnName != "Constructor" { panic("method 1 is reserved for Constructor") } diff --git a/chain/vm/gas_v0.go b/chain/vm/gas_v0.go index 7e879b8c3..7a7fb364d 100644 --- a/chain/vm/gas_v0.go +++ b/chain/vm/gas_v0.go @@ -3,12 +3,13 @@ package vm import ( "fmt" - builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" proof2 "github.com/filecoin-project/specs-actors/v2/actors/runtime/proof" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" + + "github.com/filecoin-project/lotus/chain/actors/builtin" ) type scalingCost struct { @@ -112,14 +113,14 @@ func (pl *pricelistV0) OnMethodInvocation(value abi.TokenAmount, methodNum abi.M if big.Cmp(value, abi.NewTokenAmount(0)) != 0 { ret += pl.sendTransferFunds - if methodNum == builtin2.MethodSend { + if methodNum == builtin.MethodSend { // transfer only ret += pl.sendTransferOnlyPremium } extra += "t" } - if methodNum != builtin2.MethodSend { + if methodNum != builtin.MethodSend { extra += "i" // running actors is cheaper becase we hand over to actors ret += pl.sendInvokeMethod diff --git a/chain/vm/mkactor.go b/chain/vm/mkactor.go index 56eac1e0c..885d3c0db 100644 --- a/chain/vm/mkactor.go +++ b/chain/vm/mkactor.go @@ -16,6 +16,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/actors/aerrors" "github.com/filecoin-project/lotus/chain/actors/builtin" + "github.com/filecoin-project/lotus/chain/actors/builtin/account" "github.com/filecoin-project/lotus/chain/types" ) @@ -57,7 +58,7 @@ func TryCreateAccountActor(rt *Runtime, addr address.Address) (*types.Actor, add } // call constructor on account - _, aerr = rt.internalSend(builtin.SystemActorAddr, addrID, builtin2.MethodsAccount.Constructor, big.Zero(), p) + _, aerr = rt.internalSend(builtin.SystemActorAddr, addrID, account.Methods.Constructor, big.Zero(), p) if aerr != nil { return nil, address.Undef, aerrors.Wrap(aerr, "failed to invoke account constructor") } diff --git a/cli/multisig.go b/cli/multisig.go index ebdc8ef7d..a50032cda 100644 --- a/cli/multisig.go +++ b/cli/multisig.go @@ -10,8 +10,6 @@ import ( "strconv" "text/tabwriter" - builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" - "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/actors" @@ -1170,7 +1168,7 @@ var msigLockProposeCmd = &cli.Command{ return actErr } - msgCid, err := api.MsigPropose(ctx, msig, msig, big.Zero(), from, uint64(builtin2.MethodsMultisig.LockBalance), params) + msgCid, err := api.MsigPropose(ctx, msig, msig, big.Zero(), from, uint64(multisig.Methods.LockBalance), params) if err != nil { return err } @@ -1267,7 +1265,7 @@ var msigLockApproveCmd = &cli.Command{ return actErr } - msgCid, err := api.MsigApproveTxnHash(ctx, msig, txid, prop, msig, big.Zero(), from, uint64(builtin2.MethodsMultisig.LockBalance), params) + msgCid, err := api.MsigApproveTxnHash(ctx, msig, txid, prop, msig, big.Zero(), from, uint64(multisig.Methods.LockBalance), params) if err != nil { return err } @@ -1359,7 +1357,7 @@ var msigLockCancelCmd = &cli.Command{ return actErr } - msgCid, err := api.MsigCancel(ctx, msig, txid, msig, big.Zero(), from, uint64(builtin2.MethodsMultisig.LockBalance), params) + msgCid, err := api.MsigCancel(ctx, msig, txid, msig, big.Zero(), from, uint64(multisig.Methods.LockBalance), params) if err != nil { return err } @@ -1496,7 +1494,7 @@ var msigProposeThresholdCmd = &cli.Command{ return actErr } - msgCid, err := api.MsigPropose(ctx, msig, msig, types.NewInt(0), from, uint64(builtin2.MethodsMultisig.ChangeNumApprovalsThreshold), params) + msgCid, err := api.MsigPropose(ctx, msig, msig, types.NewInt(0), from, uint64(multisig.Methods.ChangeNumApprovalsThreshold), params) if err != nil { return fmt.Errorf("failed to propose change of threshold: %w", err) } diff --git a/cmd/lotus-pcr/main.go b/cmd/lotus-pcr/main.go index 02df4a192..1d37a3ce9 100644 --- a/cmd/lotus-pcr/main.go +++ b/cmd/lotus-pcr/main.go @@ -16,7 +16,6 @@ import ( "github.com/filecoin-project/lotus/chain/actors/builtin" - builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" "github.com/filecoin-project/go-state-types/network" @@ -870,7 +869,7 @@ func (r *refunder) processTipsetStorageMarketActor(ctx context.Context, tipset * var messageMethod string switch m.Method { - case builtin2.MethodsMarket.PublishStorageDeals: + case market.Methods.PublishStorageDeals: if !r.publishStorageDealsEnabled { return false, messageMethod, types.NewInt(0), nil } @@ -897,7 +896,7 @@ func (r *refunder) processTipsetStorageMinerActor(ctx context.Context, tipset *t var messageMethod string switch m.Method { - case builtin2.MethodsMiner.SubmitWindowedPoSt: + case miner.Methods.SubmitWindowedPoSt: if !r.windowedPoStEnabled { return false, messageMethod, types.NewInt(0), nil } @@ -910,7 +909,7 @@ func (r *refunder) processTipsetStorageMinerActor(ctx context.Context, tipset *t } refundValue = types.BigMul(types.NewInt(uint64(recp.GasUsed)), tipset.Blocks()[0].ParentBaseFee) - case builtin2.MethodsMiner.ProveCommitSector: + case miner.Methods.ProveCommitSector: if !r.proveCommitEnabled { return false, messageMethod, types.NewInt(0), nil } @@ -966,7 +965,7 @@ func (r *refunder) processTipsetStorageMinerActor(ctx context.Context, tipset *t if r.refundPercent > 0 { refundValue = types.BigMul(types.BigDiv(refundValue, types.NewInt(100)), types.NewInt(uint64(r.refundPercent))) } - case builtin2.MethodsMiner.PreCommitSector: + case miner.Methods.PreCommitSector: if !r.preCommitEnabled { return false, messageMethod, types.NewInt(0), nil } diff --git a/cmd/lotus-shed/mempool-stats.go b/cmd/lotus-shed/mempool-stats.go index 408e06f71..bc4a801f0 100644 --- a/cmd/lotus-shed/mempool-stats.go +++ b/cmd/lotus-shed/mempool-stats.go @@ -14,11 +14,10 @@ import ( "go.opencensus.io/stats/view" "go.opencensus.io/tag" - builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" - "github.com/filecoin-project/go-address" lapi "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors/builtin" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" ) @@ -145,7 +144,7 @@ var mpoolStatsCmd = &cli.Command{ seen: time.Now(), } - if u.Message.Message.Method == builtin2.MethodsMiner.SubmitWindowedPoSt { + if u.Message.Message.Method == miner.Methods.SubmitWindowedPoSt { miner, err := isMiner(u.Message.Message.To) if err != nil { diff --git a/cmd/lotus-shed/sectors.go b/cmd/lotus-shed/sectors.go index cf0d5fcdb..2e78469fa 100644 --- a/cmd/lotus-shed/sectors.go +++ b/cmd/lotus-shed/sectors.go @@ -4,17 +4,19 @@ import ( "fmt" "strconv" + "golang.org/x/xerrors" + "github.com/filecoin-project/go-bitfield" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" - "github.com/filecoin-project/lotus/chain/actors" - "golang.org/x/xerrors" + "github.com/urfave/cli/v2" + miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" + + "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" - builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" - miner0 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" - "github.com/urfave/cli/v2" ) var sectorsCmd = &cli.Command{ @@ -69,7 +71,7 @@ var terminateSectorCmd = &cli.Command{ return err } - terminationDeclarationParams := []miner0.TerminationDeclaration{} + terminationDeclarationParams := []miner2.TerminationDeclaration{} for _, sn := range cctx.Args().Slice() { sectorNum, err := strconv.ParseUint(sn, 10, 64) @@ -85,7 +87,7 @@ var terminateSectorCmd = &cli.Command{ return fmt.Errorf("get state sector partition %s", err) } - para := miner0.TerminationDeclaration{ + para := miner2.TerminationDeclaration{ Deadline: loca.Deadline, Partition: loca.Partition, Sectors: sectorbit, @@ -94,7 +96,7 @@ var terminateSectorCmd = &cli.Command{ terminationDeclarationParams = append(terminationDeclarationParams, para) } - terminateSectorParams := &miner0.TerminateSectorsParams{ + terminateSectorParams := &miner2.TerminateSectorsParams{ Terminations: terminationDeclarationParams, } @@ -106,7 +108,7 @@ var terminateSectorCmd = &cli.Command{ smsg, err := nodeApi.MpoolPushMessage(ctx, &types.Message{ From: mi.Owner, To: maddr, - Method: builtin2.MethodsMiner.TerminateSectors, + Method: miner.Methods.TerminateSectors, Value: big.Zero(), Params: sp, diff --git a/cmd/lotus-shed/verifreg.go b/cmd/lotus-shed/verifreg.go index c4eada68c..df1f0d990 100644 --- a/cmd/lotus-shed/verifreg.go +++ b/cmd/lotus-shed/verifreg.go @@ -11,7 +11,6 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" - builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" verifreg2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/verifreg" "github.com/filecoin-project/lotus/api/apibstore" @@ -80,7 +79,7 @@ var verifRegAddVerifierCmd = &cli.Command{ return err } - smsg, err := api.MsigPropose(ctx, vrk, verifreg.Address, big.Zero(), sender, uint64(builtin2.MethodsVerifiedRegistry.AddVerifier), params) + smsg, err := api.MsigPropose(ctx, vrk, verifreg.Address, big.Zero(), sender, uint64(verifreg.Methods.AddVerifier), params) if err != nil { return err } @@ -151,7 +150,7 @@ var verifRegVerifyClientCmd = &cli.Command{ msg := &types.Message{ To: verifreg.Address, From: fromk, - Method: builtin2.MethodsVerifiedRegistry.AddVerifiedClient, + Method: verifreg.Methods.AddVerifiedClient, Params: params, } diff --git a/cmd/lotus-storage-miner/actor.go b/cmd/lotus-storage-miner/actor.go index ad2232e1c..611ea8f19 100644 --- a/cmd/lotus-storage-miner/actor.go +++ b/cmd/lotus-storage-miner/actor.go @@ -5,13 +5,8 @@ import ( "os" "strings" - "github.com/filecoin-project/lotus/api/apibstore" - "github.com/filecoin-project/lotus/chain/actors/adt" - "github.com/filecoin-project/lotus/chain/actors/builtin/miner" cbor "github.com/ipfs/go-ipld-cbor" - "github.com/filecoin-project/lotus/build" - "github.com/fatih/color" "github.com/libp2p/go-libp2p-core/peer" ma "github.com/multiformats/go-multiaddr" @@ -22,10 +17,13 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" - builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" + "github.com/filecoin-project/lotus/api/apibstore" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/lib/tablewriter" @@ -109,7 +107,7 @@ var actorSetAddrsCmd = &cli.Command{ From: minfo.Worker, Value: types.NewInt(0), GasLimit: gasLimit, - Method: builtin2.MethodsMiner.ChangeMultiaddrs, + Method: miner.Methods.ChangeMultiaddrs, Params: params, }, nil) if err != nil { @@ -174,7 +172,7 @@ var actorSetPeeridCmd = &cli.Command{ From: minfo.Worker, Value: types.NewInt(0), GasLimit: gasLimit, - Method: builtin2.MethodsMiner.ChangePeerID, + Method: miner.Methods.ChangePeerID, Params: params, }, nil) if err != nil { @@ -246,7 +244,7 @@ var actorWithdrawCmd = &cli.Command{ To: maddr, From: mi.Owner, Value: types.NewInt(0), - Method: builtin2.MethodsMiner.WithdrawBalance, + Method: miner.Methods.WithdrawBalance, Params: params, }, nil) if err != nil { @@ -345,7 +343,7 @@ var actorRepayDebtCmd = &cli.Command{ To: maddr, From: fromId, Value: amount, - Method: builtin2.MethodsMiner.RepayDebt, + Method: miner.Methods.RepayDebt, Params: nil, }, nil) if err != nil { @@ -572,7 +570,7 @@ var actorControlSet = &cli.Command{ smsg, err := api.MpoolPushMessage(ctx, &types.Message{ From: mi.Owner, To: maddr, - Method: builtin2.MethodsMiner.ChangeWorkerAddress, + Method: miner.Methods.ChangeWorkerAddress, Value: big.Zero(), Params: sp, @@ -650,7 +648,7 @@ var actorSetOwnerCmd = &cli.Command{ smsg, err := api.MpoolPushMessage(ctx, &types.Message{ From: mi.Owner, To: maddr, - Method: builtin2.MethodsMiner.ChangeOwnerAddress, + Method: miner.Methods.ChangeOwnerAddress, Value: big.Zero(), Params: sp, }, nil) @@ -675,7 +673,7 @@ var actorSetOwnerCmd = &cli.Command{ smsg, err = api.MpoolPushMessage(ctx, &types.Message{ From: newAddr, To: maddr, - Method: builtin2.MethodsMiner.ChangeOwnerAddress, + Method: miner.Methods.ChangeOwnerAddress, Value: big.Zero(), Params: sp, }, nil) diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index b39ede752..19de95dbf 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -31,7 +31,6 @@ import ( "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/extern/sector-storage/stores" - builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" market2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/market" miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" power2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/power" @@ -39,6 +38,7 @@ import ( lapi "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/actors/builtin/power" "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/gen/slashfilter" @@ -47,7 +47,7 @@ import ( sealing "github.com/filecoin-project/lotus/extern/storage-sealing" "github.com/filecoin-project/lotus/genesis" "github.com/filecoin-project/lotus/journal" - "github.com/filecoin-project/lotus/miner" + storageminer "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node/modules" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/repo" @@ -470,7 +470,7 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api lapi.FullNode, return fmt.Errorf("failed to open filesystem journal: %w", err) } - m := miner.NewMiner(api, epp, a, slashfilter.New(mds), j) + m := storageminer.NewMiner(api, epp, a, slashfilter.New(mds), j) { if err := m.Start(ctx); err != nil { return xerrors.Errorf("failed to start up genesis miner: %w", err) @@ -578,7 +578,7 @@ func configureStorageMiner(ctx context.Context, api lapi.FullNode, addr address. msg := &types.Message{ To: addr, From: mi.Worker, - Method: builtin2.MethodsMiner.ChangePeerID, + Method: miner.Methods.ChangePeerID, Params: enc, Value: types.NewInt(0), GasPremium: gasPrice, @@ -661,7 +661,7 @@ func createStorageMiner(ctx context.Context, api lapi.FullNode, peerid peer.ID, From: sender, Value: big.Zero(), - Method: builtin2.MethodsPower.CreateMiner, + Method: power.Methods.CreateMiner, Params: params, GasLimit: 0, diff --git a/conformance/chaos/actor.go b/conformance/chaos/actor.go index 3c280e27e..cdda1db83 100644 --- a/conformance/chaos/actor.go +++ b/conformance/chaos/actor.go @@ -6,6 +6,7 @@ import ( "github.com/filecoin-project/go-state-types/cbor" "github.com/filecoin-project/go-state-types/exitcode" "github.com/filecoin-project/go-state-types/rt" + "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/ipfs/go-cid" builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" @@ -56,7 +57,7 @@ const ( const ( _ = 0 // skip zero iota value; first usage of iota gets 1. - MethodCallerValidation = builtin2.MethodConstructor + iota + MethodCallerValidation = builtin.MethodConstructor + iota MethodCreateActor MethodResolveAddress // MethodDeleteActor is the identifier for the method that deletes this actor. @@ -77,15 +78,15 @@ const ( // Exports defines the methods this actor exposes publicly. func (a Actor) Exports() []interface{} { return []interface{}{ - builtin2.MethodConstructor: a.Constructor, - MethodCallerValidation: a.CallerValidation, - MethodCreateActor: a.CreateActor, - MethodResolveAddress: a.ResolveAddress, - MethodDeleteActor: a.DeleteActor, - MethodSend: a.Send, - MethodMutateState: a.MutateState, - MethodAbortWith: a.AbortWith, - MethodInspectRuntime: a.InspectRuntime, + builtin.MethodConstructor: a.Constructor, + MethodCallerValidation: a.CallerValidation, + MethodCreateActor: a.CreateActor, + MethodResolveAddress: a.ResolveAddress, + MethodDeleteActor: a.DeleteActor, + MethodSend: a.Send, + MethodMutateState: a.MutateState, + MethodAbortWith: a.AbortWith, + MethodInspectRuntime: a.InspectRuntime, } } diff --git a/extern/storage-sealing/states_sealing.go b/extern/storage-sealing/states_sealing.go index 1e67a0a03..7b34751c0 100644 --- a/extern/storage-sealing/states_sealing.go +++ b/extern/storage-sealing/states_sealing.go @@ -16,8 +16,6 @@ import ( "github.com/filecoin-project/go-state-types/exitcode" "github.com/filecoin-project/go-statemachine" "github.com/filecoin-project/specs-storage/storage" - - builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" ) var DealSectorPriority = 1024 @@ -221,7 +219,7 @@ func (m *Sealing) handlePreCommitting(ctx statemachine.Context, sector SectorInf deposit := big.Max(depositMinimum, collateral) log.Infof("submitting precommit for sector %d (deposit: %s): ", sector.SectorNumber, deposit) - mcid, err := m.api.SendMsg(ctx.Context(), waddr, m.maddr, builtin2.MethodsMiner.PreCommitSector, deposit, m.feeCfg.MaxPreCommitGasFee, enc.Bytes()) + mcid, err := m.api.SendMsg(ctx.Context(), waddr, m.maddr, miner.Methods.PreCommitSector, deposit, m.feeCfg.MaxPreCommitGasFee, enc.Bytes()) if err != nil { if params.ReplaceCapacity { m.remarkForUpgrade(params.ReplaceSectorNumber) @@ -402,7 +400,7 @@ func (m *Sealing) handleSubmitCommit(ctx statemachine.Context, sector SectorInfo } // TODO: check seed / ticket / deals are up to date - mcid, err := m.api.SendMsg(ctx.Context(), waddr, m.maddr, builtin2.MethodsMiner.ProveCommitSector, collateral, m.feeCfg.MaxCommitGasFee, enc.Bytes()) + mcid, err := m.api.SendMsg(ctx.Context(), waddr, m.maddr, miner.Methods.ProveCommitSector, collateral, m.feeCfg.MaxCommitGasFee, enc.Bytes()) if err != nil { return ctx.Send(SectorCommitFailed{xerrors.Errorf("pushing message to mpool: %w", err)}) } diff --git a/markets/storageadapter/provider.go b/markets/storageadapter/provider.go index 9c088e4de..ce7c8e917 100644 --- a/markets/storageadapter/provider.go +++ b/markets/storageadapter/provider.go @@ -12,7 +12,6 @@ import ( logging "github.com/ipfs/go-log/v2" "golang.org/x/xerrors" - builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" market2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/market" "github.com/filecoin-project/go-address" @@ -92,7 +91,7 @@ func (n *ProviderNodeAdapter) PublishDeals(ctx context.Context, deal storagemark To: market.Address, From: mi.Worker, Value: types.NewInt(0), - Method: builtin2.MethodsMarket.PublishStorageDeals, + Method: market.Methods.PublishStorageDeals, Params: params, }, n.publishSpec) if err != nil { @@ -192,7 +191,7 @@ func (n *ProviderNodeAdapter) AddFunds(ctx context.Context, addr address.Address To: market.Address, From: addr, Value: amount, - Method: builtin2.MethodsMarket.AddBalance, + Method: market.Methods.AddBalance, }, n.addBalanceSpec) if err != nil { return cid.Undef, err @@ -314,7 +313,7 @@ func (n *ProviderNodeAdapter) OnDealSectorCommitted(ctx context.Context, provide } switch msg.Method { - case builtin2.MethodsMiner.PreCommitSector: + case miner.Methods.PreCommitSector: var params miner.SectorPreCommitInfo if err := params.UnmarshalCBOR(bytes.NewReader(msg.Params)); err != nil { return true, false, xerrors.Errorf("unmarshal pre commit: %w", err) @@ -329,7 +328,7 @@ func (n *ProviderNodeAdapter) OnDealSectorCommitted(ctx context.Context, provide } return true, false, nil - case builtin2.MethodsMiner.ProveCommitSector: + case miner.Methods.ProveCommitSector: var params miner.ProveCommitSectorParams if err := params.UnmarshalCBOR(bytes.NewReader(msg.Params)); err != nil { return true, false, xerrors.Errorf("failed to unmarshal prove commit sector params: %w", err) diff --git a/node/impl/full/gas.go b/node/impl/full/gas.go index 4e4fc99d2..e0cbd2192 100644 --- a/node/impl/full/gas.go +++ b/node/impl/full/gas.go @@ -7,6 +7,7 @@ import ( "sort" "github.com/filecoin-project/lotus/chain/actors/builtin" + "github.com/filecoin-project/lotus/chain/actors/builtin/paych" "go.uber.org/fx" "golang.org/x/xerrors" @@ -16,8 +17,6 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/exitcode" - builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" - "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/messagepool" @@ -259,7 +258,7 @@ func gasEstimateGasLimit( if !builtin.IsPaymentChannelActor(act.Code) { return res.MsgRct.GasUsed, nil } - if msgIn.Method != builtin2.MethodsPaych.Collect { + if msgIn.Method != paych.Methods.Collect { return res.MsgRct.GasUsed, nil } diff --git a/node/impl/full/multisig.go b/node/impl/full/multisig.go index 3b13839dd..9c5f683c4 100644 --- a/node/impl/full/multisig.go +++ b/node/impl/full/multisig.go @@ -12,7 +12,6 @@ import ( "github.com/filecoin-project/lotus/chain/actors/builtin/multisig" "github.com/filecoin-project/lotus/chain/types" - builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" multisig2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/multisig" "github.com/ipfs/go-cid" @@ -85,7 +84,7 @@ func (a *MsigAPI) MsigAddPropose(ctx context.Context, msig address.Address, src return cid.Undef, actErr } - return a.MsigPropose(ctx, msig, msig, big.Zero(), src, uint64(builtin2.MethodsMultisig.AddSigner), enc) + return a.MsigPropose(ctx, msig, msig, big.Zero(), src, uint64(multisig.Methods.AddSigner), enc) } func (a *MsigAPI) MsigAddApprove(ctx context.Context, msig address.Address, src address.Address, txID uint64, proposer address.Address, newAdd address.Address, inc bool) (cid.Cid, error) { @@ -94,7 +93,7 @@ func (a *MsigAPI) MsigAddApprove(ctx context.Context, msig address.Address, src return cid.Undef, actErr } - return a.MsigApproveTxnHash(ctx, msig, txID, proposer, msig, big.Zero(), src, uint64(builtin2.MethodsMultisig.AddSigner), enc) + return a.MsigApproveTxnHash(ctx, msig, txID, proposer, msig, big.Zero(), src, uint64(multisig.Methods.AddSigner), enc) } func (a *MsigAPI) MsigAddCancel(ctx context.Context, msig address.Address, src address.Address, txID uint64, newAdd address.Address, inc bool) (cid.Cid, error) { @@ -103,7 +102,7 @@ func (a *MsigAPI) MsigAddCancel(ctx context.Context, msig address.Address, src a return cid.Undef, actErr } - return a.MsigCancel(ctx, msig, txID, msig, big.Zero(), src, uint64(builtin2.MethodsMultisig.AddSigner), enc) + return a.MsigCancel(ctx, msig, txID, msig, big.Zero(), src, uint64(multisig.Methods.AddSigner), enc) } func (a *MsigAPI) MsigSwapPropose(ctx context.Context, msig address.Address, src address.Address, oldAdd address.Address, newAdd address.Address) (cid.Cid, error) { @@ -112,7 +111,7 @@ func (a *MsigAPI) MsigSwapPropose(ctx context.Context, msig address.Address, src return cid.Undef, actErr } - return a.MsigPropose(ctx, msig, msig, big.Zero(), src, uint64(builtin2.MethodsMultisig.SwapSigner), enc) + return a.MsigPropose(ctx, msig, msig, big.Zero(), src, uint64(multisig.Methods.SwapSigner), enc) } func (a *MsigAPI) MsigSwapApprove(ctx context.Context, msig address.Address, src address.Address, txID uint64, proposer address.Address, oldAdd address.Address, newAdd address.Address) (cid.Cid, error) { @@ -121,7 +120,7 @@ func (a *MsigAPI) MsigSwapApprove(ctx context.Context, msig address.Address, src return cid.Undef, actErr } - return a.MsigApproveTxnHash(ctx, msig, txID, proposer, msig, big.Zero(), src, uint64(builtin2.MethodsMultisig.SwapSigner), enc) + return a.MsigApproveTxnHash(ctx, msig, txID, proposer, msig, big.Zero(), src, uint64(multisig.Methods.SwapSigner), enc) } func (a *MsigAPI) MsigSwapCancel(ctx context.Context, msig address.Address, src address.Address, txID uint64, oldAdd address.Address, newAdd address.Address) (cid.Cid, error) { @@ -130,7 +129,7 @@ func (a *MsigAPI) MsigSwapCancel(ctx context.Context, msig address.Address, src return cid.Undef, actErr } - return a.MsigCancel(ctx, msig, txID, msig, big.Zero(), src, uint64(builtin2.MethodsMultisig.SwapSigner), enc) + return a.MsigCancel(ctx, msig, txID, msig, big.Zero(), src, uint64(multisig.Methods.SwapSigner), enc) } func (a *MsigAPI) MsigApprove(ctx context.Context, msig address.Address, txID uint64, src address.Address) (cid.Cid, error) { @@ -151,7 +150,7 @@ func (a *MsigAPI) MsigRemoveSigner(ctx context.Context, msig address.Address, pr return cid.Undef, actErr } - return a.MsigPropose(ctx, msig, msig, types.NewInt(0), proposer, uint64(builtin2.MethodsMultisig.RemoveSigner), enc) + return a.MsigPropose(ctx, msig, msig, types.NewInt(0), proposer, uint64(multisig.Methods.RemoveSigner), enc) } func (a *MsigAPI) msigApproveOrCancelSimple(ctx context.Context, operation api.MsigProposeResponse, msig address.Address, txID uint64, src address.Address) (cid.Cid, error) { diff --git a/node/test/builder.go b/node/test/builder.go index 785e07bbe..ea9a82220 100644 --- a/node/test/builder.go +++ b/node/test/builder.go @@ -24,6 +24,7 @@ import ( "github.com/filecoin-project/lotus/api/test" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/gen" genesis2 "github.com/filecoin-project/lotus/chain/gen/genesis" @@ -40,7 +41,6 @@ import ( testing2 "github.com/filecoin-project/lotus/node/modules/testing" "github.com/filecoin-project/lotus/node/repo" "github.com/filecoin-project/lotus/storage/mockstorage" - builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" "github.com/ipfs/go-datastore" "github.com/libp2p/go-libp2p-core/crypto" @@ -93,7 +93,7 @@ func CreateTestStorageNode(ctx context.Context, t *testing.T, waddr address.Addr msg := &types.Message{ To: act, From: waddr, - Method: builtin2.MethodsMiner.ChangePeerID, + Method: miner.Methods.ChangePeerID, Params: enc, Value: types.NewInt(0), } diff --git a/paychmgr/settler/settler.go b/paychmgr/settler/settler.go index 8074e8a87..d2a0900b5 100644 --- a/paychmgr/settler/settler.go +++ b/paychmgr/settler/settler.go @@ -14,8 +14,6 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" - builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" - "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors/builtin/paych" @@ -107,7 +105,7 @@ func (pcs *paymentChannelSettler) revertHandler(ctx context.Context, ts *types.T func (pcs *paymentChannelSettler) matcher(msg *types.Message) (matchOnce bool, matched bool, err error) { // Check if this is a settle payment channel message - if msg.Method != builtin2.MethodsPaych.Settle { + if msg.Method != paych.Methods.Settle { return false, false, nil } // Check if this payment channel is of concern to this node (i.e. tracked in payment channel store), diff --git a/storage/adapter_storage_miner.go b/storage/adapter_storage_miner.go index f645c301a..0e4b53752 100644 --- a/storage/adapter_storage_miner.go +++ b/storage/adapter_storage_miner.go @@ -15,7 +15,6 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" - builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" market2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/market" "github.com/filecoin-project/lotus/api" @@ -150,7 +149,7 @@ func (s SealingAPIAdapter) StateComputeDataCommitment(ctx context.Context, maddr To: market.Address, From: maddr, Value: types.NewInt(0), - Method: builtin2.MethodsMarket.ComputeDataCommitment, + Method: market.Methods.ComputeDataCommitment, Params: ccparams, } r, err := s.delegate.StateCall(ctx, ccmt, tsk) diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index c11e5dca0..5dbe868d6 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -17,7 +17,6 @@ import ( "go.opencensus.io/trace" "golang.org/x/xerrors" - builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" proof2 "github.com/filecoin-project/specs-actors/v2/actors/runtime/proof" "github.com/filecoin-project/lotus/api" @@ -290,7 +289,7 @@ func (s *WindowPoStScheduler) checkNextRecoveries(ctx context.Context, dlIdx uin msg := &types.Message{ To: s.actor, From: s.worker, - Method: builtin2.MethodsMiner.DeclareFaultsRecovered, + Method: miner.Methods.DeclareFaultsRecovered, Params: enc, Value: types.NewInt(0), } @@ -374,7 +373,7 @@ func (s *WindowPoStScheduler) checkNextFaults(ctx context.Context, dlIdx uint64, msg := &types.Message{ To: s.actor, From: s.worker, - Method: builtin2.MethodsMiner.DeclareFaults, + Method: miner.Methods.DeclareFaults, Params: enc, Value: types.NewInt(0), // TODO: Is there a fee? } @@ -714,7 +713,7 @@ func (s *WindowPoStScheduler) submitPost(ctx context.Context, proof *miner.Submi msg := &types.Message{ To: s.actor, From: s.worker, - Method: builtin2.MethodsMiner.SubmitWindowedPoSt, + Method: miner.Methods.SubmitWindowedPoSt, Params: enc, Value: types.NewInt(0), } diff --git a/storage/wdpost_run_test.go b/storage/wdpost_run_test.go index 75588d7f8..ce3d3ad95 100644 --- a/storage/wdpost_run_test.go +++ b/storage/wdpost_run_test.go @@ -188,7 +188,7 @@ func TestWDPostDoPost(t *testing.T) { // Read the window PoST messages for i := 0; i < expectedMsgCount; i++ { msg := <-mockStgMinerAPI.pushedMessages - require.Equal(t, builtin2.MethodsMiner.SubmitWindowedPoSt, msg.Method) + require.Equal(t, miner.Methods.SubmitWindowedPoSt, msg.Method) var params miner.SubmitWindowedPoStParams err := params.UnmarshalCBOR(bytes.NewReader(msg.Params)) require.NoError(t, err) From 710ac9d54421d7c8111673f9003a24fa1507b906 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 8 Oct 2020 14:23:09 -0700 Subject: [PATCH 831/885] use the correct network version in genesis 1. Version 0 upgrades happen _after_ genesis. 2. Get rid of UseNewestNetwork. It's not useful and quite dangerous. --- build/params_shared_funcs.go | 8 -------- chain/gen/genesis/util.go | 27 +++++++++++++++++++++------ 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/build/params_shared_funcs.go b/build/params_shared_funcs.go index c48b3c507..77fd9256d 100644 --- a/build/params_shared_funcs.go +++ b/build/params_shared_funcs.go @@ -16,14 +16,6 @@ func DhtProtocolName(netName dtypes.NetworkName) protocol.ID { return protocol.ID("/fil/kad/" + string(netName)) } -func UseNewestNetwork() bool { - // TODO: Put these in a container we can iterate over - if UpgradeBreezeHeight <= 0 && UpgradeSmokeHeight <= 0 && UpgradeActorsV2Height <= 0 { - return true - } - return false -} - func SetAddressNetwork(n address.Network) { address.CurrentNetwork = n } diff --git a/chain/gen/genesis/util.go b/chain/gen/genesis/util.go index bcafb007e..54cc30cc1 100644 --- a/chain/gen/genesis/util.go +++ b/chain/gen/genesis/util.go @@ -50,12 +50,27 @@ func doExecValue(ctx context.Context, vm *vm.VM, to, from address.Address, value return ret.Return, nil } -var GenesisNetworkVersion = func() network.Version { // TODO: Get from build/ - if build.UseNewestNetwork() { // TODO: Get from build/ - return build.NewestNetworkVersion // TODO: Get from build/ - } // TODO: Get from build/ - return network.Version1 // TODO: Get from build/ -}() // TODO: Get from build/ +// TODO: Get from build +// TODO: make a list/schedule of these. +var GenesisNetworkVersion = func() network.Version { + // returns the version _before_ the first upgrade. + if build.UpgradeBreezeHeight >= 0 { + return network.Version0 + } + if build.UpgradeSmokeHeight >= 0 { + return network.Version1 + } + if build.UpgradeIgnitionHeight >= 0 { + return network.Version2 + } + if build.UpgradeActorsV2Height >= 0 { + return network.Version3 + } + if build.UpgradeLiftoffHeight >= 0 { + return network.Version3 + } + return build.ActorUpgradeNetworkVersion - 1 // genesis requires actors v0. +}() func genesisNetworkVersion(context.Context, abi.ChainEpoch) network.Version { // TODO: Get from build/ return GenesisNetworkVersion // TODO: Get from build/ From 30bda0b4c4b5a38a77eecb080f72b8187f8c7036 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 8 Oct 2020 14:57:03 -0700 Subject: [PATCH 832/885] set network name everywhere --- chain/gen/gen.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/chain/gen/gen.go b/chain/gen/gen.go index f2c3832c1..d56f285a0 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -8,12 +8,11 @@ import ( "sync/atomic" "time" - proof2 "github.com/filecoin-project/specs-actors/v2/actors/runtime/proof" - "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" + "github.com/google/uuid" block "github.com/ipfs/go-block-format" "github.com/ipfs/go-blockservice" "github.com/ipfs/go-cid" @@ -25,6 +24,8 @@ import ( "go.opencensus.io/trace" "golang.org/x/xerrors" + proof2 "github.com/filecoin-project/specs-actors/v2/actors/runtime/proof" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors/policy" @@ -226,7 +227,7 @@ func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) { }, VerifregRootKey: DefaultVerifregRootkeyActor, RemainderAccount: DefaultRemainderAccountActor, - NetworkName: "", + NetworkName: uuid.New().String(), Timestamp: uint64(build.Clock.Now().Add(-500 * time.Duration(build.BlockDelaySecs) * time.Second).Unix()), } From 0c2ec04c094f73235e4acf38376431536ad0fb3a Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 8 Oct 2020 17:36:26 -0700 Subject: [PATCH 833/885] use upgrade schedule from node when getting the network name It doesn't matter now, but may in the future. --- node/modules/chain.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/node/modules/chain.go b/node/modules/chain.go index ce3e9f749..d1414b307 100644 --- a/node/modules/chain.go +++ b/node/modules/chain.go @@ -168,14 +168,19 @@ func SetGenesis(cs *store.ChainStore, g Genesis) (dtypes.AfterGenesisSet, error) return dtypes.AfterGenesisSet{}, cs.SetGenesis(genesis) } -func NetworkName(mctx helpers.MetricsCtx, lc fx.Lifecycle, cs *store.ChainStore, _ dtypes.AfterGenesisSet) (dtypes.NetworkName, error) { +func NetworkName(mctx helpers.MetricsCtx, lc fx.Lifecycle, cs *store.ChainStore, us stmgr.UpgradeSchedule, _ dtypes.AfterGenesisSet) (dtypes.NetworkName, error) { if !build.Devnet { return "testnetnet", nil } ctx := helpers.LifecycleCtx(mctx, lc) - netName, err := stmgr.GetNetworkName(ctx, stmgr.NewStateManager(cs), cs.GetHeaviestTipSet().ParentState()) + sm, err := stmgr.NewStateManagerWithUpgradeSchedule(cs, us) + if err != nil { + return "", err + } + + netName, err := stmgr.GetNetworkName(ctx, sm, cs.GetHeaviestTipSet().ParentState()) return netName, err } From 9d15a08d27a04fe987e9e4e530baa02dd8ab7024 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 13 Oct 2020 22:10:37 -0700 Subject: [PATCH 834/885] correctly mark new actors as "important" --- chain/types/execresult.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/chain/types/execresult.go b/chain/types/execresult.go index 6fc93fac6..917b84a92 100644 --- a/chain/types/execresult.go +++ b/chain/types/execresult.go @@ -3,6 +3,7 @@ package types import ( "encoding/json" "fmt" + "regexp" "runtime" "strings" "time" @@ -68,11 +69,10 @@ func (l Loc) String() string { return fmt.Sprintf("%s@%s:%d", fnpkg, file[len(file)-1], l.Line) } +var importantRegex = regexp.MustCompile(`github.com/filecoin-project/specs-actors/(v\d+/)?actors/builtin`) + func (l Loc) Important() bool { - if strings.HasPrefix(l.Function, "github.com/filecoin-project/specs-actors/actors/builtin") { - return true - } - return false + return importantRegex.MatchString(l.Function) } func (gt *GasTrace) MarshalJSON() ([]byte, error) { From 5e23933757b1c55281869ef9a0d73fe77be33801 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 14 Oct 2020 17:29:25 -0700 Subject: [PATCH 835/885] fix import of v0 for test We're still on actors v0 in this test. --- chain/stmgr/forks_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chain/stmgr/forks_test.go b/chain/stmgr/forks_test.go index 89e051088..a2b7a179f 100644 --- a/chain/stmgr/forks_test.go +++ b/chain/stmgr/forks_test.go @@ -17,7 +17,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/cbor" - builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" + builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" init2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/init" rt2 "github.com/filecoin-project/specs-actors/v2/actors/runtime" @@ -46,7 +46,7 @@ type testActor struct { } // must use existing actor that an account is allowed to exec. -func (testActor) Code() cid.Cid { return builtin2.PaymentChannelActorCodeID } +func (testActor) Code() cid.Cid { return builtin0.PaymentChannelActorCodeID } func (testActor) State() cbor.Er { return new(testActorState) } type testActorState struct { From 5c9c08ef8549cb568db5d72040c5cf2fda21972f Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 21 Oct 2020 22:16:01 +0200 Subject: [PATCH 836/885] heaviestTipSet reurning nil is a ok Signed-off-by: Jakub Sztandera --- chain/sync_manager.go | 1 - 1 file changed, 1 deletion(-) diff --git a/chain/sync_manager.go b/chain/sync_manager.go index 17cec78d7..c7fdea726 100644 --- a/chain/sync_manager.go +++ b/chain/sync_manager.go @@ -257,7 +257,6 @@ func (stb *syncTargetBucket) add(ts *types.TipSet) { func (stb *syncTargetBucket) heaviestTipSet() *types.TipSet { if stb == nil { - log.Warn("sync target bucket was nil when heaviestTipSet got called on it") return nil } From 7bdc7aa5276fc2539f042f79daa6da355a970106 Mon Sep 17 00:00:00 2001 From: Ingar Shu Date: Tue, 20 Oct 2020 17:34:46 -0700 Subject: [PATCH 837/885] Add a market WithdrawBalance CLI --- cli/wallet.go | 93 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/cli/wallet.go b/cli/wallet.go index 3d6abc357..f504e252f 100644 --- a/cli/wallet.go +++ b/cli/wallet.go @@ -13,9 +13,13 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" + "github.com/filecoin-project/specs-actors/actors/builtin/market" + "github.com/filecoin-project/specs-actors/v2/actors/builtin" + "github.com/filecoin-project/lotus/chain/actors" types "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/tablewriter" ) @@ -34,6 +38,7 @@ var walletCmd = &cli.Command{ walletSign, walletVerify, walletDelete, + walletWithdrawMarketFunds, }, } @@ -471,3 +476,91 @@ var walletDelete = &cli.Command{ return api.WalletDelete(ctx, addr) }, } + +var walletWithdrawMarketFunds = &cli.Command{ + Name: "withdraw-market-funds", + Usage: "Withdraw funds from the Storage Market Actor", + ArgsUsage: "[amount (FIL) optional, otherwise will withdraw max available]", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "address", + Usage: "Specify address to withdraw funds from, otherwise it will use the default wallet address", + Aliases: []string{"a"}, + }, + }, + Action: func(cctx *cli.Context) error { + api, closer, err := GetFullNodeAPI(cctx) + if err != nil { + return xerrors.Errorf("getting node API: %w", err) + } + defer closer() + ctx := ReqContext(cctx) + + defAddr, err := api.WalletDefaultAddress(ctx) + if err != nil { + return xerrors.Errorf("getting default wallet address: %w", err) + } + + addr := defAddr + if cctx.String("address") != "" { + addr, err = address.NewFromString(cctx.String("address")) + if err != nil { + return xerrors.Errorf("parsing address: %w", err) + } + } + + ts, err := api.ChainHead(ctx) + if err != nil { + return xerrors.Errorf("getting chain head: %w", err) + } + + bal, err := api.StateMarketBalance(ctx, addr, ts.Key()) + if err != nil { + return xerrors.Errorf("getting market balance for address %s: %w", addr.String(), err) + } + + avail := big.Subtract(bal.Escrow, bal.Locked) + amt := avail + + if cctx.Args().Present() { + f, err := types.ParseFIL(cctx.Args().First()) + if err != nil { + return xerrors.Errorf("parsing 'amount' argument: %w", err) + } + + amt = abi.TokenAmount(f) + } + + if amt.GreaterThan(avail) { + return xerrors.Errorf("can't withdraw more funds than available; requested: %s; available: %s", amt, avail) + } + + if avail.IsZero() { + return xerrors.Errorf("zero unlocked funds available to withdraw") + } + + params, err := actors.SerializeParams(&market.WithdrawBalanceParams{ + ProviderOrClientAddress: addr, + Amount: amt, + }) + if err != nil { + return xerrors.Errorf("serializing params: %w", err) + } + + fmt.Printf("Submitting WithdrawBalance message for amount %s for address %s\n", types.FIL(amt), addr.String()) + smsg, err := api.MpoolPushMessage(ctx, &types.Message{ + To: builtin.StorageMarketActorAddr, + From: defAddr, + Value: types.NewInt(0), + Method: builtin.MethodsMarket.WithdrawBalance, + Params: params, + }, nil) + if err != nil { + return xerrors.Errorf("submitting WithdrawBalance message: %w", err) + } + + fmt.Printf("WithdrawBalance message cid: %s\n", smsg.Cid()) + + return nil + }, +} From 7f98be2920a5a20c2d8ed69d2e14e1212d89103d Mon Sep 17 00:00:00 2001 From: jennijuju Date: Wed, 21 Oct 2020 17:41:45 -0400 Subject: [PATCH 838/885] Delete wallet from local wallet cache --- chain/wallet/wallet.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/chain/wallet/wallet.go b/chain/wallet/wallet.go index 925547ea8..1f3329498 100644 --- a/chain/wallet/wallet.go +++ b/chain/wallet/wallet.go @@ -272,6 +272,7 @@ func (w *LocalWallet) WalletHas(ctx context.Context, addr address.Address) (bool func (w *LocalWallet) WalletDelete(ctx context.Context, addr address.Address) error { k, err := w.findKey(addr) + if err != nil { return xerrors.Errorf("failed to delete key %s : %w", addr, err) } @@ -279,6 +280,9 @@ func (w *LocalWallet) WalletDelete(ctx context.Context, addr address.Address) er return nil // already not there } + w.lk.Lock() + defer w.lk.Unlock() + if err := w.keystore.Put(KTrashPrefix+k.Address.String(), k.KeyInfo); err != nil { return xerrors.Errorf("failed to mark key %s as trashed: %w", addr, err) } @@ -295,6 +299,8 @@ func (w *LocalWallet) WalletDelete(ctx context.Context, addr address.Address) er // TODO: Does this always error in the not-found case? Just ignoring an error return for now. _ = w.keystore.Delete(KNamePrefix + tAddr) + delete(w.keys, addr) + return nil } From 035a762358c1bf860d682238fe6c51e6b6792cc0 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Wed, 21 Oct 2020 16:49:59 -0700 Subject: [PATCH 839/885] update drand to v1.2.1 * stop autowatching * set user-agent --- chain/beacon/drand/drand.go | 7 ++++++- go.mod | 2 +- go.sum | 4 ++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/chain/beacon/drand/drand.go b/chain/beacon/drand/drand.go index 6e8e83a20..4abc12d29 100644 --- a/chain/beacon/drand/drand.go +++ b/chain/beacon/drand/drand.go @@ -65,6 +65,11 @@ type DrandBeacon struct { localCache map[uint64]types.BeaconEntry } +// DrandHTTPClient interface overrides the user agent used by drand +type DrandHTTPClient interface { + SetUserAgent(string) +} + func NewDrandBeacon(genesisTs, interval uint64, ps *pubsub.PubSub, config dtypes.DrandConfig) (*DrandBeacon, error) { if genesisTs == 0 { panic("what are you doing this cant be zero") @@ -84,6 +89,7 @@ func NewDrandBeacon(genesisTs, interval uint64, ps *pubsub.PubSub, config dtypes if err != nil { return nil, xerrors.Errorf("could not create http drand client: %w", err) } + hc.(DrandHTTPClient).SetUserAgent("drand-client-lotus/" + build.BuildVersion) clients = append(clients, hc) } @@ -92,7 +98,6 @@ func NewDrandBeacon(genesisTs, interval uint64, ps *pubsub.PubSub, config dtypes dclient.WithChainInfo(drandChain), dclient.WithCacheSize(1024), dclient.WithLogger(dlogger), - dclient.WithAutoWatch(), } if ps != nil { diff --git a/go.mod b/go.mod index e36f821d3..8132acf24 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e github.com/dgraph-io/badger/v2 v2.2007.2 github.com/docker/go-units v0.4.0 - github.com/drand/drand v1.2.0 + github.com/drand/drand v1.2.1 github.com/drand/kyber v1.1.4 github.com/dustin/go-humanize v1.0.0 github.com/elastic/go-sysinfo v1.3.0 diff --git a/go.sum b/go.sum index 6737eff06..b59db3a75 100644 --- a/go.sum +++ b/go.sum @@ -192,8 +192,12 @@ github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/drand/bls12-381 v0.3.2 h1:RImU8Wckmx8XQx1tp1q04OV73J9Tj6mmpQLYDP7V1XE= github.com/drand/bls12-381 v0.3.2/go.mod h1:dtcLgPtYT38L3NO6mPDYH0nbpc5tjPassDqiniuAt4Y= +github.com/drand/drand v1.1.2-0.20201021232824-ead2bc9ad8b5 h1:+zGmXpQ8IKpfQKtF0tKshBBmKxQnZDAZ/lXk1M20EKY= +github.com/drand/drand v1.1.2-0.20201021232824-ead2bc9ad8b5/go.mod h1:j0P7RGmVaY7E/OuO2yQOcQj7OgeZCuhgu2gdv0JAm+g= github.com/drand/drand v1.2.0 h1:EWeIxKFxUR9A+9xZ2oV09sqlxxWhDhgBbgCaCVwnvok= github.com/drand/drand v1.2.0/go.mod h1:j0P7RGmVaY7E/OuO2yQOcQj7OgeZCuhgu2gdv0JAm+g= +github.com/drand/drand v1.2.1 h1:KB7z+69YbnQ5z22AH/LMi0ObDR8DzYmrkS6vZXTR9jI= +github.com/drand/drand v1.2.1/go.mod h1:j0P7RGmVaY7E/OuO2yQOcQj7OgeZCuhgu2gdv0JAm+g= github.com/drand/kyber v1.0.1-0.20200110225416-8de27ed8c0e2/go.mod h1:UpXoA0Upd1N9l4TvRPHr1qAUBBERj6JQ/mnKI3BPEmw= github.com/drand/kyber v1.0.2/go.mod h1:x6KOpK7avKj0GJ4emhXFP5n7M7W7ChAPmnQh/OL6vRw= github.com/drand/kyber v1.1.4 h1:YvKM03QWGvLrdTnYmxxP5iURAX+Gdb6qRDUOgg8i60Q= From 9dc9bf5eb01908343a056757d355b1290c596814 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Wed, 21 Oct 2020 19:49:38 -0700 Subject: [PATCH 840/885] go mod tidy --- go.sum | 4 ---- 1 file changed, 4 deletions(-) diff --git a/go.sum b/go.sum index b59db3a75..825ef5e63 100644 --- a/go.sum +++ b/go.sum @@ -192,10 +192,6 @@ github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/drand/bls12-381 v0.3.2 h1:RImU8Wckmx8XQx1tp1q04OV73J9Tj6mmpQLYDP7V1XE= github.com/drand/bls12-381 v0.3.2/go.mod h1:dtcLgPtYT38L3NO6mPDYH0nbpc5tjPassDqiniuAt4Y= -github.com/drand/drand v1.1.2-0.20201021232824-ead2bc9ad8b5 h1:+zGmXpQ8IKpfQKtF0tKshBBmKxQnZDAZ/lXk1M20EKY= -github.com/drand/drand v1.1.2-0.20201021232824-ead2bc9ad8b5/go.mod h1:j0P7RGmVaY7E/OuO2yQOcQj7OgeZCuhgu2gdv0JAm+g= -github.com/drand/drand v1.2.0 h1:EWeIxKFxUR9A+9xZ2oV09sqlxxWhDhgBbgCaCVwnvok= -github.com/drand/drand v1.2.0/go.mod h1:j0P7RGmVaY7E/OuO2yQOcQj7OgeZCuhgu2gdv0JAm+g= github.com/drand/drand v1.2.1 h1:KB7z+69YbnQ5z22AH/LMi0ObDR8DzYmrkS6vZXTR9jI= github.com/drand/drand v1.2.1/go.mod h1:j0P7RGmVaY7E/OuO2yQOcQj7OgeZCuhgu2gdv0JAm+g= github.com/drand/kyber v1.0.1-0.20200110225416-8de27ed8c0e2/go.mod h1:UpXoA0Upd1N9l4TvRPHr1qAUBBERj6JQ/mnKI3BPEmw= From 9d9da1a3e8a34a13afa26abd5035ca98d8f71b6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 22 Oct 2020 14:15:11 +0200 Subject: [PATCH 841/885] docsgen --- documentation/en/api-methods.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index 2684d0db5..e151ac696 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -196,7 +196,7 @@ * [WalletSignMessage](#WalletSignMessage) * [WalletValidateAddress](#WalletValidateAddress) * [WalletVerify](#WalletVerify) -## +## ### Closing @@ -977,7 +977,7 @@ Response: ClientGetDealStatus returns status given a code -Perms: +Perms: read Inputs: ```json From 90ce1c018e1d849bacc67d840dd4d15b653aaee5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 22 Oct 2020 14:46:10 +0200 Subject: [PATCH 842/885] docsgen --- documentation/en/api-methods.md | 51 +++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index e151ac696..c82440375 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -70,6 +70,9 @@ * [MinerCreateBlock](#MinerCreateBlock) * [MinerGetBaseInfo](#MinerGetBaseInfo) * [Mpool](#Mpool) + * [MpoolBatchPush](#MpoolBatchPush) + * [MpoolBatchPushMessage](#MpoolBatchPushMessage) + * [MpoolBatchPushUntrusted](#MpoolBatchPushUntrusted) * [MpoolClear](#MpoolClear) * [MpoolGetConfig](#MpoolGetConfig) * [MpoolGetNonce](#MpoolGetNonce) @@ -1697,6 +1700,54 @@ The Mpool methods are for interacting with the message pool. The message pool manages all incoming and outgoing 'messages' going over the network. +### MpoolBatchPush +MpoolBatchPush batch pushes a signed message to mempool. + + +Perms: write + +Inputs: +```json +[ + null +] +``` + +Response: `null` + +### MpoolBatchPushMessage +MpoolBatchPushMessage batch pushes a unsigned message to mempool. + + +Perms: sign + +Inputs: +```json +[ + null, + { + "MaxFee": "0" + } +] +``` + +Response: `null` + +### MpoolBatchPushUntrusted +MpoolBatchPushUntrusted batch pushes a signed message to mempool from untrusted sources. + + +Perms: write + +Inputs: +```json +[ + null +] +``` + +Response: `null` + ### MpoolClear MpoolClear clears pending messages from the mpool From 00d1e6b54900a4c514b509f9ef6e63b73c04fbd3 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Sat, 10 Oct 2020 11:49:15 -0700 Subject: [PATCH 843/885] add measure datastore wrapper around bench chain datastore --- cmd/lotus-bench/import.go | 9 ++++++++- go.mod | 2 +- go.sum | 2 ++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index 3f99d0453..00a4d22ef 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -34,6 +34,7 @@ import ( bdg "github.com/dgraph-io/badger/v2" "github.com/ipfs/go-datastore" badger "github.com/ipfs/go-ds-badger2" + measure "github.com/ipfs/go-ds-measure" pebbleds "github.com/ipfs/go-ds-pebble" "github.com/urfave/cli/v2" @@ -89,6 +90,9 @@ var importBenchCmd = &cli.Command{ &cli.BoolFlag{ Name: "only-import", }, + &cli.BoolFlag{ + Name: "use-pebble", + }, }, Action: func(cctx *cli.Context) error { vm.BatchSealVerifyParallelism = cctx.Int("batch-seal-verify-threads") @@ -126,7 +130,7 @@ var importBenchCmd = &cli.Command{ bdgOpt.Options.DetectConflicts = false var bds datastore.Batching - if false { + if cctx.Bool("use-pebble") { cache := 512 bds, err = pebbleds.NewDatastore(tdir, &pebble.Options{ // Pebble has a single combined cache area and the write @@ -155,6 +159,9 @@ var importBenchCmd = &cli.Command{ } defer bds.Close() //nolint:errcheck + under := bds + bds = measure.New("dsbench", bds) + bs := blockstore.NewBlockstore(bds) cacheOpts := blockstore.DefaultCacheOpts() cacheOpts.HasBloomFilterSize = 0 diff --git a/go.mod b/go.mod index 3fde407bb..2564f87a7 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d github.com/buger/goterm v0.0.0-20200322175922-2f3e71b85129 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e - github.com/cockroachdb/pebble v0.0.0-20200916222308-4e219a90ba5b + github.com/cockroachdb/pebble v0.0.0-20201001221639-879f3bfeef07 github.com/coreos/go-systemd/v22 v22.0.0 github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e github.com/dgraph-io/badger/v2 v2.2007.2 diff --git a/go.sum b/go.sum index 9ef3587f4..f72b5998b 100644 --- a/go.sum +++ b/go.sum @@ -136,6 +136,8 @@ github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOi github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/cockroachdb/pebble v0.0.0-20200916222308-4e219a90ba5b h1:OKALTB609+19AM7wsO0k8yMwAqjEIppcnYvyIhA+ZlQ= github.com/cockroachdb/pebble v0.0.0-20200916222308-4e219a90ba5b/go.mod h1:hU7vhtrqonEphNF+xt8/lHdaBprxmV1h8BOGrd9XwmQ= +github.com/cockroachdb/pebble v0.0.0-20201001221639-879f3bfeef07 h1:Cb2pZUCFXlLA8i7My+wrN51D41GeuhYOKa1dJeZt6NY= +github.com/cockroachdb/pebble v0.0.0-20201001221639-879f3bfeef07/go.mod h1:hU7vhtrqonEphNF+xt8/lHdaBprxmV1h8BOGrd9XwmQ= github.com/cockroachdb/redact v0.0.0-20200622112456-cd282804bbd3 h1:2+dpIJzYMSbLi0587YXpi8tOJT52qCOI/1I0UNThc/I= github.com/cockroachdb/redact v0.0.0-20200622112456-cd282804bbd3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= From 034194435031d04ac0c8a29f6cfa4ac861f1eb7a Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Sat, 10 Oct 2020 21:57:19 +0200 Subject: [PATCH 844/885] Inject prometheus Signed-off-by: Jakub Sztandera --- cmd/lotus-bench/import.go | 5 ++++- go.mod | 2 ++ go.sum | 2 ++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index 00a4d22ef..de3a0ea45 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -26,7 +26,9 @@ import ( "github.com/filecoin-project/lotus/lib/blockstore" _ "github.com/filecoin-project/lotus/lib/sigs/bls" _ "github.com/filecoin-project/lotus/lib/sigs/secp" + metricsprometheus "github.com/ipfs/go-metrics-prometheus" "github.com/ipld/go-car" + "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" @@ -95,6 +97,7 @@ var importBenchCmd = &cli.Command{ }, }, Action: func(cctx *cli.Context) error { + metricsprometheus.Inject() vm.BatchSealVerifyParallelism = cctx.Int("batch-seal-verify-threads") if !cctx.Args().Present() { fmt.Println("must pass car file of chain to benchmark importing") @@ -108,6 +111,7 @@ var importBenchCmd = &cli.Command{ defer cfi.Close() //nolint:errcheck // read only file go func() { + http.Handle("/debug/metrics/prometheus", promhttp.Handler()) http.ListenAndServe("localhost:6060", nil) //nolint:errcheck }() @@ -159,7 +163,6 @@ var importBenchCmd = &cli.Command{ } defer bds.Close() //nolint:errcheck - under := bds bds = measure.New("dsbench", bds) bs := blockstore.NewBlockstore(bds) diff --git a/go.mod b/go.mod index 2564f87a7..b79d18938 100644 --- a/go.mod +++ b/go.mod @@ -81,6 +81,7 @@ require ( github.com/ipfs/go-log v1.0.4 github.com/ipfs/go-log/v2 v2.1.2-0.20200626104915-0016c0b4b3e4 github.com/ipfs/go-merkledag v0.3.2 + github.com/ipfs/go-metrics-prometheus v0.0.2 github.com/ipfs/go-path v0.0.7 github.com/ipfs/go-unixfs v0.2.4 github.com/ipfs/interface-go-ipfs-core v0.2.3 @@ -115,6 +116,7 @@ require ( github.com/multiformats/go-multihash v0.0.14 github.com/opentracing/opentracing-go v1.2.0 github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a + github.com/prometheus/client_golang v1.6.0 github.com/raulk/clock v1.1.0 github.com/stretchr/testify v1.6.1 github.com/supranational/blst v0.1.1 diff --git a/go.sum b/go.sum index f72b5998b..be9682c83 100644 --- a/go.sum +++ b/go.sum @@ -632,6 +632,8 @@ github.com/ipfs/go-merkledag v0.3.2 h1:MRqj40QkrWkvPswXs4EfSslhZ4RVPRbxwX11js0t1 github.com/ipfs/go-merkledag v0.3.2/go.mod h1:fvkZNNZixVW6cKSZ/JfLlON5OlgTXNdRLz0p6QG/I2M= github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= +github.com/ipfs/go-metrics-prometheus v0.0.2 h1:9i2iljLg12S78OhC6UAiXi176xvQGiZaGVF1CUVdE+s= +github.com/ipfs/go-metrics-prometheus v0.0.2/go.mod h1:ELLU99AQQNi+zX6GCGm2lAgnzdSH3u5UVlCdqSXnEks= github.com/ipfs/go-path v0.0.3/go.mod h1:zIRQUez3LuQIU25zFjC2hpBTHimWx7VK5bjZgRLbbdo= github.com/ipfs/go-path v0.0.7 h1:H06hKMquQ0aYtHiHryOMLpQC1qC3QwXwkahcEVD51Ho= github.com/ipfs/go-path v0.0.7/go.mod h1:6KTKmeRnBXgqrTvzFrPV3CamxcgvXX/4z79tfAd2Sno= From 602d2b8ad50b7bd707c123cd4e01a5c989870191 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 13 Oct 2020 21:24:56 -0700 Subject: [PATCH 845/885] record datastore metrics after successful run --- cmd/lotus-bench/import.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index de3a0ea45..681d7ede2 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -320,6 +320,21 @@ var importBenchCmd = &cli.Command{ pprof.StopCPUProfile() + if true { + resp, err := http.Get("http://localhost:6060/debug/metrics/prometheus") + if err != nil { + return err + } + + metricsfi, err := os.Create("import-bench.metrics") + if err != nil { + return err + } + + io.Copy(metricsfi, resp.Body) + metricsfi.Close() + } + return nil }, From 0832d5a7d5e4566785d3d5f3d32113149690ceca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 22 Oct 2020 14:57:55 +0200 Subject: [PATCH 846/885] bench: Fix lint warnings --- cmd/lotus-bench/import.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index 681d7ede2..acbf9ebdc 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -97,7 +97,7 @@ var importBenchCmd = &cli.Command{ }, }, Action: func(cctx *cli.Context) error { - metricsprometheus.Inject() + metricsprometheus.Inject() //nolint:errcheck vm.BatchSealVerifyParallelism = cctx.Int("batch-seal-verify-threads") if !cctx.Args().Present() { fmt.Println("must pass car file of chain to benchmark importing") @@ -331,8 +331,8 @@ var importBenchCmd = &cli.Command{ return err } - io.Copy(metricsfi, resp.Body) - metricsfi.Close() + io.Copy(metricsfi, resp.Body) //nolint:errcheck + metricsfi.Close() //nolint:errcheck } return nil From 6fb3dafa955dee565767c5f2753d1d6dc1d19ed7 Mon Sep 17 00:00:00 2001 From: Travis Person Date: Sun, 11 Oct 2020 19:37:54 +0000 Subject: [PATCH 847/885] lotus-pcr: add blocklist --- cmd/lotus-pcr/main.go | 94 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/cmd/lotus-pcr/main.go b/cmd/lotus-pcr/main.go index 29db4a5fe..a615937f2 100644 --- a/cmd/lotus-pcr/main.go +++ b/cmd/lotus-pcr/main.go @@ -6,12 +6,14 @@ import ( "context" "encoding/csv" "fmt" + "io" "io/ioutil" "net/http" _ "net/http/pprof" "os" "path/filepath" "strconv" + "strings" "time" "github.com/filecoin-project/lotus/chain/actors/builtin" @@ -251,6 +253,15 @@ var recoverMinersCmd = &cli.Command{ } defer closer() + r, err := NewRepo(cctx.String("repo")) + if err != nil { + return err + } + + if err := r.Open(); err != nil { + return err + } + from, err := address.NewFromString(cctx.String("from")) if err != nil { return xerrors.Errorf("parsing source address (provide correct --from flag!): %w", err) @@ -267,6 +278,12 @@ var recoverMinersCmd = &cli.Command{ minerRecoveryCutoff := uint64(cctx.Int("miner-recovery-cutoff")) minerRecoveryBonus := uint64(cctx.Int("miner-recovery-bonus")) + blockmap := make(map[address.Address]struct{}) + + for _, addr := range r.Blocklist() { + blockmap[addr] = struct{}{} + } + rf := &refunder{ api: api, wallet: from, @@ -274,6 +291,7 @@ var recoverMinersCmd = &cli.Command{ minerRecoveryRefundPercent: minerRecoveryRefundPercent, minerRecoveryCutoff: types.FromFil(minerRecoveryCutoff), minerRecoveryBonus: types.FromFil(minerRecoveryBonus), + blockmap: blockmap, } refundTipset, err := api.ChainHead(ctx) @@ -466,6 +484,12 @@ var runCmd = &cli.Command{ return err } + blockmap := make(map[address.Address]struct{}) + + for _, addr := range r.Blocklist() { + blockmap[addr] = struct{}{} + } + rf := &refunder{ api: api, wallet: from, @@ -480,6 +504,7 @@ var runCmd = &cli.Command{ publishStorageDealsEnabled: publishStorageDealsEnabled, preFeeCapMax: types.BigInt(preFeeCapMax), proveFeeCapMax: types.BigInt(proveFeeCapMax), + blockmap: blockmap, } var refunds *MinersRefund = NewMinersRefund() @@ -487,6 +512,10 @@ var runCmd = &cli.Command{ nextMinerRecovery := r.MinerRecoveryHeight() + minerRecoveryPeriod for tipset := range tipsetsCh { + for k, _ := range rf.blockmap { + fmt.Printf("%s\n", k) + } + refunds, err = rf.ProcessTipset(ctx, tipset, refunds) if err != nil { return err @@ -634,6 +663,7 @@ type refunder struct { windowedPoStEnabled bool publishStorageDealsEnabled bool threshold big.Int + blockmap map[address.Address]struct{} preFeeCapMax big.Int proveFeeCapMax big.Int @@ -738,6 +768,11 @@ func (r *refunder) EnsureMinerMinimums(ctx context.Context, tipset *types.TipSet } for _, maddr := range miners { + if _, found := r.blockmap[maddr]; found { + log.Debugw("skipping blocked miner", "height", tipset.Height(), "key", tipset.Key(), "miner", maddr) + continue + } + mact, err := r.api.StateGetActor(ctx, maddr, types.EmptyTSK) if err != nil { log.Errorw("failed", "err", err, "height", tipset.Height(), "key", tipset.Key(), "miner", maddr) @@ -897,6 +932,11 @@ func (r *refunder) processTipsetStorageMinerActor(ctx context.Context, tipset *t refundValue := types.NewInt(0) var messageMethod string + if _, found := r.blockmap[m.To]; found { + log.Debugw("skipping blocked miner", "height", tipset.Height(), "key", tipset.Key(), "miner", m.To) + return false, messageMethod, types.NewInt(0), nil + } + switch m.Method { case builtin0.MethodsMiner.SubmitWindowedPoSt: if !r.windowedPoStEnabled { @@ -1177,6 +1217,7 @@ type Repo struct { lastHeight abi.ChainEpoch lastMinerRecoveryHeight abi.ChainEpoch path string + blocklist []address.Address } func NewRepo(path string) (*Repo, error) { @@ -1232,6 +1273,10 @@ func (r *Repo) Open() error { return err } + if err := r.loadBlockList(); err != nil { + return err + } + return nil } @@ -1257,6 +1302,51 @@ func loadChainEpoch(fn string) (abi.ChainEpoch, error) { return abi.ChainEpoch(height), nil } +func (r *Repo) loadBlockList() error { + var err error + fpath := filepath.Join(r.path, "blocklist") + f, err := os.OpenFile(fpath, os.O_RDWR|os.O_CREATE, 0644) + if err != nil { + return err + } + defer func() { + err = f.Close() + }() + + blocklist := []address.Address{} + input := bufio.NewReader(f) + for { + stra, errR := input.ReadString('\n') + stra = strings.TrimSpace(stra) + + if len(stra) == 0 { + if errR == io.EOF { + break + } + continue + } + + addr, err := address.NewFromString(stra) + if err != nil { + return err + } + + blocklist = append(blocklist, addr) + + if errR != nil && errR != io.EOF { + return err + } + + if errR == io.EOF { + break + } + } + + r.blocklist = blocklist + + return nil +} + func (r *Repo) loadHeight() error { var err error r.lastHeight, err = loadChainEpoch(filepath.Join(r.path, "height")) @@ -1269,6 +1359,10 @@ func (r *Repo) loadMinerRecoveryHeight() error { return err } +func (r *Repo) Blocklist() []address.Address { + return r.blocklist +} + func (r *Repo) Height() abi.ChainEpoch { return r.lastHeight } From ee8d8bf5e5ed6b3b4160fb3cc5f671ce4c63cf06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 22 Oct 2020 15:42:46 +0200 Subject: [PATCH 848/885] pcr: Fix lint --- cmd/lotus-pcr/main.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/lotus-pcr/main.go b/cmd/lotus-pcr/main.go index a615937f2..3c61180d9 100644 --- a/cmd/lotus-pcr/main.go +++ b/cmd/lotus-pcr/main.go @@ -507,12 +507,12 @@ var runCmd = &cli.Command{ blockmap: blockmap, } - var refunds *MinersRefund = NewMinersRefund() - var rounds int = 0 + var refunds = NewMinersRefund() + var rounds = 0 nextMinerRecovery := r.MinerRecoveryHeight() + minerRecoveryPeriod for tipset := range tipsetsCh { - for k, _ := range rf.blockmap { + for k := range rf.blockmap { fmt.Printf("%s\n", k) } From ae04f3029bb0e4ff9c32b2cf14de37ecfb22899b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 22 Oct 2020 15:58:22 +0200 Subject: [PATCH 849/885] chain: Add more robust test for ChainExport --- chain/store/store_test.go | 58 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/chain/store/store_test.go b/chain/store/store_test.go index 326899f90..60167ad86 100644 --- a/chain/store/store_test.go +++ b/chain/store/store_test.go @@ -12,6 +12,7 @@ import ( "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/gen" + "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/blockstore" @@ -107,3 +108,60 @@ func TestChainExportImport(t *testing.T) { t.Fatal("imported chain differed from exported chain") } } + +func TestChainExportImportFull(t *testing.T) { + cg, err := gen.NewGenerator() + if err != nil { + t.Fatal(err) + } + + var last *types.TipSet + for i := 0; i < 100; i++ { + ts, err := cg.NextTipSet() + if err != nil { + t.Fatal(err) + } + + last = ts.TipSet.TipSet() + } + + buf := new(bytes.Buffer) + if err := cg.ChainStore().Export(context.TODO(), last, 100, false, buf); err != nil { + t.Fatal(err) + } + + nbs := blockstore.NewTemporary() + cs := store.NewChainStore(nbs, datastore.NewMapDatastore(), nil, nil) + root, err := cs.Import(buf) + if err != nil { + t.Fatal(err) + } + + err = cs.SetHead(last) + if err != nil { + t.Fatal(err) + } + + if !root.Equals(last) { + t.Fatal("imported chain differed from exported chain") + } + + sm := stmgr.NewStateManager(cs) + for i := 0; i < 100; i++ { + ts, err := cs.GetTipsetByHeight(context.TODO(), abi.ChainEpoch(i), nil, false) + if err != nil { + t.Fatal(err) + } + + st, err := sm.ParentState(ts) + if err != nil { + t.Fatal(err) + } + + // touches a bunch of actors + _, err = sm.GetCirculatingSupply(context.TODO(), abi.ChainEpoch(i), st) + if err != nil { + t.Fatal(err) + } + } +} From f9707ba344d8b90a9e7d4e6ca09e7db4d396fd79 Mon Sep 17 00:00:00 2001 From: ZenGround0 Date: Mon, 12 Oct 2020 13:59:44 -0400 Subject: [PATCH 850/885] add borked inspection command hook up new command Fix head call Debug internal error Debug moa Debug 0 Fix 0 Fix 1 Better print Don't ignore error --- cmd/lotus-shed/borked-miners.go | 85 +++++++++++++++++++++++++++++++++ cmd/lotus-shed/main.go | 1 + 2 files changed, 86 insertions(+) create mode 100644 cmd/lotus-shed/borked-miners.go diff --git a/cmd/lotus-shed/borked-miners.go b/cmd/lotus-shed/borked-miners.go new file mode 100644 index 000000000..ecb3aeae9 --- /dev/null +++ b/cmd/lotus-shed/borked-miners.go @@ -0,0 +1,85 @@ +package main + +import ( + "fmt" + + "github.com/filecoin-project/go-state-types/abi" + lcli "github.com/filecoin-project/lotus/cli" + "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" + "github.com/urfave/cli/v2" + "golang.org/x/xerrors" +) + +var borkedMinersCmd = &cli.Command{ + Name: "borked-miners", + Description: "information about 'borked' miner actors with late or frozen deadline crons", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "tipset", + Usage: "specify tipset state to search on (pass comma separated array of cids)", + }, + &cli.BoolFlag{ + Name: "future", + Usage: "print info of miners with last deadline cron in the future (normal for v0 and early v2 actors)", + }, + }, + Action: func(c *cli.Context) error { + api, acloser, err := lcli.GetFullNodeAPI(c) + if err != nil { + return err + } + defer acloser() + ctx := lcli.ReqContext(c) + + ts, err := lcli.LoadTipSet(ctx, c, api) + if err != nil { + return err + } + if ts == nil { + ts, err = api.ChainHead(ctx) + if err != nil { + return err + } + } + + queryEpoch := ts.Height() + + mAddrs, err := api.StateListMiners(ctx, ts.Key()) + if err != nil { + return err + } + + for _, mAddr := range mAddrs { + st, err := api.StateReadState(ctx, mAddr, ts.Key()) + if err != nil { + return err + } + minerState, ok := st.State.(map[string]interface{}) + if !ok { + return xerrors.Errorf("internal error: failed to cast miner state to expected map type") + } + + ppsIface := minerState["ProvingPeriodStart"] + pps := int64(ppsIface.(float64)) + dlIdxIface := minerState["CurrentDeadline"] + dlIdx := uint64(dlIdxIface.(float64)) + latestDeadline := abi.ChainEpoch(pps) + abi.ChainEpoch(int64(dlIdx))*miner.WPoStChallengeWindow + nextDeadline := latestDeadline + miner.WPoStChallengeWindow + + // Need +1 because last epoch of the deadline queryEpoch = x + 59 cron gets run and + // state is left with latestDeadline = x + 60 + if c.Bool("future") && latestDeadline > queryEpoch+1 { + fmt.Printf("%s -- last deadline start in future epoch %d > query epoch %d + 1\n", mAddr, latestDeadline, queryEpoch) + } + + // Equality is an error because last epoch of the deadline queryEpoch = x + 59. Cron + // should get run and bump latestDeadline = x + 60 so nextDeadline = x + 120 + if queryEpoch >= nextDeadline { + fmt.Printf("%s -- next deadline start in non-future epoch %d <= query epoch %d\n", mAddr, nextDeadline, queryEpoch) + } + + } + + return nil + }, +} diff --git a/cmd/lotus-shed/main.go b/cmd/lotus-shed/main.go index 5acaa3218..c8ce3ae04 100644 --- a/cmd/lotus-shed/main.go +++ b/cmd/lotus-shed/main.go @@ -19,6 +19,7 @@ func main() { base32Cmd, base16Cmd, bitFieldCmd, + borkedMinersCmd, keyinfoCmd, jwtCmd, noncefix, From 9a58763a4e59129fed3a55b854fbe9b647be0ae1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 22 Oct 2020 16:07:42 +0200 Subject: [PATCH 851/885] shed: Rename the command to frozen-miners --- cmd/lotus-shed/{borked-miners.go => frozen-miners.go} | 6 +++--- cmd/lotus-shed/main.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) rename cmd/lotus-shed/{borked-miners.go => frozen-miners.go} (93%) diff --git a/cmd/lotus-shed/borked-miners.go b/cmd/lotus-shed/frozen-miners.go similarity index 93% rename from cmd/lotus-shed/borked-miners.go rename to cmd/lotus-shed/frozen-miners.go index ecb3aeae9..6b843f0d6 100644 --- a/cmd/lotus-shed/borked-miners.go +++ b/cmd/lotus-shed/frozen-miners.go @@ -10,9 +10,9 @@ import ( "golang.org/x/xerrors" ) -var borkedMinersCmd = &cli.Command{ - Name: "borked-miners", - Description: "information about 'borked' miner actors with late or frozen deadline crons", +var frozenMinersCmd = &cli.Command{ + Name: "frozen-miners", + Description: "information about miner actors with late or frozen deadline crons", Flags: []cli.Flag{ &cli.StringFlag{ Name: "tipset", diff --git a/cmd/lotus-shed/main.go b/cmd/lotus-shed/main.go index c8ce3ae04..488e2a6ae 100644 --- a/cmd/lotus-shed/main.go +++ b/cmd/lotus-shed/main.go @@ -19,7 +19,7 @@ func main() { base32Cmd, base16Cmd, bitFieldCmd, - borkedMinersCmd, + frozenMinersCmd, keyinfoCmd, jwtCmd, noncefix, From 70c39fe78c291063d156347ddb7ce81f86ff7bdd Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 21 Oct 2020 16:52:13 -0700 Subject: [PATCH 852/885] Fix global test options Set BELLMAN_NO_GPU on init. Set InsecurePoStValidation once in an obvious place. --- api/test/ccupgrade.go | 3 --- api/test/deals.go | 17 ----------------- api/test/mining.go | 3 --- api/test/paych.go | 3 --- api/test/test.go | 17 +++++++++++++++++ api/test/window_post.go | 8 -------- 6 files changed, 17 insertions(+), 34 deletions(-) diff --git a/api/test/ccupgrade.go b/api/test/ccupgrade.go index 4a860c661..e8010bdd4 100644 --- a/api/test/ccupgrade.go +++ b/api/test/ccupgrade.go @@ -3,7 +3,6 @@ package test import ( "context" "fmt" - "os" "sync/atomic" "testing" "time" @@ -17,8 +16,6 @@ import ( ) func TestCCUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration) { - _ = os.Setenv("BELLMAN_NO_GPU", "1") - for _, height := range []abi.ChainEpoch{ 1, // before 162, // while sealing diff --git a/api/test/deals.go b/api/test/deals.go index 8b4a7fe8b..ed856445b 100644 --- a/api/test/deals.go +++ b/api/test/deals.go @@ -16,15 +16,12 @@ import ( "github.com/ipfs/go-cid" files "github.com/ipfs/go-ipfs-files" - logging "github.com/ipfs/go-log/v2" "github.com/ipld/go-car" "github.com/filecoin-project/go-fil-markets/storagemarket" - "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" sealing "github.com/filecoin-project/lotus/extern/storage-sealing" - "github.com/filecoin-project/lotus/miner" dag "github.com/ipfs/go-merkledag" dstest "github.com/ipfs/go-merkledag/test" unixfile "github.com/ipfs/go-unixfs/file" @@ -34,18 +31,7 @@ import ( ipld "github.com/ipfs/go-ipld-format" ) -var MineNext = miner.MineReq{ - InjectNulls: 0, - Done: func(bool, abi.ChainEpoch, error) {}, -} - -func init() { - logging.SetAllLoggers(logging.LevelInfo) - build.InsecurePoStValidation = true -} - func TestDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration, carExport, fastRet bool) { - _ = os.Setenv("BELLMAN_NO_GPU", "1") ctx := context.Background() n, sn := b(t, OneFull, OneMiner) @@ -82,7 +68,6 @@ func TestDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration, carExport } func TestDoubleDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration) { - _ = os.Setenv("BELLMAN_NO_GPU", "1") ctx := context.Background() n, sn := b(t, OneFull, OneMiner) @@ -146,7 +131,6 @@ func makeDeal(t *testing.T, ctx context.Context, rseed int, client *impl.FullNod } func TestFastRetrievalDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration) { - _ = os.Setenv("BELLMAN_NO_GPU", "1") ctx := context.Background() n, sn := b(t, OneFull, OneMiner) @@ -201,7 +185,6 @@ func TestFastRetrievalDealFlow(t *testing.T, b APIBuilder, blocktime time.Durati } func TestSenondDealRetrieval(t *testing.T, b APIBuilder, blocktime time.Duration) { - _ = os.Setenv("BELLMAN_NO_GPU", "1") ctx := context.Background() n, sn := b(t, OneFull, OneMiner) diff --git a/api/test/mining.go b/api/test/mining.go index 8147c224b..90e8e8a45 100644 --- a/api/test/mining.go +++ b/api/test/mining.go @@ -5,7 +5,6 @@ import ( "context" "fmt" "math/rand" - "os" "sync/atomic" "testing" "time" @@ -88,8 +87,6 @@ func (ts *testSuite) testMiningReal(t *testing.T) { } func TestDealMining(t *testing.T, b APIBuilder, blocktime time.Duration, carExport bool) { - _ = os.Setenv("BELLMAN_NO_GPU", "1") - // test making a deal with a fresh miner, and see if it starts to mine ctx := context.Background() diff --git a/api/test/paych.go b/api/test/paych.go index 69def18bb..2bcea4369 100644 --- a/api/test/paych.go +++ b/api/test/paych.go @@ -3,7 +3,6 @@ package test import ( "context" "fmt" - "os" "sync/atomic" "testing" "time" @@ -28,8 +27,6 @@ import ( ) func TestPaymentChannels(t *testing.T, b APIBuilder, blocktime time.Duration) { - _ = os.Setenv("BELLMAN_NO_GPU", "1") - ctx := context.Background() n, sn := b(t, TwoFull, OneMiner) diff --git a/api/test/test.go b/api/test/test.go index 4074ce4a6..bae3d520e 100644 --- a/api/test/test.go +++ b/api/test/test.go @@ -2,12 +2,15 @@ package test import ( "context" + "fmt" + "os" "testing" "time" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/types" + logging "github.com/ipfs/go-log/v2" "github.com/multiformats/go-multiaddr" "github.com/stretchr/testify/assert" @@ -22,6 +25,15 @@ import ( "github.com/filecoin-project/lotus/node" ) +func init() { + logging.SetAllLoggers(logging.LevelInfo) + err := os.Setenv("BELLMAN_NO_GPU", "1") + if err != nil { + panic(fmt.Sprintf("failed to set BELLMAN_NO_GPU env variable: %s", err)) + } + build.InsecurePoStValidation = true +} + type TestNode struct { api.FullNode // ListenAddr is the address on which an API server is listening, if an @@ -110,6 +122,11 @@ var FullNodeWithUpgradeAt = func(upgradeHeight abi.ChainEpoch) FullNodeOpts { } } +var MineNext = miner.MineReq{ + InjectNulls: 0, + Done: func(bool, abi.ChainEpoch, error) {}, +} + func (ts *testSuite) testVersion(t *testing.T) { build.RunningNodeType = build.NodeFull diff --git a/api/test/window_post.go b/api/test/window_post.go index 28639cda8..55fc4ad70 100644 --- a/api/test/window_post.go +++ b/api/test/window_post.go @@ -5,7 +5,6 @@ import ( "fmt" "sync/atomic" - "os" "strings" "testing" "time" @@ -24,13 +23,6 @@ import ( "github.com/filecoin-project/lotus/node/impl" ) -func init() { - err := os.Setenv("BELLMAN_NO_GPU", "1") - if err != nil { - panic(fmt.Sprintf("failed to set BELLMAN_NO_GPU env variable: %s", err)) - } -} - func TestPledgeSector(t *testing.T, b APIBuilder, blocktime time.Duration, nSectors int) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() From 4a550d12d438f7ad87259235611d01c77044e797 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 21 Oct 2020 17:11:09 -0700 Subject: [PATCH 853/885] Always validate VRFs, even when insecure post validation is enabled We always generate them, we might as well validate them. --- chain/sync.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/chain/sync.go b/chain/sync.go index dda6f88d8..d05a3d8bb 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -1731,9 +1731,6 @@ func (syncer *Syncer) collectChain(ctx context.Context, ts *types.TipSet) error } func VerifyElectionPoStVRF(ctx context.Context, worker address.Address, rand []byte, evrf []byte) error { - if build.InsecurePoStValidation { - return nil - } return gen.VerifyVRF(ctx, worker, rand, evrf) } From defeba5576398b243cd198e000dc3e6a7cc6beee Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 21 Oct 2020 15:47:52 -0700 Subject: [PATCH 854/885] Fix lookback for worker key when computing ticket We need to compute the ticket based on our worker key from the lookback epoch, not the current epoch. --- miner/miner.go | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/miner/miner.go b/miner/miner.go index 7451d617a..7916e0405 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -386,7 +386,7 @@ func (m *Miner) mineOne(ctx context.Context, base *MiningBase) (*types.BlockMsg, rbase = bvals[len(bvals)-1] } - ticket, err := m.computeTicket(ctx, &rbase, base) + ticket, err := m.computeTicket(ctx, &rbase, base, mbi) if err != nil { return nil, xerrors.Errorf("scratching ticket failed: %w", err) } @@ -456,16 +456,7 @@ func (m *Miner) mineOne(ctx context.Context, base *MiningBase) (*types.BlockMsg, return b, nil } -func (m *Miner) computeTicket(ctx context.Context, brand *types.BeaconEntry, base *MiningBase) (*types.Ticket, error) { - mi, err := m.api.StateMinerInfo(ctx, m.address, types.EmptyTSK) - if err != nil { - return nil, err - } - worker, err := m.api.StateAccountKey(ctx, mi.Worker, types.EmptyTSK) - if err != nil { - return nil, err - } - +func (m *Miner) computeTicket(ctx context.Context, brand *types.BeaconEntry, base *MiningBase, mbi *api.MiningBaseInfo) (*types.Ticket, error) { buf := new(bytes.Buffer) if err := m.address.MarshalCBOR(buf); err != nil { return nil, xerrors.Errorf("failed to marshal address to cbor: %w", err) @@ -476,6 +467,11 @@ func (m *Miner) computeTicket(ctx context.Context, brand *types.BeaconEntry, bas buf.Write(base.TipSet.MinTicket().VRFProof) } + worker, err := m.api.StateAccountKey(ctx, mbi.WorkerKey, types.EmptyTSK) + if err != nil { + return nil, err + } + input, err := store.DrawRandomness(brand.Data, crypto.DomainSeparationTag_TicketProduction, round-build.TicketRandomnessLookback, buf.Bytes()) if err != nil { return nil, err From b05613445e709cd89f2f016aeff36fbb0e276a6a Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 21 Oct 2020 16:08:24 -0700 Subject: [PATCH 855/885] Sign blocks with the worker key from the correct lookback epoch --- chain/gen/mining.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/chain/gen/mining.go b/chain/gen/mining.go index 4889c632f..5fc56e8b2 100644 --- a/chain/gen/mining.go +++ b/chain/gen/mining.go @@ -27,7 +27,17 @@ func MinerCreateBlock(ctx context.Context, sm *stmgr.StateManager, w api.WalletA return nil, xerrors.Errorf("failed to load tipset state: %w", err) } - worker, err := stmgr.GetMinerWorkerRaw(ctx, sm, st, bt.Miner) + lbts, err := stmgr.GetLookbackTipSetForRound(ctx, sm, pts, bt.Epoch) + if err != nil { + return nil, xerrors.Errorf("getting lookback miner actor state: %w", err) + } + + lbst, _, err := sm.TipSetState(ctx, lbts) + if err != nil { + return nil, err + } + + worker, err := stmgr.GetMinerWorkerRaw(ctx, sm, lbst, bt.Miner) if err != nil { return nil, xerrors.Errorf("failed to get miner worker: %w", err) } From febc28f83f71b46d7d121e22beca1d7d40108af4 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 22 Oct 2020 04:57:29 +0200 Subject: [PATCH 856/885] Add log and Insert Signed-off-by: Jakub Sztandera --- chain/sync_manager.go | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/chain/sync_manager.go b/chain/sync_manager.go index c7fdea726..53fd07bc9 100644 --- a/chain/sync_manager.go +++ b/chain/sync_manager.go @@ -4,6 +4,7 @@ import ( "context" "sort" "sync" + "time" "github.com/filecoin-project/lotus/chain/types" peer "github.com/libp2p/go-libp2p-core/peer" @@ -224,8 +225,7 @@ func (sbs *syncBucketSet) Empty() bool { } type syncTargetBucket struct { - tips []*types.TipSet - count int + tips []*types.TipSet } func (stb *syncTargetBucket) sameChainAs(ts *types.TipSet) bool { @@ -244,7 +244,6 @@ func (stb *syncTargetBucket) sameChainAs(ts *types.TipSet) bool { } func (stb *syncTargetBucket) add(ts *types.TipSet) { - stb.count++ for _, t := range stb.tips { if t.Equals(ts) { @@ -294,6 +293,7 @@ func (sm *syncManager) selectSyncTarget() (*types.TipSet, error) { } func (sm *syncManager) syncScheduler() { + t := time.NewTicker(10 * time.Second) for { select { @@ -311,6 +311,16 @@ func (sm *syncManager) syncScheduler() { case <-sm.stop: log.Info("sync scheduler shutting down") return + case <-t.C: + activeSyncs := make([]types.TipSetKey, len(sm.activeSyncs), 0) + for tsk := range sm.activeSyncs { + activeSyncs = append(activeSyncs, tsk) + } + sort.Slice(activeSyncs, func(i, j int) bool { + return string(activeSyncs[i].Bytes()) < string(activeSyncs[j].Bytes()) + }) + + log.Infof("activeSyncs: %v, ", activeSyncs) } } } @@ -376,7 +386,9 @@ func (sm *syncManager) scheduleProcessResult(res *syncResult) { sm.nextSyncTarget = relbucket sm.workerChan = sm.syncTargets } else { - sm.syncQueue.buckets = append(sm.syncQueue.buckets, relbucket) + for _, t := range relbucket.tips { + sm.syncQueue.Insert(t) + } } return } From 5edfc527b0b0eeec035d82792e7ab463f10073fe Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 21 Oct 2020 22:52:35 +0200 Subject: [PATCH 857/885] More than one bucket Signed-off-by: Jakub Sztandera --- chain/sync_manager.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/chain/sync_manager.go b/chain/sync_manager.go index 53fd07bc9..63bc377ec 100644 --- a/chain/sync_manager.go +++ b/chain/sync_manager.go @@ -199,13 +199,18 @@ func (sbs *syncBucketSet) removeBucket(toremove *syncTargetBucket) { } func (sbs *syncBucketSet) PopRelated(ts *types.TipSet) *syncTargetBucket { + var bOut *syncTargetBucket for _, b := range sbs.buckets { if b.sameChainAs(ts) { - sbs.removeBucket(b) - return b + if bOut == nil { + sbs.removeBucket(b) + bOut = b + } else { + log.Errorf("REPORT THIS more that one related bucket for %s", ts) + } } } - return nil + return bOut } func (sbs *syncBucketSet) Heaviest() *types.TipSet { @@ -312,7 +317,7 @@ func (sm *syncManager) syncScheduler() { log.Info("sync scheduler shutting down") return case <-t.C: - activeSyncs := make([]types.TipSetKey, len(sm.activeSyncs), 0) + activeSyncs := make([]types.TipSetKey, 0, len(sm.activeSyncs)) for tsk := range sm.activeSyncs { activeSyncs = append(activeSyncs, tsk) } From 24fc7d4cbdee6871012a1cb0c9f12c52def80307 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 22 Oct 2020 23:33:05 +0200 Subject: [PATCH 858/885] Add reproduction test and fix the issue Signed-off-by: Jakub Sztandera --- chain/sync_manager.go | 19 ++++++++++-- chain/sync_manager_test.go | 61 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 2 deletions(-) diff --git a/chain/sync_manager.go b/chain/sync_manager.go index 63bc377ec..1c5951bb5 100644 --- a/chain/sync_manager.go +++ b/chain/sync_manager.go @@ -3,6 +3,7 @@ package chain import ( "context" "sort" + "strings" "sync" "time" @@ -153,6 +154,19 @@ func newSyncTargetBucket(tipsets ...*types.TipSet) *syncTargetBucket { return &stb } +func (sbs *syncBucketSet) String() string { + var bStrings []string + for _, b := range sbs.buckets { + var tsStrings []string + for _, t := range b.tips { + tsStrings = append(tsStrings, t.String()) + } + bStrings = append(bStrings, "["+strings.Join(tsStrings, ",")+"]") + } + + return "{" + strings.Join(bStrings, ";") + "}" +} + func (sbs *syncBucketSet) RelatedToAny(ts *types.TipSet) bool { for _, b := range sbs.buckets { if b.sameChainAs(ts) { @@ -325,7 +339,7 @@ func (sm *syncManager) syncScheduler() { return string(activeSyncs[i].Bytes()) < string(activeSyncs[j].Bytes()) }) - log.Infof("activeSyncs: %v, ", activeSyncs) + log.Infof("activeSyncs: %v, activeSyncTips: %s ", activeSyncs, sm.activeSyncTips.String()) } } } @@ -341,7 +355,8 @@ func (sm *syncManager) scheduleIncoming(ts *types.TipSet) { var relatedToActiveSync bool for _, acts := range sm.activeSyncs { if ts.Equals(acts) { - break + // ignore, we are already syncing it + return } if ts.Parents() == acts.Key() { diff --git a/chain/sync_manager_test.go b/chain/sync_manager_test.go index 269b3a62e..276e4bb36 100644 --- a/chain/sync_manager_test.go +++ b/chain/sync_manager_test.go @@ -67,6 +67,67 @@ func assertGetSyncOp(t *testing.T, c chan *syncOp, ts *types.TipSet) { } } +func TestSyncManagerEdgeCase(t *testing.T) { + ctx := context.Background() + + a := mock.TipSet(mock.MkBlock(genTs, 1, 1)) + t.Logf("a: %s", a) + b1 := mock.TipSet(mock.MkBlock(a, 1, 2)) + t.Logf("b1: %s", b1) + b2 := mock.TipSet(mock.MkBlock(a, 2, 3)) + t.Logf("b2: %s", b2) + c1 := mock.TipSet(mock.MkBlock(b1, 2, 4)) + t.Logf("c1: %s", c1) + c2 := mock.TipSet(mock.MkBlock(b2, 1, 5)) + t.Logf("c2: %s", c2) + + runSyncMgrTest(t, "edgeCase", 1, func(t *testing.T, sm *syncManager, stc chan *syncOp) { + sm.SetPeerHead(ctx, "peer1", a) + assertGetSyncOp(t, stc, a) + time.Sleep(10 * time.Millisecond) + t.Logf("bootstate: %d", sm.bootstrapState) + + sm.SetPeerHead(ctx, "peer1", b1) + sm.SetPeerHead(ctx, "peer1", b2) + // b1 and b2 are being processed + + b1op := <-stc + b2op := <-stc + if !b1op.ts.Equals(b1) { + b1op, b2op = b2op, b1op + } + t.Logf("activeSyncs: %s: activeSyncTips: %s", sm.activeSyncs, sm.activeSyncTips.String()) + + sm.SetPeerHead(ctx, "peer2", c2) // c2 is put into activeSyncTips at index 0 + sm.SetPeerHead(ctx, "peer2", c1) // c1 is put into activeSyncTips at index 1 + sm.SetPeerHead(ctx, "peer3", b2) // b2 is related to c2 and even though it is actively synced it is put into activeSyncTips index 0 + sm.SetPeerHead(ctx, "peer1", a) // a is related to b2 and is put into activeSyncTips index 0 + + b1op.done() // b1 completes first, is related to a, so it pops activeSyncTips index 0 + // even though correct one is index 1 + + time.Sleep(100 * time.Millisecond) + t.Logf("activeSyncs: %s: activeSyncTips: %s", sm.activeSyncs, sm.activeSyncTips.String()) + b2op.done() + // b2 completes and is not related to c1, so it leaves activeSyncTips as it is + for i := 0; i < 10; { + select { + case so := <-stc: + so.done() + default: + i++ + time.Sleep(10 * time.Millisecond) + } + } + + time.Sleep(10 * time.Millisecond) + if len(sm.activeSyncTips.buckets) != 0 { + t.Errorf("activeSyncTips expected empty but got: %s", sm.activeSyncTips.String()) + } + t.Logf("activeSyncs: %s: activeSyncTips: %s", sm.activeSyncs, sm.activeSyncTips.String()) + }) +} + func TestSyncManager(t *testing.T) { ctx := context.Background() From 810feee5a159de58a166071dc3b05bb08004a689 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 23 Oct 2020 00:33:35 +0200 Subject: [PATCH 859/885] PopRelated should pop all Signed-off-by: Jakub Sztandera --- chain/sync_manager.go | 38 ++++++++++++-------------------------- chain/sync_manager_test.go | 34 ++++++++++++++++++---------------- 2 files changed, 30 insertions(+), 42 deletions(-) diff --git a/chain/sync_manager.go b/chain/sync_manager.go index 1c5951bb5..460405a91 100644 --- a/chain/sync_manager.go +++ b/chain/sync_manager.go @@ -5,7 +5,6 @@ import ( "sort" "strings" "sync" - "time" "github.com/filecoin-project/lotus/chain/types" peer "github.com/libp2p/go-libp2p-core/peer" @@ -212,16 +211,12 @@ func (sbs *syncBucketSet) removeBucket(toremove *syncTargetBucket) { sbs.buckets = nbuckets } -func (sbs *syncBucketSet) PopRelated(ts *types.TipSet) *syncTargetBucket { - var bOut *syncTargetBucket +func (sbs *syncBucketSet) PopRelated(ts *types.TipSet) []*syncTargetBucket { + var bOut []*syncTargetBucket for _, b := range sbs.buckets { if b.sameChainAs(ts) { - if bOut == nil { - sbs.removeBucket(b) - bOut = b - } else { - log.Errorf("REPORT THIS more that one related bucket for %s", ts) - } + sbs.removeBucket(b) + bOut = append(bOut, b) } } return bOut @@ -312,8 +307,6 @@ func (sm *syncManager) selectSyncTarget() (*types.TipSet, error) { } func (sm *syncManager) syncScheduler() { - t := time.NewTicker(10 * time.Second) - for { select { case ts, ok := <-sm.incomingTipSets: @@ -330,16 +323,6 @@ func (sm *syncManager) syncScheduler() { case <-sm.stop: log.Info("sync scheduler shutting down") return - case <-t.C: - activeSyncs := make([]types.TipSetKey, 0, len(sm.activeSyncs)) - for tsk := range sm.activeSyncs { - activeSyncs = append(activeSyncs, tsk) - } - sort.Slice(activeSyncs, func(i, j int) bool { - return string(activeSyncs[i].Bytes()) < string(activeSyncs[j].Bytes()) - }) - - log.Infof("activeSyncs: %v, activeSyncTips: %s ", activeSyncs, sm.activeSyncTips.String()) } } } @@ -399,14 +382,17 @@ func (sm *syncManager) scheduleProcessResult(res *syncResult) { } delete(sm.activeSyncs, res.ts.Key()) - relbucket := sm.activeSyncTips.PopRelated(res.ts) - if relbucket != nil { + relbuckets := sm.activeSyncTips.PopRelated(res.ts) + if len(relbuckets) != 0 { if res.success { if sm.nextSyncTarget == nil { - sm.nextSyncTarget = relbucket + sm.nextSyncTarget = relbuckets[0] sm.workerChan = sm.syncTargets - } else { - for _, t := range relbucket.tips { + relbuckets = relbuckets[1:] + } + + for _, b := range relbuckets { + for _, t := range b.tips { sm.syncQueue.Insert(t) } } diff --git a/chain/sync_manager_test.go b/chain/sync_manager_test.go index 276e4bb36..709e03a41 100644 --- a/chain/sync_manager_test.go +++ b/chain/sync_manager_test.go @@ -80,12 +80,14 @@ func TestSyncManagerEdgeCase(t *testing.T) { t.Logf("c1: %s", c1) c2 := mock.TipSet(mock.MkBlock(b2, 1, 5)) t.Logf("c2: %s", c2) + d1 := mock.TipSet(mock.MkBlock(c1, 1, 6)) + t.Logf("d1: %s", d1) + e1 := mock.TipSet(mock.MkBlock(d1, 1, 7)) + t.Logf("e1: %s", e1) runSyncMgrTest(t, "edgeCase", 1, func(t *testing.T, sm *syncManager, stc chan *syncOp) { sm.SetPeerHead(ctx, "peer1", a) assertGetSyncOp(t, stc, a) - time.Sleep(10 * time.Millisecond) - t.Logf("bootstate: %d", sm.bootstrapState) sm.SetPeerHead(ctx, "peer1", b1) sm.SetPeerHead(ctx, "peer1", b2) @@ -96,7 +98,6 @@ func TestSyncManagerEdgeCase(t *testing.T) { if !b1op.ts.Equals(b1) { b1op, b2op = b2op, b1op } - t.Logf("activeSyncs: %s: activeSyncTips: %s", sm.activeSyncs, sm.activeSyncTips.String()) sm.SetPeerHead(ctx, "peer2", c2) // c2 is put into activeSyncTips at index 0 sm.SetPeerHead(ctx, "peer2", c1) // c1 is put into activeSyncTips at index 1 @@ -106,28 +107,29 @@ func TestSyncManagerEdgeCase(t *testing.T) { b1op.done() // b1 completes first, is related to a, so it pops activeSyncTips index 0 // even though correct one is index 1 - time.Sleep(100 * time.Millisecond) - t.Logf("activeSyncs: %s: activeSyncTips: %s", sm.activeSyncs, sm.activeSyncTips.String()) b2op.done() // b2 completes and is not related to c1, so it leaves activeSyncTips as it is - for i := 0; i < 10; { - select { - case so := <-stc: - so.done() - default: - i++ - time.Sleep(10 * time.Millisecond) - } - } - time.Sleep(10 * time.Millisecond) + waitUntilAllWorkersAreDone(stc) + if len(sm.activeSyncTips.buckets) != 0 { t.Errorf("activeSyncTips expected empty but got: %s", sm.activeSyncTips.String()) } - t.Logf("activeSyncs: %s: activeSyncTips: %s", sm.activeSyncs, sm.activeSyncTips.String()) }) } +func waitUntilAllWorkersAreDone(stc chan *syncOp) { + for i := 0; i < 10; { + select { + case so := <-stc: + so.done() + default: + i++ + time.Sleep(10 * time.Millisecond) + } + } +} + func TestSyncManager(t *testing.T) { ctx := context.Background() From 59cfabf8a4496716c79a3f449c33019964473e13 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 22 Oct 2020 15:48:09 -0700 Subject: [PATCH 860/885] don't resolve the worker key twice --- miner/miner.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/miner/miner.go b/miner/miner.go index 7916e0405..7de7f5b73 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -467,17 +467,12 @@ func (m *Miner) computeTicket(ctx context.Context, brand *types.BeaconEntry, bas buf.Write(base.TipSet.MinTicket().VRFProof) } - worker, err := m.api.StateAccountKey(ctx, mbi.WorkerKey, types.EmptyTSK) - if err != nil { - return nil, err - } - input, err := store.DrawRandomness(brand.Data, crypto.DomainSeparationTag_TicketProduction, round-build.TicketRandomnessLookback, buf.Bytes()) if err != nil { return nil, err } - vrfOut, err := gen.ComputeVRF(ctx, m.api.WalletSign, worker, input) + vrfOut, err := gen.ComputeVRF(ctx, m.api.WalletSign, mbi.WorkerKey, input) if err != nil { return nil, err } From c8fe21c1ef2f5292dbe5a151a305a1203b959d4d Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 23 Oct 2020 01:18:13 +0200 Subject: [PATCH 861/885] PopRelated aggretates the bucket Signed-off-by: Jakub Sztandera --- chain/sync_manager.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/chain/sync_manager.go b/chain/sync_manager.go index 460405a91..27643b054 100644 --- a/chain/sync_manager.go +++ b/chain/sync_manager.go @@ -211,12 +211,15 @@ func (sbs *syncBucketSet) removeBucket(toremove *syncTargetBucket) { sbs.buckets = nbuckets } -func (sbs *syncBucketSet) PopRelated(ts *types.TipSet) []*syncTargetBucket { - var bOut []*syncTargetBucket +func (sbs *syncBucketSet) PopRelated(ts *types.TipSet) *syncTargetBucket { + var bOut *syncTargetBucket for _, b := range sbs.buckets { if b.sameChainAs(ts) { sbs.removeBucket(b) - bOut = append(bOut, b) + if bOut == nil { + bOut = &syncTargetBucket{} + } + bOut.tips = append(bOut.tips, b.tips...) } } return bOut @@ -382,17 +385,14 @@ func (sm *syncManager) scheduleProcessResult(res *syncResult) { } delete(sm.activeSyncs, res.ts.Key()) - relbuckets := sm.activeSyncTips.PopRelated(res.ts) - if len(relbuckets) != 0 { + relbucket := sm.activeSyncTips.PopRelated(res.ts) + if relbucket != nil { if res.success { if sm.nextSyncTarget == nil { - sm.nextSyncTarget = relbuckets[0] + sm.nextSyncTarget = relbucket sm.workerChan = sm.syncTargets - relbuckets = relbuckets[1:] - } - - for _, b := range relbuckets { - for _, t := range b.tips { + } else { + for _, t := range relbucket.tips { sm.syncQueue.Insert(t) } } From 580a2f4dc6a8c301346684d6021917eb9f954ec0 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 23 Oct 2020 01:35:26 +0200 Subject: [PATCH 862/885] Add option to join by common parents Signed-off-by: Jakub Sztandera --- chain/sync_manager.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/chain/sync_manager.go b/chain/sync_manager.go index 27643b054..c25068f60 100644 --- a/chain/sync_manager.go +++ b/chain/sync_manager.go @@ -2,6 +2,7 @@ package chain import ( "context" + "os" "sort" "strings" "sync" @@ -12,6 +13,14 @@ import ( const BootstrapPeerThreshold = 2 +var coalesceForksParents = false + +func init() { + if os.Getenv("LOTUS_SYNC_REL_PARENT") == "yes" { + coalesceForksParents = true + } +} + const ( BSStateInit = 0 BSStateSelected = 1 @@ -256,6 +265,9 @@ func (stb *syncTargetBucket) sameChainAs(ts *types.TipSet) bool { if ts.Parents() == t.Key() { return true } + if coalesceForksParents && ts.Parents() == t.Parents() { + return true + } } return false } From 6842c0339984265f2d9f711d2146a0c552d133a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 23 Oct 2020 11:16:29 +0200 Subject: [PATCH 863/885] wallet cli: rename withdraw-market-funds -> market withdraw --- cli/wallet.go | 45 ++++++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/cli/wallet.go b/cli/wallet.go index f504e252f..f231f3454 100644 --- a/cli/wallet.go +++ b/cli/wallet.go @@ -38,7 +38,7 @@ var walletCmd = &cli.Command{ walletSign, walletVerify, walletDelete, - walletWithdrawMarketFunds, + walletMarket, }, } @@ -477,15 +477,23 @@ var walletDelete = &cli.Command{ }, } -var walletWithdrawMarketFunds = &cli.Command{ - Name: "withdraw-market-funds", +var walletMarket = &cli.Command{ + Name: "market", + Usage: "Interact with market balances", + Subcommands: []*cli.Command{ + walletMarketWithdraw, + }, +} + +var walletMarketWithdraw = &cli.Command{ + Name: "withdraw", Usage: "Withdraw funds from the Storage Market Actor", ArgsUsage: "[amount (FIL) optional, otherwise will withdraw max available]", Flags: []cli.Flag{ &cli.StringFlag{ - Name: "address", + Name: "from", Usage: "Specify address to withdraw funds from, otherwise it will use the default wallet address", - Aliases: []string{"a"}, + Aliases: []string{"f"}, }, }, Action: func(cctx *cli.Context) error { @@ -496,25 +504,20 @@ var walletWithdrawMarketFunds = &cli.Command{ defer closer() ctx := ReqContext(cctx) - defAddr, err := api.WalletDefaultAddress(ctx) - if err != nil { - return xerrors.Errorf("getting default wallet address: %w", err) - } - - addr := defAddr - if cctx.String("address") != "" { - addr, err = address.NewFromString(cctx.String("address")) + var addr address.Address + if cctx.String("from") != "" { + addr, err = address.NewFromString(cctx.String("from")) if err != nil { - return xerrors.Errorf("parsing address: %w", err) + return xerrors.Errorf("parsing from address: %w", err) + } + } else { + addr, err = api.WalletDefaultAddress(ctx) + if err != nil { + return xerrors.Errorf("getting default wallet address: %w", err) } } - ts, err := api.ChainHead(ctx) - if err != nil { - return xerrors.Errorf("getting chain head: %w", err) - } - - bal, err := api.StateMarketBalance(ctx, addr, ts.Key()) + bal, err := api.StateMarketBalance(ctx, addr, types.EmptyTSK) if err != nil { return xerrors.Errorf("getting market balance for address %s: %w", addr.String(), err) } @@ -550,7 +553,7 @@ var walletWithdrawMarketFunds = &cli.Command{ fmt.Printf("Submitting WithdrawBalance message for amount %s for address %s\n", types.FIL(amt), addr.String()) smsg, err := api.MpoolPushMessage(ctx, &types.Message{ To: builtin.StorageMarketActorAddr, - From: defAddr, + From: addr, Value: types.NewInt(0), Method: builtin.MethodsMarket.WithdrawBalance, Params: params, From 92395745d3e3b6842c766bc0bb21921c78c4f901 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Wed, 21 Oct 2020 10:58:07 +0100 Subject: [PATCH 864/885] tvx simulate: execute raw message against height. --- cmd/tvx/extract.go | 70 +++++------- cmd/tvx/extract_many.go | 17 +-- cmd/tvx/main.go | 50 ++++++++- cmd/tvx/simulate.go | 231 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 309 insertions(+), 59 deletions(-) create mode 100644 cmd/tvx/simulate.go diff --git a/cmd/tvx/extract.go b/cmd/tvx/extract.go index 276c38cd4..06a7d4cea 100644 --- a/cmd/tvx/extract.go +++ b/cmd/tvx/extract.go @@ -19,7 +19,6 @@ import ( "github.com/filecoin-project/lotus/chain/actors/builtin/reward" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" - lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/conformance" "github.com/filecoin-project/test-vectors/schema" @@ -88,10 +87,10 @@ var extractCmd = &cli.Command{ }, &cli.StringFlag{ Name: "precursor-select", - Usage: "precursors to apply; values: 'all', 'sender'; 'all' selects all preceding" + - "messages in the canonicalised tipset, 'sender' selects only preceding messages from the same" + - "sender. Usually, 'sender' is a good tradeoff and gives you sufficient accuracy. If the receipt sanity" + - "check fails due to gas reasons, switch to 'all', as previous messages in the tipset may have" + + Usage: "precursors to apply; values: 'all', 'sender'; 'all' selects all preceding " + + "messages in the canonicalised tipset, 'sender' selects only preceding messages from the same " + + "sender. Usually, 'sender' is a good tradeoff and gives you sufficient accuracy. If the receipt sanity " + + "check fails due to gas reasons, switch to 'all', as previous messages in the tipset may have " + "affected state in a disruptive way", Value: "sender", Destination: &extractFlags.precursor, @@ -100,41 +99,24 @@ var extractCmd = &cli.Command{ } func runExtract(c *cli.Context) error { - // LOTUS_DISABLE_VM_BUF disables what's called "VM state tree buffering", - // which stashes write operations in a BufferedBlockstore - // (https://github.com/filecoin-project/lotus/blob/b7a4dbb07fd8332b4492313a617e3458f8003b2a/lib/bufbstore/buf_bstore.go#L21) - // such that they're not written until the VM is actually flushed. - // - // For some reason, the standard behaviour was not working for me (raulk), - // and disabling it (such that the state transformations are written immediately - // to the blockstore) worked. - _ = os.Setenv("LOTUS_DISABLE_VM_BUF", "iknowitsabadidea") - - ctx := context.Background() - - // Make the API client. - fapi, closer, err := lcli.GetFullNodeAPI(c) - if err != nil { - return err - } - defer closer() - - return doExtract(ctx, fapi, extractFlags) + return doExtract(extractFlags) } -func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { +func doExtract(opts extractOpts) error { + ctx := context.Background() + mcid, err := cid.Decode(opts.cid) if err != nil { return err } - msg, execTs, incTs, err := resolveFromChain(ctx, fapi, mcid, opts.block) + msg, execTs, incTs, err := resolveFromChain(ctx, FullAPI, mcid, opts.block) if err != nil { return fmt.Errorf("failed to resolve message and tipsets from chain: %w", err) } // get the circulating supply before the message was executed. - circSupplyDetail, err := fapi.StateVMCirculatingSupplyInternal(ctx, incTs.Key()) + circSupplyDetail, err := FullAPI.StateVMCirculatingSupplyInternal(ctx, incTs.Key()) if err != nil { return fmt.Errorf("failed while fetching circulating supply: %w", err) } @@ -147,7 +129,7 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { log.Printf("finding precursor messages using mode: %s", opts.precursor) // Fetch messages in canonical order from inclusion tipset. - msgs, err := fapi.ChainGetParentMessages(ctx, execTs.Blocks()[0].Cid()) + msgs, err := FullAPI.ChainGetParentMessages(ctx, execTs.Blocks()[0].Cid()) if err != nil { return fmt.Errorf("failed to fetch messages in canonical order from inclusion tipset: %w", err) } @@ -174,8 +156,8 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { var ( // create a read-through store that uses ChainGetObject to fetch unknown CIDs. - pst = NewProxyingStores(ctx, fapi) - g = NewSurgeon(ctx, fapi, pst) + pst = NewProxyingStores(ctx, FullAPI) + g = NewSurgeon(ctx, FullAPI, pst) ) driver := conformance.NewDriver(ctx, schema.Selector{}, conformance.DriverOpts{ @@ -200,7 +182,7 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { CircSupply: circSupplyDetail.FilCirculating, BaseFee: basefee, // recorded randomness will be discarded. - Rand: conformance.NewRecordingRand(new(conformance.LogReporter), fapi), + Rand: conformance.NewRecordingRand(new(conformance.LogReporter), FullAPI), }) if err != nil { return fmt.Errorf("failed to execute precursor message: %w", err) @@ -215,7 +197,7 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { retention = opts.retain // recordingRand will record randomness so we can embed it in the test vector. - recordingRand = conformance.NewRecordingRand(new(conformance.LogReporter), fapi) + recordingRand = conformance.NewRecordingRand(new(conformance.LogReporter), FullAPI) ) log.Printf("using state retention strategy: %s", retention) @@ -248,7 +230,7 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { case "accessed-actors": log.Printf("calculating accessed actors") // get actors accessed by message. - retain, err := g.GetAccessedActors(ctx, fapi, mcid) + retain, err := g.GetAccessedActors(ctx, FullAPI, mcid) if err != nil { return fmt.Errorf("failed to calculate accessed actors: %w", err) } @@ -286,7 +268,7 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { // TODO sometimes this returns a nil receipt and no error ¯\_(ツ)_/¯ // ex: https://filfox.info/en/message/bafy2bzacebpxw3yiaxzy2bako62akig46x3imji7fewszen6fryiz6nymu2b2 // This code is lenient and skips receipt comparison in case of a nil receipt. - rec, err := fapi.StateGetReceipt(ctx, mcid, execTs.Key()) + rec, err := FullAPI.StateGetReceipt(ctx, mcid, execTs.Key()) if err != nil { return fmt.Errorf("failed to find receipt on chain: %w", err) } @@ -336,17 +318,17 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { return err } - version, err := fapi.Version(ctx) + version, err := FullAPI.Version(ctx) if err != nil { return err } - ntwkName, err := fapi.StateNetworkName(ctx) + ntwkName, err := FullAPI.StateNetworkName(ctx) if err != nil { return err } - nv, err := fapi.StateNetworkVersion(ctx, execTs.Key()) + nv, err := FullAPI.StateNetworkVersion(ctx, execTs.Key()) if err != nil { return err } @@ -399,8 +381,12 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { }, } + return writeVector(vector, opts.file) +} + +func writeVector(vector schema.TestVector, file string) (err error) { output := io.WriteCloser(os.Stdout) - if file := opts.file; file != "" { + if file := file; file != "" { dir := filepath.Dir(file) if err := os.MkdirAll(dir, 0755); err != nil { return fmt.Errorf("unable to create directory %s: %w", dir, err) @@ -415,11 +401,7 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error { enc := json.NewEncoder(output) enc.SetIndent("", " ") - if err := enc.Encode(&vector); err != nil { - return err - } - - return nil + return enc.Encode(&vector) } // resolveFromChain queries the chain for the provided message, using the block CID to diff --git a/cmd/tvx/extract_many.go b/cmd/tvx/extract_many.go index fe0ce6a6c..bf02d0880 100644 --- a/cmd/tvx/extract_many.go +++ b/cmd/tvx/extract_many.go @@ -1,7 +1,6 @@ package main import ( - "context" "encoding/csv" "fmt" "io" @@ -20,7 +19,6 @@ import ( "github.com/urfave/cli/v2" "github.com/filecoin-project/lotus/chain/stmgr" - lcli "github.com/filecoin-project/lotus/cli" ) var extractManyFlags struct { @@ -77,15 +75,6 @@ func runExtractMany(c *cli.Context) error { // to the blockstore) worked. _ = os.Setenv("LOTUS_DISABLE_VM_BUF", "iknowitsabadidea") - ctx := context.Background() - - // Make the API client. - fapi, closer, err := lcli.GetFullNodeAPI(c) - if err != nil { - return err - } - defer closer() - var ( in = extractManyFlags.in outdir = extractManyFlags.outdir @@ -198,8 +187,8 @@ func runExtractMany(c *cli.Context) error { precursor: PrecursorSelectSender, } - if err := doExtract(ctx, fapi, opts); err != nil { - log.Println(color.RedString("failed to extract vector for message %s: %s; queuing for 'canonical' precursor selection", mcid, err)) + if err := doExtract(opts); err != nil { + log.Println(color.RedString("failed to extract vector for message %s: %s; queuing for 'all' precursor selection", mcid, err)) retry = append(retry, opts) continue } @@ -215,7 +204,7 @@ func runExtractMany(c *cli.Context) error { log.Printf("retrying %s: %s", r.cid, r.id) r.precursor = PrecursorSelectAll - if err := doExtract(ctx, fapi, r); err != nil { + if err := doExtract(r); err != nil { merr = multierror.Append(merr, fmt.Errorf("failed to extract vector for message %s: %w", r.cid, err)) continue } diff --git a/cmd/tvx/main.go b/cmd/tvx/main.go index 6c887d163..5937cea8d 100644 --- a/cmd/tvx/main.go +++ b/cmd/tvx/main.go @@ -5,9 +5,21 @@ import ( "os" "sort" + "github.com/filecoin-project/go-jsonrpc" "github.com/urfave/cli/v2" + + "github.com/filecoin-project/lotus/api" + lcli "github.com/filecoin-project/lotus/cli" ) +// FullAPI is a JSON-RPC client targeting a full node. It's initialized in a +// cli.BeforeFunc. +var FullAPI api.FullNode + +// Closer is the closer for the JSON-RPC client, which must be called on +// cli.AfterFunc. +var Closer jsonrpc.ClientCloser + // DefaultLotusRepoPath is where the fallback path where to look for a Lotus // client repo. It is expanded with mitchellh/go-homedir, so it'll work with all // OSes despite the Unix twiddle notation. @@ -23,7 +35,7 @@ var repoFlag = cli.StringFlag{ func main() { app := &cli.App{ Name: "tvx", - Description: `tvx is a tool for extracting and executing test vectors. It has three subcommands. + Description: `tvx is a tool for extracting and executing test vectors. It has four subcommands. tvx extract extracts a test vector from a live network. It requires access to a Filecoin client that exposes the standard JSON-RPC API endpoint. Only @@ -35,6 +47,15 @@ func main() { tvx extract-many performs a batch extraction of many messages, supplied in a CSV file. Refer to the help of that subcommand for more info. + tvx project projects an existing test vector against a different protocol + version, reporting the result, optionally appending a new variant to the + vector in place if deemed equivalent, or producing a new vector if + non-equivalent. + + tvx simulate takes a raw message and simulates it on top of the supplied + epoch, reporting the result on stderr and writing a test vector on stdout + or into the specified file. + SETTING THE JSON-RPC API ENDPOINT You can set the JSON-RPC API endpoint through one of the following methods. @@ -57,7 +78,10 @@ func main() { extractCmd, execCmd, extractManyCmd, + simulateCmd, }, + Before: initialize, + After: destroy, } sort.Sort(cli.CommandsByName(app.Commands)) @@ -69,3 +93,27 @@ func main() { log.Fatal(err) } } + +func initialize(c *cli.Context) error { + // LOTUS_DISABLE_VM_BUF disables what's called "VM state tree buffering", + // which stashes write operations in a BufferedBlockstore + // (https://github.com/filecoin-project/lotus/blob/b7a4dbb07fd8332b4492313a617e3458f8003b2a/lib/bufbstore/buf_bstore.go#L21) + // such that they're not written until the VM is actually flushed. + // + // For some reason, the standard behaviour was not working for me (raulk), + // and disabling it (such that the state transformations are written immediately + // to the blockstore) worked. + _ = os.Setenv("LOTUS_DISABLE_VM_BUF", "iknowitsabadidea") + + // Make the API client. + var err error + FullAPI, Closer, err = lcli.GetFullNodeAPI(c) + return err +} + +func destroy(_ *cli.Context) error { + if Closer != nil { + Closer() + } + return nil +} diff --git a/cmd/tvx/simulate.go b/cmd/tvx/simulate.go new file mode 100644 index 000000000..856193e49 --- /dev/null +++ b/cmd/tvx/simulate.go @@ -0,0 +1,231 @@ +package main + +import ( + "bytes" + "compress/gzip" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "log" + "os/exec" + + "github.com/fatih/color" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/test-vectors/schema" + "github.com/urfave/cli/v2" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/conformance" +) + +var simulateFlags struct { + msg string + epoch int64 + out string + statediff bool +} + +var simulateCmd = &cli.Command{ + Name: "simulate", + Description: "simulate a raw message on top of the supplied" + + "epoch, reporting the result on stderr and writing a test vector on stdout" + + "or into the specified file", + Action: runSimulateCmd, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "msg", + Usage: "base64 cbor-encoded message", + Destination: &simulateFlags.msg, + }, + &cli.Int64Flag{ + Name: "at-epoch", + Usage: "epoch at which to run this message (or HEAD if not provided)", + Destination: &simulateFlags.epoch, + }, + &cli.StringFlag{ + Name: "out", + Usage: "file to write the test vector to; if nil, the vector will be written to stdout", + TakesFile: true, + Destination: &simulateFlags.out, + }, + &cli.BoolFlag{ + Name: "statediff", + Usage: "display a statediff of the precondition and postcondition states", + Destination: &simulateFlags.statediff, + }, + }, +} + +func runSimulateCmd(_ *cli.Context) error { + ctx := context.Background() + r := new(conformance.LogReporter) + + msgb, err := base64.StdEncoding.DecodeString(simulateFlags.msg) + if err != nil { + return fmt.Errorf("failed to base64-decode message: %w", err) + } + + msg, err := types.DecodeMessage(msgb) + if err != nil { + return fmt.Errorf("failed to deserialize message: %w", err) + } + + log.Printf("message to simulate has CID: %s", msg.Cid()) + + msgjson, err := json.Marshal(msg) + if err != nil { + return fmt.Errorf("failed to serialize message to json for printing: %w", err) + } + + log.Printf("message to simulate: %s", string(msgjson)) + + // Resolve the tipset, root, epoch. + var ts *types.TipSet + if epochIn := simulateFlags.epoch; epochIn == 0 { + ts, err = FullAPI.ChainHead(ctx) + } else { + ts, err = FullAPI.ChainGetTipSetByHeight(ctx, abi.ChainEpoch(epochIn), types.EmptyTSK) + } + + if err != nil { + return fmt.Errorf("failed to get tipset: %w", err) + } + + var ( + preroot = ts.ParentState() + epoch = ts.Height() + baseFee = ts.Blocks()[0].ParentBaseFee + circSupply api.CirculatingSupply + ) + + // Get circulating supply. + circSupply, err = FullAPI.StateVMCirculatingSupplyInternal(ctx, ts.Key()) + if err != nil { + return fmt.Errorf("failed to get circulating supply for tipset %s: %w", ts.Key(), err) + } + + // Create the driver. + stores := NewProxyingStores(ctx, FullAPI) + driver := conformance.NewDriver(ctx, schema.Selector{}, conformance.DriverOpts{ + DisableVMFlush: true, + }) + rand := conformance.NewRecordingRand(r, FullAPI) + + tbs, ok := stores.Blockstore.(TracingBlockstore) + if !ok { + return fmt.Errorf("no tracing blockstore available") + } + tbs.StartTracing() + applyret, postroot, err := driver.ExecuteMessage(stores.Blockstore, conformance.ExecuteMessageParams{ + Preroot: preroot, + Epoch: epoch, + Message: msg, + CircSupply: circSupply.FilCirculating, + BaseFee: baseFee, + Rand: rand, + }) + if err != nil { + return fmt.Errorf("failed to apply message: %w", err) + } + + accessed := tbs.FinishTracing() + + var ( + out = new(bytes.Buffer) + gw = gzip.NewWriter(out) + g = NewSurgeon(ctx, FullAPI, stores) + ) + if err := g.WriteCARIncluding(gw, accessed, preroot, postroot); err != nil { + return err + } + if err = gw.Flush(); err != nil { + return err + } + if err = gw.Close(); err != nil { + return err + } + + version, err := FullAPI.Version(ctx) + if err != nil { + log.Printf("failed to get node version: %s; falling back to unknown", err) + version = api.Version{} + } + + nv, err := FullAPI.StateNetworkVersion(ctx, ts.Key()) + if err != nil { + return err + } + + codename := GetProtocolCodename(epoch) + + // Write out the test vector. + vector := schema.TestVector{ + Class: schema.ClassMessage, + Meta: &schema.Metadata{ + ID: fmt.Sprintf("simulated-%s", msg.Cid()), + Gen: []schema.GenerationData{ + {Source: "github.com/filecoin-project/lotus", Version: version.String()}}, + }, + Selector: schema.Selector{ + schema.SelectorMinProtocolVersion: codename, + }, + Randomness: rand.Recorded(), + CAR: out.Bytes(), + Pre: &schema.Preconditions{ + Variants: []schema.Variant{ + {ID: codename, Epoch: int64(epoch), NetworkVersion: uint(nv)}, + }, + CircSupply: circSupply.FilCirculating.Int, + BaseFee: baseFee.Int, + StateTree: &schema.StateTree{ + RootCID: preroot, + }, + }, + ApplyMessages: []schema.Message{{Bytes: msgb}}, + Post: &schema.Postconditions{ + StateTree: &schema.StateTree{ + RootCID: postroot, + }, + Receipts: []*schema.Receipt{ + { + ExitCode: int64(applyret.ExitCode), + ReturnValue: applyret.Return, + GasUsed: applyret.GasUsed, + }, + }, + }, + } + + if err := writeVector(vector, simulateFlags.out); err != nil { + return fmt.Errorf("failed to write vector: %w", err) + } + + log.Printf(color.GreenString("wrote vector at: %s"), simulateFlags.out) + + if !simulateFlags.statediff { + return nil + } + + if simulateFlags.out == "" { + log.Print("omitting statediff in non-file mode") + return nil + } + + // check if statediff is installed; if not, skip. + if err := exec.Command("statediff", "--help").Run(); err != nil { + log.Printf("could not perform statediff on generated vector; command not found (%s)", err) + log.Printf("install statediff with:") + log.Printf("$ GOMODULE111=off go get github.com/filecoin-project/statediff/cmd/statediff") + return err + } + + stdiff, err := exec.Command("statediff", "vector", "--file", simulateFlags.out).CombinedOutput() + if err != nil { + return fmt.Errorf("failed to statediff: %w", err) + } + + log.Print(string(stdiff)) + return nil +} From 5301c416bc1a9476b127f1d692240b25710a8233 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Fri, 23 Oct 2020 10:46:56 +0100 Subject: [PATCH 865/885] tvx extract: --ignore-sanity-checks flag. --- cmd/tvx/extract.go | 36 +++++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/cmd/tvx/extract.go b/cmd/tvx/extract.go index 06a7d4cea..92b6d6fe3 100644 --- a/cmd/tvx/extract.go +++ b/cmd/tvx/extract.go @@ -33,13 +33,14 @@ const ( ) type extractOpts struct { - id string - block string - class string - cid string - file string - retain string - precursor string + id string + block string + class string + cid string + file string + retain string + precursor string + ignoreSanityChecks bool } var extractFlags extractOpts @@ -95,10 +96,16 @@ var extractCmd = &cli.Command{ Value: "sender", Destination: &extractFlags.precursor, }, + &cli.BoolFlag{ + Name: "ignore-sanity-checks", + Usage: "generate vector even if sanity checks fail", + Value: false, + Destination: &extractFlags.ignoreSanityChecks, + }, }, } -func runExtract(c *cli.Context) error { +func runExtract(_ *cli.Context) error { return doExtract(extractFlags) } @@ -282,13 +289,20 @@ func doExtract(opts extractOpts) error { ReturnValue: rec.Return, GasUsed: rec.GasUsed, } + reporter := new(conformance.LogReporter) conformance.AssertMsgResult(reporter, receipt, applyret, "as locally executed") if reporter.Failed() { - log.Println(color.RedString("receipt sanity check failed; aborting")) - return fmt.Errorf("vector generation aborted") + if opts.ignoreSanityChecks { + log.Println(color.YellowString("receipt sanity check failed; proceeding anyway")) + } else { + log.Println(color.RedString("receipt sanity check failed; aborting")) + return fmt.Errorf("vector generation aborted") + } + } else { + log.Println(color.GreenString("receipt sanity check succeeded")) } - log.Println(color.GreenString("receipt sanity check succeeded")) + } else { receipt = &schema.Receipt{ ExitCode: int64(applyret.ExitCode), From 39d915017586702ed49efdc356354310756e3937 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Fri, 23 Oct 2020 13:30:04 +0100 Subject: [PATCH 866/885] fix tvx init. --- cmd/tvx/extract.go | 2 ++ cmd/tvx/extract_many.go | 2 ++ cmd/tvx/main.go | 12 ++++-------- cmd/tvx/simulate.go | 8 ++++++-- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/cmd/tvx/extract.go b/cmd/tvx/extract.go index 92b6d6fe3..3dfec37d8 100644 --- a/cmd/tvx/extract.go +++ b/cmd/tvx/extract.go @@ -49,6 +49,8 @@ var extractCmd = &cli.Command{ Name: "extract", Description: "generate a test vector by extracting it from a live chain", Action: runExtract, + Before: initialize, + After: destroy, Flags: []cli.Flag{ &repoFlag, &cli.StringFlag{ diff --git a/cmd/tvx/extract_many.go b/cmd/tvx/extract_many.go index bf02d0880..048271456 100644 --- a/cmd/tvx/extract_many.go +++ b/cmd/tvx/extract_many.go @@ -43,6 +43,8 @@ var extractManyCmd = &cli.Command{ after these compulsory seven. `, Action: runExtractMany, + Before: initialize, + After: destroy, Flags: []cli.Flag{ &repoFlag, &cli.StringFlag{ diff --git a/cmd/tvx/main.go b/cmd/tvx/main.go index 5937cea8d..8de851ed5 100644 --- a/cmd/tvx/main.go +++ b/cmd/tvx/main.go @@ -1,6 +1,7 @@ package main import ( + "fmt" "log" "os" "sort" @@ -47,11 +48,6 @@ func main() { tvx extract-many performs a batch extraction of many messages, supplied in a CSV file. Refer to the help of that subcommand for more info. - tvx project projects an existing test vector against a different protocol - version, reporting the result, optionally appending a new variant to the - vector in place if deemed equivalent, or producing a new vector if - non-equivalent. - tvx simulate takes a raw message and simulates it on top of the supplied epoch, reporting the result on stderr and writing a test vector on stdout or into the specified file. @@ -80,8 +76,6 @@ func main() { extractManyCmd, simulateCmd, }, - Before: initialize, - After: destroy, } sort.Sort(cli.CommandsByName(app.Commands)) @@ -107,7 +101,9 @@ func initialize(c *cli.Context) error { // Make the API client. var err error - FullAPI, Closer, err = lcli.GetFullNodeAPI(c) + if FullAPI, Closer, err = lcli.GetFullNodeAPI(c); err != nil { + err = fmt.Errorf("failed to locate Lotus node; ") + } return err } diff --git a/cmd/tvx/simulate.go b/cmd/tvx/simulate.go index 856193e49..82b2bc118 100644 --- a/cmd/tvx/simulate.go +++ b/cmd/tvx/simulate.go @@ -29,15 +29,19 @@ var simulateFlags struct { var simulateCmd = &cli.Command{ Name: "simulate", - Description: "simulate a raw message on top of the supplied" + - "epoch, reporting the result on stderr and writing a test vector on stdout" + + Description: "simulate a raw message on top of the supplied epoch (or HEAD), " + + "reporting the result on stderr and writing a test vector on stdout " + "or into the specified file", Action: runSimulateCmd, + Before: initialize, + After: destroy, Flags: []cli.Flag{ + &repoFlag, &cli.StringFlag{ Name: "msg", Usage: "base64 cbor-encoded message", Destination: &simulateFlags.msg, + Required: true, }, &cli.Int64Flag{ Name: "at-epoch", From 9caa6ae69c18c7fecdcf3cb283b0c52f5b09cd83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 23 Oct 2020 14:42:19 +0200 Subject: [PATCH 867/885] wallet list: Add market balance and ID address flags --- cli/wallet.go | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/cli/wallet.go b/cli/wallet.go index f231f3454..ec343114d 100644 --- a/cli/wallet.go +++ b/cli/wallet.go @@ -79,6 +79,16 @@ var walletList = &cli.Command{ Usage: "Only print addresses", Aliases: []string{"a"}, }, + &cli.BoolFlag{ + Name: "id", + Usage: "Output ID addresses", + Aliases: []string{"i"}, + }, + &cli.BoolFlag{ + Name: "market", + Usage: "Output market balances", + Aliases: []string{"m"}, + }, }, Action: func(cctx *cli.Context) error { api, closer, err := GetFullNodeAPI(cctx) @@ -98,7 +108,10 @@ var walletList = &cli.Command{ tw := tablewriter.New( tablewriter.Col("Address"), + tablewriter.Col("ID"), tablewriter.Col("Balance"), + tablewriter.Col("Market(Avail)"), + tablewriter.Col("Market(Locked)"), tablewriter.Col("Nonce"), tablewriter.Col("Default"), tablewriter.NewLineCol("Error")) @@ -131,6 +144,23 @@ var walletList = &cli.Command{ row["Default"] = "X" } + if cctx.Bool("id") { + id, err := api.StateLookupID(ctx, addr, types.EmptyTSK) + if err != nil { + row["ID"] = "n/a" + } else { + row["ID"] = id + } + } + + if cctx.Bool("market") { + mbal, err := api.StateMarketBalance(ctx, addr, types.EmptyTSK) + if err == nil { + row["Market(Avail)"] = types.FIL(types.BigSub(mbal.Escrow, mbal.Locked)) + row["Market(Locked)"] = types.FIL(mbal.Locked) + } + } + tw.Write(row) } } From 92942d44d122af3d25f14130d0671db3f6d17e8b Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Fri, 9 Oct 2020 13:41:09 +0200 Subject: [PATCH 868/885] feat: lite-mode - market storage and retrieval clients --- api/api_gateway.go | 12 ++++ api/apistruct/struct.go | 85 ++++++++++++++++++----- api/test/ccupgrade.go | 2 +- api/test/deals.go | 29 +++++--- chain/stmgr/stmgr.go | 2 + chain/stmgr/utils.go | 19 ----- cmd/lotus-gateway/api.go | 107 ++++++++++++++++++++++------- cmd/lotus-gateway/api_test.go | 39 +++++++++-- cmd/lotus-gateway/endtoend_test.go | 86 +++++++++++++++++++---- markets/storageadapter/client.go | 24 +++---- node/builder.go | 1 - node/impl/full/chain.go | 17 +++-- node/impl/full/state.go | 82 +++++++++++++--------- node/impl/paych/paych.go | 5 -- node/modules/rpcstatemanager.go | 31 ++++++++- paychmgr/manager.go | 11 +-- 16 files changed, 394 insertions(+), 158 deletions(-) diff --git a/api/api_gateway.go b/api/api_gateway.go index c99ff8408..2b7b52b43 100644 --- a/api/api_gateway.go +++ b/api/api_gateway.go @@ -5,6 +5,8 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/network" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" "github.com/ipfs/go-cid" ) @@ -12,15 +14,25 @@ import ( type GatewayAPI interface { ChainHasObj(context.Context, cid.Cid) (bool, error) ChainHead(ctx context.Context) (*types.TipSet, error) + ChainGetBlockMessages(context.Context, cid.Cid) (*BlockMessages, error) + ChainGetMessage(ctx context.Context, mc cid.Cid) (*types.Message, error) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) + ChainNotify(context.Context) (<-chan []*HeadChange, error) ChainReadObj(context.Context, cid.Cid) ([]byte, error) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (DealCollateralBounds, error) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) + StateGetReceipt(context.Context, cid.Cid, types.TipSetKey) (*types.MessageReceipt, error) + StateListMiners(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + StateMarketBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (MarketBalance, error) + StateMarketStorageDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*MarketDeal, error) + StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (miner.MinerInfo, error) + StateNetworkVersion(context.Context, types.TipSetKey) (network.Version, error) StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*MsgLookup, error) } diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index e8e4ee33c..6811eff98 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -374,20 +374,29 @@ type WorkerStruct struct { type GatewayStruct struct { Internal struct { - // TODO: does the gateway need perms? - ChainHasObj func(context.Context, cid.Cid) (bool, error) - ChainGetTipSet func(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) - ChainGetTipSetByHeight func(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) - ChainHead func(ctx context.Context) (*types.TipSet, error) - ChainReadObj func(context.Context, cid.Cid) ([]byte, error) - GasEstimateMessageGas func(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) - MpoolPush func(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) - MsigGetAvailableBalance func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) - MsigGetVested func(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error) - StateAccountKey func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) - StateGetActor func(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) - StateLookupID func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) - StateWaitMsg func(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error) + ChainGetBlockMessages func(ctx context.Context, c cid.Cid) (*api.BlockMessages, error) + ChainGetMessage func(ctx context.Context, mc cid.Cid) (*types.Message, error) + ChainGetTipSet func(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) + ChainGetTipSetByHeight func(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) + ChainHasObj func(context.Context, cid.Cid) (bool, error) + ChainHead func(ctx context.Context) (*types.TipSet, error) + ChainNotify func(ctx context.Context) (<-chan []*api.HeadChange, error) + ChainReadObj func(context.Context, cid.Cid) ([]byte, error) + GasEstimateMessageGas func(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) + MpoolPush func(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) + MsigGetAvailableBalance func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) + MsigGetVested func(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error) + StateAccountKey func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + StateDealProviderCollateralBounds func(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (api.DealCollateralBounds, error) + StateGetActor func(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) + StateGetReceipt func(ctx context.Context, c cid.Cid, tsk types.TipSetKey) (*types.MessageReceipt, error) + StateLookupID func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + StateListMiners func(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error) + StateMinerInfo func(ctx context.Context, actor address.Address, tsk types.TipSetKey) (miner.MinerInfo, error) + StateMarketBalance func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (api.MarketBalance, error) + StateMarketStorageDeal func(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*api.MarketDeal, error) + StateNetworkVersion func(ctx context.Context, tsk types.TipSetKey) (stnetwork.Version, error) + StateWaitMsg func(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error) } } @@ -1450,12 +1459,12 @@ func (w *WorkerStruct) Closing(ctx context.Context) (<-chan struct{}, error) { return w.Internal.Closing(ctx) } -func (g GatewayStruct) ChainHasObj(ctx context.Context, c cid.Cid) (bool, error) { - return g.Internal.ChainHasObj(ctx, c) +func (g GatewayStruct) ChainGetBlockMessages(ctx context.Context, c cid.Cid) (*api.BlockMessages, error) { + return g.Internal.ChainGetBlockMessages(ctx, c) } -func (g GatewayStruct) ChainHead(ctx context.Context) (*types.TipSet, error) { - return g.Internal.ChainHead(ctx) +func (g GatewayStruct) ChainGetMessage(ctx context.Context, mc cid.Cid) (*types.Message, error) { + return g.Internal.ChainGetMessage(ctx, mc) } func (g GatewayStruct) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) { @@ -1466,6 +1475,18 @@ func (g GatewayStruct) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEp return g.Internal.ChainGetTipSetByHeight(ctx, h, tsk) } +func (g GatewayStruct) ChainHasObj(ctx context.Context, c cid.Cid) (bool, error) { + return g.Internal.ChainHasObj(ctx, c) +} + +func (g GatewayStruct) ChainHead(ctx context.Context) (*types.TipSet, error) { + return g.Internal.ChainHead(ctx) +} + +func (g GatewayStruct) ChainNotify(ctx context.Context) (<-chan []*api.HeadChange, error) { + return g.Internal.ChainNotify(ctx) +} + func (g GatewayStruct) ChainReadObj(ctx context.Context, c cid.Cid) ([]byte, error) { return g.Internal.ChainReadObj(ctx, c) } @@ -1490,14 +1511,42 @@ func (g GatewayStruct) StateAccountKey(ctx context.Context, addr address.Address return g.Internal.StateAccountKey(ctx, addr, tsk) } +func (g GatewayStruct) StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (api.DealCollateralBounds, error) { + return g.Internal.StateDealProviderCollateralBounds(ctx, size, verified, tsk) +} + func (g GatewayStruct) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) { return g.Internal.StateGetActor(ctx, actor, ts) } +func (g GatewayStruct) StateGetReceipt(ctx context.Context, c cid.Cid, tsk types.TipSetKey) (*types.MessageReceipt, error) { + return g.Internal.StateGetReceipt(ctx, c, tsk) +} + func (g GatewayStruct) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { return g.Internal.StateLookupID(ctx, addr, tsk) } +func (g GatewayStruct) StateListMiners(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error) { + return g.Internal.StateListMiners(ctx, tsk) +} + +func (g GatewayStruct) StateMarketBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (api.MarketBalance, error) { + return g.Internal.StateMarketBalance(ctx, addr, tsk) +} + +func (g GatewayStruct) StateMarketStorageDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*api.MarketDeal, error) { + return g.Internal.StateMarketStorageDeal(ctx, dealId, tsk) +} + +func (g GatewayStruct) StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (miner.MinerInfo, error) { + return g.Internal.StateMinerInfo(ctx, actor, tsk) +} + +func (g GatewayStruct) StateNetworkVersion(ctx context.Context, tsk types.TipSetKey) (stnetwork.Version, error) { + return g.Internal.StateNetworkVersion(ctx, tsk) +} + func (g GatewayStruct) StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error) { return g.Internal.StateWaitMsg(ctx, msg, confidence) } diff --git a/api/test/ccupgrade.go b/api/test/ccupgrade.go index e8010bdd4..75f72d861 100644 --- a/api/test/ccupgrade.go +++ b/api/test/ccupgrade.go @@ -89,7 +89,7 @@ func testCCUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration, upgradeH t.Fatal(err) } - makeDeal(t, ctx, 6, client, miner, false, false) + MakeDeal(t, ctx, 6, client, miner, false, false) // Validate upgrade diff --git a/api/test/deals.go b/api/test/deals.go index ed856445b..2317131f8 100644 --- a/api/test/deals.go +++ b/api/test/deals.go @@ -60,7 +60,7 @@ func TestDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration, carExport } }() - makeDeal(t, ctx, 6, client, miner, carExport, fastRet) + MakeDeal(t, ctx, 6, client, miner, carExport, fastRet) atomic.AddInt64(&mine, -1) fmt.Println("shutting down mining") @@ -97,24 +97,35 @@ func TestDoubleDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration) { } }() - makeDeal(t, ctx, 6, client, miner, false, false) - makeDeal(t, ctx, 7, client, miner, false, false) + MakeDeal(t, ctx, 6, client, miner, false, false) + MakeDeal(t, ctx, 7, client, miner, false, false) atomic.AddInt64(&mine, -1) fmt.Println("shutting down mining") <-done } -func makeDeal(t *testing.T, ctx context.Context, rseed int, client *impl.FullNodeAPI, miner TestStorageNode, carExport, fastRet bool) { +func MakeDeal(t *testing.T, ctx context.Context, rseed int, client api.FullNode, miner TestStorageNode, carExport, fastRet bool) { data := make([]byte, 1600) rand.New(rand.NewSource(int64(rseed))).Read(data) - r := bytes.NewReader(data) - fcid, err := client.ClientImportLocal(ctx, r) + dir, err := ioutil.TempDir(os.TempDir(), "test-make-deal-") if err != nil { t.Fatal(err) } + path := filepath.Join(dir, "sourcefile.dat") + err = ioutil.WriteFile(path, data, 0644) + if err != nil { + t.Fatal(err) + } + + res, err := client.ClientImport(ctx, api.FileRef{Path: path}) + if err != nil { + t.Fatal(err) + } + + fcid := res.Root fmt.Println("FILE CID: ", fcid) deal := startDeal(t, ctx, miner, client, fcid, fastRet) @@ -259,7 +270,7 @@ func TestSenondDealRetrieval(t *testing.T, b APIBuilder, blocktime time.Duration <-done } -func startDeal(t *testing.T, ctx context.Context, miner TestStorageNode, client *impl.FullNodeAPI, fcid cid.Cid, fastRet bool) *cid.Cid { +func startDeal(t *testing.T, ctx context.Context, miner TestStorageNode, client api.FullNode, fcid cid.Cid, fastRet bool) *cid.Cid { maddr, err := miner.ActorAddress(ctx) if err != nil { t.Fatal(err) @@ -286,7 +297,7 @@ func startDeal(t *testing.T, ctx context.Context, miner TestStorageNode, client return deal } -func waitDealSealed(t *testing.T, ctx context.Context, miner TestStorageNode, client *impl.FullNodeAPI, deal *cid.Cid, noseal bool) { +func waitDealSealed(t *testing.T, ctx context.Context, miner TestStorageNode, client api.FullNode, deal *cid.Cid, noseal bool) { loop: for { di, err := client.ClientGetDealInfo(ctx, *deal) @@ -359,7 +370,7 @@ func startSealingWaiting(t *testing.T, ctx context.Context, miner TestStorageNod } } -func testRetrieval(t *testing.T, ctx context.Context, client *impl.FullNodeAPI, fcid cid.Cid, piece *cid.Cid, carExport bool, data []byte) { +func testRetrieval(t *testing.T, ctx context.Context, client api.FullNode, fcid cid.Cid, piece *cid.Cid, carExport bool, data []byte) { offers, err := client.ClientFindData(ctx, fcid, piece) if err != nil { t.Fatal(err) diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index 4f0c26b7c..2822344b0 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -46,6 +46,8 @@ const LookbackNoLimit = abi.ChainEpoch(-1) var log = logging.Logger("statemgr") type StateManagerAPI interface { + Call(ctx context.Context, msg *types.Message, ts *types.TipSet) (*api.InvocResult, error) + GetPaychState(ctx context.Context, addr address.Address, ts *types.TipSet) (*types.Actor, paych.State, error) LoadActorTsk(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*types.Actor, error) LookupID(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) ResolveToKeyAddress(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index 7c744f60f..54f75c138 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -255,25 +255,6 @@ func GetSectorsForWinningPoSt(ctx context.Context, pv ffiwrapper.Verifier, sm *S return out, nil } -func StateMinerInfo(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) (*miner.MinerInfo, error) { - act, err := sm.LoadActor(ctx, maddr, ts) - if err != nil { - return nil, xerrors.Errorf("failed to load miner actor: %w", err) - } - - mas, err := miner.Load(sm.cs.Store(ctx), act) - if err != nil { - return nil, xerrors.Errorf("failed to load miner actor state: %w", err) - } - - mi, err := mas.Info() - if err != nil { - return nil, err - } - - return &mi, err -} - func GetMinerSlashed(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) (bool, error) { act, err := sm.LoadActor(ctx, power.Address, ts) if err != nil { diff --git a/cmd/lotus-gateway/api.go b/cmd/lotus-gateway/api.go index 7fc004327..0ebb99298 100644 --- a/cmd/lotus-gateway/api.go +++ b/cmd/lotus-gateway/api.go @@ -24,7 +24,7 @@ import ( const ( LookbackCap = time.Hour * 24 - stateWaitLookbackLimit = abi.ChainEpoch(20) + StateWaitLookbackLimit = abi.ChainEpoch(20) ) var ( @@ -35,20 +35,28 @@ var ( // (to make it easy to mock for tests) type gatewayDepsAPI interface { Version(context.Context) (api.Version, error) - - ChainHasObj(context.Context, cid.Cid) (bool, error) - ChainHead(ctx context.Context) (*types.TipSet, error) + ChainGetBlockMessages(context.Context, cid.Cid) (*api.BlockMessages, error) + ChainGetMessage(ctx context.Context, mc cid.Cid) (*types.Message, error) + ChainGetNode(ctx context.Context, p string) (*api.IpldObject, error) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) + ChainHasObj(context.Context, cid.Cid) (bool, error) + ChainHead(ctx context.Context) (*types.TipSet, error) + ChainNotify(context.Context) (<-chan []*api.HeadChange, error) ChainReadObj(context.Context, cid.Cid) ([]byte, error) - ChainGetNode(ctx context.Context, p string) (*api.IpldObject, error) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) MpoolPushUntrusted(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (api.DealCollateralBounds, error) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) + StateGetReceipt(context.Context, cid.Cid, types.TipSetKey) (*types.MessageReceipt, error) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + StateListMiners(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error) + StateMarketBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (api.MarketBalance, error) + StateMarketStorageDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*api.MarketDeal, error) + StateNetworkVersion(context.Context, types.TipSetKey) (network.Version, error) StateWaitMsgLimited(ctx context.Context, msg cid.Cid, confidence uint64, h abi.ChainEpoch) (*api.MsgLookup, error) StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*api.ActorState, error) StateMinerPower(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error) @@ -60,22 +68,22 @@ type gatewayDepsAPI interface { StateMinerProvingDeadline(context.Context, address.Address, types.TipSetKey) (*dline.Info, error) StateCirculatingSupply(context.Context, types.TipSetKey) (abi.TokenAmount, error) StateVMCirculatingSupplyInternal(context.Context, types.TipSetKey) (api.CirculatingSupply, error) - StateNetworkVersion(context.Context, types.TipSetKey) (network.Version, error) } type GatewayAPI struct { - api gatewayDepsAPI - lookbackCap time.Duration + api gatewayDepsAPI + lookbackCap time.Duration + stateWaitLookbackLimit abi.ChainEpoch } // NewGatewayAPI creates a new GatewayAPI with the default lookback cap func NewGatewayAPI(api gatewayDepsAPI) *GatewayAPI { - return newGatewayAPI(api, LookbackCap) + return newGatewayAPI(api, LookbackCap, StateWaitLookbackLimit) } // used by the tests -func newGatewayAPI(api gatewayDepsAPI, lookbackCap time.Duration) *GatewayAPI { - return &GatewayAPI{api: api, lookbackCap: lookbackCap} +func newGatewayAPI(api gatewayDepsAPI, lookbackCap time.Duration, stateWaitLookbackLimit abi.ChainEpoch) *GatewayAPI { + return &GatewayAPI{api: api, lookbackCap: lookbackCap, stateWaitLookbackLimit: stateWaitLookbackLimit} } func (a *GatewayAPI) checkTipsetKey(ctx context.Context, tsk types.TipSetKey) error { @@ -122,6 +130,10 @@ func (a *GatewayAPI) Version(ctx context.Context) (api.Version, error) { return a.api.Version(ctx) } +func (a *GatewayAPI) ChainGetBlockMessages(ctx context.Context, c cid.Cid) (*api.BlockMessages, error) { + return a.api.ChainGetBlockMessages(ctx, c) +} + func (a *GatewayAPI) ChainHasObj(ctx context.Context, c cid.Cid) (bool, error) { return a.api.ChainHasObj(ctx, c) } @@ -132,6 +144,10 @@ func (a *GatewayAPI) ChainHead(ctx context.Context) (*types.TipSet, error) { return a.api.ChainHead(ctx) } +func (a *GatewayAPI) ChainGetMessage(ctx context.Context, mc cid.Cid) (*types.Message, error) { + return a.api.ChainGetMessage(ctx, mc) +} + func (a *GatewayAPI) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) { return a.api.ChainGetTipSet(ctx, tsk) } @@ -165,14 +181,18 @@ func (a *GatewayAPI) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoc return a.api.ChainGetTipSetByHeight(ctx, h, tsk) } -func (a *GatewayAPI) ChainReadObj(ctx context.Context, c cid.Cid) ([]byte, error) { - return a.api.ChainReadObj(ctx, c) -} - func (a *GatewayAPI) ChainGetNode(ctx context.Context, p string) (*api.IpldObject, error) { return a.api.ChainGetNode(ctx, p) } +func (a *GatewayAPI) ChainNotify(ctx context.Context) (<-chan []*api.HeadChange, error) { + return a.api.ChainNotify(ctx) +} + +func (a *GatewayAPI) ChainReadObj(ctx context.Context, c cid.Cid) ([]byte, error) { + return a.api.ChainReadObj(ctx, c) +} + func (a *GatewayAPI) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) { if err := a.checkTipsetKey(ctx, tsk); err != nil { return nil, err @@ -213,6 +233,14 @@ func (a *GatewayAPI) StateAccountKey(ctx context.Context, addr address.Address, return a.api.StateAccountKey(ctx, addr, tsk) } +func (a *GatewayAPI) StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (api.DealCollateralBounds, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return api.DealCollateralBounds{}, err + } + + return a.api.StateDealProviderCollateralBounds(ctx, size, verified, tsk) +} + func (a *GatewayAPI) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) { if err := a.checkTipsetKey(ctx, tsk); err != nil { return nil, err @@ -221,6 +249,22 @@ func (a *GatewayAPI) StateGetActor(ctx context.Context, actor address.Address, t return a.api.StateGetActor(ctx, actor, tsk) } +func (a *GatewayAPI) StateGetReceipt(ctx context.Context, c cid.Cid, tsk types.TipSetKey) (*types.MessageReceipt, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return nil, err + } + + return a.api.StateGetReceipt(ctx, c, tsk) +} + +func (a *GatewayAPI) StateListMiners(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return nil, err + } + + return a.api.StateListMiners(ctx, tsk) +} + func (a *GatewayAPI) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) { if err := a.checkTipsetKey(ctx, tsk); err != nil { return address.Undef, err @@ -229,8 +273,32 @@ func (a *GatewayAPI) StateLookupID(ctx context.Context, addr address.Address, ts return a.api.StateLookupID(ctx, addr, tsk) } +func (a *GatewayAPI) StateMarketBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (api.MarketBalance, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return api.MarketBalance{}, err + } + + return a.api.StateMarketBalance(ctx, addr, tsk) +} + +func (a *GatewayAPI) StateMarketStorageDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*api.MarketDeal, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return nil, err + } + + return a.api.StateMarketStorageDeal(ctx, dealId, tsk) +} + +func (a *GatewayAPI) StateNetworkVersion(ctx context.Context, tsk types.TipSetKey) (network.Version, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return network.VersionMax, err + } + + return a.api.StateNetworkVersion(ctx, tsk) +} + func (a *GatewayAPI) StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error) { - return a.api.StateWaitMsgLimited(ctx, msg, confidence, stateWaitLookbackLimit) + return a.api.StateWaitMsgLimited(ctx, msg, confidence, a.stateWaitLookbackLimit) } func (a *GatewayAPI) StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*api.ActorState, error) { @@ -303,13 +371,6 @@ func (a *GatewayAPI) StateVMCirculatingSupplyInternal(ctx context.Context, tsk t return a.api.StateVMCirculatingSupplyInternal(ctx, tsk) } -func (a *GatewayAPI) StateNetworkVersion(ctx context.Context, tsk types.TipSetKey) (network.Version, error) { - if err := a.checkTipsetKey(ctx, tsk); err != nil { - return 0, err - } - return a.api.StateNetworkVersion(ctx, tsk) -} - func (a *GatewayAPI) WalletVerify(ctx context.Context, k address.Address, msg []byte, sig *crypto.Signature) (bool, error) { return sigs.Verify(sig, k, msg) == nil, nil } diff --git a/cmd/lotus-gateway/api_test.go b/cmd/lotus-gateway/api_test.go index ae161945b..23d2cbf3a 100644 --- a/cmd/lotus-gateway/api_test.go +++ b/cmd/lotus-gateway/api_test.go @@ -6,6 +6,9 @@ import ( "testing" "time" + "github.com/filecoin-project/go-state-types/network" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + "github.com/filecoin-project/lotus/build" "github.com/stretchr/testify/require" @@ -116,6 +119,38 @@ func (m *mockGatewayDepsAPI) ChainHasObj(context.Context, cid.Cid) (bool, error) panic("implement me") } +func (m *mockGatewayDepsAPI) ChainGetMessage(ctx context.Context, mc cid.Cid) (*types.Message, error) { + panic("implement me") +} + +func (m *mockGatewayDepsAPI) ChainReadObj(ctx context.Context, c cid.Cid) ([]byte, error) { + panic("implement me") +} + +func (m *mockGatewayDepsAPI) StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (api.DealCollateralBounds, error) { + panic("implement me") +} + +func (m *mockGatewayDepsAPI) StateListMiners(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error) { + panic("implement me") +} + +func (m *mockGatewayDepsAPI) StateMarketBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (api.MarketBalance, error) { + panic("implement me") +} + +func (m *mockGatewayDepsAPI) StateMarketStorageDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*api.MarketDeal, error) { + panic("implement me") +} + +func (m *mockGatewayDepsAPI) StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (miner.MinerInfo, error) { + panic("implement me") +} + +func (m *mockGatewayDepsAPI) StateNetworkVersion(ctx context.Context, key types.TipSetKey) (network.Version, error) { + panic("implement me") +} + func (m *mockGatewayDepsAPI) ChainHead(ctx context.Context) (*types.TipSet, error) { m.lk.RLock() defer m.lk.RUnlock() @@ -165,10 +200,6 @@ func (m *mockGatewayDepsAPI) ChainGetTipSetByHeight(ctx context.Context, h abi.C return m.tipsets[h], nil } -func (m *mockGatewayDepsAPI) ChainReadObj(ctx context.Context, c cid.Cid) ([]byte, error) { - panic("implement me") -} - func (m *mockGatewayDepsAPI) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) { panic("implement me") } diff --git a/cmd/lotus-gateway/endtoend_test.go b/cmd/lotus-gateway/endtoend_test.go index b043ce28c..c5b513f44 100644 --- a/cmd/lotus-gateway/endtoend_test.go +++ b/cmd/lotus-gateway/endtoend_test.go @@ -11,6 +11,7 @@ import ( "github.com/filecoin-project/lotus/cli" clitest "github.com/filecoin-project/lotus/cli/test" + logging "github.com/ipfs/go-log/v2" init2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/init" multisig2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/multisig" @@ -25,12 +26,14 @@ import ( "github.com/filecoin-project/lotus/api/client" "github.com/filecoin-project/lotus/api/test" "github.com/filecoin-project/lotus/chain/actors/policy" + "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node" builder "github.com/filecoin-project/lotus/node/test" ) const maxLookbackCap = time.Duration(math.MaxInt64) +const maxStateWaitLookbackLimit = stmgr.LookbackNoLimit func init() { policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg2KiBV1) @@ -38,15 +41,18 @@ func init() { policy.SetMinVerifiedDealSize(abi.NewStoragePower(256)) } -// TestEndToEndWalletMsig tests that wallet and msig API calls can be made -// on a lite node that is connected through a gateway to a full API node -func TestEndToEndWalletMsig(t *testing.T) { +// TestWalletMsig tests that API calls to wallet and msig can be made on a lite +// node that is connected through a gateway to a full API node +func TestWalletMsig(t *testing.T) { _ = os.Setenv("BELLMAN_NO_GPU", "1") blocktime := 5 * time.Millisecond ctx := context.Background() - full, lite, closer := startNodes(ctx, t, blocktime, maxLookbackCap) - defer closer() + nodes := startNodes(ctx, t, blocktime, maxLookbackCap, maxStateWaitLookbackLimit) + defer nodes.closer() + + lite := nodes.lite + full := nodes.full // The full node starts with a wallet fullWalletAddr, err := full.WalletDefaultAddress(ctx) @@ -141,16 +147,19 @@ func TestEndToEndWalletMsig(t *testing.T) { require.True(t, approveReturn.Applied) } -// TestEndToEndMsigCLI tests that msig CLI calls can be made +// TestMsigCLI tests that msig CLI calls can be made // on a lite node that is connected through a gateway to a full API node -func TestEndToEndMsigCLI(t *testing.T) { +func TestMsigCLI(t *testing.T) { _ = os.Setenv("BELLMAN_NO_GPU", "1") clitest.QuietMiningLogs() blocktime := 5 * time.Millisecond ctx := context.Background() - full, lite, closer := startNodes(ctx, t, blocktime, maxLookbackCap) - defer closer() + nodes := startNodes(ctx, t, blocktime, maxLookbackCap, maxStateWaitLookbackLimit) + defer nodes.closer() + + lite := nodes.lite + full := nodes.full // The full node starts with a wallet fullWalletAddr, err := full.WalletDefaultAddress(ctx) @@ -190,7 +199,52 @@ func sendFunds(ctx context.Context, fromNode test.TestNode, fromAddr address.Add return nil } -func startNodes(ctx context.Context, t *testing.T, blocktime time.Duration, lookbackCap time.Duration) (test.TestNode, test.TestNode, jsonrpc.ClientCloser) { +func TestDealFlow(t *testing.T) { + _ = os.Setenv("BELLMAN_NO_GPU", "1") + + logging.SetLogLevel("miner", "ERROR") + logging.SetLogLevel("chainstore", "ERROR") + logging.SetLogLevel("chain", "ERROR") + logging.SetLogLevel("sub", "ERROR") + logging.SetLogLevel("storageminer", "ERROR") + + blocktime := 5 * time.Millisecond + ctx := context.Background() + nodes := startNodes(ctx, t, blocktime, maxLookbackCap, maxStateWaitLookbackLimit) + defer nodes.closer() + + full := nodes.full + lite := nodes.lite + + // The full node starts with a wallet + fullWalletAddr, err := full.WalletDefaultAddress(ctx) + require.NoError(t, err) + + // Create a wallet on the lite node + liteWalletAddr, err := lite.WalletNew(ctx, types.KTSecp256k1) + require.NoError(t, err) + + // Send some funds from the full node to the lite node + err = sendFunds(ctx, full, fullWalletAddr, liteWalletAddr, types.NewInt(1e18)) + require.NoError(t, err) + + test.MakeDeal(t, ctx, 6, lite, nodes.miner, false, false) +} + +type testNodes struct { + lite test.TestNode + full test.TestNode + miner test.TestStorageNode + closer jsonrpc.ClientCloser +} + +func startNodes( + ctx context.Context, + t *testing.T, + blocktime time.Duration, + lookbackCap time.Duration, + stateWaitLookbackLimit abi.ChainEpoch, +) *testNodes { var closer jsonrpc.ClientCloser // Create one miner and two full nodes. @@ -207,7 +261,8 @@ func startNodes(ctx context.Context, t *testing.T, blocktime time.Duration, look fullNode := nodes[0] // Create a gateway server in front of the full node - _, addr, err := builder.CreateRPCServer(newGatewayAPI(fullNode, lookbackCap)) + gapiImpl := newGatewayAPI(fullNode, lookbackCap, stateWaitLookbackLimit) + _, addr, err := builder.CreateRPCServer(gapiImpl) require.NoError(t, err) // Create a gateway client API that connects to the gateway server @@ -234,9 +289,16 @@ func startNodes(ctx context.Context, t *testing.T, blocktime time.Duration, look err = miner.NetConnect(ctx, fullAddr) require.NoError(t, err) + // Connect the miner and the lite node (so that the lite node can send + // data to the miner) + liteAddr, err := lite.NetAddrsListen(ctx) + require.NoError(t, err) + err = miner.NetConnect(ctx, liteAddr) + require.NoError(t, err) + // Start mining blocks bm := test.NewBlockMiner(ctx, t, miner, blocktime) bm.MineBlocks() - return full, lite, closer + return &testNodes{lite: lite, full: full, miner: miner, closer: closer} } diff --git a/markets/storageadapter/client.go b/markets/storageadapter/client.go index ad2b2df8a..f299dd4d5 100644 --- a/markets/storageadapter/client.go +++ b/markets/storageadapter/client.go @@ -27,8 +27,6 @@ import ( "github.com/filecoin-project/lotus/chain/events" "github.com/filecoin-project/lotus/chain/events/state" "github.com/filecoin-project/lotus/chain/market" - "github.com/filecoin-project/lotus/chain/stmgr" - "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/sigs" "github.com/filecoin-project/lotus/markets/utils" @@ -40,8 +38,6 @@ type ClientNodeAdapter struct { full.ChainAPI full.MpoolAPI - sm *stmgr.StateManager - cs *store.ChainStore fm *market.FundMgr ev *events.Events } @@ -51,14 +47,12 @@ type clientApi struct { full.StateAPI } -func NewClientNodeAdapter(state full.StateAPI, chain full.ChainAPI, mpool full.MpoolAPI, sm *stmgr.StateManager, cs *store.ChainStore, fm *market.FundMgr) storagemarket.StorageClientNode { +func NewClientNodeAdapter(state full.StateAPI, chain full.ChainAPI, mpool full.MpoolAPI, fm *market.FundMgr) storagemarket.StorageClientNode { return &ClientNodeAdapter{ StateAPI: state, ChainAPI: chain, MpoolAPI: mpool, - sm: sm, - cs: cs, fm: fm, ev: events.NewEvents(context.TODO(), &clientApi{chain, state}), } @@ -138,12 +132,12 @@ func (c *ClientNodeAdapter) GetBalance(ctx context.Context, addr address.Address func (c *ClientNodeAdapter) ValidatePublishedDeal(ctx context.Context, deal storagemarket.ClientDeal) (abi.DealID, error) { log.Infow("DEAL ACCEPTED!") - pubmsg, err := c.cs.GetMessage(*deal.PublishMessage) + pubmsg, err := c.ChainGetMessage(ctx, *deal.PublishMessage) if err != nil { return 0, xerrors.Errorf("getting deal publish message: %w", err) } - mi, err := stmgr.StateMinerInfo(ctx, c.sm, c.cs.GetHeaviestTipSet(), deal.Proposal.Provider) + mi, err := c.StateMinerInfo(ctx, deal.Proposal.Provider, types.EmptyTSK) if err != nil { return 0, xerrors.Errorf("getting miner worker failed: %w", err) } @@ -189,16 +183,16 @@ func (c *ClientNodeAdapter) ValidatePublishedDeal(ctx context.Context, deal stor } // TODO: timeout - _, ret, _, err := c.sm.WaitForMessage(ctx, *deal.PublishMessage, build.MessageConfidence, stmgr.LookbackNoLimit) + ret, err := c.StateWaitMsg(ctx, *deal.PublishMessage, build.MessageConfidence) if err != nil { return 0, xerrors.Errorf("waiting for deal publish message: %w", err) } - if ret.ExitCode != 0 { - return 0, xerrors.Errorf("deal publish failed: exit=%d", ret.ExitCode) + if ret.Receipt.ExitCode != 0 { + return 0, xerrors.Errorf("deal publish failed: exit=%d", ret.Receipt.ExitCode) } var res market2.PublishStorageDealsReturn - if err := res.UnmarshalCBOR(bytes.NewReader(ret.Return)); err != nil { + if err := res.UnmarshalCBOR(bytes.NewReader(ret.Receipt.Return)); err != nil { return 0, err } @@ -218,7 +212,7 @@ func (c *ClientNodeAdapter) DealProviderCollateralBounds(ctx context.Context, si func (c *ClientNodeAdapter) OnDealSectorCommitted(ctx context.Context, provider address.Address, dealId abi.DealID, cb storagemarket.DealSectorCommittedCallback) error { checkFunc := func(ts *types.TipSet) (done bool, more bool, err error) { - sd, err := stmgr.GetStorageDeal(ctx, c.StateManager, dealId, ts) + sd, err := c.StateMarketStorageDeal(ctx, dealId, ts.Key()) if err != nil { // TODO: This may be fine for some errors @@ -245,7 +239,7 @@ func (c *ClientNodeAdapter) OnDealSectorCommitted(ctx context.Context, provider return false, nil } - sd, err := stmgr.GetStorageDeal(ctx, c.StateManager, dealId, ts) + sd, err := c.StateMarketStorageDeal(ctx, dealId, ts.Key()) if err != nil { return false, xerrors.Errorf("failed to look up deal on chain: %w", err) } diff --git a/node/builder.go b/node/builder.go index 1dc12e80a..1d632c35c 100644 --- a/node/builder.go +++ b/node/builder.go @@ -250,7 +250,6 @@ func Online() Option { Override(new(*store.ChainStore), modules.ChainStore), Override(new(stmgr.UpgradeSchedule), stmgr.DefaultUpgradeSchedule()), Override(new(*stmgr.StateManager), stmgr.NewStateManagerWithUpgradeSchedule), - Override(new(stmgr.StateManagerAPI), From(new(*stmgr.StateManager))), Override(new(*wallet.LocalWallet), wallet.NewWallet), Override(new(wallet.Default), From(new(*wallet.LocalWallet))), Override(new(api.WalletAPI), From(new(wallet.MultiWallet))), diff --git a/node/impl/full/chain.go b/node/impl/full/chain.go index 095b51ed8..a3410b8db 100644 --- a/node/impl/full/chain.go +++ b/node/impl/full/chain.go @@ -40,8 +40,11 @@ import ( var log = logging.Logger("fullnode") type ChainModuleAPI interface { + ChainNotify(context.Context) (<-chan []*api.HeadChange, error) + ChainGetBlockMessages(context.Context, cid.Cid) (*api.BlockMessages, error) ChainHasObj(context.Context, cid.Cid) (bool, error) ChainHead(context.Context) (*types.TipSet, error) + ChainGetMessage(ctx context.Context, mc cid.Cid) (*types.Message, error) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) ChainReadObj(context.Context, cid.Cid) ([]byte, error) @@ -67,8 +70,8 @@ type ChainAPI struct { Chain *store.ChainStore } -func (a *ChainAPI) ChainNotify(ctx context.Context) (<-chan []*api.HeadChange, error) { - return a.Chain.SubHeadChanges(ctx), nil +func (m *ChainModule) ChainNotify(ctx context.Context) (<-chan []*api.HeadChange, error) { + return m.Chain.SubHeadChanges(ctx), nil } func (m *ChainModule) ChainHead(context.Context) (*types.TipSet, error) { @@ -101,13 +104,13 @@ func (m *ChainModule) ChainGetTipSet(ctx context.Context, key types.TipSetKey) ( return m.Chain.LoadTipSet(key) } -func (a *ChainAPI) ChainGetBlockMessages(ctx context.Context, msg cid.Cid) (*api.BlockMessages, error) { - b, err := a.Chain.GetBlock(msg) +func (m *ChainModule) ChainGetBlockMessages(ctx context.Context, msg cid.Cid) (*api.BlockMessages, error) { + b, err := m.Chain.GetBlock(msg) if err != nil { return nil, err } - bmsgs, smsgs, err := a.Chain.MessagesForBlock(b) + bmsgs, smsgs, err := m.Chain.MessagesForBlock(b) if err != nil { return nil, err } @@ -532,8 +535,8 @@ func (a *ChainAPI) ChainGetNode(ctx context.Context, p string) (*api.IpldObject, }, nil } -func (a *ChainAPI) ChainGetMessage(ctx context.Context, mc cid.Cid) (*types.Message, error) { - cm, err := a.Chain.GetCMessage(mc) +func (m *ChainModule) ChainGetMessage(ctx context.Context, mc cid.Cid) (*types.Message, error) { + cm, err := m.Chain.GetCMessage(mc) if err != nil { return nil, err } diff --git a/node/impl/full/state.go b/node/impl/full/state.go index a6564296b..75ad03e7b 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -40,12 +40,19 @@ import ( ) type StateModuleAPI interface { - StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) - StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) - StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) - StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error) MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error) + StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (api.DealCollateralBounds, error) + StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) + StateGetReceipt(context.Context, cid.Cid, types.TipSetKey) (*types.MessageReceipt, error) + StateListMiners(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error) + StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) + StateMarketBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (api.MarketBalance, error) + StateMarketStorageDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*api.MarketDeal, error) + StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (miner.MinerInfo, error) + StateNetworkVersion(ctx context.Context, key types.TipSetKey) (network.Version, error) + StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error) } // StateModule provides a default implementation of StateModuleAPI. @@ -112,13 +119,13 @@ func (a *StateAPI) StateMinerActiveSectors(ctx context.Context, maddr address.Ad return stmgr.GetMinerSectorSet(ctx, a.StateManager, ts, maddr, &activeSectors) } -func (a *StateAPI) StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (miner.MinerInfo, error) { - act, err := a.StateManager.LoadActorTsk(ctx, actor, tsk) +func (m *StateModule) StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (miner.MinerInfo, error) { + act, err := m.StateManager.LoadActorTsk(ctx, actor, tsk) if err != nil { return miner.MinerInfo{}, xerrors.Errorf("failed to load miner actor: %w", err) } - mas, err := miner.Load(a.StateManager.ChainStore().Store(ctx), act) + mas, err := miner.Load(m.StateManager.ChainStore().Store(ctx), act) if err != nil { return miner.MinerInfo{}, xerrors.Errorf("failed to load miner actor state: %w", err) } @@ -563,20 +570,20 @@ func (a *StateAPI) StateSearchMsg(ctx context.Context, msg cid.Cid) (*api.MsgLoo return nil, nil } -func (a *StateAPI) StateGetReceipt(ctx context.Context, msg cid.Cid, tsk types.TipSetKey) (*types.MessageReceipt, error) { - ts, err := a.Chain.GetTipSetFromKey(tsk) +func (m *StateModule) StateGetReceipt(ctx context.Context, msg cid.Cid, tsk types.TipSetKey) (*types.MessageReceipt, error) { + ts, err := m.Chain.GetTipSetFromKey(tsk) if err != nil { return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - return a.StateManager.GetReceipt(ctx, msg, ts) + return m.StateManager.GetReceipt(ctx, msg, ts) } -func (a *StateAPI) StateListMiners(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error) { - ts, err := a.Chain.GetTipSetFromKey(tsk) +func (m *StateModule) StateListMiners(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error) { + ts, err := m.Chain.GetTipSetFromKey(tsk) if err != nil { return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - return stmgr.ListMinerActors(ctx, a.StateManager, ts) + return stmgr.ListMinerActors(ctx, m.StateManager, ts) } func (a *StateAPI) StateListActors(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error) { @@ -587,12 +594,12 @@ func (a *StateAPI) StateListActors(ctx context.Context, tsk types.TipSetKey) ([] return a.StateManager.ListAllActors(ctx, ts) } -func (a *StateAPI) StateMarketBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (api.MarketBalance, error) { - ts, err := a.Chain.GetTipSetFromKey(tsk) +func (m *StateModule) StateMarketBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (api.MarketBalance, error) { + ts, err := m.Chain.GetTipSetFromKey(tsk) if err != nil { return api.MarketBalance{}, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - return a.StateManager.MarketBalance(ctx, addr, ts) + return m.StateManager.MarketBalance(ctx, addr, ts) } func (a *StateAPI) StateMarketParticipants(ctx context.Context, tsk types.TipSetKey) (map[string]api.MarketBalance, error) { @@ -676,12 +683,12 @@ func (a *StateAPI) StateMarketDeals(ctx context.Context, tsk types.TipSetKey) (m return out, nil } -func (a *StateAPI) StateMarketStorageDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*api.MarketDeal, error) { - ts, err := a.Chain.GetTipSetFromKey(tsk) +func (m *StateModule) StateMarketStorageDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*api.MarketDeal, error) { + ts, err := m.Chain.GetTipSetFromKey(tsk) if err != nil { return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - return stmgr.GetStorageDeal(ctx, a.StateManager, dealId, ts) + return stmgr.GetStorageDeal(ctx, m.StateManager, dealId, ts) } func (a *StateAPI) StateChangedActors(ctx context.Context, old cid.Cid, new cid.Cid) (map[string]types.Actor, error) { @@ -1205,33 +1212,33 @@ var dealProviderCollateralDen = types.NewInt(100) // StateDealProviderCollateralBounds returns the min and max collateral a storage provider // can issue. It takes the deal size and verified status as parameters. -func (a *StateAPI) StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (api.DealCollateralBounds, error) { - ts, err := a.Chain.GetTipSetFromKey(tsk) +func (m *StateModule) StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (api.DealCollateralBounds, error) { + ts, err := m.Chain.GetTipSetFromKey(tsk) if err != nil { return api.DealCollateralBounds{}, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - pact, err := a.StateGetActor(ctx, power.Address, tsk) + pact, err := m.StateGetActor(ctx, power.Address, tsk) if err != nil { return api.DealCollateralBounds{}, xerrors.Errorf("failed to load power actor: %w", err) } - ract, err := a.StateGetActor(ctx, reward.Address, tsk) + ract, err := m.StateGetActor(ctx, reward.Address, tsk) if err != nil { return api.DealCollateralBounds{}, xerrors.Errorf("failed to load reward actor: %w", err) } - pst, err := power.Load(a.StateManager.ChainStore().Store(ctx), pact) + pst, err := power.Load(m.StateManager.ChainStore().Store(ctx), pact) if err != nil { return api.DealCollateralBounds{}, xerrors.Errorf("failed to load power actor state: %w", err) } - rst, err := reward.Load(a.StateManager.ChainStore().Store(ctx), ract) + rst, err := reward.Load(m.StateManager.ChainStore().Store(ctx), ract) if err != nil { return api.DealCollateralBounds{}, xerrors.Errorf("failed to load reward actor state: %w", err) } - circ, err := a.StateVMCirculatingSupplyInternal(ctx, ts.Key()) + circ, err := stateVMCirculatingSupplyInternal(ctx, ts.Key(), m.Chain, m.StateManager) if err != nil { return api.DealCollateralBounds{}, xerrors.Errorf("getting total circulating supply: %w", err) } @@ -1252,7 +1259,7 @@ func (a *StateAPI) StateDealProviderCollateralBounds(ctx context.Context, size a powClaim.QualityAdjPower, rewPow, circ.FilCirculating, - a.StateManager.GetNtwkVersion(ctx, ts.Height())) + m.StateManager.GetNtwkVersion(ctx, ts.Height())) return api.DealCollateralBounds{ Min: types.BigDiv(types.BigMul(min, dealProviderCollateralNum), dealProviderCollateralDen), Max: max, @@ -1273,23 +1280,32 @@ func (a *StateAPI) StateCirculatingSupply(ctx context.Context, tsk types.TipSetK } func (a *StateAPI) StateVMCirculatingSupplyInternal(ctx context.Context, tsk types.TipSetKey) (api.CirculatingSupply, error) { - ts, err := a.Chain.GetTipSetFromKey(tsk) + return stateVMCirculatingSupplyInternal(ctx, tsk, a.Chain, a.StateManager) +} +func stateVMCirculatingSupplyInternal( + ctx context.Context, + tsk types.TipSetKey, + cstore *store.ChainStore, + smgr *stmgr.StateManager, +) (api.CirculatingSupply, error) { + ts, err := cstore.GetTipSetFromKey(tsk) if err != nil { return api.CirculatingSupply{}, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - sTree, err := a.stateForTs(ctx, ts) + sTree, err := stateForTs(ctx, ts, cstore, smgr) if err != nil { return api.CirculatingSupply{}, err } - return a.StateManager.GetVMCirculatingSupplyDetailed(ctx, ts.Height(), sTree) + + return smgr.GetVMCirculatingSupplyDetailed(ctx, ts.Height(), sTree) } -func (a *StateAPI) StateNetworkVersion(ctx context.Context, tsk types.TipSetKey) (network.Version, error) { - ts, err := a.Chain.GetTipSetFromKey(tsk) +func (m *StateModule) StateNetworkVersion(ctx context.Context, tsk types.TipSetKey) (network.Version, error) { + ts, err := m.Chain.GetTipSetFromKey(tsk) if err != nil { return network.VersionMax, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - return a.StateManager.GetNtwkVersion(ctx, ts.Height()), nil + return m.StateManager.GetNtwkVersion(ctx, ts.Height()), nil } diff --git a/node/impl/paych/paych.go b/node/impl/paych/paych.go index af0a1db15..773a5efab 100644 --- a/node/impl/paych/paych.go +++ b/node/impl/paych/paych.go @@ -13,17 +13,12 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors/builtin/paych" "github.com/filecoin-project/lotus/chain/types" - full "github.com/filecoin-project/lotus/node/impl/full" "github.com/filecoin-project/lotus/paychmgr" ) type PaychAPI struct { fx.In - full.MpoolAPI - full.WalletAPI - full.ChainAPI - PaychMgr *paychmgr.Manager } diff --git a/node/modules/rpcstatemanager.go b/node/modules/rpcstatemanager.go index 0ed054d45..7d7b92437 100644 --- a/node/modules/rpcstatemanager.go +++ b/node/modules/rpcstatemanager.go @@ -3,19 +3,40 @@ package modules import ( "context" - "github.com/filecoin-project/lotus/api" + "golang.org/x/xerrors" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/api/apibstore" + "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lotus/chain/actors/builtin/paych" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/types" + cbor "github.com/ipfs/go-ipld-cbor" ) type RPCStateManager struct { - gapi api.GatewayAPI + gapi api.GatewayAPI + cstore *cbor.BasicIpldStore } func NewRPCStateManager(api api.GatewayAPI) *RPCStateManager { - return &RPCStateManager{gapi: api} + cstore := cbor.NewCborStore(apibstore.NewAPIBlockstore(api)) + return &RPCStateManager{gapi: api, cstore: cstore} +} + +func (s *RPCStateManager) GetPaychState(ctx context.Context, addr address.Address, ts *types.TipSet) (*types.Actor, paych.State, error) { + act, err := s.gapi.StateGetActor(ctx, addr, ts.Key()) + if err != nil { + return nil, nil, err + } + + actState, err := paych.Load(adt.WrapStore(ctx, s.cstore), act) + if err != nil { + return nil, nil, err + } + return act, actState, nil + } func (s *RPCStateManager) LoadActorTsk(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*types.Actor, error) { @@ -30,4 +51,8 @@ func (s *RPCStateManager) ResolveToKeyAddress(ctx context.Context, addr address. return s.gapi.StateAccountKey(ctx, addr, ts.Key()) } +func (s *RPCStateManager) Call(ctx context.Context, msg *types.Message, ts *types.TipSet) (*api.InvocResult, error) { + return nil, xerrors.Errorf("RPCStateManager does not implement StateManager.Call") +} + var _ stmgr.StateManagerAPI = (*RPCStateManager)(nil) diff --git a/paychmgr/manager.go b/paychmgr/manager.go index f2fc190c7..5e0aa88ce 100644 --- a/paychmgr/manager.go +++ b/paychmgr/manager.go @@ -16,7 +16,6 @@ import ( "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/actors/builtin/paych" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/types" @@ -61,14 +60,10 @@ type managerAPI interface { // managerAPIImpl is used to create a composite that implements managerAPI type managerAPIImpl struct { - *stmgr.StateManager + stmgr.StateManagerAPI paychAPI } -func (m *managerAPIImpl) AdtStore(ctx context.Context) adt.Store { - return m.ChainStore().Store(ctx) -} - type Manager struct { // The Manager context is used to terminate wait operations on shutdown ctx context.Context @@ -82,11 +77,11 @@ type Manager struct { channels map[string]*channelAccessor } -func NewManager(mctx helpers.MetricsCtx, lc fx.Lifecycle, sm *stmgr.StateManager, pchstore *Store, api PaychAPI) *Manager { +func NewManager(mctx helpers.MetricsCtx, lc fx.Lifecycle, sm stmgr.StateManagerAPI, pchstore *Store, api PaychAPI) *Manager { ctx := helpers.LifecycleCtx(mctx, lc) ctx, shutdown := context.WithCancel(ctx) - impl := &managerAPIImpl{StateManager: sm, paychAPI: &api} + impl := &managerAPIImpl{StateManagerAPI: sm, paychAPI: &api} return &Manager{ ctx: ctx, shutdown: shutdown, From 906286fdbe245a286cb4e0340d0dafeee1a01bc9 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Tue, 20 Oct 2020 17:08:25 +0200 Subject: [PATCH 869/885] feat: lite-mode - CLI tests for `lotus client` commands --- api/api_gateway.go | 1 + api/apistruct/struct.go | 5 ++ api/test/deals.go | 38 +++++---- cli/client.go | 74 +++++++++--------- cli/client_test.go | 22 ++++++ cli/helper.go | 43 +++++++++-- cli/multisig_test.go | 37 +-------- cli/test/client.go | 115 +++++++++++++++++++++++++++ cli/test/mockcli.go | 9 +++ cli/test/multisig.go | 9 --- cli/test/net.go | 41 ++++++++++ cli/test/util.go | 12 +++ cmd/lotus-gateway/api.go | 8 ++ cmd/lotus-gateway/endtoend_test.go | 120 +++++++++++++++-------------- node/impl/full/state.go | 9 ++- 15 files changed, 381 insertions(+), 162 deletions(-) create mode 100644 cli/client_test.go create mode 100644 cli/test/client.go create mode 100644 cli/test/net.go create mode 100644 cli/test/util.go diff --git a/api/api_gateway.go b/api/api_gateway.go index 2b7b52b43..5f31b8d31 100644 --- a/api/api_gateway.go +++ b/api/api_gateway.go @@ -34,5 +34,6 @@ type GatewayAPI interface { StateMarketStorageDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*MarketDeal, error) StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (miner.MinerInfo, error) StateNetworkVersion(context.Context, types.TipSetKey) (network.Version, error) + StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*MsgLookup, error) } diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 6811eff98..6c6c19f23 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -396,6 +396,7 @@ type GatewayStruct struct { StateMarketBalance func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (api.MarketBalance, error) StateMarketStorageDeal func(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*api.MarketDeal, error) StateNetworkVersion func(ctx context.Context, tsk types.TipSetKey) (stnetwork.Version, error) + StateVerifiedClientStatus func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) StateWaitMsg func(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error) } } @@ -1547,6 +1548,10 @@ func (g GatewayStruct) StateNetworkVersion(ctx context.Context, tsk types.TipSet return g.Internal.StateNetworkVersion(ctx, tsk) } +func (g GatewayStruct) StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) { + return g.Internal.StateVerifiedClientStatus(ctx, addr, tsk) +} + func (g GatewayStruct) StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error) { return g.Internal.StateWaitMsg(ctx, msg, confidence) } diff --git a/api/test/deals.go b/api/test/deals.go index 2317131f8..b81099d90 100644 --- a/api/test/deals.go +++ b/api/test/deals.go @@ -106,21 +106,7 @@ func TestDoubleDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration) { } func MakeDeal(t *testing.T, ctx context.Context, rseed int, client api.FullNode, miner TestStorageNode, carExport, fastRet bool) { - data := make([]byte, 1600) - rand.New(rand.NewSource(int64(rseed))).Read(data) - - dir, err := ioutil.TempDir(os.TempDir(), "test-make-deal-") - if err != nil { - t.Fatal(err) - } - - path := filepath.Join(dir, "sourcefile.dat") - err = ioutil.WriteFile(path, data, 0644) - if err != nil { - t.Fatal(err) - } - - res, err := client.ClientImport(ctx, api.FileRef{Path: path}) + res, data, err := CreateClientFile(ctx, client, rseed) if err != nil { t.Fatal(err) } @@ -141,6 +127,28 @@ func MakeDeal(t *testing.T, ctx context.Context, rseed int, client api.FullNode, testRetrieval(t, ctx, client, fcid, &info.PieceCID, carExport, data) } +func CreateClientFile(ctx context.Context, client api.FullNode, rseed int) (*api.ImportRes, []byte, error) { + data := make([]byte, 1600) + rand.New(rand.NewSource(int64(rseed))).Read(data) + + dir, err := ioutil.TempDir(os.TempDir(), "test-make-deal-") + if err != nil { + return nil, nil, err + } + + path := filepath.Join(dir, "sourcefile.dat") + err = ioutil.WriteFile(path, data, 0644) + if err != nil { + return nil, nil, err + } + + res, err := client.ClientImport(ctx, api.FileRef{Path: path}) + if err != nil { + return nil, nil, err + } + return res, data, nil +} + func TestFastRetrievalDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration) { ctx := context.Background() diff --git a/cli/client.go b/cli/client.go index 83c452555..c3479a725 100644 --- a/cli/client.go +++ b/cli/client.go @@ -343,6 +343,7 @@ var clientDealCmd = &cli.Command{ } defer closer() ctx := ReqContext(cctx) + afmt := NewAppFmt(cctx.App) if cctx.NArg() != 4 { return xerrors.New("expected 4 args: dataCid, miner, price, duration") @@ -462,7 +463,7 @@ var clientDealCmd = &cli.Command{ return err } - fmt.Println(encoder.Encode(*proposal)) + afmt.Println(encoder.Encode(*proposal)) return nil }, @@ -477,6 +478,7 @@ func interactiveDeal(cctx *cli.Context) error { ctx := ReqContext(cctx) ctx, cancel := context.WithCancel(ctx) defer cancel() + afmt := NewAppFmt(cctx.App) state := "import" gib := types.NewInt(1 << 30) @@ -517,10 +519,10 @@ func interactiveDeal(cctx *cli.Context) error { } printErr := func(err error) { - fmt.Printf("%s %s\n", color.RedString("Error:"), err.Error()) + afmt.Printf("%s %s\n", color.RedString("Error:"), err.Error()) } - cs := readline.NewCancelableStdin(os.Stdin) + cs := readline.NewCancelableStdin(afmt.Stdin) go func() { <-ctx.Done() cs.Close() // nolint:errcheck @@ -537,7 +539,7 @@ uiLoop: switch state { case "import": - fmt.Print("Data CID (from " + color.YellowString("lotus client import") + "): ") + afmt.Print("Data CID (from " + color.YellowString("lotus client import") + "): ") _cidStr, _, err := rl.ReadLine() cidStr := string(_cidStr) @@ -560,7 +562,7 @@ uiLoop: state = "duration" case "duration": - fmt.Print("Deal duration (days): ") + afmt.Print("Deal duration (days): ") _daystr, _, err := rl.ReadLine() daystr := string(_daystr) @@ -605,7 +607,7 @@ uiLoop: continue } - fmt.Print("\nMake this a verified deal? (yes/no): ") + afmt.Print("\nMake this a verified deal? (yes/no): ") _yn, _, err := rl.ReadLine() yn := string(_yn) @@ -619,13 +621,13 @@ uiLoop: case "no": verified = false default: - fmt.Println("Type in full 'yes' or 'no'") + afmt.Println("Type in full 'yes' or 'no'") continue } state = "miner" case "miner": - fmt.Print("Miner Addresses (f0.. f0..), none to find: ") + afmt.Print("Miner Addresses (f0.. f0..), none to find: ") _maddrsStr, _, err := rl.ReadLine() maddrsStr := string(_maddrsStr) @@ -664,11 +666,11 @@ uiLoop: candidateAsks = append(candidateAsks, ask) } - fmt.Printf("Found %d candidate asks\n", len(candidateAsks)) + afmt.Printf("Found %d candidate asks\n", len(candidateAsks)) state = "find-budget" case "find-budget": - fmt.Printf("Proposing from %s, Current Balance: %s\n", a, types.FIL(fromBal)) - fmt.Print("Maximum budget (FIL): ") // TODO: Propose some default somehow? + afmt.Printf("Proposing from %s, Current Balance: %s\n", a, types.FIL(fromBal)) + afmt.Print("Maximum budget (FIL): ") // TODO: Propose some default somehow? _budgetStr, _, err := rl.ReadLine() budgetStr := string(_budgetStr) @@ -698,10 +700,10 @@ uiLoop: } } candidateAsks = goodAsks - fmt.Printf("%d asks within budget\n", len(candidateAsks)) + afmt.Printf("%d asks within budget\n", len(candidateAsks)) state = "find-count" case "find-count": - fmt.Print("Deals to make (1): ") + afmt.Print("Deals to make (1): ") dealcStr, _, err := rl.ReadLine() if err != nil { printErr(xerrors.Errorf("reading deal count: %w", err)) @@ -780,12 +782,12 @@ uiLoop: case "confirm": // TODO: do some more or epochs math (round to miner PP, deal start buffer) - fmt.Printf("-----\n") - fmt.Printf("Proposing from %s\n", a) - fmt.Printf("\tBalance: %s\n", types.FIL(fromBal)) - fmt.Printf("\n") - fmt.Printf("Piece size: %s (Payload size: %s)\n", units.BytesSize(float64(ds.PieceSize)), units.BytesSize(float64(ds.PayloadSize))) - fmt.Printf("Duration: %s\n", dur) + afmt.Printf("-----\n") + afmt.Printf("Proposing from %s\n", a) + afmt.Printf("\tBalance: %s\n", types.FIL(fromBal)) + afmt.Printf("\n") + afmt.Printf("Piece size: %s (Payload size: %s)\n", units.BytesSize(float64(ds.PieceSize)), units.BytesSize(float64(ds.PayloadSize))) + afmt.Printf("Duration: %s\n", dur) pricePerGib := big.Zero() for _, a := range ask { @@ -804,7 +806,7 @@ uiLoop: if len(ask) > 1 { totalPrice := types.BigMul(epochPrice, types.NewInt(uint64(epochs))) - fmt.Printf("Miner %s (Power:%s) price: ~%s (%s per epoch)\n", color.YellowString(a.Miner.String()), color.GreenString(types.SizeStr(mpow.MinerPower.QualityAdjPower)), color.BlueString(types.FIL(totalPrice).String()), types.FIL(epochPrice)) + afmt.Printf("Miner %s (Power:%s) price: ~%s (%s per epoch)\n", color.YellowString(a.Miner.String()), color.GreenString(types.SizeStr(mpow.MinerPower.QualityAdjPower)), color.BlueString(types.FIL(totalPrice).String()), types.FIL(epochPrice)) } } @@ -812,12 +814,12 @@ uiLoop: epochPrice := types.BigDiv(types.BigMul(pricePerGib, types.NewInt(uint64(ds.PieceSize))), gib) totalPrice := types.BigMul(epochPrice, types.NewInt(uint64(epochs))) - fmt.Printf("Total price: ~%s (%s per epoch)\n", color.CyanString(types.FIL(totalPrice).String()), types.FIL(epochPrice)) - fmt.Printf("Verified: %v\n", verified) + afmt.Printf("Total price: ~%s (%s per epoch)\n", color.CyanString(types.FIL(totalPrice).String()), types.FIL(epochPrice)) + afmt.Printf("Verified: %v\n", verified) state = "accept" case "accept": - fmt.Print("\nAccept (yes/no): ") + afmt.Print("\nAccept (yes/no): ") _yn, _, err := rl.ReadLine() yn := string(_yn) @@ -830,7 +832,7 @@ uiLoop: } if yn != "yes" { - fmt.Println("Type in full 'yes' or 'no'") + afmt.Println("Type in full 'yes' or 'no'") continue } @@ -861,7 +863,7 @@ uiLoop: return err } - fmt.Printf("Deal (%s) CID: %s\n", maddr, color.GreenString(encoder.Encode(*proposal))) + afmt.Printf("Deal (%s) CID: %s\n", maddr, color.GreenString(encoder.Encode(*proposal))) } return nil @@ -975,6 +977,7 @@ var clientRetrieveCmd = &cli.Command{ } defer closer() ctx := ReqContext(cctx) + afmt := NewAppFmt(cctx.App) var payer address.Address if cctx.String("from") != "" { @@ -1083,14 +1086,14 @@ var clientRetrieveCmd = &cli.Command{ select { case evt, ok := <-updates: if ok { - fmt.Printf("> Recv: %s, Paid %s, %s (%s)\n", + afmt.Printf("> Recv: %s, Paid %s, %s (%s)\n", types.SizeStr(types.NewInt(evt.BytesReceived)), types.FIL(evt.FundsSpent), retrievalmarket.ClientEvents[evt.Event], retrievalmarket.DealStatuses[evt.Status], ) } else { - fmt.Println("Success") + afmt.Println("Success") return nil } @@ -1269,8 +1272,9 @@ var clientQueryAskCmd = &cli.Command{ }, }, Action: func(cctx *cli.Context) error { + afmt := NewAppFmt(cctx.App) if cctx.NArg() != 1 { - fmt.Println("Usage: query-ask [minerAddress]") + afmt.Println("Usage: query-ask [minerAddress]") return nil } @@ -1311,23 +1315,23 @@ var clientQueryAskCmd = &cli.Command{ return err } - fmt.Printf("Ask: %s\n", maddr) - fmt.Printf("Price per GiB: %s\n", types.FIL(ask.Price)) - fmt.Printf("Verified Price per GiB: %s\n", types.FIL(ask.VerifiedPrice)) - fmt.Printf("Max Piece size: %s\n", types.SizeStr(types.NewInt(uint64(ask.MaxPieceSize)))) + afmt.Printf("Ask: %s\n", maddr) + afmt.Printf("Price per GiB: %s\n", types.FIL(ask.Price)) + afmt.Printf("Verified Price per GiB: %s\n", types.FIL(ask.VerifiedPrice)) + afmt.Printf("Max Piece size: %s\n", types.SizeStr(types.NewInt(uint64(ask.MaxPieceSize)))) size := cctx.Int64("size") if size == 0 { return nil } perEpoch := types.BigDiv(types.BigMul(ask.Price, types.NewInt(uint64(size))), types.NewInt(1<<30)) - fmt.Printf("Price per Block: %s\n", types.FIL(perEpoch)) + afmt.Printf("Price per Block: %s\n", types.FIL(perEpoch)) duration := cctx.Int64("duration") if duration == 0 { return nil } - fmt.Printf("Total Price: %s\n", types.FIL(types.BigMul(perEpoch, types.NewInt(uint64(duration))))) + afmt.Printf("Total Price: %s\n", types.FIL(types.BigMul(perEpoch, types.NewInt(uint64(duration))))) return nil }, @@ -1410,7 +1414,7 @@ var clientListDeals = &cli.Command{ } } - return outputStorageDeals(ctx, os.Stdout, api, localDeals, cctx.Bool("verbose"), cctx.Bool("color"), showFailed) + return outputStorageDeals(ctx, cctx.App.Writer, api, localDeals, cctx.Bool("verbose"), cctx.Bool("color"), showFailed) }, } diff --git a/cli/client_test.go b/cli/client_test.go new file mode 100644 index 000000000..f0e8efda8 --- /dev/null +++ b/cli/client_test.go @@ -0,0 +1,22 @@ +package cli + +import ( + "context" + "os" + "testing" + "time" + + clitest "github.com/filecoin-project/lotus/cli/test" +) + +// TestClient does a basic test to exercise the client CLI +// commands +func TestClient(t *testing.T) { + _ = os.Setenv("BELLMAN_NO_GPU", "1") + clitest.QuietMiningLogs() + + blocktime := 5 * time.Millisecond + ctx := context.Background() + clientNode, _ := clitest.StartOneNodeOneMiner(ctx, t, blocktime) + clitest.RunClientTest(t, Commands, clientNode) +} diff --git a/cli/helper.go b/cli/helper.go index 9398ead71..da236bcae 100644 --- a/cli/helper.go +++ b/cli/helper.go @@ -2,15 +2,16 @@ package cli import ( "fmt" + "io" "os" - "github.com/urfave/cli/v2" + ufcli "github.com/urfave/cli/v2" "golang.org/x/xerrors" ) type PrintHelpErr struct { Err error - Ctx *cli.Context + Ctx *ufcli.Context } func (e *PrintHelpErr) Error() string { @@ -26,11 +27,11 @@ func (e *PrintHelpErr) Is(o error) bool { return ok } -func ShowHelp(cctx *cli.Context, err error) error { +func ShowHelp(cctx *ufcli.Context, err error) error { return &PrintHelpErr{Err: err, Ctx: cctx} } -func RunApp(app *cli.App) { +func RunApp(app *ufcli.App) { if err := app.Run(os.Args); err != nil { if os.Getenv("LOTUS_DEV") != "" { log.Warnf("%+v", err) @@ -39,8 +40,40 @@ func RunApp(app *cli.App) { } var phe *PrintHelpErr if xerrors.As(err, &phe) { - _ = cli.ShowCommandHelp(phe.Ctx, phe.Ctx.Command.Name) + _ = ufcli.ShowCommandHelp(phe.Ctx, phe.Ctx.Command.Name) } os.Exit(1) } } + +type AppFmt struct { + app *ufcli.App + Stdin io.Reader +} + +func NewAppFmt(a *ufcli.App) *AppFmt { + var stdin io.Reader + istdin, ok := a.Metadata["stdin"] + if ok { + stdin = istdin.(io.Reader) + } else { + stdin = os.Stdin + } + return &AppFmt{app: a, Stdin: stdin} +} + +func (a *AppFmt) Print(args ...interface{}) { + fmt.Fprint(a.app.Writer, args...) +} + +func (a *AppFmt) Println(args ...interface{}) { + fmt.Fprintln(a.app.Writer, args...) +} + +func (a *AppFmt) Printf(fmtstr string, args ...interface{}) { + fmt.Fprintf(a.app.Writer, fmtstr, args...) +} + +func (a *AppFmt) Scan(args ...interface{}) (int, error) { + return fmt.Fscan(a.Stdin, args...) +} diff --git a/cli/multisig_test.go b/cli/multisig_test.go index 09cdbe056..82472cd62 100644 --- a/cli/multisig_test.go +++ b/cli/multisig_test.go @@ -6,50 +6,17 @@ import ( "testing" "time" - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/api/test" clitest "github.com/filecoin-project/lotus/cli/test" - builder "github.com/filecoin-project/lotus/node/test" ) // TestMultisig does a basic test to exercise the multisig CLI // commands func TestMultisig(t *testing.T) { _ = os.Setenv("BELLMAN_NO_GPU", "1") + clitest.QuietMiningLogs() blocktime := 5 * time.Millisecond ctx := context.Background() - nodes, _ := startNodes(ctx, t, blocktime) - clientNode := nodes[0] + clientNode, _ := clitest.StartOneNodeOneMiner(ctx, t, blocktime) clitest.RunMultisigTest(t, Commands, clientNode) } - -func startNodes(ctx context.Context, t *testing.T, blocktime time.Duration) ([]test.TestNode, []address.Address) { - n, sn := builder.RPCMockSbBuilder(t, test.OneFull, test.OneMiner) - - full := n[0] - miner := sn[0] - - // Get everyone connected - addrs, err := full.NetAddrsListen(ctx) - if err != nil { - t.Fatal(err) - } - - if err := miner.NetConnect(ctx, addrs); err != nil { - t.Fatal(err) - } - - // Start mining blocks - bm := test.NewBlockMiner(ctx, t, miner, blocktime) - bm.MineBlocks() - - // Get the creator's address - creatorAddr, err := full.WalletDefaultAddress(ctx) - if err != nil { - t.Fatal(err) - } - - // Create mock CLI - return n, []address.Address{creatorAddr} -} diff --git a/cli/test/client.go b/cli/test/client.go new file mode 100644 index 000000000..3a5146219 --- /dev/null +++ b/cli/test/client.go @@ -0,0 +1,115 @@ +package test + +import ( + "context" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + "time" + + "github.com/filecoin-project/lotus/api/test" + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v2/actors/builtin" + "github.com/stretchr/testify/require" + lcli "github.com/urfave/cli/v2" +) + +// RunClientTest exercises some of the client CLI commands +func RunClientTest(t *testing.T, cmds []*lcli.Command, clientNode test.TestNode) { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + // Create mock CLI + mockCLI := newMockCLI(t, cmds) + clientCLI := mockCLI.client(clientNode.ListenAddr) + + // Get the miner address + addrs, err := clientNode.StateListMiners(ctx, types.EmptyTSK) + require.NoError(t, err) + require.Len(t, addrs, 1) + + minerAddr := addrs[0] + fmt.Println("Miner:", minerAddr) + + // client query-ask + cmd := []string{ + "client", "query-ask", minerAddr.String(), + } + out := clientCLI.runCmd(cmd) + require.Regexp(t, regexp.MustCompile("Ask:"), out) + + // Create a deal (non-interactive) + // client deal 1000000attofil + res, _, err := test.CreateClientFile(ctx, clientNode, 1) + require.NoError(t, err) + dataCid := res.Root + price := "1000000attofil" + duration := fmt.Sprintf("%d", build.MinDealDuration) + cmd = []string{ + "client", "deal", dataCid.String(), minerAddr.String(), price, duration, + } + out = clientCLI.runCmd(cmd) + fmt.Println("client deal", out) + + // Create a deal (interactive) + // client deal + // + // (in days) + // + // "no" (verified client) + // "yes" (confirm deal) + res, _, err = test.CreateClientFile(ctx, clientNode, 2) + require.NoError(t, err) + dataCid2 := res.Root + duration = fmt.Sprintf("%d", build.MinDealDuration/builtin.EpochsInDay) + cmd = []string{ + "client", "deal", + } + interactiveCmds := []string{ + dataCid2.String(), + duration, + minerAddr.String(), + "no", + "yes", + } + out = clientCLI.runInteractiveCmd(cmd, interactiveCmds) + fmt.Println("client deal:\n", out) + + // Wait for provider to start sealing deal + dealStatus := "" + for dealStatus != "StorageDealSealing" { + // client list-deals + cmd = []string{"client", "list-deals"} + out = clientCLI.runCmd(cmd) + fmt.Println("list-deals:\n", out) + + lines := strings.Split(out, "\n") + require.Len(t, lines, 2) + re := regexp.MustCompile(`\s+`) + parts := re.Split(lines[1], -1) + if len(parts) < 4 { + require.Fail(t, "bad list-deals output format") + } + dealStatus = parts[3] + fmt.Println(" Deal status:", dealStatus) + + time.Sleep(time.Second) + } + + // Retrieve the first file from the miner + // client retrieve + tmpdir, err := ioutil.TempDir(os.TempDir(), "test-cli-client") + require.NoError(t, err) + path := filepath.Join(tmpdir, "outfile.dat") + cmd = []string{ + "client", "retrieve", dataCid.String(), path, + } + out = clientCLI.runCmd(cmd) + fmt.Println("retrieve:\n", out) + require.Regexp(t, regexp.MustCompile("Success"), out) +} diff --git a/cli/test/mockcli.go b/cli/test/mockcli.go index f5d5cfcba..c7eb70092 100644 --- a/cli/test/mockcli.go +++ b/cli/test/mockcli.go @@ -122,3 +122,12 @@ func (c *mockCLIClient) flagSet(cmd *lcli.Command) *flag.FlagSet { } return fs } + +func (c *mockCLIClient) runInteractiveCmd(cmd []string, interactive []string) string { + c.toStdin(strings.Join(interactive, "\n") + "\n") + return c.runCmd(cmd) +} + +func (c *mockCLIClient) toStdin(s string) { + c.cctx.App.Metadata["stdin"] = bytes.NewBufferString(s) +} diff --git a/cli/test/multisig.go b/cli/test/multisig.go index e77879346..d2c0238d2 100644 --- a/cli/test/multisig.go +++ b/cli/test/multisig.go @@ -10,19 +10,10 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/api/test" "github.com/filecoin-project/lotus/chain/types" - logging "github.com/ipfs/go-log/v2" "github.com/stretchr/testify/require" lcli "github.com/urfave/cli/v2" ) -func QuietMiningLogs() { - logging.SetLogLevel("miner", "ERROR") - logging.SetLogLevel("chainstore", "ERROR") - logging.SetLogLevel("chain", "ERROR") - logging.SetLogLevel("sub", "ERROR") - logging.SetLogLevel("storageminer", "ERROR") -} - func RunMultisigTest(t *testing.T, cmds []*lcli.Command, clientNode test.TestNode) { ctx := context.Background() diff --git a/cli/test/net.go b/cli/test/net.go new file mode 100644 index 000000000..d13993d16 --- /dev/null +++ b/cli/test/net.go @@ -0,0 +1,41 @@ +package test + +import ( + "context" + "testing" + "time" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/api/test" + test2 "github.com/filecoin-project/lotus/node/test" +) + +func StartOneNodeOneMiner(ctx context.Context, t *testing.T, blocktime time.Duration) (test.TestNode, address.Address) { + n, sn := test2.RPCMockSbBuilder(t, test.OneFull, test.OneMiner) + + full := n[0] + miner := sn[0] + + // Get everyone connected + addrs, err := full.NetAddrsListen(ctx) + if err != nil { + t.Fatal(err) + } + + if err := miner.NetConnect(ctx, addrs); err != nil { + t.Fatal(err) + } + + // Start mining blocks + bm := test.NewBlockMiner(ctx, t, miner, blocktime) + bm.MineBlocks() + + // Get the full node's wallet address + fullAddr, err := full.WalletDefaultAddress(ctx) + if err != nil { + t.Fatal(err) + } + + // Create mock CLI + return full, fullAddr +} diff --git a/cli/test/util.go b/cli/test/util.go new file mode 100644 index 000000000..e3930dc83 --- /dev/null +++ b/cli/test/util.go @@ -0,0 +1,12 @@ +package test + +import "github.com/ipfs/go-log/v2" + +func QuietMiningLogs() { + _ = log.SetLogLevel("miner", "ERROR") + _ = log.SetLogLevel("chainstore", "ERROR") + _ = log.SetLogLevel("chain", "ERROR") + _ = log.SetLogLevel("sub", "ERROR") + _ = log.SetLogLevel("storageminer", "ERROR") + _ = log.SetLogLevel("pubsub", "ERROR") +} diff --git a/cmd/lotus-gateway/api.go b/cmd/lotus-gateway/api.go index 0ebb99298..875eaac7d 100644 --- a/cmd/lotus-gateway/api.go +++ b/cmd/lotus-gateway/api.go @@ -67,6 +67,7 @@ type gatewayDepsAPI interface { StateMinerAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) StateMinerProvingDeadline(context.Context, address.Address, types.TipSetKey) (*dline.Info, error) StateCirculatingSupply(context.Context, types.TipSetKey) (abi.TokenAmount, error) + StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) StateVMCirculatingSupplyInternal(context.Context, types.TipSetKey) (api.CirculatingSupply, error) } @@ -364,6 +365,13 @@ func (a *GatewayAPI) StateCirculatingSupply(ctx context.Context, tsk types.TipSe } +func (a *GatewayAPI) StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) { + if err := a.checkTipsetKey(ctx, tsk); err != nil { + return nil, err + } + return a.api.StateVerifiedClientStatus(ctx, addr, tsk) +} + func (a *GatewayAPI) StateVMCirculatingSupplyInternal(ctx context.Context, tsk types.TipSetKey) (api.CirculatingSupply, error) { if err := a.checkTipsetKey(ctx, tsk); err != nil { return api.CirculatingSupply{}, err diff --git a/cmd/lotus-gateway/endtoend_test.go b/cmd/lotus-gateway/endtoend_test.go index c5b513f44..1e1e5e229 100644 --- a/cmd/lotus-gateway/endtoend_test.go +++ b/cmd/lotus-gateway/endtoend_test.go @@ -11,7 +11,6 @@ import ( "github.com/filecoin-project/lotus/cli" clitest "github.com/filecoin-project/lotus/cli/test" - logging "github.com/ipfs/go-log/v2" init2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/init" multisig2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/multisig" @@ -45,6 +44,7 @@ func init() { // node that is connected through a gateway to a full API node func TestWalletMsig(t *testing.T) { _ = os.Setenv("BELLMAN_NO_GPU", "1") + clitest.QuietMiningLogs() blocktime := 5 * time.Millisecond ctx := context.Background() @@ -155,80 +155,35 @@ func TestMsigCLI(t *testing.T) { blocktime := 5 * time.Millisecond ctx := context.Background() - nodes := startNodes(ctx, t, blocktime, maxLookbackCap, maxStateWaitLookbackLimit) + nodes := startNodesWithFunds(ctx, t, blocktime, maxLookbackCap, maxStateWaitLookbackLimit) defer nodes.closer() lite := nodes.lite - full := nodes.full - - // The full node starts with a wallet - fullWalletAddr, err := full.WalletDefaultAddress(ctx) - require.NoError(t, err) - - // Create a wallet on the lite node - liteWalletAddr, err := lite.WalletNew(ctx, types.KTSecp256k1) - require.NoError(t, err) - - // Send some funds from the full node to the lite node - err = sendFunds(ctx, full, fullWalletAddr, liteWalletAddr, types.NewInt(1e18)) - require.NoError(t, err) - clitest.RunMultisigTest(t, cli.Commands, lite) } -func sendFunds(ctx context.Context, fromNode test.TestNode, fromAddr address.Address, toAddr address.Address, amt types.BigInt) error { - msg := &types.Message{ - From: fromAddr, - To: toAddr, - Value: amt, - } - - sm, err := fromNode.MpoolPushMessage(ctx, msg, nil) - if err != nil { - return err - } - - res, err := fromNode.StateWaitMsg(ctx, sm.Cid(), 1) - if err != nil { - return err - } - if res.Receipt.ExitCode != 0 { - return xerrors.Errorf("send funds failed with exit code %d", res.Receipt.ExitCode) - } - - return nil -} - func TestDealFlow(t *testing.T) { _ = os.Setenv("BELLMAN_NO_GPU", "1") - - logging.SetLogLevel("miner", "ERROR") - logging.SetLogLevel("chainstore", "ERROR") - logging.SetLogLevel("chain", "ERROR") - logging.SetLogLevel("sub", "ERROR") - logging.SetLogLevel("storageminer", "ERROR") + clitest.QuietMiningLogs() blocktime := 5 * time.Millisecond ctx := context.Background() - nodes := startNodes(ctx, t, blocktime, maxLookbackCap, maxStateWaitLookbackLimit) + nodes := startNodesWithFunds(ctx, t, blocktime, maxLookbackCap, maxStateWaitLookbackLimit) defer nodes.closer() - full := nodes.full - lite := nodes.lite + test.MakeDeal(t, ctx, 6, nodes.lite, nodes.miner, false, false) +} - // The full node starts with a wallet - fullWalletAddr, err := full.WalletDefaultAddress(ctx) - require.NoError(t, err) +func TestCLIDealFlow(t *testing.T) { + _ = os.Setenv("BELLMAN_NO_GPU", "1") + clitest.QuietMiningLogs() - // Create a wallet on the lite node - liteWalletAddr, err := lite.WalletNew(ctx, types.KTSecp256k1) - require.NoError(t, err) + blocktime := 5 * time.Millisecond + ctx := context.Background() + nodes := startNodesWithFunds(ctx, t, blocktime, maxLookbackCap, maxStateWaitLookbackLimit) + defer nodes.closer() - // Send some funds from the full node to the lite node - err = sendFunds(ctx, full, fullWalletAddr, liteWalletAddr, types.NewInt(1e18)) - require.NoError(t, err) - - test.MakeDeal(t, ctx, 6, lite, nodes.miner, false, false) + clitest.RunClientTest(t, cli.Commands, nodes.lite) } type testNodes struct { @@ -238,6 +193,30 @@ type testNodes struct { closer jsonrpc.ClientCloser } +func startNodesWithFunds( + ctx context.Context, + t *testing.T, + blocktime time.Duration, + lookbackCap time.Duration, + stateWaitLookbackLimit abi.ChainEpoch, +) *testNodes { + nodes := startNodes(ctx, t, blocktime, lookbackCap, stateWaitLookbackLimit) + + // The full node starts with a wallet + fullWalletAddr, err := nodes.full.WalletDefaultAddress(ctx) + require.NoError(t, err) + + // Create a wallet on the lite node + liteWalletAddr, err := nodes.lite.WalletNew(ctx, types.KTSecp256k1) + require.NoError(t, err) + + // Send some funds from the full node to the lite node + err = sendFunds(ctx, nodes.full, fullWalletAddr, liteWalletAddr, types.NewInt(1e18)) + require.NoError(t, err) + + return nodes +} + func startNodes( ctx context.Context, t *testing.T, @@ -302,3 +281,26 @@ func startNodes( return &testNodes{lite: lite, full: full, miner: miner, closer: closer} } + +func sendFunds(ctx context.Context, fromNode test.TestNode, fromAddr address.Address, toAddr address.Address, amt types.BigInt) error { + msg := &types.Message{ + From: fromAddr, + To: toAddr, + Value: amt, + } + + sm, err := fromNode.MpoolPushMessage(ctx, msg, nil) + if err != nil { + return err + } + + res, err := fromNode.StateWaitMsg(ctx, sm.Cid(), 1) + if err != nil { + return err + } + if res.Receipt.ExitCode != 0 { + return xerrors.Errorf("send funds failed with exit code %d", res.Receipt.ExitCode) + } + + return nil +} diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 75ad03e7b..d0e813758 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -52,6 +52,7 @@ type StateModuleAPI interface { StateMarketStorageDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*api.MarketDeal, error) StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (miner.MinerInfo, error) StateNetworkVersion(ctx context.Context, key types.TipSetKey) (network.Version, error) + StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error) } @@ -1165,19 +1166,19 @@ func (a *StateAPI) StateVerifierStatus(ctx context.Context, addr address.Address // StateVerifiedClientStatus returns the data cap for the given address. // Returns zero if there is no entry in the data cap table for the // address. -func (a *StateAPI) StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) { - act, err := a.StateGetActor(ctx, verifreg.Address, tsk) +func (m *StateModule) StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) { + act, err := m.StateGetActor(ctx, verifreg.Address, tsk) if err != nil { return nil, err } - aid, err := a.StateLookupID(ctx, addr, tsk) + aid, err := m.StateLookupID(ctx, addr, tsk) if err != nil { log.Warnf("lookup failure %v", err) return nil, err } - vrs, err := verifreg.Load(a.StateManager.ChainStore().Store(ctx), act) + vrs, err := verifreg.Load(m.StateManager.ChainStore().Store(ctx), act) if err != nil { return nil, xerrors.Errorf("failed to load verified registry state: %w", err) } From 36816fb4f3281ad23c72a66b00b90eb73063036f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 23 Oct 2020 16:51:27 +0200 Subject: [PATCH 870/885] Fix list-asks, deals with non-genesis miners in lite-mode --- api/api_gateway.go | 7 ++++++- api/apistruct/struct.go | 10 ++++++++++ node/impl/client/client.go | 4 ++-- node/impl/full/state.go | 18 ++++++++++-------- 4 files changed, 28 insertions(+), 11 deletions(-) diff --git a/api/api_gateway.go b/api/api_gateway.go index 5f31b8d31..07fb5deb3 100644 --- a/api/api_gateway.go +++ b/api/api_gateway.go @@ -3,12 +3,15 @@ package api import ( "context" + "github.com/ipfs/go-cid" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/dline" "github.com/filecoin-project/go-state-types/network" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" - "github.com/ipfs/go-cid" ) type GatewayAPI interface { @@ -33,6 +36,8 @@ type GatewayAPI interface { StateMarketBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (MarketBalance, error) StateMarketStorageDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*MarketDeal, error) StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (miner.MinerInfo, error) + StateMinerProvingDeadline(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*dline.Info, error) + StateMinerPower(context.Context, address.Address, types.TipSetKey) (*MinerPower, error) StateNetworkVersion(context.Context, types.TipSetKey) (network.Version, error) StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*MsgLookup, error) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 6c6c19f23..883eeda46 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -393,6 +393,8 @@ type GatewayStruct struct { StateLookupID func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) StateListMiners func(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error) StateMinerInfo func(ctx context.Context, actor address.Address, tsk types.TipSetKey) (miner.MinerInfo, error) + StateMinerProvingDeadline func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*dline.Info, error) + StateMinerPower func(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error) StateMarketBalance func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (api.MarketBalance, error) StateMarketStorageDeal func(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*api.MarketDeal, error) StateNetworkVersion func(ctx context.Context, tsk types.TipSetKey) (stnetwork.Version, error) @@ -1544,6 +1546,14 @@ func (g GatewayStruct) StateMinerInfo(ctx context.Context, actor address.Address return g.Internal.StateMinerInfo(ctx, actor, tsk) } +func (g GatewayStruct) StateMinerProvingDeadline(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*dline.Info, error) { + return g.Internal.StateMinerProvingDeadline(ctx, addr, tsk) +} + +func (g GatewayStruct) StateMinerPower(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*api.MinerPower, error) { + return g.Internal.StateMinerPower(ctx, addr, tsk) +} + func (g GatewayStruct) StateNetworkVersion(ctx context.Context, tsk types.TipSetKey) (stnetwork.Version, error) { return g.Internal.StateNetworkVersion(ctx, tsk) } diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 0cd164826..37a24998e 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -65,9 +65,9 @@ type API struct { fx.In full.ChainAPI - full.StateAPI full.WalletAPI paych.PaychAPI + full.StateAPI SMDealClient storagemarket.StorageClient RetDiscovery discovery.PeerResolver @@ -117,7 +117,7 @@ func (a *API) ClientStartDeal(ctx context.Context, params *api.StartDealParams) } } - walletKey, err := a.StateAPI.StateManager.ResolveToKeyAddress(ctx, params.Wallet, nil) + walletKey, err := a.StateAccountKey(ctx, params.Wallet, types.EmptyTSK) if err != nil { return nil, xerrors.Errorf("failed resolving params.Wallet addr: %w", params.Wallet) } diff --git a/node/impl/full/state.go b/node/impl/full/state.go index d0e813758..882b3e93c 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -51,6 +51,8 @@ type StateModuleAPI interface { StateMarketBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (api.MarketBalance, error) StateMarketStorageDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*api.MarketDeal, error) StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (miner.MinerInfo, error) + StateMinerProvingDeadline(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*dline.Info, error) + StateMinerPower(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error) StateNetworkVersion(ctx context.Context, key types.TipSetKey) (network.Version, error) StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error) @@ -223,18 +225,18 @@ func (a *StateAPI) StateMinerPartitions(ctx context.Context, m address.Address, return out, err } -func (a *StateAPI) StateMinerProvingDeadline(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*dline.Info, error) { - ts, err := a.StateManager.ChainStore().GetTipSetFromKey(tsk) +func (m *StateModule) StateMinerProvingDeadline(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*dline.Info, error) { + ts, err := m.Chain.GetTipSetFromKey(tsk) if err != nil { return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - act, err := a.StateManager.LoadActor(ctx, addr, ts) + act, err := m.StateManager.LoadActor(ctx, addr, ts) if err != nil { return nil, xerrors.Errorf("failed to load miner actor: %w", err) } - mas, err := miner.Load(a.StateManager.ChainStore().Store(ctx), act) + mas, err := miner.Load(m.StateManager.ChainStore().Store(ctx), act) if err != nil { return nil, xerrors.Errorf("failed to load miner actor state: %w", err) } @@ -318,19 +320,19 @@ func (a *StateAPI) StateMinerRecoveries(ctx context.Context, addr address.Addres return miner.AllPartSectors(mas, miner.Partition.RecoveringSectors) } -func (a *StateAPI) StateMinerPower(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*api.MinerPower, error) { - ts, err := a.Chain.GetTipSetFromKey(tsk) +func (m *StateModule) StateMinerPower(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*api.MinerPower, error) { + ts, err := m.Chain.GetTipSetFromKey(tsk) if err != nil { return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) } - m, net, hmp, err := stmgr.GetPower(ctx, a.StateManager, ts, addr) + mp, net, hmp, err := stmgr.GetPower(ctx, m.StateManager, ts, addr) if err != nil { return nil, err } return &api.MinerPower{ - MinerPower: m, + MinerPower: mp, TotalPower: net, HasMinPower: hmp, }, nil From 16a911bc39085c545f0c67f26a13dc9555d8f9be Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 23 Oct 2020 19:30:42 +0200 Subject: [PATCH 871/885] Fix random test failures If block 1 was a null block then blockSet would include genesis which would lead to us trying to load parent of a genesis block. Signed-off-by: Jakub Sztandera --- chain/sync.go | 5 +++++ miner/miner.go | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/chain/sync.go b/chain/sync.go index d05a3d8bb..8da093cb6 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -1396,6 +1396,11 @@ loop: } base := blockSet[len(blockSet)-1] + if base.Equals(known) { + blockSet = blockSet[:len(blockSet)-1] + base = blockSet[len(blockSet)-1] + } + if base.IsChildOf(known) { // common case: receiving blocks that are building on top of our best tipset return blockSet, nil diff --git a/miner/miner.go b/miner/miner.go index 7de7f5b73..f2468a911 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -280,7 +280,7 @@ minerLoop: m.minedBlockHeights.Add(blkKey, true) if err := m.api.SyncSubmitBlock(ctx, b); err != nil { - log.Errorf("failed to submit newly mined block: %s", err) + log.Errorf("failed to submit newly mined block: %+v", err) } } else { base.NullRounds++ From ea69386519519b3c3b309467b96d2e6dd9bd2e76 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 23 Oct 2020 20:46:06 +0200 Subject: [PATCH 872/885] Fix flaky TestTimedBSSimple Signed-off-by: Jakub Sztandera --- lib/timedbs/timedbs.go | 14 ++++++++++---- lib/timedbs/timedbs_test.go | 23 +++++++++++++++-------- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/lib/timedbs/timedbs.go b/lib/timedbs/timedbs.go index bb03a59e9..c5c1a8fe0 100644 --- a/lib/timedbs/timedbs.go +++ b/lib/timedbs/timedbs.go @@ -8,6 +8,7 @@ import ( blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" + "github.com/raulk/clock" "go.uber.org/multierr" "github.com/filecoin-project/lotus/build" @@ -24,9 +25,10 @@ import ( type TimedCacheBS struct { mu sync.RWMutex active, inactive blockstore.MemStore - - interval time.Duration - closeCh chan struct{} + clock clock.Clock + interval time.Duration + closeCh chan struct{} + doneRotatingCh chan struct{} } func NewTimedCacheBS(cacheTime time.Duration) *TimedCacheBS { @@ -34,6 +36,7 @@ func NewTimedCacheBS(cacheTime time.Duration) *TimedCacheBS { active: blockstore.NewTemporary(), inactive: blockstore.NewTemporary(), interval: cacheTime, + clock: build.Clock, } } @@ -45,12 +48,15 @@ func (t *TimedCacheBS) Start(ctx context.Context) error { } t.closeCh = make(chan struct{}) go func() { - ticker := build.Clock.Ticker(t.interval) + ticker := t.clock.Ticker(t.interval) defer ticker.Stop() for { select { case <-ticker.C: t.rotate() + if t.doneRotatingCh != nil { + t.doneRotatingCh <- struct{}{} + } case <-t.closeCh: return } diff --git a/lib/timedbs/timedbs_test.go b/lib/timedbs/timedbs_test.go index 2126b9287..db21a5d79 100644 --- a/lib/timedbs/timedbs_test.go +++ b/lib/timedbs/timedbs_test.go @@ -1,21 +1,27 @@ -package timedbs_test +package timedbs import ( "context" "testing" "time" + "github.com/raulk/clock" "github.com/stretchr/testify/require" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" - - "github.com/filecoin-project/lotus/lib/timedbs" ) func TestTimedBSSimple(t *testing.T) { - tc := timedbs.NewTimedCacheBS(10 * time.Millisecond) - _ = tc.Start(context.Background()) + tc := NewTimedCacheBS(10 * time.Millisecond) + mClock := clock.NewMock() + mClock.Set(time.Now()) + tc.clock = mClock + tc.doneRotatingCh = make(chan struct{}) + + tc.Start(context.Background()) + mClock.Add(1) // IDK why it is needed but it makes it work + defer func() { _ = tc.Stop(context.Background()) }() @@ -36,7 +42,8 @@ func TestTimedBSSimple(t *testing.T) { require.NoError(t, err) require.True(t, has) - time.Sleep(15 * time.Millisecond) + mClock.Add(10 * time.Millisecond) + <-tc.doneRotatingCh // We should still have everything. has, err = tc.Has(b1.Cid()) @@ -60,8 +67,8 @@ func TestTimedBSSimple(t *testing.T) { require.NoError(t, err) require.ElementsMatch(t, ks, []cid.Cid{b1.Cid(), b2.Cid(), b3.Cid()}) - time.Sleep(10 * time.Millisecond) - + mClock.Add(10 * time.Millisecond) + <-tc.doneRotatingCh // should still have b2, and b3, but not b1 has, err = tc.Has(b1.Cid()) From 9fbd2a50ffb9c93a56647ee59bcab2b446bc5099 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 23 Oct 2020 20:50:17 +0200 Subject: [PATCH 873/885] Fix lint Signed-off-by: Jakub Sztandera --- lib/timedbs/timedbs_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/timedbs/timedbs_test.go b/lib/timedbs/timedbs_test.go index db21a5d79..e01215bbd 100644 --- a/lib/timedbs/timedbs_test.go +++ b/lib/timedbs/timedbs_test.go @@ -19,7 +19,7 @@ func TestTimedBSSimple(t *testing.T) { tc.clock = mClock tc.doneRotatingCh = make(chan struct{}) - tc.Start(context.Background()) + _ = tc.Start(context.Background()) mClock.Add(1) // IDK why it is needed but it makes it work defer func() { From c7e6c8eacc3b7b399a535a635c69e3e5712d6ac0 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 23 Oct 2020 21:45:08 +0200 Subject: [PATCH 874/885] Fix flaky TestChainExportImportFull Signed-off-by: Jakub Sztandera --- chain/store/store_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chain/store/store_test.go b/chain/store/store_test.go index 60167ad86..160527104 100644 --- a/chain/store/store_test.go +++ b/chain/store/store_test.go @@ -126,7 +126,7 @@ func TestChainExportImportFull(t *testing.T) { } buf := new(bytes.Buffer) - if err := cg.ChainStore().Export(context.TODO(), last, 100, false, buf); err != nil { + if err := cg.ChainStore().Export(context.TODO(), last, last.Height(), false, buf); err != nil { t.Fatal(err) } From 109b326ae2733725f172604503483890b8459a57 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 23 Oct 2020 12:32:07 -0700 Subject: [PATCH 875/885] Use older randomness for the PoSt commit on specs-actors version 2 After the upgrade, we're allowed to use randomness up to the challenge epoch. This safe-guards us against very large reorgs where a miner computes a PoSt very quickly. --- storage/wdpost_run.go | 9 +++++++++ storage/wdpost_run_test.go | 3 ++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 54d3d3dcc..fd3053a34 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -12,6 +12,7 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/go-state-types/dline" + "github.com/filecoin-project/go-state-types/network" "github.com/ipfs/go-cid" "go.opencensus.io/trace" @@ -151,7 +152,15 @@ func (s *WindowPoStScheduler) runSubmitPoST( defer span.End() // Get randomness from tickets + // use the challenge epoch if we've upgraded to network version 4 + // (actors version 2). We want to go back as far as possible to be safe. commEpoch := deadline.Open + if ver, err := s.api.StateNetworkVersion(ctx, types.EmptyTSK); err != nil { + log.Errorw("failed to get network version to determine PoSt epoch randomness lookback", "error", err) + } else if ver >= network.Version4 { + commEpoch = deadline.Challenge + } + commRand, err := s.api.ChainGetRandomnessFromTickets(ctx, ts.Key(), crypto.DomainSeparationTag_PoStChainCommit, commEpoch, nil) if err != nil { err = xerrors.Errorf("failed to get chain randomness from tickets for windowPost (ts=%d; deadline=%d): %w", ts.Height(), commEpoch, err) diff --git a/storage/wdpost_run_test.go b/storage/wdpost_run_test.go index bac86f022..4145a6e70 100644 --- a/storage/wdpost_run_test.go +++ b/storage/wdpost_run_test.go @@ -23,6 +23,7 @@ import ( tutils "github.com/filecoin-project/specs-actors/v2/support/testing" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/journal" @@ -47,7 +48,7 @@ func (m *mockStorageMinerAPI) StateMinerInfo(ctx context.Context, a address.Addr } func (m *mockStorageMinerAPI) StateNetworkVersion(ctx context.Context, key types.TipSetKey) (network.Version, error) { - panic("implement me") + return build.NewestNetworkVersion, nil } func (m *mockStorageMinerAPI) ChainGetRandomnessFromTickets(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error) { From b7b6ab779ae5607a0a9b398b82652510b7ff0e99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 23 Oct 2020 14:54:52 +0200 Subject: [PATCH 876/885] Make wallet market withdraw usable with miner addresses --- cli/wallet.go | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/cli/wallet.go b/cli/wallet.go index f231f3454..2501618f8 100644 --- a/cli/wallet.go +++ b/cli/wallet.go @@ -495,6 +495,11 @@ var walletMarketWithdraw = &cli.Command{ Usage: "Specify address to withdraw funds from, otherwise it will use the default wallet address", Aliases: []string{"f"}, }, + &cli.StringFlag{ + Name: "address", + Usage: "Market address to withdraw from (account or miner actor address, defaults to --from address)", + Aliases: []string{"a"}, + }, }, Action: func(cctx *cli.Context) error { api, closer, err := GetFullNodeAPI(cctx) @@ -504,19 +509,27 @@ var walletMarketWithdraw = &cli.Command{ defer closer() ctx := ReqContext(cctx) - var addr address.Address + var from address.Address if cctx.String("from") != "" { - addr, err = address.NewFromString(cctx.String("from")) + from, err = address.NewFromString(cctx.String("from")) if err != nil { return xerrors.Errorf("parsing from address: %w", err) } } else { - addr, err = api.WalletDefaultAddress(ctx) + from, err = api.WalletDefaultAddress(ctx) if err != nil { return xerrors.Errorf("getting default wallet address: %w", err) } } + addr := from + if cctx.String("address") != "" { + addr, err = address.NewFromString(cctx.String("address")) + if err != nil { + return xerrors.Errorf("parsing market address: %w", err) + } + } + bal, err := api.StateMarketBalance(ctx, addr, types.EmptyTSK) if err != nil { return xerrors.Errorf("getting market balance for address %s: %w", addr.String(), err) @@ -535,7 +548,7 @@ var walletMarketWithdraw = &cli.Command{ } if amt.GreaterThan(avail) { - return xerrors.Errorf("can't withdraw more funds than available; requested: %s; available: %s", amt, avail) + return xerrors.Errorf("can't withdraw more funds than available; requested: %s; available: %s", types.FIL(amt), types.FIL(avail)) } if avail.IsZero() { @@ -550,10 +563,10 @@ var walletMarketWithdraw = &cli.Command{ return xerrors.Errorf("serializing params: %w", err) } - fmt.Printf("Submitting WithdrawBalance message for amount %s for address %s\n", types.FIL(amt), addr.String()) + fmt.Printf("Submitting WithdrawBalance message for amount %s for address %s\n", types.FIL(amt), from.String()) smsg, err := api.MpoolPushMessage(ctx, &types.Message{ To: builtin.StorageMarketActorAddr, - From: addr, + From: from, Value: types.NewInt(0), Method: builtin.MethodsMarket.WithdrawBalance, Params: params, From 067de4508bdd4776aa1efb61540a5ffbbea654bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 23 Oct 2020 21:51:10 +0200 Subject: [PATCH 877/885] shed: Commad to decode messages --- cli/chain.go | 2 +- cli/state.go | 4 +- cmd/lotus-shed/main.go | 1 + cmd/lotus-shed/msg.go | 280 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 284 insertions(+), 3 deletions(-) create mode 100644 cmd/lotus-shed/msg.go diff --git a/cli/chain.go b/cli/chain.go index b1351549f..e2d0ebb4a 100644 --- a/cli/chain.go +++ b/cli/chain.go @@ -1291,7 +1291,7 @@ var chainDecodeParamsCmd = &cli.Command{ return xerrors.Errorf("getting actor: %w", err) } - pstr, err := jsonParams(act.Code, abi.MethodNum(method), params) + pstr, err := JsonParams(act.Code, abi.MethodNum(method), params) if err != nil { return err } diff --git a/cli/state.go b/cli/state.go index d0fcc4f95..13aa5c39b 100644 --- a/cli/state.go +++ b/cli/state.go @@ -1211,7 +1211,7 @@ func ComputeStateHTMLTempl(w io.Writer, ts *types.TipSet, o *api.ComputeStateOut "GetCode": getCode, "GetMethod": getMethod, "ToFil": toFil, - "JsonParams": jsonParams, + "JsonParams": JsonParams, "JsonReturn": jsonReturn, "IsSlow": isSlow, "IsVerySlow": isVerySlow, @@ -1298,7 +1298,7 @@ func sumGas(changes []*types.GasTrace) types.GasTrace { return out } -func jsonParams(code cid.Cid, method abi.MethodNum, params []byte) (string, error) { +func JsonParams(code cid.Cid, method abi.MethodNum, params []byte) (string, error) { methodMeta, found := stmgr.MethodsMap[code][method] if !found { return "", fmt.Errorf("method %d not found on actor %s", method, code) diff --git a/cmd/lotus-shed/main.go b/cmd/lotus-shed/main.go index 488e2a6ae..8201ec111 100644 --- a/cmd/lotus-shed/main.go +++ b/cmd/lotus-shed/main.go @@ -45,6 +45,7 @@ func main() { datastoreCmd, ledgerCmd, sectorsCmd, + msgCmd, } app := &cli.App{ diff --git a/cmd/lotus-shed/msg.go b/cmd/lotus-shed/msg.go new file mode 100644 index 000000000..63cfc86b9 --- /dev/null +++ b/cmd/lotus-shed/msg.go @@ -0,0 +1,280 @@ +package main + +import ( + "bytes" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + + "github.com/fatih/color" + + "github.com/ipfs/go-cid" + "github.com/urfave/cli/v2" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/big" + + "github.com/filecoin-project/lotus/chain/stmgr" + "github.com/filecoin-project/lotus/chain/types" + lcli "github.com/filecoin-project/lotus/cli" + "github.com/filecoin-project/specs-actors/v2/actors/builtin/multisig" +) + +var msgCmd = &cli.Command{ + Name: "msg", + Usage: "Translate message between various formats", + ArgsUsage: "Message in any form", + Action: func(cctx *cli.Context) error { + if cctx.Args().Len() != 1 { + return xerrors.Errorf("expected 1 argument") + } + + msg, err := messageFromString(cctx, cctx.Args().First()) + if err != nil { + return err + } + + switch msg := msg.(type) { + case *types.SignedMessage: + return printSignedMessage(cctx, msg) + case *types.Message: + return printMessage(cctx, msg) + default: + return xerrors.Errorf("this error message can't be printed") + } + }, +} + +func printSignedMessage(cctx *cli.Context, smsg *types.SignedMessage) error { + color.Green("Signed:") + color.Blue("CID: %s\n", smsg.Cid()) + + b, err := smsg.Serialize() + if err != nil { + return err + } + color.Magenta("HEX: %x\n", b) + color.Blue("B64: %s\n", base64.StdEncoding.EncodeToString(b)) + jm, err := json.MarshalIndent(smsg, "", " ") + if err != nil { + return xerrors.Errorf("marshaling as json: %w", err) + } + + color.Magenta("JSON: %s\n", string(jm)) + fmt.Println() + fmt.Println("---") + color.Green("Signed Message Details:") + fmt.Printf("Signature(hex): %x\n", smsg.Signature.Data) + fmt.Printf("Signature(b64): %s\n", base64.StdEncoding.EncodeToString(smsg.Signature.Data)) + + sigtype, err := smsg.Signature.Type.Name() + if err != nil { + sigtype = err.Error() + } + fmt.Printf("Signature type: %d (%s)\n", smsg.Signature.Type, sigtype) + + fmt.Println("-------") + return printMessage(cctx, &smsg.Message) +} + +func printMessage(cctx *cli.Context, msg *types.Message) error { + if msg.Version != 0x6d736967 { + color.Green("Unsigned:") + color.Yellow("CID: %s\n", msg.Cid()) + + b, err := msg.Serialize() + if err != nil { + return err + } + color.Cyan("HEX: %x\n", b) + color.Yellow("B64: %s\n", base64.StdEncoding.EncodeToString(b)) + + jm, err := json.MarshalIndent(msg, "", " ") + if err != nil { + return xerrors.Errorf("marshaling as json: %w", err) + } + + color.Cyan("JSON: %s\n", string(jm)) + fmt.Println() + } else { + color.Green("Msig Propose:") + pp := &multisig.ProposeParams{ + To: msg.To, + Value: msg.Value, + Method: msg.Method, + Params: msg.Params, + } + var b bytes.Buffer + if err := pp.MarshalCBOR(&b); err != nil { + return err + } + + color.Cyan("HEX: %x\n", b.Bytes()) + color.Yellow("B64: %s\n", base64.StdEncoding.EncodeToString(b.Bytes())) + jm, err := json.MarshalIndent(pp, "", " ") + if err != nil { + return xerrors.Errorf("marshaling as json: %w", err) + } + + color.Cyan("JSON: %s\n", string(jm)) + fmt.Println() + } + + fmt.Println("---") + color.Green("Message Details:") + fmt.Println("Value:", types.FIL(msg.Value)) + fmt.Println("Max Fees:", types.FIL(msg.RequiredFunds())) + fmt.Println("Max Total Cost:", types.FIL(big.Add(msg.RequiredFunds(), msg.Value))) + + api, closer, err := lcli.GetFullNodeAPI(cctx) + if err != nil { + return err + } + + defer closer() + ctx := lcli.ReqContext(cctx) + + toact, err := api.StateGetActor(ctx, msg.To, types.EmptyTSK) + if err != nil { + return nil + } + + fmt.Println("Method:", stmgr.MethodsMap[toact.Code][msg.Method].Name) + p, err := lcli.JsonParams(toact.Code, msg.Method, msg.Params) + if err != nil { + return err + } + + fmt.Println("Params:", p) + + return nil +} + +func messageFromString(cctx *cli.Context, smsg string) (types.ChainMsg, error) { + // a CID is least likely to just decode + if c, err := cid.Parse(smsg); err == nil { + return messageFromCID(cctx, c) + } + + // try baseX serializations next + { + // hex first, some hay strings may be decodable as b64 + if b, err := hex.DecodeString(smsg); err == nil { + return messageFromBytes(cctx, b) + } + + // b64 next + if b, err := base64.StdEncoding.DecodeString(smsg); err == nil { + return messageFromBytes(cctx, b) + } + + // b64u?? + if b, err := base64.URLEncoding.DecodeString(smsg); err == nil { + return messageFromBytes(cctx, b) + } + } + + // maybe it's json? + if _, err := messageFromJson(cctx, []byte(smsg)); err == nil { + return nil, err + } + + // declare defeat + return nil, xerrors.Errorf("couldn't decode the message") +} + +func messageFromJson(cctx *cli.Context, msgb []byte) (types.ChainMsg, error) { + // Unsigned + { + var msg types.Message + if err := json.Unmarshal(msgb, &msg); err == nil { + if msg.To != address.Undef { + return &msg, nil + } + } + } + + // Signed + { + var msg types.SignedMessage + if err := json.Unmarshal(msgb, &msg); err == nil { + if msg.Message.To != address.Undef { + return &msg, nil + } + } + } + + return nil, xerrors.New("probably not a json-serialized message") +} + +func messageFromBytes(cctx *cli.Context, msgb []byte) (types.ChainMsg, error) { + // Signed + { + var msg types.SignedMessage + if err := msg.UnmarshalCBOR(bytes.NewReader(msgb)); err == nil { + return &msg, nil + } + } + + // Unsigned + { + var msg types.Message + if err := msg.UnmarshalCBOR(bytes.NewReader(msgb)); err == nil { + return &msg, nil + } + } + + // Multisig propose? + { + var pp multisig.ProposeParams + if err := pp.UnmarshalCBOR(bytes.NewReader(msgb)); err == nil { + i, err := address.NewIDAddress(0) + if err != nil { + return nil, err + } + + return &types.Message{ + // Hack(-ish) + Version: 0x6d736967, + From: i, + + To: pp.To, + Value: pp.Value, + + Method: pp.Method, + Params: pp.Params, + + GasFeeCap: big.Zero(), + GasPremium: big.Zero(), + }, nil + } + } + + // Encoded json??? + { + if msg, err := messageFromJson(cctx, msgb); err == nil { + return msg, nil + } + } + + return nil, xerrors.New("probably not a cbor-serialized message") +} + +func messageFromCID(cctx *cli.Context, c cid.Cid) (types.ChainMsg, error) { + api, closer, err := lcli.GetFullNodeAPI(cctx) + if err != nil { + return nil, err + } + + defer closer() + ctx := lcli.ReqContext(cctx) + + msgb, err := api.ChainReadObj(ctx, c) + if err != nil { + return nil, err + } + + return messageFromBytes(cctx, msgb) +} From 8bb69f1e9cfe070d1334eb04ce99db713105d994 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 23 Oct 2020 14:52:26 -0700 Subject: [PATCH 878/885] Use pre-computed parent state root for lookback info There's no need to call TipSetState and recompute it. --- chain/gen/mining.go | 7 +------ chain/stmgr/utils.go | 37 ++++++++++++++++++++++++------------- chain/sub/incoming.go | 2 +- chain/sync.go | 7 +------ 4 files changed, 27 insertions(+), 26 deletions(-) diff --git a/chain/gen/mining.go b/chain/gen/mining.go index 5fc56e8b2..cca4b6169 100644 --- a/chain/gen/mining.go +++ b/chain/gen/mining.go @@ -27,16 +27,11 @@ func MinerCreateBlock(ctx context.Context, sm *stmgr.StateManager, w api.WalletA return nil, xerrors.Errorf("failed to load tipset state: %w", err) } - lbts, err := stmgr.GetLookbackTipSetForRound(ctx, sm, pts, bt.Epoch) + _, lbst, err := stmgr.GetLookbackTipSetForRound(ctx, sm, pts, bt.Epoch) if err != nil { return nil, xerrors.Errorf("getting lookback miner actor state: %w", err) } - lbst, _, err := sm.TipSetState(ctx, lbts) - if err != nil { - return nil, err - } - worker, err := stmgr.GetMinerWorkerRaw(ctx, sm, lbst, bt.Miner) if err != nil { return nil, xerrors.Errorf("failed to get miner worker: %w", err) diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index 54f75c138..486805786 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -391,7 +391,7 @@ func ComputeState(ctx context.Context, sm *StateManager, height abi.ChainEpoch, return root, trace, nil } -func GetLookbackTipSetForRound(ctx context.Context, sm *StateManager, ts *types.TipSet, round abi.ChainEpoch) (*types.TipSet, error) { +func GetLookbackTipSetForRound(ctx context.Context, sm *StateManager, ts *types.TipSet, round abi.ChainEpoch) (*types.TipSet, cid.Cid, error) { var lbr abi.ChainEpoch lb := policy.GetWinningPoStSectorSetLookback(sm.GetNtwkVersion(ctx, round)) if round > lb { @@ -399,16 +399,33 @@ func GetLookbackTipSetForRound(ctx context.Context, sm *StateManager, ts *types. } // more null blocks than our lookback - if lbr > ts.Height() { - return ts, nil + if lbr >= ts.Height() { + // This should never happen at this point, but may happen before + // network version 3 (where the lookback was only 10 blocks). + st, _, err := sm.TipSetState(ctx, ts) + if err != nil { + return nil, cid.Undef, err + } + return ts, st, nil } - lbts, err := sm.ChainStore().GetTipsetByHeight(ctx, lbr, ts, true) + // Get the tipset after the lookback tipset, or the next non-null one. + nextTs, err := sm.ChainStore().GetTipsetByHeight(ctx, lbr+1, ts, false) if err != nil { - return nil, xerrors.Errorf("failed to get lookback tipset: %w", err) + return nil, cid.Undef, xerrors.Errorf("failed to get lookback tipset+1: %w", err) } - return lbts, nil + if lbr > nextTs.Height() { + return nil, cid.Undef, xerrors.Errorf("failed to find non-null tipset %s (%d) which is known to exist, found %s (%d)", ts.Key(), ts.Height(), nextTs.Key(), nextTs.Height()) + + } + + lbts, err := sm.ChainStore().GetTipSetFromKey(nextTs.Parents()) + if err != nil { + return nil, cid.Undef, xerrors.Errorf("failed to resolve lookback tipset: %w", err) + } + + return lbts, nextTs.ParentState(), nil } func MinerGetBaseInfo(ctx context.Context, sm *StateManager, bcs beacon.Schedule, tsk types.TipSetKey, round abi.ChainEpoch, maddr address.Address, pv ffiwrapper.Verifier) (*api.MiningBaseInfo, error) { @@ -436,17 +453,11 @@ func MinerGetBaseInfo(ctx context.Context, sm *StateManager, bcs beacon.Schedule rbase = entries[len(entries)-1] } - lbts, err := GetLookbackTipSetForRound(ctx, sm, ts, round) + lbts, lbst, err := GetLookbackTipSetForRound(ctx, sm, ts, round) if err != nil { return nil, xerrors.Errorf("getting lookback miner actor state: %w", err) } - // TODO: load the state instead of computing it? - lbst, _, err := sm.TipSetState(ctx, lbts) - if err != nil { - return nil, err - } - act, err := sm.LoadActorRaw(ctx, maddr, lbst) if xerrors.Is(err, types.ErrActorNotFound) { _, err := sm.LoadActor(ctx, maddr, ts) diff --git a/chain/sub/incoming.go b/chain/sub/incoming.go index 625c8d1e2..be103876d 100644 --- a/chain/sub/incoming.go +++ b/chain/sub/incoming.go @@ -484,7 +484,7 @@ func (bv *BlockValidator) checkPowerAndGetWorkerKey(ctx context.Context, bh *typ // we check that the miner met the minimum power at the lookback tipset baseTs := bv.chain.GetHeaviestTipSet() - lbts, err := stmgr.GetLookbackTipSetForRound(ctx, bv.stmgr, baseTs, bh.Height) + lbts, _, err := stmgr.GetLookbackTipSetForRound(ctx, bv.stmgr, baseTs, bh.Height) if err != nil { log.Warnf("failed to load lookback tipset for incoming block: %s", err) return address.Undef, ErrSoftFailure diff --git a/chain/sync.go b/chain/sync.go index 8da093cb6..1410dd2a7 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -730,16 +730,11 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock, use return xerrors.Errorf("load parent tipset failed (%s): %w", h.Parents, err) } - lbts, err := stmgr.GetLookbackTipSetForRound(ctx, syncer.sm, baseTs, h.Height) + lbts, lbst, err := stmgr.GetLookbackTipSetForRound(ctx, syncer.sm, baseTs, h.Height) if err != nil { return xerrors.Errorf("failed to get lookback tipset for block: %w", err) } - lbst, _, err := syncer.sm.TipSetState(ctx, lbts) - if err != nil { - return xerrors.Errorf("failed to compute lookback tipset state (epoch %d): %w", lbts.Height(), err) - } - prevBeacon, err := syncer.store.GetLatestBeaconEntry(baseTs) if err != nil { return xerrors.Errorf("failed to get latest beacon entry: %w", err) From 06af2fb52be05a20351d4a1e22edf6a2ba214d73 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 23 Oct 2020 14:57:21 -0700 Subject: [PATCH 879/885] Remove the worker key cache and use the correct worker key We need to use the worker key from the lookback tipset state. We can't reasonably maintain this cache, unfortunately (unless we kept some form of "per-height" cache). --- chain/sub/incoming.go | 65 ++++++++----------------------------------- 1 file changed, 11 insertions(+), 54 deletions(-) diff --git a/chain/sub/incoming.go b/chain/sub/incoming.go index be103876d..1701866eb 100644 --- a/chain/sub/incoming.go +++ b/chain/sub/incoming.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "sync" "time" "golang.org/x/xerrors" @@ -27,14 +26,11 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain" - "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/messagepool" - "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/blockstore" - "github.com/filecoin-project/lotus/lib/bufbstore" "github.com/filecoin-project/lotus/lib/sigs" "github.com/filecoin-project/lotus/metrics" "github.com/filecoin-project/lotus/node/impl/client" @@ -227,9 +223,6 @@ type BlockValidator struct { // necessary for block validation chain *store.ChainStore stmgr *stmgr.StateManager - - mx sync.Mutex - keycache map[string]address.Address } func NewBlockValidator(self peer.ID, chain *store.ChainStore, stmgr *stmgr.StateManager, blacklist func(peer.ID)) *BlockValidator { @@ -242,7 +235,6 @@ func NewBlockValidator(self peer.ID, chain *store.ChainStore, stmgr *stmgr.State recvBlocks: newBlockReceiptCache(), chain: chain, stmgr: stmgr, - keycache: make(map[string]address.Address), } } @@ -436,60 +428,25 @@ func (bv *BlockValidator) validateMsgMeta(ctx context.Context, msg *types.BlockM } func (bv *BlockValidator) checkPowerAndGetWorkerKey(ctx context.Context, bh *types.BlockHeader) (address.Address, error) { - addr := bh.Miner - - bv.mx.Lock() - key, ok := bv.keycache[addr.String()] - bv.mx.Unlock() - if !ok { - // TODO I have a feeling all this can be simplified by cleverer DI to use the API - ts := bv.chain.GetHeaviestTipSet() - st, _, err := bv.stmgr.TipSetState(ctx, ts) - if err != nil { - return address.Undef, err - } - - buf := bufbstore.NewBufferedBstore(bv.chain.Blockstore()) - cst := cbor.NewCborStore(buf) - state, err := state.LoadStateTree(cst, st) - if err != nil { - return address.Undef, err - } - act, err := state.GetActor(addr) - if err != nil { - return address.Undef, err - } - - mst, err := miner.Load(bv.chain.Store(ctx), act) - if err != nil { - return address.Undef, err - } - - info, err := mst.Info() - if err != nil { - return address.Undef, err - } - - worker := info.Worker - key, err = bv.stmgr.ResolveToKeyAddress(ctx, worker, ts) - if err != nil { - return address.Undef, err - } - - bv.mx.Lock() - bv.keycache[addr.String()] = key - bv.mx.Unlock() - } - // we check that the miner met the minimum power at the lookback tipset baseTs := bv.chain.GetHeaviestTipSet() - lbts, _, err := stmgr.GetLookbackTipSetForRound(ctx, bv.stmgr, baseTs, bh.Height) + lbts, lbst, err := stmgr.GetLookbackTipSetForRound(ctx, bv.stmgr, baseTs, bh.Height) if err != nil { log.Warnf("failed to load lookback tipset for incoming block: %s", err) return address.Undef, ErrSoftFailure } + key, err := stmgr.GetMinerWorkerRaw(ctx, bv.stmgr, lbst, bh.Miner) + if err != nil { + log.Warnf("failed to resolve worker key for miner %s: %s", bh.Miner, err) + return address.Undef, ErrSoftFailure + } + + // NOTE: we check to see if the miner was eligible in the lookback + // tipset - 1 for historical reasons. DO NOT use the lookback state + // returned by GetLookbackTipSetForRound. + eligible, err := stmgr.MinerEligibleToMine(ctx, bv.stmgr, bh.Miner, baseTs, lbts) if err != nil { log.Warnf("failed to determine if incoming block's miner has minimum power: %s", err) From 8b8d81e767064c1eb46d8e49a75d56655804479a Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Sat, 24 Oct 2020 00:16:45 -0400 Subject: [PATCH 880/885] Lotus version 1.1.1 --- CHANGELOG.md | 45 +++++++++++++++++++++++++++++++++++++++++++++ build/version.go | 2 +- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c21ffb685..070d2414c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,50 @@ # Lotus changelog +# 1.1.1 / 2020-10-24 + +This is a patch release of Lotus that addresses some issues caused by when miners change their worker keys. Miners and node operators should update to this release as soon as possible, especially any miner who has changed their worker key recently. + +## Changes + +- Miner finder for interactive client deal CLI (https://github.com/filecoin-project/lotus/pull/4504) +- Disable blockstore bloom filter (https://github.com/filecoin-project/lotus/pull/4512) +- Add api for getting status given a code (https://github.com/filecoin-project/lotus/pull/4210) +- add batch api for push messages (https://github.com/filecoin-project/lotus/pull/4236) +- add measure datastore wrapper around bench chain datastore (https://github.com/filecoin-project/lotus/pull/4302) +- Look at block base fee for PCR (https://github.com/filecoin-project/lotus/pull/4313) +- Add a shed util to determine % of power that has won a block (https://github.com/filecoin-project/lotus/pull/4318) +- Shed/borked cmd (https://github.com/filecoin-project/lotus/pull/4339) +- optimize mining code (https://github.com/filecoin-project/lotus/pull/4379) +- heaviestTipSet reurning nil is a ok (https://github.com/filecoin-project/lotus/pull/4523) +- Remove most v0 actor imports (https://github.com/filecoin-project/lotus/pull/4383) +- Small chain export optimization (https://github.com/filecoin-project/lotus/pull/4536) +- Add block list to pcr (https://github.com/filecoin-project/lotus/pull/4314) +- Fix circ supply default in conformance (https://github.com/filecoin-project/lotus/pull/4449) +- miner: fix init --create-worker-key (https://github.com/filecoin-project/lotus/pull/4475) +- make push and addLocal atomic (https://github.com/filecoin-project/lotus/pull/4500) +- add some methods that oni needs (https://github.com/filecoin-project/lotus/pull/4501) +- MinerGetBaseInfo: if miner is not found in lookback, check current (https://github.com/filecoin-project/lotus/pull/4508) +- Delete wallet from local wallet cache (https://github.com/filecoin-project/lotus/pull/4526) +- Fix lotus-shed ledger list (https://github.com/filecoin-project/lotus/pull/4521) +- Manage sectors by size instead of proof type (https://github.com/filecoin-project/lotus/pull/4511) +- Feat/api request metrics wrapper (https://github.com/filecoin-project/lotus/pull/4516) +- Fix chain sync stopping to sync (https://github.com/filecoin-project/lotus/pull/4541) +- Use the correct lookback for the worker key when creating blocks (https://github.com/filecoin-project/lotus/pull/4539) +- Cleanup test initialization and always validate VRFs in tests (https://github.com/filecoin-project/lotus/pull/4538) +- Add a market WithdrawBalance CLI (https://github.com/filecoin-project/lotus/pull/4524) +- wallet list: Add market balance and ID address flags (https://github.com/filecoin-project/lotus/pull/4555) +- tvx simulate command; tvx extract --ignore-sanity-checks (https://github.com/filecoin-project/lotus/pull/4554) +- lotus-lite: CLI tests for `lotus client` commands (https://github.com/filecoin-project/lotus/pull/4497) +- lite-mode - market storage and retrieval clients (https://github.com/filecoin-project/lotus/pull/4263) +- Chore: update drand to v1.2.0 (https://github.com/filecoin-project/lotus/pull/4420) +- Fix random test failures (https://github.com/filecoin-project/lotus/pull/4559) +- Fix flaky TestTimedBSSimple (https://github.com/filecoin-project/lotus/pull/4561) +- Make wallet market withdraw usable with miner addresses (https://github.com/filecoin-project/lotus/pull/4556) +- Fix flaky TestChainExportImportFull (https://github.com/filecoin-project/lotus/pull/4564) +- Use older randomness for the PoSt commit on specs-actors version 2 (https://github.com/filecoin-project/lotus/pull/4563) +- shed: Commad to decode messages (https://github.com/filecoin-project/lotus/pull/4565) +- Fetch worker key from correct block on sync (https://github.com/filecoin-project/lotus/pull/4573) + # 1.1.0 / 2020-10-20 This is a mandatory release that introduces the first post-liftoff upgrade to the Filecoin network. The changes that break consensus are an upgrade to specs-actors v2.2.0 at epoch 170000. diff --git a/build/version.go b/build/version.go index b8941cb7a..701e47d4a 100644 --- a/build/version.go +++ b/build/version.go @@ -29,7 +29,7 @@ func buildType() string { } // BuildVersion is the local build version, set by build system -const BuildVersion = "1.1.0" +const BuildVersion = "1.1.1" func UserVersion() string { return BuildVersion + buildType() + CurrentCommit From f23034305fc4cc9129dd29384e135506ac936ae2 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Sat, 24 Oct 2020 02:57:17 -0700 Subject: [PATCH 881/885] Handle worker key changes correctly in runtime --- chain/gen/genesis/miners.go | 4 +-- chain/stmgr/call.go | 2 ++ chain/stmgr/stmgr.go | 1 + chain/stmgr/utils.go | 11 +++++++ chain/vm/syscalls.go | 59 ++++++++++++++++++++++--------------- chain/vm/vm.go | 15 ++++++---- 6 files changed, 61 insertions(+), 31 deletions(-) diff --git a/chain/gen/genesis/miners.go b/chain/gen/genesis/miners.go index 49e09d792..be83a8711 100644 --- a/chain/gen/genesis/miners.go +++ b/chain/gen/genesis/miners.go @@ -56,9 +56,9 @@ func (fss *fakedSigSyscalls) VerifySignature(signature crypto.Signature, signer } func mkFakedSigSyscalls(base vm.SyscallBuilder) vm.SyscallBuilder { - return func(ctx context.Context, cstate *state.StateTree, cst cbor.IpldStore) runtime2.Syscalls { + return func(ctx context.Context, rt *vm.Runtime) runtime2.Syscalls { return &fakedSigSyscalls{ - base(ctx, cstate, cst), + base(ctx, rt), } } } diff --git a/chain/stmgr/call.go b/chain/stmgr/call.go index 3d7d284bc..bb0f0e5ec 100644 --- a/chain/stmgr/call.go +++ b/chain/stmgr/call.go @@ -64,6 +64,7 @@ func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types. CircSupplyCalc: sm.GetVMCirculatingSupply, NtwkVersion: sm.GetNtwkVersion, BaseFee: types.NewInt(0), + LookbackState: LookbackStateGetterForTipset(sm, ts), } vmi, err := sm.newVM(ctx, vmopt) @@ -178,6 +179,7 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri CircSupplyCalc: sm.GetVMCirculatingSupply, NtwkVersion: sm.GetNtwkVersion, BaseFee: ts.Blocks()[0].ParentBaseFee, + LookbackState: LookbackStateGetterForTipset(sm, ts), } vmi, err := sm.newVM(ctx, vmopt) if err != nil { diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index 2822344b0..7e5809a84 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -241,6 +241,7 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp CircSupplyCalc: sm.GetVMCirculatingSupply, NtwkVersion: sm.GetNtwkVersion, BaseFee: baseFee, + LookbackState: LookbackStateGetterForTipset(sm, ts), } return sm.newVM(ctx, vmopt) diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index 486805786..5b144281d 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -366,6 +366,7 @@ func ComputeState(ctx context.Context, sm *StateManager, height abi.ChainEpoch, CircSupplyCalc: sm.GetVMCirculatingSupply, NtwkVersion: sm.GetNtwkVersion, BaseFee: ts.Blocks()[0].ParentBaseFee, + LookbackState: LookbackStateGetterForTipset(sm, ts), } vmi, err := sm.newVM(ctx, vmopt) if err != nil { @@ -391,6 +392,16 @@ func ComputeState(ctx context.Context, sm *StateManager, height abi.ChainEpoch, return root, trace, nil } +func LookbackStateGetterForTipset(sm *StateManager, ts *types.TipSet) vm.LookbackStateGetter { + return func(ctx context.Context, round abi.ChainEpoch) (*state.StateTree, error) { + _, st, err := GetLookbackTipSetForRound(ctx, sm, ts, round) + if err != nil { + return nil, err + } + return sm.StateTree(st) + } +} + func GetLookbackTipSetForRound(ctx context.Context, sm *StateManager, ts *types.TipSet, round abi.ChainEpoch) (*types.TipSet, cid.Cid, error) { var lbr abi.ChainEpoch lb := policy.GetWinningPoStSectorSetLookback(sm.GetNtwkVersion(ctx, round)) diff --git a/chain/vm/syscalls.go b/chain/vm/syscalls.go index 347c3409c..d2f1f77d3 100644 --- a/chain/vm/syscalls.go +++ b/chain/vm/syscalls.go @@ -34,15 +34,18 @@ func init() { // Actual type is defined in chain/types/vmcontext.go because the VMContext interface is there -type SyscallBuilder func(ctx context.Context, cstate *state.StateTree, cst cbor.IpldStore) runtime2.Syscalls +type SyscallBuilder func(ctx context.Context, rt *Runtime) runtime2.Syscalls func Syscalls(verifier ffiwrapper.Verifier) SyscallBuilder { - return func(ctx context.Context, cstate *state.StateTree, cst cbor.IpldStore) runtime2.Syscalls { + return func(ctx context.Context, rt *Runtime) runtime2.Syscalls { + return &syscallShim{ ctx: ctx, - cstate: cstate, - cst: cst, + actor: rt.Receiver(), + cstate: rt.state, + cst: rt.cst, + lbState: rt.vm.lbStateGet, verifier: verifier, } @@ -52,6 +55,8 @@ func Syscalls(verifier ffiwrapper.Verifier) SyscallBuilder { type syscallShim struct { ctx context.Context + lbState LookbackStateGetter + actor address.Address cstate *state.StateTree cst cbor.IpldStore verifier ffiwrapper.Verifier @@ -184,26 +189,7 @@ func (ss *syscallShim) VerifyConsensusFault(a, b, extra []byte) (*runtime2.Conse } func (ss *syscallShim) VerifyBlockSig(blk *types.BlockHeader) error { - - // get appropriate miner actor - act, err := ss.cstate.GetActor(blk.Miner) - if err != nil { - return err - } - - // use that to get the miner state - mas, err := miner.Load(adt.WrapStore(ss.ctx, ss.cst), act) - if err != nil { - return err - } - - info, err := mas.Info() - if err != nil { - return err - } - - // and use to get resolved workerKey - waddr, err := ResolveToKeyAddr(ss.cstate, ss.cst, info.Worker) + waddr, err := ss.workerKeyAtLookback(blk.Height) if err != nil { return err } @@ -215,6 +201,31 @@ func (ss *syscallShim) VerifyBlockSig(blk *types.BlockHeader) error { return nil } +func (ss *syscallShim) workerKeyAtLookback(height abi.ChainEpoch) (address.Address, error) { + lbState, err := ss.lbState(ss.ctx, height) + if err != nil { + return address.Undef, err + } + // get appropriate miner actor + act, err := lbState.GetActor(ss.actor) + if err != nil { + return address.Undef, err + } + + // use that to get the miner state + mas, err := miner.Load(adt.WrapStore(ss.ctx, ss.cst), act) + if err != nil { + return address.Undef, err + } + + info, err := mas.Info() + if err != nil { + return address.Undef, err + } + + return ResolveToKeyAddr(ss.cstate, ss.cst, info.Worker) +} + func (ss *syscallShim) VerifyPoSt(proof proof2.WindowPoStVerifyInfo) error { ok, err := ss.verifier.VerifyWindowPoSt(context.TODO(), proof) if err != nil { diff --git a/chain/vm/vm.go b/chain/vm/vm.go index a7aa05719..8b7f78074 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -134,11 +134,6 @@ func (vm *VM) makeRuntime(ctx context.Context, msg *types.Message, parent *Runti Blocks: &gasChargingBlocks{rt.chargeGasFunc(2), rt.pricelist, vm.cst.Blocks}, Atlas: vm.cst.Atlas, } - rt.Syscalls = pricedSyscalls{ - under: vm.Syscalls(ctx, vm.cstate, rt.cst), - chargeGas: rt.chargeGasFunc(1), - pl: rt.pricelist, - } vmm := *msg resF, ok := rt.ResolveAddress(msg.From) @@ -156,6 +151,12 @@ func (vm *VM) makeRuntime(ctx context.Context, msg *types.Message, parent *Runti rt.Message = &Message{msg: vmm} } + rt.Syscalls = pricedSyscalls{ + under: vm.Syscalls(ctx, rt), + chargeGas: rt.chargeGasFunc(1), + pl: rt.pricelist, + } + return rt } @@ -169,6 +170,7 @@ func (vm *UnsafeVM) MakeRuntime(ctx context.Context, msg *types.Message) *Runtim type CircSupplyCalculator func(context.Context, abi.ChainEpoch, *state.StateTree) (abi.TokenAmount, error) type NtwkVersionGetter func(context.Context, abi.ChainEpoch) network.Version +type LookbackStateGetter func(context.Context, abi.ChainEpoch) (*state.StateTree, error) type VM struct { cstate *state.StateTree @@ -181,6 +183,7 @@ type VM struct { circSupplyCalc CircSupplyCalculator ntwkVersion NtwkVersionGetter baseFee abi.TokenAmount + lbStateGet LookbackStateGetter Syscalls SyscallBuilder } @@ -194,6 +197,7 @@ type VMOpts struct { CircSupplyCalc CircSupplyCalculator NtwkVersion NtwkVersionGetter // TODO: stebalien: In what cases do we actually need this? It seems like even when creating new networks we want to use the 'global'/build-default version getter BaseFee abi.TokenAmount + LookbackState LookbackStateGetter } func NewVM(ctx context.Context, opts *VMOpts) (*VM, error) { @@ -216,6 +220,7 @@ func NewVM(ctx context.Context, opts *VMOpts) (*VM, error) { ntwkVersion: opts.NtwkVersion, Syscalls: opts.Syscalls, baseFee: opts.BaseFee, + lbStateGet: opts.LookbackState, }, nil } From 7b3f1c2b85a0eff2d7f8677aaf79da6dfac07bd9 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Sat, 24 Oct 2020 06:33:31 -0400 Subject: [PATCH 882/885] Lotus version 1.1.2 --- CHANGELOG.md | 8 ++++++++ build/version.go | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 070d2414c..88a30c91d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Lotus changelog +# 1.1.2 / 2020-10-24 + +This is a patch release of Lotus that builds on the fixes involving worker keys that was introduced in v1.1.1. Miners and node operators should update to this release as soon as possible in order to ensure their blocks are propagated and validated. + +## Changes + +- Handle worker key changes correctly in runtime (https://github.com/filecoin-project/lotus/pull/4579) + # 1.1.1 / 2020-10-24 This is a patch release of Lotus that addresses some issues caused by when miners change their worker keys. Miners and node operators should update to this release as soon as possible, especially any miner who has changed their worker key recently. diff --git a/build/version.go b/build/version.go index 701e47d4a..a8487f98b 100644 --- a/build/version.go +++ b/build/version.go @@ -29,7 +29,7 @@ func buildType() string { } // BuildVersion is the local build version, set by build system -const BuildVersion = "1.1.1" +const BuildVersion = "1.1.2" func UserVersion() string { return BuildVersion + buildType() + CurrentCommit From 8bf8757a3e08324ac203d2e7205009af6232d74d Mon Sep 17 00:00:00 2001 From: Peter Rabbitson Date: Fri, 11 Sep 2020 16:42:17 +0200 Subject: [PATCH 883/885] Nerpa-specific consensus adjustments --- build/params_mainnet.go | 50 ++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/build/params_mainnet.go b/build/params_mainnet.go index 94deedfec..d48ba6aa2 100644 --- a/build/params_mainnet.go +++ b/build/params_mainnet.go @@ -5,54 +5,52 @@ package build import ( - "math" - "os" - - "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/actors/policy" + "github.com/filecoin-project/specs-actors/actors/builtin/miner" builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin" ) var DrandSchedule = map[abi.ChainEpoch]DrandEnum{ - 0: DrandIncentinet, - UpgradeSmokeHeight: DrandMainnet, + 0: DrandMainnet, } -const UpgradeBreezeHeight = 41280 -const BreezeGasTampingDuration = 120 +const UpgradeBreezeHeight = -1 +const BreezeGasTampingDuration = 0 -const UpgradeSmokeHeight = 51000 +const UpgradeSmokeHeight = -1 -const UpgradeIgnitionHeight = 94000 -const UpgradeRefuelHeight = 130800 +const UpgradeIgnitionHeight = -2 +const UpgradeRefuelHeight = -3 -var UpgradeActorsV2Height = abi.ChainEpoch(138720) +const UpgradeTapeHeight = -4 -const UpgradeTapeHeight = 140760 +const UpgradeLiftoffHeight = -5 +const UpgradeActorsV2Height = 120 // critical: the network can bootstrap from v1 only -// This signals our tentative epoch for mainnet launch. Can make it later, but not earlier. -// Miners, clients, developers, custodians all need time to prepare. -// We still have upgrades and state changes to do, but can happen after signaling timing here. -const UpgradeLiftoffHeight = 148888 - -const UpgradeKumquatHeight = 170000 +const UpgradeKumquatHeight = -6 func init() { - policy.SetConsensusMinerMinPower(abi.NewStoragePower(10 << 40)) + // Minimum block production power is set to 4 TiB + // Rationale is to discourage small-scale miners from trying to take over the network + // One needs to invest in ~2.3x the compute to break consensus, making it not worth it + // + // DOWNSIDE: the fake-seals need to be kept alive/protected, otherwise network will seize + // + policy.SetConsensusMinerMinPower(abi.NewStoragePower(4 << 40)) + policy.SetSupportedProofTypes( + abi.RegisteredSealProof_StackedDrg512MiBV1, abi.RegisteredSealProof_StackedDrg32GiBV1, abi.RegisteredSealProof_StackedDrg64GiBV1, ) - if os.Getenv("LOTUS_USE_TEST_ADDRESSES") != "1" { - SetAddressNetwork(address.Mainnet) - } + // Lower the most time-consuming parts of PoRep + miner.PreCommitChallengeDelay = abi.ChainEpoch(10) - if os.Getenv("LOTUS_DISABLE_V2_ACTOR_MIGRATION") == "1" { - UpgradeActorsV2Height = math.MaxInt64 - } + // TODO - make this a variable + //miner.WPoStChallengeLookback = abi.ChainEpoch(2) Devnet = false } From 4fd4665e33fced7e78b85d0908c9d194b4ecdba5 Mon Sep 17 00:00:00 2001 From: Peter Rabbitson Date: Tue, 27 Oct 2020 14:10:02 +0100 Subject: [PATCH 884/885] New verifreg key not to bother Why --- chain/gen/gen.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chain/gen/gen.go b/chain/gen/gen.go index d56f285a0..f0ac0af35 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -110,7 +110,7 @@ var DefaultVerifregRootkeyActor = genesis.Actor{ Meta: rootkeyMultisig.ActorMeta(), } -var remAccTestKey, _ = address.NewFromString("t1ceb34gnsc6qk5dt6n7xg6ycwzasjhbxm3iylkiy") +var remAccTestKey, _ = address.NewFromString("t1arngf6wtnep7psj5etoepzcdbf4mhy27gxagbxq") var remAccMeta = genesis.MultisigMeta{ Signers: []address.Address{remAccTestKey}, Threshold: 1, From 609b6fa72b15e0790913647ac8e8f3d8750c8414 Mon Sep 17 00:00:00 2001 From: Liviu Damian Date: Tue, 27 Oct 2020 18:05:33 +0200 Subject: [PATCH 885/885] Nerpa v1.1.2 genesis --- build/bootstrap/bootstrappers.pi | 8 ++++---- build/genesis/devnet.car | Bin 2637324 -> 2637323 bytes 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build/bootstrap/bootstrappers.pi b/build/bootstrap/bootstrappers.pi index 5b7c861f2..78df3029a 100644 --- a/build/bootstrap/bootstrappers.pi +++ b/build/bootstrap/bootstrappers.pi @@ -1,4 +1,4 @@ -/dns4/bootstrap-0.nerpa.interplanetary.dev/tcp/1347/p2p/12D3KooWH18xUp3ywhrxdt1NKPKdKJN9PUUoWVvghf7SHig7tabB -/dns4/bootstrap-1.nerpa.interplanetary.dev/tcp/1347/p2p/12D3KooWPPYqe8dNkcvqDbdFUZkq6FAgjUYgZ5UCb8JEXVpvYpeW -/dns4/bootstrap-2.nerpa.interplanetary.dev/tcp/1347/p2p/12D3KooWANTw75WxNzbM699kNPSjoV7P9T82nX5Mk9cb2Phxx7U1 -/dns4/bootstrap-3.nerpa.interplanetary.dev/tcp/1347/p2p/12D3KooWBxkzgLoqiXrNaCcHf3NW3LCrQfwp41GVK288cric8MvC +/dns4/bootstrap-0.nerpa.interplanetary.dev/tcp/1347/p2p/12D3KooWBscfpDkMHL2zCPeHARxDj9NVJTD8XUMVMs6gzZhjt8RS +/dns4/bootstrap-1.nerpa.interplanetary.dev/tcp/1347/p2p/12D3KooWKvPTUjuxbTaGcrnDnBic646GLhZVoWntEtvNa8VAsC23 +/dns4/bootstrap-2.nerpa.interplanetary.dev/tcp/1347/p2p/12D3KooWPgn23WdadmxEkpCYynbEtDsqJGjQP4GNP14seXheD4xg +/dns4/bootstrap-3.nerpa.interplanetary.dev/tcp/1347/p2p/12D3KooWFqL3sbuj4yZhHpiyngdUHLXf26ArYzmWAji8gZ7814KM diff --git a/build/genesis/devnet.car b/build/genesis/devnet.car index 89702a856ca463849b4051017dfeab844189561a..f57a3f59987be4165e83638665ff8f4b05df1d91 100644 GIT binary patch delta 1604 zcmeDAvw*Q1h$o2D&t;Lh6lMSZ)RtEa*-_JiA2MY3PkYNOcCfbM#|jsfZQs(%Qj3Z+ z^Ya)tGBFk|c*3MWNW%mt28PB61)-edOVfU}r6_Z*5tw%W#e``(cZ@rpB%WH>mMbR5 z{Z*@p!J)BNQI{vI;ZycdG@ntUHTLJXi|3Py`qxkn%k5NCcg;DD0EJq9(aRM zd$KvB9`g*_7n?H}BNz#4nf!q1u>x1-bgtXn%cs0elGi))e*amWid#Q!Nv+Uxv40x> z_rgT4&5xPW85JH)FEH-k)fLy5_$_SSC4G*Yv)9@A`W7=D)9IIGI=^?-<}B6{Mg=2t z){D}M*Cv$wx}nK=nMq+!Tt@i}EH1-7~sB`5&9MLcrbr-v3Wl?z_)q%y|D6 z)4HSXJ3oK7z2_|Vz9oTsCy)PRPj+*Ks56}`vn2ImIvzxZA2oP8>qpzwS7y=SHmbZV z79!T-oRb%@Yb%6IxOpl}%=7-FjB_^cSmni@cd$j26wIF~C3v~D?}@>N$*+NmKIF;v ztnWyRaopc9PMTtj6lo;#LPg<0>rF9%m&2l z+s!yQ(z!zjg;HaLA@ib&@9`o+qNfw~b}|d+It&N=h;< z7`#M9+ox}C_<1~S`$u7pOcrxOLE9M0_^ycg)E0w{2mJW9YN;1G@0-Wb*WEr*f&+*- zftU-3xq+AmhnSOSP8fmjNNrGZ!mh-HCT z4v6KqPn1yDv%j8Dls4bcicn{OB>HHdhyD3{ijs4(x^Go%TgaXfcyo8!E++FSCGQZBqOD!r$%u6kK2hAvi^wc*wd-yErSeR(o*n~YGB+{XhEKy0nE@#SrKWVFS z$Nh;jXT{;i@e&zeNnq^s9B#N7A{|_kwpE6$nYG;grsV9|+PxV|Q)kL9-Fn@o^}=7h z9VdQsPj5VxtN31i#YKq}kk0xhG9p$Y72+bMrz{(^Yft9fmyV6|UZrf;P$@oW-`6X2LV8j35$rtO*rMOLPR$lnUY5r)Ouk3O2u*2_MrH^rpm51&_+>YA;wIeI70HTB$U_L85{o_?0^ z>9M;DB+{ZFu|z2Ufs!&ju_M?8tu;C zsW(Mkm&TQC*Px# delta 1614 zcmeDFvw*P&h$o2D=NQChUd*}K*7dOdh}vRzH7hmEW6}Q4IA;|eT;(qGPAR=CwWv5V zKaX)E6Jz0mCrk>2G)!<}U}%g`IJD|bknA!~=cg;%8InDcW=6yx3t#u^L+9$(%qzDf zKCNtGaA<5|h?qDzQbAlIK}VUvBfaui-K-RgzWvATj4UcEQiT6B*K_)$+^U`)c!N=U z`U6`H?&(u(H5AysNyR7h{n?zs7{N$T@8kzej}<1zojLwGvnJ@slC7&d4t!MVHb2UE zP;B|jG=0A2>SPw_&5xPW85NwCUkkF^D|I#1-nE}S?4`x21^Z`IYpu0;``ub$!&!@K zo3mI;7!^ui#rf7Rx;4GOz5iK|g6*XGQ`2p)u2<-k`BfKDarAxC{6Asgj{M) zoO+U{dEZx;U-_lS(@uLu^=B_TsJ;E8Fh?efIia9!3}u}9C4S9Xqnnv468f&3)>6n( z@VNiIeWC;h5OV@C7Z7s;F%J;)0x=&D^8>K}5DNma5D*Iku?P^00x{hAqJqS{)ROnm978}){gft`mI;Oq9Sai;8=FFzxFuHe zXI-!Ld(bj(VeSSM-udde_a&KMLS(UqlSDdHYxDl94PMdpdv>+_YkpsG#&L74VeH%2vJ7?$=*=r7~rgL>kmhg3S9QH;FG;*H@SSEFP(0Ykb7iUy85c z(^|gz?A3k2GbQTxUh8N$*vlr75(SASLKzU0mSKk16KlFH#B4(O7MPf^1qjq-m>Ier zHAjP-wzlSTM_t{V)6En2eDd+qej!yU*0?h4?>=?0@*VmNllu#~i;L~kMEk72^;OQT uJ39|*HlcC=WHe?Z%i)jZ_LSQSK&%MFN