lpweb: Page for sealing pipeline sector list

provider/lpweb/hapi/routes.go

@@ -33,6 +33,9 @@ func Routes(r *mux.Router, deps *deps.Deps) error {
 	r.HandleFunc("/simpleinfo/tasks", a.indexTasks)
 	r.HandleFunc("/simpleinfo/taskhistory", a.indexTasksHistory)
 	r.HandleFunc("/simpleinfo/pipeline-porep", a.indexPipelinePorep)
+
+	// pipeline-porep page
+	r.HandleFunc("/simpleinfo/pipeline-porep/sectors", a.pipelinePorepSectors)
 	return nil
 }
 

provider/lpweb/hapi/simpleinfo_pipeline_porep.go

@@ -0,0 +1,195 @@
+package hapi
+
+import (
+	"net/http"
+	"time"
+
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-bitfield"
+
+	"github.com/filecoin-project/lotus/blockstore"
+	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
+	"github.com/filecoin-project/lotus/chain/store"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/lib/must"
+)
+
+func (a *app) pipelinePorepSectors(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+
+	type PipelineTask struct {
+		SpID         int64 `db:"sp_id"`
+		SectorNumber int64 `db:"sector_number"`
+
+		CreateTime time.Time `db:"create_time"`
+
+		TaskSDR  *int64 `db:"task_id_sdr"`
+		AfterSDR bool   `db:"after_sdr"`
+
+		TaskTreeD  *int64 `db:"task_id_tree_d"`
+		AfterTreeD bool   `db:"after_tree_d"`
+
+		TaskTreeC  *int64 `db:"task_id_tree_c"`
+		AfterTreeC bool   `db:"after_tree_c"`
+
+		TaskTreeR  *int64 `db:"task_id_tree_r"`
+		AfterTreeR bool   `db:"after_tree_r"`
+
+		TaskPrecommitMsg  *int64 `db:"task_id_precommit_msg"`
+		AfterPrecommitMsg bool   `db:"after_precommit_msg"`
+
+		AfterPrecommitMsgSuccess bool   `db:"after_precommit_msg_success"`
+		SeedEpoch                *int64 `db:"seed_epoch"`
+
+		TaskPoRep  *int64 `db:"task_id_porep"`
+		PoRepProof []byte `db:"porep_proof"`
+		AfterPoRep bool   `db:"after_porep"`
+
+		TaskFinalize  *int64 `db:"task_id_finalize"`
+		AfterFinalize bool   `db:"after_finalize"`
+
+		TaskMoveStorage  *int64 `db:"task_id_move_storage"`
+		AfterMoveStorage bool   `db:"after_move_storage"`
+
+		TaskCommitMsg  *int64 `db:"task_id_commit_msg"`
+		AfterCommitMsg bool   `db:"after_commit_msg"`
+
+		AfterCommitMsgSuccess bool `db:"after_commit_msg_success"`
+
+		Failed       bool   `db:"failed"`
+		FailedReason string `db:"failed_reason"`
+	}
+
+	var tasks []PipelineTask
+
+	err := a.db.Select(ctx, &tasks, `SELECT 
+       sp_id, sector_number,
+       create_time,
+       task_id_sdr, after_sdr,
+       task_id_tree_d, after_tree_d,
+       task_id_tree_c, after_tree_c,
+       task_id_tree_r, after_tree_r,
+       task_id_precommit_msg, after_precommit_msg,
+       after_precommit_msg_success, seed_epoch,
+       task_id_porep, porep_proof, after_porep,
+       task_id_finalize, after_finalize,
+       task_id_move_storage, after_move_storage,
+       task_id_commit_msg, after_commit_msg,
+       after_commit_msg_success,
+       failed, failed_reason
+    FROM sectors_sdr_pipeline order by sp_id, sector_number`) // todo where constrain list
+	if err != nil {
+		http.Error(w, xerrors.Errorf("failed to fetch pipeline tasks: %w", err).Error(), http.StatusInternalServerError)
+		return
+	}
+
+	type sectorListEntry struct {
+		PipelineTask
+
+		Address    address.Address
+		CreateTime string
+		AfterSeed  bool
+
+		ChainAlloc, ChainSector, ChainActive, ChainUnproven, ChainFaulty bool
+	}
+
+	head, err := a.workingApi.ChainHead(ctx)
+	if err != nil {
+		http.Error(w, xerrors.Errorf("failed to fetch chain head: %w", err).Error(), http.StatusInternalServerError)
+		return
+	}
+	epoch := head.Height()
+
+	stor := store.ActorStore(ctx, blockstore.NewAPIBlockstore(a.workingApi))
+
+	type minerBitfields struct {
+		alloc, sectorSet, active, unproven, faulty bitfield.BitField
+	}
+	minerBitfieldCache := map[address.Address]minerBitfields{}
+
+	sectorList := make([]sectorListEntry, 0, len(tasks))
+	for _, task := range tasks {
+		task := task
+
+		task.CreateTime = task.CreateTime.Local()
+
+		addr, err := address.NewIDAddress(uint64(task.SpID))
+		if err != nil {
+			http.Error(w, xerrors.Errorf("failed to create actor address: %w", err).Error(), http.StatusInternalServerError)
+			return
+		}
+
+		mbf, ok := minerBitfieldCache[addr]
+		if !ok {
+			act, err := a.workingApi.StateGetActor(ctx, addr, types.EmptyTSK)
+			if err != nil {
+				http.Error(w, xerrors.Errorf("failed to load actor: %w", err).Error(), http.StatusInternalServerError)
+				return
+			}
+
+			mas, err := miner.Load(stor, act)
+			if err != nil {
+				http.Error(w, xerrors.Errorf("failed to load miner actor: %w", err).Error(), http.StatusInternalServerError)
+				return
+			}
+
+			activeSectors, err := miner.AllPartSectors(mas, miner.Partition.ActiveSectors)
+			if err != nil {
+				http.Error(w, xerrors.Errorf("failed to load active sectors: %w", err).Error(), http.StatusInternalServerError)
+				return
+			}
+
+			allSectors, err := miner.AllPartSectors(mas, miner.Partition.AllSectors)
+			if err != nil {
+				http.Error(w, xerrors.Errorf("failed to load all sectors: %w", err).Error(), http.StatusInternalServerError)
+				return
+			}
+
+			unproved, err := miner.AllPartSectors(mas, miner.Partition.UnprovenSectors)
+			if err != nil {
+				http.Error(w, xerrors.Errorf("failed to load unproven sectors: %w", err).Error(), http.StatusInternalServerError)
+				return
+			}
+
+			faulty, err := miner.AllPartSectors(mas, miner.Partition.FaultySectors)
+			if err != nil {
+				http.Error(w, xerrors.Errorf("failed to load faulty sectors: %w", err).Error(), http.StatusInternalServerError)
+				return
+			}
+
+			alloc, err := mas.GetAllocatedSectors()
+			if err != nil {
+				http.Error(w, xerrors.Errorf("failed to load allocated sectors: %w", err).Error(), http.StatusInternalServerError)
+				return
+			}
+
+			mbf = minerBitfields{
+				alloc:     *alloc,
+				sectorSet: allSectors,
+				active:    activeSectors,
+				unproven:  unproved,
+				faulty:    faulty,
+			}
+			minerBitfieldCache[addr] = mbf
+		}
+
+		afterSeed := task.SeedEpoch != nil && *task.SeedEpoch <= int64(epoch)
+
+		sectorList = append(sectorList, sectorListEntry{
+			PipelineTask: task,
+			Address:      addr,
+			CreateTime:   task.CreateTime.Format(time.DateTime),
+			AfterSeed:    afterSeed,
+
+			ChainAlloc:    must.One(mbf.alloc.IsSet(uint64(task.SectorNumber))),
+			ChainSector:   must.One(mbf.sectorSet.IsSet(uint64(task.SectorNumber))),
+			ChainActive:   must.One(mbf.active.IsSet(uint64(task.SectorNumber))),
+			ChainUnproven: must.One(mbf.unproven.IsSet(uint64(task.SectorNumber))),
+			ChainFaulty:   must.One(mbf.faulty.IsSet(uint64(task.SectorNumber))),
+		})
+	}
+
+	a.executeTemplate(w, "pipeline_porep_sectors", sectorList)
+}

provider/lpweb/hapi/web/pipeline_porep_sectors.gohtml

@@ -0,0 +1,138 @@
+{{define "pipeline_porep_sectors"}}
+    {{range .}}
+        <tr>
+            <td>{{.Address}}</td>
+            <td rowspan="2">{{.CreateTime}}</td>
+            <td rowspan="2">
+                <table class="porep-state">
+                    <tbody>
+                    <tr>
+                        <td class="{{if ne .TaskSDR nil}}pipeline-active{{end}} {{if .AfterSDR}}pipeline-success{{end}}">
+                            <div>SDR</div>
+                            <div>
+                                {{if .AfterSDR}}done{{else}}
+                                    {{if ne .TaskSDR nil}}T:{{.TaskSDR}}{{else}}--{{end}}
+                                {{end}}
+                            </div>
+                        </td>
+                        <td class="{{if ne .TaskTreeC nil}}pipeline-active{{end}} {{if .AfterTreeC}}pipeline-success{{end}}">
+                            <div>TreeC</div>
+                            <div>
+                                {{if .AfterTreeC}}done{{else}}
+                                    {{if ne .TaskTreeC nil}}T:{{.TaskTreeC}}{{else}}--{{end}}
+                                {{end}}
+                            </div>
+                        </td>
+                        <td rowspan="2" class="{{if ne .TaskPrecommitMsg nil}}pipeline-active{{end}} {{if .AfterPrecommitMsg}}pipeline-success{{end}}">
+                            <div>PComm Msg</div>
+                            <div>
+                                {{if .AfterPrecommitMsg}}done{{else}}
+                                    {{if ne .TaskPrecommitMsg nil}}T:{{.TaskPrecommitMsg}}{{else}}--{{end}}
+                                {{end}}
+                            </div>
+                        </td>
+                        <td rowspan="2" class="{{if .AfterPrecommitMsg}}pipeline-active{{end}} {{if .AfterPrecommitMsgSuccess}}pipeline-success{{end}}">
+                            <div>PComm Wait</div>
+                            <div>
+                                {{if .AfterPrecommitMsgSuccess}}done{{else}}
+                                    --
+                                {{end}}
+                            </div>
+                        </td>
+                        <td rowspan="2" class="{{if .AfterPrecommitMsgSuccess}}pipeline-active{{end}} {{if .AfterSeed}}pipeline-success{{end}}">
+                            <div>Wait Seed</div>
+                            <div>
+                                {{if .AfterSeed}}done{{else}}
+                                    {{if ne .SeedEpoch nil}}@{{.SeedEpoch}}{{else}}--{{end}}
+                                {{end}}
+                            </div>
+                        </td>
+                        <td rowspan="2" class="{{if ne .TaskPoRep nil}}pipeline-active{{end}} {{if .AfterPoRep}}pipeline-success{{end}}">
+                            <div>PoRep</div>
+                            <div>
+                                {{if .AfterPoRep}}done{{else}}
+                                    {{if ne .TaskPoRep nil}}T:{{.TaskPoRep}}{{else}}--{{end}}
+                                {{end}}
+                            </div>
+                        </td>
+                        <td class="{{if ne .TaskFinalize nil}}pipeline-active{{end}} {{if .AfterFinalize}}pipeline-success{{end}}">
+                            <div>Clear Cache</div>
+                            <div>
+                                {{if .AfterFinalize}}done{{else}}
+                                    {{if ne .TaskFinalize nil}}T:{{.TaskFinalize}}{{else}}--{{end}}
+                                {{end}}
+                            </div>
+                        </td>
+                        <td class="{{if ne .TaskMoveStorage nil}}pipeline-active{{end}} {{if .AfterMoveStorage}}pipeline-success{{end}}">
+                            <div>Move Storage</div>
+                            <div>
+                                {{if .AfterMoveStorage}}done{{else}}
+                                    {{if ne .TaskMoveStorage nil}}T:{{.TaskMoveStorage}}{{else}}--{{end}}
+                                {{end}}
+                            </div>
+                        </td>
+                        <td class="{{if .ChainSector}}pipeline-success{{else}}{{if .ChainAlloc}}pipeline-active{{else}}pipeline-failed{{end}}{{end}}">
+                            <div>On Chain</div>
+                            <div>{{if .ChainSector}}yes{{else}}{{if .ChainAlloc}}allocated{{else}}no{{end}}{{end}}</div>
+                        </td>
+                    </tr>
+                    <tr>
+                        <td class="{{if ne .TaskTreeD nil}}pipeline-active{{end}} {{if .AfterTreeD}}pipeline-success{{end}}">
+                            <div>TreeD</div>
+                            <div>
+                                {{if .AfterTreeD}}done{{else}}
+                                    {{if ne .TaskTreeD nil}}T:{{.TaskTreeD}}{{else}}--{{end}}
+                                {{end}}
+                            </div>
+                        </td>
+                        <td class="{{if ne .TaskTreeR nil}}pipeline-active{{end}} {{if .AfterTreeR}}pipeline-success{{end}}">
+                            <div>TreeR</div>
+                            <div>
+                                {{if .AfterTreeR}}done{{else}}
+                                    {{if ne .TaskTreeR nil}}T:{{.TaskTreeR}}{{else}}--{{end}}
+                                {{end}}
+                            </div>
+                        </td>
+                        <!-- PC-S -->
+                        <!-- PC-W -->
+                        <!-- WS -->
+                        <!-- PoRep -->
+                        <td class="{{if ne .TaskCommitMsg nil}}pipeline-active{{end}} {{if .AfterCommitMsg}}pipeline-success{{end}}">
+                            <div>Commit Msg</div>
+                            <div>
+                                {{if .AfterCommitMsg}}done{{else}}
+                                    {{if ne .TaskCommitMsg nil}}T:{{.TaskCommitMsg}}{{else}}--{{end}}
+                                {{end}}
+                            </div>
+                        </td>
+                        <td class="{{if .AfterCommitMsg}}pipeline-active{{end}} {{if .AfterCommitMsgSuccess}}pipeline-success{{end}}">
+                            <div>Commit Wait</div>
+                            <div>
+                                {{if .AfterCommitMsgSuccess}}done{{else}}
+                                    --
+                                {{end}}
+                            </div>
+                        </td>
+                        <td class="{{if .ChainActive}}pipeline-success{{else}}pipeline-failed{{end}}">
+                            <div>Active</div>
+                            <div>{{if .ChainActive}}yes{{else}}
+                                {{if .ChainUnproven}}unproven{{else}}
+                                {{if .ChainFaulty}}faulty{{else}}no{{end}}
+                                {{end}}
+                                {{end}}
+                            </div>
+                        </td>
+                    </tbody>
+                </table>
+            </td>
+            <td rowspan="2">
+                <a href="#">DETAILS</a>
+            </td>
+        </tr>
+        <tr>
+            <td>
+                {{.SectorNumber}}
+            </td>
+        </tr>
+    {{end}}
+{{end}}

provider/lpweb/static/chain-connectivity.js

@@ -65,9 +65,6 @@ window.customElements.define('chain-connectivity', class MyElement extends LitEl
             <td>${item.Version}</td>
         </tr>
         `)}
-        <tr>
-            <td colspan="4">Data incoming...</td>
-        </tr>
     </tbody>
   </table>`
 });

provider/lpweb/static/index.html

@@ -3,79 +3,9 @@
     <title>Lotus Provider Cluster Overview</title>
     <script src="https://unpkg.com/htmx.org@1.9.5" integrity="sha384-xcuj3WpfgjlKF+FXhSQFQ0ZNr39ln+hwjN3npfM9VBnUskLolQAcN80McRIVOPuO" crossorigin="anonymous"></script>
     <script type="module" src="chain-connectivity.js"></script>
+    <link rel="stylesheet" href="main.css">
     <link rel='stylesheet' href='https://cdn.jsdelivr.net/npm/hack-font@3.3.0/build/web/hack-subset.css'>
     <style>
-        html, body {
-            background: #0f0f0f;
-            color: #ffffff;
-            padding: 0;
-            margin: 0;
-
-            font-family: 'Hack', monospace;
-        }
-
-        table td, table th {
-            font-size: 13px;
-        }
-
-        .app-head {
-            width: 100%;
-        }
-        .head-left {
-            display: inline-block;
-        }
-        .head-right {
-            display: inline-block;
-            float: right;
-        }
-
-        table {
-            border-collapse: collapse;
-        }
-
-        table td, table th {
-            border-left: 1px solid #f0f0f0;
-            padding: 1px 5px;
-        }
-
-        table tr td:first-child, table tr th:first-child {
-            border-left: none;
-        }
-
-        a:link {
-            color: #cfc;
-        }
-
-        a:visited {
-            color: #dfa;
-        }
-
-        a:hover {
-            color: #af7;
-        }
-
-        .success {
-            color: greenyellow;
-        }
-        .warning {
-            color: yellow;
-        }
-        .error {
-            color: red;
-        }
-
-        .dash-tile {
-            display: flex;
-            flex-direction: column;
-            padding: 0.75rem;
-            background: #3f3f3f;
-
-            & b {
-                padding-bottom: 0.5rem;
-                color: deeppink;
-            }
-        }
-
         .deadline-box {
             display: grid;
             grid-template-columns: repeat(16, auto);
@@ -139,7 +69,7 @@
     </div>
     <hr>
     <div class="info-block">
-        <h2>PoRep Pipeline</h2>
+        <h2><a href="/pipeline_porep.html">PoRep Pipeline</a></h2>
         <table>
             <thead>
             <tr>

provider/lpweb/static/main.css

@@ -0,0 +1,70 @@
+html, body {
+    background: #0f0f0f;
+    color: #ffffff;
+    padding: 0;
+    margin: 0;
+
+    font-family: 'Hack', monospace;
+}
+
+table td, table th {
+    font-size: 13px;
+}
+
+.app-head {
+    width: 100%;
+}
+.head-left {
+    display: inline-block;
+}
+.head-right {
+    display: inline-block;
+    float: right;
+}
+
+table {
+    border-collapse: collapse;
+}
+
+table td, table th {
+    border-left: 1px solid #f0f0f0;
+    padding: 1px 5px;
+}
+
+table tr td:first-child, table tr th:first-child {
+    border-left: none;
+}
+
+a:link {
+    color: #cfc;
+}
+
+a:visited {
+    color: #dfa;
+}
+
+a:hover {
+    color: #af7;
+}
+
+.success {
+    color: greenyellow;
+}
+.warning {
+    color: yellow;
+}
+.error {
+    color: red;
+}
+
+.dash-tile {
+    display: flex;
+    flex-direction: column;
+    padding: 0.75rem;
+    background: #3f3f3f;
+
+    & b {
+        padding-bottom: 0.5rem;
+        color: deeppink;
+    }
+}

provider/lpweb/static/pipeline_porep.html

@@ -0,0 +1,84 @@
+<html>
+<head>
+    <title>Lotus Provider PoRep Pipeline</title>
+    <script src="https://unpkg.com/htmx.org@1.9.5" integrity="sha384-xcuj3WpfgjlKF+FXhSQFQ0ZNr39ln+hwjN3npfM9VBnUskLolQAcN80McRIVOPuO" crossorigin="anonymous"></script>
+    <script type="module" src="chain-connectivity.js"></script>
+    <link rel="stylesheet" href="main.css">
+    <link rel='stylesheet' href='https://cdn.jsdelivr.net/npm/hack-font@3.3.0/build/web/hack-subset.css'>
+    <style>
+        .porep-pipeline-table, .porep-state {
+            color: #d0d0d0;
+        }
+
+        .porep-pipeline-table td, .porep-pipeline-table th {
+            border-left: none;
+            border-collapse: collapse;
+        }
+
+        .porep-pipeline-table tr:nth-child(odd) {
+            border-top: 6px solid #999999;
+
+        }
+
+        .porep-pipeline-table tr:first-child, .porep-pipeline-table tr:first-child {
+            border-top: none;
+        }
+
+        .porep-state {
+            border-collapse: collapse;
+        }
+
+        .porep-state td, .porep-state th {
+            border-left: 1px solid #f0f0f0;
+            border-right: 1px solid #f0f0f0;
+
+            padding: 1px 5px;
+
+            text-align: center;
+            font-size: 0.7em;
+        }
+
+        .porep-state tr {
+            border-top: 1px solid #f0f0f0;
+        }
+        .porep-state tr:first-child {
+            border-top: none;
+        }
+
+        .pipeline-active {
+            background-color: #303060;
+        }
+        .pipeline-success {
+            background-color: #306030;
+        }
+        .pipeline-failed {
+            background-color: #603030;
+        }
+
+    </style>
+</head>
+<body>
+<div class="app-head">
+    <div class="head-left">
+        <h1>Lotus Provider PoRep Pipeline</h1>
+    </div>
+</div>
+<hr/>
+<div class="page">
+    <div class="info-block">
+        <h2>Sectors</h2>
+        <table class="porep-pipeline-table">
+            <thead>
+            <tr>
+                <th>Sector ID</th>
+                <th>Create Time</th>
+                <th>State</th>
+            </tr>
+            </thead>
+            <tbody hx-get="/hapi/simpleinfo/pipeline-porep/sectors" hx-trigger="load,every 3s">
+            </tbody>
+        </table>
+    </div>
+</div>
+</body>
+</html>