v1.27.0-a #10

Closed
jonathanface wants to merge 473 commits from v1.27.0-a into master
7 changed files with 492 additions and 75 deletions
Showing only changes of commit 0b76f3e683 - Show all commits

View File

@ -33,6 +33,9 @@ func Routes(r *mux.Router, deps *deps.Deps) error {
r.HandleFunc("/simpleinfo/tasks", a.indexTasks)
r.HandleFunc("/simpleinfo/taskhistory", a.indexTasksHistory)
r.HandleFunc("/simpleinfo/pipeline-porep", a.indexPipelinePorep)
// pipeline-porep page
r.HandleFunc("/simpleinfo/pipeline-porep/sectors", a.pipelinePorepSectors)
return nil
}

View File

@ -0,0 +1,195 @@
package hapi
import (
"net/http"
"time"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-bitfield"
"github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/must"
)
func (a *app) pipelinePorepSectors(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
type PipelineTask struct {
SpID int64 `db:"sp_id"`
SectorNumber int64 `db:"sector_number"`
CreateTime time.Time `db:"create_time"`
TaskSDR *int64 `db:"task_id_sdr"`
AfterSDR bool `db:"after_sdr"`
TaskTreeD *int64 `db:"task_id_tree_d"`
AfterTreeD bool `db:"after_tree_d"`
TaskTreeC *int64 `db:"task_id_tree_c"`
AfterTreeC bool `db:"after_tree_c"`
TaskTreeR *int64 `db:"task_id_tree_r"`
AfterTreeR bool `db:"after_tree_r"`
TaskPrecommitMsg *int64 `db:"task_id_precommit_msg"`
AfterPrecommitMsg bool `db:"after_precommit_msg"`
AfterPrecommitMsgSuccess bool `db:"after_precommit_msg_success"`
SeedEpoch *int64 `db:"seed_epoch"`
TaskPoRep *int64 `db:"task_id_porep"`
PoRepProof []byte `db:"porep_proof"`
AfterPoRep bool `db:"after_porep"`
TaskFinalize *int64 `db:"task_id_finalize"`
AfterFinalize bool `db:"after_finalize"`
TaskMoveStorage *int64 `db:"task_id_move_storage"`
AfterMoveStorage bool `db:"after_move_storage"`
TaskCommitMsg *int64 `db:"task_id_commit_msg"`
AfterCommitMsg bool `db:"after_commit_msg"`
AfterCommitMsgSuccess bool `db:"after_commit_msg_success"`
Failed bool `db:"failed"`
FailedReason string `db:"failed_reason"`
}
var tasks []PipelineTask
err := a.db.Select(ctx, &tasks, `SELECT
sp_id, sector_number,
create_time,
task_id_sdr, after_sdr,
task_id_tree_d, after_tree_d,
task_id_tree_c, after_tree_c,
task_id_tree_r, after_tree_r,
task_id_precommit_msg, after_precommit_msg,
after_precommit_msg_success, seed_epoch,
task_id_porep, porep_proof, after_porep,
task_id_finalize, after_finalize,
task_id_move_storage, after_move_storage,
task_id_commit_msg, after_commit_msg,
after_commit_msg_success,
failed, failed_reason
FROM sectors_sdr_pipeline order by sp_id, sector_number`) // todo where constrain list
if err != nil {
http.Error(w, xerrors.Errorf("failed to fetch pipeline tasks: %w", err).Error(), http.StatusInternalServerError)
return
}
type sectorListEntry struct {
PipelineTask
Address address.Address
CreateTime string
AfterSeed bool
ChainAlloc, ChainSector, ChainActive, ChainUnproven, ChainFaulty bool
}
head, err := a.workingApi.ChainHead(ctx)
if err != nil {
http.Error(w, xerrors.Errorf("failed to fetch chain head: %w", err).Error(), http.StatusInternalServerError)
return
}
epoch := head.Height()
stor := store.ActorStore(ctx, blockstore.NewAPIBlockstore(a.workingApi))
type minerBitfields struct {
alloc, sectorSet, active, unproven, faulty bitfield.BitField
}
minerBitfieldCache := map[address.Address]minerBitfields{}
sectorList := make([]sectorListEntry, 0, len(tasks))
for _, task := range tasks {
task := task
task.CreateTime = task.CreateTime.Local()
addr, err := address.NewIDAddress(uint64(task.SpID))
if err != nil {
http.Error(w, xerrors.Errorf("failed to create actor address: %w", err).Error(), http.StatusInternalServerError)
return
}
mbf, ok := minerBitfieldCache[addr]
if !ok {
act, err := a.workingApi.StateGetActor(ctx, addr, types.EmptyTSK)
if err != nil {
http.Error(w, xerrors.Errorf("failed to load actor: %w", err).Error(), http.StatusInternalServerError)
return
}
mas, err := miner.Load(stor, act)
if err != nil {
http.Error(w, xerrors.Errorf("failed to load miner actor: %w", err).Error(), http.StatusInternalServerError)
return
}
activeSectors, err := miner.AllPartSectors(mas, miner.Partition.ActiveSectors)
if err != nil {
http.Error(w, xerrors.Errorf("failed to load active sectors: %w", err).Error(), http.StatusInternalServerError)
return
}
allSectors, err := miner.AllPartSectors(mas, miner.Partition.AllSectors)
if err != nil {
http.Error(w, xerrors.Errorf("failed to load all sectors: %w", err).Error(), http.StatusInternalServerError)
return
}
unproved, err := miner.AllPartSectors(mas, miner.Partition.UnprovenSectors)
if err != nil {
http.Error(w, xerrors.Errorf("failed to load unproven sectors: %w", err).Error(), http.StatusInternalServerError)
return
}
faulty, err := miner.AllPartSectors(mas, miner.Partition.FaultySectors)
if err != nil {
http.Error(w, xerrors.Errorf("failed to load faulty sectors: %w", err).Error(), http.StatusInternalServerError)
return
}
alloc, err := mas.GetAllocatedSectors()
if err != nil {
http.Error(w, xerrors.Errorf("failed to load allocated sectors: %w", err).Error(), http.StatusInternalServerError)
return
}
mbf = minerBitfields{
alloc: *alloc,
sectorSet: allSectors,
active: activeSectors,
unproven: unproved,
faulty: faulty,
}
minerBitfieldCache[addr] = mbf
}
afterSeed := task.SeedEpoch != nil && *task.SeedEpoch <= int64(epoch)
sectorList = append(sectorList, sectorListEntry{
PipelineTask: task,
Address: addr,
CreateTime: task.CreateTime.Format(time.DateTime),
AfterSeed: afterSeed,
ChainAlloc: must.One(mbf.alloc.IsSet(uint64(task.SectorNumber))),
ChainSector: must.One(mbf.sectorSet.IsSet(uint64(task.SectorNumber))),
ChainActive: must.One(mbf.active.IsSet(uint64(task.SectorNumber))),
ChainUnproven: must.One(mbf.unproven.IsSet(uint64(task.SectorNumber))),
ChainFaulty: must.One(mbf.faulty.IsSet(uint64(task.SectorNumber))),
})
}
a.executeTemplate(w, "pipeline_porep_sectors", sectorList)
}

View File

@ -0,0 +1,138 @@
{{define "pipeline_porep_sectors"}}
{{range .}}
<tr>
<td>{{.Address}}</td>
<td rowspan="2">{{.CreateTime}}</td>
<td rowspan="2">
<table class="porep-state">
<tbody>
<tr>
<td class="{{if ne .TaskSDR nil}}pipeline-active{{end}} {{if .AfterSDR}}pipeline-success{{end}}">
<div>SDR</div>
<div>
{{if .AfterSDR}}done{{else}}
{{if ne .TaskSDR nil}}T:{{.TaskSDR}}{{else}}--{{end}}
{{end}}
</div>
</td>
<td class="{{if ne .TaskTreeC nil}}pipeline-active{{end}} {{if .AfterTreeC}}pipeline-success{{end}}">
<div>TreeC</div>
<div>
{{if .AfterTreeC}}done{{else}}
{{if ne .TaskTreeC nil}}T:{{.TaskTreeC}}{{else}}--{{end}}
{{end}}
</div>
</td>
<td rowspan="2" class="{{if ne .TaskPrecommitMsg nil}}pipeline-active{{end}} {{if .AfterPrecommitMsg}}pipeline-success{{end}}">
<div>PComm Msg</div>
<div>
{{if .AfterPrecommitMsg}}done{{else}}
{{if ne .TaskPrecommitMsg nil}}T:{{.TaskPrecommitMsg}}{{else}}--{{end}}
{{end}}
</div>
</td>
<td rowspan="2" class="{{if .AfterPrecommitMsg}}pipeline-active{{end}} {{if .AfterPrecommitMsgSuccess}}pipeline-success{{end}}">
<div>PComm Wait</div>
<div>
{{if .AfterPrecommitMsgSuccess}}done{{else}}
--
{{end}}
</div>
</td>
<td rowspan="2" class="{{if .AfterPrecommitMsgSuccess}}pipeline-active{{end}} {{if .AfterSeed}}pipeline-success{{end}}">
<div>Wait Seed</div>
<div>
{{if .AfterSeed}}done{{else}}
{{if ne .SeedEpoch nil}}@{{.SeedEpoch}}{{else}}--{{end}}
{{end}}
</div>
</td>
<td rowspan="2" class="{{if ne .TaskPoRep nil}}pipeline-active{{end}} {{if .AfterPoRep}}pipeline-success{{end}}">
<div>PoRep</div>
<div>
{{if .AfterPoRep}}done{{else}}
{{if ne .TaskPoRep nil}}T:{{.TaskPoRep}}{{else}}--{{end}}
{{end}}
</div>
</td>
<td class="{{if ne .TaskFinalize nil}}pipeline-active{{end}} {{if .AfterFinalize}}pipeline-success{{end}}">
<div>Clear Cache</div>
<div>
{{if .AfterFinalize}}done{{else}}
{{if ne .TaskFinalize nil}}T:{{.TaskFinalize}}{{else}}--{{end}}
{{end}}
</div>
</td>
<td class="{{if ne .TaskMoveStorage nil}}pipeline-active{{end}} {{if .AfterMoveStorage}}pipeline-success{{end}}">
<div>Move Storage</div>
<div>
{{if .AfterMoveStorage}}done{{else}}
{{if ne .TaskMoveStorage nil}}T:{{.TaskMoveStorage}}{{else}}--{{end}}
{{end}}
</div>
</td>
<td class="{{if .ChainSector}}pipeline-success{{else}}{{if .ChainAlloc}}pipeline-active{{else}}pipeline-failed{{end}}{{end}}">
<div>On Chain</div>
<div>{{if .ChainSector}}yes{{else}}{{if .ChainAlloc}}allocated{{else}}no{{end}}{{end}}</div>
</td>
</tr>
<tr>
<td class="{{if ne .TaskTreeD nil}}pipeline-active{{end}} {{if .AfterTreeD}}pipeline-success{{end}}">
<div>TreeD</div>
<div>
{{if .AfterTreeD}}done{{else}}
{{if ne .TaskTreeD nil}}T:{{.TaskTreeD}}{{else}}--{{end}}
{{end}}
</div>
</td>
<td class="{{if ne .TaskTreeR nil}}pipeline-active{{end}} {{if .AfterTreeR}}pipeline-success{{end}}">
<div>TreeR</div>
<div>
{{if .AfterTreeR}}done{{else}}
{{if ne .TaskTreeR nil}}T:{{.TaskTreeR}}{{else}}--{{end}}
{{end}}
</div>
</td>
<!-- PC-S -->
<!-- PC-W -->
<!-- WS -->
<!-- PoRep -->
<td class="{{if ne .TaskCommitMsg nil}}pipeline-active{{end}} {{if .AfterCommitMsg}}pipeline-success{{end}}">
<div>Commit Msg</div>
<div>
{{if .AfterCommitMsg}}done{{else}}
{{if ne .TaskCommitMsg nil}}T:{{.TaskCommitMsg}}{{else}}--{{end}}
{{end}}
</div>
</td>
<td class="{{if .AfterCommitMsg}}pipeline-active{{end}} {{if .AfterCommitMsgSuccess}}pipeline-success{{end}}">
<div>Commit Wait</div>
<div>
{{if .AfterCommitMsgSuccess}}done{{else}}
--
{{end}}
</div>
</td>
<td class="{{if .ChainActive}}pipeline-success{{else}}pipeline-failed{{end}}">
<div>Active</div>
<div>{{if .ChainActive}}yes{{else}}
{{if .ChainUnproven}}unproven{{else}}
{{if .ChainFaulty}}faulty{{else}}no{{end}}
{{end}}
{{end}}
</div>
</td>
</tbody>
</table>
</td>
<td rowspan="2">
<a href="#">DETAILS</a>
</td>
</tr>
<tr>
<td>
{{.SectorNumber}}
</td>
</tr>
{{end}}
{{end}}

View File

@ -65,9 +65,6 @@ window.customElements.define('chain-connectivity', class MyElement extends LitEl
<td>${item.Version}</td>
</tr>
`)}
<tr>
<td colspan="4">Data incoming...</td>
</tr>
</tbody>
</table>`
});

View File

@ -3,79 +3,9 @@
<title>Lotus Provider Cluster Overview</title>
<script src="https://unpkg.com/htmx.org@1.9.5" integrity="sha384-xcuj3WpfgjlKF+FXhSQFQ0ZNr39ln+hwjN3npfM9VBnUskLolQAcN80McRIVOPuO" crossorigin="anonymous"></script>
<script type="module" src="chain-connectivity.js"></script>
<link rel="stylesheet" href="main.css">
<link rel='stylesheet' href='https://cdn.jsdelivr.net/npm/hack-font@3.3.0/build/web/hack-subset.css'>
<style>
html, body {
background: #0f0f0f;
color: #ffffff;
padding: 0;
margin: 0;
font-family: 'Hack', monospace;
}
table td, table th {
font-size: 13px;
}
.app-head {
width: 100%;
}
.head-left {
display: inline-block;
}
.head-right {
display: inline-block;
float: right;
}
table {
border-collapse: collapse;
}
table td, table th {
border-left: 1px solid #f0f0f0;
padding: 1px 5px;
}
table tr td:first-child, table tr th:first-child {
border-left: none;
}
a:link {
color: #cfc;
}
a:visited {
color: #dfa;
}
a:hover {
color: #af7;
}
.success {
color: greenyellow;
}
.warning {
color: yellow;
}
.error {
color: red;
}
.dash-tile {
display: flex;
flex-direction: column;
padding: 0.75rem;
background: #3f3f3f;
& b {
padding-bottom: 0.5rem;
color: deeppink;
}
}
.deadline-box {
display: grid;
grid-template-columns: repeat(16, auto);
@ -139,7 +69,7 @@
</div>
<hr>
<div class="info-block">
<h2>PoRep Pipeline</h2>
<h2><a href="/pipeline_porep.html">PoRep Pipeline</a></h2>
<table>
<thead>
<tr>

View File

@ -0,0 +1,70 @@
html, body {
background: #0f0f0f;
color: #ffffff;
padding: 0;
margin: 0;
font-family: 'Hack', monospace;
}
table td, table th {
font-size: 13px;
}
.app-head {
width: 100%;
}
.head-left {
display: inline-block;
}
.head-right {
display: inline-block;
float: right;
}
table {
border-collapse: collapse;
}
table td, table th {
border-left: 1px solid #f0f0f0;
padding: 1px 5px;
}
table tr td:first-child, table tr th:first-child {
border-left: none;
}
a:link {
color: #cfc;
}
a:visited {
color: #dfa;
}
a:hover {
color: #af7;
}
.success {
color: greenyellow;
}
.warning {
color: yellow;
}
.error {
color: red;
}
.dash-tile {
display: flex;
flex-direction: column;
padding: 0.75rem;
background: #3f3f3f;
& b {
padding-bottom: 0.5rem;
color: deeppink;
}
}

View File

@ -0,0 +1,84 @@
<html>
<head>
<title>Lotus Provider PoRep Pipeline</title>
<script src="https://unpkg.com/htmx.org@1.9.5" integrity="sha384-xcuj3WpfgjlKF+FXhSQFQ0ZNr39ln+hwjN3npfM9VBnUskLolQAcN80McRIVOPuO" crossorigin="anonymous"></script>
<script type="module" src="chain-connectivity.js"></script>
<link rel="stylesheet" href="main.css">
<link rel='stylesheet' href='https://cdn.jsdelivr.net/npm/hack-font@3.3.0/build/web/hack-subset.css'>
<style>
.porep-pipeline-table, .porep-state {
color: #d0d0d0;
}
.porep-pipeline-table td, .porep-pipeline-table th {
border-left: none;
border-collapse: collapse;
}
.porep-pipeline-table tr:nth-child(odd) {
border-top: 6px solid #999999;
}
.porep-pipeline-table tr:first-child, .porep-pipeline-table tr:first-child {
border-top: none;
}
.porep-state {
border-collapse: collapse;
}
.porep-state td, .porep-state th {
border-left: 1px solid #f0f0f0;
border-right: 1px solid #f0f0f0;
padding: 1px 5px;
text-align: center;
font-size: 0.7em;
}
.porep-state tr {
border-top: 1px solid #f0f0f0;
}
.porep-state tr:first-child {
border-top: none;
}
.pipeline-active {
background-color: #303060;
}
.pipeline-success {
background-color: #306030;
}
.pipeline-failed {
background-color: #603030;
}
</style>
</head>
<body>
<div class="app-head">
<div class="head-left">
<h1>Lotus Provider PoRep Pipeline</h1>
</div>
</div>
<hr/>
<div class="page">
<div class="info-block">
<h2>Sectors</h2>
<table class="porep-pipeline-table">
<thead>
<tr>
<th>Sector ID</th>
<th>Create Time</th>
<th>State</th>
</tr>
</thead>
<tbody hx-get="/hapi/simpleinfo/pipeline-porep/sectors" hx-trigger="load,every 3s">
</tbody>
</table>
</div>
</div>
</body>
</html>