cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

docsgen: Support miner/worker structs

Browse Source
This commit is contained in:
Łukasz Magiera 2020-11-12 10:29:42 +01:00
parent 3a3986320c
commit 8409e155d4
4 changed files with 135 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Makefile
View File

@ -304,7 +304,9 @@ method-gen:
gen: type-gen method-gen
docsgen:
go run ./api/docgen > documentation/en/api-methods.md
go run ./api/docgen "api/api_full.go" "FullNode" > documentation/en/api-methods.md
go run ./api/docgen "api/api_storage.go" "StorageMiner" > documentation/en/api-methods-miner.md
go run ./api/docgen "api/api_worker.go" "WorkerAPI" > documentation/en/api-methods-worker.md
print-%:
@echo $*=$($*)

149
api/docgen/docgen.go
View File

@ -3,9 +3,16 @@ package main
import (
"encoding/json"
"fmt"
filestore2 "github.com/filecoin-project/go-fil-markets/filestore"
"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
"github.com/filecoin-project/lotus/extern/sector-storage/stores"
"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
sealing "github.com/filecoin-project/lotus/extern/storage-sealing"
"github.com/google/uuid"
"go/ast"
"go/parser"
"go/token"
"os"
"reflect"
"sort"
"strings"
@ -117,17 +124,17 @@ func init() {
addExample(network.ReachabilityPublic)
addExample(build.NewestNetworkVersion)
addExample(&types.ExecutionTrace{
Msg: exampleValue(reflect.TypeOf(&types.Message{}), nil).(*types.Message),
MsgRct: exampleValue(reflect.TypeOf(&types.MessageReceipt{}), nil).(*types.MessageReceipt),
Msg: exampleValue("init", reflect.TypeOf(&types.Message{}), nil).(*types.Message),
MsgRct: exampleValue("init", reflect.TypeOf(&types.MessageReceipt{}), nil).(*types.MessageReceipt),
})
addExample(map[string]types.Actor{
"t01236": exampleValue(reflect.TypeOf(types.Actor{}), nil).(types.Actor),
"t01236": exampleValue("init", reflect.TypeOf(types.Actor{}), nil).(types.Actor),
})
addExample(map[string]api.MarketDeal{
"t026363": exampleValue(reflect.TypeOf(api.MarketDeal{}), nil).(api.MarketDeal),
"t026363": exampleValue("init", reflect.TypeOf(api.MarketDeal{}), nil).(api.MarketDeal),
})
addExample(map[string]api.MarketBalance{
"t026363": exampleValue(reflect.TypeOf(api.MarketBalance{}), nil).(api.MarketBalance),
"t026363": exampleValue("init", reflect.TypeOf(api.MarketBalance{}), nil).(api.MarketBalance),
})
addExample(map[string]*pubsub.TopicScoreSnapshot{
"/blocks": {
@ -162,9 +169,82 @@ func init() {
// because reflect.TypeOf(maddr) returns the concrete type...
ExampleValues[reflect.TypeOf(struct{ A multiaddr.Multiaddr }{}).Field(0).Type] = maddr
// miner specific
addExample(filestore2.Path(".lotusminer/fstmp123"))
si := multistore.StoreID(12)
addExample(&si)
addExample(retrievalmarket.DealID(5))
addExample(abi.ActorID(1000))
addExample(map[string][]api.SealedRef{
"98000": {
api.SealedRef{
SectorID: 100,
Offset: 10 << 20,
Size: 1 << 20,
},
},
})
addExample(api.SectorState(sealing.Proving))
addExample(stores.ID("76f1988b-ef30-4d7e-b3ec-9a627f4ba5a8"))
addExample(storiface.FTUnsealed)
addExample(storiface.PathSealing)
addExample(map[stores.ID][]stores.Decl{
"76f1988b-ef30-4d7e-b3ec-9a627f4ba5a8": {
{
SectorID: abi.SectorID{Miner: 1000, Number: 100},
SectorFileType: storiface.FTSealed,
},
},
})
addExample(map[stores.ID]string{
"76f1988b-ef30-4d7e-b3ec-9a627f4ba5a8": "/data/path",
})
addExample(map[uuid.UUID][]storiface.WorkerJob{
uuid.MustParse("ef8d99a2-6865-4189-8ffa-9fef0f806eee"): {
{
ID: storiface.CallID{
Sector: abi.SectorID{Miner: 1000, Number: 100},
ID: uuid.MustParse("76081ba0-61bd-45a5-bc08-af05f1c26e5d"),
},
Sector: abi.SectorID{Miner: 1000, Number: 100},
Task: sealtasks.TTPreCommit2,
RunWait: 0,
Start: time.Unix(1605172927, 0),
Hostname: "host",
},
},
})
addExample(map[uuid.UUID]storiface.WorkerStats{
uuid.MustParse("ef8d99a2-6865-4189-8ffa-9fef0f806eee"): {
Info: storiface.WorkerInfo{
Hostname: "host",
Resources: storiface.WorkerResources{
MemPhysical: 256 << 30,
MemSwap: 120 << 30,
MemReserved: 2 << 30,
CPUs: 64,
GPUs: []string{"aGPU 1337"},
},
},
Enabled: true,
MemUsedMin: 0,
MemUsedMax: 0,
GpuUsed: false,
CpuUse: 0,
},
})
// worker specific
addExample(storiface.AcquireMove)
addExample(storiface.UnpaddedByteIndex(abi.PaddedPieceSize(1 << 20).Unpadded()))
addExample(map[sealtasks.TaskType]struct {}{
sealtasks.TTPreCommit2: {},
sealtasks.TTCommit1: {},
sealtasks.TTCommit2: {},
})
}
func exampleValue(t, parent reflect.Type) interface{} {
func exampleValue(method string, t, parent reflect.Type) interface{} {
v, ok := ExampleValues[t]
if ok {
return v
@ -173,25 +253,25 @@ func exampleValue(t, parent reflect.Type) interface{} {
switch t.Kind() {
case reflect.Slice:
out := reflect.New(t).Elem()
reflect.Append(out, reflect.ValueOf(exampleValue(t.Elem(), t)))
reflect.Append(out, reflect.ValueOf(exampleValue(method, t.Elem(), t)))
return out.Interface()
case reflect.Chan:
return exampleValue(t.Elem(), nil)
return exampleValue(method, t.Elem(), nil)
case reflect.Struct:
es := exampleStruct(t, parent)
es := exampleStruct(method, t, parent)
v := reflect.ValueOf(es).Elem().Interface()
ExampleValues[t] = v
return v
case reflect.Array:
out := reflect.New(t).Elem()
for i := 0; i < t.Len(); i++ {
out.Index(i).Set(reflect.ValueOf(exampleValue(t.Elem(), t)))
out.Index(i).Set(reflect.ValueOf(exampleValue(method, t.Elem(), t)))
}
return out.Interface()
case reflect.Ptr:
if t.Elem().Kind() == reflect.Struct {
es := exampleStruct(t.Elem(), t)
es := exampleStruct(method, t.Elem(), t)
//ExampleValues[t] = es
return es
}
@ -199,10 +279,10 @@ func exampleValue(t, parent reflect.Type) interface{} {
return struct{}{}
}
panic(fmt.Sprintf("No example value for type: %s", t))
panic(fmt.Sprintf("No example value for type: %s (method '%s')", t, method))
}
func exampleStruct(t, parent reflect.Type) interface{} {
func exampleStruct(method string, t, parent reflect.Type) interface{} {
ns := reflect.New(t)
for i := 0; i < t.NumField(); i++ {
f := t.Field(i)
@ -210,7 +290,7 @@ func exampleStruct(t, parent reflect.Type) interface{} {
continue
}
if strings.Title(f.Name) == f.Name {
ns.Elem().Field(i).Set(reflect.ValueOf(exampleValue(f.Type, t)))
ns.Elem().Field(i).Set(reflect.ValueOf(exampleValue(method, f.Type, t)))
}
}
@ -218,6 +298,7 @@ func exampleStruct(t, parent reflect.Type) interface{} {
}
type Visitor struct {
Root string
Methods map[string]ast.Node
}
@ -227,7 +308,7 @@ func (v *Visitor) Visit(node ast.Node) ast.Visitor {
return v
}
if st.Name.Name != "FullNode" {
if st.Name.Name != v.Root {
return nil
}
@ -243,7 +324,7 @@ func (v *Visitor) Visit(node ast.Node) ast.Visitor {
const noComment = "There are not yet any comments for this method."
func parseApiASTInfo() (map[string]string, map[string]string) { //nolint:golint
func parseApiASTInfo(apiFile, iface string) (map[string]string, map[string]string) { //nolint:golint
fset := token.NewFileSet()
pkgs, err := parser.ParseDir(fset, "./api", nil, parser.AllErrors|parser.ParseComments)
if err != nil {
@ -252,11 +333,11 @@ func parseApiASTInfo() (map[string]string, map[string]string) { //nolint:golint
ap := pkgs["api"]
f := ap.Files["api/api_full.go"]
f := ap.Files[apiFile]
cmap := ast.NewCommentMap(fset, f, f.Comments)
v := &Visitor{make(map[string]ast.Node)}
v := &Visitor{iface, make(map[string]ast.Node)}
ast.Walk(v, pkgs["api"])
groupDocs := make(map[string]string)
@ -312,13 +393,30 @@ func methodGroupFromName(mn string) string {
}
func main() {
comments, groupComments := parseApiASTInfo()
comments, groupComments := parseApiASTInfo(os.Args[1], os.Args[2])
groups := make(map[string]*MethodGroup)
var api struct{ api.FullNode }
t := reflect.TypeOf(api)
var t reflect.Type
var permStruct, commonPermStruct reflect.Type
switch os.Args[2] {
case "FullNode":
t = reflect.TypeOf(new(struct{ api.FullNode })).Elem()
permStruct = reflect.TypeOf(apistruct.FullNodeStruct{}.Internal)
commonPermStruct = reflect.TypeOf(apistruct.CommonStruct{}.Internal)
case "StorageMiner":
t = reflect.TypeOf(new(struct{ api.StorageMiner })).Elem()
permStruct = reflect.TypeOf(apistruct.StorageMinerStruct{}.Internal)
commonPermStruct = reflect.TypeOf(apistruct.CommonStruct{}.Internal)
case "WorkerAPI":
t = reflect.TypeOf(new(struct{ api.WorkerAPI })).Elem()
permStruct = reflect.TypeOf(apistruct.WorkerStruct{}.Internal)
commonPermStruct = reflect.TypeOf(apistruct.WorkerStruct{}.Internal)
default:
panic("unknown type")
}
for i := 0; i < t.NumMethod(); i++ {
m := t.Method(i)
@ -336,7 +434,7 @@ func main() {
ft := m.Func.Type()
for j := 2; j < ft.NumIn(); j++ {
inp := ft.In(j)
args = append(args, exampleValue(inp, nil))
args = append(args, exampleValue(m.Name, inp, nil))
}
v, err := json.MarshalIndent(args, "", " ")
@ -344,7 +442,7 @@ func main() {
panic(err)
}
outv := exampleValue(ft.Out(0), nil)
outv := exampleValue(m.Name, ft.Out(0), nil)
ov, err := json.MarshalIndent(outv, "", " ")
if err != nil {
@ -377,9 +475,6 @@ func main() {
}
}
permStruct := reflect.TypeOf(apistruct.FullNodeStruct{}.Internal)
commonPermStruct := reflect.TypeOf(apistruct.CommonStruct{}.Internal)
for _, g := range groupslice {
g := g
fmt.Printf("## %s\n", g.GroupName)

7
extern/sector-storage/stores/index.go vendored
View File

@ -2,6 +2,7 @@ package stores
import (
"context"
"errors"
"net/url"
gopath "path"
"sort"
@ -35,7 +36,7 @@ type StorageInfo struct {
type HealthReport struct {
Stat fsutil.FsStat
Err error
Err string
}
type SectorStorageInfo struct {
@ -175,7 +176,9 @@ func (i *Index) StorageReportHealth(ctx context.Context, id ID, report HealthRep
}
ent.fsi = report.Stat
ent.heartbeatErr = report.Err
if report.Err != "" {
ent.heartbeatErr = errors.New(report.Err)
}
ent.lastHeartbeat = time.Now()
return nil

9
extern/sector-storage/stores/local.go vendored
View File

@ -308,11 +308,12 @@ func (st *Local) reportStorage(ctx context.Context) {
toReport := map[ID]HealthReport{}
for id, p := range st.paths {
stat, err := p.stat(st.localStorage)
toReport[id] = HealthReport{
Stat: stat,
Err: err,
r := HealthReport{Stat: stat}
if err != nil {
r.Err = err.Error()
}
toReport[id] = r
}
st.localLk.RUnlock()