cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge pull request #110 from filecoin-project/feat/pond-storage-miners

Browse Source 
Storage Miners in Pond
This commit is contained in:
Łukasz Magiera 2019-08-06 18:11:51 +02:00 committed by GitHub
commit b1aee461ae
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 346 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/lotus-storage-miner/init.go
View File

@ -155,7 +155,7 @@ var initCmd = &cli.Command{
return err
}
log.Infof("Waiting for confirmation (TODO: actually wait)")
log.Infof("Waiting for confirmation")
mw, err := api.ChainWaitMsg(ctx, signed.Cid())
if err != nil {

2
extern/go-sectorbuilder vendored

@ -1 +1 @@
Subproject commit 9b090e700325b1c9a7d3b1556d447897ae039a58
Subproject commit e75bc9b0aaeab4e1c8ea7eda42d63a39ae80728f

19
lotuspond/front/src/App.css
View File

@ -4,6 +4,14 @@
font-family: monospace;
}
.NodeList {
background: #f9be77;
user-select: text;
font-family: monospace;
min-width: 40em;
display: inline-block;
}
.FullNode {
background: #f9be77;
user-select: text;
@ -12,9 +20,18 @@
display: inline-block;
}
.StorageNode {
background: #f9be77;
user-select: text;
font-family: monospace;
min-width: 40em;
display: inline-block;
}
.CristalScroll {
display: flex;
min-width: 100%;
height: 100%;
min-height: 100%;
overflow: auto;
}

23
lotuspond/front/src/App.js
View File

@ -13,7 +13,23 @@ class App extends React.Component {
this.setState(() => ({client: client}))
})
this.state = {}
this.state = {
windows: {},
nextWindow: 0,
}
this.mountWindow = this.mountWindow.bind(this)
}
mountWindow(cb) {
const id = this.state.nextWindow
this.setState({nextWindow: id + 1})
const window = cb(() => {
this.setState(prev => ({windows: {...prev.windows, [id]: undefined}}))
})
this.setState(prev => ({windows: {...prev.windows, [id]: window}}))
}
render() {
@ -27,7 +43,10 @@ class App extends React.Component {
return (
<div className="App">
<NodeList client={this.state.client}/>
<NodeList client={this.state.client} mountWindow={this.mountWindow}/>
<div>
{Object.keys(this.state.windows).map((w, i) => <div key={i}>{this.state.windows[w]}</div>)}
</div>
</div>
)
}

24
lotuspond/front/src/FullNode.js
View File

@ -2,6 +2,7 @@ import React from 'react';
import { Client } from 'rpc-websockets'
import Cristal from 'react-cristal'
import { BlockLinks } from "./BlockLink";
import StorageNodeInit from "./StorageNodeInit";
const stateConnected = 'connected'
const stateConnecting = 'connecting'
@ -31,13 +32,13 @@ class FullNode extends React.Component {
this.loadInfo = this.loadInfo.bind(this)
this.startMining = this.startMining.bind(this)
this.newScepAddr = this.newScepAddr.bind(this)
this.startStorageMiner = this.startStorageMiner.bind(this)
this.connect()
}
async connect() {
console.log("gettok")
const token = await this.props.pondClient.call('Pond.TokenFor', [this.props.node.ID])
this.setState(() => ({
@ -101,6 +102,7 @@ class FullNode extends React.Component {
async startMining() {
// TODO: Use actual miner address
// see cli/miner.go
this.setState({mining: true})
let addr = "t0523423423" // in case we have no wallets
if (this.state.defaultAddr) {
addr = this.state.defaultAddr
@ -110,6 +112,16 @@ class FullNode extends React.Component {
await this.state.client.call("Filecoin.MinerStart", [addr])
}
async newScepAddr() {
const t = "secp256k1"
await this.state.client.call("Filecoin.WalletNew", [t])
this.loadInfo()
}
async startStorageMiner() {
this.props.mountWindow((onClose) => <StorageNodeInit fullRepo={this.props.node.Repo} fullConn={this.props.conn} pondClient={this.props.pondClient} onClose={onClose} mountWindow={this.props.mountWindow}/>)
}
render() {
let runtime = <div></div>
if (this.state.state === stateConnected) {
@ -129,6 +141,8 @@ class FullNode extends React.Component {
mine = "[Mining]"
}
let storageMine = <a href="#" onClick={this.startStorageMiner}>[Spawn Storage Miner]</a>
let balances = this.state.balances.map(([addr, balance]) => {
let line = <span>{truncAddr(addr)}:&nbsp;{balance}&nbsp;(ActTyp)</span>
if (this.state.defaultAddr === addr) {
@ -142,9 +156,11 @@ class FullNode extends React.Component {
<div>v{this.state.version.Version}, <abbr title={this.state.id}>{this.state.id.substr(-8)}</abbr>, {this.state.peers} peers</div>
<div>Repo: LOTUS_PATH={this.props.node.Repo}</div>
{chainInfo}
{mine}
<div>
<div>Balances:</div>
{mine} {storageMine}
</div>
<div>
<div>Balances: [New <a href="#" onClick={this.newScepAddr}>[Secp256k1]</a>]</div>
<div>{balances}</div>
</div>

91
lotuspond/front/src/NodeList.js
View File

@ -2,6 +2,8 @@ import React from 'react';
import FullNode from "./FullNode";
import ConnMgr from "./ConnMgr";
import Consensus from "./Consensus";
import {Cristal} from "react-cristal";
import StorageNode from "./StorageNode";
class NodeList extends React.Component {
constructor(props) {
@ -12,30 +14,52 @@ class NodeList extends React.Component {
showConnMgr: false,
showConsensus: false,
windows: {},
nextWindow: 0,
}
// This binding is necessary to make `this` work in the callback
this.spawnNode = this.spawnNode.bind(this)
this.connMgr = this.connMgr.bind(this)
this.consensus = this.consensus.bind(this)
this.mountWindow = this.mountWindow.bind(this)
this.getNodes()
}
mountNode(node) {
if (!node.Storage) {
this.props.mountWindow((onClose) =>
<FullNode key={node.ID}
node={{...node}}
pondClient={this.props.client}
onConnect={(conn, id) => this.setState(prev => ({
nodes: {
...prev.nodes,
[node.ID]: {...node, conn: conn, peerid: id}
}
}))}
mountWindow={this.props.mountWindow}/>)
} else {
this.props.mountWindow((onClose) =>
<StorageNode node={{...node}}
pondClient={this.props.client}
mountWindow={this.props.mountWindow}/>)
}
}
async getNodes() {
const nds = await this.props.client.call('Pond.Nodes')
const nodes = nds.reduce((o, i) => {o[i.ID] = i; return o}, {})
console.log('nds', nodes)
Object.keys(nodes).map(n => nodes[n]).forEach(n => this.mountNode(n))
this.setState({existingLoaded: true, nodes: nodes})
}
async spawnNode() {
const node = await this.props.client.call('Pond.Spawn')
console.log(node)
this.mountNode(node)
this.setState(state => ({nodes: {...state.nodes, [node.ID]: node}}))
}
@ -47,17 +71,6 @@ class NodeList extends React.Component {
this.setState({showConsensus: true})
}
mountWindow(cb) {
const id = this.state.nextWindow
this.setState({nextWindow: id + 1})
const window = cb(() => {
console.log("umount wnd todo")
})
this.setState(prev => ({windows: {...prev.windows, [id]: window}}))
}
render() {
let connMgr
if (this.state.showConnMgr) {
@ -66,35 +79,41 @@ class NodeList extends React.Component {
let consensus
if (this.state.showConsensus) {
consensus = (<Consensus nodes={this.state.nodes} mountWindow={this.mountWindow}/>)
consensus = (<Consensus nodes={this.state.nodes} mountWindow={this.props.mountWindow}/>)
}
return (
<div>
<div>
<button onClick={this.spawnNode} disabled={!this.state.existingLoaded}>Spawn Node</button>
<button onClick={this.connMgr} disabled={!this.state.existingLoaded && !this.state.showConnMgr}>Connections</button>
<button onClick={this.consensus} disabled={!this.state.existingLoaded && !this.state.showConsensus}>Consensus</button>
<Cristal title={"Node List"} initialPosition="bottom-left">
<div className={'NodeList'}>
<div>
<button onClick={this.spawnNode} disabled={!this.state.existingLoaded}>Spawn Node</button>
<button onClick={this.connMgr} disabled={!this.state.existingLoaded && !this.state.showConnMgr}>Connections</button>
<button onClick={this.consensus} disabled={!this.state.existingLoaded && !this.state.showConsensus}>Consensus</button>
</div>
<div>
{Object.keys(this.state.nodes).map(n => {
const nd = this.state.nodes[n]
let type = "FULL"
if (nd.Storage) {
type = "STOR"
}
let info = "[CONNECTING..]"
if (nd.conn) {
info = <span>{nd.peerid}</span>
}
return <div key={n}>
{n} {type} {info}
</div>
})}
</div>
</div>
<div>
{
Object.keys(this.state.nodes).map(n => {
const node = this.state.nodes[n]
return (<FullNode key={node.ID}
node={{...node}}
pondClient={this.props.client}
onConnect={(conn, id) => this.setState(prev => ({nodes: {...prev.nodes, [n]: {...node, conn: conn, peerid: id}}}))}
mountWindow={this.mountWindow}/>)
})
}
{connMgr}
{consensus}
</div>
<div>
{Object.keys(this.state.windows).map((w, i) => <div key={i}>{this.state.windows[w]}</div>)}
</div>
</div>
</Cristal>
);
}
}

136
lotuspond/front/src/StorageNode.js Normal file
View File

@ -0,0 +1,136 @@
import React from 'react';
import {Cristal} from "react-cristal";
import { Client } from 'rpc-websockets'
const stateConnected = 'connected'
const stateConnecting = 'connecting'
const stateGettingToken = 'getting-token'
let sealCodes = [
'Sealed',
'Pending',
'Failed',
'Sealing'
]
class StorageNode extends React.Component {
constructor(props) {
super(props)
this.state = {
state: stateGettingToken,
id: "~",
mining: false,
statusCounts: [0, 0, 0, 0]
}
this.loadInfo = this.loadInfo.bind(this)
this.sealGarbage = this.sealGarbage.bind(this)
this.connect()
}
async connect() {
const token = await this.props.pondClient.call('Pond.TokenFor', [this.props.node.ID])
this.setState(() => ({
state: stateConnecting,
token: token,
}))
const client = new Client(`ws://127.0.0.1:${this.props.node.ApiPort}/rpc/v0?token=${token}`)
client.on('open', async () => {
this.setState(() => ({
state: stateConnected,
client: client,
version: {Version: "~version~"},
id: "~peerid~",
peers: -1,
balances: []
}))
const id = await this.state.client.call("Filecoin.ID", [])
this.setState(() => ({id: id}))
// this.props.onConnect(client, id) // TODO: dedupe connecting part
this.loadInfo()
setInterval(this.loadInfo, 1050)
})
console.log(token) // todo: use
}
async loadInfo() {
const version = await this.state.client.call("Filecoin.Version", [])
this.setState(() => ({version: version}))
const peers = await this.state.client.call("Filecoin.NetPeers", [])
this.setState(() => ({peers: peers.length}))
/*const addrss = await this.state.client.call('Filecoin.WalletList', [])
let defaultAddr = ""
if (addrss.length > 0) {
defaultAddr = await this.state.client.call('Filecoin.WalletDefaultAddress', [])
}
this.setState(() => ({defaultAddr: defaultAddr}))
*/
await this.stagedList()
}
async stagedList() {
let stagedList = await this.state.client.call("Filecoin.SectorsStagedList", [])
let staged = await stagedList
.map(sector => this.state.client.call("Filecoin.SectorsStatus", [sector.SectorID]))
.reduce(async (p, n) => [...await p, await n], Promise.resolve([]))
let statusCounts = staged.reduce((p, n) => p.map((e, i) => e + (i === n.SealStatusCode ? 1 : 0) ), [0, 0, 0, 0])
this.setState({staged, statusCounts})
}
async sealGarbage() {
await this.state.client.call("Filecoin.StoreGarbageData", [])
}
render() {
let runtime = <div></div>
if (this.state.state === stateConnected) {
const sealGarbage = <a href="#" onClick={this.sealGarbage}>[Seal Garbage]</a>
runtime = (
<div>
<div>v{this.state.version.Version}, <abbr title={this.state.id}>{this.state.id.substr(-8)}</abbr>, {this.state.peers} peers</div>
<div>Repo: LOTUS_STORAGE_PATH={this.props.node.Repo}</div>
<div>
{sealGarbage}
</div>
<div>{this.state.statusCounts.map((c, i) => <span>{sealCodes[i]}: {c} | </span>)}</div>
<div>
{this.state.staged ? this.state.staged.map(s => (
<div>{s.SectorID} {sealCodes[s.SealStatusCode]}</div>
)) : <div></div>}
</div>
</div>
)
}
return <Cristal
title={"Storage Miner Node " + this.props.node.ID}
initialPosition={{x: this.props.node.ID*30, y: this.props.node.ID * 30}}>
<div className="CristalScroll">
<div className="StorageNode">
{runtime}
</div>
</div>
</Cristal>
}
}
export default StorageNode

26
lotuspond/front/src/StorageNodeInit.js Normal file
View File

@ -0,0 +1,26 @@
import React from 'react';
import {Cristal} from "react-cristal";
import StorageNode from "./StorageNode";
class StorageNodeInit extends React.Component {
async componentDidMount() {
const info = await this.props.pondClient.call('Pond.SpawnStorage', [this.props.fullRepo])
this.props.onClose()
this.props.mountWindow((onClose) => <StorageNode node={info} fullRepo={this.props.fullRepo} fullConn={this.props.fullConn} pondClient={this.props.pondClient} onClose={onClose} mountWindow={this.props.mountWindow}/>)
}
render() {
return <Cristal
title={"Storage miner initializing"}
initialPosition={'center'}>
<div className="CristalScroll">
<div className="StorageNodeInit">
Waiting for init, make sure at least one miner is enabled
</div>
</div>
</Cristal>
}
}
export default StorageNodeInit

61
lotuspond/main.go
View File

@ -37,6 +37,8 @@ type nodeInfo struct {
Repo string
ID int32
ApiPort int32
Storage bool
}
func (api *api) Spawn() (nodeInfo, error) {
@ -137,6 +139,65 @@ func (api *api) TokenFor(id int32) (string, error) {
return string(t), nil
}
func (api *api) SpawnStorage(fullNodeRepo string) (nodeInfo, error) {
dir, err := ioutil.TempDir(os.TempDir(), "lotus-storage-")
if err != nil {
return nodeInfo{}, err
}
errlogfile, err := os.OpenFile(dir+".err.log", os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
return nodeInfo{}, err
}
logfile, err := os.OpenFile(dir+".out.log", os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
return nodeInfo{}, err
}
id := atomic.AddInt32(&api.cmds, 1)
cmd := exec.Command("./lotus-storage-miner", "init")
cmd.Stderr = io.MultiWriter(os.Stderr, errlogfile)
cmd.Stdout = io.MultiWriter(os.Stdout, logfile)
cmd.Env = []string{"LOTUS_STORAGE_PATH=" + dir, "LOTUS_PATH=" + fullNodeRepo}
if err := cmd.Run(); err != nil {
return nodeInfo{}, err
}
time.Sleep(time.Millisecond * 300)
cmd = exec.Command("./lotus-storage-miner", "run", "--api", fmt.Sprintf("%d", 2500+id))
cmd.Stderr = io.MultiWriter(os.Stderr, errlogfile)
cmd.Stdout = io.MultiWriter(os.Stdout, logfile)
cmd.Env = []string{"LOTUS_STORAGE_PATH=" + dir, "LOTUS_PATH=" + fullNodeRepo}
if err := cmd.Start(); err != nil {
return nodeInfo{}, err
}
info := nodeInfo{
Repo: dir,
ID: id,
ApiPort: 2500 + id,
Storage: true,
}
api.runningLk.Lock()
api.running[id] = runningNode{
cmd: cmd,
meta: info,
stop: func() {
defer errlogfile.Close()
defer logfile.Close()
},
}
api.runningLk.Unlock()
time.Sleep(time.Millisecond * 750) // TODO: Something less terrible
return info, nil
}
func main() {
rpcServer := jsonrpc.NewServer()
rpcServer.Register("Pond", &api{running: map[int32]runningNode{}})

12
node/repo/fsrepo.go
View File

@ -67,15 +67,17 @@ func (fsr *FsRepo) Exists() (bool, error) {
}
func (fsr *FsRepo) Init() error {
if _, err := os.Stat(fsr.path); err == nil {
return fsr.initKeystore()
} else if !os.IsNotExist(err) {
exist, err := fsr.Exists()
if err != nil {
return err
}
if exist {
return nil
}
log.Infof("Initializing repo at '%s'", fsr.path)
err := os.Mkdir(fsr.path, 0755) //nolint: gosec
if err != nil {
err = os.Mkdir(fsr.path, 0755) //nolint: gosec
if err != nil && !os.IsExist(err) {
return err
}
c, err := os.Create(filepath.Join(fsr.path, fsConfig))