Merge pull request #396 from filecoin-project/travisperson-feat/rate-limit-fountain

fountain: Create miner endpoint

cmd/lotus-fountain/main.go

@@ -5,14 +5,17 @@ import (
 	"fmt"
 	"net/http"
 	"os"
+	"time"
 
 	rice "github.com/GeertJohan/go.rice"
 	logging "github.com/ipfs/go-log"
+	peer "github.com/libp2p/go-libp2p-peer"
 	"golang.org/x/xerrors"
 	"gopkg.in/urfave/cli.v2"
 
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/address"
 	"github.com/filecoin-project/lotus/chain/types"
 	lcli "github.com/filecoin-project/lotus/cli"
@@ -88,6 +91,22 @@ var runCmd = &cli.Command{
 			ctx:  ctx,
 			api:  nodeApi,
 			from: from,
+			limiter: NewLimiter(LimiterConfig{
+				TotalRate:   time.Second,
+				TotalBurst:  20,
+				IPRate:      time.Minute,
+				IPBurst:     5,
+				WalletRate:  15 * time.Minute,
+				WalletBurst: 1,
+			}),
+			colLimiter: NewLimiter(LimiterConfig{
+				TotalRate:   time.Second,
+				TotalBurst:  20,
+				IPRate:      10 * time.Minute,
+				IPBurst:     1,
+				WalletRate:  1 * time.Hour,
+				WalletBurst: 1,
+			}),
 		}
 
 		http.Handle("/", http.FileServer(rice.MustFindBox("site").HTTPBox()))
@@ -110,9 +129,25 @@ type handler struct {
 	api api.FullNode
 
 	from address.Address
+
+	limiter    *Limiter
+	colLimiter *Limiter
 }
 
 func (h *handler) send(w http.ResponseWriter, r *http.Request) {
+	// General limiter to allow throttling all messages that can make it into the mpool
+	if !h.limiter.Allow() {
+		http.Error(w, http.StatusText(http.StatusTooManyRequests), http.StatusTooManyRequests)
+		return
+	}
+
+	// Limit based on IP
+	limiter := h.limiter.GetIPLimiter(r.RemoteAddr)
+	if !limiter.Allow() {
+		http.Error(w, http.StatusText(http.StatusTooManyRequests), http.StatusTooManyRequests)
+		return
+	}
+
 	to, err := address.NewFromString(r.FormValue("address"))
 	if err != nil {
 		w.WriteHeader(400)
@@ -120,6 +155,13 @@ func (h *handler) send(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// Limit based on wallet address
+	limiter = h.limiter.GetWalletLimiter(to.String())
+	if !limiter.Allow() {
+		http.Error(w, http.StatusText(http.StatusTooManyRequests), http.StatusTooManyRequests)
+		return
+	}
+
 	smsg, err := h.api.MpoolPushMessage(h.ctx, &types.Message{
 		Value: sendPerRequest,
 		From:  h.from,
@@ -138,6 +180,132 @@ func (h *handler) send(w http.ResponseWriter, r *http.Request) {
 }
 
 func (h *handler) mkminer(w http.ResponseWriter, r *http.Request) {
-	w.WriteHeader(400)
+	// General limiter owner allow throttling all messages that can make it into the mpool
-	// todo
+	if !h.colLimiter.Allow() {
+		http.Error(w, http.StatusText(http.StatusTooManyRequests), http.StatusTooManyRequests)
+		return
+	}
+
+	// Limit based on IP
+	limiter := h.colLimiter.GetIPLimiter(r.RemoteAddr)
+	if !limiter.Allow() {
+		http.Error(w, http.StatusText(http.StatusTooManyRequests), http.StatusTooManyRequests)
+		return
+	}
+
+	owner, err := address.NewFromString(r.FormValue("address"))
+	if err != nil {
+		w.WriteHeader(400)
+		w.Write([]byte(err.Error()))
+		return
+	}
+
+	if owner.Protocol() != address.BLS {
+		w.WriteHeader(400)
+		w.Write([]byte("Miner address must use BLS"))
+		return
+	}
+
+	log.Infof("mkactor on %s", owner)
+
+	// Limit based on wallet address
+	limiter = h.colLimiter.GetWalletLimiter(owner.String())
+	if !limiter.Allow() {
+		http.Error(w, http.StatusText(http.StatusTooManyRequests), http.StatusTooManyRequests)
+		return
+	}
+	collateral, err := h.api.StatePledgeCollateral(r.Context(), nil)
+	if err != nil {
+		w.WriteHeader(400)
+		w.Write([]byte(err.Error()))
+		return
+	}
+
+	smsg, err := h.api.MpoolPushMessage(h.ctx, &types.Message{
+		Value: sendPerRequest,
+		From:  h.from,
+		To:    owner,
+
+		GasPrice: types.NewInt(0),
+		GasLimit: types.NewInt(1000),
+	})
+	if err != nil {
+		w.WriteHeader(400)
+		w.Write([]byte("pushfunds: " + err.Error()))
+		return
+	}
+	log.Infof("push funds to %s: %s", owner, smsg.Cid())
+
+	mw, err := h.api.StateWaitMsg(r.Context(), smsg.Cid())
+	if err != nil {
+		w.WriteHeader(400)
+		w.Write([]byte(err.Error()))
+		return
+	}
+
+	if mw.Receipt.ExitCode != 0 {
+		w.WriteHeader(400)
+		w.Write([]byte(xerrors.Errorf("create storage miner failed: exit code %d", mw.Receipt.ExitCode).Error()))
+		return
+	}
+
+	log.Infof("sendto %s ok", owner)
+
+	params, err := actors.SerializeParams(&actors.CreateStorageMinerParams{
+		Owner:      owner,
+		Worker:     owner,
+		SectorSize: build.SectorSizes[0], // TODO: dropdown allowing selection
+		PeerID:     peer.ID("SETME"),
+	})
+	if err != nil {
+		w.WriteHeader(400)
+		w.Write([]byte(err.Error()))
+		return
+	}
+
+	createStorageMinerMsg := &types.Message{
+		To:    actors.StorageMarketAddress,
+		From:  h.from,
+		Value: collateral,
+
+		Method: actors.SPAMethods.CreateStorageMiner,
+		Params: params,
+
+		GasLimit: types.NewInt(10000000),
+		GasPrice: types.NewInt(0),
+	}
+
+	signed, err := h.api.MpoolPushMessage(r.Context(), createStorageMinerMsg)
+	if err != nil {
+		w.WriteHeader(400)
+		w.Write([]byte(err.Error()))
+		return
+	}
+
+	log.Infof("smc %s", owner)
+
+	mw, err = h.api.StateWaitMsg(r.Context(), signed.Cid())
+	if err != nil {
+		w.WriteHeader(400)
+		w.Write([]byte(err.Error()))
+		return
+	}
+
+	if mw.Receipt.ExitCode != 0 {
+		w.WriteHeader(400)
+		w.Write([]byte(xerrors.Errorf("create storage miner failed: exit code %d", mw.Receipt.ExitCode).Error()))
+		return
+	}
+
+	addr, err := address.NewFromBytes(mw.Receipt.Return)
+	if err != nil {
+		w.WriteHeader(400)
+		w.Write([]byte(err.Error()))
+		return
+	}
+
+	w.Header().Set("Content-Type", "text/plain")
+	w.WriteHeader(200)
+	fmt.Fprintf(w, "New storage miners address is: %s\n", addr)
+	fmt.Fprintf(w, "Run lotus-storage-miner init --actor=%s --owner=%s", addr, owner)
 }

cmd/lotus-fountain/rate_limiter.go

@@ -0,0 +1,94 @@
+package main
+
+import (
+	"sync"
+	"time"
+
+	"golang.org/x/time/rate"
+)
+
+type Limiter struct {
+	control *rate.Limiter
+
+	ips     map[string]*rate.Limiter
+	wallets map[string]*rate.Limiter
+	mu      *sync.RWMutex
+
+	config LimiterConfig
+}
+
+type LimiterConfig struct {
+	TotalRate  time.Duration
+	TotalBurst int
+
+	IPRate  time.Duration
+	IPBurst int
+
+	WalletRate  time.Duration
+	WalletBurst int
+}
+
+func NewLimiter(c LimiterConfig) *Limiter {
+	return &Limiter{
+		control: rate.NewLimiter(rate.Every(c.TotalRate), c.TotalBurst),
+		mu:      &sync.RWMutex{},
+		ips:     make(map[string]*rate.Limiter),
+		wallets: make(map[string]*rate.Limiter),
+
+		config: c,
+	}
+}
+
+func (i *Limiter) Allow() bool {
+	return i.control.Allow()
+}
+
+func (i *Limiter) AddIPLimiter(ip string) *rate.Limiter {
+	i.mu.Lock()
+	defer i.mu.Unlock()
+
+	limiter := rate.NewLimiter(rate.Every(i.config.IPRate), i.config.IPBurst)
+
+	i.ips[ip] = limiter
+
+	return limiter
+}
+
+func (i *Limiter) GetIPLimiter(ip string) *rate.Limiter {
+	i.mu.Lock()
+	limiter, exists := i.ips[ip]
+
+	if !exists {
+		i.mu.Unlock()
+		return i.AddIPLimiter(ip)
+	}
+
+	i.mu.Unlock()
+
+	return limiter
+}
+
+func (i *Limiter) AddWalletLimiter(addr string) *rate.Limiter {
+	i.mu.Lock()
+	defer i.mu.Unlock()
+
+	limiter := rate.NewLimiter(rate.Every(i.config.WalletRate), i.config.WalletBurst)
+
+	i.wallets[addr] = limiter
+
+	return limiter
+}
+
+func (i *Limiter) GetWalletLimiter(wallet string) *rate.Limiter {
+	i.mu.Lock()
+	limiter, exists := i.wallets[wallet]
+
+	if !exists {
+		i.mu.Unlock()
+		return i.AddWalletLimiter(wallet)
+	}
+
+	i.mu.Unlock()
+
+	return limiter
+}

cmd/lotus-fountain/rate_limiter_test.go

@@ -0,0 +1,38 @@
+package main
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRateLimit(t *testing.T) {
+	limiter := NewLimiter(LimiterConfig{
+		TotalRate:   time.Second,
+		TotalBurst:  20,
+		IPRate:      time.Second,
+		IPBurst:     1,
+		WalletRate:  time.Second,
+		WalletBurst: 1,
+	})
+
+	for i := 0; i < 20; i++ {
+		assert.True(t, limiter.Allow())
+	}
+
+	assert.False(t, limiter.Allow())
+
+	time.Sleep(time.Second)
+	assert.True(t, limiter.Allow())
+
+	assert.True(t, limiter.GetIPLimiter("127.0.0.1").Allow())
+	assert.False(t, limiter.GetIPLimiter("127.0.0.1").Allow())
+	time.Sleep(time.Second)
+	assert.True(t, limiter.GetIPLimiter("127.0.0.1").Allow())
+
+	assert.True(t, limiter.GetWalletLimiter("abc123").Allow())
+	assert.False(t, limiter.GetWalletLimiter("abc123").Allow())
+	time.Sleep(time.Second)
+	assert.True(t, limiter.GetWalletLimiter("abc123").Allow())
+}

cmd/lotus-fountain/site/funds.html

@@ -0,0 +1,29 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <title>Sending Funds - Lotus Fountain</title>
+    <link rel="stylesheet" type="text/css" href="main.css">
+</head>
+<body>
+<div class="Index">
+    <div class="Index-nodes">
+        <div class="Index-node">
+            [SENDING FUNDS]
+        </div>
+        <div class="Index-node">
+            <form action='/send' method='get'>
+                <span>Enter destination address:</span>
+                <input type='text' name='address' style="width: 300px">
+                <button type='submit'>Send Funds</button>
+            </form>
+        </div>
+    </div>
+    <div class="Index-footer">
+        <div>
+            <a href="index.html">[Back]</a>
+            <span style="float: right">Not dispensing real Filecoin tokens</span>
+        </div>
+    </div>
+</div>
+</body>
+</html>

cmd/lotus-fountain/site/index.html

@@ -2,19 +2,26 @@
 <html>
 <head>
     <title>Lotus Fountain</title>
-    <style>
+    <link rel="stylesheet" type="text/css" href="main.css">
-        body {
-            font-family: 'monospace';
-            background: #1f1f1f;
-            color: #f0f0f0;
-        }
-    </style>
 </head>
 <body>
-<form action='/send' method='get'>
+<div class="Index">
-    <span>Enter destination address:</span>
+    <div class="Index-nodes">
-    <input type='text' name='address' style="width: 300px">
+        <div class="Index-node">
-    <button type='submit'>Send</button>
+            [LOTUS DEVNET FAUCET]
-</form>
+        </div>
+        <div class="Index-node">
+            <a href="funds.html">[Send Funds]</a>
+        </div>
+        <div class="Index-node">
+            <a href="miner.html">[Create Miner]</a>
+        </div>
+    </div>
+    <div class="Index-footer">
+        <div>
+            <span style="float: right">Not dispensing real Filecoin tokens</span>
+        </div>
+    </div>
+</div>
 </body>
 </html>

cmd/lotus-fountain/site/main.css

@@ -0,0 +1,56 @@
+body {
+    font-family: 'monospace';
+    background: #1f1f1f;
+    color: #f0f0f0;
+    padding: 0;
+    margin: 0;
+}
+
+.Index {
+    width: 100vw;
+    height: 100vh;
+    background: #1a1a1a;
+    color: #f0f0f0;
+    font-family: monospace;
+
+    display: grid;
+    grid-template-columns: auto 40vw auto;
+    grid-template-rows: auto auto auto 3em;
+    grid-template-areas:
+            ". . ."
+            ". main ."
+            ". . ."
+            "footer footer footer";
+}
+.Index-footer {
+    background: #2a2a2a;
+    grid-area: footer;
+}
+
+.Index-footer > div {
+    padding-left: 0.7em;
+    padding-top: 0.7em;
+}
+
+.Index-nodes {
+    grid-area: main;
+    background: #2a2a2a;
+}
+
+.Index-node {
+    margin: 5px;
+    padding: 15px;
+    background: #1f1f1f;
+}
+
+a:link {
+    color: #50f020;
+}
+
+a:visited {
+    color: #50f020;
+}
+
+a:hover {
+    color: #30a00a;
+}

cmd/lotus-fountain/site/miner.html

@@ -0,0 +1,43 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <title>Creating Storage Miner - Lotus Fountain</title>
+    <link rel="stylesheet" type="text/css" href="main.css">
+</head>
+<body>
+<div class="Index">
+    <div class="Index-nodes">
+        <div class="Index-node">
+            [CREATING STORAGE MINER]
+        </div>
+        <div class="Index-node" id="formnd">
+            <form id="f" action='/mkminer' method='get'>
+                <span>Enter destination address:</span>
+                <input type='text' name='address' style="width: 300px">
+                <button type='submit'>Send Funds</button>
+            </form>
+        </div>
+        <div id="plswait" style="display: none" class="Index-node">
+            <b>Waiting for transaction on chain..</b>
+        </div>
+        <div class="Index-node">
+            <span>When creating storage miner, DO NOT REFRESH THE PAGE, wait for it to load. This can take more than 5min.</span>
+        </div>
+    </div>
+    <div class="Index-footer">
+        <div>
+            <a href="index.html">[Back]</a>
+            <span style="float: right">Not dispensing real Filecoin tokens</span>
+        </div>
+    </div>
+</div>
+<script>
+    let f = document.getElementById('f')
+    f.onsubmit = ev => {
+        document.getElementById('plswait').style.display = 'block'
+        document.getElementById('formnd').style.display = 'none'
+    }
+
+</script>
+</body>
+</html>

go.mod

@@ -83,6 +83,7 @@ require (
 	go4.org v0.0.0-20190313082347-94abd6928b1d // indirect
 	golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472 // indirect
 	golang.org/x/sys v0.0.0-20190904154756-749cb33beabd // indirect
+	golang.org/x/time v0.0.0-20181108054448-85acf8d2951c
 	golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7
 	google.golang.org/api v0.9.0 // indirect
 	gopkg.in/cheggaaa/pb.v1 v1.0.28

go.sum

@@ -640,6 +640,7 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=