diff --git a/miner/remote_agent.go b/miner/remote_agent.go
index 18ddf121c..00b5f7e08 100644
--- a/miner/remote_agent.go
+++ b/miner/remote_agent.go
@@ -20,6 +20,7 @@ import (
 	"errors"
 	"math/big"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/ethereum/ethash"
@@ -45,6 +46,8 @@ type RemoteAgent struct {
 
 	hashrateMu sync.RWMutex
 	hashrate   map[common.Hash]hashrate
+
+	running int32 // running indicates whether the agent is active. Call atomically
 }
 
 func NewRemoteAgent() *RemoteAgent {
@@ -70,18 +73,22 @@ func (a *RemoteAgent) SetReturnCh(returnCh chan<- *Result) {
 }
 
 func (a *RemoteAgent) Start() {
+	if !atomic.CompareAndSwapInt32(&a.running, 0, 1) {
+		return
+	}
+
 	a.quit = make(chan struct{})
 	a.workCh = make(chan *Work, 1)
 	go a.maintainLoop()
 }
 
 func (a *RemoteAgent) Stop() {
-	if a.quit != nil {
-		close(a.quit)
-	}
-	if a.workCh != nil {
-		close(a.workCh)
+	if !atomic.CompareAndSwapInt32(&a.running, 1, 0) {
+		return
 	}
+
+	close(a.quit)
+	close(a.workCh)
 }
 
 // GetHashRate returns the accumulated hashrate of all identifier combined