split backfill range up into smaller bins and process them concurrently; improve tests; review fixes

libraries/shared/streamer/statediff_streamer.go

@@ -1,16 +1,18 @@
-// Copyright 2019 Vulcanize
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 package streamer
 
@@ -18,24 +20,29 @@ import (
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/ethereum/go-ethereum/statediff"
 	"github.com/sirupsen/logrus"
+
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 )
 
+// Streamer is the interface for streaming a statediff subscription
 type Streamer interface {
-	Stream(chan statediff.Payload) (*rpc.ClientSubscription, error)
+	Stream(payloadChan chan statediff.Payload) (*rpc.ClientSubscription, error)
 }
 
+// StateDiffStreamer is the underlying struct for the StateDiffStreamer interface
 type StateDiffStreamer struct {
-	client core.RPCClient
+	Client core.RPCClient
 }
 
-func (streamer *StateDiffStreamer) Stream(payloadChan chan statediff.Payload) (*rpc.ClientSubscription, error) {
-	logrus.Info("streaming diffs from geth")
-	return streamer.client.Subscribe("statediff", payloadChan, "stream")
-}
-
-func NewStateDiffStreamer(client core.RPCClient) StateDiffStreamer {
-	return StateDiffStreamer{
-		client: client,
+// NewStateDiffStreamer creates a pointer to a new StateDiffStreamer which satisfies the IStateDiffStreamer interface
+func NewStateDiffStreamer(client core.RPCClient) Streamer {
+	return &StateDiffStreamer{
+		Client: client,
 	}
 }
+
+// Stream is the main loop for subscribing to data from the Geth state diff process
+func (sds *StateDiffStreamer) Stream(payloadChan chan statediff.Payload) (*rpc.ClientSubscription, error) {
+	logrus.Info("streaming diffs from geth")
+	return sds.Client.Subscribe("statediff", payloadChan, "stream")
+}