Write event logs to database before transforming

- enables decoupling event extraction/persistence from transformation
- modifies event transformer, converter, and log chunker to accept
  payload that includes internal log database ID with log data
- remove alias for transformer pkg as shared_t
- remove unused mock watcher repository

libraries/shared/chunker/log_chunker.go

@@ -20,14 +20,13 @@ import (
 	"strings"
 
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/types"
-
-	shared_t "github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
 )
 
 type Chunker interface {
-	AddConfigs(transformerConfigs []shared_t.EventTransformerConfig)
-	ChunkLogs(logs []types.Log) map[string][]types.Log
+	AddConfigs(transformerConfigs []transformer.EventTransformerConfig)
+	ChunkLogs(logs []core.HeaderSyncLog) map[string][]core.HeaderSyncLog
 }
 
 type LogChunker struct {
@@ -45,7 +44,7 @@ func NewLogChunker() *LogChunker {
 }
 
 // Configures the chunker by adding more addreses and topics to consider.
-func (chunker *LogChunker) AddConfigs(transformerConfigs []shared_t.EventTransformerConfig) {
+func (chunker *LogChunker) AddConfigs(transformerConfigs []transformer.EventTransformerConfig) {
 	for _, config := range transformerConfigs {
 		for _, address := range config.ContractAddresses {
 			var lowerCaseAddress = strings.ToLower(address)
@@ -56,15 +55,15 @@ func (chunker *LogChunker) AddConfigs(transformerConfigs []shared_t.EventTransfo
 }
 
 // Goes through an array of logs, associating relevant logs (matching addresses and topic) with transformers
-func (chunker *LogChunker) ChunkLogs(logs []types.Log) map[string][]types.Log {
-	chunks := map[string][]types.Log{}
+func (chunker *LogChunker) ChunkLogs(logs []core.HeaderSyncLog) map[string][]core.HeaderSyncLog {
+	chunks := map[string][]core.HeaderSyncLog{}
 	for _, log := range logs {
 		// Topic0 is not unique to each transformer, also need to consider the contract address
-		relevantTransformers := chunker.AddressToNames[strings.ToLower(log.Address.String())]
+		relevantTransformers := chunker.AddressToNames[strings.ToLower(log.Log.Address.Hex())]
 
-		for _, transformer := range relevantTransformers {
-			if chunker.NameToTopic0[transformer] == log.Topics[0] {
-				chunks[transformer] = append(chunks[transformer], log)
+		for _, t := range relevantTransformers {
+			if chunker.NameToTopic0[t] == log.Log.Topics[0] {
+				chunks[t] = append(chunks[t], log)
 			}
 		}
 	}

libraries/shared/chunker/log_chunker_test.go

@@ -23,34 +23,35 @@ import (
 	. "github.com/onsi/gomega"
 
 	chunk "github.com/vulcanize/vulcanizedb/libraries/shared/chunker"
-	shared_t "github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
 )
 
 var _ = Describe("Log chunker", func() {
 	var (
-		configs []shared_t.EventTransformerConfig
+		configs []transformer.EventTransformerConfig
 		chunker *chunk.LogChunker
 	)
 
 	BeforeEach(func() {
-		configA := shared_t.EventTransformerConfig{
+		configA := transformer.EventTransformerConfig{
 			TransformerName:   "TransformerA",
 			ContractAddresses: []string{"0x00000000000000000000000000000000000000A1", "0x00000000000000000000000000000000000000A2"},
 			Topic:             "0xA",
 		}
-		configB := shared_t.EventTransformerConfig{
+		configB := transformer.EventTransformerConfig{
 			TransformerName:   "TransformerB",
 			ContractAddresses: []string{"0x00000000000000000000000000000000000000B1"},
 			Topic:             "0xB",
 		}
 
-		configC := shared_t.EventTransformerConfig{
+		configC := transformer.EventTransformerConfig{
 			TransformerName:   "TransformerC",
 			ContractAddresses: []string{"0x00000000000000000000000000000000000000A2"},
 			Topic:             "0xC",
 		}
 
-		configs = []shared_t.EventTransformerConfig{configA, configB, configC}
+		configs = []transformer.EventTransformerConfig{configA, configB, configC}
 		chunker = chunk.NewLogChunker()
 		chunker.AddConfigs(configs)
 	})
@@ -73,24 +74,24 @@ var _ = Describe("Log chunker", func() {
 
 	Describe("AddConfigs", func() {
 		It("can add more configs later", func() {
-			configD := shared_t.EventTransformerConfig{
+			configD := transformer.EventTransformerConfig{
 				TransformerName:   "TransformerD",
 				ContractAddresses: []string{"0x000000000000000000000000000000000000000D"},
 				Topic:             "0xD",
 			}
-			chunker.AddConfigs([]shared_t.EventTransformerConfig{configD})
+			chunker.AddConfigs([]transformer.EventTransformerConfig{configD})
 
 			Expect(chunker.AddressToNames).To(ContainElement([]string{"TransformerD"}))
 			Expect(chunker.NameToTopic0).To(ContainElement(common.HexToHash("0xD")))
 		})
 
 		It("lower cases address", func() {
-			configD := shared_t.EventTransformerConfig{
+			configD := transformer.EventTransformerConfig{
 				TransformerName:   "TransformerD",
 				ContractAddresses: []string{"0x000000000000000000000000000000000000000D"},
 				Topic:             "0xD",
 			}
-			chunker.AddConfigs([]shared_t.EventTransformerConfig{configD})
+			chunker.AddConfigs([]transformer.EventTransformerConfig{configD})
 
 			Expect(chunker.AddressToNames["0x000000000000000000000000000000000000000d"]).To(Equal([]string{"TransformerD"}))
 		})
@@ -98,7 +99,7 @@ var _ = Describe("Log chunker", func() {
 
 	Describe("ChunkLogs", func() {
 		It("only associates logs with relevant topic0 and address to transformers", func() {
-			logs := []types.Log{log1, log2, log3, log4, log5}
+			logs := []core.HeaderSyncLog{log1, log2, log3, log4, log5}
 			chunks := chunker.ChunkLogs(logs)
 
 			Expect(chunks["TransformerA"]).To(And(ContainElement(log1), ContainElement(log4)))
@@ -110,43 +111,53 @@ var _ = Describe("Log chunker", func() {
 
 var (
 	// Match TransformerA
-	log1 = types.Log{
-		Address: common.HexToAddress("0xA1"),
-		Topics: []common.Hash{
-			common.HexToHash("0xA"),
-			common.HexToHash("0xLogTopic1"),
+	log1 = core.HeaderSyncLog{
+		Log: types.Log{
+			Address: common.HexToAddress("0xA1"),
+			Topics: []common.Hash{
+				common.HexToHash("0xA"),
+				common.HexToHash("0xLogTopic1"),
+			},
 		},
 	}
 	// Match TransformerA address, but not topic0
-	log2 = types.Log{
-		Address: common.HexToAddress("0xA1"),
-		Topics: []common.Hash{
-			common.HexToHash("0xB"),
-			common.HexToHash("0xLogTopic2"),
+	log2 = core.HeaderSyncLog{
+		Log: types.Log{
+			Address: common.HexToAddress("0xA1"),
+			Topics: []common.Hash{
+				common.HexToHash("0xB"),
+				common.HexToHash("0xLogTopic2"),
+			},
 		},
 	}
 	// Match TransformerA topic, but TransformerB address
-	log3 = types.Log{
-		Address: common.HexToAddress("0xB1"),
-		Topics: []common.Hash{
-			common.HexToHash("0xA"),
-			common.HexToHash("0xLogTopic3"),
+	log3 = core.HeaderSyncLog{
+		Log: types.Log{
+			Address: common.HexToAddress("0xB1"),
+			Topics: []common.Hash{
+				common.HexToHash("0xA"),
+				common.HexToHash("0xLogTopic3"),
+			},
 		},
 	}
 	// Match TransformerA, with the other address
-	log4 = types.Log{
-		Address: common.HexToAddress("0xA2"),
-		Topics: []common.Hash{
-			common.HexToHash("0xA"),
-			common.HexToHash("0xLogTopic4"),
+	log4 = core.HeaderSyncLog{
+		Log: types.Log{
+			Address: common.HexToAddress("0xA2"),
+			Topics: []common.Hash{
+				common.HexToHash("0xA"),
+				common.HexToHash("0xLogTopic4"),
+			},
 		},
 	}
 	// Match TransformerC, which shares address with TransformerA
-	log5 = types.Log{
-		Address: common.HexToAddress("0xA2"),
-		Topics: []common.Hash{
-			common.HexToHash("0xC"),
-			common.HexToHash("0xLogTopic5"),
+	log5 = core.HeaderSyncLog{
+		Log: types.Log{
+			Address: common.HexToAddress("0xA2"),
+			Topics: []common.Hash{
+				common.HexToHash("0xC"),
+				common.HexToHash("0xLogTopic5"),
+			},
 		},
 	}
 )