finish method polling with hash or address type argument values

collected from watched events; 'MissingMethodsCheckedEventsIntersection'
method to find headers which have been checked for each event of
interest but methods have not yet been polled at that header; tests for
new features; travis ci go version 1.9 -> 1.11 ; consolidate omniWatcher and lightOmniWatcher into single command with light as default

pkg/omni/light/transformer/transformer.go

@@ -70,10 +70,20 @@ type transformer struct {
 	EventArgs  map[string][]string
 	MethodArgs map[string][]string
 
-	// Whether or not to create a list of token holder addresses for the contract in postgres
+	// Whether or not to create a list of emitted address or hashes for the contract in postgres
 	CreateAddrList map[string]bool
+	CreateHashList map[string]bool
+
+	// Method piping on/off for a contract
+	Piping map[string]bool
 }
 
+// Order-of-operations:
+// 1. Create new transformer
+// 2. Load contract addresses and their parameters
+// 3. Init
+// 3. Execute
+
 // Transformer takes in config for blockchain, database, and network id
 func NewTransformer(network string, bc core.BlockChain, db *postgres.DB) *transformer {
 
@@ -92,6 +102,8 @@ func NewTransformer(network string, bc core.BlockChain, db *postgres.DB) *transf
 		EventArgs:        map[string][]string{},
 		MethodArgs:       map[string][]string{},
 		CreateAddrList:   map[string]bool{},
+		CreateHashList:   map[string]bool{},
+		Piping:           map[string]bool{},
 	}
 }
 
@@ -154,6 +166,8 @@ func (tr *transformer) Init() error {
 			FilterArgs:     eventArgs,
 			MethodArgs:     methodArgs,
 			CreateAddrList: tr.CreateAddrList[contractAddr],
+			CreateHashList: tr.CreateHashList[contractAddr],
+			Piping:         tr.Piping[contractAddr],
 		}.Init()
 	}
 
@@ -164,24 +178,31 @@ func (tr *transformer) Execute() error {
 	if len(tr.Contracts) == 0 {
 		return errors.New("error: transformer has no initialized contracts")
 	}
-	// Iterate through all internal contracts
+	// Iterate through all initialized contracts
 	for _, con := range tr.Contracts {
 		// Update converter with current contract
-		tr.Update(con)
-
+		tr.Converter.Update(con)
+		// This is so that same header slice is retrieved for each event iteration
+		last, err := tr.BlockRetriever.RetrieveMostRecentBlock()
+		if err != nil {
+			return err
+		}
 		// Iterate through events
+		eventIds := make([]string, 0, len(con.Events))
 		for _, event := range con.Events {
 			// Filter using the event signature
 			topics := [][]common.Hash{{common.HexToHash(helpers.GenerateSignature(event.Sig()))}}
 
 			// Generate eventID and use it to create a checked_header column if one does not already exist
 			eventId := strings.ToLower(event.Name + "_" + con.Address)
-			if err := tr.AddCheckColumn(eventId); err != nil {
+			eventIds = append(eventIds, eventId)
+			err := tr.HeaderRepository.AddCheckColumn(eventId)
+			if err != nil {
 				return err
 			}
 
 			// Find unchecked headers for this event
-			missingHeaders, err := tr.MissingHeaders(con.StartingBlock, con.LastBlock, eventId)
+			missingHeaders, err := tr.HeaderRepository.MissingHeaders(con.StartingBlock, last, eventId)
 			if err != nil {
 				return err
 			}
@@ -189,14 +210,14 @@ func (tr *transformer) Execute() error {
 			// Iterate over headers
 			for _, header := range missingHeaders {
 				// And fetch event logs using the header, contract address, and topics filter
-				logs, err := tr.FetchLogs([]string{con.Address}, topics, header)
+				logs, err := tr.Fetcher.FetchLogs([]string{con.Address}, topics, header)
 				if err != nil {
 					return err
 				}
 
 				// Mark the header checked for this eventID and continue to next iteration if no logs are found
 				if len(logs) < 1 {
-					err = tr.MarkHeaderChecked(header.Id, eventId)
+					err = tr.HeaderRepository.MarkHeaderChecked(header.Id, eventId)
 					if err != nil {
 						return err
 					}
@@ -204,7 +225,7 @@ func (tr *transformer) Execute() error {
 				}
 
 				// Convert logs into custom type
-				convertedLogs, err := tr.Convert(logs, event, header.Id)
+				convertedLogs, err := tr.Converter.Convert(logs, event, header.Id)
 				if err != nil {
 					return err
 				}
@@ -213,18 +234,46 @@ func (tr *transformer) Execute() error {
 				}
 
 				// If logs aren't empty, persist them
-				err = tr.PersistLogs(convertedLogs, event, con.Address, con.Name)
+				err = tr.EventRepository.PersistLogs(convertedLogs, event, con.Address, con.Name)
 				if err != nil {
 					return err
 				}
-
-				// Poll contract methods at this header's block height
-				// with arguments collected from event logs up to this point
-				if err := tr.PollContractAt(*con, header.BlockNumber); err != nil {
-					return err
-				}
 			}
 		}
+
+		if len(con.Methods) == 0 {
+			continue
+		}
+
+		// Create checked_headers columns for each method id
+		methodIds := make([]string, 0, len(con.Methods))
+		for _, m := range con.Methods {
+			methodId := strings.ToLower(m.Name + "_" + con.Address)
+			err = tr.HeaderRepository.AddCheckColumn(methodId)
+			if err != nil {
+				return err
+			}
+			methodIds = append(methodIds, methodId)
+		}
+
+		// Retrieve headers that have been checked for all events but haven not been checked for the methods
+		missingHeaders, err := tr.HeaderRepository.MissingMethodsCheckedEventsIntersection(con.StartingBlock, last, methodIds, eventIds)
+		if err != nil {
+			return err
+		}
+		// Poll over the missing headers
+		for _, header := range missingHeaders {
+			err = tr.Poller.PollContractAt(*con, header.BlockNumber)
+			if err != nil {
+				return err
+			}
+		}
+		// Mark those headers checked for the methods
+		err = tr.HeaderRepository.MarkHeadersChecked(missingHeaders, methodIds)
+		if err != nil {
+			return err
+		}
+
 	}
 
 	return nil
@@ -255,7 +304,17 @@ func (tr *transformer) SetRange(contractAddr string, rng [2]int64) {
 	tr.ContractRanges[contractAddr] = rng
 }
 
-// Used to set the block range to watch for a given address
+// Used to set whether or not to persist an account address list
 func (tr *transformer) SetCreateAddrList(contractAddr string, on bool) {
 	tr.CreateAddrList[contractAddr] = on
 }
+
+// Used to set whether or not to persist an hash list
+func (tr *transformer) SetCreateHashList(contractAddr string, on bool) {
+	tr.CreateHashList[contractAddr] = on
+}
+
+// Used to turn method piping on for a contract
+func (tr *transformer) SetPiping(contractAddr string, on bool) {
+	tr.Piping[contractAddr] = on
+}

pkg/omni/light/transformer/transformer_test.go

@@ -93,6 +93,22 @@ var _ = Describe("Transformer", func() {
 		})
 	})
 
+	Describe("SetCreateAddrList", func() {
+		It("Sets the block range that the contract should be watched within", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetCreateAddrList(constants.TusdContractAddress, true)
+			Expect(t.CreateAddrList[constants.TusdContractAddress]).To(Equal(true))
+		})
+	})
+
+	Describe("SetCreateHashList", func() {
+		It("Sets the block range that the contract should be watched within", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetCreateHashList(constants.TusdContractAddress, true)
+			Expect(t.CreateHashList[constants.TusdContractAddress]).To(Equal(true))
+		})
+	})
+
 	Describe("Init", func() {
 		It("Initializes transformer's contract objects", func() {
 			headerRepository.CreateOrUpdateHeader(mocks.MockHeader1)
@@ -211,18 +227,6 @@ var _ = Describe("Transformer", func() {
 			Expect(err).ToNot(HaveOccurred())
 
 			res := test_helpers.BalanceOf{}
-
-			c, ok := t.Contracts[constants.TusdContractAddress]
-			Expect(ok).To(Equal(true))
-
-			b, ok := c.EmittedAddrs[common.HexToAddress("0x1062a747393198f70F71ec65A582423Dba7E5Ab3")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			b, ok = c.EmittedAddrs[common.HexToAddress("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
 			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x1062a747393198f70F71ec65A582423Dba7E5Ab3' AND block = '6791669'", constants.TusdContractAddress)).StructScan(&res)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(res.Balance).To(Equal("55849938025000000000000"))
@@ -241,4 +245,130 @@ var _ = Describe("Transformer", func() {
 			Expect(err).To(HaveOccurred())
 		})
 	})
+
+	Describe("Execute- against ENS registry contract", func() {
+		BeforeEach(func() {
+			header1, err := blockChain.GetHeaderByNumber(6885695)
+			Expect(err).ToNot(HaveOccurred())
+			header2, err := blockChain.GetHeaderByNumber(6885696)
+			Expect(err).ToNot(HaveOccurred())
+			header3, err := blockChain.GetHeaderByNumber(6885697)
+			Expect(err).ToNot(HaveOccurred())
+			headerRepository.CreateOrUpdateHeader(header1)
+			headerID, err = headerRepository.CreateOrUpdateHeader(header2)
+			Expect(err).ToNot(HaveOccurred())
+			headerRepository.CreateOrUpdateHeader(header3)
+		})
+
+		It("Transforms watched contract data into custom repositories", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
+			t.SetMethods(constants.EnsContractAddress, nil)
+
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+
+			log := test_helpers.LightNewOwnerLog{}
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", constants.EnsContractAddress)).StructScan(&log)
+
+			// We don't know vulcID, so compare individual fields instead of complete structures
+			Expect(log.HeaderID).To(Equal(headerID))
+			Expect(log.Node).To(Equal("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"))
+			Expect(log.Label).To(Equal("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047"))
+			Expect(log.Owner).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
+		})
+
+		It("Keeps track of contract-related hashes while transforming event data if they need to be used for later method polling", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
+			t.SetMethods(constants.EnsContractAddress, []string{"owner"})
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+
+			c, ok := t.Contracts[constants.EnsContractAddress]
+			Expect(ok).To(Equal(true))
+
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(c.EmittedHashes)).To(Equal(2))
+
+			b, ok := c.EmittedHashes[common.HexToHash("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae")]
+			Expect(ok).To(Equal(true))
+			Expect(b).To(Equal(true))
+
+			b, ok = c.EmittedHashes[common.HexToHash("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047")]
+			Expect(ok).To(Equal(true))
+			Expect(b).To(Equal(true))
+
+			// Doesn't keep track of address since it wouldn't be used in calling the 'owner' method
+			_, ok = c.EmittedAddrs[common.HexToAddress("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef")]
+			Expect(ok).To(Equal(false))
+		})
+
+		It("Polls given methods using generated token holder address", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
+			t.SetMethods(constants.EnsContractAddress, []string{"owner"})
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+
+			res := test_helpers.Owner{}
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", constants.EnsContractAddress)).StructScan(&res)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
+			Expect(res.TokenName).To(Equal(""))
+
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", constants.EnsContractAddress)).StructScan(&res)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
+			Expect(res.TokenName).To(Equal(""))
+
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ceMADEUPaaf4HASHc186badTHIS288IS625bFAKE' AND block = '6885696'", constants.EnsContractAddress)).StructScan(&res)
+			Expect(err).To(HaveOccurred())
+		})
+
+		It("It does not perist events if they do not pass the emitted arg filter", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
+			t.SetMethods(constants.EnsContractAddress, nil)
+			t.SetEventArgs(constants.EnsContractAddress, []string{"fake_filter_value"})
+
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+
+			log := test_helpers.LightNewOwnerLog{}
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", constants.EnsContractAddress)).StructScan(&log)
+			Expect(err).To(HaveOccurred())
+		})
+
+		It("If a method arg filter is applied, only those arguments are used in polling", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
+			t.SetMethods(constants.EnsContractAddress, []string{"owner"})
+			t.SetMethodArgs(constants.EnsContractAddress, []string{"0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"})
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+
+			res := test_helpers.Owner{}
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", constants.EnsContractAddress)).StructScan(&res)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
+			Expect(res.TokenName).To(Equal(""))
+
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", constants.EnsContractAddress)).StructScan(&res)
+			Expect(err).To(HaveOccurred())
+		})
+	})
 })