Update checked headers for new transformers

- If a header was marked as checked before a transformer was added to the watcher, mark all headers since the new transformer's starting block number as unchecked.
2019-08-28 09:25:14 -05:00
parent d76be4962b
commit 666ea1c325
19 changed files with 538 additions and 49 deletions
@@ -41,6 +41,12 @@ func (repo CheckedHeadersRepository) MarkHeaderChecked(headerID int64) error {
 	return err
 }

+// Remove checked_headers rows with block number >= starting block number
+func (repo CheckedHeadersRepository) MarkHeadersUnchecked(startingBlockNumber int64) error {
+	_, err := repo.db.Exec(`DELETE FROM public.checked_headers WHERE header_id IN (SELECT id FROM public.headers WHERE block_number >= $1)`, startingBlockNumber)
+	return err
+}
+
 // Return header_id if not present in checked_headers or its check_count is < passed checkCount
 func (repo CheckedHeadersRepository) MissingHeaders(startingBlockNumber, endingBlockNumber, checkCount int64) ([]core.Header, error) {
 	var result []core.Header
@@ -73,6 +73,47 @@ var _ = Describe("Checked Headers repository", func() {
 		})
 	})

+	Describe("MarkHeadersUnchecked", func() {
+		It("removes rows for headers <= starting block number", func() {
+			blockNumberOne := rand.Int63()
+			blockNumberTwo := blockNumberOne + 1
+			blockNumberThree := blockNumberOne + 2
+			fakeHeaderOne := fakes.GetFakeHeader(blockNumberOne)
+			fakeHeaderTwo := fakes.GetFakeHeader(blockNumberTwo)
+			fakeHeaderThree := fakes.GetFakeHeader(blockNumberThree)
+			headerRepository := repositories.NewHeaderRepository(db)
+			// insert three headers with incrementing block number
+			headerIdOne, insertHeaderOneErr := headerRepository.CreateOrUpdateHeader(fakeHeaderOne)
+			Expect(insertHeaderOneErr).NotTo(HaveOccurred())
+			headerIdTwo, insertHeaderTwoErr := headerRepository.CreateOrUpdateHeader(fakeHeaderTwo)
+			Expect(insertHeaderTwoErr).NotTo(HaveOccurred())
+			headerIdThree, insertHeaderThreeErr := headerRepository.CreateOrUpdateHeader(fakeHeaderThree)
+			Expect(insertHeaderThreeErr).NotTo(HaveOccurred())
+			// mark all headers checked
+			markHeaderOneCheckedErr := repo.MarkHeaderChecked(headerIdOne)
+			Expect(markHeaderOneCheckedErr).NotTo(HaveOccurred())
+			markHeaderTwoCheckedErr := repo.MarkHeaderChecked(headerIdTwo)
+			Expect(markHeaderTwoCheckedErr).NotTo(HaveOccurred())
+			markHeaderThreeCheckedErr := repo.MarkHeaderChecked(headerIdThree)
+			Expect(markHeaderThreeCheckedErr).NotTo(HaveOccurred())
+
+			// mark headers unchecked since blockNumberTwo
+			err := repo.MarkHeadersUnchecked(blockNumberTwo)
+
+			Expect(err).NotTo(HaveOccurred())
+			var headerOneChecked, headerTwoChecked, headerThreeChecked bool
+			getHeaderOneErr := db.Get(&headerOneChecked, `SELECT EXISTS(SELECT 1 FROM public.checked_headers WHERE header_id = $1)`, headerIdOne)
+			Expect(getHeaderOneErr).NotTo(HaveOccurred())
+			Expect(headerOneChecked).To(BeTrue())
+			getHeaderTwoErr := db.Get(&headerTwoChecked, `SELECT EXISTS(SELECT 1 FROM public.checked_headers WHERE header_id = $1)`, headerIdTwo)
+			Expect(getHeaderTwoErr).NotTo(HaveOccurred())
+			Expect(headerTwoChecked).To(BeFalse())
+			getHeaderThreeErr := db.Get(&headerThreeChecked, `SELECT EXISTS(SELECT 1 FROM public.checked_headers WHERE header_id = $1)`, headerIdThree)
+			Expect(getHeaderThreeErr).NotTo(HaveOccurred())
+			Expect(headerThreeChecked).To(BeFalse())
+		})
+	})
+
 	Describe("MissingHeaders", func() {
 		var (
 			headerRepository      datastore.HeaderRepository
@@ -0,0 +1,69 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package repositories
+
+import (
+	"github.com/sirupsen/logrus"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+type CheckedLogsRepository struct {
+	db *postgres.DB
+}
+
+func NewCheckedLogsRepository(db *postgres.DB) CheckedLogsRepository {
+	return CheckedLogsRepository{db: db}
+}
+
+// Return whether a given address + topic0 has been fetched on a previous run of vDB
+func (repository CheckedLogsRepository) HaveLogsBeenChecked(addresses []string, topic0 string) (bool, error) {
+	for _, address := range addresses {
+		var addressExists bool
+		getAddressExistsErr := repository.db.Get(&addressExists, `SELECT EXISTS(SELECT 1 FROM public.checked_logs WHERE contract_address = $1)`, address)
+		if getAddressExistsErr != nil {
+			return false, getAddressExistsErr
+		}
+		if !addressExists {
+			return false, nil
+		}
+	}
+	var topicZeroExists bool
+	getTopicZeroExistsErr := repository.db.Get(&topicZeroExists, `SELECT EXISTS(SELECT 1 FROM public.checked_logs WHERE topic_zero = $1)`, topic0)
+	if getTopicZeroExistsErr != nil {
+		return false, getTopicZeroExistsErr
+	}
+	return topicZeroExists, nil
+}
+
+// Persist that a given address + topic0 has is being fetched on this run of vDB
+func (repository CheckedLogsRepository) MarkLogsChecked(addresses []string, topic0 string) error {
+	tx, txErr := repository.db.Beginx()
+	if txErr != nil {
+		return txErr
+	}
+	for _, address := range addresses {
+		_, insertErr := tx.Exec(`INSERT INTO public.checked_logs (contract_address, topic_zero) VALUES ($1, $2)`, address, topic0)
+		if insertErr != nil {
+			rollbackErr := tx.Rollback()
+			if rollbackErr != nil {
+				logrus.Errorf("error rolling back transaction inserting checked logs: %s", rollbackErr.Error())
+			}
+			return insertErr
+		}
+	}
+	return tx.Commit()
+}
@@ -0,0 +1,110 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package repositories_test
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"github.com/vulcanize/vulcanizedb/test_config"
+)
+
+var _ = Describe("Checked logs repository", func() {
+	var (
+		db            *postgres.DB
+		fakeAddress   = fakes.FakeAddress.Hex()
+		fakeAddresses = []string{fakeAddress}
+		fakeTopicZero = fakes.FakeHash.Hex()
+		repository    datastore.CheckedLogsRepository
+	)
+
+	BeforeEach(func() {
+		db = test_config.NewTestDB(test_config.NewTestNode())
+		test_config.CleanTestDB(db)
+		repository = repositories.NewCheckedLogsRepository(db)
+	})
+
+	Describe("HaveLogsBeenChecked", func() {
+		It("returns true if all addresses and the topic0 are already present in the db", func() {
+			_, insertErr := db.Exec(`INSERT INTO public.checked_logs (contract_address, topic_zero) VALUES ($1, $2)`, fakeAddress, fakeTopicZero)
+			Expect(insertErr).NotTo(HaveOccurred())
+
+			hasBeenChecked, err := repository.HaveLogsBeenChecked(fakeAddresses, fakeTopicZero)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(hasBeenChecked).To(BeTrue())
+		})
+
+		It("returns true if addresses and topic0 were fetched because of a combination of other transformers", func() {
+			anotherFakeAddress := common.HexToAddress("0x" + fakes.RandomString(40)).Hex()
+			anotherFakeTopicZero := common.HexToHash("0x" + fakes.RandomString(64)).Hex()
+			// insert row with matching address but different topic0
+			_, insertOneErr := db.Exec(`INSERT INTO public.checked_logs (contract_address, topic_zero) VALUES ($1, $2)`, fakeAddress, anotherFakeTopicZero)
+			Expect(insertOneErr).NotTo(HaveOccurred())
+			// insert row with matching topic0 but different address
+			_, insertTwoErr := db.Exec(`INSERT INTO public.checked_logs (contract_address, topic_zero) VALUES ($1, $2)`, anotherFakeAddress, fakeTopicZero)
+			Expect(insertTwoErr).NotTo(HaveOccurred())
+
+			hasBeenChecked, err := repository.HaveLogsBeenChecked(fakeAddresses, fakeTopicZero)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(hasBeenChecked).To(BeTrue())
+		})
+
+		It("returns false if any address has not been checked", func() {
+			anotherFakeAddress := common.HexToAddress("0x" + fakes.RandomString(40)).Hex()
+			_, insertErr := db.Exec(`INSERT INTO public.checked_logs (contract_address, topic_zero) VALUES ($1, $2)`, fakeAddress, fakeTopicZero)
+			Expect(insertErr).NotTo(HaveOccurred())
+
+			hasBeenChecked, err := repository.HaveLogsBeenChecked(append(fakeAddresses, anotherFakeAddress), fakeTopicZero)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(hasBeenChecked).To(BeFalse())
+		})
+
+		It("returns false if topic0 has not been checked", func() {
+			anotherFakeTopicZero := common.HexToHash("0x" + fakes.RandomString(64)).Hex()
+			_, insertErr := db.Exec(`INSERT INTO public.checked_logs (contract_address, topic_zero) VALUES ($1, $2)`, fakeAddress, anotherFakeTopicZero)
+			Expect(insertErr).NotTo(HaveOccurred())
+
+			hasBeenChecked, err := repository.HaveLogsBeenChecked(fakeAddresses, fakeTopicZero)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(hasBeenChecked).To(BeFalse())
+		})
+	})
+
+	Describe("MarkLogsChecked", func() {
+		It("adds a row for all of transformer's addresses + topic0", func() {
+			anotherFakeAddress := common.HexToAddress("0x" + fakes.RandomString(40)).Hex()
+			err := repository.MarkLogsChecked(append(fakeAddresses, anotherFakeAddress), fakeTopicZero)
+
+			Expect(err).NotTo(HaveOccurred())
+			var comboOneExists, comboTwoExists bool
+			getComboOneErr := db.Get(&comboOneExists, `SELECT EXISTS(SELECT 1 FROM public.checked_logs WHERE contract_address = $1 AND topic_zero = $2)`, fakeAddress, fakeTopicZero)
+			Expect(getComboOneErr).NotTo(HaveOccurred())
+			Expect(comboOneExists).To(BeTrue())
+			getComboTwoErr := db.Get(&comboTwoExists, `SELECT EXISTS(SELECT 1 FROM public.checked_logs WHERE contract_address = $1 AND topic_zero = $2)`, anotherFakeAddress, fakeTopicZero)
+			Expect(getComboTwoErr).NotTo(HaveOccurred())
+			Expect(comboTwoExists).To(BeTrue())
+		})
+	})
+})