remove pkg/transformers; extract shared files needed to

libraries/shared; work on automated db migration management
2019-02-24 21:38:47 -06:00
parent 55fa9b8364
commit decc2a3caf
545 changed files with 431 additions and 15922 deletions
@@ -0,0 +1,192 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package repository
+
+import (
+	"bytes"
+	"database/sql"
+	"database/sql/driver"
+	"fmt"
+
+	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+func GetOrCreateIlk(ilk string, db *postgres.DB) (int, error) {
+	var ilkID int
+	err := db.Get(&ilkID, `SELECT id FROM maker.ilks WHERE ilk = $1`, ilk)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			insertErr := db.QueryRow(`INSERT INTO maker.ilks (ilk) VALUES ($1) RETURNING id`, ilk).Scan(&ilkID)
+			return ilkID, insertErr
+		}
+	}
+	return ilkID, err
+}
+
+func GetOrCreateIlkInTransaction(ilk string, tx *sql.Tx) (int, error) {
+	var ilkID int
+	err := tx.QueryRow(`SELECT id FROM maker.ilks WHERE ilk = $1`, ilk).Scan(&ilkID)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			insertErr := tx.QueryRow(`INSERT INTO maker.ilks (ilk) VALUES ($1) RETURNING id`, ilk).Scan(&ilkID)
+			return ilkID, insertErr
+		}
+	}
+	return ilkID, err
+}
+
+func MarkHeaderChecked(headerID int64, db *postgres.DB, checkedHeadersColumn string) error {
+	_, err := db.Exec(`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`)
+		VALUES ($1, $2)
+		ON CONFLICT (header_id) DO
+			UPDATE SET `+checkedHeadersColumn+` = checked_headers.`+checkedHeadersColumn+` + 1`, headerID, 1)
+	return err
+}
+
+func MarkHeaderCheckedInTransaction(headerID int64, tx *sql.Tx, checkedHeadersColumn string) error {
+	_, err := tx.Exec(`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`)
+		VALUES ($1, $2)
+		ON CONFLICT (header_id) DO
+			UPDATE SET `+checkedHeadersColumn+` = checked_headers.`+checkedHeadersColumn+` + 1`, headerID, 1)
+	return err
+}
+
+// Treats a header as missing if it's not in the headers table, or not checked for some log type
+func MissingHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.DB, notCheckedSQL string) ([]core.Header, error) {
+	var result []core.Header
+	var query string
+	var err error
+
+	if endingBlockNumber == -1 {
+		query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
+				LEFT JOIN checked_headers on headers.id = header_id
+				WHERE (header_id ISNULL OR ` + notCheckedSQL + `)
+				AND headers.block_number >= $1
+				AND headers.eth_node_fingerprint = $2`
+		err = db.Select(&result, query, startingBlockNumber, db.Node.ID)
+	} else {
+		query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
+				LEFT JOIN checked_headers on headers.id = header_id
+				WHERE (header_id ISNULL OR ` + notCheckedSQL + `)
+				AND headers.block_number >= $1
+				AND headers.block_number <= $2
+				AND headers.eth_node_fingerprint = $3`
+		err = db.Select(&result, query, startingBlockNumber, endingBlockNumber, db.Node.ID)
+	}
+
+	return result, err
+}
+
+func RecheckHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.DB, checkedHeadersColumn string) ([]core.Header, error) {
+	var result []core.Header
+	var query string
+	var err error
+
+	if endingBlockNumber == -1 {
+		query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
+				LEFT JOIN checked_headers on headers.id = header_id
+				WHERE ` + checkedHeadersColumn + ` between 1 and ` + constants.RecheckHeaderCap + `
+				AND headers.block_number >= $1
+				AND headers.eth_node_fingerprint = $2`
+		err = db.Select(&result, query, startingBlockNumber, db.Node.ID)
+	} else {
+		query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
+				LEFT JOIN checked_headers on headers.id = header_id
+				WHERE ` + checkedHeadersColumn + ` between 1 and ` + constants.RecheckHeaderCap + `
+				AND headers.block_number >= $1
+				AND headers.block_number <= $2
+				AND headers.eth_node_fingerprint = $3`
+		err = db.Select(&result, query, startingBlockNumber, endingBlockNumber, db.Node.ID)
+	}
+
+	return result, err
+}
+
+func GetCheckedColumnNames(db *postgres.DB) ([]string, error) {
+	// Query returns `[]driver.Value`, nullable polymorphic interface
+	var queryResult []driver.Value
+	columnNamesQuery :=
+		`SELECT column_name FROM information_schema.columns
+		WHERE table_schema = 'public'
+			AND table_name = 'checked_headers'
+			AND column_name ~ '_checked';`
+
+	err := db.Select(&queryResult, columnNamesQuery)
+	if err != nil {
+		return []string{}, err
+	}
+
+	// Transform column names from `driver.Value` to strings
+	var columnNames []string
+	for _, result := range queryResult {
+		if columnName, ok := result.(string); ok {
+			columnNames = append(columnNames, columnName)
+		} else {
+			return []string{}, fmt.Errorf("incorrect value for checked_headers column name")
+		}
+	}
+
+	return columnNames, nil
+}
+
+// Builds a SQL string that checks if any column value is 0, given the column names.
+// Defaults to FALSE when no columns are provided.
+// Ex: ["columnA", "columnB"] => "NOT (columnA!=0 AND columnB!=0)"
+//     [] => "FALSE"
+func CreateNotCheckedSQL(boolColumns []string, recheckHeaders constants.TransformerExecution) string {
+
+	var result bytes.Buffer
+
+	if len(boolColumns) == 0 {
+		return "FALSE"
+	}
+
+	result.WriteString("NOT (")
+
+	// Loop excluding last column name
+	for _, column := range boolColumns[:len(boolColumns)-1] {
+
+		if recheckHeaders {
+			result.WriteString(fmt.Sprintf("%v>=%s AND ", column, constants.RecheckHeaderCap))
+		} else {
+			result.WriteString(fmt.Sprintf("%v!=0 AND ", column))
+		}
+	}
+
+	// No trailing "OR" for the last column name
+	if recheckHeaders {
+		result.WriteString(fmt.Sprintf("%v>=%s)", boolColumns[len(boolColumns)-1], constants.RecheckHeaderCap))
+	} else {
+		result.WriteString(fmt.Sprintf("%v!=0)", boolColumns[len(boolColumns)-1]))
+
+	}
+
+	return result.String()
+}
+
+func GetTicInTx(headerID int64, tx *sql.Tx) (int64, error) {
+	var blockTimestamp int64
+	err := tx.QueryRow(`SELECT block_timestamp FROM public.headers WHERE id = $1;`, headerID).Scan(&blockTimestamp)
+	if err != nil {
+		return 0, err
+	}
+
+	tic := blockTimestamp + constants.TTL
+	return tic, nil
+}
@@ -0,0 +1,183 @@
+// VulcanizeDB
+// Copyright © 2018 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package repository_test
+
+import (
+	"fmt"
+	"math/rand"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
+	shared "github.com/vulcanize/vulcanizedb/libraries/shared/repository"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"github.com/vulcanize/vulcanizedb/test_config"
+)
+
+var _ = Describe("Repository utilities", func() {
+	Describe("MissingHeaders", func() {
+		var (
+			db                       *postgres.DB
+			headerRepository         datastore.HeaderRepository
+			startingBlockNumber      int64
+			endingBlockNumber        int64
+			eventSpecificBlockNumber int64
+			blockNumbers             []int64
+			headerIDs                []int64
+			notCheckedSQL            string
+			err                      error
+		)
+
+		BeforeEach(func() {
+			db = test_config.NewTestDB(test_config.NewTestNode())
+			test_config.CleanTestDB(db)
+			headerRepository = repositories.NewHeaderRepository(db)
+
+			columnNames, err := shared.GetCheckedColumnNames(db)
+			Expect(err).NotTo(HaveOccurred())
+			notCheckedSQL = shared.CreateNotCheckedSQL(columnNames, constants.HeaderMissing)
+
+			startingBlockNumber = rand.Int63()
+			eventSpecificBlockNumber = startingBlockNumber + 1
+			endingBlockNumber = startingBlockNumber + 2
+			outOfRangeBlockNumber := endingBlockNumber + 1
+
+			blockNumbers = []int64{startingBlockNumber, eventSpecificBlockNumber, endingBlockNumber, outOfRangeBlockNumber}
+
+			headerIDs = []int64{}
+			for _, n := range blockNumbers {
+				headerID, err := headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(n))
+				headerIDs = append(headerIDs, headerID)
+				Expect(err).NotTo(HaveOccurred())
+			}
+		})
+
+		It("only treats headers as checked if the event specific logs have been checked", func() {
+			_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
+			Expect(err).NotTo(HaveOccurred())
+
+			headers, err := shared.MissingHeaders(startingBlockNumber, endingBlockNumber, db, notCheckedSQL)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(headers)).To(Equal(3))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber)))
+			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber)))
+		})
+
+		It("only returns headers associated with the current node", func() {
+			dbTwo := test_config.NewTestDB(core.Node{ID: "second"})
+			headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
+			for _, n := range blockNumbers {
+				_, err = headerRepositoryTwo.CreateOrUpdateHeader(fakes.GetFakeHeader(n + 10))
+				Expect(err).NotTo(HaveOccurred())
+			}
+
+			Expect(err).NotTo(HaveOccurred())
+			nodeOneMissingHeaders, err := shared.MissingHeaders(startingBlockNumber, endingBlockNumber, db, notCheckedSQL)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(nodeOneMissingHeaders)).To(Equal(3))
+			Expect(nodeOneMissingHeaders[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(eventSpecificBlockNumber), Equal(endingBlockNumber)))
+			Expect(nodeOneMissingHeaders[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(eventSpecificBlockNumber), Equal(endingBlockNumber)))
+			Expect(nodeOneMissingHeaders[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(startingBlockNumber), Equal(eventSpecificBlockNumber), Equal(endingBlockNumber)))
+
+			nodeTwoMissingHeaders, err := shared.MissingHeaders(startingBlockNumber, endingBlockNumber+10, dbTwo, notCheckedSQL)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(nodeTwoMissingHeaders)).To(Equal(3))
+			Expect(nodeTwoMissingHeaders[0].BlockNumber).To(Or(Equal(startingBlockNumber+10), Equal(eventSpecificBlockNumber+10), Equal(endingBlockNumber+10)))
+			Expect(nodeTwoMissingHeaders[1].BlockNumber).To(Or(Equal(startingBlockNumber+10), Equal(eventSpecificBlockNumber+10), Equal(endingBlockNumber+10)))
+		})
+	})
+
+	Describe("GetCheckedColumnNames", func() {
+		It("gets the column names from checked_headers", func() {
+			db := test_config.NewTestDB(test_config.NewTestNode())
+			test_config.CleanTestDB(db)
+			expectedColumnNames := getExpectedColumnNames()
+			actualColumnNames, err := shared.GetCheckedColumnNames(db)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(actualColumnNames).To(Equal(expectedColumnNames))
+		})
+	})
+
+	Describe("CreateNotCheckedSQL", func() {
+		It("generates a correct SQL string for one column", func() {
+			columns := []string{"columnA"}
+			expected := "NOT (columnA!=0)"
+			actual := shared.CreateNotCheckedSQL(columns, constants.HeaderMissing)
+			Expect(actual).To(Equal(expected))
+		})
+
+		It("generates a correct SQL string for several columns", func() {
+			columns := []string{"columnA", "columnB"}
+			expected := "NOT (columnA!=0 AND columnB!=0)"
+			actual := shared.CreateNotCheckedSQL(columns, constants.HeaderMissing)
+			Expect(actual).To(Equal(expected))
+		})
+
+		It("defaults to FALSE when there are no columns", func() {
+			expected := "FALSE"
+			actual := shared.CreateNotCheckedSQL([]string{}, constants.HeaderMissing)
+			Expect(actual).To(Equal(expected))
+		})
+
+		It("generates a correct SQL string for rechecking headers", func() {
+			columns := []string{"columnA", "columnB"}
+			expected := fmt.Sprintf("NOT (columnA>=%s AND columnB>=%s)", constants.RecheckHeaderCap, constants.RecheckHeaderCap)
+			actual := shared.CreateNotCheckedSQL(columns, constants.HeaderRecheck)
+			Expect(actual).To(Equal(expected))
+		})
+	})
+})
+
+func getExpectedColumnNames() []string {
+	return []string{
+		"price_feeds_checked",
+		"flip_kick_checked",
+		"frob_checked",
+		"tend_checked",
+		"bite_checked",
+		"dent_checked",
+		"pit_file_debt_ceiling_checked",
+		"pit_file_ilk_checked",
+		"vat_init_checked",
+		"drip_file_ilk_checked",
+		"drip_file_repo_checked",
+		"drip_file_vow_checked",
+		"deal_checked",
+		"drip_drip_checked",
+		"cat_file_chop_lump_checked",
+		"cat_file_flip_checked",
+		"cat_file_pit_vow_checked",
+		"flop_kick_checked",
+		"vat_move_checked",
+		"vat_fold_checked",
+		"vat_heal_checked",
+		"vat_toll_checked",
+		"vat_tune_checked",
+		"vat_grab_checked",
+		"vat_flux_checked",
+		"vat_slip_checked",
+		"vow_flog_checked",
+		"flap_kick_checked",
+	}
+}