deep-stack/ipld-eth-server
6
0
Fork 0
forked from cerc-io/ipld-eth-server
Code Pull Requests Activity

Address repo updates (#134)

Browse Source 
* Factor out get or create address into one sql string

* Factor out getChecksumAddress method in address repo

* Update address repo methods to not need a receiver

* Move address repository to libraries/shared
This commit is contained in:
Elizabeth 2019-10-03 11:17:08 -05:00 committed by GitHub
parent 031043130e
commit f6ab9382b2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 115 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
libraries/shared/repository/address_repository.go Normal file
View File

@ -0,0 +1,73 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package repository
import (
"github.com/ethereum/go-ethereum/common"
"github.com/jmoiron/sqlx"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
const getOrCreateAddressQuery = `WITH addressId AS (
INSERT INTO addresses (address) VALUES ($1) ON CONFLICT DO NOTHING RETURNING id
)
SELECT id FROM addresses WHERE address = $1
UNION
SELECT id FROM addressId`
func GetOrCreateAddress(db *postgres.DB, address string) (int64, error) {
checksumAddress := getChecksumAddress(address)
var addressId int64
getOrCreateErr := db.Get(&addressId, getOrCreateAddressQuery, checksumAddress)
return addressId, getOrCreateErr
}
func GetOrCreateAddressInTransaction(tx *sqlx.Tx, address string) (int64, error) {
checksumAddress := getChecksumAddress(address)
var addressId int64
getOrCreateErr := tx.Get(&addressId, getOrCreateAddressQuery, checksumAddress)
return addressId, getOrCreateErr
}
func GetAddressById(db *postgres.DB, id int64) (string, error) {
var address string
getErr := db.Get(&address, `SELECT address FROM public.addresses WHERE id = $1`, id)
return address, getErr
}
func getChecksumAddress(address string) string {
stringAddressToCommonAddress := common.HexToAddress(address)
return stringAddressToCommonAddress.Hex()
}

46
pkg/datastore/postgres/repositories/address_repository_test.go → libraries/shared/repository/address_repository_test.go
View File

@ -1,22 +1,23 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package repositories_test
package repository_test
import (
"github.com/vulcanize/vulcanizedb/libraries/shared/repository"
"strings"
"github.com/jmoiron/sqlx"
@ -24,7 +25,6 @@ import (
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/test_config"
)
@ -32,13 +32,11 @@ import (
var _ = Describe("address lookup", func() {
var (
db *postgres.DB
repo repositories.AddressRepository
address = fakes.FakeAddress.Hex()
)
BeforeEach(func() {
db = test_config.NewTestDB(test_config.NewTestNode())
test_config.CleanTestDB(db)
repo = repositories.AddressRepository{}
})
AfterEach(func() {
@ -52,7 +50,7 @@ var _ = Describe("address lookup", func() {
Describe("GetOrCreateAddress", func() {
It("creates an address record", func() {
addressId, createErr := repo.GetOrCreateAddress(db, address)
addressId, createErr := repository.GetOrCreateAddress(db, address)
Expect(createErr).NotTo(HaveOccurred())
var actualAddress dbAddress
@ -63,10 +61,10 @@ var _ = Describe("address lookup", func() {
})
It("returns the existing record id if the address already exists", func() {
createId, createErr := repo.GetOrCreateAddress(db, address)
createId, createErr := repository.GetOrCreateAddress(db, address)
Expect(createErr).NotTo(HaveOccurred())
getId, getErr := repo.GetOrCreateAddress(db, address)
getId, getErr := repository.GetOrCreateAddress(db, address)
Expect(getErr).NotTo(HaveOccurred())
var addressCount int
@ -78,20 +76,20 @@ var _ = Describe("address lookup", func() {
It("gets upper-cased addresses", func() {
upperAddress := strings.ToUpper(address)
upperAddressId, createErr := repo.GetOrCreateAddress(db, upperAddress)
upperAddressId, createErr := repository.GetOrCreateAddress(db, upperAddress)
Expect(createErr).NotTo(HaveOccurred())
mixedCaseAddressId, getErr := repo.GetOrCreateAddress(db, address)
mixedCaseAddressId, getErr := repository.GetOrCreateAddress(db, address)
Expect(getErr).NotTo(HaveOccurred())
Expect(upperAddressId).To(Equal(mixedCaseAddressId))
})
It("gets lower-cased addresses", func() {
lowerAddress := strings.ToLower(address)
upperAddressId, createErr := repo.GetOrCreateAddress(db, lowerAddress)
upperAddressId, createErr := repository.GetOrCreateAddress(db, lowerAddress)
Expect(createErr).NotTo(HaveOccurred())
mixedCaseAddressId, getErr := repo.GetOrCreateAddress(db, address)
mixedCaseAddressId, getErr := repository.GetOrCreateAddress(db, address)
Expect(getErr).NotTo(HaveOccurred())
Expect(upperAddressId).To(Equal(mixedCaseAddressId))
})
@ -112,7 +110,7 @@ var _ = Describe("address lookup", func() {
})
It("creates an address record", func() {
addressId, createErr := repo.GetOrCreateAddressInTransaction(tx, address)
addressId, createErr := repository.GetOrCreateAddressInTransaction(tx, address)
Expect(createErr).NotTo(HaveOccurred())
commitErr := tx.Commit()
Expect(commitErr).NotTo(HaveOccurred())
@ -125,10 +123,10 @@ var _ = Describe("address lookup", func() {
})
It("returns the existing record id if the address already exists", func() {
_, createErr := repo.GetOrCreateAddressInTransaction(tx, address)
_, createErr := repository.GetOrCreateAddressInTransaction(tx, address)
Expect(createErr).NotTo(HaveOccurred())
_, getErr := repo.GetOrCreateAddressInTransaction(tx, address)
_, getErr := repository.GetOrCreateAddressInTransaction(tx, address)
Expect(getErr).NotTo(HaveOccurred())
tx.Commit()
@ -139,10 +137,10 @@ var _ = Describe("address lookup", func() {
It("gets upper-cased addresses", func() {
upperAddress := strings.ToUpper(address)
upperAddressId, createErr := repo.GetOrCreateAddressInTransaction(tx, upperAddress)
upperAddressId, createErr := repository.GetOrCreateAddressInTransaction(tx, upperAddress)
Expect(createErr).NotTo(HaveOccurred())
mixedCaseAddressId, getErr := repo.GetOrCreateAddressInTransaction(tx, address)
mixedCaseAddressId, getErr := repository.GetOrCreateAddressInTransaction(tx, address)
Expect(getErr).NotTo(HaveOccurred())
tx.Commit()
@ -151,10 +149,10 @@ var _ = Describe("address lookup", func() {
It("gets lower-cased addresses", func() {
lowerAddress := strings.ToLower(address)
upperAddressId, createErr := repo.GetOrCreateAddressInTransaction(tx, lowerAddress)
upperAddressId, createErr := repository.GetOrCreateAddressInTransaction(tx, lowerAddress)
Expect(createErr).NotTo(HaveOccurred())
mixedCaseAddressId, getErr := repo.GetOrCreateAddressInTransaction(tx, address)
mixedCaseAddressId, getErr := repository.GetOrCreateAddressInTransaction(tx, address)
Expect(getErr).NotTo(HaveOccurred())
tx.Commit()
@ -164,16 +162,16 @@ var _ = Describe("address lookup", func() {
Describe("GetAddressById", func() {
It("gets and address by it's id", func() {
addressId, createErr := repo.GetOrCreateAddress(db, address)
addressId, createErr := repository.GetOrCreateAddress(db, address)
Expect(createErr).NotTo(HaveOccurred())
actualAddress, getErr := repo.GetAddressById(db, addressId)
actualAddress, getErr := repository.GetAddressById(db, addressId)
Expect(getErr).NotTo(HaveOccurred())
Expect(actualAddress).To(Equal(address))
})
It("returns an error if the id doesn't exist", func() {
_, getErr := repo.GetAddressById(db, 0)
_, getErr := repository.GetAddressById(db, 0)
Expect(getErr).To(HaveOccurred())
Expect(getErr).To(MatchError("sql: no rows in result set"))
})

8
pkg/contract_watcher/full/retriever/block_retriever.go
View File

@ -18,8 +18,8 @@ package retriever
import (
"database/sql"
"github.com/vulcanize/vulcanizedb/libraries/shared/repository"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
)
// Block retriever is used to retrieve the first block for a given contract and the most recent block
@ -55,7 +55,7 @@ func (r *blockRetriever) RetrieveFirstBlock(contractAddr string) (int64, error)
// For some contracts the contract creation transaction receipt doesn't have the contract address so this doesn't work (e.g. Sai)
func (r *blockRetriever) retrieveFirstBlockFromReceipts(contractAddr string) (int64, error) {
var firstBlock int64
addressId, getAddressErr := addressRepository().GetOrCreateAddress(r.db, contractAddr)
addressId, getAddressErr := repository.GetOrCreateAddress(r.db, contractAddr)
if getAddressErr != nil {
return firstBlock, getAddressErr
}
@ -72,10 +72,6 @@ func (r *blockRetriever) retrieveFirstBlockFromReceipts(contractAddr string) (in
return firstBlock, err
}
func addressRepository() repositories.AddressRepository {
return repositories.AddressRepository{}
}
// In which case this servers as a heuristic to find the first block by finding the first contract event log
func (r *blockRetriever) retrieveFirstBlockFromLogs(contractAddr string) (int64, error) {
var firstBlock int

62
pkg/datastore/postgres/repositories/address_repository.go
View File

@ -1,62 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package repositories
import (
"database/sql"
"github.com/ethereum/go-ethereum/common"
"github.com/jmoiron/sqlx"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
type AddressRepository struct{}
func (AddressRepository) GetOrCreateAddress(db *postgres.DB, address string) (int64, error) {
stringAddressToCommonAddress := common.HexToAddress(address)
hexAddress := stringAddressToCommonAddress.Hex()
var addressId int64
getErr := db.Get(&addressId, `SELECT id FROM public.addresses WHERE address = $1`, hexAddress)
if getErr == sql.ErrNoRows {
insertErr := db.QueryRow(`INSERT INTO public.addresses (address) VALUES($1) RETURNING id`, hexAddress).Scan(&addressId)
return addressId, insertErr
}
return addressId, getErr
}
func (AddressRepository) GetOrCreateAddressInTransaction(tx *sqlx.Tx, address string) (int64, error) {
stringAddressToCommonAddress := common.HexToAddress(address)
hexAddress := stringAddressToCommonAddress.Hex()
var addressId int64
getErr := tx.Get(&addressId, `SELECT id FROM public.addresses WHERE address = $1`, hexAddress)
if getErr == sql.ErrNoRows {
insertErr := tx.QueryRow(`INSERT INTO public.addresses (address) VALUES($1) RETURNING id`, hexAddress).Scan(&addressId)
return addressId, insertErr
}
return addressId, getErr
}
func (AddressRepository) GetAddressById(db *postgres.DB, id int64) (string, error) {
var address string
getErr := db.Get(&address, `SELECT address FROM public.addresses WHERE id = $1`, id)
return address, getErr
}

3
pkg/datastore/postgres/repositories/full_sync_receipt_repository.go
View File

@ -20,6 +20,7 @@ import (
"database/sql"
"github.com/jmoiron/sqlx"
"github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/libraries/shared/repository"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
@ -84,7 +85,7 @@ func createLogs(logs []core.FullSyncLog, receiptId int64, tx *sqlx.Tx) error {
func (FullSyncReceiptRepository) CreateFullSyncReceiptInTx(blockId int64, receipt core.Receipt, tx *sqlx.Tx) (int64, error) {
var receiptId int64
addressId, getAddressErr := AddressRepository{}.GetOrCreateAddressInTransaction(tx, receipt.ContractAddress)
addressId, getAddressErr := repository.GetOrCreateAddressInTransaction(tx, receipt.ContractAddress)
if getAddressErr != nil {
logrus.Error("createReceipt: Error getting address id: ", getAddressErr)
return receiptId, getAddressErr

23
pkg/datastore/postgres/repositories/header_sync_log_repository.go
View File

@ -22,6 +22,7 @@ import (
"github.com/jmoiron/sqlx"
"github.com/lib/pq"
"github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/libraries/shared/repository"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
@ -31,14 +32,12 @@ const insertHeaderSyncLogQuery = `INSERT INTO header_sync_logs
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) ON CONFLICT DO NOTHING`
type HeaderSyncLogRepository struct {
db *postgres.DB
addressRepository AddressRepository
db *postgres.DB
}
func NewHeaderSyncLogRepository(db *postgres.DB) HeaderSyncLogRepository {
return HeaderSyncLogRepository{
db: db,
addressRepository: AddressRepository{},
db: db,
}
}
@ -57,8 +56,8 @@ type headerSyncLog struct {
Raw []byte
}
func (repository HeaderSyncLogRepository) GetUntransformedHeaderSyncLogs() ([]core.HeaderSyncLog, error) {
rows, queryErr := repository.db.Queryx(`SELECT * FROM public.header_sync_logs WHERE transformed = false`)
func (repo HeaderSyncLogRepository) GetUntransformedHeaderSyncLogs() ([]core.HeaderSyncLog, error) {
rows, queryErr := repo.db.Queryx(`SELECT * FROM public.header_sync_logs WHERE transformed = false`)
if queryErr != nil {
return nil, queryErr
}
@ -74,7 +73,7 @@ func (repository HeaderSyncLogRepository) GetUntransformedHeaderSyncLogs() ([]co
for _, topic := range rawLog.Topics {
logTopics = append(logTopics, common.BytesToHash(topic))
}
address, addrErr := repository.addressRepository.GetAddressById(repository.db, rawLog.Address)
address, addrErr := repository.GetAddressById(repo.db, rawLog.Address)
if addrErr != nil {
return nil, addrErr
}
@ -104,13 +103,13 @@ func (repository HeaderSyncLogRepository) GetUntransformedHeaderSyncLogs() ([]co
return results, nil
}
func (repository HeaderSyncLogRepository) CreateHeaderSyncLogs(headerID int64, logs []types.Log) error {
tx, txErr := repository.db.Beginx()
func (repo HeaderSyncLogRepository) CreateHeaderSyncLogs(headerID int64, logs []types.Log) error {
tx, txErr := repo.db.Beginx()
if txErr != nil {
return txErr
}
for _, log := range logs {
err := repository.insertLog(headerID, log, tx)
err := repo.insertLog(headerID, log, tx)
if err != nil {
rollbackErr := tx.Rollback()
if rollbackErr != nil {
@ -122,13 +121,13 @@ func (repository HeaderSyncLogRepository) CreateHeaderSyncLogs(headerID int64, l
return tx.Commit()
}
func (repository HeaderSyncLogRepository) insertLog(headerID int64, log types.Log, tx *sqlx.Tx) error {
func (repo HeaderSyncLogRepository) insertLog(headerID int64, log types.Log, tx *sqlx.Tx) error {
topics := buildTopics(log)
raw, jsonErr := log.MarshalJSON()
if jsonErr != nil {
return jsonErr
}
addressID, addrErr := repository.addressRepository.GetOrCreateAddressInTransaction(tx, log.Address.Hex())
addressID, addrErr := repository.GetOrCreateAddressInTransaction(tx, log.Address.Hex())
if addrErr != nil {
return addrErr
}

7
pkg/datastore/postgres/repositories/header_sync_log_repository_test.go
View File

@ -22,6 +22,7 @@ import (
"github.com/lib/pq"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
repository2 "github.com/vulcanize/vulcanizedb/libraries/shared/repository"
"github.com/vulcanize/vulcanizedb/libraries/shared/test_data"
"github.com/vulcanize/vulcanizedb/pkg/datastore"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
@ -79,8 +80,7 @@ var _ = Describe("Header sync log repository", func() {
Expect(lookupErr).NotTo(HaveOccurred())
Expect(dbLog.ID).NotTo(BeZero())
Expect(dbLog.HeaderID).To(Equal(headerID))
addressRepository := repositories.AddressRepository{}
actualAddress, addressErr := addressRepository.GetAddressById(db, dbLog.Address)
actualAddress, addressErr := repository2.GetAddressById(db, dbLog.Address)
Expect(addressErr).NotTo(HaveOccurred())
Expect(actualAddress).To(Equal(log.Address.Hex()))
Expect(dbLog.Topics[0]).To(Equal(log.Topics[0].Bytes()))
@ -128,8 +128,7 @@ var _ = Describe("Header sync log repository", func() {
logTopics = append(logTopics, common.BytesToHash(topic))
}
addressRepository := repositories.AddressRepository{}
actualAddress, addressErr := addressRepository.GetAddressById(db, dbLog.Address)
actualAddress, addressErr := repository2.GetAddressById(db, dbLog.Address)
Expect(addressErr).NotTo(HaveOccurred())
reconstructedLog := types.Log{
Address: common.HexToAddress(actualAddress),

3
pkg/datastore/postgres/repositories/header_sync_receipt_repository.go
View File

@ -19,6 +19,7 @@ package repositories
import (
"github.com/ethereum/go-ethereum/log"
"github.com/jmoiron/sqlx"
"github.com/vulcanize/vulcanizedb/libraries/shared/repository"
"github.com/vulcanize/vulcanizedb/pkg/core"
)
@ -26,7 +27,7 @@ type HeaderSyncReceiptRepository struct{}
func (HeaderSyncReceiptRepository) CreateHeaderSyncReceiptInTx(headerID, transactionID int64, receipt core.Receipt, tx *sqlx.Tx) (int64, error) {
var receiptId int64
addressId, getAddressErr := AddressRepository{}.GetOrCreateAddressInTransaction(tx, receipt.ContractAddress)
addressId, getAddressErr := repository.GetOrCreateAddressInTransaction(tx, receipt.ContractAddress)
if getAddressErr != nil {
log.Error("createReceipt: Error getting address id: ", getAddressErr)
return receiptId, getAddressErr