retriever for generating list of all token holder addresses + updated transformer to use said addresses to populate balanceOf and allowance information and added database migrations for balance and allowance tables
This commit is contained in:
Ian Norden
@@ -59,8 +59,8 @@ var _ = Describe("Everyblock transformers", func() {
|
||||
})
|
||||
|
||||
It("creates a token_supply record for each block in the given range", func() {
|
||||
initializer := every_block.TokenSupplyTransformerInitializer{Config: erc20_watcher.DaiConfig}
|
||||
transformer := initializer.NewTokenSupplyTransformer(db, blockChain)
|
||||
initializer := every_block.ERC20TokenTransformerInitializer{Config: erc20_watcher.DaiConfig}
|
||||
transformer := initializer.NewERC20TokenTransformer(db, blockChain)
|
||||
transformer.Execute()
|
||||
|
||||
var tokenSupplyCount int
|
||||
|
||||
@@ -16,7 +16,9 @@ package every_block
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/vulcanize/vulcanizedb/examples/erc20_watcher"
|
||||
"github.com/vulcanize/vulcanizedb/examples/generic"
|
||||
"github.com/vulcanize/vulcanizedb/libraries/shared"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
@@ -27,6 +29,7 @@ import (
|
||||
type Transformer struct {
|
||||
Getter ERC20GetterInterface
|
||||
Repository ERC20RepositoryInterface
|
||||
Retriever generic.Retriever
|
||||
Config erc20_watcher.ContractConfig
|
||||
}
|
||||
|
||||
@@ -34,16 +37,18 @@ func (t *Transformer) SetConfiguration(config erc20_watcher.ContractConfig) {
|
||||
t.Config = config
|
||||
}
|
||||
|
||||
type TokenSupplyTransformerInitializer struct {
|
||||
type ERC20TokenTransformerInitializer struct {
|
||||
Config erc20_watcher.ContractConfig
|
||||
}
|
||||
|
||||
func (i TokenSupplyTransformerInitializer) NewTokenSupplyTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer {
|
||||
getter := NewGetter(blockChain)
|
||||
func (i ERC20TokenTransformerInitializer) NewERC20TokenTransformer(db *postgres.DB, blockchain core.BlockChain) shared.Transformer {
|
||||
getter := NewGetter(blockchain)
|
||||
repository := ERC20TokenRepository{DB: db}
|
||||
retriever := generic.NewRetriever(db, i.Config.Address)
|
||||
transformer := Transformer{
|
||||
Getter: &getter,
|
||||
Repository: &repository,
|
||||
Retriever: retriever,
|
||||
Config: i.Config,
|
||||
}
|
||||
|
||||
@@ -51,9 +56,14 @@ func (i TokenSupplyTransformerInitializer) NewTokenSupplyTransformer(db *postgre
|
||||
}
|
||||
|
||||
const (
|
||||
FetchingBlocksError = "Error getting missing blocks starting at block number %d: %s"
|
||||
GetSupplyError = "Error getting supply for block %d: %s"
|
||||
CreateSupplyError = "Error inserting token_supply for block %d: %s"
|
||||
FetchingBlocksError = "Error fetching missing blocks starting at block number %d: %s"
|
||||
FetchingSupplyError = "Error fetching supply for block %d: %s"
|
||||
CreateSupplyError = "Error inserting token_supply for block %d: %s"
|
||||
FetchingTokenAddressesError = "Error fetching token holder addresses at block %d: %s"
|
||||
FetchingBalanceError = "Error fetching balance at block %d: %s"
|
||||
CreateBalanceError = "Error inserting token_balance at block %d: %s"
|
||||
FetchingAllowanceError = "Error fetching allowance at block %d: %s"
|
||||
CreateAllowanceError = "Error inserting allowance at block %d: %s"
|
||||
)
|
||||
|
||||
type transformerError struct {
|
||||
@@ -74,8 +84,8 @@ func newTransformerError(err error, blockNumber int64, msg string) error {
|
||||
|
||||
func (t Transformer) Execute() error {
|
||||
var upperBoundBlock int64
|
||||
blockChain := t.Getter.GetBlockChain()
|
||||
lastBlock := blockChain.LastBlock().Int64()
|
||||
blockchain := t.Getter.GetBlockChain()
|
||||
lastBlock := blockchain.LastBlock().Int64()
|
||||
|
||||
if t.Config.LastBlock == -1 {
|
||||
upperBoundBlock = lastBlock
|
||||
@@ -93,14 +103,14 @@ func (t Transformer) Execute() error {
|
||||
}
|
||||
|
||||
// Fetch supply for missing blocks
|
||||
log.Printf("Gets totalSupply for %d blocks", len(blocks))
|
||||
log.Printf("Fetching totalSupply for %d blocks", len(blocks))
|
||||
|
||||
// For each block missing total supply, create supply model and feed the missing data into the repository
|
||||
for _, blockNumber := range blocks {
|
||||
totalSupply, err := t.Getter.GetTotalSupply(t.Config.Abi, t.Config.Address, blockNumber)
|
||||
|
||||
if err != nil {
|
||||
return newTransformerError(err, blockNumber, GetSupplyError)
|
||||
return newTransformerError(err, blockNumber, FetchingSupplyError)
|
||||
}
|
||||
// Create the supply model
|
||||
model := createTokenSupplyModel(totalSupply, t.Config.Address, blockNumber)
|
||||
@@ -112,6 +122,93 @@ func (t Transformer) Execute() error {
|
||||
}
|
||||
}
|
||||
|
||||
// Balance and allowance transformations:
|
||||
|
||||
// Retrieve all token holder addresses for the given contract configuration
|
||||
|
||||
tokenHolderAddresses, err := t.Retriever.RetrieveContractAssociatedAddresses()
|
||||
if err != nil {
|
||||
return newTransformerError(err, t.Config.FirstBlock, FetchingTokenAddressesError)
|
||||
}
|
||||
|
||||
// Iterate over the addresses and add their balances and allowances at each block height to the repository
|
||||
for holderAddr := range tokenHolderAddresses {
|
||||
|
||||
// Balance transformations:
|
||||
|
||||
blocks, err := t.Repository.MissingBalanceBlocks(t.Config.FirstBlock, upperBoundBlock, t.Config.Address, holderAddr.String())
|
||||
|
||||
if err != nil {
|
||||
return newTransformerError(err, t.Config.FirstBlock, FetchingBlocksError)
|
||||
}
|
||||
|
||||
log.Printf("Fetching balances for %d blocks", len(blocks))
|
||||
|
||||
// For each block missing balances for the given address, create a balance model and feed the missing data into the repository
|
||||
for _, blockNumber := range blocks {
|
||||
|
||||
hashArgs := []common.Address{holderAddr}
|
||||
balanceOfArgs := make([]interface{}, len(hashArgs))
|
||||
for i, s := range hashArgs {
|
||||
balanceOfArgs[i] = s
|
||||
}
|
||||
|
||||
totalSupply, err := t.Getter.GetBalance(t.Config.Abi, t.Config.Address, blockNumber, balanceOfArgs)
|
||||
|
||||
if err != nil {
|
||||
return newTransformerError(err, blockNumber, FetchingBalanceError)
|
||||
}
|
||||
|
||||
model := createTokenBalanceModel(totalSupply, t.Config.Address, blockNumber, holderAddr.String())
|
||||
|
||||
err = t.Repository.CreateBalance(model)
|
||||
|
||||
if err != nil {
|
||||
return newTransformerError(err, blockNumber, CreateBalanceError)
|
||||
}
|
||||
}
|
||||
|
||||
// Allowance transformations:
|
||||
|
||||
for spenderAddr := range tokenHolderAddresses {
|
||||
|
||||
blocks, err := t.Repository.MissingAllowanceBlocks(t.Config.FirstBlock, upperBoundBlock, t.Config.Address, holderAddr.String(), spenderAddr.String())
|
||||
|
||||
if err != nil {
|
||||
return newTransformerError(err, t.Config.FirstBlock, FetchingBlocksError)
|
||||
}
|
||||
|
||||
log.Printf("Fetching allowances for %d blocks", len(blocks))
|
||||
|
||||
// For each block missing allowances for the given holder and spender addresses, create a allowance model and feed the missing data into the repository
|
||||
for _, blockNumber := range blocks {
|
||||
|
||||
hashArgs := []common.Address{holderAddr, spenderAddr}
|
||||
allowanceArgs := make([]interface{}, len(hashArgs))
|
||||
for i, s := range hashArgs {
|
||||
allowanceArgs[i] = s
|
||||
}
|
||||
|
||||
totalSupply, err := t.Getter.GetAllowance(t.Config.Abi, t.Config.Address, blockNumber, allowanceArgs)
|
||||
|
||||
if err != nil {
|
||||
return newTransformerError(err, blockNumber, FetchingAllowanceError)
|
||||
}
|
||||
|
||||
model := createTokenAllowanceModel(totalSupply, t.Config.Address, blockNumber, holderAddr.String(), spenderAddr.String())
|
||||
|
||||
err = t.Repository.CreateAllowance(model)
|
||||
|
||||
if err != nil {
|
||||
return newTransformerError(err, blockNumber, CreateAllowanceError)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -122,3 +219,22 @@ func createTokenSupplyModel(totalSupply big.Int, address string, blockNumber int
|
||||
BlockNumber: blockNumber,
|
||||
}
|
||||
}
|
||||
|
||||
func createTokenBalanceModel(tokenBalance big.Int, tokenAddress string, blockNumber int64, tokenHolderAddress string) TokenBalance {
|
||||
return TokenBalance{
|
||||
Value: tokenBalance.String(),
|
||||
TokenAddress: tokenAddress,
|
||||
BlockNumber: blockNumber,
|
||||
TokenHolderAddress: tokenHolderAddress,
|
||||
}
|
||||
}
|
||||
|
||||
func createTokenAllowanceModel(tokenBalance big.Int, tokenAddress string, blockNumber int64, tokenHolderAddress, tokenSpenderAddress string) TokenAllowance {
|
||||
return TokenAllowance{
|
||||
Value: tokenBalance.String(),
|
||||
TokenAddress: tokenAddress,
|
||||
BlockNumber: blockNumber,
|
||||
TokenHolderAddress: tokenHolderAddress,
|
||||
TokenSpenderAddress: tokenSpenderAddress,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,7 +20,9 @@ import (
|
||||
"github.com/vulcanize/vulcanizedb/examples/constants"
|
||||
"github.com/vulcanize/vulcanizedb/examples/erc20_watcher"
|
||||
"github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block"
|
||||
"github.com/vulcanize/vulcanizedb/examples/generic"
|
||||
"github.com/vulcanize/vulcanizedb/examples/mocks"
|
||||
"github.com/vulcanize/vulcanizedb/examples/test_helpers"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/fakes"
|
||||
"math/big"
|
||||
"math/rand"
|
||||
@@ -55,11 +57,15 @@ var _ = Describe("Everyblock transformer", func() {
|
||||
getter.Fetcher.SetSupply(initialSupply)
|
||||
repository = mocks.ERC20TokenRepository{}
|
||||
repository.SetMissingSupplyBlocks([]int64{config.FirstBlock})
|
||||
db := test_helpers.CreateNewDatabase()
|
||||
rt := generic.NewRetriever(db, config.Address)
|
||||
//setting the mock repository to return the first block as the missing blocks
|
||||
|
||||
transformer = every_block.Transformer{
|
||||
Getter: &getter,
|
||||
Repository: &repository,
|
||||
Retriever: rt,
|
||||
Config: config,
|
||||
}
|
||||
transformer.SetConfiguration(config)
|
||||
})
|
||||
@@ -152,7 +158,7 @@ var _ = Describe("Everyblock transformer", func() {
|
||||
err := transformer.Execute()
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring(fakes.FakeError.Error()))
|
||||
Expect(err.Error()).To(ContainSubstring("getting missing blocks"))
|
||||
Expect(err.Error()).To(ContainSubstring("fetching missing blocks"))
|
||||
})
|
||||
|
||||
It("returns an error if the call to the blockChain fails", func() {
|
||||
|
||||
@@ -21,8 +21,8 @@ import (
|
||||
|
||||
func TransformerInitializers() []shared.TransformerInitializer {
|
||||
config := erc20_watcher.DaiConfig
|
||||
initializer := TokenSupplyTransformerInitializer{config}
|
||||
initializer := ERC20TokenTransformerInitializer{config}
|
||||
return []shared.TransformerInitializer{
|
||||
initializer.NewTokenSupplyTransformer,
|
||||
initializer.NewERC20TokenTransformer,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
// Copyright 2018 Vulcanize
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package generic
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
"log"
|
||||
)
|
||||
|
||||
// Retriever is used to iterate over addresses going into or out of a contract
|
||||
// address in an attempt to generate a list of token holder addresses
|
||||
|
||||
type RetrieverInterface interface {
|
||||
RetrieveSendingAddresses() ([]string, error)
|
||||
RetrieveReceivingAddresses() ([]string, error)
|
||||
RetrieveContractAssociatedAddresses() (map[common.Address]bool, error)
|
||||
}
|
||||
|
||||
type Retriever struct {
|
||||
Database *postgres.DB
|
||||
ContractAddress string
|
||||
}
|
||||
|
||||
type retrieverError struct {
|
||||
err string
|
||||
msg string
|
||||
address string
|
||||
}
|
||||
|
||||
// Retriever error method
|
||||
func (re *retrieverError) Error() string {
|
||||
return fmt.Sprintf(re.msg, re.address, re.err)
|
||||
}
|
||||
|
||||
// Used to create a new retriever error for a given error and fetch method
|
||||
func newRetrieverError(err error, msg string, address string) error {
|
||||
e := retrieverError{err.Error(), msg, address}
|
||||
log.Println(e.Error())
|
||||
return &e
|
||||
}
|
||||
|
||||
// Constant error definitions
|
||||
const (
|
||||
GetSenderError = "Error fetching addresses receiving from contract %s: %s"
|
||||
GetReceiverError = "Error fetching addresses sending to contract %s: %s"
|
||||
)
|
||||
|
||||
func NewRetriever(db *postgres.DB, address string) Retriever {
|
||||
return Retriever{
|
||||
Database: db,
|
||||
ContractAddress: address,
|
||||
}
|
||||
}
|
||||
|
||||
func (rt Retriever) RetrieveReceivingAddresses() ([]string, error) {
|
||||
|
||||
receiversFromContract := make([]string, 0)
|
||||
|
||||
err := rt.Database.DB.Select(
|
||||
&receiversFromContract,
|
||||
`SELECT tx_to FROM TRANSACTIONS
|
||||
WHERE tx_from = $1
|
||||
LIMIT 20`,
|
||||
rt.ContractAddress,
|
||||
)
|
||||
if err != nil {
|
||||
return []string{}, newRetrieverError(err, GetReceiverError, rt.ContractAddress)
|
||||
}
|
||||
return receiversFromContract, err
|
||||
}
|
||||
|
||||
func (rt Retriever) RetrieveSendingAddresses() ([]string, error) {
|
||||
|
||||
sendersToContract := make([]string, 0)
|
||||
|
||||
err := rt.Database.DB.Select(
|
||||
&sendersToContract,
|
||||
`SELECT tx_from FROM TRANSACTIONS
|
||||
WHERE tx_to = $1
|
||||
LIMIT 20`,
|
||||
rt.ContractAddress,
|
||||
)
|
||||
if err != nil {
|
||||
return []string{}, newRetrieverError(err, GetSenderError, rt.ContractAddress)
|
||||
}
|
||||
return sendersToContract, err
|
||||
}
|
||||
|
||||
func (rt Retriever) RetrieveContractAssociatedAddresses() (map[common.Address]bool, error) {
|
||||
|
||||
sending, err := rt.RetrieveSendingAddresses()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
receiving, err := rt.RetrieveReceivingAddresses()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
contractAddresses := make(map[common.Address]bool)
|
||||
|
||||
for _, addr := range sending {
|
||||
contractAddresses[common.HexToAddress(addr)] = true
|
||||
}
|
||||
|
||||
for _, addr := range receiving {
|
||||
contractAddresses[common.HexToAddress(addr)] = true
|
||||
}
|
||||
|
||||
return contractAddresses, nil
|
||||
}
|
||||
Reference in New Issue
Block a user