deep-stack/ipld-eth-server
6
0
Fork 0
forked from cerc-io/ipld-eth-server
Code Pull Requests Activity

Merge pull request #92 from vulcanize/vdb-560-rename-lightsync-again

Browse Source 
(VDB-560) Rename lightSync to headerSync
This commit is contained in:
Gabe Laughlin 2019-05-01 12:20:33 -05:00 committed by GitHub
commit c93e6adb27
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
48 changed files with 391 additions and 391 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -102,7 +102,7 @@ Usage is broken up into two processes:
### Data syncing
To provide data for transformations, raw Ethereum data must first be synced into vDB.
This is accomplished through the use of the `lightSync`, `sync`, or `coldImport` commands.
This is accomplished through the use of the `headerSync`, `sync`, or `coldImport` commands.
These commands are described in detail [here](../staging/documentation/sync.md).
### Data transformation

6
cmd/contractWatcher.go
View File

@ -26,7 +26,7 @@ import (
st "github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
ft "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/transformer"
lt "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/transformer"
lt "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/transformer"
"github.com/vulcanize/vulcanizedb/utils"
)
@ -98,7 +98,7 @@ func contractWatcher() {
con := config.ContractConfig{}
con.PrepConfig()
switch mode {
case "light":
case "header":
t = lt.NewTransformer(con, blockChain, &db)
case "full":
t = ft.NewTransformer(con, blockChain, &db)
@ -121,5 +121,5 @@ func contractWatcher() {
func init() {
rootCmd.AddCommand(contractWatcherCmd)
contractWatcherCmd.Flags().StringVarP(&mode, "mode", "o", "light", "'light' or 'full' mode to work with either light synced or fully synced vDB (default is light)")
contractWatcherCmd.Flags().StringVarP(&mode, "mode", "o", "header", "'header' or 'full' mode to work with either header synced or fully synced vDB (default is header)")
}

18
cmd/lightSync.go → cmd/headerSync.go
View File

@ -30,14 +30,14 @@ import (
"github.com/vulcanize/vulcanizedb/utils"
)
// lightSyncCmd represents the lightSync command
var lightSyncCmd = &cobra.Command{
Use: "lightSync",
// headerSyncCmd represents the headerSync command
var headerSyncCmd = &cobra.Command{
Use: "headerSync",
Short: "Syncs VulcanizeDB with local ethereum node's block headers",
Long: `Syncs VulcanizeDB with local ethereum node. Populates
Postgres with block headers.
./vulcanizedb lightSync --starting-block-number 0 --config public.toml
./vulcanizedb headerSync --starting-block-number 0 --config public.toml
Expects ethereum node to be running and requires a .toml config:
@ -50,13 +50,13 @@ Expects ethereum node to be running and requires a .toml config:
ipcPath = "/Users/user/Library/Ethereum/geth.ipc"
`,
Run: func(cmd *cobra.Command, args []string) {
lightSync()
headerSync()
},
}
func init() {
rootCmd.AddCommand(lightSyncCmd)
lightSyncCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block number to start syncing from")
rootCmd.AddCommand(headerSyncCmd)
headerSyncCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block number to start syncing from")
}
func backFillAllHeaders(blockchain core.BlockChain, headerRepository datastore.HeaderRepository, missingBlocksPopulated chan int, startingBlockNumber int64) {
@ -69,7 +69,7 @@ func backFillAllHeaders(blockchain core.BlockChain, headerRepository datastore.H
missingBlocksPopulated <- populated
}
func lightSync() {
func headerSync() {
ticker := time.NewTicker(pollingInterval)
defer ticker.Stop()
blockChain := getBlockChain()
@ -86,7 +86,7 @@ func lightSync() {
case <-ticker.C:
window, err := validator.ValidateHeaders()
if err != nil {
log.Error("lightSync: ValidateHeaders failed: ", err)
log.Error("headerSync: ValidateHeaders failed: ", err)
}
log.Info(window.GetString())
case n := <-missingBlocksPopulated:

4
db/migrations/00025_create_light_sync_transactions_table.sql → db/migrations/00025_create_header_sync_transactions_table.sql
View File

@ -1,5 +1,5 @@
-- +goose Up
CREATE TABLE light_sync_transactions (
CREATE TABLE header_sync_transactions (
id SERIAL PRIMARY KEY,
header_id INTEGER NOT NULL REFERENCES headers(id) ON DELETE CASCADE,
hash VARCHAR(66),
@ -16,4 +16,4 @@ CREATE TABLE light_sync_transactions (
);
-- +goose Down
DROP TABLE light_sync_transactions;
DROP TABLE header_sync_transactions;

6
db/migrations/00026_create_light_sync_receipts_table.sql → db/migrations/00026_create_header_sync_receipts_table.sql
View File

@ -1,7 +1,7 @@
-- +goose Up
CREATE TABLE light_sync_receipts(
CREATE TABLE header_sync_receipts(
id SERIAL PRIMARY KEY,
transaction_id INTEGER NOT NULL REFERENCES light_sync_transactions(id) ON DELETE CASCADE,
transaction_id INTEGER NOT NULL REFERENCES header_sync_transactions(id) ON DELETE CASCADE,
header_id INTEGER NOT NULL REFERENCES headers(id) ON DELETE CASCADE,
contract_address VARCHAR(42),
cumulative_gas_used NUMERIC,
@ -15,4 +15,4 @@ CREATE TABLE light_sync_receipts(
-- +goose Down
DROP TABLE light_sync_receipts;
DROP TABLE header_sync_receipts;

296
db/schema.sql
View File

@ -259,6 +259,84 @@ CREATE SEQUENCE public.goose_db_version_id_seq
ALTER SEQUENCE public.goose_db_version_id_seq OWNED BY public.goose_db_version.id;
--
-- Name: header_sync_receipts; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.header_sync_receipts (
id integer NOT NULL,
transaction_id integer NOT NULL,
header_id integer NOT NULL,
contract_address character varying(42),
cumulative_gas_used numeric,
gas_used numeric,
state_root character varying(66),
status integer,
tx_hash character varying(66),
rlp bytea
);
--
-- Name: header_sync_receipts_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.header_sync_receipts_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: header_sync_receipts_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.header_sync_receipts_id_seq OWNED BY public.header_sync_receipts.id;
--
-- Name: header_sync_transactions; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.header_sync_transactions (
id integer NOT NULL,
header_id integer NOT NULL,
hash character varying(66),
gas_limit numeric,
gas_price numeric,
input_data bytea,
nonce numeric,
raw bytea,
tx_from character varying(44),
tx_index integer,
tx_to character varying(44),
value numeric
);
--
-- Name: header_sync_transactions_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.header_sync_transactions_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: header_sync_transactions_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.header_sync_transactions_id_seq OWNED BY public.header_sync_transactions.id;
--
-- Name: headers; Type: TABLE; Schema: public; Owner: -
--
@ -294,84 +372,6 @@ CREATE SEQUENCE public.headers_id_seq
ALTER SEQUENCE public.headers_id_seq OWNED BY public.headers.id;
--
-- Name: light_sync_receipts; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.light_sync_receipts (
id integer NOT NULL,
transaction_id integer NOT NULL,
header_id integer NOT NULL,
contract_address character varying(42),
cumulative_gas_used numeric,
gas_used numeric,
state_root character varying(66),
status integer,
tx_hash character varying(66),
rlp bytea
);
--
-- Name: light_sync_receipts_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.light_sync_receipts_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: light_sync_receipts_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.light_sync_receipts_id_seq OWNED BY public.light_sync_receipts.id;
--
-- Name: light_sync_transactions; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.light_sync_transactions (
id integer NOT NULL,
header_id integer NOT NULL,
hash character varying(66),
gas_limit numeric,
gas_price numeric,
input_data bytea,
nonce numeric,
raw bytea,
tx_from character varying(44),
tx_index integer,
tx_to character varying(44),
value numeric
);
--
-- Name: light_sync_transactions_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.light_sync_transactions_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: light_sync_transactions_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.light_sync_transactions_id_seq OWNED BY public.light_sync_transactions.id;
--
-- Name: log_filters; Type: TABLE; Schema: public; Owner: -
--
@ -619,6 +619,20 @@ ALTER TABLE ONLY public.full_sync_transactions ALTER COLUMN id SET DEFAULT nextv
ALTER TABLE ONLY public.goose_db_version ALTER COLUMN id SET DEFAULT nextval('public.goose_db_version_id_seq'::regclass);
--
-- Name: header_sync_receipts id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.header_sync_receipts ALTER COLUMN id SET DEFAULT nextval('public.header_sync_receipts_id_seq'::regclass);
--
-- Name: header_sync_transactions id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.header_sync_transactions ALTER COLUMN id SET DEFAULT nextval('public.header_sync_transactions_id_seq'::regclass);
--
-- Name: headers id; Type: DEFAULT; Schema: public; Owner: -
--
@ -626,20 +640,6 @@ ALTER TABLE ONLY public.goose_db_version ALTER COLUMN id SET DEFAULT nextval('pu
ALTER TABLE ONLY public.headers ALTER COLUMN id SET DEFAULT nextval('public.headers_id_seq'::regclass);
--
-- Name: light_sync_receipts id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.light_sync_receipts ALTER COLUMN id SET DEFAULT nextval('public.light_sync_receipts_id_seq'::regclass);
--
-- Name: light_sync_transactions id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.light_sync_transactions ALTER COLUMN id SET DEFAULT nextval('public.light_sync_transactions_id_seq'::regclass);
--
-- Name: log_filters id; Type: DEFAULT; Schema: public; Owner: -
--
@ -739,6 +739,38 @@ ALTER TABLE ONLY public.goose_db_version
ADD CONSTRAINT goose_db_version_pkey PRIMARY KEY (id);
--
-- Name: header_sync_receipts header_sync_receipts_header_id_transaction_id_key; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.header_sync_receipts
ADD CONSTRAINT header_sync_receipts_header_id_transaction_id_key UNIQUE (header_id, transaction_id);
--
-- Name: header_sync_receipts header_sync_receipts_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.header_sync_receipts
ADD CONSTRAINT header_sync_receipts_pkey PRIMARY KEY (id);
--
-- Name: header_sync_transactions header_sync_transactions_header_id_hash_key; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.header_sync_transactions
ADD CONSTRAINT header_sync_transactions_header_id_hash_key UNIQUE (header_id, hash);
--
-- Name: header_sync_transactions header_sync_transactions_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.header_sync_transactions
ADD CONSTRAINT header_sync_transactions_pkey PRIMARY KEY (id);
--
-- Name: headers headers_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
@ -747,38 +779,6 @@ ALTER TABLE ONLY public.headers
ADD CONSTRAINT headers_pkey PRIMARY KEY (id);
--
-- Name: light_sync_receipts light_sync_receipts_header_id_transaction_id_key; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.light_sync_receipts
ADD CONSTRAINT light_sync_receipts_header_id_transaction_id_key UNIQUE (header_id, transaction_id);
--
-- Name: light_sync_receipts light_sync_receipts_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.light_sync_receipts
ADD CONSTRAINT light_sync_receipts_pkey PRIMARY KEY (id);
--
-- Name: light_sync_transactions light_sync_transactions_header_id_hash_key; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.light_sync_transactions
ADD CONSTRAINT light_sync_transactions_header_id_hash_key UNIQUE (header_id, hash);
--
-- Name: light_sync_transactions light_sync_transactions_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.light_sync_transactions
ADD CONSTRAINT light_sync_transactions_pkey PRIMARY KEY (id);
--
-- Name: logs logs_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
@ -909,6 +909,30 @@ ALTER TABLE ONLY public.full_sync_transactions
ADD CONSTRAINT full_sync_transactions_block_id_fkey FOREIGN KEY (block_id) REFERENCES public.blocks(id) ON DELETE CASCADE;
--
-- Name: header_sync_receipts header_sync_receipts_header_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.header_sync_receipts
ADD CONSTRAINT header_sync_receipts_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE;
--
-- Name: header_sync_receipts header_sync_receipts_transaction_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.header_sync_receipts
ADD CONSTRAINT header_sync_receipts_transaction_id_fkey FOREIGN KEY (transaction_id) REFERENCES public.header_sync_transactions(id) ON DELETE CASCADE;
--
-- Name: header_sync_transactions header_sync_transactions_header_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.header_sync_transactions
ADD CONSTRAINT header_sync_transactions_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE;
--
-- Name: headers headers_eth_node_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
--
@ -917,30 +941,6 @@ ALTER TABLE ONLY public.headers
ADD CONSTRAINT headers_eth_node_id_fkey FOREIGN KEY (eth_node_id) REFERENCES public.eth_nodes(id) ON DELETE CASCADE;
--
-- Name: light_sync_receipts light_sync_receipts_header_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.light_sync_receipts
ADD CONSTRAINT light_sync_receipts_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE;
--
-- Name: light_sync_receipts light_sync_receipts_transaction_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.light_sync_receipts
ADD CONSTRAINT light_sync_receipts_transaction_id_fkey FOREIGN KEY (transaction_id) REFERENCES public.light_sync_transactions(id) ON DELETE CASCADE;
--
-- Name: light_sync_transactions light_sync_transactions_header_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.light_sync_transactions
ADD CONSTRAINT light_sync_transactions_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE;
--
-- Name: blocks node_fk; Type: FK CONSTRAINT; Schema: public; Owner: -
--

6
dockerfiles/README.md
View File

@ -2,7 +2,7 @@ S
`Dockerfile` will build an alpine image containing:
- vDB as a binary with runtime deps statically linked: `/app/vulcanizedb`
- The migration tool goose: `/app/goose`
- Two services for running `lightSync` and `continuousLogSync`, started with the default configuration `environments/staging.toml`.
- Two services for running `headerSync` and `continuousLogSync`, started with the default configuration `environments/staging.toml`.
By default, vDB is configured towards the Kovan deploy. The configuration values can be overridden using environment variables, using the same hierarchical naming pattern but in CAPS and using underscores. For example, the contract address for the `Pit` can be set with the variable `CONTRACT_ADDRESS_PIT="0x123..."`.
@ -18,8 +18,8 @@ e`
* `CONTRACT_ADDRESS_[CONTRACT NAME]=0x123...`
* `CONTRACT_ABI_[CONTRACT NAME]="ABI STRING"`
* `CONTRACT_DEPLOYMENT-BLOCK_[CONTRACT NAME]=0` (doesn't really matter on a short chain, just avoids long unnecessary searching)
6. Start the `lightSync` and `continuousLogSync` services:
* `./vulcanizedb lightSync --config environments/staging.toml`
6. Start the `headerSync` and `continuousLogSync` services:
* `./vulcanizedb headerSync --config environments/staging.toml`
* `./vulcanizedb continuousLogSync --config environments/staging.toml`
### Automated

4
dockerfiles/startup_script.sh
View File

@ -1,5 +1,5 @@
#!/bin/sh
# Runs the migrations and starts the lightSync and continuousLogSync services
# Runs the migrations and starts the headerSync and continuousLogSync services
# Exit if the variable tests fail
set -e
@ -21,7 +21,7 @@ set +e
./goose postgres "$CONNECT_STRING" up
if [ $? -eq 0 ]; then
# Fire up the services
./vulcanizedb lightSync --config environments/staging.toml &
./vulcanizedb headerSync --config environments/staging.toml &
./vulcanizedb continuousLogSync --config environments/staging.toml &
else
echo "Could not run migrations. Are the database details correct?"

2
documentation/composeAndExecute.md
View File

@ -101,7 +101,7 @@ The config provides information for composing a set of transformers from externa
that fetches event logs from an ETH node
- `eth_contract` indicates the transformer works with the [contract watcher](../staging/libraries/shared/watcher/contract_watcher.go)
that is made to work with [contract_watcher pkg](../../staging/pkg/contract_watcher)
based transformers which work with either a light or full sync vDB to watch events and poll public methods ([example1](https://github.com/vulcanize/account_transformers/tree/master/transformers/account/light), [example2](https://github.com/vulcanize/ens_transformers/tree/working/transformers/domain_records))
based transformers which work with either a header or full sync vDB to watch events and poll public methods ([example1](https://github.com/vulcanize/account_transformers/tree/master/transformers/account/light), [example2](https://github.com/vulcanize/ens_transformers/tree/working/transformers/domain_records))
- `migrations` is the relative path from `repository` to the db migrations directory for the transformer
- `rank` determines the order that migrations are ran, with lower ranked migrations running first
- this is to help isolate any potential conflicts between transformer migrations

36
documentation/contractWatcher.md
View File

@ -5,11 +5,11 @@ It also provides some state variable coverage by automating polling of public me
1. The method's arguments must all be of type address or bytes32 (hash)
1. The method must return a single value
This command operates in two modes- `light` and `full`- which require a light or full-synced vulcanizeDB, respectively.
This command operates in two modes- `header` and `full`- which require a header or full-synced vulcanizeDB, respectively.
This command requires the contract ABI be available on Etherscan if it is not provided in the config file by the user.
If method polling is turned on we require an archival node at the ETH ipc endpoint in our config, whether or not we are operating in `light` or `full` mode.
If method polling is turned on we require an archival node at the ETH ipc endpoint in our config, whether or not we are operating in `header` or `full` mode.
Otherwise we only need to connect to a full node.
## Configuration
@ -97,16 +97,16 @@ Modify `./environments/example.toml` to replace the empty `ipcPath` with a path
This endpoint should be for an archival eth node if we want to perform method polling as this configuration is currently set up to do. To work with a non-archival full node,
remove the `balanceOf` method from the `0x8dd5fbce2f6a956c3022ba3663759011dd51e73e` (TrueUSD) contract.
If you are operating a light sync vDB, run:
If you are operating a header sync vDB, run:
`./vulcanizedb contractWatcher --config=./environments/example.toml --mode=light`
`./vulcanizedb contractWatcher --config=./environments/example.toml --mode=header`
If instead you are operating a full sync vDB and provided an archival node IPC path, run in full mode:
`./vulcanizedb contractWatcher --config=./environments/example.toml --mode=full`
This will run the contractWatcher and configures it to watch the contracts specified in the config file. Note that
by default we operate in `light` mode but the flag is included here to demonstrate its use.
by default we operate in `header` mode but the flag is included here to demonstrate its use.
The example config we link to in this example watches two contracts, the ENS Registry (0x314159265dD8dbb310642f98f50C066173C1259b) and TrueUSD (0x8dd5fbCe2F6a956C3022bA3663759011Dd51e73E).
@ -117,24 +117,24 @@ The TrueUSD contract is configured with two events (`Transfer` and `Mint`) and a
to poll the `balanceOf` method with those addresses at every block. Note that we do not provide an ABI for TrueUSD as its ABI can be fetched from Etherscan.
For the ENS contract, it produces and populates a schema with four tables"
`light_0x314159265dd8dbb310642f98f50c066173c1259b.newowner_event`
`light_0x314159265dd8dbb310642f98f50c066173c1259b.newresolver_event`
`light_0x314159265dd8dbb310642f98f50c066173c1259b.newttl_event`
`light_0x314159265dd8dbb310642f98f50c066173c1259b.transfer_event`
`header_0x314159265dd8dbb310642f98f50c066173c1259b.newowner_event`
`header_0x314159265dd8dbb310642f98f50c066173c1259b.newresolver_event`
`header_0x314159265dd8dbb310642f98f50c066173c1259b.newttl_event`
`header_0x314159265dd8dbb310642f98f50c066173c1259b.transfer_event`
For the TrusUSD contract, it produces and populates a schema with three tables:
`light_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.transfer_event`
`light_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.mint_event`
`light_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.balanceof_method`
`header_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.transfer_event`
`header_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.mint_event`
`header_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.balanceof_method`
Column ids and types for these tables are generated based on the event and method argument names and types and method return types, resulting in tables such as:
Table "light_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.transfer_event"
Table "header_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.transfer_event"
| Column | Type | Collation | Nullable | Default | Storage | Stats target | Description
|:----------:|:---------------------:|:---------:|:--------:|:-------------------------------------------------------------------------------------------:|:--------:|:------------:|:-----------:|
| id | integer | | not null | nextval('light_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.transfer_event_id_seq'::regclass) | plain | | |
|:----------:|:---------------------:|:---------:|:--------:|:--------------------------------------------------------------------------------------------:|:--------:|:------------:|:-----------:|
| id | integer | | not null | nextval('header_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.transfer_event_id_seq'::regclass) | plain | | |
| header_id | integer | | not null | | plain | | |
| token_name | character varying(66) | | not null | | extended | | |
| raw_log | jsonb | | | | extended | | |
@ -145,11 +145,11 @@ Table "light_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.transfer_event"
| value_ | numeric | | not null | | main | | |
Table "light_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.balanceof_method"
Table "header_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.balanceof_method"
| Column | Type | Collation | Nullable | Default | Storage | Stats target | Description |
|:----------:|:---------------------:|:---------:|:--------:|:-------------------------------------------------------------------------------------------:|:--------:|:------------:|:-----------:|
| id | integer | | not null | nextval('light_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.balanceof_method_id_seq'::regclass) | plain | | |
|:----------:|:---------------------:|:---------:|:--------:|:----------------------------------------------------------------------------------------------:|:--------:|:------------:|:-----------:|
| id | integer | | not null | nextval('header_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.balanceof_method_id_seq'::regclass) | plain | | |
| token_name | character varying(66) | | not null | | extended | | |
| block | integer | | not null | | plain | | |
| who_ | character varying(66) | | not null | | extended | | |

4
documentation/sync.md
View File

@ -1,12 +1,12 @@
# Syncing commands
These commands are used to sync raw Ethereum data into Postgres.
## lightSync
## headerSync
Syncs VulcanizeDB with the configured Ethereum node, populating only block headers.
This command is useful when you want a minimal baseline from which to track targeted data on the blockchain (e.g. individual smart contract storage values or event logs).
1. Start Ethereum node
1. In a separate terminal start VulcanizeDB:
- `./vulcanizedb lightSync --config <config.toml> --starting-block-number <block-number>`
- `./vulcanizedb headerSync --config <config.toml> --starting-block-number <block-number>`
## sync
Syncs VulcanizeDB with the configured Ethereum node, populating blocks, transactions, receipts, and logs.

2
integration_test/contract_watcher_full_transformer_test.go
View File

@ -286,7 +286,7 @@ var _ = Describe("contractWatcher full transformer", func() {
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
log := test_helpers.LightNewOwnerLog{}
log := test_helpers.HeaderSyncNewOwnerLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.newowner_event", ensAddr)).StructScan(&log)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("does not exist"))

48
integration_test/contract_watcher_light_transformer_test.go → integration_test/contract_watcher_header_sync_transformer_test.go
View File

@ -9,7 +9,7 @@ import (
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/transformer"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/transformer"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks"
@ -18,7 +18,7 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
)
var _ = Describe("contractWatcher light transformer", func() {
var _ = Describe("contractWatcher headerSync transformer", func() {
var db *postgres.DB
var err error
var blockChain core.BlockChain
@ -107,8 +107,8 @@ var _ = Describe("contractWatcher light transformer", func() {
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
log := test_helpers.LightTransferLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.transfer_event", tusdAddr)).StructScan(&log)
log := test_helpers.HeaderSyncTransferLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.transfer_event", tusdAddr)).StructScan(&log)
Expect(err).ToNot(HaveOccurred())
// We don't know vulcID, so compare individual fields instead of complete structures
Expect(log.HeaderID).To(Equal(headerID))
@ -175,12 +175,12 @@ var _ = Describe("contractWatcher light transformer", func() {
Expect(err).ToNot(HaveOccurred())
res := test_helpers.BalanceOf{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x1062a747393198f70F71ec65A582423Dba7E5Ab3' AND block = '6791669'", tusdAddr)).StructScan(&res)
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.balanceof_method WHERE who_ = '0x1062a747393198f70F71ec65A582423Dba7E5Ab3' AND block = '6791669'", tusdAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Balance).To(Equal("55849938025000000000000"))
Expect(res.TokenName).To(Equal("TrueUSD"))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6791669'", tusdAddr)).StructScan(&res)
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6791669'", tusdAddr)).StructScan(&res)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("no rows in result set"))
})
@ -215,8 +215,8 @@ var _ = Describe("contractWatcher light transformer", func() {
Expect(err).ToNot(HaveOccurred())
Expect(t.Start).To(Equal(int64(6885698)))
log := test_helpers.LightNewOwnerLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&log)
log := test_helpers.HeaderSyncNewOwnerLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.newowner_event", ensAddr)).StructScan(&log)
Expect(err).ToNot(HaveOccurred())
// We don't know vulcID, so compare individual fields instead of complete structures
Expect(log.HeaderID).To(Equal(headerID))
@ -267,17 +267,17 @@ var _ = Describe("contractWatcher light transformer", func() {
Expect(err).ToNot(HaveOccurred())
res := test_helpers.Owner{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&res)
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
Expect(res.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&res)
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
Expect(res.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6885696'", ensAddr)).StructScan(&res)
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6885696'", ensAddr)).StructScan(&res)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("no rows in result set"))
})
@ -294,8 +294,8 @@ var _ = Describe("contractWatcher light transformer", func() {
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
log := test_helpers.LightNewOwnerLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&log)
log := test_helpers.HeaderSyncNewOwnerLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.newowner_event", ensAddr)).StructScan(&log)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("does not exist"))
})
@ -316,12 +316,12 @@ var _ = Describe("contractWatcher light transformer", func() {
Expect(err).ToNot(HaveOccurred())
res := test_helpers.Owner{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&res)
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
Expect(res.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&res)
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&res)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("no rows in result set"))
})
@ -345,16 +345,16 @@ var _ = Describe("contractWatcher light transformer", func() {
Expect(err).ToNot(HaveOccurred())
Expect(t.Start).To(Equal(int64(6885702)))
newOwnerLog := test_helpers.LightNewOwnerLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&newOwnerLog)
newOwnerLog := test_helpers.HeaderSyncNewOwnerLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.newowner_event", ensAddr)).StructScan(&newOwnerLog)
Expect(err).ToNot(HaveOccurred())
// We don't know vulcID, so compare individual fields instead of complete structures
Expect(newOwnerLog.Node).To(Equal("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"))
Expect(newOwnerLog.Label).To(Equal("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047"))
Expect(newOwnerLog.Owner).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
transferLog := test_helpers.LightTransferLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.transfer_event", tusdAddr)).StructScan(&transferLog)
transferLog := test_helpers.HeaderSyncTransferLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.transfer_event", tusdAddr)).StructScan(&transferLog)
Expect(err).ToNot(HaveOccurred())
// We don't know vulcID, so compare individual fields instead of complete structures
Expect(transferLog.From).To(Equal("0x8cA465764873E71CEa525F5EB6AE973d650c22C2"))
@ -417,27 +417,27 @@ var _ = Describe("contractWatcher light transformer", func() {
Expect(err).ToNot(HaveOccurred())
owner := test_helpers.Owner{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&owner)
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&owner)
Expect(err).ToNot(HaveOccurred())
Expect(owner.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
Expect(owner.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&owner)
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&owner)
Expect(err).ToNot(HaveOccurred())
Expect(owner.Address).To(Equal("0x0000000000000000000000000000000000000000"))
Expect(owner.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ceMADEUPaaf4HASHc186badTHItransformers.8IS625bFAKE' AND block = '6885696'", ensAddr)).StructScan(&owner)
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ceMADEUPaaf4HASHc186badTHItransformers.8IS625bFAKE' AND block = '6885696'", ensAddr)).StructScan(&owner)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("no rows in result set"))
bal := test_helpers.BalanceOf{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x8cA465764873E71CEa525F5EB6AE973d650c22C2' AND block = '6885701'", tusdAddr)).StructScan(&bal)
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.balanceof_method WHERE who_ = '0x8cA465764873E71CEa525F5EB6AE973d650c22C2' AND block = '6885701'", tusdAddr)).StructScan(&bal)
Expect(err).ToNot(HaveOccurred())
Expect(bal.Balance).To(Equal("1954436000000000000000"))
Expect(bal.TokenName).To(Equal("TrueUSD"))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6885701'", tusdAddr)).StructScan(&bal)
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6885701'", tusdAddr)).StructScan(&bal)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("no rows in result set"))
})

24
integration_test/poller_test.go
View File

@ -130,10 +130,10 @@ var _ = Describe("Poller", func() {
})
})
Describe("Light sync mode", func() {
Describe("Header sync mode", func() {
BeforeEach(func() {
db, bc = test_helpers.SetupDBandBC()
contractPoller = poller.NewPoller(bc, db, types.LightSync)
contractPoller = poller.NewPoller(bc, db, types.HeaderSync)
})
Describe("PollContract", func() {
@ -148,22 +148,22 @@ var _ = Describe("Poller", func() {
scanStruct := test_helpers.BalanceOf{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0xfE9e8709d3215310075d67E3ed32A380CCf451C8' AND block = '6707322'", constants.TusdContractAddress)).StructScan(&scanStruct)
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.balanceof_method WHERE who_ = '0xfE9e8709d3215310075d67E3ed32A380CCf451C8' AND block = '6707322'", constants.TusdContractAddress)).StructScan(&scanStruct)
Expect(err).ToNot(HaveOccurred())
Expect(scanStruct.Balance).To(Equal("66386309548896882859581786"))
Expect(scanStruct.TokenName).To(Equal("TrueUSD"))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0xfE9e8709d3215310075d67E3ed32A380CCf451C8' AND block = '6707323'", constants.TusdContractAddress)).StructScan(&scanStruct)
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.balanceof_method WHERE who_ = '0xfE9e8709d3215310075d67E3ed32A380CCf451C8' AND block = '6707323'", constants.TusdContractAddress)).StructScan(&scanStruct)
Expect(err).ToNot(HaveOccurred())
Expect(scanStruct.Balance).To(Equal("66386309548896882859581786"))
Expect(scanStruct.TokenName).To(Equal("TrueUSD"))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x3f5CE5FBFe3E9af3971dD833D26bA9b5C936f0bE' AND block = '6707322'", constants.TusdContractAddress)).StructScan(&scanStruct)
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.balanceof_method WHERE who_ = '0x3f5CE5FBFe3E9af3971dD833D26bA9b5C936f0bE' AND block = '6707322'", constants.TusdContractAddress)).StructScan(&scanStruct)
Expect(err).ToNot(HaveOccurred())
Expect(scanStruct.Balance).To(Equal("17982350181394112023885864"))
Expect(scanStruct.TokenName).To(Equal("TrueUSD"))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x3f5CE5FBFe3E9af3971dD833D26bA9b5C936f0bE' AND block = '6707323'", constants.TusdContractAddress)).StructScan(&scanStruct)
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.balanceof_method WHERE who_ = '0x3f5CE5FBFe3E9af3971dD833D26bA9b5C936f0bE' AND block = '6707323'", constants.TusdContractAddress)).StructScan(&scanStruct)
Expect(err).ToNot(HaveOccurred())
Expect(scanStruct.Balance).To(Equal("17982350181394112023885864"))
Expect(scanStruct.TokenName).To(Equal("TrueUSD"))
@ -180,12 +180,12 @@ var _ = Describe("Poller", func() {
scanStruct := test_helpers.Owner{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x7e74a86b6e146964fb965db04dc2590516da77f720bb6759337bf5632415fd86' AND block = '6885877'", constants.EnsContractAddress)).StructScan(&scanStruct)
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.owner_method WHERE node_ = '0x7e74a86b6e146964fb965db04dc2590516da77f720bb6759337bf5632415fd86' AND block = '6885877'", constants.EnsContractAddress)).StructScan(&scanStruct)
Expect(err).ToNot(HaveOccurred())
Expect(scanStruct.Address).To(Equal("0x546aA2EaE2514494EeaDb7bbb35243348983C59d"))
Expect(scanStruct.TokenName).To(Equal("ENS-Registry"))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885877'", constants.EnsContractAddress)).StructScan(&scanStruct)
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885877'", constants.EnsContractAddress)).StructScan(&scanStruct)
Expect(err).ToNot(HaveOccurred())
Expect(scanStruct.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
Expect(scanStruct.TokenName).To(Equal("ENS-Registry"))
@ -202,7 +202,7 @@ var _ = Describe("Poller", func() {
scanStruct := test_helpers.BalanceOf{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0xfE9e8709d3215310075d67E3ed32A380CCf451C8' AND block = '6707322'", constants.TusdContractAddress)).StructScan(&scanStruct)
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.balanceof_method WHERE who_ = '0xfE9e8709d3215310075d67E3ed32A380CCf451C8' AND block = '6707322'", constants.TusdContractAddress)).StructScan(&scanStruct)
Expect(err).To(HaveOccurred())
})
@ -217,7 +217,7 @@ var _ = Describe("Poller", func() {
Expect(err).ToNot(HaveOccurred())
scanStruct := test_helpers.Resolver{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.resolver_method WHERE node_ = '0x495b6e6efdedb750aa519919b5cf282bdaa86067b82a2293a3ff5723527141e8' AND block = '6921967'", constants.EnsContractAddress)).StructScan(&scanStruct)
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.resolver_method WHERE node_ = '0x495b6e6efdedb750aa519919b5cf282bdaa86067b82a2293a3ff5723527141e8' AND block = '6921967'", constants.EnsContractAddress)).StructScan(&scanStruct)
Expect(err).ToNot(HaveOccurred())
Expect(scanStruct.Address).To(Equal("0x5FfC014343cd971B7eb70732021E26C35B744cc4"))
Expect(scanStruct.TokenName).To(Equal("ENS-Registry"))
@ -225,13 +225,13 @@ var _ = Describe("Poller", func() {
test_helpers.TearDown(db)
db, bc = test_helpers.SetupDBandBC()
contractPoller = poller.NewPoller(bc, db, types.LightSync)
contractPoller = poller.NewPoller(bc, db, types.HeaderSync)
con.Piping = true
err = contractPoller.PollContract(*con, 6921968)
Expect(err).ToNot(HaveOccurred())
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.resolver_method WHERE node_ = '0x495b6e6efdedb750aa519919b5cf282bdaa86067b82a2293a3ff5723527141e8' AND block = '6921967'", constants.EnsContractAddress)).StructScan(&scanStruct)
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.resolver_method WHERE node_ = '0x495b6e6efdedb750aa519919b5cf282bdaa86067b82a2293a3ff5723527141e8' AND block = '6921967'", constants.EnsContractAddress)).StructScan(&scanStruct)
Expect(err).ToNot(HaveOccurred())
Expect(scanStruct.Address).To(Equal("0x5FfC014343cd971B7eb70732021E26C35B744cc4"))
Expect(scanStruct.TokenName).To(Equal("ENS-Registry"))

6
libraries/shared/factories/event/README.md
View File

@ -2,13 +2,13 @@
One approach VulcanizeDB takes to caching and indexing smart contracts is to watch contract events emitted in receipt logs.
With a light synced vDB we can watch events by iterating over headers retrieved from the synced `headers` table and using these headers to
With a header synced vDB we can watch events by iterating over headers retrieved from the synced `headers` table and using these headers to
fetch and verify relevant event logs from a full Ethereum node, keeping track of which headers we have checked for which events
with our `checked_headers` table.
## Assumptions
This approach assumes you are running a vDB light sync which is itself run against a light Ethereum node,
This approach assumes you are running a vDB header sync which is run against a light Ethereum node;
this approach also assumes there is a full node available.
Looking forward, we will be building fetchers that enable sourcing data from IPFS instead of an ETH node.
@ -17,7 +17,7 @@ Looking forward, we will be building fetchers that enable sourcing data from IPF
VulcanizeDB has shared code built out for building and plugging in event transformers
### [Event Watcher (light sync)](../staging/libraries/shared/watcher/event_watcher.go)
### [Event Watcher (header sync)](../staging/libraries/shared/watcher/event_watcher.go)
The event watcher is responsible for continuously fetching and delegating chunks of logs and their associated header to the appropriate transformers.

2
libraries/shared/repository/repository_test.go
View File

@ -26,7 +26,7 @@ import (
"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
shared "github.com/vulcanize/vulcanizedb/libraries/shared/repository"
r2 "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/repository"
r2 "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/repository"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"

4
pkg/contract_watcher/full/transformer/transformer_test.go
View File

@ -39,7 +39,7 @@ var _ = Describe("Transformer", func() {
Describe("Init", func() {
It("Initializes transformer's contract objects", func() {
blockRetriever := &fakes.MockFullBlockRetriever{}
blockRetriever := &fakes.MockFullSyncBlockRetriever{}
firstBlock := int64(1)
mostRecentBlock := int64(2)
blockRetriever.FirstBlock = firstBlock
@ -74,7 +74,7 @@ var _ = Describe("Transformer", func() {
})
It("Fails to initialize if first and most recent blocks cannot be fetched from vDB", func() {
blockRetriever := &fakes.MockFullBlockRetriever{}
blockRetriever := &fakes.MockFullSyncBlockRetriever{}
blockRetriever.FirstBlockErr = fakes.FakeError
t := getTransformer(blockRetriever, &fakes.MockParser{}, &fakes.MockPoller{})

0
pkg/contract_watcher/light/converter/converter.go → pkg/contract_watcher/header/converter/converter.go
View File

2
pkg/contract_watcher/light/converter/converter_suite_test.go → pkg/contract_watcher/header/converter/converter_suite_test.go
View File

@ -27,7 +27,7 @@ import (
func TestConverter(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Light Converter Suite Test")
RunSpecs(t, "Header Sync Converter Suite Test")
}
var _ = BeforeSuite(func() {

2
pkg/contract_watcher/light/converter/converter_test.go → pkg/contract_watcher/header/converter/converter_test.go
View File

@ -22,7 +22,7 @@ import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/converter"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/converter"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers"

0
pkg/contract_watcher/light/fetcher/fetcher.go → pkg/contract_watcher/header/fetcher/fetcher.go
View File

0
pkg/contract_watcher/light/fetcher/fetcher_suite_test.go → pkg/contract_watcher/header/fetcher/fetcher_suite_test.go
View File

2
pkg/contract_watcher/light/fetcher/fetcher_test.go → pkg/contract_watcher/header/fetcher/fetcher_test.go
View File

@ -22,7 +22,7 @@ import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/fetcher"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/fetcher"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
)

0
pkg/contract_watcher/light/repository/header_repository.go → pkg/contract_watcher/header/repository/header_repository.go
View File

4
pkg/contract_watcher/light/repository/header_repository_test.go → pkg/contract_watcher/header/repository/header_repository_test.go
View File

@ -23,7 +23,7 @@ import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/repository"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/repository"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
@ -32,7 +32,7 @@ import (
var _ = Describe("Repository", func() {
var db *postgres.DB
var contractHeaderRepo repository.HeaderRepository // contract_watcher light header repository
var contractHeaderRepo repository.HeaderRepository // contract_watcher headerSync header repository
var coreHeaderRepo repositories.HeaderRepository // pkg/datastore header repository
var eventIDs = []string{
"eventName_contractAddr",

2
pkg/contract_watcher/light/repository/repository_suite_test.go → pkg/contract_watcher/header/repository/repository_suite_test.go
View File

@ -27,7 +27,7 @@ import (
func TestRepository(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Light Repository Suite Test")
RunSpecs(t, "Header Sync Repository Suite Test")
}
var _ = BeforeSuite(func() {

0
pkg/contract_watcher/light/retriever/block_retriever.go → pkg/contract_watcher/header/retriever/block_retriever.go
View File

2
pkg/contract_watcher/light/retriever/block_retriever_test.go → pkg/contract_watcher/header/retriever/block_retriever_test.go
View File

@ -20,7 +20,7 @@ import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/retriever"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/retriever"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"

2
pkg/contract_watcher/light/retriever/retriever_suite_test.go → pkg/contract_watcher/header/retriever/retriever_suite_test.go
View File

@ -27,7 +27,7 @@ import (
func TestRetriever(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Light Block Number Retriever Suite Test")
RunSpecs(t, "Header Sync Block Number Retriever Suite Test")
}
var _ = BeforeSuite(func() {

14
pkg/contract_watcher/light/transformer/transformer.go → pkg/contract_watcher/header/transformer/transformer.go
View File

@ -24,10 +24,10 @@ import (
gethTypes "github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/config"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/converter"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/fetcher"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/repository"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/retriever"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/converter"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/fetcher"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/repository"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/retriever"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/parser"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/poller"
@ -37,7 +37,7 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
// Requires a light synced vDB (headers) and a running eth node (or infura)
// Requires a header synced vDB (headers) and a running eth node (or infura)
type Transformer struct {
// Database interfaces
EventRepository srep.EventRepository // Holds transformed watched event log data
@ -77,14 +77,14 @@ type Transformer struct {
func NewTransformer(con config.ContractConfig, bc core.BlockChain, db *postgres.DB) *Transformer {
return &Transformer{
Poller: poller.NewPoller(bc, db, types.LightSync),
Poller: poller.NewPoller(bc, db, types.HeaderSync),
Fetcher: fetcher.NewFetcher(bc),
Parser: parser.NewParser(con.Network),
HeaderRepository: repository.NewHeaderRepository(db),
Retriever: retriever.NewBlockRetriever(db),
Converter: &converter.Converter{},
Contracts: map[string]*contract.Contract{},
EventRepository: srep.NewEventRepository(db, types.LightSync),
EventRepository: srep.NewEventRepository(db, types.HeaderSync),
Config: con,
}
}

2
pkg/contract_watcher/light/transformer/transformer_suite_test.go → pkg/contract_watcher/header/transformer/transformer_suite_test.go
View File

@ -27,7 +27,7 @@ import (
func TestTransformer(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Light Transformer Suite Test")
RunSpecs(t, "Header Sync Transformer Suite Test")
}
var _ = BeforeSuite(func() {

14
pkg/contract_watcher/light/transformer/transformer_test.go → pkg/contract_watcher/header/transformer/transformer_test.go
View File

@ -20,8 +20,8 @@ import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/retriever"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/transformer"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/retriever"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/transformer"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/parser"
@ -33,7 +33,7 @@ var _ = Describe("Transformer", func() {
var fakeAddress = "0x1234567890abcdef"
Describe("Init", func() {
It("Initializes transformer's contract objects", func() {
blockRetriever := &fakes.MockLightBlockRetriever{}
blockRetriever := &fakes.MockHeaderSyncBlockRetriever{}
firstBlock := int64(1)
blockRetriever.FirstBlock = firstBlock
@ -61,7 +61,7 @@ var _ = Describe("Transformer", func() {
})
It("Fails to initialize if first block cannot be fetched from vDB headers table", func() {
blockRetriever := &fakes.MockLightBlockRetriever{}
blockRetriever := &fakes.MockHeaderSyncBlockRetriever{}
blockRetriever.FirstBlockErr = fakes.FakeError
t := getFakeTransformer(blockRetriever, &fakes.MockParser{}, &fakes.MockPoller{})
@ -74,7 +74,7 @@ var _ = Describe("Transformer", func() {
Describe("Execute", func() {
It("Executes contract transformations", func() {
blockRetriever := &fakes.MockLightBlockRetriever{}
blockRetriever := &fakes.MockHeaderSyncBlockRetriever{}
firstBlock := int64(1)
blockRetriever.FirstBlock = firstBlock
@ -102,7 +102,7 @@ var _ = Describe("Transformer", func() {
})
It("Fails to initialize if first block cannot be fetched from vDB headers table", func() {
blockRetriever := &fakes.MockLightBlockRetriever{}
blockRetriever := &fakes.MockHeaderSyncBlockRetriever{}
blockRetriever.FirstBlockErr = fakes.FakeError
t := getFakeTransformer(blockRetriever, &fakes.MockParser{}, &fakes.MockPoller{})
@ -119,7 +119,7 @@ func getFakeTransformer(blockRetriever retriever.BlockRetriever, parsr parser.Pa
Parser: parsr,
Retriever: blockRetriever,
Poller: pollr,
HeaderRepository: &fakes.MockLightHeaderRepository{},
HeaderRepository: &fakes.MockHeaderSyncHeaderRepository{},
Contracts: map[string]*contract.Contract{},
Config: mocks.MockConfig,
}

12
pkg/contract_watcher/shared/helpers/test_helpers/database.go
View File

@ -59,7 +59,7 @@ type NewOwnerLog struct {
Owner string `db:"owner_"`
}
type LightTransferLog struct {
type HeaderSyncTransferLog struct {
Id int64 `db:"id"`
HeaderID int64 `db:"header_id"`
TokenName string `db:"token_name"`
@ -71,7 +71,7 @@ type LightTransferLog struct {
RawLog []byte `db:"raw_log"`
}
type LightNewOwnerLog struct {
type HeaderSyncNewOwnerLog struct {
Id int64 `db:"id"`
HeaderID int64 `db:"header_id"`
TokenName string `db:"token_name"`
@ -240,13 +240,13 @@ func TearDown(db *postgres.DB) {
_, err = tx.Exec(`DELETE FROM full_sync_transactions`)
Expect(err).NotTo(HaveOccurred())
_, err = tx.Exec("DELETE FROM light_sync_transactions")
_, err = tx.Exec("DELETE FROM header_sync_transactions")
Expect(err).NotTo(HaveOccurred())
_, err = tx.Exec(`DELETE FROM full_sync_receipts`)
Expect(err).NotTo(HaveOccurred())
_, err = tx.Exec(`DELETE FROM light_sync_receipts`)
_, err = tx.Exec(`DELETE FROM header_sync_receipts`)
Expect(err).NotTo(HaveOccurred())
_, err = tx.Exec(`DROP TABLE checked_headers`)
@ -258,13 +258,13 @@ func TearDown(db *postgres.DB) {
_, err = tx.Exec(`DROP SCHEMA IF EXISTS full_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e CASCADE`)
Expect(err).NotTo(HaveOccurred())
_, err = tx.Exec(`DROP SCHEMA IF EXISTS light_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e CASCADE`)
_, err = tx.Exec(`DROP SCHEMA IF EXISTS header_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e CASCADE`)
Expect(err).NotTo(HaveOccurred())
_, err = tx.Exec(`DROP SCHEMA IF EXISTS full_0x314159265dd8dbb310642f98f50c066173c1259b CASCADE`)
Expect(err).NotTo(HaveOccurred())
_, err = tx.Exec(`DROP SCHEMA IF EXISTS light_0x314159265dd8dbb310642f98f50c066173c1259b CASCADE`)
_, err = tx.Exec(`DROP SCHEMA IF EXISTS header_0x314159265dd8dbb310642f98f50c066173c1259b CASCADE`)
Expect(err).NotTo(HaveOccurred())
err = tx.Commit()

10
pkg/contract_watcher/shared/repository/event_repository.go
View File

@ -84,8 +84,8 @@ func (r *eventRepository) PersistLogs(logs []types.Log, eventInfo types.Event, c
func (r *eventRepository) persistLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error {
var err error
switch r.mode {
case types.LightSync:
err = r.persistLightSyncLogs(logs, eventInfo, contractAddr, contractName)
case types.HeaderSync:
err = r.persistHeaderSyncLogs(logs, eventInfo, contractAddr, contractName)
case types.FullSync:
err = r.persistFullSyncLogs(logs, eventInfo, contractAddr, contractName)
default:
@ -95,8 +95,8 @@ func (r *eventRepository) persistLogs(logs []types.Log, eventInfo types.Event, c
return err
}
// Creates a custom postgres command to persist logs for the given event (compatible with light synced vDB)
func (r *eventRepository) persistLightSyncLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error {
// Creates a custom postgres command to persist logs for the given event (compatible with header synced vDB)
func (r *eventRepository) persistHeaderSyncLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error {
tx, err := r.db.Beginx()
if err != nil {
return err
@ -232,7 +232,7 @@ func (r *eventRepository) newEventTable(tableID string, event types.Event) error
pgStr = pgStr + fmt.Sprintf(" %s_ %s NOT NULL,", strings.ToLower(field.Name), field.PgType)
}
pgStr = pgStr + " CONSTRAINT log_index_fk FOREIGN KEY (vulcanize_log_id) REFERENCES logs (id) ON DELETE CASCADE)"
case types.LightSync:
case types.HeaderSync:
pgStr = pgStr + "(id SERIAL, header_id INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE, token_name CHARACTER VARYING(66) NOT NULL, raw_log JSONB, log_idx INTEGER NOT NULL, tx_idx INTEGER NOT NULL,"
for _, field := range event.Fields {

18
pkg/contract_watcher/shared/repository/event_repository_test.go
View File

@ -27,8 +27,8 @@ import (
. "github.com/onsi/gomega"
fc "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/converter"
lc "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/converter"
lr "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/repository"
lc "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/converter"
lr "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/repository"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract"
"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers"
@ -208,9 +208,9 @@ var _ = Describe("Repository", func() {
})
})
Describe("Light sync mode", func() {
Describe("Header sync mode", func() {
BeforeEach(func() {
dataStore = sr.NewEventRepository(db, types.LightSync)
dataStore = sr.NewEventRepository(db, types.HeaderSync)
})
Describe("CreateContractSchema", func() {
@ -242,7 +242,7 @@ var _ = Describe("Repository", func() {
Expect(err).ToNot(HaveOccurred())
Expect(created).To(Equal(true))
tableID := fmt.Sprintf("%s_%s.%s_event", types.LightSync, strings.ToLower(con.Address), strings.ToLower(event.Name))
tableID := fmt.Sprintf("%s_%s.%s_event", types.HeaderSync, strings.ToLower(con.Address), strings.ToLower(event.Name))
_, ok := dataStore.CheckTableCache(tableID)
Expect(ok).To(Equal(false))
@ -292,12 +292,12 @@ var _ = Describe("Repository", func() {
Expect(err).ToNot(HaveOccurred())
var count int
err = db.Get(&count, fmt.Sprintf("SELECT COUNT(*) FROM light_%s.transfer_event", constants.TusdContractAddress))
err = db.Get(&count, fmt.Sprintf("SELECT COUNT(*) FROM header_%s.transfer_event", constants.TusdContractAddress))
Expect(err).ToNot(HaveOccurred())
Expect(count).To(Equal(2))
scanLog := test_helpers.LightTransferLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.transfer_event LIMIT 1", constants.TusdContractAddress)).StructScan(&scanLog)
scanLog := test_helpers.HeaderSyncTransferLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.transfer_event LIMIT 1", constants.TusdContractAddress)).StructScan(&scanLog)
Expect(err).ToNot(HaveOccurred())
Expect(scanLog.HeaderID).To(Equal(headerID))
Expect(scanLog.TokenName).To(Equal("TrueUSD"))
@ -334,7 +334,7 @@ var _ = Describe("Repository", func() {
// Show that no new logs were entered
var count int
err = db.Get(&count, fmt.Sprintf("SELECT COUNT(*) FROM light_%s.transfer_event", constants.TusdContractAddress))
err = db.Get(&count, fmt.Sprintf("SELECT COUNT(*) FROM header_%s.transfer_event", constants.TusdContractAddress))
Expect(err).ToNot(HaveOccurred())
Expect(count).To(Equal(2))
})

10
pkg/contract_watcher/shared/repository/method_repository_test.go
View File

@ -149,9 +149,9 @@ var _ = Describe("Repository", func() {
})
})
Describe("Light Sync Mode", func() {
Describe("Header Sync Mode", func() {
BeforeEach(func() {
dataStore = repository.NewMethodRepository(db, types.LightSync)
dataStore = repository.NewMethodRepository(db, types.HeaderSync)
})
Describe("CreateContractSchema", func() {
@ -199,7 +199,7 @@ var _ = Describe("Repository", func() {
Expect(err).ToNot(HaveOccurred())
Expect(created).To(Equal(true))
tableID := fmt.Sprintf("%s_%s.%s_method", types.LightSync, strings.ToLower(con.Address), strings.ToLower(method.Name))
tableID := fmt.Sprintf("%s_%s.%s_method", types.HeaderSync, strings.ToLower(con.Address), strings.ToLower(method.Name))
_, ok := dataStore.CheckTableCache(tableID)
Expect(ok).To(Equal(false))
@ -214,13 +214,13 @@ var _ = Describe("Repository", func() {
})
Describe("PersistResult", func() {
It("Persists result from method polling in custom pg table for light sync mode vDB", func() {
It("Persists result from method polling in custom pg table for header sync mode vDB", func() {
err = dataStore.PersistResults([]types.Result{mockResult}, method, con.Address, con.Name)
Expect(err).ToNot(HaveOccurred())
scanStruct := test_helpers.BalanceOf{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method", constants.TusdContractAddress)).StructScan(&scanStruct)
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.balanceof_method", constants.TusdContractAddress)).StructScan(&scanStruct)
expectedLog := test_helpers.BalanceOf{
Id: 1,
TokenName: "TrueUSD",

4
pkg/contract_watcher/shared/types/event.go
View File

@ -38,14 +38,14 @@ type Field struct {
// Struct to hold instance of an event log data
type Log struct {
Id int64 // VulcanizeIdLog for full sync and header ID for light sync contract watcher
Id int64 // VulcanizeIdLog for full sync and header ID for header sync contract watcher
Values map[string]string // Map of event input names to their values
// Used for full sync only
Block int64
Tx string
// Used for lightSync only
// Used for headerSync only
LogIndex uint
TransactionIndex uint
Raw []byte // json.Unmarshalled byte array of geth/core/types.Log{}

20
pkg/contract_watcher/shared/types/mode.go
View File

@ -21,18 +21,18 @@ import "fmt"
type Mode int
const (
LightSync Mode = iota
HeaderSync Mode = iota
FullSync
)
func (mode Mode) IsValid() bool {
return mode >= LightSync && mode <= FullSync
return mode >= HeaderSync && mode <= FullSync
}
func (mode Mode) String() string {
switch mode {
case LightSync:
return "light"
case HeaderSync:
return "header"
case FullSync:
return "full"
default:
@ -42,23 +42,23 @@ func (mode Mode) String() string {
func (mode Mode) MarshalText() ([]byte, error) {
switch mode {
case LightSync:
return []byte("light"), nil
case HeaderSync:
return []byte("header"), nil
case FullSync:
return []byte("full"), nil
default:
return nil, fmt.Errorf("contract watcher: unknown mode %d, want LightSync or FullSync", mode)
return nil, fmt.Errorf("contract watcher: unknown mode %d, want HeaderSync or FullSync", mode)
}
}
func (mode *Mode) UnmarshalText(text []byte) error {
switch string(text) {
case "light":
*mode = LightSync
case "header":
*mode = HeaderSync
case "full":
*mode = FullSync
default:
return fmt.Errorf(`contract watcher: unknown mode %q, want "light" or "full"`, text)
return fmt.Errorf(`contract watcher: unknown mode %q, want "header" or "full"`, text)
}
return nil
}

6
pkg/datastore/postgres/repositories/header_repository.go
View File

@ -54,7 +54,7 @@ func (repository HeaderRepository) CreateOrUpdateHeader(header core.Header) (int
func (repository HeaderRepository) CreateTransactions(headerID int64, transactions []core.TransactionModel) error {
for _, transaction := range transactions {
_, err := repository.database.Exec(`INSERT INTO public.light_sync_transactions
_, err := repository.database.Exec(`INSERT INTO public.header_sync_transactions
(header_id, hash, gas_limit, gas_price, input_data, nonce, raw, tx_from, tx_index, tx_to, "value")
VALUES ($1, $2, $3::NUMERIC, $4::NUMERIC, $5, $6::NUMERIC, $7, $8, $9::NUMERIC, $10, $11::NUMERIC)
ON CONFLICT DO NOTHING`, headerID, transaction.Hash, transaction.GasLimit, transaction.GasPrice,
@ -69,7 +69,7 @@ func (repository HeaderRepository) CreateTransactions(headerID int64, transactio
func (repository HeaderRepository) CreateTransactionInTx(tx *sqlx.Tx, headerID int64, transaction core.TransactionModel) (int64, error) {
var txId int64
err := tx.QueryRowx(`INSERT INTO public.light_sync_transactions
err := tx.QueryRowx(`INSERT INTO public.header_sync_transactions
(header_id, hash, gas_limit, gas_price, input_data, nonce, raw, tx_from, tx_index, tx_to, "value")
VALUES ($1, $2, $3::NUMERIC, $4::NUMERIC, $5, $6::NUMERIC, $7, $8, $9::NUMERIC, $10, $11::NUMERIC)
ON CONFLICT (header_id, hash) DO UPDATE
@ -87,7 +87,7 @@ func (repository HeaderRepository) CreateTransactionInTx(tx *sqlx.Tx, headerID i
func (repository HeaderRepository) CreateReceiptInTx(tx *sqlx.Tx, headerID, transactionID int64, receipt core.Receipt) (int64, error) {
var receiptId int64
err := tx.QueryRowx(`INSERT INTO public.light_sync_receipts
err := tx.QueryRowx(`INSERT INTO public.header_sync_receipts
(header_id, transaction_id, contract_address, cumulative_gas_used, gas_used, state_root, status, tx_hash, rlp)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
ON CONFLICT (header_id, transaction_id) DO UPDATE

10
pkg/datastore/postgres/repositories/header_repository_test.go
View File

@ -228,7 +228,7 @@ var _ = Describe("Block header repository", func() {
var dbReceipt idModel
err = db.Get(&dbReceipt,
`SELECT transaction_id, contract_address, cumulative_gas_used, gas_used, state_root, status, tx_hash, rlp
FROM public.light_sync_receipts WHERE header_id = $1`, headerID)
FROM public.header_sync_receipts WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(dbReceipt.TransactionId).To(Equal(txId))
Expect(dbReceipt.TxHash).To(Equal(txHash.Hex()))
@ -288,7 +288,7 @@ var _ = Describe("Block header repository", func() {
var dbTransactions []core.TransactionModel
err = db.Select(&dbTransactions,
`SELECT hash, gas_limit, gas_price, input_data, nonce, raw, tx_from, tx_index, tx_to, "value"
FROM public.light_sync_transactions WHERE header_id = $1`, headerID)
FROM public.header_sync_transactions WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(dbTransactions).To(ConsistOf(transactions))
})
@ -300,7 +300,7 @@ var _ = Describe("Block header repository", func() {
var dbTransactions []core.TransactionModel
err = db.Select(&dbTransactions,
`SELECT hash, gas_limit, gas_price, input_data, nonce, raw, tx_from, tx_index, tx_to, "value"
FROM public.light_sync_transactions WHERE header_id = $1`, headerID)
FROM public.header_sync_transactions WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(len(dbTransactions)).To(Equal(2))
})
@ -337,7 +337,7 @@ var _ = Describe("Block header repository", func() {
var dbTransaction core.TransactionModel
err = db.Get(&dbTransaction,
`SELECT hash, gas_limit, gas_price, input_data, nonce, raw, tx_from, tx_index, tx_to, "value"
FROM public.light_sync_transactions WHERE header_id = $1`, headerID)
FROM public.header_sync_transactions WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(dbTransaction).To(Equal(transaction))
})
@ -381,7 +381,7 @@ var _ = Describe("Block header repository", func() {
var dbTransactions []core.TransactionModel
err = db.Select(&dbTransactions,
`SELECT hash, gas_limit, gas_price, input_data, nonce, raw, tx_from, tx_index, tx_to, "value"
FROM public.light_sync_transactions WHERE header_id = $1`, headerID)
FROM public.header_sync_transactions WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(len(dbTransactions)).To(Equal(1))
})

15
pkg/fakes/mock_full_block_retriever.go
View File

@ -1,15 +0,0 @@
package fakes
type MockFullBlockRetriever struct {
FirstBlock int64
FirstBlockErr error
MostRecentBlock int64
}
func (retriever *MockFullBlockRetriever) RetrieveFirstBlock(contractAddr string) (int64, error) {
return retriever.FirstBlock, retriever.FirstBlockErr
}
func (retriever *MockFullBlockRetriever) RetrieveMostRecentBlock() (int64, error) {
return retriever.MostRecentBlock, nil
}

15
pkg/fakes/mock_full_sync_block_retriever.go Normal file
View File

@ -0,0 +1,15 @@
package fakes
type MockFullSyncBlockRetriever struct {
FirstBlock int64
FirstBlockErr error
MostRecentBlock int64
}
func (retriever *MockFullSyncBlockRetriever) RetrieveFirstBlock(contractAddr string) (int64, error) {
return retriever.FirstBlock, retriever.FirstBlockErr
}
func (retriever *MockFullSyncBlockRetriever) RetrieveMostRecentBlock() (int64, error) {
return retriever.MostRecentBlock, nil
}

14
pkg/fakes/mock_header_sync_block_retriever.go Normal file
View File

@ -0,0 +1,14 @@
package fakes
type MockHeaderSyncBlockRetriever struct {
FirstBlock int64
FirstBlockErr error
}
func (retriever *MockHeaderSyncBlockRetriever) RetrieveFirstBlock() (int64, error) {
return retriever.FirstBlock, retriever.FirstBlockErr
}
func (retriever *MockHeaderSyncBlockRetriever) RetrieveMostRecentBlock() (int64, error) {
return 0, nil
}

42
pkg/fakes/mock_header_sync_header_repository.go Normal file
View File

@ -0,0 +1,42 @@
package fakes
import "github.com/vulcanize/vulcanizedb/pkg/core"
type MockHeaderSyncHeaderRepository struct {
}
func (*MockHeaderSyncHeaderRepository) AddCheckColumn(id string) error {
return nil
}
func (*MockHeaderSyncHeaderRepository) AddCheckColumns(ids []string) error {
panic("implement me")
}
func (*MockHeaderSyncHeaderRepository) MarkHeaderChecked(headerID int64, eventID string) error {
panic("implement me")
}
func (*MockHeaderSyncHeaderRepository) MarkHeaderCheckedForAll(headerID int64, ids []string) error {
panic("implement me")
}
func (*MockHeaderSyncHeaderRepository) MarkHeadersCheckedForAll(headers []core.Header, ids []string) error {
panic("implement me")
}
func (*MockHeaderSyncHeaderRepository) MissingHeaders(startingBlockNumber int64, endingBlockNumber int64, eventID string) ([]core.Header, error) {
panic("implement me")
}
func (*MockHeaderSyncHeaderRepository) MissingMethodsCheckedEventsIntersection(startingBlockNumber, endingBlockNumber int64, methodIds, eventIds []string) ([]core.Header, error) {
panic("implement me")
}
func (*MockHeaderSyncHeaderRepository) MissingHeadersForAll(startingBlockNumber, endingBlockNumber int64, ids []string) ([]core.Header, error) {
panic("implement me")
}
func (*MockHeaderSyncHeaderRepository) CheckCache(key string) (interface{}, bool) {
panic("implement me")
}

14
pkg/fakes/mock_light_block_retriever.go
View File

@ -1,14 +0,0 @@
package fakes
type MockLightBlockRetriever struct {
FirstBlock int64
FirstBlockErr error
}
func (retriever *MockLightBlockRetriever) RetrieveFirstBlock() (int64, error) {
return retriever.FirstBlock, retriever.FirstBlockErr
}
func (retriever *MockLightBlockRetriever) RetrieveMostRecentBlock() (int64, error) {
return 0, nil
}

42
pkg/fakes/mock_light_header_repository.go
View File

@ -1,42 +0,0 @@
package fakes
import "github.com/vulcanize/vulcanizedb/pkg/core"
type MockLightHeaderRepository struct {
}
func (*MockLightHeaderRepository) AddCheckColumn(id string) error {
return nil
}
func (*MockLightHeaderRepository) AddCheckColumns(ids []string) error {
panic("implement me")
}
func (*MockLightHeaderRepository) MarkHeaderChecked(headerID int64, eventID string) error {
panic("implement me")
}
func (*MockLightHeaderRepository) MarkHeaderCheckedForAll(headerID int64, ids []string) error {
panic("implement me")
}
func (*MockLightHeaderRepository) MarkHeadersCheckedForAll(headers []core.Header, ids []string) error {
panic("implement me")
}
func (*MockLightHeaderRepository) MissingHeaders(startingBlockNumber int64, endingBlockNumber int64, eventID string) ([]core.Header, error) {
panic("implement me")
}
func (*MockLightHeaderRepository) MissingMethodsCheckedEventsIntersection(startingBlockNumber, endingBlockNumber int64, methodIds, eventIds []string) ([]core.Header, error) {
panic("implement me")
}
func (*MockLightHeaderRepository) MissingHeadersForAll(startingBlockNumber, endingBlockNumber int64, ids []string) ([]core.Header, error) {
panic("implement me")
}
func (*MockLightHeaderRepository) CheckCache(key string) (interface{}, bool) {
panic("implement me")
}

2
pkg/geth/converters/rpc/transaction_converter.go
View File

@ -78,7 +78,7 @@ func (converter *RpcTransactionConverter) ConvertRpcTransactionsToModels(transac
Hash: transaction.Hash,
Nonce: txData.AccountNonce,
Raw: txRLP,
// NOTE: Light Sync transactions don't include receipt; would require separate RPC call
// NOTE: Header Sync transactions don't include receipt; would require separate RPC call
To: transaction.Recipient,
TxIndex: txIndex.Int64(),
Value: txData.Amount.String(),

4
test_config/test_config.go
View File

@ -111,12 +111,12 @@ func CleanTestDB(db *postgres.DB) {
db.MustExec("DELETE FROM full_sync_transactions")
db.MustExec("DELETE FROM goose_db_version")
db.MustExec("DELETE FROM headers")
db.MustExec("DELETE FROM light_sync_transactions")
db.MustExec("DELETE FROM header_sync_transactions")
db.MustExec("DELETE FROM log_filters")
db.MustExec("DELETE FROM logs")
db.MustExec("DELETE FROM queued_storage")
db.MustExec("DELETE FROM full_sync_receipts")
db.MustExec("DELETE FROM light_sync_receipts")
db.MustExec("DELETE FROM header_sync_receipts")
db.MustExec("DELETE FROM watched_contracts")
}