cerc-io/ipld-eth-server
17
0
Fork 1
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases 37 Wiki Activity

retriever for generating list of all token holder addresses + updated transformer to use said addresses to populate balanceOf and allowance information and added database migrations for balance and allowance tables

Browse Source
This commit is contained in:
Ian Norden 2018-08-22 14:11:15 -05:00
parent 1d50a0ace0
commit 44e0a8d303
10 changed files with 353 additions and 404 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
db/migrations/1535055894_create_token_balance_table.down.sql Normal file
View File

@ -0,0 +1 @@
DROP TABLE token_balance;

10
db/migrations/1535055894_create_token_balance_table.up.sql Normal file
View File

@ -0,0 +1,10 @@
CREATE TABLE token_balance (
id SERIAL,
block_id INTEGER NOT NULL,
balance DECIMAL NOT NULL,
token_address CHARACTER VARYING(66) NOT NULL,
token_holder_address CHARACTER VARYING(66) NOT NULL,
CONSTRAINT blocks_fk FOREIGN KEY (block_id)
REFERENCES blocks (id)
ON DELETE CASCADE
)

1
db/migrations/20180823160858_create_token_allowance_table.down.sql Normal file
View File

@ -0,0 +1 @@
DROP TABLE token_allowance;

11
db/migrations/20180823160858_create_token_allowance_table.up.sql Normal file
View File

@ -0,0 +1,11 @@
CREATE TABLE token_allowance (
id SERIAL,
block_id INTEGER NOT NULL,
allowance DECIMAL NOT NULL,
token_address CHARACTER VARYING(66) NOT NULL,
token_holder_address CHARACTER VARYING(66) NOT NULL,
token_spender_address CHARACTER VARYING(66) NOT NULL,
CONSTRAINT blocks_fk FOREIGN KEY (block_id)
REFERENCES blocks (id)
ON DELETE CASCADE
)

456
db/schema.sql
View File

@ -2,8 +2,8 @@
-- PostgreSQL database dump
--
-- Dumped from database version 10.5
-- Dumped by pg_dump version 10.4
-- Dumped from database version 10.3
-- Dumped by pg_dump version 10.3
SET statement_timeout = 0;
SET lock_timeout = 0;
@ -16,14 +16,14 @@ SET client_min_messages = warning;
SET row_security = off;
--
-- Name: plpgsql; Type: EXTENSION; Schema: -; Owner:
-- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: -
--
CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog;
--
-- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner:
-- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: -
--
COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language';
@ -34,7 +34,7 @@ SET default_tablespace = '';
SET default_with_oids = false;
--
-- Name: logs; Type: TABLE; Schema: public; Owner: iannorden
-- Name: logs; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.logs (
@ -52,10 +52,8 @@ CREATE TABLE public.logs (
);
ALTER TABLE public.logs OWNER TO iannorden;
--
-- Name: block_stats; Type: VIEW; Schema: public; Owner: iannorden
-- Name: block_stats; Type: VIEW; Schema: public; Owner: -
--
CREATE VIEW public.block_stats AS
@ -64,10 +62,8 @@ CREATE VIEW public.block_stats AS
FROM public.logs;
ALTER TABLE public.block_stats OWNER TO iannorden;
--
-- Name: blocks; Type: TABLE; Schema: public; Owner: iannorden
-- Name: blocks; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.blocks (
@ -92,10 +88,8 @@ CREATE TABLE public.blocks (
);
ALTER TABLE public.blocks OWNER TO iannorden;
--
-- Name: blocks_id_seq; Type: SEQUENCE; Schema: public; Owner: iannorden
-- Name: blocks_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.blocks_id_seq
@ -107,17 +101,15 @@ CREATE SEQUENCE public.blocks_id_seq
CACHE 1;
ALTER TABLE public.blocks_id_seq OWNER TO iannorden;
--
-- Name: blocks_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: iannorden
-- Name: blocks_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.blocks_id_seq OWNED BY public.blocks.id;
--
-- Name: eth_nodes; Type: TABLE; Schema: public; Owner: iannorden
-- Name: eth_nodes; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.eth_nodes (
@ -129,10 +121,8 @@ CREATE TABLE public.eth_nodes (
);
ALTER TABLE public.eth_nodes OWNER TO iannorden;
--
-- Name: headers; Type: TABLE; Schema: public; Owner: iannorden
-- Name: headers; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.headers (
@ -145,10 +135,8 @@ CREATE TABLE public.headers (
);
ALTER TABLE public.headers OWNER TO iannorden;
--
-- Name: headers_id_seq; Type: SEQUENCE; Schema: public; Owner: iannorden
-- Name: headers_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.headers_id_seq
@ -160,17 +148,15 @@ CREATE SEQUENCE public.headers_id_seq
CACHE 1;
ALTER TABLE public.headers_id_seq OWNER TO iannorden;
--
-- Name: headers_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: iannorden
-- Name: headers_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.headers_id_seq OWNED BY public.headers.id;
--
-- Name: log_filters; Type: TABLE; Schema: public; Owner: iannorden
-- Name: log_filters; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.log_filters (
@ -189,10 +175,8 @@ CREATE TABLE public.log_filters (
);
ALTER TABLE public.log_filters OWNER TO iannorden;
--
-- Name: log_filters_id_seq; Type: SEQUENCE; Schema: public; Owner: iannorden
-- Name: log_filters_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.log_filters_id_seq
@ -204,17 +188,15 @@ CREATE SEQUENCE public.log_filters_id_seq
CACHE 1;
ALTER TABLE public.log_filters_id_seq OWNER TO iannorden;
--
-- Name: log_filters_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: iannorden
-- Name: log_filters_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.log_filters_id_seq OWNED BY public.log_filters.id;
--
-- Name: logs_id_seq; Type: SEQUENCE; Schema: public; Owner: iannorden
-- Name: logs_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.logs_id_seq
@ -226,17 +208,15 @@ CREATE SEQUENCE public.logs_id_seq
CACHE 1;
ALTER TABLE public.logs_id_seq OWNER TO iannorden;
--
-- Name: logs_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: iannorden
-- Name: logs_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.logs_id_seq OWNED BY public.logs.id;
--
-- Name: nodes_id_seq; Type: SEQUENCE; Schema: public; Owner: iannorden
-- Name: nodes_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.nodes_id_seq
@ -248,17 +228,15 @@ CREATE SEQUENCE public.nodes_id_seq
CACHE 1;
ALTER TABLE public.nodes_id_seq OWNER TO iannorden;
--
-- Name: nodes_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: iannorden
-- Name: nodes_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.nodes_id_seq OWNED BY public.eth_nodes.id;
--
-- Name: receipts; Type: TABLE; Schema: public; Owner: iannorden
-- Name: receipts; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.receipts (
@ -273,10 +251,8 @@ CREATE TABLE public.receipts (
);
ALTER TABLE public.receipts OWNER TO iannorden;
--
-- Name: receipts_id_seq; Type: SEQUENCE; Schema: public; Owner: iannorden
-- Name: receipts_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.receipts_id_seq
@ -288,17 +264,15 @@ CREATE SEQUENCE public.receipts_id_seq
CACHE 1;
ALTER TABLE public.receipts_id_seq OWNER TO iannorden;
--
-- Name: receipts_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: iannorden
-- Name: receipts_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.receipts_id_seq OWNED BY public.receipts.id;
--
-- Name: schema_migrations; Type: TABLE; Schema: public; Owner: iannorden
-- Name: schema_migrations; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.schema_migrations (
@ -307,10 +281,8 @@ CREATE TABLE public.schema_migrations (
);
ALTER TABLE public.schema_migrations OWNER TO iannorden;
--
-- Name: token_supply; Type: TABLE; Schema: public; Owner: iannorden
-- Name: token_supply; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.token_supply (
@ -321,10 +293,8 @@ CREATE TABLE public.token_supply (
);
ALTER TABLE public.token_supply OWNER TO iannorden;
--
-- Name: token_supply_id_seq; Type: SEQUENCE; Schema: public; Owner: iannorden
-- Name: token_supply_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.token_supply_id_seq
@ -336,92 +306,15 @@ CREATE SEQUENCE public.token_supply_id_seq
CACHE 1;
ALTER TABLE public.token_supply_id_seq OWNER TO iannorden;
--
-- Name: token_supply_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: iannorden
-- Name: token_supply_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.token_supply_id_seq OWNED BY public.token_supply.id;
--
-- Name: token_allowance; Type: TABLE; Schema: public; Owner: iannorden
--
CREATE TABLE public.token_allowance (
id integer DEFAULT nextval('public.token_supply_id_seq'::regclass) NOT NULL,
block_id integer NOT NULL,
allowance numeric NOT NULL,
token_address character varying(66) NOT NULL,
token_holder_address character varying(66) NOT NULL,
token_spender_address character varying(66) NOT NULL
);
ALTER TABLE public.token_allowance OWNER TO iannorden;
--
-- Name: token_allowance_id_seq; Type: SEQUENCE; Schema: public; Owner: iannorden
--
CREATE SEQUENCE public.token_allowance_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
ALTER TABLE public.token_allowance_id_seq OWNER TO iannorden;
--
-- Name: token_allowance_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: iannorden
--
ALTER SEQUENCE public.token_allowance_id_seq OWNED BY public.token_allowance.id;
--
-- Name: token_balance; Type: TABLE; Schema: public; Owner: iannorden
--
CREATE TABLE public.token_balance (
id integer DEFAULT nextval('public.token_supply_id_seq'::regclass) NOT NULL,
block_id integer NOT NULL,
balance numeric NOT NULL,
token_address character varying(66) NOT NULL,
token_holder_address character varying(66) NOT NULL
);
ALTER TABLE public.token_balance OWNER TO iannorden;
--
-- Name: token_balance_id_seq; Type: SEQUENCE; Schema: public; Owner: iannorden
--
CREATE SEQUENCE public.token_balance_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
ALTER TABLE public.token_balance_id_seq OWNER TO iannorden;
--
-- Name: token_balance_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: iannorden
--
ALTER SEQUENCE public.token_balance_id_seq OWNED BY public.token_balance.id;
--
-- Name: transactions; Type: TABLE; Schema: public; Owner: iannorden
-- Name: transactions; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.transactions (
@ -438,10 +331,8 @@ CREATE TABLE public.transactions (
);
ALTER TABLE public.transactions OWNER TO iannorden;
--
-- Name: transactions_id_seq; Type: SEQUENCE; Schema: public; Owner: iannorden
-- Name: transactions_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.transactions_id_seq
@ -453,17 +344,15 @@ CREATE SEQUENCE public.transactions_id_seq
CACHE 1;
ALTER TABLE public.transactions_id_seq OWNER TO iannorden;
--
-- Name: transactions_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: iannorden
-- Name: transactions_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.transactions_id_seq OWNED BY public.transactions.id;
--
-- Name: watched_contracts; Type: TABLE; Schema: public; Owner: iannorden
-- Name: watched_contracts; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.watched_contracts (
@ -473,10 +362,8 @@ CREATE TABLE public.watched_contracts (
);
ALTER TABLE public.watched_contracts OWNER TO iannorden;
--
-- Name: watched_contracts_contract_id_seq; Type: SEQUENCE; Schema: public; Owner: iannorden
-- Name: watched_contracts_contract_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.watched_contracts_contract_id_seq
@ -488,17 +375,15 @@ CREATE SEQUENCE public.watched_contracts_contract_id_seq
CACHE 1;
ALTER TABLE public.watched_contracts_contract_id_seq OWNER TO iannorden;
--
-- Name: watched_contracts_contract_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: iannorden
-- Name: watched_contracts_contract_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.watched_contracts_contract_id_seq OWNED BY public.watched_contracts.contract_id;
--
-- Name: watched_event_logs; Type: VIEW; Schema: public; Owner: iannorden
-- Name: watched_event_logs; Type: VIEW; Schema: public; Owner: -
--
CREATE VIEW public.watched_event_logs AS
@ -520,261 +405,71 @@ CREATE VIEW public.watched_event_logs AS
WHERE ((((log_filters.topic0)::text = (logs.topic0)::text) OR (log_filters.topic0 IS NULL)) AND (((log_filters.topic1)::text = (logs.topic1)::text) OR (log_filters.topic1 IS NULL)) AND (((log_filters.topic2)::text = (logs.topic2)::text) OR (log_filters.topic2 IS NULL)) AND (((log_filters.topic3)::text = (logs.topic3)::text) OR (log_filters.topic3 IS NULL)));
ALTER TABLE public.watched_event_logs OWNER TO iannorden;
--
-- Name: blocks id; Type: DEFAULT; Schema: public; Owner: iannorden
-- Name: blocks id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.blocks ALTER COLUMN id SET DEFAULT nextval('public.blocks_id_seq'::regclass);
--
-- Name: eth_nodes id; Type: DEFAULT; Schema: public; Owner: iannorden
-- Name: eth_nodes id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.eth_nodes ALTER COLUMN id SET DEFAULT nextval('public.nodes_id_seq'::regclass);
--
-- Name: headers id; Type: DEFAULT; Schema: public; Owner: iannorden
-- Name: headers id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.headers ALTER COLUMN id SET DEFAULT nextval('public.headers_id_seq'::regclass);
--
-- Name: log_filters id; Type: DEFAULT; Schema: public; Owner: iannorden
-- Name: log_filters id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.log_filters ALTER COLUMN id SET DEFAULT nextval('public.log_filters_id_seq'::regclass);
--
-- Name: logs id; Type: DEFAULT; Schema: public; Owner: iannorden
-- Name: logs id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.logs ALTER COLUMN id SET DEFAULT nextval('public.logs_id_seq'::regclass);
--
-- Name: receipts id; Type: DEFAULT; Schema: public; Owner: iannorden
-- Name: receipts id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.receipts ALTER COLUMN id SET DEFAULT nextval('public.receipts_id_seq'::regclass);
--
-- Name: token_supply id; Type: DEFAULT; Schema: public; Owner: iannorden
-- Name: token_supply id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.token_supply ALTER COLUMN id SET DEFAULT nextval('public.token_supply_id_seq'::regclass);
--
-- Name: transactions id; Type: DEFAULT; Schema: public; Owner: iannorden
-- Name: transactions id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.transactions ALTER COLUMN id SET DEFAULT nextval('public.transactions_id_seq'::regclass);
--
-- Name: watched_contracts contract_id; Type: DEFAULT; Schema: public; Owner: iannorden
-- Name: watched_contracts contract_id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.watched_contracts ALTER COLUMN contract_id SET DEFAULT nextval('public.watched_contracts_contract_id_seq'::regclass);
--
-- Data for Name: blocks; Type: TABLE DATA; Schema: public; Owner: iannorden
--
COPY public.blocks (number, gaslimit, gasused, "time", id, difficulty, hash, nonce, parenthash, size, uncle_hash, eth_node_id, is_final, miner, extra_data, reward, uncles_reward, eth_node_fingerprint) FROM stdin;
\.
--
-- Data for Name: eth_nodes; Type: TABLE DATA; Schema: public; Owner: iannorden
--
COPY public.eth_nodes (id, genesis_block, network_id, eth_node_id, client_name) FROM stdin;
1 GENESIS 1 2ea672a45c4c7b96e3c4b130b21a22af390a552fd0b3cff96420b4bda26568d470dc56e05e453823f64f2556a6e4460ad1d4d00eb2d8b8fc16fcb1be73e86522 Geth/v1.7.2-stable-1db4ecdc/darwin-amd64/go1.9
111 0x456 1
66 GENESIS 1 x123 geth
42 GENESIS 1 b6f90c0fdd8ec9607aed8ee45c69322e47b7063f0bfb7a29c8ecafab24d0a22d24dd2329b5ee6ed4125a03cb14e57fd584e67f9e53e6c631055cbbd82f080845 Geth/v1.7.2-stable-1db4ecdc/darwin-amd64/go1.9
70 0 EthNodeFingerprint
104 0x456 1 x123456 Geth
73 0 Fingerprint
74 0 FingerprintTwo
5 GENESIS 1 testNodeId Geth/v1.7.2-stable-1db4ecdc/darwin-amd64/go1.9
69 0
81 0 NodeFingerprint
79 0 NodeFingerprintTwo
\.
--
-- Data for Name: headers; Type: TABLE DATA; Schema: public; Owner: iannorden
--
COPY public.headers (id, hash, block_number, raw, eth_node_id, eth_node_fingerprint) FROM stdin;
304 1 \\x 81 NodeFingerprint
305 3 \\x 81 NodeFingerprint
306 5 \\x 81 NodeFingerprint
\.
--
-- Data for Name: log_filters; Type: TABLE DATA; Schema: public; Owner: iannorden
--
COPY public.log_filters (id, name, from_block, to_block, address, topic0, topic1, topic2, topic3) FROM stdin;
\.
--
-- Data for Name: logs; Type: TABLE DATA; Schema: public; Owner: iannorden
--
COPY public.logs (id, block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data, receipt_id) FROM stdin;
\.
--
-- Data for Name: receipts; Type: TABLE DATA; Schema: public; Owner: iannorden
--
COPY public.receipts (id, contract_address, cumulative_gas_used, gas_used, state_root, status, tx_hash, block_id) FROM stdin;
\.
--
-- Data for Name: schema_migrations; Type: TABLE DATA; Schema: public; Owner: iannorden
--
COPY public.schema_migrations (version, dirty) FROM stdin;
\.
--
-- Data for Name: token_allowance; Type: TABLE DATA; Schema: public; Owner: iannorden
--
COPY public.token_allowance (id, block_id, allowance, token_address, token_holder_address, token_spender_address) FROM stdin;
\.
--
-- Data for Name: token_balance; Type: TABLE DATA; Schema: public; Owner: iannorden
--
COPY public.token_balance (id, block_id, balance, token_address, token_holder_address) FROM stdin;
\.
--
-- Data for Name: token_supply; Type: TABLE DATA; Schema: public; Owner: iannorden
--
COPY public.token_supply (id, block_id, supply, token_address) FROM stdin;
\.
--
-- Data for Name: transactions; Type: TABLE DATA; Schema: public; Owner: iannorden
--
COPY public.transactions (id, hash, nonce, tx_to, gaslimit, gasprice, value, block_id, tx_from, input_data) FROM stdin;
\.
--
-- Data for Name: watched_contracts; Type: TABLE DATA; Schema: public; Owner: iannorden
--
COPY public.watched_contracts (contract_id, contract_hash, contract_abi) FROM stdin;
\.
--
-- Name: blocks_id_seq; Type: SEQUENCE SET; Schema: public; Owner: iannorden
--
SELECT pg_catalog.setval('public.blocks_id_seq', 1902, true);
--
-- Name: headers_id_seq; Type: SEQUENCE SET; Schema: public; Owner: iannorden
--
SELECT pg_catalog.setval('public.headers_id_seq', 306, true);
--
-- Name: log_filters_id_seq; Type: SEQUENCE SET; Schema: public; Owner: iannorden
--
SELECT pg_catalog.setval('public.log_filters_id_seq', 102, true);
--
-- Name: logs_id_seq; Type: SEQUENCE SET; Schema: public; Owner: iannorden
--
SELECT pg_catalog.setval('public.logs_id_seq', 290, true);
--
-- Name: nodes_id_seq; Type: SEQUENCE SET; Schema: public; Owner: iannorden
--
SELECT pg_catalog.setval('public.nodes_id_seq', 1770, true);
--
-- Name: receipts_id_seq; Type: SEQUENCE SET; Schema: public; Owner: iannorden
--
SELECT pg_catalog.setval('public.receipts_id_seq', 153, true);
--
-- Name: token_allowance_id_seq; Type: SEQUENCE SET; Schema: public; Owner: iannorden
--
SELECT pg_catalog.setval('public.token_allowance_id_seq', 1, false);
--
-- Name: token_balance_id_seq; Type: SEQUENCE SET; Schema: public; Owner: iannorden
--
SELECT pg_catalog.setval('public.token_balance_id_seq', 1, false);
--
-- Name: token_supply_id_seq; Type: SEQUENCE SET; Schema: public; Owner: iannorden
--
SELECT pg_catalog.setval('public.token_supply_id_seq', 400, true);
--
-- Name: transactions_id_seq; Type: SEQUENCE SET; Schema: public; Owner: iannorden
--
SELECT pg_catalog.setval('public.transactions_id_seq', 340, true);
--
-- Name: watched_contracts_contract_id_seq; Type: SEQUENCE SET; Schema: public; Owner: iannorden
--
SELECT pg_catalog.setval('public.watched_contracts_contract_id_seq', 102, true);
--
-- Name: blocks blocks_pkey; Type: CONSTRAINT; Schema: public; Owner: iannorden
-- Name: blocks blocks_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.blocks
@ -782,7 +477,7 @@ ALTER TABLE ONLY public.blocks
--
-- Name: watched_contracts contract_hash_uc; Type: CONSTRAINT; Schema: public; Owner: iannorden
-- Name: watched_contracts contract_hash_uc; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.watched_contracts
@ -790,7 +485,7 @@ ALTER TABLE ONLY public.watched_contracts
--
-- Name: blocks eth_node_id_block_number_uc; Type: CONSTRAINT; Schema: public; Owner: iannorden
-- Name: blocks eth_node_id_block_number_uc; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.blocks
@ -798,7 +493,7 @@ ALTER TABLE ONLY public.blocks
--
-- Name: eth_nodes eth_node_uc; Type: CONSTRAINT; Schema: public; Owner: iannorden
-- Name: eth_nodes eth_node_uc; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.eth_nodes
@ -806,7 +501,7 @@ ALTER TABLE ONLY public.eth_nodes
--
-- Name: headers headers_pkey; Type: CONSTRAINT; Schema: public; Owner: iannorden
-- Name: headers headers_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.headers
@ -814,7 +509,7 @@ ALTER TABLE ONLY public.headers
--
-- Name: logs logs_pkey; Type: CONSTRAINT; Schema: public; Owner: iannorden
-- Name: logs logs_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.logs
@ -822,7 +517,7 @@ ALTER TABLE ONLY public.logs
--
-- Name: log_filters name_uc; Type: CONSTRAINT; Schema: public; Owner: iannorden
-- Name: log_filters name_uc; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.log_filters
@ -830,7 +525,7 @@ ALTER TABLE ONLY public.log_filters
--
-- Name: eth_nodes nodes_pkey; Type: CONSTRAINT; Schema: public; Owner: iannorden
-- Name: eth_nodes nodes_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.eth_nodes
@ -838,7 +533,7 @@ ALTER TABLE ONLY public.eth_nodes
--
-- Name: receipts receipts_pkey; Type: CONSTRAINT; Schema: public; Owner: iannorden
-- Name: receipts receipts_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.receipts
@ -846,7 +541,7 @@ ALTER TABLE ONLY public.receipts
--
-- Name: schema_migrations schema_migrations_pkey; Type: CONSTRAINT; Schema: public; Owner: iannorden
-- Name: schema_migrations schema_migrations_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.schema_migrations
@ -854,7 +549,7 @@ ALTER TABLE ONLY public.schema_migrations
--
-- Name: transactions transactions_pkey; Type: CONSTRAINT; Schema: public; Owner: iannorden
-- Name: transactions transactions_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.transactions
@ -862,7 +557,7 @@ ALTER TABLE ONLY public.transactions
--
-- Name: watched_contracts watched_contracts_pkey; Type: CONSTRAINT; Schema: public; Owner: iannorden
-- Name: watched_contracts watched_contracts_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.watched_contracts
@ -870,42 +565,42 @@ ALTER TABLE ONLY public.watched_contracts
--
-- Name: block_id_index; Type: INDEX; Schema: public; Owner: iannorden
-- Name: block_id_index; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX block_id_index ON public.transactions USING btree (block_id);
--
-- Name: block_number_index; Type: INDEX; Schema: public; Owner: iannorden
-- Name: block_number_index; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX block_number_index ON public.blocks USING btree (number);
--
-- Name: node_id_index; Type: INDEX; Schema: public; Owner: iannorden
-- Name: node_id_index; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX node_id_index ON public.blocks USING btree (eth_node_id);
--
-- Name: tx_from_index; Type: INDEX; Schema: public; Owner: iannorden
-- Name: tx_from_index; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX tx_from_index ON public.transactions USING btree (tx_from);
--
-- Name: tx_to_index; Type: INDEX; Schema: public; Owner: iannorden
-- Name: tx_to_index; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX tx_to_index ON public.transactions USING btree (tx_to);
--
-- Name: transactions blocks_fk; Type: FK CONSTRAINT; Schema: public; Owner: iannorden
-- Name: transactions blocks_fk; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.transactions
@ -913,7 +608,7 @@ ALTER TABLE ONLY public.transactions
--
-- Name: receipts blocks_fk; Type: FK CONSTRAINT; Schema: public; Owner: iannorden
-- Name: receipts blocks_fk; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.receipts
@ -921,7 +616,7 @@ ALTER TABLE ONLY public.receipts
--
-- Name: token_supply blocks_fk; Type: FK CONSTRAINT; Schema: public; Owner: iannorden
-- Name: token_supply blocks_fk; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.token_supply
@ -929,23 +624,7 @@ ALTER TABLE ONLY public.token_supply
--
-- Name: token_balance blocks_fk; Type: FK CONSTRAINT; Schema: public; Owner: iannorden
--
ALTER TABLE ONLY public.token_balance
ADD CONSTRAINT blocks_fk FOREIGN KEY (block_id) REFERENCES public.blocks(id) ON DELETE CASCADE;
--
-- Name: token_allowance blocks_fk; Type: FK CONSTRAINT; Schema: public; Owner: iannorden
--
ALTER TABLE ONLY public.token_allowance
ADD CONSTRAINT blocks_fk FOREIGN KEY (block_id) REFERENCES public.blocks(id) ON DELETE CASCADE;
--
-- Name: headers eth_nodes_fk; Type: FK CONSTRAINT; Schema: public; Owner: iannorden
-- Name: headers eth_nodes_fk; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.headers
@ -953,7 +632,7 @@ ALTER TABLE ONLY public.headers
--
-- Name: blocks node_fk; Type: FK CONSTRAINT; Schema: public; Owner: iannorden
-- Name: blocks node_fk; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.blocks
@ -961,7 +640,7 @@ ALTER TABLE ONLY public.blocks
--
-- Name: logs receipts_fk; Type: FK CONSTRAINT; Schema: public; Owner: iannorden
-- Name: logs receipts_fk; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.logs
@ -970,5 +649,4 @@ ALTER TABLE ONLY public.logs
--
-- PostgreSQL database dump complete
--
--

4
examples/erc20_watcher/every_block/integration_test.go
View File

@ -59,8 +59,8 @@ var _ = Describe("Everyblock transformers", func() {
})
It("creates a token_supply record for each block in the given range", func() {
initializer := every_block.TokenSupplyTransformerInitializer{Config: erc20_watcher.DaiConfig}
transformer := initializer.NewTokenSupplyTransformer(db, blockChain)
initializer := every_block.ERC20TokenTransformerInitializer{Config: erc20_watcher.DaiConfig}
transformer := initializer.NewERC20TokenTransformer(db, blockChain)
transformer.Execute()
var tokenSupplyCount int

136
examples/erc20_watcher/every_block/transformer.go
View File

@ -16,7 +16,9 @@ package every_block
import (
"fmt"
"github.com/ethereum/go-ethereum/common"
"github.com/vulcanize/vulcanizedb/examples/erc20_watcher"
"github.com/vulcanize/vulcanizedb/examples/generic"
"github.com/vulcanize/vulcanizedb/libraries/shared"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
@ -27,6 +29,7 @@ import (
type Transformer struct {
Getter ERC20GetterInterface
Repository ERC20RepositoryInterface
Retriever generic.Retriever
Config erc20_watcher.ContractConfig
}
@ -34,16 +37,18 @@ func (t *Transformer) SetConfiguration(config erc20_watcher.ContractConfig) {
t.Config = config
}
type TokenSupplyTransformerInitializer struct {
type ERC20TokenTransformerInitializer struct {
Config erc20_watcher.ContractConfig
}
func (i TokenSupplyTransformerInitializer) NewTokenSupplyTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer {
getter := NewGetter(blockChain)
func (i ERC20TokenTransformerInitializer) NewERC20TokenTransformer(db *postgres.DB, blockchain core.BlockChain) shared.Transformer {
getter := NewGetter(blockchain)
repository := ERC20TokenRepository{DB: db}
retriever := generic.NewRetriever(db, i.Config.Address)
transformer := Transformer{
Getter: &getter,
Repository: &repository,
Retriever: retriever,
Config: i.Config,
}
@ -51,9 +56,14 @@ func (i TokenSupplyTransformerInitializer) NewTokenSupplyTransformer(db *postgre
}
const (
FetchingBlocksError = "Error getting missing blocks starting at block number %d: %s"
GetSupplyError = "Error getting supply for block %d: %s"
CreateSupplyError = "Error inserting token_supply for block %d: %s"
FetchingBlocksError = "Error fetching missing blocks starting at block number %d: %s"
FetchingSupplyError = "Error fetching supply for block %d: %s"
CreateSupplyError = "Error inserting token_supply for block %d: %s"
FetchingTokenAddressesError = "Error fetching token holder addresses at block %d: %s"
FetchingBalanceError = "Error fetching balance at block %d: %s"
CreateBalanceError = "Error inserting token_balance at block %d: %s"
FetchingAllowanceError = "Error fetching allowance at block %d: %s"
CreateAllowanceError = "Error inserting allowance at block %d: %s"
)
type transformerError struct {
@ -74,8 +84,8 @@ func newTransformerError(err error, blockNumber int64, msg string) error {
func (t Transformer) Execute() error {
var upperBoundBlock int64
blockChain := t.Getter.GetBlockChain()
lastBlock := blockChain.LastBlock().Int64()
blockchain := t.Getter.GetBlockChain()
lastBlock := blockchain.LastBlock().Int64()
if t.Config.LastBlock == -1 {
upperBoundBlock = lastBlock
@ -93,14 +103,14 @@ func (t Transformer) Execute() error {
}
// Fetch supply for missing blocks
log.Printf("Gets totalSupply for %d blocks", len(blocks))
log.Printf("Fetching totalSupply for %d blocks", len(blocks))
// For each block missing total supply, create supply model and feed the missing data into the repository
for _, blockNumber := range blocks {
totalSupply, err := t.Getter.GetTotalSupply(t.Config.Abi, t.Config.Address, blockNumber)
if err != nil {
return newTransformerError(err, blockNumber, GetSupplyError)
return newTransformerError(err, blockNumber, FetchingSupplyError)
}
// Create the supply model
model := createTokenSupplyModel(totalSupply, t.Config.Address, blockNumber)
@ -112,6 +122,93 @@ func (t Transformer) Execute() error {
}
}
// Balance and allowance transformations:
// Retrieve all token holder addresses for the given contract configuration
tokenHolderAddresses, err := t.Retriever.RetrieveContractAssociatedAddresses()
if err != nil {
return newTransformerError(err, t.Config.FirstBlock, FetchingTokenAddressesError)
}
// Iterate over the addresses and add their balances and allowances at each block height to the repository
for holderAddr := range tokenHolderAddresses {
// Balance transformations:
blocks, err := t.Repository.MissingBalanceBlocks(t.Config.FirstBlock, upperBoundBlock, t.Config.Address, holderAddr.String())
if err != nil {
return newTransformerError(err, t.Config.FirstBlock, FetchingBlocksError)
}
log.Printf("Fetching balances for %d blocks", len(blocks))
// For each block missing balances for the given address, create a balance model and feed the missing data into the repository
for _, blockNumber := range blocks {
hashArgs := []common.Address{holderAddr}
balanceOfArgs := make([]interface{}, len(hashArgs))
for i, s := range hashArgs {
balanceOfArgs[i] = s
}
totalSupply, err := t.Getter.GetBalance(t.Config.Abi, t.Config.Address, blockNumber, balanceOfArgs)
if err != nil {
return newTransformerError(err, blockNumber, FetchingBalanceError)
}
model := createTokenBalanceModel(totalSupply, t.Config.Address, blockNumber, holderAddr.String())
err = t.Repository.CreateBalance(model)
if err != nil {
return newTransformerError(err, blockNumber, CreateBalanceError)
}
}
// Allowance transformations:
for spenderAddr := range tokenHolderAddresses {
blocks, err := t.Repository.MissingAllowanceBlocks(t.Config.FirstBlock, upperBoundBlock, t.Config.Address, holderAddr.String(), spenderAddr.String())
if err != nil {
return newTransformerError(err, t.Config.FirstBlock, FetchingBlocksError)
}
log.Printf("Fetching allowances for %d blocks", len(blocks))
// For each block missing allowances for the given holder and spender addresses, create a allowance model and feed the missing data into the repository
for _, blockNumber := range blocks {
hashArgs := []common.Address{holderAddr, spenderAddr}
allowanceArgs := make([]interface{}, len(hashArgs))
for i, s := range hashArgs {
allowanceArgs[i] = s
}
totalSupply, err := t.Getter.GetAllowance(t.Config.Abi, t.Config.Address, blockNumber, allowanceArgs)
if err != nil {
return newTransformerError(err, blockNumber, FetchingAllowanceError)
}
model := createTokenAllowanceModel(totalSupply, t.Config.Address, blockNumber, holderAddr.String(), spenderAddr.String())
err = t.Repository.CreateAllowance(model)
if err != nil {
return newTransformerError(err, blockNumber, CreateAllowanceError)
}
}
}
}
return nil
}
@ -122,3 +219,22 @@ func createTokenSupplyModel(totalSupply big.Int, address string, blockNumber int
BlockNumber: blockNumber,
}
}
func createTokenBalanceModel(tokenBalance big.Int, tokenAddress string, blockNumber int64, tokenHolderAddress string) TokenBalance {
return TokenBalance{
Value: tokenBalance.String(),
TokenAddress: tokenAddress,
BlockNumber: blockNumber,
TokenHolderAddress: tokenHolderAddress,
}
}
func createTokenAllowanceModel(tokenBalance big.Int, tokenAddress string, blockNumber int64, tokenHolderAddress, tokenSpenderAddress string) TokenAllowance {
return TokenAllowance{
Value: tokenBalance.String(),
TokenAddress: tokenAddress,
BlockNumber: blockNumber,
TokenHolderAddress: tokenHolderAddress,
TokenSpenderAddress: tokenSpenderAddress,
}
}

8
examples/erc20_watcher/every_block/transformer_test.go
View File

@ -20,7 +20,9 @@ import (
"github.com/vulcanize/vulcanizedb/examples/constants"
"github.com/vulcanize/vulcanizedb/examples/erc20_watcher"
"github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block"
"github.com/vulcanize/vulcanizedb/examples/generic"
"github.com/vulcanize/vulcanizedb/examples/mocks"
"github.com/vulcanize/vulcanizedb/examples/test_helpers"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"math/big"
"math/rand"
@ -55,11 +57,15 @@ var _ = Describe("Everyblock transformer", func() {
getter.Fetcher.SetSupply(initialSupply)
repository = mocks.ERC20TokenRepository{}
repository.SetMissingSupplyBlocks([]int64{config.FirstBlock})
db := test_helpers.CreateNewDatabase()
rt := generic.NewRetriever(db, config.Address)
//setting the mock repository to return the first block as the missing blocks
transformer = every_block.Transformer{
Getter: &getter,
Repository: &repository,
Retriever: rt,
Config: config,
}
transformer.SetConfiguration(config)
})
@ -152,7 +158,7 @@ var _ = Describe("Everyblock transformer", func() {
err := transformer.Execute()
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring(fakes.FakeError.Error()))
Expect(err.Error()).To(ContainSubstring("getting missing blocks"))
Expect(err.Error()).To(ContainSubstring("fetching missing blocks"))
})
It("returns an error if the call to the blockChain fails", func() {

4
examples/erc20_watcher/every_block/transformers.go
View File

@ -21,8 +21,8 @@ import (
func TransformerInitializers() []shared.TransformerInitializer {
config := erc20_watcher.DaiConfig
initializer := TokenSupplyTransformerInitializer{config}
initializer := ERC20TokenTransformerInitializer{config}
return []shared.TransformerInitializer{
initializer.NewTokenSupplyTransformer,
initializer.NewERC20TokenTransformer,
}
}

126
examples/generic/retriever.go Normal file
View File

@ -0,0 +1,126 @@
// Copyright 2018 Vulcanize
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package generic
import (
"fmt"
"github.com/ethereum/go-ethereum/common"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"log"
)
// Retriever is used to iterate over addresses going into or out of a contract
// address in an attempt to generate a list of token holder addresses
type RetrieverInterface interface {
RetrieveSendingAddresses() ([]string, error)
RetrieveReceivingAddresses() ([]string, error)
RetrieveContractAssociatedAddresses() (map[common.Address]bool, error)
}
type Retriever struct {
Database *postgres.DB
ContractAddress string
}
type retrieverError struct {
err string
msg string
address string
}
// Retriever error method
func (re *retrieverError) Error() string {
return fmt.Sprintf(re.msg, re.address, re.err)
}
// Used to create a new retriever error for a given error and fetch method
func newRetrieverError(err error, msg string, address string) error {
e := retrieverError{err.Error(), msg, address}
log.Println(e.Error())
return &e
}
// Constant error definitions
const (
GetSenderError = "Error fetching addresses receiving from contract %s: %s"
GetReceiverError = "Error fetching addresses sending to contract %s: %s"
)
func NewRetriever(db *postgres.DB, address string) Retriever {
return Retriever{
Database: db,
ContractAddress: address,
}
}
func (rt Retriever) RetrieveReceivingAddresses() ([]string, error) {
receiversFromContract := make([]string, 0)
err := rt.Database.DB.Select(
&receiversFromContract,
`SELECT tx_to FROM TRANSACTIONS
WHERE tx_from = $1
LIMIT 20`,
rt.ContractAddress,
)
if err != nil {
return []string{}, newRetrieverError(err, GetReceiverError, rt.ContractAddress)
}
return receiversFromContract, err
}
func (rt Retriever) RetrieveSendingAddresses() ([]string, error) {
sendersToContract := make([]string, 0)
err := rt.Database.DB.Select(
&sendersToContract,
`SELECT tx_from FROM TRANSACTIONS
WHERE tx_to = $1
LIMIT 20`,
rt.ContractAddress,
)
if err != nil {
return []string{}, newRetrieverError(err, GetSenderError, rt.ContractAddress)
}
return sendersToContract, err
}
func (rt Retriever) RetrieveContractAssociatedAddresses() (map[common.Address]bool, error) {
sending, err := rt.RetrieveSendingAddresses()
if err != nil {
return nil, err
}
receiving, err := rt.RetrieveReceivingAddresses()
if err != nil {
return nil, err
}
contractAddresses := make(map[common.Address]bool)
for _, addr := range sending {
contractAddresses[common.HexToAddress(addr)] = true
}
for _, addr := range receiving {
contractAddresses[common.HexToAddress(addr)] = true
}
return contractAddresses, nil
}