diff --git a/cmd/resync.go b/cmd/resync.go index 83aad59a..d61feff9 100644 --- a/cmd/resync.go +++ b/cmd/resync.go @@ -67,6 +67,7 @@ func init() { resyncCmd.PersistentFlags().Int("resync-batch-size", 0, "data fetching batch size") resyncCmd.PersistentFlags().Int("resync-batch-number", 0, "how many goroutines to fetch data concurrently") resyncCmd.PersistentFlags().Bool("resync-clear-old-cache", false, "if true, clear out old data of the provided type within the resync range before resyncing") + resyncCmd.PersistentFlags().Bool("resync-reset-validation", false, "if true, reset times_validated to 0") resyncCmd.PersistentFlags().String("btc-http-path", "", "http url for bitcoin node") resyncCmd.PersistentFlags().String("btc-password", "", "password for btc node") @@ -88,6 +89,7 @@ func init() { viper.BindPFlag("resync.batchSize", resyncCmd.PersistentFlags().Lookup("resync-batch-size")) viper.BindPFlag("resync.batchNumber", resyncCmd.PersistentFlags().Lookup("resync-batch-number")) viper.BindPFlag("resync.clearOldCache", resyncCmd.PersistentFlags().Lookup("resync-clear-old-cache")) + viper.BindPFlag("resync.resetValidation", resyncCmd.PersistentFlags().Lookup("resync-reset-validation")) viper.BindPFlag("bitcoin.httpPath", resyncCmd.PersistentFlags().Lookup("btc-http-path")) viper.BindPFlag("bitcoin.pass", resyncCmd.PersistentFlags().Lookup("btc-password")) diff --git a/cmd/superNode.go b/cmd/superNode.go index b1239fd2..f9e8ae31 100644 --- a/cmd/superNode.go +++ b/cmd/superNode.go @@ -117,6 +117,7 @@ func init() { superNodeCmd.PersistentFlags().Int("supernode-frequency", 0, "how often (in seconds) the backfill process checks for gaps") superNodeCmd.PersistentFlags().Int("supernode-batch-size", 0, "data fetching batch size") superNodeCmd.PersistentFlags().Int("supernode-batch-number", 0, "how many goroutines to fetch data concurrently") + superNodeCmd.PersistentFlags().Int("supernode-validation-level", 0, "backfill will resync any data below this level") superNodeCmd.PersistentFlags().String("btc-ws-path", "", "ws url for bitcoin node") superNodeCmd.PersistentFlags().String("btc-http-path", "", "http url for bitcoin node") @@ -144,6 +145,7 @@ func init() { viper.BindPFlag("superNode.frequency", superNodeCmd.PersistentFlags().Lookup("supernode-frequency")) viper.BindPFlag("superNode.batchSize", superNodeCmd.PersistentFlags().Lookup("supernode-batch-size")) viper.BindPFlag("superNode.batchNumber", superNodeCmd.PersistentFlags().Lookup("supernode-batch-number")) + viper.BindPFlag("superNode.validationLevel", superNodeCmd.PersistentFlags().Lookup("supernode-validation-level")) viper.BindPFlag("bitcoin.wsPath", superNodeCmd.PersistentFlags().Lookup("btc-ws-path")) viper.BindPFlag("bitcoin.httpPath", superNodeCmd.PersistentFlags().Lookup("btc-http-path")) diff --git a/db/migrations/00032_update_receipt_cids.sql b/db/migrations/00032_update_receipt_cids.sql index a4ca85f3..250fb474 100644 --- a/db/migrations/00032_update_receipt_cids.sql +++ b/db/migrations/00032_update_receipt_cids.sql @@ -5,6 +5,9 @@ ADD COLUMN log_contracts VARCHAR(66)[]; ALTER TABLE eth.receipt_cids RENAME COLUMN contract TO contract_hash; +WITH uniques AS (SELECT DISTINCT ON (tx_id) * FROM eth.receipt_cids) +DELETE FROM eth.receipt_cids WHERE receipt_cids.id NOT IN (SELECT id FROM uniques); + ALTER TABLE eth.receipt_cids ADD CONSTRAINT receipt_cids_tx_id_key UNIQUE (tx_id); diff --git a/db/schema.sql b/db/schema.sql index e69de29b..d00e8e07 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -0,0 +1,1725 @@ +-- +-- PostgreSQL database dump +-- + +-- Dumped from database version 10.10 +-- Dumped by pg_dump version 10.10 + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SELECT pg_catalog.set_config('search_path', '', false); +SET check_function_bodies = false; +SET xmloption = content; +SET client_min_messages = warning; +SET row_security = off; + +-- +-- Name: btc; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA btc; + + +-- +-- Name: eth; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA eth; + + +-- +-- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog; + + +-- +-- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language'; + + +SET default_tablespace = ''; + +SET default_with_oids = false; + +-- +-- Name: header_cids; Type: TABLE; Schema: btc; Owner: - +-- + +CREATE TABLE btc.header_cids ( + id integer NOT NULL, + block_number bigint NOT NULL, + block_hash character varying(66) NOT NULL, + parent_hash character varying(66) NOT NULL, + cid text NOT NULL, + "timestamp" numeric NOT NULL, + bits bigint NOT NULL, + node_id integer NOT NULL, + times_validated integer DEFAULT 1 NOT NULL +); + + +-- +-- Name: TABLE header_cids; Type: COMMENT; Schema: btc; Owner: - +-- + +COMMENT ON TABLE btc.header_cids IS '@name BtcHeaderCids'; + + +-- +-- Name: COLUMN header_cids.node_id; Type: COMMENT; Schema: btc; Owner: - +-- + +COMMENT ON COLUMN btc.header_cids.node_id IS '@name BtcNodeID'; + + +-- +-- Name: header_cids_id_seq; Type: SEQUENCE; Schema: btc; Owner: - +-- + +CREATE SEQUENCE btc.header_cids_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: header_cids_id_seq; Type: SEQUENCE OWNED BY; Schema: btc; Owner: - +-- + +ALTER SEQUENCE btc.header_cids_id_seq OWNED BY btc.header_cids.id; + + +-- +-- Name: queue_data; Type: TABLE; Schema: btc; Owner: - +-- + +CREATE TABLE btc.queue_data ( + id integer NOT NULL, + data bytea NOT NULL, + height bigint NOT NULL +); + + +-- +-- Name: TABLE queue_data; Type: COMMENT; Schema: btc; Owner: - +-- + +COMMENT ON TABLE btc.queue_data IS '@name BtcQueueData'; + + +-- +-- Name: queue_data_id_seq; Type: SEQUENCE; Schema: btc; Owner: - +-- + +CREATE SEQUENCE btc.queue_data_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: queue_data_id_seq; Type: SEQUENCE OWNED BY; Schema: btc; Owner: - +-- + +ALTER SEQUENCE btc.queue_data_id_seq OWNED BY btc.queue_data.id; + + +-- +-- Name: transaction_cids; Type: TABLE; Schema: btc; Owner: - +-- + +CREATE TABLE btc.transaction_cids ( + id integer NOT NULL, + header_id integer NOT NULL, + index integer NOT NULL, + tx_hash character varying(66) NOT NULL, + cid text NOT NULL, + segwit boolean NOT NULL, + witness_hash character varying(66) +); + + +-- +-- Name: TABLE transaction_cids; Type: COMMENT; Schema: btc; Owner: - +-- + +COMMENT ON TABLE btc.transaction_cids IS '@name BtcTransactionCids'; + + +-- +-- Name: transaction_cids_id_seq; Type: SEQUENCE; Schema: btc; Owner: - +-- + +CREATE SEQUENCE btc.transaction_cids_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: transaction_cids_id_seq; Type: SEQUENCE OWNED BY; Schema: btc; Owner: - +-- + +ALTER SEQUENCE btc.transaction_cids_id_seq OWNED BY btc.transaction_cids.id; + + +-- +-- Name: tx_inputs; Type: TABLE; Schema: btc; Owner: - +-- + +CREATE TABLE btc.tx_inputs ( + id integer NOT NULL, + tx_id integer NOT NULL, + index integer NOT NULL, + witness character varying[], + sig_script bytea NOT NULL, + outpoint_tx_hash character varying(66) NOT NULL, + outpoint_index numeric NOT NULL +); + + +-- +-- Name: tx_inputs_id_seq; Type: SEQUENCE; Schema: btc; Owner: - +-- + +CREATE SEQUENCE btc.tx_inputs_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: tx_inputs_id_seq; Type: SEQUENCE OWNED BY; Schema: btc; Owner: - +-- + +ALTER SEQUENCE btc.tx_inputs_id_seq OWNED BY btc.tx_inputs.id; + + +-- +-- Name: tx_outputs; Type: TABLE; Schema: btc; Owner: - +-- + +CREATE TABLE btc.tx_outputs ( + id integer NOT NULL, + tx_id integer NOT NULL, + index integer NOT NULL, + value bigint NOT NULL, + pk_script bytea NOT NULL, + script_class integer NOT NULL, + addresses character varying(66)[], + required_sigs integer NOT NULL +); + + +-- +-- Name: tx_outputs_id_seq; Type: SEQUENCE; Schema: btc; Owner: - +-- + +CREATE SEQUENCE btc.tx_outputs_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: tx_outputs_id_seq; Type: SEQUENCE OWNED BY; Schema: btc; Owner: - +-- + +ALTER SEQUENCE btc.tx_outputs_id_seq OWNED BY btc.tx_outputs.id; + + +-- +-- Name: header_cids; Type: TABLE; Schema: eth; Owner: - +-- + +CREATE TABLE eth.header_cids ( + id integer NOT NULL, + block_number bigint NOT NULL, + block_hash character varying(66) NOT NULL, + parent_hash character varying(66) NOT NULL, + cid text NOT NULL, + td numeric NOT NULL, + node_id integer NOT NULL, + reward numeric NOT NULL, + state_root character varying(66), + tx_root character varying(66), + receipt_root character varying(66), + uncle_root character varying(66), + bloom bytea, + "timestamp" numeric, + times_validated integer DEFAULT 1 NOT NULL +); + + +-- +-- Name: TABLE header_cids; Type: COMMENT; Schema: eth; Owner: - +-- + +COMMENT ON TABLE eth.header_cids IS '@name EthHeaderCids'; + + +-- +-- Name: COLUMN header_cids.node_id; Type: COMMENT; Schema: eth; Owner: - +-- + +COMMENT ON COLUMN eth.header_cids.node_id IS '@name EthNodeID'; + + +-- +-- Name: header_cids_id_seq; Type: SEQUENCE; Schema: eth; Owner: - +-- + +CREATE SEQUENCE eth.header_cids_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: header_cids_id_seq; Type: SEQUENCE OWNED BY; Schema: eth; Owner: - +-- + +ALTER SEQUENCE eth.header_cids_id_seq OWNED BY eth.header_cids.id; + + +-- +-- Name: queue_data; Type: TABLE; Schema: eth; Owner: - +-- + +CREATE TABLE eth.queue_data ( + id integer NOT NULL, + data bytea NOT NULL, + height bigint NOT NULL +); + + +-- +-- Name: TABLE queue_data; Type: COMMENT; Schema: eth; Owner: - +-- + +COMMENT ON TABLE eth.queue_data IS '@name EthQueueData'; + + +-- +-- Name: queue_data_id_seq; Type: SEQUENCE; Schema: eth; Owner: - +-- + +CREATE SEQUENCE eth.queue_data_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: queue_data_id_seq; Type: SEQUENCE OWNED BY; Schema: eth; Owner: - +-- + +ALTER SEQUENCE eth.queue_data_id_seq OWNED BY eth.queue_data.id; + + +-- +-- Name: receipt_cids; Type: TABLE; Schema: eth; Owner: - +-- + +CREATE TABLE eth.receipt_cids ( + id integer NOT NULL, + tx_id integer NOT NULL, + cid text NOT NULL, + contract_hash character varying(66), + topic0s character varying(66)[], + topic1s character varying(66)[], + topic2s character varying(66)[], + topic3s character varying(66)[], + log_contracts character varying(66)[] +); + + +-- +-- Name: receipt_cids_id_seq; Type: SEQUENCE; Schema: eth; Owner: - +-- + +CREATE SEQUENCE eth.receipt_cids_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: receipt_cids_id_seq; Type: SEQUENCE OWNED BY; Schema: eth; Owner: - +-- + +ALTER SEQUENCE eth.receipt_cids_id_seq OWNED BY eth.receipt_cids.id; + + +-- +-- Name: state_accounts; Type: TABLE; Schema: eth; Owner: - +-- + +CREATE TABLE eth.state_accounts ( + id integer NOT NULL, + state_id integer NOT NULL, + balance numeric NOT NULL, + nonce integer NOT NULL, + code_hash bytea NOT NULL, + storage_root character varying(66) NOT NULL +); + + +-- +-- Name: state_accounts_id_seq; Type: SEQUENCE; Schema: eth; Owner: - +-- + +CREATE SEQUENCE eth.state_accounts_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: state_accounts_id_seq; Type: SEQUENCE OWNED BY; Schema: eth; Owner: - +-- + +ALTER SEQUENCE eth.state_accounts_id_seq OWNED BY eth.state_accounts.id; + + +-- +-- Name: state_cids; Type: TABLE; Schema: eth; Owner: - +-- + +CREATE TABLE eth.state_cids ( + id integer NOT NULL, + header_id integer NOT NULL, + state_leaf_key character varying(66), + cid text NOT NULL, + state_path bytea, + node_type integer +); + + +-- +-- Name: state_cids_id_seq; Type: SEQUENCE; Schema: eth; Owner: - +-- + +CREATE SEQUENCE eth.state_cids_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: state_cids_id_seq; Type: SEQUENCE OWNED BY; Schema: eth; Owner: - +-- + +ALTER SEQUENCE eth.state_cids_id_seq OWNED BY eth.state_cids.id; + + +-- +-- Name: storage_cids; Type: TABLE; Schema: eth; Owner: - +-- + +CREATE TABLE eth.storage_cids ( + id integer NOT NULL, + state_id integer NOT NULL, + storage_leaf_key character varying(66), + cid text NOT NULL, + storage_path bytea, + node_type integer +); + + +-- +-- Name: storage_cids_id_seq; Type: SEQUENCE; Schema: eth; Owner: - +-- + +CREATE SEQUENCE eth.storage_cids_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: storage_cids_id_seq; Type: SEQUENCE OWNED BY; Schema: eth; Owner: - +-- + +ALTER SEQUENCE eth.storage_cids_id_seq OWNED BY eth.storage_cids.id; + + +-- +-- Name: transaction_cids; Type: TABLE; Schema: eth; Owner: - +-- + +CREATE TABLE eth.transaction_cids ( + id integer NOT NULL, + header_id integer NOT NULL, + tx_hash character varying(66) NOT NULL, + index integer NOT NULL, + cid text NOT NULL, + dst character varying(66) NOT NULL, + src character varying(66) NOT NULL +); + + +-- +-- Name: TABLE transaction_cids; Type: COMMENT; Schema: eth; Owner: - +-- + +COMMENT ON TABLE eth.transaction_cids IS '@name EthTransactionCids'; + + +-- +-- Name: transaction_cids_id_seq; Type: SEQUENCE; Schema: eth; Owner: - +-- + +CREATE SEQUENCE eth.transaction_cids_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: transaction_cids_id_seq; Type: SEQUENCE OWNED BY; Schema: eth; Owner: - +-- + +ALTER SEQUENCE eth.transaction_cids_id_seq OWNED BY eth.transaction_cids.id; + + +-- +-- Name: uncle_cids; Type: TABLE; Schema: eth; Owner: - +-- + +CREATE TABLE eth.uncle_cids ( + id integer NOT NULL, + header_id integer NOT NULL, + block_hash character varying(66) NOT NULL, + parent_hash character varying(66) NOT NULL, + cid text NOT NULL, + reward numeric NOT NULL +); + + +-- +-- Name: uncle_cids_id_seq; Type: SEQUENCE; Schema: eth; Owner: - +-- + +CREATE SEQUENCE eth.uncle_cids_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: uncle_cids_id_seq; Type: SEQUENCE OWNED BY; Schema: eth; Owner: - +-- + +ALTER SEQUENCE eth.uncle_cids_id_seq OWNED BY eth.uncle_cids.id; + + +-- +-- Name: addresses; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.addresses ( + id integer NOT NULL, + address character varying(42), + hashed_address character varying(66) +); + + +-- +-- Name: addresses_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.addresses_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: addresses_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.addresses_id_seq OWNED BY public.addresses.id; + + +-- +-- Name: blocks; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.blocks ( + key text NOT NULL, + data bytea NOT NULL +); + + +-- +-- Name: checked_headers; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.checked_headers ( + id integer NOT NULL, + header_id integer NOT NULL +); + + +-- +-- Name: checked_headers_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.checked_headers_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: checked_headers_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.checked_headers_id_seq OWNED BY public.checked_headers.id; + + +-- +-- Name: goose_db_version; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.goose_db_version ( + id integer NOT NULL, + version_id bigint NOT NULL, + is_applied boolean NOT NULL, + tstamp timestamp without time zone DEFAULT now() +); + + +-- +-- Name: goose_db_version_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.goose_db_version_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: goose_db_version_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.goose_db_version_id_seq OWNED BY public.goose_db_version.id; + + +-- +-- Name: header_sync_logs; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.header_sync_logs ( + id integer NOT NULL, + header_id integer NOT NULL, + address integer NOT NULL, + topics bytea[], + data bytea, + block_number bigint, + block_hash character varying(66), + tx_hash character varying(66), + tx_index integer, + log_index integer, + raw jsonb, + transformed boolean DEFAULT false NOT NULL +); + + +-- +-- Name: header_sync_logs_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.header_sync_logs_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: header_sync_logs_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.header_sync_logs_id_seq OWNED BY public.header_sync_logs.id; + + +-- +-- Name: header_sync_receipts; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.header_sync_receipts ( + id integer NOT NULL, + transaction_id integer NOT NULL, + header_id integer NOT NULL, + contract_address_id integer NOT NULL, + cumulative_gas_used numeric, + gas_used numeric, + state_root character varying(66), + status integer, + tx_hash character varying(66), + rlp bytea +); + + +-- +-- Name: header_sync_receipts_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.header_sync_receipts_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: header_sync_receipts_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.header_sync_receipts_id_seq OWNED BY public.header_sync_receipts.id; + + +-- +-- Name: header_sync_transactions; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.header_sync_transactions ( + id integer NOT NULL, + header_id integer NOT NULL, + hash character varying(66), + gas_limit numeric, + gas_price numeric, + input_data bytea, + nonce numeric, + raw bytea, + tx_from character varying(44), + tx_index integer, + tx_to character varying(44), + value numeric +); + + +-- +-- Name: header_sync_transactions_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.header_sync_transactions_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: header_sync_transactions_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.header_sync_transactions_id_seq OWNED BY public.header_sync_transactions.id; + + +-- +-- Name: headers; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.headers ( + id integer NOT NULL, + hash character varying(66), + block_number bigint, + raw jsonb, + block_timestamp numeric, + check_count integer DEFAULT 0 NOT NULL, + node_id integer NOT NULL, + eth_node_fingerprint character varying(128) +); + + +-- +-- Name: TABLE headers; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON TABLE public.headers IS '@name EthHeaders'; + + +-- +-- Name: COLUMN headers.node_id; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.headers.node_id IS '@name EthNodeID'; + + +-- +-- Name: headers_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.headers_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: headers_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.headers_id_seq OWNED BY public.headers.id; + + +-- +-- Name: nodes; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.nodes ( + id integer NOT NULL, + client_name character varying, + genesis_block character varying(66), + network_id character varying, + node_id character varying(128) +); + + +-- +-- Name: TABLE nodes; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON TABLE public.nodes IS '@name NodeInfo'; + + +-- +-- Name: COLUMN nodes.node_id; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.nodes.node_id IS '@name ChainNodeID'; + + +-- +-- Name: nodes_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.nodes_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: nodes_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.nodes_id_seq OWNED BY public.nodes.id; + + +-- +-- Name: queued_storage; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.queued_storage ( + id integer NOT NULL, + diff_id bigint NOT NULL +); + + +-- +-- Name: queued_storage_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.queued_storage_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: queued_storage_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.queued_storage_id_seq OWNED BY public.queued_storage.id; + + +-- +-- Name: storage_diff; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.storage_diff ( + id integer NOT NULL, + block_height bigint, + block_hash bytea, + hashed_address bytea, + storage_key bytea, + storage_value bytea +); + + +-- +-- Name: storage_diff_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.storage_diff_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: storage_diff_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.storage_diff_id_seq OWNED BY public.storage_diff.id; + + +-- +-- Name: watched_logs; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.watched_logs ( + id integer NOT NULL, + contract_address character varying(42), + topic_zero character varying(66) +); + + +-- +-- Name: watched_logs_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.watched_logs_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: watched_logs_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.watched_logs_id_seq OWNED BY public.watched_logs.id; + + +-- +-- Name: header_cids id; Type: DEFAULT; Schema: btc; Owner: - +-- + +ALTER TABLE ONLY btc.header_cids ALTER COLUMN id SET DEFAULT nextval('btc.header_cids_id_seq'::regclass); + + +-- +-- Name: queue_data id; Type: DEFAULT; Schema: btc; Owner: - +-- + +ALTER TABLE ONLY btc.queue_data ALTER COLUMN id SET DEFAULT nextval('btc.queue_data_id_seq'::regclass); + + +-- +-- Name: transaction_cids id; Type: DEFAULT; Schema: btc; Owner: - +-- + +ALTER TABLE ONLY btc.transaction_cids ALTER COLUMN id SET DEFAULT nextval('btc.transaction_cids_id_seq'::regclass); + + +-- +-- Name: tx_inputs id; Type: DEFAULT; Schema: btc; Owner: - +-- + +ALTER TABLE ONLY btc.tx_inputs ALTER COLUMN id SET DEFAULT nextval('btc.tx_inputs_id_seq'::regclass); + + +-- +-- Name: tx_outputs id; Type: DEFAULT; Schema: btc; Owner: - +-- + +ALTER TABLE ONLY btc.tx_outputs ALTER COLUMN id SET DEFAULT nextval('btc.tx_outputs_id_seq'::regclass); + + +-- +-- Name: header_cids id; Type: DEFAULT; Schema: eth; Owner: - +-- + +ALTER TABLE ONLY eth.header_cids ALTER COLUMN id SET DEFAULT nextval('eth.header_cids_id_seq'::regclass); + + +-- +-- Name: queue_data id; Type: DEFAULT; Schema: eth; Owner: - +-- + +ALTER TABLE ONLY eth.queue_data ALTER COLUMN id SET DEFAULT nextval('eth.queue_data_id_seq'::regclass); + + +-- +-- Name: receipt_cids id; Type: DEFAULT; Schema: eth; Owner: - +-- + +ALTER TABLE ONLY eth.receipt_cids ALTER COLUMN id SET DEFAULT nextval('eth.receipt_cids_id_seq'::regclass); + + +-- +-- Name: state_accounts id; Type: DEFAULT; Schema: eth; Owner: - +-- + +ALTER TABLE ONLY eth.state_accounts ALTER COLUMN id SET DEFAULT nextval('eth.state_accounts_id_seq'::regclass); + + +-- +-- Name: state_cids id; Type: DEFAULT; Schema: eth; Owner: - +-- + +ALTER TABLE ONLY eth.state_cids ALTER COLUMN id SET DEFAULT nextval('eth.state_cids_id_seq'::regclass); + + +-- +-- Name: storage_cids id; Type: DEFAULT; Schema: eth; Owner: - +-- + +ALTER TABLE ONLY eth.storage_cids ALTER COLUMN id SET DEFAULT nextval('eth.storage_cids_id_seq'::regclass); + + +-- +-- Name: transaction_cids id; Type: DEFAULT; Schema: eth; Owner: - +-- + +ALTER TABLE ONLY eth.transaction_cids ALTER COLUMN id SET DEFAULT nextval('eth.transaction_cids_id_seq'::regclass); + + +-- +-- Name: uncle_cids id; Type: DEFAULT; Schema: eth; Owner: - +-- + +ALTER TABLE ONLY eth.uncle_cids ALTER COLUMN id SET DEFAULT nextval('eth.uncle_cids_id_seq'::regclass); + + +-- +-- Name: addresses id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.addresses ALTER COLUMN id SET DEFAULT nextval('public.addresses_id_seq'::regclass); + + +-- +-- Name: checked_headers id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.checked_headers ALTER COLUMN id SET DEFAULT nextval('public.checked_headers_id_seq'::regclass); + + +-- +-- Name: goose_db_version id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.goose_db_version ALTER COLUMN id SET DEFAULT nextval('public.goose_db_version_id_seq'::regclass); + + +-- +-- Name: header_sync_logs id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.header_sync_logs ALTER COLUMN id SET DEFAULT nextval('public.header_sync_logs_id_seq'::regclass); + + +-- +-- Name: header_sync_receipts id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.header_sync_receipts ALTER COLUMN id SET DEFAULT nextval('public.header_sync_receipts_id_seq'::regclass); + + +-- +-- Name: header_sync_transactions id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.header_sync_transactions ALTER COLUMN id SET DEFAULT nextval('public.header_sync_transactions_id_seq'::regclass); + + +-- +-- Name: headers id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.headers ALTER COLUMN id SET DEFAULT nextval('public.headers_id_seq'::regclass); + + +-- +-- Name: nodes id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.nodes ALTER COLUMN id SET DEFAULT nextval('public.nodes_id_seq'::regclass); + + +-- +-- Name: queued_storage id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.queued_storage ALTER COLUMN id SET DEFAULT nextval('public.queued_storage_id_seq'::regclass); + + +-- +-- Name: storage_diff id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.storage_diff ALTER COLUMN id SET DEFAULT nextval('public.storage_diff_id_seq'::regclass); + + +-- +-- Name: watched_logs id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.watched_logs ALTER COLUMN id SET DEFAULT nextval('public.watched_logs_id_seq'::regclass); + + +-- +-- Name: header_cids header_cids_block_number_block_hash_key; Type: CONSTRAINT; Schema: btc; Owner: - +-- + +ALTER TABLE ONLY btc.header_cids + ADD CONSTRAINT header_cids_block_number_block_hash_key UNIQUE (block_number, block_hash); + + +-- +-- Name: header_cids header_cids_pkey; Type: CONSTRAINT; Schema: btc; Owner: - +-- + +ALTER TABLE ONLY btc.header_cids + ADD CONSTRAINT header_cids_pkey PRIMARY KEY (id); + + +-- +-- Name: queue_data queue_data_height_key; Type: CONSTRAINT; Schema: btc; Owner: - +-- + +ALTER TABLE ONLY btc.queue_data + ADD CONSTRAINT queue_data_height_key UNIQUE (height); + + +-- +-- Name: queue_data queue_data_pkey; Type: CONSTRAINT; Schema: btc; Owner: - +-- + +ALTER TABLE ONLY btc.queue_data + ADD CONSTRAINT queue_data_pkey PRIMARY KEY (id); + + +-- +-- Name: transaction_cids transaction_cids_pkey; Type: CONSTRAINT; Schema: btc; Owner: - +-- + +ALTER TABLE ONLY btc.transaction_cids + ADD CONSTRAINT transaction_cids_pkey PRIMARY KEY (id); + + +-- +-- Name: transaction_cids transaction_cids_tx_hash_key; Type: CONSTRAINT; Schema: btc; Owner: - +-- + +ALTER TABLE ONLY btc.transaction_cids + ADD CONSTRAINT transaction_cids_tx_hash_key UNIQUE (tx_hash); + + +-- +-- Name: tx_inputs tx_inputs_pkey; Type: CONSTRAINT; Schema: btc; Owner: - +-- + +ALTER TABLE ONLY btc.tx_inputs + ADD CONSTRAINT tx_inputs_pkey PRIMARY KEY (id); + + +-- +-- Name: tx_inputs tx_inputs_tx_id_index_key; Type: CONSTRAINT; Schema: btc; Owner: - +-- + +ALTER TABLE ONLY btc.tx_inputs + ADD CONSTRAINT tx_inputs_tx_id_index_key UNIQUE (tx_id, index); + + +-- +-- Name: tx_outputs tx_outputs_pkey; Type: CONSTRAINT; Schema: btc; Owner: - +-- + +ALTER TABLE ONLY btc.tx_outputs + ADD CONSTRAINT tx_outputs_pkey PRIMARY KEY (id); + + +-- +-- Name: tx_outputs tx_outputs_tx_id_index_key; Type: CONSTRAINT; Schema: btc; Owner: - +-- + +ALTER TABLE ONLY btc.tx_outputs + ADD CONSTRAINT tx_outputs_tx_id_index_key UNIQUE (tx_id, index); + + +-- +-- Name: header_cids header_cids_block_number_block_hash_key; Type: CONSTRAINT; Schema: eth; Owner: - +-- + +ALTER TABLE ONLY eth.header_cids + ADD CONSTRAINT header_cids_block_number_block_hash_key UNIQUE (block_number, block_hash); + + +-- +-- Name: header_cids header_cids_pkey; Type: CONSTRAINT; Schema: eth; Owner: - +-- + +ALTER TABLE ONLY eth.header_cids + ADD CONSTRAINT header_cids_pkey PRIMARY KEY (id); + + +-- +-- Name: queue_data queue_data_height_key; Type: CONSTRAINT; Schema: eth; Owner: - +-- + +ALTER TABLE ONLY eth.queue_data + ADD CONSTRAINT queue_data_height_key UNIQUE (height); + + +-- +-- Name: queue_data queue_data_pkey; Type: CONSTRAINT; Schema: eth; Owner: - +-- + +ALTER TABLE ONLY eth.queue_data + ADD CONSTRAINT queue_data_pkey PRIMARY KEY (id); + + +-- +-- Name: receipt_cids receipt_cids_pkey; Type: CONSTRAINT; Schema: eth; Owner: - +-- + +ALTER TABLE ONLY eth.receipt_cids + ADD CONSTRAINT receipt_cids_pkey PRIMARY KEY (id); + + +-- +-- Name: receipt_cids receipt_cids_tx_id_key; Type: CONSTRAINT; Schema: eth; Owner: - +-- + +ALTER TABLE ONLY eth.receipt_cids + ADD CONSTRAINT receipt_cids_tx_id_key UNIQUE (tx_id); + + +-- +-- Name: state_accounts state_accounts_pkey; Type: CONSTRAINT; Schema: eth; Owner: - +-- + +ALTER TABLE ONLY eth.state_accounts + ADD CONSTRAINT state_accounts_pkey PRIMARY KEY (id); + + +-- +-- Name: state_accounts state_accounts_state_id_key; Type: CONSTRAINT; Schema: eth; Owner: - +-- + +ALTER TABLE ONLY eth.state_accounts + ADD CONSTRAINT state_accounts_state_id_key UNIQUE (state_id); + + +-- +-- Name: state_cids state_cids_header_id_state_path_key; Type: CONSTRAINT; Schema: eth; Owner: - +-- + +ALTER TABLE ONLY eth.state_cids + ADD CONSTRAINT state_cids_header_id_state_path_key UNIQUE (header_id, state_path); + + +-- +-- Name: state_cids state_cids_pkey; Type: CONSTRAINT; Schema: eth; Owner: - +-- + +ALTER TABLE ONLY eth.state_cids + ADD CONSTRAINT state_cids_pkey PRIMARY KEY (id); + + +-- +-- Name: storage_cids storage_cids_pkey; Type: CONSTRAINT; Schema: eth; Owner: - +-- + +ALTER TABLE ONLY eth.storage_cids + ADD CONSTRAINT storage_cids_pkey PRIMARY KEY (id); + + +-- +-- Name: storage_cids storage_cids_state_id_storage_path_key; Type: CONSTRAINT; Schema: eth; Owner: - +-- + +ALTER TABLE ONLY eth.storage_cids + ADD CONSTRAINT storage_cids_state_id_storage_path_key UNIQUE (state_id, storage_path); + + +-- +-- Name: transaction_cids transaction_cids_header_id_tx_hash_key; Type: CONSTRAINT; Schema: eth; Owner: - +-- + +ALTER TABLE ONLY eth.transaction_cids + ADD CONSTRAINT transaction_cids_header_id_tx_hash_key UNIQUE (header_id, tx_hash); + + +-- +-- Name: transaction_cids transaction_cids_pkey; Type: CONSTRAINT; Schema: eth; Owner: - +-- + +ALTER TABLE ONLY eth.transaction_cids + ADD CONSTRAINT transaction_cids_pkey PRIMARY KEY (id); + + +-- +-- Name: uncle_cids uncle_cids_header_id_block_hash_key; Type: CONSTRAINT; Schema: eth; Owner: - +-- + +ALTER TABLE ONLY eth.uncle_cids + ADD CONSTRAINT uncle_cids_header_id_block_hash_key UNIQUE (header_id, block_hash); + + +-- +-- Name: uncle_cids uncle_cids_pkey; Type: CONSTRAINT; Schema: eth; Owner: - +-- + +ALTER TABLE ONLY eth.uncle_cids + ADD CONSTRAINT uncle_cids_pkey PRIMARY KEY (id); + + +-- +-- Name: addresses addresses_address_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.addresses + ADD CONSTRAINT addresses_address_key UNIQUE (address); + + +-- +-- Name: addresses addresses_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.addresses + ADD CONSTRAINT addresses_pkey PRIMARY KEY (id); + + +-- +-- Name: blocks blocks_key_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.blocks + ADD CONSTRAINT blocks_key_key UNIQUE (key); + + +-- +-- Name: checked_headers checked_headers_header_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.checked_headers + ADD CONSTRAINT checked_headers_header_id_key UNIQUE (header_id); + + +-- +-- Name: checked_headers checked_headers_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.checked_headers + ADD CONSTRAINT checked_headers_pkey PRIMARY KEY (id); + + +-- +-- Name: goose_db_version goose_db_version_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.goose_db_version + ADD CONSTRAINT goose_db_version_pkey PRIMARY KEY (id); + + +-- +-- Name: header_sync_logs header_sync_logs_header_id_tx_index_log_index_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.header_sync_logs + ADD CONSTRAINT header_sync_logs_header_id_tx_index_log_index_key UNIQUE (header_id, tx_index, log_index); + + +-- +-- Name: header_sync_logs header_sync_logs_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.header_sync_logs + ADD CONSTRAINT header_sync_logs_pkey PRIMARY KEY (id); + + +-- +-- Name: header_sync_receipts header_sync_receipts_header_id_transaction_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.header_sync_receipts + ADD CONSTRAINT header_sync_receipts_header_id_transaction_id_key UNIQUE (header_id, transaction_id); + + +-- +-- Name: header_sync_receipts header_sync_receipts_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.header_sync_receipts + ADD CONSTRAINT header_sync_receipts_pkey PRIMARY KEY (id); + + +-- +-- Name: header_sync_transactions header_sync_transactions_header_id_hash_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.header_sync_transactions + ADD CONSTRAINT header_sync_transactions_header_id_hash_key UNIQUE (header_id, hash); + + +-- +-- Name: header_sync_transactions header_sync_transactions_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.header_sync_transactions + ADD CONSTRAINT header_sync_transactions_pkey PRIMARY KEY (id); + + +-- +-- Name: headers headers_block_number_hash_eth_node_fingerprint_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.headers + ADD CONSTRAINT headers_block_number_hash_eth_node_fingerprint_key UNIQUE (block_number, hash, eth_node_fingerprint); + + +-- +-- Name: headers headers_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.headers + ADD CONSTRAINT headers_pkey PRIMARY KEY (id); + + +-- +-- Name: nodes node_uc; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.nodes + ADD CONSTRAINT node_uc UNIQUE (genesis_block, network_id, node_id); + + +-- +-- Name: nodes nodes_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.nodes + ADD CONSTRAINT nodes_pkey PRIMARY KEY (id); + + +-- +-- Name: queued_storage queued_storage_diff_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.queued_storage + ADD CONSTRAINT queued_storage_diff_id_key UNIQUE (diff_id); + + +-- +-- Name: queued_storage queued_storage_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.queued_storage + ADD CONSTRAINT queued_storage_pkey PRIMARY KEY (id); + + +-- +-- Name: storage_diff storage_diff_block_height_block_hash_hashed_address_storage_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.storage_diff + ADD CONSTRAINT storage_diff_block_height_block_hash_hashed_address_storage_key UNIQUE (block_height, block_hash, hashed_address, storage_key, storage_value); + + +-- +-- Name: storage_diff storage_diff_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.storage_diff + ADD CONSTRAINT storage_diff_pkey PRIMARY KEY (id); + + +-- +-- Name: watched_logs watched_logs_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.watched_logs + ADD CONSTRAINT watched_logs_pkey PRIMARY KEY (id); + + +-- +-- Name: header_sync_receipts_header; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX header_sync_receipts_header ON public.header_sync_receipts USING btree (header_id); + + +-- +-- Name: header_sync_receipts_transaction; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX header_sync_receipts_transaction ON public.header_sync_receipts USING btree (transaction_id); + + +-- +-- Name: header_sync_transactions_header; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX header_sync_transactions_header ON public.header_sync_transactions USING btree (header_id); + + +-- +-- Name: header_sync_transactions_tx_index; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX header_sync_transactions_tx_index ON public.header_sync_transactions USING btree (tx_index); + + +-- +-- Name: headers_block_number; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX headers_block_number ON public.headers USING btree (block_number); + + +-- +-- Name: headers_block_timestamp; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX headers_block_timestamp ON public.headers USING btree (block_timestamp); + + +-- +-- Name: header_cids header_cids_node_id_fkey; Type: FK CONSTRAINT; Schema: btc; Owner: - +-- + +ALTER TABLE ONLY btc.header_cids + ADD CONSTRAINT header_cids_node_id_fkey FOREIGN KEY (node_id) REFERENCES public.nodes(id) ON DELETE CASCADE; + + +-- +-- Name: transaction_cids transaction_cids_header_id_fkey; Type: FK CONSTRAINT; Schema: btc; Owner: - +-- + +ALTER TABLE ONLY btc.transaction_cids + ADD CONSTRAINT transaction_cids_header_id_fkey FOREIGN KEY (header_id) REFERENCES btc.header_cids(id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; + + +-- +-- Name: tx_inputs tx_inputs_tx_id_fkey; Type: FK CONSTRAINT; Schema: btc; Owner: - +-- + +ALTER TABLE ONLY btc.tx_inputs + ADD CONSTRAINT tx_inputs_tx_id_fkey FOREIGN KEY (tx_id) REFERENCES btc.transaction_cids(id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; + + +-- +-- Name: tx_outputs tx_outputs_tx_id_fkey; Type: FK CONSTRAINT; Schema: btc; Owner: - +-- + +ALTER TABLE ONLY btc.tx_outputs + ADD CONSTRAINT tx_outputs_tx_id_fkey FOREIGN KEY (tx_id) REFERENCES btc.transaction_cids(id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; + + +-- +-- Name: header_cids header_cids_node_id_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: - +-- + +ALTER TABLE ONLY eth.header_cids + ADD CONSTRAINT header_cids_node_id_fkey FOREIGN KEY (node_id) REFERENCES public.nodes(id) ON DELETE CASCADE; + + +-- +-- Name: receipt_cids receipt_cids_tx_id_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: - +-- + +ALTER TABLE ONLY eth.receipt_cids + ADD CONSTRAINT receipt_cids_tx_id_fkey FOREIGN KEY (tx_id) REFERENCES eth.transaction_cids(id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; + + +-- +-- Name: state_accounts state_accounts_state_id_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: - +-- + +ALTER TABLE ONLY eth.state_accounts + ADD CONSTRAINT state_accounts_state_id_fkey FOREIGN KEY (state_id) REFERENCES eth.state_cids(id) ON DELETE CASCADE; + + +-- +-- Name: state_cids state_cids_header_id_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: - +-- + +ALTER TABLE ONLY eth.state_cids + ADD CONSTRAINT state_cids_header_id_fkey FOREIGN KEY (header_id) REFERENCES eth.header_cids(id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; + + +-- +-- Name: storage_cids storage_cids_state_id_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: - +-- + +ALTER TABLE ONLY eth.storage_cids + ADD CONSTRAINT storage_cids_state_id_fkey FOREIGN KEY (state_id) REFERENCES eth.state_cids(id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; + + +-- +-- Name: transaction_cids transaction_cids_header_id_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: - +-- + +ALTER TABLE ONLY eth.transaction_cids + ADD CONSTRAINT transaction_cids_header_id_fkey FOREIGN KEY (header_id) REFERENCES eth.header_cids(id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; + + +-- +-- Name: uncle_cids uncle_cids_header_id_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: - +-- + +ALTER TABLE ONLY eth.uncle_cids + ADD CONSTRAINT uncle_cids_header_id_fkey FOREIGN KEY (header_id) REFERENCES eth.header_cids(id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; + + +-- +-- Name: checked_headers checked_headers_header_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.checked_headers + ADD CONSTRAINT checked_headers_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE; + + +-- +-- Name: header_sync_logs header_sync_logs_address_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.header_sync_logs + ADD CONSTRAINT header_sync_logs_address_fkey FOREIGN KEY (address) REFERENCES public.addresses(id) ON DELETE CASCADE; + + +-- +-- Name: header_sync_logs header_sync_logs_header_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.header_sync_logs + ADD CONSTRAINT header_sync_logs_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE; + + +-- +-- Name: header_sync_receipts header_sync_receipts_contract_address_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.header_sync_receipts + ADD CONSTRAINT header_sync_receipts_contract_address_id_fkey FOREIGN KEY (contract_address_id) REFERENCES public.addresses(id) ON DELETE CASCADE; + + +-- +-- Name: header_sync_receipts header_sync_receipts_header_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.header_sync_receipts + ADD CONSTRAINT header_sync_receipts_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE; + + +-- +-- Name: header_sync_receipts header_sync_receipts_transaction_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.header_sync_receipts + ADD CONSTRAINT header_sync_receipts_transaction_id_fkey FOREIGN KEY (transaction_id) REFERENCES public.header_sync_transactions(id) ON DELETE CASCADE; + + +-- +-- Name: header_sync_transactions header_sync_transactions_header_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.header_sync_transactions + ADD CONSTRAINT header_sync_transactions_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE; + + +-- +-- Name: headers headers_node_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.headers + ADD CONSTRAINT headers_node_id_fkey FOREIGN KEY (node_id) REFERENCES public.nodes(id) ON DELETE CASCADE; + + +-- +-- Name: queued_storage queued_storage_diff_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.queued_storage + ADD CONSTRAINT queued_storage_diff_id_fkey FOREIGN KEY (diff_id) REFERENCES public.storage_diff(id); + + +-- +-- PostgreSQL database dump complete +-- + diff --git a/environments/superNodeBTC.toml b/environments/superNodeBTC.toml index d6ee3f6f..2f114137 100644 --- a/environments/superNodeBTC.toml +++ b/environments/superNodeBTC.toml @@ -16,7 +16,7 @@ batchSize = 1 # $RESYNC_BATCH_SIZE batchNumber = 50 # $RESYNC_BATCH_NUMBER clearOldCache = false # $RESYNC_CLEAR_OLD_CACHE - resetValidation = false # $RESYNC_RESET_VALIDATION + resetValidation = true # $RESYNC_RESET_VALIDATION [superNode] chain = "bitcoin" # $SUPERNODE_CHAIN @@ -30,6 +30,7 @@ frequency = 45 # $SUPERNODE_FREQUENCY batchSize = 1 # $SUPERNODE_BATCH_SIZE batchNumber = 50 # $SUPERNODE_BATCH_NUMBER + validationLevel = 1 # $SUPERNODE_VALIDATION_LEVEL [bitcoin] wsPath = "127.0.0.1:8332" # $BTC_WS_PATH diff --git a/environments/superNodeETH.toml b/environments/superNodeETH.toml index 65c339ec..3fc68fe1 100644 --- a/environments/superNodeETH.toml +++ b/environments/superNodeETH.toml @@ -15,7 +15,7 @@ stop = 0 # $RESYNC_STOP batchSize = 5 # $RESYNC_BATCH_SIZE batchNumber = 50 # $RESYNC_BATCH_NUMBER - clearOldCache = true # $RESYNC_CLEAR_OLD_CACHE + clearOldCache = false # $RESYNC_CLEAR_OLD_CACHE resetValidation = true # $RESYNC_RESET_VALIDATION [superNode] @@ -30,6 +30,7 @@ frequency = 15 # $SUPERNODE_FREQUENCY batchSize = 5 # $SUPERNODE_BATCH_SIZE batchNumber = 50 # $SUPERNODE_BATCH_NUMBER + validationLevel = 1 # $SUPERNODE_VALIDATION_LEVEL [ethereum] wsPath = "127.0.0.1:8546" # $ETH_WS_PATH diff --git a/libraries/shared/storage/utils/bins_test.go b/libraries/shared/storage/utils/bins_test.go index 84f9405e..19985b3b 100644 --- a/libraries/shared/storage/utils/bins_test.go +++ b/libraries/shared/storage/utils/bins_test.go @@ -44,6 +44,14 @@ var _ = Describe("GetBlockHeightBins", func() { Expect(err).ToNot(HaveOccurred()) Expect(len(blockRangeBins)).To(Equal(100)) Expect(blockRangeBins[99]).To(Equal(lastBin)) + + startingBlock = 1 + endingBlock = 1 + batchSize = 100 + blockRangeBins, err = utils.GetBlockHeightBins(startingBlock, endingBlock, batchSize) + Expect(err).ToNot(HaveOccurred()) + Expect(len(blockRangeBins)).To(Equal(1)) + Expect(blockRangeBins[0]).To(Equal([]uint64{1})) }) It("throws an error if the starting block is higher than the ending block", func() { diff --git a/pkg/super_node/backfiller.go b/pkg/super_node/backfiller.go index 247a30f4..29059253 100644 --- a/pkg/super_node/backfiller.go +++ b/pkg/super_node/backfiller.go @@ -63,6 +63,8 @@ type BackFillService struct { QuitChan chan bool // Chain type chain shared.ChainType + // Headers with times_validated lower than this will be resynced + validationLevel int } // NewBackFillService returns a new BackFillInterface @@ -107,6 +109,7 @@ func NewBackFillService(settings *Config, screenAndServeChan chan shared.Convert ScreenAndServeChan: screenAndServeChan, QuitChan: settings.Quit, chain: settings.Chain, + validationLevel: settings.ValidationLevel, }, nil } @@ -135,7 +138,7 @@ func (bfs *BackFillService) FillGapsInSuperNode(wg *sync.WaitGroup) { log.Error(err) } } - gaps, err := bfs.Retriever.RetrieveGapsInData() + gaps, err := bfs.Retriever.RetrieveGapsInData(bfs.validationLevel) if err != nil { log.Errorf("super node db backfill RetrieveGapsInData error for chain %s: %v", bfs.chain.String(), err) continue @@ -158,7 +161,6 @@ func (bfs *BackFillService) backFill(startingBlock, endingBlock uint64) error { if endingBlock < startingBlock { return fmt.Errorf("super node %s db backfill: ending block number needs to be greater than starting block number", bfs.chain.String()) } - // // break the range up into bins of smaller ranges blockRangeBins, err := utils.GetBlockHeightBins(startingBlock, endingBlock, bfs.BatchSize) if err != nil { diff --git a/pkg/super_node/btc/retriever.go b/pkg/super_node/btc/retriever.go index 864812bd..4a08cbfd 100644 --- a/pkg/super_node/btc/retriever.go +++ b/pkg/super_node/btc/retriever.go @@ -161,7 +161,7 @@ func (ecr *CIDRetriever) RetrieveTxCIDs(tx *sqlx.Tx, txFilter TxFilter, headerID } // RetrieveGapsInData is used to find the the block numbers at which we are missing data in the db -func (ecr *CIDRetriever) RetrieveGapsInData() ([]shared.Gap, error) { +func (ecr *CIDRetriever) RetrieveGapsInData(validationLevel int) ([]shared.Gap, error) { pgStr := `SELECT header_cids.block_number + 1 AS start, min(fr.block_number) - 1 AS stop FROM btc.header_cids LEFT JOIN btc.header_cids r on btc.header_cids.block_number = r.block_number - 1 LEFT JOIN btc.header_cids fr on btc.header_cids.block_number < fr.block_number @@ -171,18 +171,45 @@ func (ecr *CIDRetriever) RetrieveGapsInData() ([]shared.Gap, error) { Start uint64 `db:"start"` Stop uint64 `db:"stop"` }, 0) - err := ecr.db.Select(&results, pgStr) - if err != nil { + if err := ecr.db.Select(&results, pgStr); err != nil { return nil, err } - gaps := make([]shared.Gap, len(results)) + emptyGaps := make([]shared.Gap, len(results)) for i, res := range results { - gaps[i] = shared.Gap{ + emptyGaps[i] = shared.Gap{ Start: res.Start, Stop: res.Stop, } } - return gaps, nil + + // Find sections of blocks where we are below the validation level + // There will be no overlap between these "gaps" and the ones above + pgStr = `SELECT block_number FROM btc.header_cids + WHERE times_validated < $1 + ORDER BY block_number` + var heights []uint64 + if err := ecr.db.Select(&heights, pgStr, validationLevel); err != nil { + return nil, err + } + if len(heights) == 0 { + return emptyGaps, nil + } + validationGaps := make([]shared.Gap, 0) + start := heights[0] + lastHeight := start + for _, height := range heights[1:] { + if height == lastHeight+1 { + lastHeight = height + continue + } + validationGaps = append(validationGaps, shared.Gap{ + Start: start, + Stop: lastHeight, + }) + start = height + lastHeight = start + } + return append(emptyGaps, validationGaps...), nil } // RetrieveBlockByHash returns all of the CIDs needed to compose an entire block, for a given block hash diff --git a/pkg/super_node/config.go b/pkg/super_node/config.go index e8afe345..5614c1e2 100644 --- a/pkg/super_node/config.go +++ b/pkg/super_node/config.go @@ -33,17 +33,18 @@ import ( // Env variables const ( - SUPERNODE_CHAIN = "SUPERNODE_CHAIN" - SUPERNODE_SYNC = "SUPERNODE_SYNC" - SUPERNODE_WORKERS = "SUPERNODE_WORKERS" - SUPERNODE_SERVER = "SUPERNODE_SERVER" - SUPERNODE_WS_PATH = "SUPERNODE_WS_PATH" - SUPERNODE_IPC_PATH = "SUPERNODE_IPC_PATH" - SUPERNODE_HTTP_PATH = "SUPERNODE_HTTP_PATH" - SUPERNODE_BACKFILL = "SUPERNODE_BACKFILL" - SUPERNODE_FREQUENCY = "SUPERNODE_FREQUENCY" - SUPERNODE_BATCH_SIZE = "SUPERNODE_BATCH_SIZE" - SUPERNODE_BATCH_NUMBER = "SUPERNODE_BATCH_NUMBER" + SUPERNODE_CHAIN = "SUPERNODE_CHAIN" + SUPERNODE_SYNC = "SUPERNODE_SYNC" + SUPERNODE_WORKERS = "SUPERNODE_WORKERS" + SUPERNODE_SERVER = "SUPERNODE_SERVER" + SUPERNODE_WS_PATH = "SUPERNODE_WS_PATH" + SUPERNODE_IPC_PATH = "SUPERNODE_IPC_PATH" + SUPERNODE_HTTP_PATH = "SUPERNODE_HTTP_PATH" + SUPERNODE_BACKFILL = "SUPERNODE_BACKFILL" + SUPERNODE_FREQUENCY = "SUPERNODE_FREQUENCY" + SUPERNODE_BATCH_SIZE = "SUPERNODE_BATCH_SIZE" + SUPERNODE_BATCH_NUMBER = "SUPERNODE_BATCH_NUMBER" + SUPERNODE_VALIDATION_LEVEL = "SUPERNODE_VALIDATION_LEVEL" ) // Config struct @@ -65,11 +66,12 @@ type Config struct { WSClient interface{} NodeInfo core.Node // Backfiller params - BackFill bool - HTTPClient interface{} - Frequency time.Duration - BatchSize uint64 - BatchNumber uint64 + BackFill bool + HTTPClient interface{} + Frequency time.Duration + BatchSize uint64 + BatchNumber uint64 + ValidationLevel int } // NewSuperNodeConfig is used to initialize a SuperNode config from a .toml file @@ -167,6 +169,7 @@ func (c *Config) BackFillFields() error { viper.BindEnv("superNode.frequency", SUPERNODE_FREQUENCY) viper.BindEnv("superNode.batchSize", SUPERNODE_BATCH_SIZE) viper.BindEnv("superNode.batchNumber", SUPERNODE_BATCH_NUMBER) + viper.BindEnv("superNode.validationLevel", SUPERNODE_VALIDATION_LEVEL) switch c.Chain { case shared.Ethereum: @@ -190,5 +193,6 @@ func (c *Config) BackFillFields() error { c.Frequency = frequency c.BatchSize = uint64(viper.GetInt64("superNode.batchSize")) c.BatchNumber = uint64(viper.GetInt64("superNode.batchNumber")) + c.ValidationLevel = viper.GetInt("superNode.validationLevel") return nil } diff --git a/pkg/super_node/eth/retriever.go b/pkg/super_node/eth/retriever.go index bfe3f5d1..0251888d 100644 --- a/pkg/super_node/eth/retriever.go +++ b/pkg/super_node/eth/retriever.go @@ -445,7 +445,8 @@ func (ecr *CIDRetriever) RetrieveStorageCIDs(tx *sqlx.Tx, storageFilter StorageF } // RetrieveGapsInData is used to find the the block numbers at which we are missing data in the db -func (ecr *CIDRetriever) RetrieveGapsInData() ([]shared.Gap, error) { +// it finds the union of heights where no data exists and where the times_validated is lower than the validation level +func (ecr *CIDRetriever) RetrieveGapsInData(validationLevel int) ([]shared.Gap, error) { pgStr := `SELECT header_cids.block_number + 1 AS start, min(fr.block_number) - 1 AS stop FROM eth.header_cids LEFT JOIN eth.header_cids r on eth.header_cids.block_number = r.block_number - 1 LEFT JOIN eth.header_cids fr on eth.header_cids.block_number < fr.block_number @@ -455,18 +456,45 @@ func (ecr *CIDRetriever) RetrieveGapsInData() ([]shared.Gap, error) { Start uint64 `db:"start"` Stop uint64 `db:"stop"` }, 0) - err := ecr.db.Select(&results, pgStr) - if err != nil { + if err := ecr.db.Select(&results, pgStr); err != nil { return nil, err } - gaps := make([]shared.Gap, len(results)) + emptyGaps := make([]shared.Gap, len(results)) for i, res := range results { - gaps[i] = shared.Gap{ + emptyGaps[i] = shared.Gap{ Start: res.Start, Stop: res.Stop, } } - return gaps, nil + + // Find sections of blocks where we are below the validation level + // There will be no overlap between these "gaps" and the ones above + pgStr = `SELECT block_number FROM eth.header_cids + WHERE times_validated < $1 + ORDER BY block_number` + var heights []uint64 + if err := ecr.db.Select(&heights, pgStr, validationLevel); err != nil { + return nil, err + } + if len(heights) == 0 { + return emptyGaps, nil + } + validationGaps := make([]shared.Gap, 0) + start := heights[0] + lastHeight := start + for _, height := range heights[1:] { + if height == lastHeight+1 { + lastHeight = height + continue + } + validationGaps = append(validationGaps, shared.Gap{ + Start: start, + Stop: lastHeight, + }) + start = height + lastHeight = start + } + return append(emptyGaps, validationGaps...), nil } // RetrieveBlockByHash returns all of the CIDs needed to compose an entire block, for a given block hash diff --git a/pkg/super_node/eth/retriever_test.go b/pkg/super_node/eth/retriever_test.go index c822f344..71f4b1d8 100644 --- a/pkg/super_node/eth/retriever_test.go +++ b/pkg/super_node/eth/retriever_test.go @@ -484,7 +484,7 @@ var _ = Describe("Retriever", func() { Expect(err).ToNot(HaveOccurred()) err = repo.Index(&payload2) Expect(err).ToNot(HaveOccurred()) - gaps, err := retriever.RetrieveGapsInData() + gaps, err := retriever.RetrieveGapsInData(1) Expect(err).ToNot(HaveOccurred()) Expect(len(gaps)).To(Equal(0)) }) @@ -494,11 +494,29 @@ var _ = Describe("Retriever", func() { payload.HeaderCID.BlockNumber = "5" err := repo.Index(&payload) Expect(err).ToNot(HaveOccurred()) - gaps, err := retriever.RetrieveGapsInData() + gaps, err := retriever.RetrieveGapsInData(1) Expect(err).ToNot(HaveOccurred()) Expect(len(gaps)).To(Equal(0)) }) + It("Can handle single block gaps", func() { + payload1 := *mocks.MockCIDPayload + payload1.HeaderCID.BlockNumber = "2" + payload2 := payload1 + payload2.HeaderCID.BlockNumber = "4" + err := repo.Index(mocks.MockCIDPayload) + Expect(err).ToNot(HaveOccurred()) + err = repo.Index(&payload1) + Expect(err).ToNot(HaveOccurred()) + err = repo.Index(&payload2) + Expect(err).ToNot(HaveOccurred()) + gaps, err := retriever.RetrieveGapsInData(1) + Expect(err).ToNot(HaveOccurred()) + Expect(len(gaps)).To(Equal(1)) + Expect(gaps[0].Start).To(Equal(uint64(3))) + Expect(gaps[0].Stop).To(Equal(uint64(3))) + }) + It("Finds gap between two entries", func() { payload1 := *mocks.MockCIDPayload payload1.HeaderCID.BlockNumber = "1010101" @@ -508,7 +526,7 @@ var _ = Describe("Retriever", func() { Expect(err).ToNot(HaveOccurred()) err = repo.Index(&payload2) Expect(err).ToNot(HaveOccurred()) - gaps, err := retriever.RetrieveGapsInData() + gaps, err := retriever.RetrieveGapsInData(1) Expect(err).ToNot(HaveOccurred()) Expect(len(gaps)).To(Equal(1)) Expect(gaps[0].Start).To(Equal(uint64(6))) @@ -540,7 +558,7 @@ var _ = Describe("Retriever", func() { Expect(err).ToNot(HaveOccurred()) err = repo.Index(&payload6) Expect(err).ToNot(HaveOccurred()) - gaps, err := retriever.RetrieveGapsInData() + gaps, err := retriever.RetrieveGapsInData(1) Expect(err).ToNot(HaveOccurred()) Expect(len(gaps)).To(Equal(3)) Expect(shared.ListContainsGap(gaps, shared.Gap{Start: 6, Stop: 99})).To(BeTrue()) diff --git a/pkg/super_node/shared/intefaces.go b/pkg/super_node/shared/intefaces.go index 77b8a8fe..9cf8abff 100644 --- a/pkg/super_node/shared/intefaces.go +++ b/pkg/super_node/shared/intefaces.go @@ -57,7 +57,7 @@ type CIDRetriever interface { Retrieve(filter SubscriptionSettings, blockNumber int64) ([]CIDsForFetching, bool, error) RetrieveFirstBlockNumber() (int64, error) RetrieveLastBlockNumber() (int64, error) - RetrieveGapsInData() ([]Gap, error) + RetrieveGapsInData(validationLevel int) ([]Gap, error) } // IPLDFetcher uses a CID wrapper to fetch an IPLD wrapper diff --git a/pkg/super_node/shared/mocks/retriever.go b/pkg/super_node/shared/mocks/retriever.go index 93efc9a5..d899d0b2 100644 --- a/pkg/super_node/shared/mocks/retriever.go +++ b/pkg/super_node/shared/mocks/retriever.go @@ -46,7 +46,7 @@ func (mcr *CIDRetriever) RetrieveFirstBlockNumber() (int64, error) { } // RetrieveGapsInData mock method -func (mcr *CIDRetriever) RetrieveGapsInData() ([]shared.Gap, error) { +func (mcr *CIDRetriever) RetrieveGapsInData(int) ([]shared.Gap, error) { mcr.CalledTimes++ return mcr.GapsToRetrieve, mcr.GapsToRetrieveErr }