From 562ee3e782e43cfd831bacfdd3df595d4c5def94 Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Thu, 30 Sep 2021 18:09:48 +0530 Subject: [PATCH 1/5] Create a docker image with postgres DB and concise migration. --- .github/workflow/on-master.yaml | 25 +++++ .github/workflow/on-pr.yaml | 40 ++++++++ .github/workflow/publish.yaml | 24 +++++ Dockerfile | 1 + Makefile | 11 +++ db/Dockerfile | 23 +++++ .../00001_create_ipfs_blocks_table.sql | 0 .../migrations}/00002_create_nodes_table.sql | 0 .../migrations}/00003_create_eth_schema.sql | 0 .../00004_create_eth_header_cids_table.sql | 0 .../00005_create_eth_uncle_cids_table.sql | 0 ...0006_create_eth_transaction_cids_table.sql | 0 .../00007_create_eth_receipt_cids_table.sql | 6 +- .../00008_create_eth_state_cids_table.sql | 0 .../00009_create_eth_storage_cids_table.sql | 0 .../00010_create_eth_state_accouts_table.sql | 0 .../00011_create_postgraphile_comments.sql | 0 .../00012_potgraphile_triggers.sql | 0 .../migrations}/00013_create_cid_indexes.sql | 10 +- .../00014_create_stored_functions.sql | 52 ++++------ .../00016_create_eth_log_cids_table.sql | 3 +- docker-compose.test.yml | 19 ++++ docker-compose.yml | 16 ++++ migrations/00015_create_access_list_table.sql | 15 --- schema.sql | 95 +++++++------------ scripts/check_diff.sh | 7 ++ scripts/startup_script.sh | 19 ++++ 27 files changed, 246 insertions(+), 120 deletions(-) create mode 100644 .github/workflow/on-master.yaml create mode 100644 .github/workflow/on-pr.yaml create mode 100644 .github/workflow/publish.yaml create mode 100644 Dockerfile create mode 100644 db/Dockerfile rename {migrations => db/migrations}/00001_create_ipfs_blocks_table.sql (100%) rename {migrations => db/migrations}/00002_create_nodes_table.sql (100%) rename {migrations => db/migrations}/00003_create_eth_schema.sql (100%) rename {migrations => db/migrations}/00004_create_eth_header_cids_table.sql (100%) rename {migrations => db/migrations}/00005_create_eth_uncle_cids_table.sql (100%) rename {migrations => db/migrations}/00006_create_eth_transaction_cids_table.sql (100%) rename {migrations => db/migrations}/00007_create_eth_receipt_cids_table.sql (78%) rename {migrations => db/migrations}/00008_create_eth_state_cids_table.sql (100%) rename {migrations => db/migrations}/00009_create_eth_storage_cids_table.sql (100%) rename {migrations => db/migrations}/00010_create_eth_state_accouts_table.sql (100%) rename {migrations => db/migrations}/00011_create_postgraphile_comments.sql (100%) rename {migrations => db/migrations}/00012_potgraphile_triggers.sql (100%) rename {migrations => db/migrations}/00013_create_cid_indexes.sql (93%) rename {migrations => db/migrations}/00014_create_stored_functions.sql (69%) rename {migrations => db/migrations}/00016_create_eth_log_cids_table.sql (97%) create mode 100644 docker-compose.test.yml create mode 100644 docker-compose.yml delete mode 100644 migrations/00015_create_access_list_table.sql create mode 100755 scripts/check_diff.sh create mode 100755 scripts/startup_script.sh diff --git a/.github/workflow/on-master.yaml b/.github/workflow/on-master.yaml new file mode 100644 index 0000000..87f72bd --- /dev/null +++ b/.github/workflow/on-master.yaml @@ -0,0 +1,25 @@ +name: Docker Compose Build + +on: + push: + branches: + - master + +jobs: + build: + name: Run docker build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Get the version + id: vars + run: echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7}) + - name: Run docker build + run: make docker-build + - name: Tag docker image + run: docker tag vulcanize/ipld-eth-db docker.pkg.github.com/vulcanize/ipl-eth-db/:${{steps.vars.outputs.sha}} + - name: Docker Login + run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin + - name: Docker Push + run: docker push docker.pkg.github.com/vulcanize/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.sha}} + diff --git a/.github/workflow/on-pr.yaml b/.github/workflow/on-pr.yaml new file mode 100644 index 0000000..01898ca --- /dev/null +++ b/.github/workflow/on-pr.yaml @@ -0,0 +1,40 @@ +name: Docker Build + +on: [pull_request] + +jobs: + concise_migration_diff: + name: Verify concise migration and generated schema + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Run docker concise migration build + run: make docker-concise-migration-build + - name: Run database + run: docker-compose -f docker-compose.test.yml up -d test-db + - name: Test concise migration + run: | + sleep 10 + docker run --rm --network host -e DATABASE_USER=vdbm -e DATABASE_PASSWORD=password \ + -e DATABASE_HOSTNAME=127.0.0.1 -e DATABASE_PORT=8066 -e DATABASE_NAME=vulcanize_testing \ + vulcanize/docker-concise-migration-build + - name: Verify schema is latest + run: | + PGPASSWORD="password" pg_dump -h localhost -p 8066 -U vdbm vulcanize_testing --no-owner --schema-only > ./db/migration_schema.sql + ./scripts/check_diff.sh + incremental_migration_diff: + name: Compare conscise migration schema with incremental migration. + runs-on: ubuntu-latest + steps: + - name: Run database + run: docker-compose -f docker-compose.test.yml up -d test-db statediff-migrations + - name: Test incremental migration + run: | + sleep 10 + docker run --rm --network host -e DATABASE_USER=vdbm -e DATABASE_PASSWORD=password \ + -e DATABASE_HOSTNAME=127.0.0.1 -e DATABASE_PORT=8066 -e DATABASE_NAME=vulcanize_testing \ + vulcanize/statediff-migrations:v0.8.0 + - name: Verify schema is latest + run: | + PGPASSWORD="password" pg_dump -h localhost -p 8066 -U vdbm vulcanize_testing --no-owner --schema-only > ./db/migration_schema.sql + ./scripts/check_diff.sh diff --git a/.github/workflow/publish.yaml b/.github/workflow/publish.yaml new file mode 100644 index 0000000..305372a --- /dev/null +++ b/.github/workflow/publish.yaml @@ -0,0 +1,24 @@ +name: Publish Docker image +on: + release: + types: [published] +jobs: + push_to_registries: + name: Push Docker image to Docker Hub + runs-on: ubuntu-latest + steps: + - name: Get the version + id: vars + run: | + echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7}) + echo ::set-output name=tag::$(echo ${GITHUB_REF#refs/tags/}) + - name: Docker Login to Github Registry + run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin + - name: Docker Pull + run: docker pull docker.pkg.github.com/vulcanize/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.sha}} + - name: Docker Login to Docker Registry + run: echo ${{ secrets.VULCANIZEJENKINS_PAT }} | docker login -u vulcanizejenkins --password-stdin + - name: Tag docker image + run: docker tag docker.pkg.github.com/vulcanize/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.sha}} vulcanize/ipld-eth-db:${{steps.vars.outputs.tag}} + - name: Docker Push to Docker Hub + run: docker push vulcanize/ipld-eth-db:${{steps.vars.outputs.tag}} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..12e25da --- /dev/null +++ b/Dockerfile @@ -0,0 +1 @@ +FROM postgres:12-alpine \ No newline at end of file diff --git a/Makefile b/Makefile index 64dfe01..81a976f 100644 --- a/Makefile +++ b/Makefile @@ -77,3 +77,14 @@ version_migrations: import: test -n "$(NAME)" # $$NAME psql $(NAME) < schema.sql + + +## Build docker image with schema +.PHONY: docker-build +docker-build: + docker-compose build + +## Build docker image for migration +.PHONY: docker-concise-migration-build +docker-concise-migration-build: + docker build -t vulcanize/docker-concise-migration-build -f ./db/Dockerfile . \ No newline at end of file diff --git a/db/Dockerfile b/db/Dockerfile new file mode 100644 index 0000000..5087196 --- /dev/null +++ b/db/Dockerfile @@ -0,0 +1,23 @@ +FROM golang:1.16-alpine as builder + +RUN apk --update --no-cache add make git g++ linux-headers + +ADD . /go/src/github.com/vulcanize/ipld-eth-db + +# Build migration tool +WORKDIR /go/src/github.com/pressly +RUN git clone https://github.com/pressly/goose.git +WORKDIR /go/src/github.com/pressly/goose/cmd/goose +RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -tags='no_mysql no_sqlite' -o goose . + +# app container +FROM alpine + +WORKDIR /app + +COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-db/scripts/startup_script.sh . + +COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose goose +COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-db/db/migrations migrations/vulcanizedb + +ENTRYPOINT ["/app/startup_script.sh"] \ No newline at end of file diff --git a/migrations/00001_create_ipfs_blocks_table.sql b/db/migrations/00001_create_ipfs_blocks_table.sql similarity index 100% rename from migrations/00001_create_ipfs_blocks_table.sql rename to db/migrations/00001_create_ipfs_blocks_table.sql diff --git a/migrations/00002_create_nodes_table.sql b/db/migrations/00002_create_nodes_table.sql similarity index 100% rename from migrations/00002_create_nodes_table.sql rename to db/migrations/00002_create_nodes_table.sql diff --git a/migrations/00003_create_eth_schema.sql b/db/migrations/00003_create_eth_schema.sql similarity index 100% rename from migrations/00003_create_eth_schema.sql rename to db/migrations/00003_create_eth_schema.sql diff --git a/migrations/00004_create_eth_header_cids_table.sql b/db/migrations/00004_create_eth_header_cids_table.sql similarity index 100% rename from migrations/00004_create_eth_header_cids_table.sql rename to db/migrations/00004_create_eth_header_cids_table.sql diff --git a/migrations/00005_create_eth_uncle_cids_table.sql b/db/migrations/00005_create_eth_uncle_cids_table.sql similarity index 100% rename from migrations/00005_create_eth_uncle_cids_table.sql rename to db/migrations/00005_create_eth_uncle_cids_table.sql diff --git a/migrations/00006_create_eth_transaction_cids_table.sql b/db/migrations/00006_create_eth_transaction_cids_table.sql similarity index 100% rename from migrations/00006_create_eth_transaction_cids_table.sql rename to db/migrations/00006_create_eth_transaction_cids_table.sql diff --git a/migrations/00007_create_eth_receipt_cids_table.sql b/db/migrations/00007_create_eth_receipt_cids_table.sql similarity index 78% rename from migrations/00007_create_eth_receipt_cids_table.sql rename to db/migrations/00007_create_eth_receipt_cids_table.sql index 856322f..ee1cc35 100644 --- a/migrations/00007_create_eth_receipt_cids_table.sql +++ b/db/migrations/00007_create_eth_receipt_cids_table.sql @@ -2,8 +2,8 @@ CREATE TABLE eth.receipt_cids ( id SERIAL PRIMARY KEY, tx_id INTEGER NOT NULL REFERENCES eth.transaction_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, - cid TEXT NOT NULL, - mh_key TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, + leaf_cid TEXT NOT NULL, + leaf_mh_key TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, contract VARCHAR(66), contract_hash VARCHAR(66), post_state VARCHAR(66), @@ -13,4 +13,4 @@ CREATE TABLE eth.receipt_cids ( ); -- +goose Down -DROP TABLE eth.receipt_cids; +DROP TABLE eth.receipt_cids; \ No newline at end of file diff --git a/migrations/00008_create_eth_state_cids_table.sql b/db/migrations/00008_create_eth_state_cids_table.sql similarity index 100% rename from migrations/00008_create_eth_state_cids_table.sql rename to db/migrations/00008_create_eth_state_cids_table.sql diff --git a/migrations/00009_create_eth_storage_cids_table.sql b/db/migrations/00009_create_eth_storage_cids_table.sql similarity index 100% rename from migrations/00009_create_eth_storage_cids_table.sql rename to db/migrations/00009_create_eth_storage_cids_table.sql diff --git a/migrations/00010_create_eth_state_accouts_table.sql b/db/migrations/00010_create_eth_state_accouts_table.sql similarity index 100% rename from migrations/00010_create_eth_state_accouts_table.sql rename to db/migrations/00010_create_eth_state_accouts_table.sql diff --git a/migrations/00011_create_postgraphile_comments.sql b/db/migrations/00011_create_postgraphile_comments.sql similarity index 100% rename from migrations/00011_create_postgraphile_comments.sql rename to db/migrations/00011_create_postgraphile_comments.sql diff --git a/migrations/00012_potgraphile_triggers.sql b/db/migrations/00012_potgraphile_triggers.sql similarity index 100% rename from migrations/00012_potgraphile_triggers.sql rename to db/migrations/00012_potgraphile_triggers.sql diff --git a/migrations/00013_create_cid_indexes.sql b/db/migrations/00013_create_cid_indexes.sql similarity index 93% rename from migrations/00013_create_cid_indexes.sql rename to db/migrations/00013_create_cid_indexes.sql index e123510..5fd4161 100644 --- a/migrations/00013_create_cid_indexes.sql +++ b/db/migrations/00013_create_cid_indexes.sql @@ -28,9 +28,9 @@ CREATE INDEX tx_src_index ON eth.transaction_cids USING btree (src); -- receipt indexes CREATE INDEX rct_tx_id_index ON eth.receipt_cids USING btree (tx_id); -CREATE INDEX rct_cid_index ON eth.receipt_cids USING btree (cid); +CREATE INDEX rct_leaf_cid_index ON eth.receipt_cids USING btree (leaf_cid); -CREATE INDEX rct_mh_index ON eth.receipt_cids USING btree (mh_key); +CREATE INDEX rct_leaf_mh_index ON eth.receipt_cids USING btree (leaf_mh_key); CREATE INDEX rct_contract_index ON eth.receipt_cids USING btree (contract); @@ -91,8 +91,8 @@ DROP INDEX eth.state_header_id_index; -- receipt indexes DROP INDEX eth.rct_contract_hash_index; DROP INDEX eth.rct_contract_index; -DROP INDEX eth.rct_mh_index; -DROP INDEX eth.rct_cid_index; +DROP INDEX eth.rct_leaf_mh_index; +DROP INDEX eth.rct_leaf_cid_index; DROP INDEX eth.rct_tx_id_index; -- transaction indexes @@ -109,4 +109,4 @@ DROP INDEX eth.state_root_index; DROP INDEX eth.header_mh_index; DROP INDEX eth.header_cid_index; DROP INDEX eth.block_hash_index; -DROP INDEX eth.block_number_index; +DROP INDEX eth.block_number_index; \ No newline at end of file diff --git a/migrations/00014_create_stored_functions.sql b/db/migrations/00014_create_stored_functions.sql similarity index 69% rename from migrations/00014_create_stored_functions.sql rename to db/migrations/00014_create_stored_functions.sql index 8db60a3..81d7a40 100644 --- a/migrations/00014_create_stored_functions.sql +++ b/db/migrations/00014_create_stored_functions.sql @@ -1,37 +1,18 @@ -- +goose Up -- +goose StatementBegin --- returns if a storage node at the provided path was removed in the range > the provided height and <= the provided block hash -CREATE OR REPLACE FUNCTION was_storage_removed(path BYTEA, height BIGINT, hash VARCHAR(66)) RETURNS BOOLEAN -AS $$ -SELECT exists(SELECT 1 - FROM eth.storage_cids - INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id) - INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) - WHERE storage_path = path - AND block_number > height - AND block_number <= (SELECT block_number - FROM eth.header_cids - WHERE block_hash = hash) - AND storage_cids.node_type = 3 - LIMIT 1); -$$ LANGUAGE SQL; --- +goose StatementEnd - --- +goose StatementBegin --- returns if a state node at the provided path was removed in the range > the provided height and <= the provided block hash -CREATE OR REPLACE FUNCTION was_state_removed(path BYTEA, height BIGINT, hash VARCHAR(66)) RETURNS BOOLEAN -AS $$ -SELECT exists(SELECT 1 - FROM eth.state_cids - INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) - WHERE state_path = path - AND block_number > height - AND block_number <= (SELECT block_number - FROM eth.header_cids - WHERE block_hash = hash) - AND state_cids.node_type = 3 - LIMIT 1); -$$ LANGUAGE SQL; +-- returns if a state leaf node was removed within the provided block number +CREATE OR REPLACE FUNCTION was_state_leaf_removed(key character varying, hash character varying) + RETURNS boolean AS $$ + SELECT state_cids.node_type = 3 + FROM eth.state_cids + INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) + WHERE state_leaf_key = key + AND block_number <= (SELECT block_number + FROM eth.header_cids + WHERE block_hash = hash) + ORDER BY block_number DESC LIMIT 1; +$$ +language sql; -- +goose StatementEnd -- +goose StatementBegin @@ -63,7 +44,7 @@ BEGIN new_child_result.children = array_append(new_child_result.children, temp_child); END LOOP; END IF; -RETURN new_child_result; + RETURN new_child_result; END $BODY$ LANGUAGE 'plpgsql'; @@ -97,7 +78,7 @@ BEGIN -- if none of the headers had children, none is more canonical than the other IF has_children_count = 0 THEN -- return the first one selected - SELECT * INTO canonical_header FROM unnest(headers) LIMIT 1; + SELECT * INTO canonical_header FROM unnest(headers) LIMIT 1; -- if only one header had children, it can be considered the heaviest/canonical header of the set ELSIF has_children_count = 1 THEN -- return the only header with a child @@ -150,8 +131,7 @@ LANGUAGE 'plpgsql'; -- +goose StatementEnd -- +goose Down -DROP FUNCTION was_storage_removed; -DROP FUNCTION was_state_removed; +DROP FUNCTION was_state_leaf_removed; DROP FUNCTION canonical_header_id; DROP FUNCTION canonical_header_from_array; DROP FUNCTION has_child; diff --git a/migrations/00016_create_eth_log_cids_table.sql b/db/migrations/00016_create_eth_log_cids_table.sql similarity index 97% rename from migrations/00016_create_eth_log_cids_table.sql rename to db/migrations/00016_create_eth_log_cids_table.sql index b2373f0..85de978 100644 --- a/migrations/00016_create_eth_log_cids_table.sql +++ b/db/migrations/00016_create_eth_log_cids_table.sql @@ -4,7 +4,7 @@ CREATE TABLE eth.log_cids ( leaf_cid TEXT NOT NULL, leaf_mh_key TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, receipt_id INTEGER NOT NULL REFERENCES eth.receipt_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, - address VARCHAR(66), + address VARCHAR(66) NOT NULL, log_data BYTEA, index INTEGER NOT NULL, topic0 VARCHAR(66), @@ -19,6 +19,7 @@ CREATE INDEX log_mh_index ON eth.log_cids USING btree (leaf_mh_key); CREATE INDEX log_cid_index ON eth.log_cids USING btree (leaf_cid); CREATE INDEX log_rct_id_index ON eth.log_cids USING btree (receipt_id); + -- -- Name: log_topic0_index; Type: INDEX; Schema: eth; Owner: - -- diff --git a/docker-compose.test.yml b/docker-compose.test.yml new file mode 100644 index 0000000..be5b5dc --- /dev/null +++ b/docker-compose.test.yml @@ -0,0 +1,19 @@ +version: '3.2' + +services: + statediff-migrations: + restart: on-failure + depends_on: + - test-db + image: vulcanize/statediff-migrations:latest + + test-db: + restart: always + image: postgres:10.12-alpine + command: ["postgres", "-c", "log_statement=all"] + environment: + POSTGRES_USER: "vdbm" + POSTGRES_DB: "vulcanize_testing" + POSTGRES_PASSWORD: "password" + ports: + - "127.0.0.1:8066:5432" \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..f991859 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,16 @@ +version: '3.2' + +services: + ipld-eth-db: + restart: always + image: vulcanize/ipld-eth-db + build: . + environment: + POSTGRES_USER: "vdbm" + POSTGRES_DB: "vulcanize_public" + POSTGRES_PASSWORD: "password" + hostname: db + volumes: + - ./schema.sql:/docker-entrypoint-initdb.d/init.sql + ports: + - "127.0.0.1:8077:5432" \ No newline at end of file diff --git a/migrations/00015_create_access_list_table.sql b/migrations/00015_create_access_list_table.sql deleted file mode 100644 index 2e30cd5..0000000 --- a/migrations/00015_create_access_list_table.sql +++ /dev/null @@ -1,15 +0,0 @@ --- +goose Up -CREATE TABLE eth.access_list_element ( - id SERIAL PRIMARY KEY, - tx_id INTEGER NOT NULL REFERENCES eth.transaction_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, - index INTEGER NOT NULL, - address VARCHAR(66), - storage_keys VARCHAR(66)[], - UNIQUE (tx_id, index) -); - -CREATE INDEX accesss_list_element_address_index ON eth.access_list_element USING btree (address); - --- +goose Down -DROP INDEX eth.accesss_list_element_address_index; -DROP TABLE eth.access_list_element; diff --git a/schema.sql b/schema.sql index 3303174..7d06e8b 100644 --- a/schema.sql +++ b/schema.sql @@ -2,8 +2,8 @@ -- PostgreSQL database dump -- --- Dumped from database version 14beta3 --- Dumped by pg_dump version 14beta3 +-- Dumped from database version 10.12 +-- Dumped by pg_dump version 13.4 SET statement_timeout = 0; SET lock_timeout = 0; @@ -25,8 +25,6 @@ CREATE SCHEMA eth; SET default_tablespace = ''; -SET default_table_access_method = heap; - -- -- Name: header_cids; Type: TABLE; Schema: eth; Owner: - -- @@ -136,7 +134,7 @@ BEGIN -- if none of the headers had children, none is more canonical than the other IF has_children_count = 0 THEN -- return the first one selected - SELECT * INTO canonical_header FROM unnest(headers) LIMIT 1; + SELECT * INTO canonical_header FROM unnest(headers) LIMIT 1; -- if only one header had children, it can be considered the heaviest/canonical header of the set ELSIF has_children_count = 1 THEN -- return the only header with a child @@ -218,49 +216,26 @@ BEGIN new_child_result.children = array_append(new_child_result.children, temp_child); END LOOP; END IF; -RETURN new_child_result; + RETURN new_child_result; END $$; -- --- Name: was_state_removed(bytea, bigint, character varying); Type: FUNCTION; Schema: public; Owner: - +-- Name: was_state_leaf_removed(character varying, character varying); Type: FUNCTION; Schema: public; Owner: - -- -CREATE FUNCTION public.was_state_removed(path bytea, height bigint, hash character varying) RETURNS boolean +CREATE FUNCTION public.was_state_leaf_removed(key character varying, hash character varying) RETURNS boolean LANGUAGE sql AS $$ -SELECT exists(SELECT 1 - FROM eth.state_cids - INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) - WHERE state_path = path - AND block_number > height - AND block_number <= (SELECT block_number - FROM eth.header_cids - WHERE block_hash = hash) - AND state_cids.node_type = 3 - LIMIT 1); -$$; - - --- --- Name: was_storage_removed(bytea, bigint, character varying); Type: FUNCTION; Schema: public; Owner: - --- - -CREATE FUNCTION public.was_storage_removed(path bytea, height bigint, hash character varying) RETURNS boolean - LANGUAGE sql - AS $$ -SELECT exists(SELECT 1 - FROM eth.storage_cids - INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id) - INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) - WHERE storage_path = path - AND block_number > height - AND block_number <= (SELECT block_number - FROM eth.header_cids - WHERE block_hash = hash) - AND storage_cids.node_type = 3 - LIMIT 1); + SELECT state_cids.node_type = 3 + FROM eth.state_cids + INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) + WHERE state_leaf_key = key + AND block_number <= (SELECT block_number + FROM eth.header_cids + WHERE block_hash = hash) + ORDER BY block_number DESC LIMIT 1; $$; @@ -326,7 +301,7 @@ CREATE TABLE eth.log_cids ( leaf_cid text NOT NULL, leaf_mh_key text NOT NULL, receipt_id integer NOT NULL, - address character varying(66), + address character varying(66) NOT NULL, log_data bytea, index integer NOT NULL, topic0 character varying(66), @@ -363,8 +338,8 @@ ALTER SEQUENCE eth.log_cids_id_seq OWNED BY eth.log_cids.id; CREATE TABLE eth.receipt_cids ( id integer NOT NULL, tx_id integer NOT NULL, - cid text NOT NULL, - mh_key text NOT NULL, + leaf_cid text NOT NULL, + leaf_mh_key text NOT NULL, contract character varying(66), contract_hash character varying(66), post_state character varying(66), @@ -1011,13 +986,6 @@ CREATE INDEX log_topic2_index ON eth.log_cids USING btree (topic2); CREATE INDEX log_topic3_index ON eth.log_cids USING btree (topic3); --- --- Name: rct_cid_index; Type: INDEX; Schema: eth; Owner: - --- - -CREATE INDEX rct_cid_index ON eth.receipt_cids USING btree (cid); - - -- -- Name: rct_contract_hash_index; Type: INDEX; Schema: eth; Owner: - -- @@ -1033,10 +1001,17 @@ CREATE INDEX rct_contract_index ON eth.receipt_cids USING btree (contract); -- --- Name: rct_mh_index; Type: INDEX; Schema: eth; Owner: - +-- Name: rct_leaf_cid_index; Type: INDEX; Schema: eth; Owner: - -- -CREATE INDEX rct_mh_index ON eth.receipt_cids USING btree (mh_key); +CREATE INDEX rct_leaf_cid_index ON eth.receipt_cids USING btree (leaf_cid); + + +-- +-- Name: rct_leaf_mh_index; Type: INDEX; Schema: eth; Owner: - +-- + +CREATE INDEX rct_leaf_mh_index ON eth.receipt_cids USING btree (leaf_mh_key); -- @@ -1197,49 +1172,49 @@ CREATE INDEX tx_src_index ON eth.transaction_cids USING btree (src); -- Name: header_cids header_cids_ai; Type: TRIGGER; Schema: eth; Owner: - -- -CREATE TRIGGER header_cids_ai AFTER INSERT ON eth.header_cids FOR EACH ROW EXECUTE FUNCTION eth.graphql_subscription('header_cids', 'id'); +CREATE TRIGGER header_cids_ai AFTER INSERT ON eth.header_cids FOR EACH ROW EXECUTE PROCEDURE eth.graphql_subscription('header_cids', 'id'); -- -- Name: receipt_cids receipt_cids_ai; Type: TRIGGER; Schema: eth; Owner: - -- -CREATE TRIGGER receipt_cids_ai AFTER INSERT ON eth.receipt_cids FOR EACH ROW EXECUTE FUNCTION eth.graphql_subscription('receipt_cids', 'id'); +CREATE TRIGGER receipt_cids_ai AFTER INSERT ON eth.receipt_cids FOR EACH ROW EXECUTE PROCEDURE eth.graphql_subscription('receipt_cids', 'id'); -- -- Name: state_accounts state_accounts_ai; Type: TRIGGER; Schema: eth; Owner: - -- -CREATE TRIGGER state_accounts_ai AFTER INSERT ON eth.state_accounts FOR EACH ROW EXECUTE FUNCTION eth.graphql_subscription('state_accounts', 'id'); +CREATE TRIGGER state_accounts_ai AFTER INSERT ON eth.state_accounts FOR EACH ROW EXECUTE PROCEDURE eth.graphql_subscription('state_accounts', 'id'); -- -- Name: state_cids state_cids_ai; Type: TRIGGER; Schema: eth; Owner: - -- -CREATE TRIGGER state_cids_ai AFTER INSERT ON eth.state_cids FOR EACH ROW EXECUTE FUNCTION eth.graphql_subscription('state_cids', 'id'); +CREATE TRIGGER state_cids_ai AFTER INSERT ON eth.state_cids FOR EACH ROW EXECUTE PROCEDURE eth.graphql_subscription('state_cids', 'id'); -- -- Name: storage_cids storage_cids_ai; Type: TRIGGER; Schema: eth; Owner: - -- -CREATE TRIGGER storage_cids_ai AFTER INSERT ON eth.storage_cids FOR EACH ROW EXECUTE FUNCTION eth.graphql_subscription('storage_cids', 'id'); +CREATE TRIGGER storage_cids_ai AFTER INSERT ON eth.storage_cids FOR EACH ROW EXECUTE PROCEDURE eth.graphql_subscription('storage_cids', 'id'); -- -- Name: transaction_cids transaction_cids_ai; Type: TRIGGER; Schema: eth; Owner: - -- -CREATE TRIGGER transaction_cids_ai AFTER INSERT ON eth.transaction_cids FOR EACH ROW EXECUTE FUNCTION eth.graphql_subscription('transaction_cids', 'id'); +CREATE TRIGGER transaction_cids_ai AFTER INSERT ON eth.transaction_cids FOR EACH ROW EXECUTE PROCEDURE eth.graphql_subscription('transaction_cids', 'id'); -- -- Name: uncle_cids uncle_cids_ai; Type: TRIGGER; Schema: eth; Owner: - -- -CREATE TRIGGER uncle_cids_ai AFTER INSERT ON eth.uncle_cids FOR EACH ROW EXECUTE FUNCTION eth.graphql_subscription('uncle_cids', 'id'); +CREATE TRIGGER uncle_cids_ai AFTER INSERT ON eth.uncle_cids FOR EACH ROW EXECUTE PROCEDURE eth.graphql_subscription('uncle_cids', 'id'); -- @@ -1283,11 +1258,11 @@ ALTER TABLE ONLY eth.log_cids -- --- Name: receipt_cids receipt_cids_mh_key_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: - +-- Name: receipt_cids receipt_cids_leaf_mh_key_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: - -- ALTER TABLE ONLY eth.receipt_cids - ADD CONSTRAINT receipt_cids_mh_key_fkey FOREIGN KEY (mh_key) REFERENCES public.blocks(key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; + ADD CONSTRAINT receipt_cids_leaf_mh_key_fkey FOREIGN KEY (leaf_mh_key) REFERENCES public.blocks(key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; -- diff --git a/scripts/check_diff.sh b/scripts/check_diff.sh new file mode 100755 index 0000000..3eaa9b8 --- /dev/null +++ b/scripts/check_diff.sh @@ -0,0 +1,7 @@ +lines=$( git diff --no-index ./db/migration_schema.sql schema.sql | wc -l ) +if [ $lines -gt 0 ]; then + echo "There are differences in schema" + git diff --no-index ./db/migration_schema.sql schema.sql + exit 1 +fi +echo "Schema is latest" \ No newline at end of file diff --git a/scripts/startup_script.sh b/scripts/startup_script.sh new file mode 100755 index 0000000..3bf21d8 --- /dev/null +++ b/scripts/startup_script.sh @@ -0,0 +1,19 @@ +#!/bin/sh +# Runs the db migrations +set +x + +# Construct the connection string for postgres +VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable + +# Run the DB migrations +echo "Connecting with: $VDB_PG_CONNECT" +echo "Running database migrations" +./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up + +# If the db migrations ran without err +if [[ $? -eq 0 ]]; then + echo "Migration process ran successfully" +else + echo "Could not run migrations. Are the database details correct?" + exit 1 +fi From c89a73f7388d537170e87976af824fcf8ea43e16 Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Tue, 5 Oct 2021 16:47:45 +0530 Subject: [PATCH 2/5] Minor fix. --- .github/workflow/on-pr.yaml | 2 +- Makefile | 2 +- db/migrations/00015_create_access_list_table.sql | 15 +++++++++++++++ docker-compose.test.yml | 2 +- 4 files changed, 18 insertions(+), 3 deletions(-) create mode 100644 db/migrations/00015_create_access_list_table.sql diff --git a/.github/workflow/on-pr.yaml b/.github/workflow/on-pr.yaml index 01898ca..c5935d7 100644 --- a/.github/workflow/on-pr.yaml +++ b/.github/workflow/on-pr.yaml @@ -17,7 +17,7 @@ jobs: sleep 10 docker run --rm --network host -e DATABASE_USER=vdbm -e DATABASE_PASSWORD=password \ -e DATABASE_HOSTNAME=127.0.0.1 -e DATABASE_PORT=8066 -e DATABASE_NAME=vulcanize_testing \ - vulcanize/docker-concise-migration-build + vulcanize/concise-migration-build - name: Verify schema is latest run: | PGPASSWORD="password" pg_dump -h localhost -p 8066 -U vdbm vulcanize_testing --no-owner --schema-only > ./db/migration_schema.sql diff --git a/Makefile b/Makefile index 81a976f..884eb96 100644 --- a/Makefile +++ b/Makefile @@ -87,4 +87,4 @@ docker-build: ## Build docker image for migration .PHONY: docker-concise-migration-build docker-concise-migration-build: - docker build -t vulcanize/docker-concise-migration-build -f ./db/Dockerfile . \ No newline at end of file + docker build -t vulcanize/concise-migration-build -f ./db/Dockerfile . \ No newline at end of file diff --git a/db/migrations/00015_create_access_list_table.sql b/db/migrations/00015_create_access_list_table.sql new file mode 100644 index 0000000..46a54ce --- /dev/null +++ b/db/migrations/00015_create_access_list_table.sql @@ -0,0 +1,15 @@ +-- +goose Up +CREATE TABLE eth.access_list_element ( + id SERIAL PRIMARY KEY, + tx_id INTEGER NOT NULL REFERENCES eth.transaction_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, + index INTEGER NOT NULL, + address VARCHAR(66), + storage_keys VARCHAR(66)[], + UNIQUE (tx_id, index) +); + +CREATE INDEX accesss_list_element_address_index ON eth.access_list_element USING btree (address); + +-- +goose Down +DROP INDEX eth.accesss_list_element_address_index; +DROP TABLE eth.access_list_element; diff --git a/docker-compose.test.yml b/docker-compose.test.yml index be5b5dc..6fc915d 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -5,7 +5,7 @@ services: restart: on-failure depends_on: - test-db - image: vulcanize/statediff-migrations:latest + image: vulcanize/statediff-migrations:v0.8.0 test-db: restart: always From 1db8a682df1e4871b4298912b84f595d887c9e97 Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Fri, 8 Oct 2021 18:33:37 +0530 Subject: [PATCH 3/5] Upgrade statediff image to v0.9.0. --- .github/workflow/on-pr.yaml | 2 +- db/migrations/00014_create_stored_functions.sql | 2 +- db/migrations/00015_create_access_list_table.sql | 12 ++++++------ docker-compose.test.yml | 2 +- schema.sql | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflow/on-pr.yaml b/.github/workflow/on-pr.yaml index c5935d7..1ec6c59 100644 --- a/.github/workflow/on-pr.yaml +++ b/.github/workflow/on-pr.yaml @@ -33,7 +33,7 @@ jobs: sleep 10 docker run --rm --network host -e DATABASE_USER=vdbm -e DATABASE_PASSWORD=password \ -e DATABASE_HOSTNAME=127.0.0.1 -e DATABASE_PORT=8066 -e DATABASE_NAME=vulcanize_testing \ - vulcanize/statediff-migrations:v0.8.0 + vulcanize/statediff-migrations:v0.9.0 - name: Verify schema is latest run: | PGPASSWORD="password" pg_dump -h localhost -p 8066 -U vdbm vulcanize_testing --no-owner --schema-only > ./db/migration_schema.sql diff --git a/db/migrations/00014_create_stored_functions.sql b/db/migrations/00014_create_stored_functions.sql index 81d7a40..cdf579c 100644 --- a/db/migrations/00014_create_stored_functions.sql +++ b/db/migrations/00014_create_stored_functions.sql @@ -78,7 +78,7 @@ BEGIN -- if none of the headers had children, none is more canonical than the other IF has_children_count = 0 THEN -- return the first one selected - SELECT * INTO canonical_header FROM unnest(headers) LIMIT 1; + SELECT * INTO canonical_header FROM unnest(headers) LIMIT 1; -- if only one header had children, it can be considered the heaviest/canonical header of the set ELSIF has_children_count = 1 THEN -- return the only header with a child diff --git a/db/migrations/00015_create_access_list_table.sql b/db/migrations/00015_create_access_list_table.sql index 46a54ce..3e27a63 100644 --- a/db/migrations/00015_create_access_list_table.sql +++ b/db/migrations/00015_create_access_list_table.sql @@ -1,11 +1,11 @@ -- +goose Up CREATE TABLE eth.access_list_element ( - id SERIAL PRIMARY KEY, - tx_id INTEGER NOT NULL REFERENCES eth.transaction_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, - index INTEGER NOT NULL, - address VARCHAR(66), - storage_keys VARCHAR(66)[], - UNIQUE (tx_id, index) + id SERIAL PRIMARY KEY, + tx_id INTEGER NOT NULL REFERENCES eth.transaction_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, + index INTEGER NOT NULL, + address VARCHAR(66), + storage_keys VARCHAR(66)[], + UNIQUE (tx_id, index) ); CREATE INDEX accesss_list_element_address_index ON eth.access_list_element USING btree (address); diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 6fc915d..8e93c52 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -5,7 +5,7 @@ services: restart: on-failure depends_on: - test-db - image: vulcanize/statediff-migrations:v0.8.0 + image: vulcanize/statediff-migrations:v0.9.0 test-db: restart: always diff --git a/schema.sql b/schema.sql index 7d06e8b..805961e 100644 --- a/schema.sql +++ b/schema.sql @@ -134,7 +134,7 @@ BEGIN -- if none of the headers had children, none is more canonical than the other IF has_children_count = 0 THEN -- return the first one selected - SELECT * INTO canonical_header FROM unnest(headers) LIMIT 1; + SELECT * INTO canonical_header FROM unnest(headers) LIMIT 1; -- if only one header had children, it can be considered the heaviest/canonical header of the set ELSIF has_children_count = 1 THEN -- return the only header with a child From d8bf76a48fb13e03575cf791ecba3b78b5dcae89 Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Fri, 8 Oct 2021 18:45:37 +0530 Subject: [PATCH 4/5] Fix branch name in workflow. --- .github/{workflow/on-master.yaml => workflows/on-main.yaml} | 2 +- .github/{workflow => workflows}/on-pr.yaml | 0 .github/{workflow => workflows}/publish.yaml | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename .github/{workflow/on-master.yaml => workflows/on-main.yaml} (98%) rename .github/{workflow => workflows}/on-pr.yaml (100%) rename .github/{workflow => workflows}/publish.yaml (100%) diff --git a/.github/workflow/on-master.yaml b/.github/workflows/on-main.yaml similarity index 98% rename from .github/workflow/on-master.yaml rename to .github/workflows/on-main.yaml index 87f72bd..bc0dae7 100644 --- a/.github/workflow/on-master.yaml +++ b/.github/workflows/on-main.yaml @@ -3,7 +3,7 @@ name: Docker Compose Build on: push: branches: - - master + - main jobs: build: diff --git a/.github/workflow/on-pr.yaml b/.github/workflows/on-pr.yaml similarity index 100% rename from .github/workflow/on-pr.yaml rename to .github/workflows/on-pr.yaml diff --git a/.github/workflow/publish.yaml b/.github/workflows/publish.yaml similarity index 100% rename from .github/workflow/publish.yaml rename to .github/workflows/publish.yaml From 91d92bc0bc72255d431660148df6d88cdae679cb Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Fri, 8 Oct 2021 18:55:49 +0530 Subject: [PATCH 5/5] Fix CI workflow. --- .github/workflows/on-pr.yaml | 1 + schema.sql | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/on-pr.yaml b/.github/workflows/on-pr.yaml index 1ec6c59..4888e38 100644 --- a/.github/workflows/on-pr.yaml +++ b/.github/workflows/on-pr.yaml @@ -26,6 +26,7 @@ jobs: name: Compare conscise migration schema with incremental migration. runs-on: ubuntu-latest steps: + - uses: actions/checkout@v2 - name: Run database run: docker-compose -f docker-compose.test.yml up -d test-db statediff-migrations - name: Test incremental migration diff --git a/schema.sql b/schema.sql index 805961e..d23164d 100644 --- a/schema.sql +++ b/schema.sql @@ -3,7 +3,7 @@ -- -- Dumped from database version 10.12 --- Dumped by pg_dump version 13.4 +-- Dumped by pg_dump version 14.0 (Ubuntu 14.0-1.pgdg20.04+1) SET statement_timeout = 0; SET lock_timeout = 0;