Merge pull request #10 from vulcanize/concise-migration-db

Create a docker image with postgres DB and concise migration.
This commit is contained in:
Arijit Das 2021-10-08 19:04:24 +05:30 committed by GitHub
commit e5af6a1abe
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
28 changed files with 260 additions and 118 deletions

25
.github/workflows/on-main.yaml vendored Normal file
View File

@ -0,0 +1,25 @@
name: Docker Compose Build
on:
push:
branches:
- main
jobs:
build:
name: Run docker build
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Get the version
id: vars
run: echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
- name: Run docker build
run: make docker-build
- name: Tag docker image
run: docker tag vulcanize/ipld-eth-db docker.pkg.github.com/vulcanize/ipl-eth-db/:${{steps.vars.outputs.sha}}
- name: Docker Login
run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
- name: Docker Push
run: docker push docker.pkg.github.com/vulcanize/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.sha}}

41
.github/workflows/on-pr.yaml vendored Normal file
View File

@ -0,0 +1,41 @@
name: Docker Build
on: [pull_request]
jobs:
concise_migration_diff:
name: Verify concise migration and generated schema
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Run docker concise migration build
run: make docker-concise-migration-build
- name: Run database
run: docker-compose -f docker-compose.test.yml up -d test-db
- name: Test concise migration
run: |
sleep 10
docker run --rm --network host -e DATABASE_USER=vdbm -e DATABASE_PASSWORD=password \
-e DATABASE_HOSTNAME=127.0.0.1 -e DATABASE_PORT=8066 -e DATABASE_NAME=vulcanize_testing \
vulcanize/concise-migration-build
- name: Verify schema is latest
run: |
PGPASSWORD="password" pg_dump -h localhost -p 8066 -U vdbm vulcanize_testing --no-owner --schema-only > ./db/migration_schema.sql
./scripts/check_diff.sh
incremental_migration_diff:
name: Compare conscise migration schema with incremental migration.
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Run database
run: docker-compose -f docker-compose.test.yml up -d test-db statediff-migrations
- name: Test incremental migration
run: |
sleep 10
docker run --rm --network host -e DATABASE_USER=vdbm -e DATABASE_PASSWORD=password \
-e DATABASE_HOSTNAME=127.0.0.1 -e DATABASE_PORT=8066 -e DATABASE_NAME=vulcanize_testing \
vulcanize/statediff-migrations:v0.9.0
- name: Verify schema is latest
run: |
PGPASSWORD="password" pg_dump -h localhost -p 8066 -U vdbm vulcanize_testing --no-owner --schema-only > ./db/migration_schema.sql
./scripts/check_diff.sh

24
.github/workflows/publish.yaml vendored Normal file
View File

@ -0,0 +1,24 @@
name: Publish Docker image
on:
release:
types: [published]
jobs:
push_to_registries:
name: Push Docker image to Docker Hub
runs-on: ubuntu-latest
steps:
- name: Get the version
id: vars
run: |
echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
echo ::set-output name=tag::$(echo ${GITHUB_REF#refs/tags/})
- name: Docker Login to Github Registry
run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
- name: Docker Pull
run: docker pull docker.pkg.github.com/vulcanize/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.sha}}
- name: Docker Login to Docker Registry
run: echo ${{ secrets.VULCANIZEJENKINS_PAT }} | docker login -u vulcanizejenkins --password-stdin
- name: Tag docker image
run: docker tag docker.pkg.github.com/vulcanize/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.sha}} vulcanize/ipld-eth-db:${{steps.vars.outputs.tag}}
- name: Docker Push to Docker Hub
run: docker push vulcanize/ipld-eth-db:${{steps.vars.outputs.tag}}

1
Dockerfile Normal file
View File

@ -0,0 +1 @@
FROM postgres:12-alpine

View File

@ -77,3 +77,14 @@ version_migrations:
import:
test -n "$(NAME)" # $$NAME
psql $(NAME) < schema.sql
## Build docker image with schema
.PHONY: docker-build
docker-build:
docker-compose build
## Build docker image for migration
.PHONY: docker-concise-migration-build
docker-concise-migration-build:
docker build -t vulcanize/concise-migration-build -f ./db/Dockerfile .

23
db/Dockerfile Normal file
View File

@ -0,0 +1,23 @@
FROM golang:1.16-alpine as builder
RUN apk --update --no-cache add make git g++ linux-headers
ADD . /go/src/github.com/vulcanize/ipld-eth-db
# Build migration tool
WORKDIR /go/src/github.com/pressly
RUN git clone https://github.com/pressly/goose.git
WORKDIR /go/src/github.com/pressly/goose/cmd/goose
RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -tags='no_mysql no_sqlite' -o goose .
# app container
FROM alpine
WORKDIR /app
COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-db/scripts/startup_script.sh .
COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose goose
COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-db/db/migrations migrations/vulcanizedb
ENTRYPOINT ["/app/startup_script.sh"]

View File

@ -2,8 +2,8 @@
CREATE TABLE eth.receipt_cids (
id SERIAL PRIMARY KEY,
tx_id INTEGER NOT NULL REFERENCES eth.transaction_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
cid TEXT NOT NULL,
mh_key TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
leaf_cid TEXT NOT NULL,
leaf_mh_key TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
contract VARCHAR(66),
contract_hash VARCHAR(66),
post_state VARCHAR(66),

View File

@ -28,9 +28,9 @@ CREATE INDEX tx_src_index ON eth.transaction_cids USING btree (src);
-- receipt indexes
CREATE INDEX rct_tx_id_index ON eth.receipt_cids USING btree (tx_id);
CREATE INDEX rct_cid_index ON eth.receipt_cids USING btree (cid);
CREATE INDEX rct_leaf_cid_index ON eth.receipt_cids USING btree (leaf_cid);
CREATE INDEX rct_mh_index ON eth.receipt_cids USING btree (mh_key);
CREATE INDEX rct_leaf_mh_index ON eth.receipt_cids USING btree (leaf_mh_key);
CREATE INDEX rct_contract_index ON eth.receipt_cids USING btree (contract);
@ -91,8 +91,8 @@ DROP INDEX eth.state_header_id_index;
-- receipt indexes
DROP INDEX eth.rct_contract_hash_index;
DROP INDEX eth.rct_contract_index;
DROP INDEX eth.rct_mh_index;
DROP INDEX eth.rct_cid_index;
DROP INDEX eth.rct_leaf_mh_index;
DROP INDEX eth.rct_leaf_cid_index;
DROP INDEX eth.rct_tx_id_index;
-- transaction indexes

View File

@ -1,37 +1,18 @@
-- +goose Up
-- +goose StatementBegin
-- returns if a storage node at the provided path was removed in the range > the provided height and <= the provided block hash
CREATE OR REPLACE FUNCTION was_storage_removed(path BYTEA, height BIGINT, hash VARCHAR(66)) RETURNS BOOLEAN
AS $$
SELECT exists(SELECT 1
FROM eth.storage_cids
INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id)
INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
WHERE storage_path = path
AND block_number > height
AND block_number <= (SELECT block_number
FROM eth.header_cids
WHERE block_hash = hash)
AND storage_cids.node_type = 3
LIMIT 1);
$$ LANGUAGE SQL;
-- +goose StatementEnd
-- +goose StatementBegin
-- returns if a state node at the provided path was removed in the range > the provided height and <= the provided block hash
CREATE OR REPLACE FUNCTION was_state_removed(path BYTEA, height BIGINT, hash VARCHAR(66)) RETURNS BOOLEAN
AS $$
SELECT exists(SELECT 1
FROM eth.state_cids
INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
WHERE state_path = path
AND block_number > height
AND block_number <= (SELECT block_number
FROM eth.header_cids
WHERE block_hash = hash)
AND state_cids.node_type = 3
LIMIT 1);
$$ LANGUAGE SQL;
-- returns if a state leaf node was removed within the provided block number
CREATE OR REPLACE FUNCTION was_state_leaf_removed(key character varying, hash character varying)
RETURNS boolean AS $$
SELECT state_cids.node_type = 3
FROM eth.state_cids
INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
WHERE state_leaf_key = key
AND block_number <= (SELECT block_number
FROM eth.header_cids
WHERE block_hash = hash)
ORDER BY block_number DESC LIMIT 1;
$$
language sql;
-- +goose StatementEnd
-- +goose StatementBegin
@ -63,7 +44,7 @@ BEGIN
new_child_result.children = array_append(new_child_result.children, temp_child);
END LOOP;
END IF;
RETURN new_child_result;
RETURN new_child_result;
END
$BODY$
LANGUAGE 'plpgsql';
@ -150,8 +131,7 @@ LANGUAGE 'plpgsql';
-- +goose StatementEnd
-- +goose Down
DROP FUNCTION was_storage_removed;
DROP FUNCTION was_state_removed;
DROP FUNCTION was_state_leaf_removed;
DROP FUNCTION canonical_header_id;
DROP FUNCTION canonical_header_from_array;
DROP FUNCTION has_child;

View File

@ -0,0 +1,15 @@
-- +goose Up
CREATE TABLE eth.access_list_element (
id SERIAL PRIMARY KEY,
tx_id INTEGER NOT NULL REFERENCES eth.transaction_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
index INTEGER NOT NULL,
address VARCHAR(66),
storage_keys VARCHAR(66)[],
UNIQUE (tx_id, index)
);
CREATE INDEX accesss_list_element_address_index ON eth.access_list_element USING btree (address);
-- +goose Down
DROP INDEX eth.accesss_list_element_address_index;
DROP TABLE eth.access_list_element;

View File

@ -4,7 +4,7 @@ CREATE TABLE eth.log_cids (
leaf_cid TEXT NOT NULL,
leaf_mh_key TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
receipt_id INTEGER NOT NULL REFERENCES eth.receipt_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
address VARCHAR(66),
address VARCHAR(66) NOT NULL,
log_data BYTEA,
index INTEGER NOT NULL,
topic0 VARCHAR(66),
@ -19,6 +19,7 @@ CREATE INDEX log_mh_index ON eth.log_cids USING btree (leaf_mh_key);
CREATE INDEX log_cid_index ON eth.log_cids USING btree (leaf_cid);
CREATE INDEX log_rct_id_index ON eth.log_cids USING btree (receipt_id);
--
-- Name: log_topic0_index; Type: INDEX; Schema: eth; Owner: -
--

19
docker-compose.test.yml Normal file
View File

@ -0,0 +1,19 @@
version: '3.2'
services:
statediff-migrations:
restart: on-failure
depends_on:
- test-db
image: vulcanize/statediff-migrations:v0.9.0
test-db:
restart: always
image: postgres:10.12-alpine
command: ["postgres", "-c", "log_statement=all"]
environment:
POSTGRES_USER: "vdbm"
POSTGRES_DB: "vulcanize_testing"
POSTGRES_PASSWORD: "password"
ports:
- "127.0.0.1:8066:5432"

16
docker-compose.yml Normal file
View File

@ -0,0 +1,16 @@
version: '3.2'
services:
ipld-eth-db:
restart: always
image: vulcanize/ipld-eth-db
build: .
environment:
POSTGRES_USER: "vdbm"
POSTGRES_DB: "vulcanize_public"
POSTGRES_PASSWORD: "password"
hostname: db
volumes:
- ./schema.sql:/docker-entrypoint-initdb.d/init.sql
ports:
- "127.0.0.1:8077:5432"

View File

@ -1,15 +0,0 @@
-- +goose Up
CREATE TABLE eth.access_list_element (
id SERIAL PRIMARY KEY,
tx_id INTEGER NOT NULL REFERENCES eth.transaction_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
index INTEGER NOT NULL,
address VARCHAR(66),
storage_keys VARCHAR(66)[],
UNIQUE (tx_id, index)
);
CREATE INDEX accesss_list_element_address_index ON eth.access_list_element USING btree (address);
-- +goose Down
DROP INDEX eth.accesss_list_element_address_index;
DROP TABLE eth.access_list_element;

View File

@ -2,8 +2,8 @@
-- PostgreSQL database dump
--
-- Dumped from database version 14beta3
-- Dumped by pg_dump version 14beta3
-- Dumped from database version 10.12
-- Dumped by pg_dump version 14.0 (Ubuntu 14.0-1.pgdg20.04+1)
SET statement_timeout = 0;
SET lock_timeout = 0;
@ -25,8 +25,6 @@ CREATE SCHEMA eth;
SET default_tablespace = '';
SET default_table_access_method = heap;
--
-- Name: header_cids; Type: TABLE; Schema: eth; Owner: -
--
@ -218,49 +216,26 @@ BEGIN
new_child_result.children = array_append(new_child_result.children, temp_child);
END LOOP;
END IF;
RETURN new_child_result;
RETURN new_child_result;
END
$$;
--
-- Name: was_state_removed(bytea, bigint, character varying); Type: FUNCTION; Schema: public; Owner: -
-- Name: was_state_leaf_removed(character varying, character varying); Type: FUNCTION; Schema: public; Owner: -
--
CREATE FUNCTION public.was_state_removed(path bytea, height bigint, hash character varying) RETURNS boolean
CREATE FUNCTION public.was_state_leaf_removed(key character varying, hash character varying) RETURNS boolean
LANGUAGE sql
AS $$
SELECT exists(SELECT 1
FROM eth.state_cids
INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
WHERE state_path = path
AND block_number > height
AND block_number <= (SELECT block_number
FROM eth.header_cids
WHERE block_hash = hash)
AND state_cids.node_type = 3
LIMIT 1);
$$;
--
-- Name: was_storage_removed(bytea, bigint, character varying); Type: FUNCTION; Schema: public; Owner: -
--
CREATE FUNCTION public.was_storage_removed(path bytea, height bigint, hash character varying) RETURNS boolean
LANGUAGE sql
AS $$
SELECT exists(SELECT 1
FROM eth.storage_cids
INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id)
INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
WHERE storage_path = path
AND block_number > height
AND block_number <= (SELECT block_number
FROM eth.header_cids
WHERE block_hash = hash)
AND storage_cids.node_type = 3
LIMIT 1);
SELECT state_cids.node_type = 3
FROM eth.state_cids
INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
WHERE state_leaf_key = key
AND block_number <= (SELECT block_number
FROM eth.header_cids
WHERE block_hash = hash)
ORDER BY block_number DESC LIMIT 1;
$$;
@ -326,7 +301,7 @@ CREATE TABLE eth.log_cids (
leaf_cid text NOT NULL,
leaf_mh_key text NOT NULL,
receipt_id integer NOT NULL,
address character varying(66),
address character varying(66) NOT NULL,
log_data bytea,
index integer NOT NULL,
topic0 character varying(66),
@ -363,8 +338,8 @@ ALTER SEQUENCE eth.log_cids_id_seq OWNED BY eth.log_cids.id;
CREATE TABLE eth.receipt_cids (
id integer NOT NULL,
tx_id integer NOT NULL,
cid text NOT NULL,
mh_key text NOT NULL,
leaf_cid text NOT NULL,
leaf_mh_key text NOT NULL,
contract character varying(66),
contract_hash character varying(66),
post_state character varying(66),
@ -1011,13 +986,6 @@ CREATE INDEX log_topic2_index ON eth.log_cids USING btree (topic2);
CREATE INDEX log_topic3_index ON eth.log_cids USING btree (topic3);
--
-- Name: rct_cid_index; Type: INDEX; Schema: eth; Owner: -
--
CREATE INDEX rct_cid_index ON eth.receipt_cids USING btree (cid);
--
-- Name: rct_contract_hash_index; Type: INDEX; Schema: eth; Owner: -
--
@ -1033,10 +1001,17 @@ CREATE INDEX rct_contract_index ON eth.receipt_cids USING btree (contract);
--
-- Name: rct_mh_index; Type: INDEX; Schema: eth; Owner: -
-- Name: rct_leaf_cid_index; Type: INDEX; Schema: eth; Owner: -
--
CREATE INDEX rct_mh_index ON eth.receipt_cids USING btree (mh_key);
CREATE INDEX rct_leaf_cid_index ON eth.receipt_cids USING btree (leaf_cid);
--
-- Name: rct_leaf_mh_index; Type: INDEX; Schema: eth; Owner: -
--
CREATE INDEX rct_leaf_mh_index ON eth.receipt_cids USING btree (leaf_mh_key);
--
@ -1197,49 +1172,49 @@ CREATE INDEX tx_src_index ON eth.transaction_cids USING btree (src);
-- Name: header_cids header_cids_ai; Type: TRIGGER; Schema: eth; Owner: -
--
CREATE TRIGGER header_cids_ai AFTER INSERT ON eth.header_cids FOR EACH ROW EXECUTE FUNCTION eth.graphql_subscription('header_cids', 'id');
CREATE TRIGGER header_cids_ai AFTER INSERT ON eth.header_cids FOR EACH ROW EXECUTE PROCEDURE eth.graphql_subscription('header_cids', 'id');
--
-- Name: receipt_cids receipt_cids_ai; Type: TRIGGER; Schema: eth; Owner: -
--
CREATE TRIGGER receipt_cids_ai AFTER INSERT ON eth.receipt_cids FOR EACH ROW EXECUTE FUNCTION eth.graphql_subscription('receipt_cids', 'id');
CREATE TRIGGER receipt_cids_ai AFTER INSERT ON eth.receipt_cids FOR EACH ROW EXECUTE PROCEDURE eth.graphql_subscription('receipt_cids', 'id');
--
-- Name: state_accounts state_accounts_ai; Type: TRIGGER; Schema: eth; Owner: -
--
CREATE TRIGGER state_accounts_ai AFTER INSERT ON eth.state_accounts FOR EACH ROW EXECUTE FUNCTION eth.graphql_subscription('state_accounts', 'id');
CREATE TRIGGER state_accounts_ai AFTER INSERT ON eth.state_accounts FOR EACH ROW EXECUTE PROCEDURE eth.graphql_subscription('state_accounts', 'id');
--
-- Name: state_cids state_cids_ai; Type: TRIGGER; Schema: eth; Owner: -
--
CREATE TRIGGER state_cids_ai AFTER INSERT ON eth.state_cids FOR EACH ROW EXECUTE FUNCTION eth.graphql_subscription('state_cids', 'id');
CREATE TRIGGER state_cids_ai AFTER INSERT ON eth.state_cids FOR EACH ROW EXECUTE PROCEDURE eth.graphql_subscription('state_cids', 'id');
--
-- Name: storage_cids storage_cids_ai; Type: TRIGGER; Schema: eth; Owner: -
--
CREATE TRIGGER storage_cids_ai AFTER INSERT ON eth.storage_cids FOR EACH ROW EXECUTE FUNCTION eth.graphql_subscription('storage_cids', 'id');
CREATE TRIGGER storage_cids_ai AFTER INSERT ON eth.storage_cids FOR EACH ROW EXECUTE PROCEDURE eth.graphql_subscription('storage_cids', 'id');
--
-- Name: transaction_cids transaction_cids_ai; Type: TRIGGER; Schema: eth; Owner: -
--
CREATE TRIGGER transaction_cids_ai AFTER INSERT ON eth.transaction_cids FOR EACH ROW EXECUTE FUNCTION eth.graphql_subscription('transaction_cids', 'id');
CREATE TRIGGER transaction_cids_ai AFTER INSERT ON eth.transaction_cids FOR EACH ROW EXECUTE PROCEDURE eth.graphql_subscription('transaction_cids', 'id');
--
-- Name: uncle_cids uncle_cids_ai; Type: TRIGGER; Schema: eth; Owner: -
--
CREATE TRIGGER uncle_cids_ai AFTER INSERT ON eth.uncle_cids FOR EACH ROW EXECUTE FUNCTION eth.graphql_subscription('uncle_cids', 'id');
CREATE TRIGGER uncle_cids_ai AFTER INSERT ON eth.uncle_cids FOR EACH ROW EXECUTE PROCEDURE eth.graphql_subscription('uncle_cids', 'id');
--
@ -1283,11 +1258,11 @@ ALTER TABLE ONLY eth.log_cids
--
-- Name: receipt_cids receipt_cids_mh_key_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: -
-- Name: receipt_cids receipt_cids_leaf_mh_key_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: -
--
ALTER TABLE ONLY eth.receipt_cids
ADD CONSTRAINT receipt_cids_mh_key_fkey FOREIGN KEY (mh_key) REFERENCES public.blocks(key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
ADD CONSTRAINT receipt_cids_leaf_mh_key_fkey FOREIGN KEY (leaf_mh_key) REFERENCES public.blocks(key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
--

7
scripts/check_diff.sh Executable file
View File

@ -0,0 +1,7 @@
lines=$( git diff --no-index ./db/migration_schema.sql schema.sql | wc -l )
if [ $lines -gt 0 ]; then
echo "There are differences in schema"
git diff --no-index ./db/migration_schema.sql schema.sql
exit 1
fi
echo "Schema is latest"

19
scripts/startup_script.sh Executable file
View File

@ -0,0 +1,19 @@
#!/bin/sh
# Runs the db migrations
set +x
# Construct the connection string for postgres
VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable
# Run the DB migrations
echo "Connecting with: $VDB_PG_CONNECT"
echo "Running database migrations"
./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up
# If the db migrations ran without err
if [[ $? -eq 0 ]]; then
echo "Migration process ran successfully"
else
echo "Could not run migrations. Are the database details correct?"
exit 1
fi