From 141b537b7970937bdbcc14bef415565ec8f0a980 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Mon, 24 Jun 2024 22:52:53 +0800 Subject: [PATCH 1/4] Remove unused Makefile, workflows --- .gitea/workflows/on-pr.yaml | 58 ---------------- .github/workflows/on-pr.yaml | 78 --------------------- .github/workflows/publish.yaml | 26 ------- Makefile | 121 --------------------------------- 4 files changed, 283 deletions(-) delete mode 100644 .github/workflows/on-pr.yaml delete mode 100644 .github/workflows/publish.yaml delete mode 100644 Makefile diff --git a/.gitea/workflows/on-pr.yaml b/.gitea/workflows/on-pr.yaml index 84ba726..6278c52 100644 --- a/.gitea/workflows/on-pr.yaml +++ b/.gitea/workflows/on-pr.yaml @@ -18,61 +18,3 @@ jobs: steps: - name: Run dummy job run: echo "Empty dummy job" - # concise_migration_diff: - # name: Verify concise migration and generated schema - # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v2 - # - name: Run docker concise migration build - # run: make docker-concise-migration-build - # - name: Run database - # run: docker-compose -f docker-compose.test.yml up -d test-db - # - name: Test concise migration - # run: | - # sleep 10 - # docker run --rm --network host -e DATABASE_USER=vdbm -e DATABASE_PASSWORD=password \ - # -e DATABASE_HOSTNAME=127.0.0.1 -e DATABASE_PORT=8066 -e DATABASE_NAME=vulcanize_testing \ - # vulcanize/concise-migration-build - # - name: Verify schema is latest - # run: | - # PGPASSWORD="password" pg_dump -h localhost -p 8066 -U vdbm vulcanize_testing --no-owner --schema-only > ./db/migration_schema.sql - # ./scripts/check_diff.sh ./db/migration_schema.sql db/schema.sql - - # incremental_migration_diff: - # name: Compare conscise migration schema with incremental migration. - # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v2 - # - name: Run database - # run: docker-compose -f docker-compose.test.yml up -d test-db statediff-migrations - # - name: Test incremental migration - # run: | - # sleep 10 - # docker run --rm --network host -e DATABASE_USER=vdbm -e DATABASE_PASSWORD=password \ - # -e DATABASE_HOSTNAME=127.0.0.1 -e DATABASE_PORT=8066 -e DATABASE_NAME=vulcanize_testing \ - # vulcanize/statediff-migrations:v0.9.0 - # - name: Verify schema is latest - # run: | - # PGPASSWORD="password" pg_dump -h localhost -p 8066 -U vdbm vulcanize_testing --no-owner --schema-only > ./db/migration_schema.sql - # ./scripts/check_diff.sh db/schema.sql ./db/migration_schema.sql - - # migration: - # name: Compare up and down migration - # env: - # GOPATH: /tmp/go - # strategy: - # matrix: - # go-version: [ 1.16.x ] - # os: [ ubuntu-latest ] - # runs-on: ${{ matrix.os }} - # steps: - # - name: Create GOPATH - # run: mkdir -p /tmp/go - # - name: Install Go - # uses: actions/setup-go@v2 - # with: - # go-version: ${{ matrix.go-version }} - # - uses: actions/checkout@v2 - # - name: Test migration - # run: | - # timeout 5m make test-migrations diff --git a/.github/workflows/on-pr.yaml b/.github/workflows/on-pr.yaml deleted file mode 100644 index 84ba726..0000000 --- a/.github/workflows/on-pr.yaml +++ /dev/null @@ -1,78 +0,0 @@ -name: Docker Build - -on: [pull_request] - -jobs: - build: - name: Run docker build - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Run docker build - run: docker compose build - test: - # Add a dummy job to avoid failing GitHub CI checks. - # Other checks to be added later. - name: Dummy job - runs-on: ubuntu-latest - steps: - - name: Run dummy job - run: echo "Empty dummy job" - # concise_migration_diff: - # name: Verify concise migration and generated schema - # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v2 - # - name: Run docker concise migration build - # run: make docker-concise-migration-build - # - name: Run database - # run: docker-compose -f docker-compose.test.yml up -d test-db - # - name: Test concise migration - # run: | - # sleep 10 - # docker run --rm --network host -e DATABASE_USER=vdbm -e DATABASE_PASSWORD=password \ - # -e DATABASE_HOSTNAME=127.0.0.1 -e DATABASE_PORT=8066 -e DATABASE_NAME=vulcanize_testing \ - # vulcanize/concise-migration-build - # - name: Verify schema is latest - # run: | - # PGPASSWORD="password" pg_dump -h localhost -p 8066 -U vdbm vulcanize_testing --no-owner --schema-only > ./db/migration_schema.sql - # ./scripts/check_diff.sh ./db/migration_schema.sql db/schema.sql - - # incremental_migration_diff: - # name: Compare conscise migration schema with incremental migration. - # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v2 - # - name: Run database - # run: docker-compose -f docker-compose.test.yml up -d test-db statediff-migrations - # - name: Test incremental migration - # run: | - # sleep 10 - # docker run --rm --network host -e DATABASE_USER=vdbm -e DATABASE_PASSWORD=password \ - # -e DATABASE_HOSTNAME=127.0.0.1 -e DATABASE_PORT=8066 -e DATABASE_NAME=vulcanize_testing \ - # vulcanize/statediff-migrations:v0.9.0 - # - name: Verify schema is latest - # run: | - # PGPASSWORD="password" pg_dump -h localhost -p 8066 -U vdbm vulcanize_testing --no-owner --schema-only > ./db/migration_schema.sql - # ./scripts/check_diff.sh db/schema.sql ./db/migration_schema.sql - - # migration: - # name: Compare up and down migration - # env: - # GOPATH: /tmp/go - # strategy: - # matrix: - # go-version: [ 1.16.x ] - # os: [ ubuntu-latest ] - # runs-on: ${{ matrix.os }} - # steps: - # - name: Create GOPATH - # run: mkdir -p /tmp/go - # - name: Install Go - # uses: actions/setup-go@v2 - # with: - # go-version: ${{ matrix.go-version }} - # - uses: actions/checkout@v2 - # - name: Test migration - # run: | - # timeout 5m make test-migrations diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml deleted file mode 100644 index 1880d0e..0000000 --- a/.github/workflows/publish.yaml +++ /dev/null @@ -1,26 +0,0 @@ -name: Publish Docker image -on: - release: - types: [published, edited] -jobs: - build: - name: Build and publish image - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - id: vars - name: Output SHA and version tag - run: | - echo "sha=${GITHUB_SHA:0:7}" >> $GITHUB_OUTPUT - echo "tag=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT - - name: Build and tag image - run: | - docker build . \ - -t cerc-io/ipld-eth-db \ - -t git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.sha}} \ - -t git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.tag}} - - name: Push image tags - run: | - echo ${{ secrets.GITEA_PUBLISH_TOKEN }} | docker login https://git.vdb.to -u cerccicd --password-stdin - docker push git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.sha}} - docker push git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.tag}} diff --git a/Makefile b/Makefile deleted file mode 100644 index 79a0cba..0000000 --- a/Makefile +++ /dev/null @@ -1,121 +0,0 @@ -ifndef GOPATH -override GOPATH = $(HOME)/go -endif - -BIN = $(GOPATH)/bin - -# Tools -## Migration tool -GOOSE = $(BIN)/goose -$(BIN)/goose: - go get -u github.com/pressly/goose/cmd/goose - -.PHONY: installtools -installtools: | $(GOOSE) - echo "Installing tools" - -#Database -HOST_NAME = localhost -PORT = 5432 -NAME = -USER = postgres -PASSWORD = password -CONNECT_STRING=postgresql://$(USER):$(PASSWORD)@$(HOST_NAME):$(PORT)/$(NAME)?sslmode=disable - -# Parameter checks -## Check that DB variables are provided -.PHONY: checkdbvars -checkdbvars: - test -n "$(HOST_NAME)" # $$HOST_NAME - test -n "$(PORT)" # $$PORT - test -n "$(NAME)" # $$NAME - @echo $(CONNECT_STRING) - -## Check that the migration variable (id/timestamp) is provided -.PHONY: checkmigration -checkmigration: - test -n "$(MIGRATION)" # $$MIGRATION - -# Check that the migration name is provided -.PHONY: checkmigname -checkmigname: - test -n "$(NAME)" # $$NAME - -# Migration operations -## Rollback the last migration -.PHONY: rollback -rollback: $(GOOSE) checkdbvars - $(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" down - pg_dump -O -s $(CONNECT_STRING) > schema.sql - -## Rollback to a select migration (id/timestamp) -.PHONY: rollback_to -rollback_to: $(GOOSE) checkmigration checkdbvars - $(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" down-to "$(MIGRATION)" - -## Rollback pre_batch_set -.PHONY: `rollback_pre_batch_set` -rollback_pre_batch_set: $(GOOSE) checkdbvars - $(GOOSE) -dir db/pre_batch_processing_migrations postgres "$(CONNECT_STRING)" down - -## Rollback post_batch_set -.PHONY: rollback_post_batch_set -rollback_post_batch_set: $(GOOSE) checkdbvars - $(GOOSE) -dir db/post_batch_processing_migrations postgres "$(CONNECT_STRING)" down - -## Apply the next up migration -.PHONY: migrate_up_by_one -migrate_up_by_one: $(GOOSE) checkdbvars - $(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" up-by-one - -## Apply all migrations not already run -.PHONY: migrate -migrate: $(GOOSE) checkdbvars - $(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" up - pg_dump -O -s $(CONNECT_STRING) > schema.sql - -## Apply all the migrations used to generate a UML diagram (containing FKs) -.PHONY: migrate_for_uml -migrate_for_uml: $(GOOSE) checkdbvars - $(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" up-to 00018 - -## Apply migrations to be ran before a batch processing -.PHONY: migrate_pre_batch_set -migrate_pre_batch_set: $(GOOSE) checkdbvars - $(GOOSE) -dir db/pre_batch_processing_migrations postgres "$(CONNECT_STRING)" up - -## Apply migrations to be ran after a batch processing, one-by-one -.PHONY: migrate_post_batch_set_up_by_one -migrate_post_batch_set_up_by_one: $(GOOSE) checkdbvars - $(GOOSE) -dir db/post_batch_processing_migrations postgres "$(CONNECT_STRING)" up-by-one - -## Apply migrations to be ran after a batch processing -.PHONY: migrate_post_batch_set -migrate_post_batch_set: $(GOOSE) checkdbvars - $(GOOSE) -dir db/post_batch_processing_migrations postgres "$(CONNECT_STRING)" up - -## Create a new migration file -.PHONY: new_migration -new_migration: $(GOOSE) checkmigname - $(GOOSE) -dir db/migrations create $(NAME) sql - -## Check which migrations are applied at the moment -.PHONY: migration_status -migration_status: $(GOOSE) checkdbvars - $(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" status - -# Convert timestamped migrations to versioned (to be run in CI); -# merge timestamped files to prevent conflict -.PHONY: version_migrations -version_migrations: - $(GOOSE) -dir db/migrations fix - -# Import a psql schema to the database -.PHONY: import -import: - test -n "$(NAME)" # $$NAME - psql $(NAME) < schema.sql - -.PHONY: test-migrations -test-migrations: $(GOOSE) - ./scripts/test_migration.sh -- 2.45.2 From 60e83a1e2c1170c3bd4aed4904e05e585172a2a1 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Mon, 8 Jul 2024 17:48:16 +0800 Subject: [PATCH 2/4] Simplify startup script --- scripts/startup_script.sh | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/scripts/startup_script.sh b/scripts/startup_script.sh index 9e07753..66a29a5 100755 --- a/scripts/startup_script.sh +++ b/scripts/startup_script.sh @@ -1,5 +1,4 @@ #!/bin/sh -# Runs the db migrations set -e # Default command is "goose up" @@ -11,14 +10,5 @@ fi VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable # Run the DB migrations -echo "Connecting with: $VDB_PG_CONNECT" -echo "Running database migrations" -./goose -dir migrations postgres "$VDB_PG_CONNECT" "$@" - -# If the db migrations ran without err -if [[ $? -eq 0 ]]; then - echo "Migration process ran successfully" -else - echo "Could not run migrations. Are the database details correct?" - exit 1 -fi +set -x +exec ./goose -dir migrations postgres "$VDB_PG_CONNECT" "$@" -- 2.45.2 From 5661187a3298a92b8f24e073fca266398557e5ef Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Mon, 8 Jul 2024 17:49:37 +0800 Subject: [PATCH 3/4] Add CI sanity check + compose healthcheck --- .gitea/workflows/on-pr.yaml | 30 ++++++++++++++++-------------- compose.yml | 10 ++++++++-- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/.gitea/workflows/on-pr.yaml b/.gitea/workflows/on-pr.yaml index 6278c52..0949d02 100644 --- a/.gitea/workflows/on-pr.yaml +++ b/.gitea/workflows/on-pr.yaml @@ -1,20 +1,22 @@ -name: Docker Build - +name: Basic test on: [pull_request] jobs: - build: - name: Run docker build + basic-test: + name: Build and sanity check runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Run docker build + - uses: actions/checkout@v4 + - name: Build Docker images run: docker compose build - test: - # Add a dummy job to avoid failing GitHub CI checks. - # Other checks to be added later. - name: Dummy job - runs-on: ubuntu-latest - steps: - - name: Run dummy job - run: echo "Empty dummy job" + - name: Run Docker containers + run: docker compose up -d + - name: Check migration version + timeout-minutes: 1 + run: | + while + version=$(docker compose run --rm migrations version 2>&1 | tail -1 | awk '{print $(NF)}') + [[ $version != 21 ]]; do + echo "Incorrect version: $version" + echo "Retrying..." + done diff --git a/compose.yml b/compose.yml index 2849e33..8caf840 100644 --- a/compose.yml +++ b/compose.yml @@ -2,13 +2,14 @@ services: migrations: restart: on-failure depends_on: - - ipld-eth-db + ipld-eth-db: + condition: service_healthy # Use local build build: context: . dockerfile: Dockerfile # Use an existing image - image: cerc/ipld-eth-db + image: cerc/ipld-eth-db:local environment: DATABASE_USER: "vdbm" DATABASE_NAME: "cerc_testing" @@ -26,3 +27,8 @@ services: POSTGRES_PASSWORD: "password" ports: - "127.0.0.1:8077:5432" + healthcheck: + test: ["CMD", "pg_isready", "-U", "vdbm"] + interval: 2s + timeout: 1s + retries: 3 -- 2.45.2 From b16620bf4d8d468ca05b3aa3fcb65bc62ba837e0 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Fri, 28 Jun 2024 20:36:18 +0800 Subject: [PATCH 4/4] Update schema.sql --- schema.sql | 58 ++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 48 insertions(+), 10 deletions(-) diff --git a/schema.sql b/schema.sql index 5c924ba..d426acc 100644 --- a/schema.sql +++ b/schema.sql @@ -2,8 +2,8 @@ -- PostgreSQL database dump -- --- Dumped from database version 14.8 --- Dumped by pg_dump version 14.8 (Ubuntu 14.8-0ubuntu0.22.04.1) +-- Dumped from database version 14.12 +-- Dumped by pg_dump version 14.12 SET statement_timeout = 0; SET lock_timeout = 0; @@ -209,7 +209,8 @@ CREATE TABLE eth.header_cids ( bloom bytea NOT NULL, "timestamp" bigint NOT NULL, coinbase character varying(66) NOT NULL, - canonical boolean DEFAULT true NOT NULL + canonical boolean DEFAULT true NOT NULL, + withdrawals_root character varying(66) NOT NULL ); @@ -333,6 +334,21 @@ CREATE TABLE eth.uncle_cids ( ); +-- +-- Name: withdrawal_cids; Type: TABLE; Schema: eth; Owner: - +-- + +CREATE TABLE eth.withdrawal_cids ( + block_number bigint NOT NULL, + header_id character varying(66) NOT NULL, + cid text NOT NULL, + index integer NOT NULL, + validator integer NOT NULL, + address character varying(66) NOT NULL, + amount numeric NOT NULL +); + + -- -- Name: watched_addresses; Type: TABLE; Schema: eth_meta; Owner: - -- @@ -490,6 +506,14 @@ ALTER TABLE ONLY eth.uncle_cids ADD CONSTRAINT uncle_cids_pkey PRIMARY KEY (block_hash, block_number); +-- +-- Name: withdrawal_cids withdrawal_cids_pkey; Type: CONSTRAINT; Schema: eth; Owner: - +-- + +ALTER TABLE ONLY eth.withdrawal_cids + ADD CONSTRAINT withdrawal_cids_pkey PRIMARY KEY (index, header_id, block_number); + + -- -- Name: watched_addresses watched_addresses_pkey; Type: CONSTRAINT; Schema: eth_meta; Owner: - -- @@ -782,6 +806,13 @@ CREATE UNIQUE INDEX uncle_cid_block_number_index ON eth.uncle_cids USING btree ( CREATE INDEX uncle_header_id_index ON eth.uncle_cids USING btree (header_id); +-- +-- Name: withdrawal_cids_block_number_idx; Type: INDEX; Schema: eth; Owner: - +-- + +CREATE INDEX withdrawal_cids_block_number_idx ON eth.withdrawal_cids USING btree (block_number DESC); + + -- -- Name: blocks_block_number_idx; Type: INDEX; Schema: ipld; Owner: - -- @@ -793,49 +824,56 @@ CREATE INDEX blocks_block_number_idx ON ipld.blocks USING btree (block_number DE -- Name: log_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: - -- -CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.log_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_internal.insert_blocker(); +CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.log_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.insert_blocker(); -- -- Name: receipt_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: - -- -CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.receipt_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_internal.insert_blocker(); +CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.receipt_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.insert_blocker(); -- -- Name: state_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: - -- -CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.state_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_internal.insert_blocker(); +CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.state_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.insert_blocker(); -- -- Name: storage_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: - -- -CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.storage_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_internal.insert_blocker(); +CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.storage_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.insert_blocker(); -- -- Name: transaction_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: - -- -CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.transaction_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_internal.insert_blocker(); +CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.transaction_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.insert_blocker(); -- -- Name: uncle_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: - -- -CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.uncle_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_internal.insert_blocker(); +CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.uncle_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.insert_blocker(); + + +-- +-- Name: withdrawal_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: - +-- + +CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.withdrawal_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.insert_blocker(); -- -- Name: blocks ts_insert_blocker; Type: TRIGGER; Schema: ipld; Owner: - -- -CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON ipld.blocks FOR EACH ROW EXECUTE FUNCTION _timescaledb_internal.insert_blocker(); +CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON ipld.blocks FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.insert_blocker(); -- -- 2.45.2