Merge branch 'v5' into add-blob-hashes-table

This commit is contained in:
Roy Crihfield 2024-07-25 19:09:10 +08:00
commit d0fffa2deb
8 changed files with 79 additions and 329 deletions

View File

@ -1,78 +1,22 @@
name: Docker Build name: Basic test
on: [pull_request] on: [pull_request]
jobs: jobs:
build: basic-test:
name: Run docker build name: Build and sanity check
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v4
- name: Run docker build - name: Build Docker images
run: docker compose build run: docker compose build
test: - name: Run Docker containers
# Add a dummy job to avoid failing GitHub CI checks. run: docker compose up -d
# Other checks to be added later. - name: Check migration version
name: Dummy job timeout-minutes: 1
runs-on: ubuntu-latest run: |
steps: while
- name: Run dummy job version=$(docker compose run --rm migrations version 2>&1 | tail -1 | awk '{print $(NF)}')
run: echo "Empty dummy job" [[ $version != 21 ]]; do
# concise_migration_diff: echo "Incorrect version: $version"
# name: Verify concise migration and generated schema echo "Retrying..."
# runs-on: ubuntu-latest done
# steps:
# - uses: actions/checkout@v2
# - name: Run docker concise migration build
# run: make docker-concise-migration-build
# - name: Run database
# run: docker-compose -f docker-compose.test.yml up -d test-db
# - name: Test concise migration
# run: |
# sleep 10
# docker run --rm --network host -e DATABASE_USER=vdbm -e DATABASE_PASSWORD=password \
# -e DATABASE_HOSTNAME=127.0.0.1 -e DATABASE_PORT=8066 -e DATABASE_NAME=vulcanize_testing \
# vulcanize/concise-migration-build
# - name: Verify schema is latest
# run: |
# PGPASSWORD="password" pg_dump -h localhost -p 8066 -U vdbm vulcanize_testing --no-owner --schema-only > ./db/migration_schema.sql
# ./scripts/check_diff.sh ./db/migration_schema.sql db/schema.sql
# incremental_migration_diff:
# name: Compare conscise migration schema with incremental migration.
# runs-on: ubuntu-latest
# steps:
# - uses: actions/checkout@v2
# - name: Run database
# run: docker-compose -f docker-compose.test.yml up -d test-db statediff-migrations
# - name: Test incremental migration
# run: |
# sleep 10
# docker run --rm --network host -e DATABASE_USER=vdbm -e DATABASE_PASSWORD=password \
# -e DATABASE_HOSTNAME=127.0.0.1 -e DATABASE_PORT=8066 -e DATABASE_NAME=vulcanize_testing \
# vulcanize/statediff-migrations:v0.9.0
# - name: Verify schema is latest
# run: |
# PGPASSWORD="password" pg_dump -h localhost -p 8066 -U vdbm vulcanize_testing --no-owner --schema-only > ./db/migration_schema.sql
# ./scripts/check_diff.sh db/schema.sql ./db/migration_schema.sql
# migration:
# name: Compare up and down migration
# env:
# GOPATH: /tmp/go
# strategy:
# matrix:
# go-version: [ 1.16.x ]
# os: [ ubuntu-latest ]
# runs-on: ${{ matrix.os }}
# steps:
# - name: Create GOPATH
# run: mkdir -p /tmp/go
# - name: Install Go
# uses: actions/setup-go@v2
# with:
# go-version: ${{ matrix.go-version }}
# - uses: actions/checkout@v2
# - name: Test migration
# run: |
# timeout 5m make test-migrations

View File

@ -1,78 +0,0 @@
name: Docker Build
on: [pull_request]
jobs:
build:
name: Run docker build
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Run docker build
run: docker compose build
test:
# Add a dummy job to avoid failing GitHub CI checks.
# Other checks to be added later.
name: Dummy job
runs-on: ubuntu-latest
steps:
- name: Run dummy job
run: echo "Empty dummy job"
# concise_migration_diff:
# name: Verify concise migration and generated schema
# runs-on: ubuntu-latest
# steps:
# - uses: actions/checkout@v2
# - name: Run docker concise migration build
# run: make docker-concise-migration-build
# - name: Run database
# run: docker-compose -f docker-compose.test.yml up -d test-db
# - name: Test concise migration
# run: |
# sleep 10
# docker run --rm --network host -e DATABASE_USER=vdbm -e DATABASE_PASSWORD=password \
# -e DATABASE_HOSTNAME=127.0.0.1 -e DATABASE_PORT=8066 -e DATABASE_NAME=vulcanize_testing \
# vulcanize/concise-migration-build
# - name: Verify schema is latest
# run: |
# PGPASSWORD="password" pg_dump -h localhost -p 8066 -U vdbm vulcanize_testing --no-owner --schema-only > ./db/migration_schema.sql
# ./scripts/check_diff.sh ./db/migration_schema.sql db/schema.sql
# incremental_migration_diff:
# name: Compare conscise migration schema with incremental migration.
# runs-on: ubuntu-latest
# steps:
# - uses: actions/checkout@v2
# - name: Run database
# run: docker-compose -f docker-compose.test.yml up -d test-db statediff-migrations
# - name: Test incremental migration
# run: |
# sleep 10
# docker run --rm --network host -e DATABASE_USER=vdbm -e DATABASE_PASSWORD=password \
# -e DATABASE_HOSTNAME=127.0.0.1 -e DATABASE_PORT=8066 -e DATABASE_NAME=vulcanize_testing \
# vulcanize/statediff-migrations:v0.9.0
# - name: Verify schema is latest
# run: |
# PGPASSWORD="password" pg_dump -h localhost -p 8066 -U vdbm vulcanize_testing --no-owner --schema-only > ./db/migration_schema.sql
# ./scripts/check_diff.sh db/schema.sql ./db/migration_schema.sql
# migration:
# name: Compare up and down migration
# env:
# GOPATH: /tmp/go
# strategy:
# matrix:
# go-version: [ 1.16.x ]
# os: [ ubuntu-latest ]
# runs-on: ${{ matrix.os }}
# steps:
# - name: Create GOPATH
# run: mkdir -p /tmp/go
# - name: Install Go
# uses: actions/setup-go@v2
# with:
# go-version: ${{ matrix.go-version }}
# - uses: actions/checkout@v2
# - name: Test migration
# run: |
# timeout 5m make test-migrations

View File

@ -1,26 +0,0 @@
name: Publish Docker image
on:
release:
types: [published, edited]
jobs:
build:
name: Build and publish image
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- id: vars
name: Output SHA and version tag
run: |
echo "sha=${GITHUB_SHA:0:7}" >> $GITHUB_OUTPUT
echo "tag=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
- name: Build and tag image
run: |
docker build . \
-t cerc-io/ipld-eth-db \
-t git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.sha}} \
-t git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.tag}}
- name: Push image tags
run: |
echo ${{ secrets.GITEA_PUBLISH_TOKEN }} | docker login https://git.vdb.to -u cerccicd --password-stdin
docker push git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.sha}}
docker push git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.tag}}

View File

@ -1,12 +1,10 @@
FROM golang:1.18-alpine as builder FROM alpine as builder
ADD . /go/src/github.com/cerc-io/ipld-eth-db
# Get migration tool # Get migration tool
WORKDIR / WORKDIR /
ARG GOOSE_VER="v3.6.1" ARG GOOSE_VERSION="v3.6.1"
RUN arch=$(arch | sed s/aarch64/arm64/) && \ RUN arch=$(arch | sed s/aarch64/arm64/) && \
wget -O ./goose https://github.com/pressly/goose/releases/download/${GOOSE_VER}/goose_linux_${arch} wget -O ./goose https://github.com/pressly/goose/releases/download/${GOOSE_VERSION}/goose_linux_${arch}
RUN chmod +x ./goose RUN chmod +x ./goose
# app container # app container
@ -14,9 +12,8 @@ FROM alpine
WORKDIR /app WORKDIR /app
COPY --from=builder /go/src/github.com/cerc-io/ipld-eth-db/scripts/startup_script.sh .
COPY --from=builder /goose goose COPY --from=builder /goose goose
COPY --from=builder /go/src/github.com/cerc-io/ipld-eth-db/db/migrations migrations ADD scripts/startup_script.sh .
ADD db/migrations migrations
ENTRYPOINT ["/app/startup_script.sh"] ENTRYPOINT ["/app/startup_script.sh"]

121
Makefile
View File

@ -1,121 +0,0 @@
ifndef GOPATH
override GOPATH = $(HOME)/go
endif
BIN = $(GOPATH)/bin
# Tools
## Migration tool
GOOSE = $(BIN)/goose
$(BIN)/goose:
go get -u github.com/pressly/goose/cmd/goose
.PHONY: installtools
installtools: | $(GOOSE)
echo "Installing tools"
#Database
HOST_NAME = localhost
PORT = 5432
NAME =
USER = postgres
PASSWORD = password
CONNECT_STRING=postgresql://$(USER):$(PASSWORD)@$(HOST_NAME):$(PORT)/$(NAME)?sslmode=disable
# Parameter checks
## Check that DB variables are provided
.PHONY: checkdbvars
checkdbvars:
test -n "$(HOST_NAME)" # $$HOST_NAME
test -n "$(PORT)" # $$PORT
test -n "$(NAME)" # $$NAME
@echo $(CONNECT_STRING)
## Check that the migration variable (id/timestamp) is provided
.PHONY: checkmigration
checkmigration:
test -n "$(MIGRATION)" # $$MIGRATION
# Check that the migration name is provided
.PHONY: checkmigname
checkmigname:
test -n "$(NAME)" # $$NAME
# Migration operations
## Rollback the last migration
.PHONY: rollback
rollback: $(GOOSE) checkdbvars
$(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" down
pg_dump -O -s $(CONNECT_STRING) > schema.sql
## Rollback to a select migration (id/timestamp)
.PHONY: rollback_to
rollback_to: $(GOOSE) checkmigration checkdbvars
$(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" down-to "$(MIGRATION)"
## Rollback pre_batch_set
.PHONY: `rollback_pre_batch_set`
rollback_pre_batch_set: $(GOOSE) checkdbvars
$(GOOSE) -dir db/pre_batch_processing_migrations postgres "$(CONNECT_STRING)" down
## Rollback post_batch_set
.PHONY: rollback_post_batch_set
rollback_post_batch_set: $(GOOSE) checkdbvars
$(GOOSE) -dir db/post_batch_processing_migrations postgres "$(CONNECT_STRING)" down
## Apply the next up migration
.PHONY: migrate_up_by_one
migrate_up_by_one: $(GOOSE) checkdbvars
$(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" up-by-one
## Apply all migrations not already run
.PHONY: migrate
migrate: $(GOOSE) checkdbvars
$(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" up
pg_dump -O -s $(CONNECT_STRING) > schema.sql
## Apply all the migrations used to generate a UML diagram (containing FKs)
.PHONY: migrate_for_uml
migrate_for_uml: $(GOOSE) checkdbvars
$(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" up-to 00018
## Apply migrations to be ran before a batch processing
.PHONY: migrate_pre_batch_set
migrate_pre_batch_set: $(GOOSE) checkdbvars
$(GOOSE) -dir db/pre_batch_processing_migrations postgres "$(CONNECT_STRING)" up
## Apply migrations to be ran after a batch processing, one-by-one
.PHONY: migrate_post_batch_set_up_by_one
migrate_post_batch_set_up_by_one: $(GOOSE) checkdbvars
$(GOOSE) -dir db/post_batch_processing_migrations postgres "$(CONNECT_STRING)" up-by-one
## Apply migrations to be ran after a batch processing
.PHONY: migrate_post_batch_set
migrate_post_batch_set: $(GOOSE) checkdbvars
$(GOOSE) -dir db/post_batch_processing_migrations postgres "$(CONNECT_STRING)" up
## Create a new migration file
.PHONY: new_migration
new_migration: $(GOOSE) checkmigname
$(GOOSE) -dir db/migrations create $(NAME) sql
## Check which migrations are applied at the moment
.PHONY: migration_status
migration_status: $(GOOSE) checkdbvars
$(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" status
# Convert timestamped migrations to versioned (to be run in CI);
# merge timestamped files to prevent conflict
.PHONY: version_migrations
version_migrations:
$(GOOSE) -dir db/migrations fix
# Import a psql schema to the database
.PHONY: import
import:
test -n "$(NAME)" # $$NAME
psql $(NAME) < schema.sql
.PHONY: test-migrations
test-migrations: $(GOOSE)
./scripts/test_migration.sh

View File

@ -2,13 +2,14 @@ services:
migrations: migrations:
restart: on-failure restart: on-failure
depends_on: depends_on:
- ipld-eth-db ipld-eth-db:
condition: service_healthy
# Use local build # Use local build
build: build:
context: . context: .
dockerfile: Dockerfile dockerfile: Dockerfile
# Use an existing image # Use an existing image
image: cerc/ipld-eth-db image: cerc/ipld-eth-db:local
environment: environment:
DATABASE_USER: "vdbm" DATABASE_USER: "vdbm"
DATABASE_NAME: "cerc_testing" DATABASE_NAME: "cerc_testing"
@ -26,3 +27,8 @@ services:
POSTGRES_PASSWORD: "password" POSTGRES_PASSWORD: "password"
ports: ports:
- "127.0.0.1:8077:5432" - "127.0.0.1:8077:5432"
healthcheck:
test: ["CMD", "pg_isready", "-U", "vdbm"]
interval: 2s
timeout: 1s
retries: 3

View File

@ -2,8 +2,8 @@
-- PostgreSQL database dump -- PostgreSQL database dump
-- --
-- Dumped from database version 14.8 -- Dumped from database version 14.12
-- Dumped by pg_dump version 14.8 (Ubuntu 14.8-0ubuntu0.22.04.1) -- Dumped by pg_dump version 14.12
SET statement_timeout = 0; SET statement_timeout = 0;
SET lock_timeout = 0; SET lock_timeout = 0;
@ -209,7 +209,8 @@ CREATE TABLE eth.header_cids (
bloom bytea NOT NULL, bloom bytea NOT NULL,
"timestamp" bigint NOT NULL, "timestamp" bigint NOT NULL,
coinbase character varying(66) NOT NULL, coinbase character varying(66) NOT NULL,
canonical boolean DEFAULT true NOT NULL canonical boolean DEFAULT true NOT NULL,
withdrawals_root character varying(66) NOT NULL
); );
@ -333,6 +334,21 @@ CREATE TABLE eth.uncle_cids (
); );
--
-- Name: withdrawal_cids; Type: TABLE; Schema: eth; Owner: -
--
CREATE TABLE eth.withdrawal_cids (
block_number bigint NOT NULL,
header_id character varying(66) NOT NULL,
cid text NOT NULL,
index integer NOT NULL,
validator integer NOT NULL,
address character varying(66) NOT NULL,
amount numeric NOT NULL
);
-- --
-- Name: watched_addresses; Type: TABLE; Schema: eth_meta; Owner: - -- Name: watched_addresses; Type: TABLE; Schema: eth_meta; Owner: -
-- --
@ -490,6 +506,14 @@ ALTER TABLE ONLY eth.uncle_cids
ADD CONSTRAINT uncle_cids_pkey PRIMARY KEY (block_hash, block_number); ADD CONSTRAINT uncle_cids_pkey PRIMARY KEY (block_hash, block_number);
--
-- Name: withdrawal_cids withdrawal_cids_pkey; Type: CONSTRAINT; Schema: eth; Owner: -
--
ALTER TABLE ONLY eth.withdrawal_cids
ADD CONSTRAINT withdrawal_cids_pkey PRIMARY KEY (index, header_id, block_number);
-- --
-- Name: watched_addresses watched_addresses_pkey; Type: CONSTRAINT; Schema: eth_meta; Owner: - -- Name: watched_addresses watched_addresses_pkey; Type: CONSTRAINT; Schema: eth_meta; Owner: -
-- --
@ -782,6 +806,13 @@ CREATE UNIQUE INDEX uncle_cid_block_number_index ON eth.uncle_cids USING btree (
CREATE INDEX uncle_header_id_index ON eth.uncle_cids USING btree (header_id); CREATE INDEX uncle_header_id_index ON eth.uncle_cids USING btree (header_id);
--
-- Name: withdrawal_cids_block_number_idx; Type: INDEX; Schema: eth; Owner: -
--
CREATE INDEX withdrawal_cids_block_number_idx ON eth.withdrawal_cids USING btree (block_number DESC);
-- --
-- Name: blocks_block_number_idx; Type: INDEX; Schema: ipld; Owner: - -- Name: blocks_block_number_idx; Type: INDEX; Schema: ipld; Owner: -
-- --
@ -793,49 +824,56 @@ CREATE INDEX blocks_block_number_idx ON ipld.blocks USING btree (block_number DE
-- Name: log_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: - -- Name: log_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: -
-- --
CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.log_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_internal.insert_blocker(); CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.log_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.insert_blocker();
-- --
-- Name: receipt_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: - -- Name: receipt_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: -
-- --
CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.receipt_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_internal.insert_blocker(); CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.receipt_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.insert_blocker();
-- --
-- Name: state_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: - -- Name: state_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: -
-- --
CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.state_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_internal.insert_blocker(); CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.state_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.insert_blocker();
-- --
-- Name: storage_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: - -- Name: storage_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: -
-- --
CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.storage_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_internal.insert_blocker(); CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.storage_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.insert_blocker();
-- --
-- Name: transaction_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: - -- Name: transaction_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: -
-- --
CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.transaction_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_internal.insert_blocker(); CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.transaction_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.insert_blocker();
-- --
-- Name: uncle_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: - -- Name: uncle_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: -
-- --
CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.uncle_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_internal.insert_blocker(); CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.uncle_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.insert_blocker();
--
-- Name: withdrawal_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: -
--
CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.withdrawal_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.insert_blocker();
-- --
-- Name: blocks ts_insert_blocker; Type: TRIGGER; Schema: ipld; Owner: - -- Name: blocks ts_insert_blocker; Type: TRIGGER; Schema: ipld; Owner: -
-- --
CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON ipld.blocks FOR EACH ROW EXECUTE FUNCTION _timescaledb_internal.insert_blocker(); CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON ipld.blocks FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.insert_blocker();
-- --

View File

@ -1,5 +1,4 @@
#!/bin/sh #!/bin/sh
# Runs the db migrations
set -e set -e
# Default command is "goose up" # Default command is "goose up"
@ -11,14 +10,5 @@ fi
VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable
# Run the DB migrations # Run the DB migrations
echo "Connecting with: $VDB_PG_CONNECT" set -x
echo "Running database migrations" exec ./goose -dir migrations postgres "$VDB_PG_CONNECT" "$@"
./goose -dir migrations postgres "$VDB_PG_CONNECT" "$@"
# If the db migrations ran without err
if [[ $? -eq 0 ]]; then
echo "Migration process ran successfully"
else
echo "Could not run migrations. Are the database details correct?"
exit 1
fi