Merge branch 'v5' into add-blob-hashes-table

.gitea/workflows/on-pr.yaml

@@ -1,78 +1,22 @@
-name: Docker Build
+name: Basic test
-
 on: [pull_request]
 
 jobs:
-  build:
+  basic-test:
-    name: Run docker build
+    name: Build and sanity check
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
-      - name: Run docker build
+      - name: Build Docker images
         run: docker compose build
-  test:
+      - name: Run Docker containers
-    # Add a dummy job to avoid failing GitHub CI checks.
+        run: docker compose up -d
-    # Other checks to be added later.
+      - name: Check migration version
-    name: Dummy job
+        timeout-minutes: 1
-    runs-on: ubuntu-latest
+        run: |
-    steps:
+          while
-      - name: Run dummy job
+            version=$(docker compose run --rm migrations version 2>&1 | tail -1 | awk '{print $(NF)}')
-        run: echo "Empty dummy job"
+            [[ $version != 21 ]]; do
-  # concise_migration_diff:
+            echo "Incorrect version: $version"
-  #   name: Verify concise migration and generated schema
+            echo "Retrying..."
-  #   runs-on: ubuntu-latest
+          done
-  #   steps:
-  #     - uses: actions/checkout@v2
-  #     - name: Run docker concise migration build
-  #       run: make docker-concise-migration-build
-  #     - name: Run database
-  #       run: docker-compose -f docker-compose.test.yml up -d test-db
-  #     - name: Test concise migration
-  #       run: |
-  #         sleep 10
-  #         docker run --rm --network host -e DATABASE_USER=vdbm -e DATABASE_PASSWORD=password \
-  #         -e DATABASE_HOSTNAME=127.0.0.1 -e DATABASE_PORT=8066 -e DATABASE_NAME=vulcanize_testing \
-  #         vulcanize/concise-migration-build
-  #     - name: Verify schema is latest
-  #       run: |
-  #         PGPASSWORD="password" pg_dump -h localhost -p 8066 -U vdbm vulcanize_testing --no-owner --schema-only > ./db/migration_schema.sql
-  #         ./scripts/check_diff.sh ./db/migration_schema.sql db/schema.sql
-
-  # incremental_migration_diff:
-  #   name: Compare conscise migration schema with incremental migration.
-  #   runs-on: ubuntu-latest
-  #   steps:
-  #     - uses: actions/checkout@v2
-  #     - name: Run database
-  #       run: docker-compose -f docker-compose.test.yml up -d test-db statediff-migrations
-  #     - name: Test incremental migration
-  #       run: |
-  #         sleep 10
-  #         docker run --rm --network host -e DATABASE_USER=vdbm -e DATABASE_PASSWORD=password \
-  #         -e DATABASE_HOSTNAME=127.0.0.1 -e DATABASE_PORT=8066 -e DATABASE_NAME=vulcanize_testing \
-  #         vulcanize/statediff-migrations:v0.9.0
-  #     - name: Verify schema is latest
-  #       run: |
-  #         PGPASSWORD="password" pg_dump -h localhost -p 8066 -U vdbm vulcanize_testing --no-owner --schema-only > ./db/migration_schema.sql
-  #         ./scripts/check_diff.sh db/schema.sql ./db/migration_schema.sql
-
-  # migration:
-  #   name: Compare up and down migration
-  #   env:
-  #     GOPATH: /tmp/go
-  #   strategy:
-  #     matrix:
-  #       go-version: [ 1.16.x ]
-  #       os: [ ubuntu-latest ]
-  #   runs-on: ${{ matrix.os }}
-  #   steps:
-  #     - name: Create GOPATH
-  #       run: mkdir -p /tmp/go
-  #     - name: Install Go
-  #       uses: actions/setup-go@v2
-  #       with:
-  #         go-version: ${{ matrix.go-version }}
-  #     - uses: actions/checkout@v2
-  #     - name: Test migration
-  #       run: |
-  #         timeout 5m make test-migrations

.github/workflows/on-pr.yaml

@@ -1,78 +0,0 @@
-name: Docker Build
-
-on: [pull_request]
-
-jobs:
-  build:
-    name: Run docker build
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Run docker build
-        run: docker compose build
-  test:
-    # Add a dummy job to avoid failing GitHub CI checks.
-    # Other checks to be added later.
-    name: Dummy job
-    runs-on: ubuntu-latest
-    steps:
-      - name: Run dummy job
-        run: echo "Empty dummy job"
-  # concise_migration_diff:
-  #   name: Verify concise migration and generated schema
-  #   runs-on: ubuntu-latest
-  #   steps:
-  #     - uses: actions/checkout@v2
-  #     - name: Run docker concise migration build
-  #       run: make docker-concise-migration-build
-  #     - name: Run database
-  #       run: docker-compose -f docker-compose.test.yml up -d test-db
-  #     - name: Test concise migration
-  #       run: |
-  #         sleep 10
-  #         docker run --rm --network host -e DATABASE_USER=vdbm -e DATABASE_PASSWORD=password \
-  #         -e DATABASE_HOSTNAME=127.0.0.1 -e DATABASE_PORT=8066 -e DATABASE_NAME=vulcanize_testing \
-  #         vulcanize/concise-migration-build
-  #     - name: Verify schema is latest
-  #       run: |
-  #         PGPASSWORD="password" pg_dump -h localhost -p 8066 -U vdbm vulcanize_testing --no-owner --schema-only > ./db/migration_schema.sql
-  #         ./scripts/check_diff.sh ./db/migration_schema.sql db/schema.sql
-
-  # incremental_migration_diff:
-  #   name: Compare conscise migration schema with incremental migration.
-  #   runs-on: ubuntu-latest
-  #   steps:
-  #     - uses: actions/checkout@v2
-  #     - name: Run database
-  #       run: docker-compose -f docker-compose.test.yml up -d test-db statediff-migrations
-  #     - name: Test incremental migration
-  #       run: |
-  #         sleep 10
-  #         docker run --rm --network host -e DATABASE_USER=vdbm -e DATABASE_PASSWORD=password \
-  #         -e DATABASE_HOSTNAME=127.0.0.1 -e DATABASE_PORT=8066 -e DATABASE_NAME=vulcanize_testing \
-  #         vulcanize/statediff-migrations:v0.9.0
-  #     - name: Verify schema is latest
-  #       run: |
-  #         PGPASSWORD="password" pg_dump -h localhost -p 8066 -U vdbm vulcanize_testing --no-owner --schema-only > ./db/migration_schema.sql
-  #         ./scripts/check_diff.sh db/schema.sql ./db/migration_schema.sql
-
-  # migration:
-  #   name: Compare up and down migration
-  #   env:
-  #     GOPATH: /tmp/go
-  #   strategy:
-  #     matrix:
-  #       go-version: [ 1.16.x ]
-  #       os: [ ubuntu-latest ]
-  #   runs-on: ${{ matrix.os }}
-  #   steps:
-  #     - name: Create GOPATH
-  #       run: mkdir -p /tmp/go
-  #     - name: Install Go
-  #       uses: actions/setup-go@v2
-  #       with:
-  #         go-version: ${{ matrix.go-version }}
-  #     - uses: actions/checkout@v2
-  #     - name: Test migration
-  #       run: |
-  #         timeout 5m make test-migrations

.github/workflows/publish.yaml

@@ -1,26 +0,0 @@
-name: Publish Docker image
-on:
-  release:
-    types: [published, edited]
-jobs:
-  build:
-    name: Build and publish image
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - id: vars
-        name: Output SHA and version tag
-        run: |
-          echo "sha=${GITHUB_SHA:0:7}" >> $GITHUB_OUTPUT
-          echo "tag=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
-      - name: Build and tag image
-        run: |
-          docker build . \
-          -t cerc-io/ipld-eth-db \
-          -t git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.sha}} \
-          -t git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.tag}}
-      - name: Push image tags
-        run: |
-          echo ${{ secrets.GITEA_PUBLISH_TOKEN }} | docker login https://git.vdb.to -u cerccicd --password-stdin
-          docker push git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.sha}}
-          docker push git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.tag}}

Dockerfile

@@ -1,12 +1,10 @@
-FROM golang:1.18-alpine as builder
+FROM alpine as builder
-
-ADD . /go/src/github.com/cerc-io/ipld-eth-db
 
 # Get migration tool
 WORKDIR /
-ARG GOOSE_VER="v3.6.1"
+ARG GOOSE_VERSION="v3.6.1"
 RUN arch=$(arch | sed s/aarch64/arm64/) && \
-  wget -O ./goose https://github.com/pressly/goose/releases/download/${GOOSE_VER}/goose_linux_${arch}
+  wget -O ./goose https://github.com/pressly/goose/releases/download/${GOOSE_VERSION}/goose_linux_${arch}
 RUN chmod +x ./goose
 
 # app container
@@ -14,9 +12,8 @@ FROM alpine
 
 WORKDIR /app
 
-COPY --from=builder /go/src/github.com/cerc-io/ipld-eth-db/scripts/startup_script.sh .
-
 COPY --from=builder /goose goose
-COPY --from=builder /go/src/github.com/cerc-io/ipld-eth-db/db/migrations migrations
+ADD scripts/startup_script.sh .
+ADD db/migrations migrations
 
 ENTRYPOINT ["/app/startup_script.sh"]

Makefile

@@ -1,121 +0,0 @@
-ifndef GOPATH
-override GOPATH = $(HOME)/go
-endif
-
-BIN = $(GOPATH)/bin
-
-# Tools
-## Migration tool
-GOOSE = $(BIN)/goose
-$(BIN)/goose:
-	go get -u github.com/pressly/goose/cmd/goose
-
-.PHONY: installtools
-installtools: | $(GOOSE)
-	echo "Installing tools"
-
-#Database
-HOST_NAME = localhost
-PORT = 5432
-NAME =
-USER = postgres
-PASSWORD = password
-CONNECT_STRING=postgresql://$(USER):$(PASSWORD)@$(HOST_NAME):$(PORT)/$(NAME)?sslmode=disable
-
-# Parameter checks
-## Check that DB variables are provided
-.PHONY: checkdbvars
-checkdbvars:
-	test -n "$(HOST_NAME)" # $$HOST_NAME
-	test -n "$(PORT)" # $$PORT
-	test -n "$(NAME)" # $$NAME
-	@echo $(CONNECT_STRING)
-
-## Check that the migration variable (id/timestamp) is provided
-.PHONY: checkmigration
-checkmigration:
-	test -n "$(MIGRATION)" # $$MIGRATION
-
-# Check that the migration name is provided
-.PHONY: checkmigname
-checkmigname:
-	test -n "$(NAME)" # $$NAME
-
-# Migration operations
-## Rollback the last migration
-.PHONY: rollback
-rollback: $(GOOSE) checkdbvars
-	$(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" down
-	pg_dump -O -s $(CONNECT_STRING) > schema.sql
-
-## Rollback to a select migration (id/timestamp)
-.PHONY: rollback_to
-rollback_to: $(GOOSE) checkmigration checkdbvars
-	$(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" down-to "$(MIGRATION)"
-
-## Rollback pre_batch_set
-.PHONY: `rollback_pre_batch_set`
-rollback_pre_batch_set: $(GOOSE) checkdbvars
-	$(GOOSE) -dir db/pre_batch_processing_migrations postgres "$(CONNECT_STRING)" down
-
-## Rollback post_batch_set
-.PHONY: rollback_post_batch_set
-rollback_post_batch_set: $(GOOSE) checkdbvars
-	$(GOOSE) -dir db/post_batch_processing_migrations postgres "$(CONNECT_STRING)" down
-
-## Apply the next up migration
-.PHONY: migrate_up_by_one
-migrate_up_by_one: $(GOOSE) checkdbvars
-	$(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" up-by-one
-
-## Apply all migrations not already run
-.PHONY: migrate
-migrate: $(GOOSE) checkdbvars
-	$(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" up
-	pg_dump -O -s $(CONNECT_STRING) > schema.sql
-
-## Apply all the migrations used to generate a UML diagram (containing FKs)
-.PHONY: migrate_for_uml
-migrate_for_uml: $(GOOSE) checkdbvars
-	$(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" up-to 00018
-
-## Apply migrations to be ran before a batch processing
-.PHONY: migrate_pre_batch_set
-migrate_pre_batch_set: $(GOOSE) checkdbvars
-	$(GOOSE) -dir db/pre_batch_processing_migrations postgres "$(CONNECT_STRING)" up
-
-## Apply migrations to be ran after a batch processing, one-by-one
-.PHONY: migrate_post_batch_set_up_by_one
-migrate_post_batch_set_up_by_one: $(GOOSE) checkdbvars
-	$(GOOSE) -dir db/post_batch_processing_migrations postgres "$(CONNECT_STRING)" up-by-one
-
-## Apply migrations to be ran after a batch processing
-.PHONY: migrate_post_batch_set
-migrate_post_batch_set: $(GOOSE) checkdbvars
-	$(GOOSE) -dir db/post_batch_processing_migrations postgres "$(CONNECT_STRING)" up
-
-## Create a new migration file
-.PHONY: new_migration
-new_migration: $(GOOSE) checkmigname
-	$(GOOSE) -dir db/migrations create $(NAME) sql
-
-## Check which migrations are applied at the moment
-.PHONY: migration_status
-migration_status: $(GOOSE) checkdbvars
-	$(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" status
-
-# Convert timestamped migrations to versioned (to be run in CI);
-# merge timestamped files to prevent conflict
-.PHONY: version_migrations
-version_migrations:
-	$(GOOSE) -dir db/migrations fix
-
-# Import a psql schema to the database
-.PHONY: import
-import:
-	test -n "$(NAME)" # $$NAME
-	psql $(NAME) < schema.sql
-
-.PHONY: test-migrations
-test-migrations: $(GOOSE)
-	./scripts/test_migration.sh

compose.yml

@@ -2,13 +2,14 @@ services:
   migrations:
     restart: on-failure
     depends_on:
-      - ipld-eth-db
+      ipld-eth-db:
+        condition: service_healthy
     # Use local build
     build:
       context: .
       dockerfile: Dockerfile
     # Use an existing image
-    image: cerc/ipld-eth-db
+    image: cerc/ipld-eth-db:local
     environment:
       DATABASE_USER: "vdbm"
       DATABASE_NAME: "cerc_testing"
@@ -26,3 +27,8 @@ services:
       POSTGRES_PASSWORD: "password"
     ports:
       - "127.0.0.1:8077:5432"
+    healthcheck:
+      test: ["CMD", "pg_isready", "-U", "vdbm"]
+      interval: 2s
+      timeout: 1s
+      retries: 3

schema.sql

@@ -2,8 +2,8 @@
 -- PostgreSQL database dump
 --
 
--- Dumped from database version 14.8
+-- Dumped from database version 14.12
--- Dumped by pg_dump version 14.8 (Ubuntu 14.8-0ubuntu0.22.04.1)
+-- Dumped by pg_dump version 14.12
 
 SET statement_timeout = 0;
 SET lock_timeout = 0;
@@ -209,7 +209,8 @@ CREATE TABLE eth.header_cids (
     bloom bytea NOT NULL,
     "timestamp" bigint NOT NULL,
     coinbase character varying(66) NOT NULL,
-    canonical boolean DEFAULT true NOT NULL
+    canonical boolean DEFAULT true NOT NULL,
+    withdrawals_root character varying(66) NOT NULL
 );
 
 
@@ -333,6 +334,21 @@ CREATE TABLE eth.uncle_cids (
 );
 
 
+--
+-- Name: withdrawal_cids; Type: TABLE; Schema: eth; Owner: -
+--
+
+CREATE TABLE eth.withdrawal_cids (
+    block_number bigint NOT NULL,
+    header_id character varying(66) NOT NULL,
+    cid text NOT NULL,
+    index integer NOT NULL,
+    validator integer NOT NULL,
+    address character varying(66) NOT NULL,
+    amount numeric NOT NULL
+);
+
+
 --
 -- Name: watched_addresses; Type: TABLE; Schema: eth_meta; Owner: -
 --
@@ -490,6 +506,14 @@ ALTER TABLE ONLY eth.uncle_cids
     ADD CONSTRAINT uncle_cids_pkey PRIMARY KEY (block_hash, block_number);
 
 
+--
+-- Name: withdrawal_cids withdrawal_cids_pkey; Type: CONSTRAINT; Schema: eth; Owner: -
+--
+
+ALTER TABLE ONLY eth.withdrawal_cids
+    ADD CONSTRAINT withdrawal_cids_pkey PRIMARY KEY (index, header_id, block_number);
+
+
 --
 -- Name: watched_addresses watched_addresses_pkey; Type: CONSTRAINT; Schema: eth_meta; Owner: -
 --
@@ -782,6 +806,13 @@ CREATE UNIQUE INDEX uncle_cid_block_number_index ON eth.uncle_cids USING btree (
 CREATE INDEX uncle_header_id_index ON eth.uncle_cids USING btree (header_id);
 
 
+--
+-- Name: withdrawal_cids_block_number_idx; Type: INDEX; Schema: eth; Owner: -
+--
+
+CREATE INDEX withdrawal_cids_block_number_idx ON eth.withdrawal_cids USING btree (block_number DESC);
+
+
 --
 -- Name: blocks_block_number_idx; Type: INDEX; Schema: ipld; Owner: -
 --
@@ -793,49 +824,56 @@ CREATE INDEX blocks_block_number_idx ON ipld.blocks USING btree (block_number DE
 -- Name: log_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: -
 --
 
-CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.log_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_internal.insert_blocker();
+CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.log_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.insert_blocker();
 
 
 --
 -- Name: receipt_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: -
 --
 
-CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.receipt_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_internal.insert_blocker();
+CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.receipt_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.insert_blocker();
 
 
 --
 -- Name: state_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: -
 --
 
-CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.state_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_internal.insert_blocker();
+CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.state_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.insert_blocker();
 
 
 --
 -- Name: storage_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: -
 --
 
-CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.storage_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_internal.insert_blocker();
+CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.storage_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.insert_blocker();
 
 
 --
 -- Name: transaction_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: -
 --
 
-CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.transaction_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_internal.insert_blocker();
+CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.transaction_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.insert_blocker();
 
 
 --
 -- Name: uncle_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: -
 --
 
-CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.uncle_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_internal.insert_blocker();
+CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.uncle_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.insert_blocker();
+
+
+--
+-- Name: withdrawal_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: -
+--
+
+CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.withdrawal_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.insert_blocker();
 
 
 --
 -- Name: blocks ts_insert_blocker; Type: TRIGGER; Schema: ipld; Owner: -
 --
 
-CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON ipld.blocks FOR EACH ROW EXECUTE FUNCTION _timescaledb_internal.insert_blocker();
+CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON ipld.blocks FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.insert_blocker();
 
 
 --

scripts/startup_script.sh

@@ -1,5 +1,4 @@
 #!/bin/sh
-# Runs the db migrations
 set -e
 
 # Default command is "goose up"
@@ -11,14 +10,5 @@ fi
 VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable
 
 # Run the DB migrations
-echo "Connecting with: $VDB_PG_CONNECT"
+set -x
-echo "Running database migrations"
+exec ./goose -dir migrations postgres "$VDB_PG_CONNECT" "$@"
-./goose -dir migrations postgres "$VDB_PG_CONNECT" "$@"
-
-# If the db migrations ran without err
-if [[ $? -eq 0 ]]; then
-    echo "Migration process ran successfully"
-else
-    echo "Could not run migrations. Are the database details correct?"
-    exit 1
-fi