.gitea/workflows/on-pr.yaml

@@ -1,22 +1,78 @@
-name: Basic test
+name: Docker Build
+
 on: [pull_request]
 
 jobs:
-  basic-test:
+  build:
-    name: Build and sanity check
+    name: Run docker build
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v2
-      - name: Build Docker images
+      - name: Run docker build
         run: docker compose build
-      - name: Run Docker containers
+  test:
-        run: docker compose up -d
+    # Add a dummy job to avoid failing GitHub CI checks.
-      - name: Check migration version
+    # Other checks to be added later.
-        timeout-minutes: 1
+    name: Dummy job
-        run: |
+    runs-on: ubuntu-latest
-          while
+    steps:
-            version=$(docker compose run --rm migrations version 2>&1 | tail -1 | awk '{print $(NF)}')
+      - name: Run dummy job
-            [[ $version != 21 ]]; do
+        run: echo "Empty dummy job"
-            echo "Incorrect version: $version"
+  # concise_migration_diff:
-            echo "Retrying..."
+  #   name: Verify concise migration and generated schema
-          done
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - uses: actions/checkout@v2
+  #     - name: Run docker concise migration build
+  #       run: make docker-concise-migration-build
+  #     - name: Run database
+  #       run: docker-compose -f docker-compose.test.yml up -d test-db
+  #     - name: Test concise migration
+  #       run: |
+  #         sleep 10
+  #         docker run --rm --network host -e DATABASE_USER=vdbm -e DATABASE_PASSWORD=password \
+  #         -e DATABASE_HOSTNAME=127.0.0.1 -e DATABASE_PORT=8066 -e DATABASE_NAME=vulcanize_testing \
+  #         vulcanize/concise-migration-build
+  #     - name: Verify schema is latest
+  #       run: |
+  #         PGPASSWORD="password" pg_dump -h localhost -p 8066 -U vdbm vulcanize_testing --no-owner --schema-only > ./db/migration_schema.sql
+  #         ./scripts/check_diff.sh ./db/migration_schema.sql db/schema.sql
+
+  # incremental_migration_diff:
+  #   name: Compare conscise migration schema with incremental migration.
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - uses: actions/checkout@v2
+  #     - name: Run database
+  #       run: docker-compose -f docker-compose.test.yml up -d test-db statediff-migrations
+  #     - name: Test incremental migration
+  #       run: |
+  #         sleep 10
+  #         docker run --rm --network host -e DATABASE_USER=vdbm -e DATABASE_PASSWORD=password \
+  #         -e DATABASE_HOSTNAME=127.0.0.1 -e DATABASE_PORT=8066 -e DATABASE_NAME=vulcanize_testing \
+  #         vulcanize/statediff-migrations:v0.9.0
+  #     - name: Verify schema is latest
+  #       run: |
+  #         PGPASSWORD="password" pg_dump -h localhost -p 8066 -U vdbm vulcanize_testing --no-owner --schema-only > ./db/migration_schema.sql
+  #         ./scripts/check_diff.sh db/schema.sql ./db/migration_schema.sql
+
+  # migration:
+  #   name: Compare up and down migration
+  #   env:
+  #     GOPATH: /tmp/go
+  #   strategy:
+  #     matrix:
+  #       go-version: [ 1.16.x ]
+  #       os: [ ubuntu-latest ]
+  #   runs-on: ${{ matrix.os }}
+  #   steps:
+  #     - name: Create GOPATH
+  #       run: mkdir -p /tmp/go
+  #     - name: Install Go
+  #       uses: actions/setup-go@v2
+  #       with:
+  #         go-version: ${{ matrix.go-version }}
+  #     - uses: actions/checkout@v2
+  #     - name: Test migration
+  #       run: |
+  #         timeout 5m make test-migrations

.github/workflows/on-pr.yaml

@@ -0,0 +1,78 @@
+name: Docker Build
+
+on: [pull_request]
+
+jobs:
+  build:
+    name: Run docker build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Run docker build
+        run: docker compose build
+  test:
+    # Add a dummy job to avoid failing GitHub CI checks.
+    # Other checks to be added later.
+    name: Dummy job
+    runs-on: ubuntu-latest
+    steps:
+      - name: Run dummy job
+        run: echo "Empty dummy job"
+  # concise_migration_diff:
+  #   name: Verify concise migration and generated schema
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - uses: actions/checkout@v2
+  #     - name: Run docker concise migration build
+  #       run: make docker-concise-migration-build
+  #     - name: Run database
+  #       run: docker-compose -f docker-compose.test.yml up -d test-db
+  #     - name: Test concise migration
+  #       run: |
+  #         sleep 10
+  #         docker run --rm --network host -e DATABASE_USER=vdbm -e DATABASE_PASSWORD=password \
+  #         -e DATABASE_HOSTNAME=127.0.0.1 -e DATABASE_PORT=8066 -e DATABASE_NAME=vulcanize_testing \
+  #         vulcanize/concise-migration-build
+  #     - name: Verify schema is latest
+  #       run: |
+  #         PGPASSWORD="password" pg_dump -h localhost -p 8066 -U vdbm vulcanize_testing --no-owner --schema-only > ./db/migration_schema.sql
+  #         ./scripts/check_diff.sh ./db/migration_schema.sql db/schema.sql
+
+  # incremental_migration_diff:
+  #   name: Compare conscise migration schema with incremental migration.
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - uses: actions/checkout@v2
+  #     - name: Run database
+  #       run: docker-compose -f docker-compose.test.yml up -d test-db statediff-migrations
+  #     - name: Test incremental migration
+  #       run: |
+  #         sleep 10
+  #         docker run --rm --network host -e DATABASE_USER=vdbm -e DATABASE_PASSWORD=password \
+  #         -e DATABASE_HOSTNAME=127.0.0.1 -e DATABASE_PORT=8066 -e DATABASE_NAME=vulcanize_testing \
+  #         vulcanize/statediff-migrations:v0.9.0
+  #     - name: Verify schema is latest
+  #       run: |
+  #         PGPASSWORD="password" pg_dump -h localhost -p 8066 -U vdbm vulcanize_testing --no-owner --schema-only > ./db/migration_schema.sql
+  #         ./scripts/check_diff.sh db/schema.sql ./db/migration_schema.sql
+
+  # migration:
+  #   name: Compare up and down migration
+  #   env:
+  #     GOPATH: /tmp/go
+  #   strategy:
+  #     matrix:
+  #       go-version: [ 1.16.x ]
+  #       os: [ ubuntu-latest ]
+  #   runs-on: ${{ matrix.os }}
+  #   steps:
+  #     - name: Create GOPATH
+  #       run: mkdir -p /tmp/go
+  #     - name: Install Go
+  #       uses: actions/setup-go@v2
+  #       with:
+  #         go-version: ${{ matrix.go-version }}
+  #     - uses: actions/checkout@v2
+  #     - name: Test migration
+  #       run: |
+  #         timeout 5m make test-migrations

.github/workflows/publish.yaml

@@ -0,0 +1,26 @@
+name: Publish Docker image
+on:
+  release:
+    types: [published, edited]
+jobs:
+  build:
+    name: Build and publish image
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - id: vars
+        name: Output SHA and version tag
+        run: |
+          echo "sha=${GITHUB_SHA:0:7}" >> $GITHUB_OUTPUT
+          echo "tag=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
+      - name: Build and tag image
+        run: |
+          docker build . \
+          -t cerc-io/ipld-eth-db \
+          -t git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.sha}} \
+          -t git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.tag}}
+      - name: Push image tags
+        run: |
+          echo ${{ secrets.GITEA_PUBLISH_TOKEN }} | docker login https://git.vdb.to -u cerccicd --password-stdin
+          docker push git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.sha}}
+          docker push git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.tag}}

Dockerfile

@@ -1,10 +1,12 @@
-FROM alpine as builder
+FROM golang:1.18-alpine as builder
+
+ADD . /go/src/github.com/cerc-io/ipld-eth-db
 
 # Get migration tool
 WORKDIR /
-ARG GOOSE_VERSION="v3.6.1"
+ARG GOOSE_VER="v3.6.1"
 RUN arch=$(arch | sed s/aarch64/arm64/) && \
-  wget -O ./goose https://github.com/pressly/goose/releases/download/${GOOSE_VERSION}/goose_linux_${arch}
+  wget -O ./goose https://github.com/pressly/goose/releases/download/${GOOSE_VER}/goose_linux_${arch}
 RUN chmod +x ./goose
 
 # app container
@@ -12,8 +14,9 @@ FROM alpine
 
 WORKDIR /app
 
+COPY --from=builder /go/src/github.com/cerc-io/ipld-eth-db/scripts/startup_script.sh .
+
 COPY --from=builder /goose goose
-ADD scripts/startup_script.sh .
+COPY --from=builder /go/src/github.com/cerc-io/ipld-eth-db/db/migrations migrations
-ADD db/migrations migrations
 
 ENTRYPOINT ["/app/startup_script.sh"]

Makefile

@@ -0,0 +1,121 @@
+ifndef GOPATH
+override GOPATH = $(HOME)/go
+endif
+
+BIN = $(GOPATH)/bin
+
+# Tools
+## Migration tool
+GOOSE = $(BIN)/goose
+$(BIN)/goose:
+	go get -u github.com/pressly/goose/cmd/goose
+
+.PHONY: installtools
+installtools: | $(GOOSE)
+	echo "Installing tools"
+
+#Database
+HOST_NAME = localhost
+PORT = 5432
+NAME =
+USER = postgres
+PASSWORD = password
+CONNECT_STRING=postgresql://$(USER):$(PASSWORD)@$(HOST_NAME):$(PORT)/$(NAME)?sslmode=disable
+
+# Parameter checks
+## Check that DB variables are provided
+.PHONY: checkdbvars
+checkdbvars:
+	test -n "$(HOST_NAME)" # $$HOST_NAME
+	test -n "$(PORT)" # $$PORT
+	test -n "$(NAME)" # $$NAME
+	@echo $(CONNECT_STRING)
+
+## Check that the migration variable (id/timestamp) is provided
+.PHONY: checkmigration
+checkmigration:
+	test -n "$(MIGRATION)" # $$MIGRATION
+
+# Check that the migration name is provided
+.PHONY: checkmigname
+checkmigname:
+	test -n "$(NAME)" # $$NAME
+
+# Migration operations
+## Rollback the last migration
+.PHONY: rollback
+rollback: $(GOOSE) checkdbvars
+	$(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" down
+	pg_dump -O -s $(CONNECT_STRING) > schema.sql
+
+## Rollback to a select migration (id/timestamp)
+.PHONY: rollback_to
+rollback_to: $(GOOSE) checkmigration checkdbvars
+	$(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" down-to "$(MIGRATION)"
+
+## Rollback pre_batch_set
+.PHONY: `rollback_pre_batch_set`
+rollback_pre_batch_set: $(GOOSE) checkdbvars
+	$(GOOSE) -dir db/pre_batch_processing_migrations postgres "$(CONNECT_STRING)" down
+
+## Rollback post_batch_set
+.PHONY: rollback_post_batch_set
+rollback_post_batch_set: $(GOOSE) checkdbvars
+	$(GOOSE) -dir db/post_batch_processing_migrations postgres "$(CONNECT_STRING)" down
+
+## Apply the next up migration
+.PHONY: migrate_up_by_one
+migrate_up_by_one: $(GOOSE) checkdbvars
+	$(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" up-by-one
+
+## Apply all migrations not already run
+.PHONY: migrate
+migrate: $(GOOSE) checkdbvars
+	$(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" up
+	pg_dump -O -s $(CONNECT_STRING) > schema.sql
+
+## Apply all the migrations used to generate a UML diagram (containing FKs)
+.PHONY: migrate_for_uml
+migrate_for_uml: $(GOOSE) checkdbvars
+	$(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" up-to 00018
+
+## Apply migrations to be ran before a batch processing
+.PHONY: migrate_pre_batch_set
+migrate_pre_batch_set: $(GOOSE) checkdbvars
+	$(GOOSE) -dir db/pre_batch_processing_migrations postgres "$(CONNECT_STRING)" up
+
+## Apply migrations to be ran after a batch processing, one-by-one
+.PHONY: migrate_post_batch_set_up_by_one
+migrate_post_batch_set_up_by_one: $(GOOSE) checkdbvars
+	$(GOOSE) -dir db/post_batch_processing_migrations postgres "$(CONNECT_STRING)" up-by-one
+
+## Apply migrations to be ran after a batch processing
+.PHONY: migrate_post_batch_set
+migrate_post_batch_set: $(GOOSE) checkdbvars
+	$(GOOSE) -dir db/post_batch_processing_migrations postgres "$(CONNECT_STRING)" up
+
+## Create a new migration file
+.PHONY: new_migration
+new_migration: $(GOOSE) checkmigname
+	$(GOOSE) -dir db/migrations create $(NAME) sql
+
+## Check which migrations are applied at the moment
+.PHONY: migration_status
+migration_status: $(GOOSE) checkdbvars
+	$(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" status
+
+# Convert timestamped migrations to versioned (to be run in CI);
+# merge timestamped files to prevent conflict
+.PHONY: version_migrations
+version_migrations:
+	$(GOOSE) -dir db/migrations fix
+
+# Import a psql schema to the database
+.PHONY: import
+import:
+	test -n "$(NAME)" # $$NAME
+	psql $(NAME) < schema.sql
+
+.PHONY: test-migrations
+test-migrations: $(GOOSE)
+	./scripts/test_migration.sh

compose.yml

@@ -2,14 +2,13 @@ services:
   migrations:
     restart: on-failure
     depends_on:
-      ipld-eth-db:
+      - ipld-eth-db
-        condition: service_healthy
     # Use local build
     build:
       context: .
       dockerfile: Dockerfile
     # Use an existing image
-    image: cerc/ipld-eth-db:local
+    image: cerc/ipld-eth-db
     environment:
       DATABASE_USER: "vdbm"
       DATABASE_NAME: "cerc_testing"
@@ -27,8 +26,3 @@ services:
       POSTGRES_PASSWORD: "password"
     ports:
       - "127.0.0.1:8077:5432"
-    healthcheck:
-      test: ["CMD", "pg_isready", "-U", "vdbm"]
-      interval: 2s
-      timeout: 1s
-      retries: 3

schema.sql

@@ -2,8 +2,8 @@
 -- PostgreSQL database dump
 --
 
--- Dumped from database version 14.12
+-- Dumped from database version 14.8
--- Dumped by pg_dump version 14.12
+-- Dumped by pg_dump version 14.8 (Ubuntu 14.8-0ubuntu0.22.04.1)
 
 SET statement_timeout = 0;
 SET lock_timeout = 0;
@@ -190,17 +190,6 @@ SET default_tablespace = '';
 
 SET default_table_access_method = heap;
 
---
--- Name: blob_hashes; Type: TABLE; Schema: eth; Owner: -
---
-
-CREATE TABLE eth.blob_hashes (
-    tx_hash character varying(66) NOT NULL,
-    index integer NOT NULL,
-    blob_hash bytea NOT NULL
-);
-
-
 --
 -- Name: header_cids; Type: TABLE; Schema: eth; Owner: -
 --
@@ -220,8 +209,7 @@ CREATE TABLE eth.header_cids (
     bloom bytea NOT NULL,
     "timestamp" bigint NOT NULL,
     coinbase character varying(66) NOT NULL,
-    canonical boolean DEFAULT true NOT NULL,
+    canonical boolean DEFAULT true NOT NULL
-    withdrawals_root character varying(66) NOT NULL
 );
 
 
@@ -345,21 +333,6 @@ CREATE TABLE eth.uncle_cids (
 );
 
 
---
--- Name: withdrawal_cids; Type: TABLE; Schema: eth; Owner: -
---
-
-CREATE TABLE eth.withdrawal_cids (
-    block_number bigint NOT NULL,
-    header_id character varying(66) NOT NULL,
-    cid text NOT NULL,
-    index integer NOT NULL,
-    validator integer NOT NULL,
-    address character varying(66) NOT NULL,
-    amount numeric NOT NULL
-);
-
-
 --
 -- Name: watched_addresses; Type: TABLE; Schema: eth_meta; Owner: -
 --
@@ -517,14 +490,6 @@ ALTER TABLE ONLY eth.uncle_cids
     ADD CONSTRAINT uncle_cids_pkey PRIMARY KEY (block_hash, block_number);
 
 
---
--- Name: withdrawal_cids withdrawal_cids_pkey; Type: CONSTRAINT; Schema: eth; Owner: -
---
-
-ALTER TABLE ONLY eth.withdrawal_cids
-    ADD CONSTRAINT withdrawal_cids_pkey PRIMARY KEY (index, header_id, block_number);
-
-
 --
 -- Name: watched_addresses watched_addresses_pkey; Type: CONSTRAINT; Schema: eth_meta; Owner: -
 --
@@ -565,13 +530,6 @@ ALTER TABLE ONLY public.nodes
     ADD CONSTRAINT nodes_pkey PRIMARY KEY (node_id);
 
 
---
--- Name: blob_hashes_tx_hash_index; Type: INDEX; Schema: eth; Owner: -
---
-
-CREATE UNIQUE INDEX blob_hashes_tx_hash_index ON eth.blob_hashes USING btree (tx_hash, index);
-
-
 --
 -- Name: header_block_number_index; Type: INDEX; Schema: eth; Owner: -
 --
@@ -824,13 +782,6 @@ CREATE UNIQUE INDEX uncle_cid_block_number_index ON eth.uncle_cids USING btree (
 CREATE INDEX uncle_header_id_index ON eth.uncle_cids USING btree (header_id);
 
 
---
--- Name: withdrawal_cids_block_number_idx; Type: INDEX; Schema: eth; Owner: -
---
-
-CREATE INDEX withdrawal_cids_block_number_idx ON eth.withdrawal_cids USING btree (block_number DESC);
-
-
 --
 -- Name: blocks_block_number_idx; Type: INDEX; Schema: ipld; Owner: -
 --
@@ -842,56 +793,49 @@ CREATE INDEX blocks_block_number_idx ON ipld.blocks USING btree (block_number DE
 -- Name: log_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: -
 --
 
-CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.log_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.insert_blocker();
+CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.log_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_internal.insert_blocker();
 
 
 --
 -- Name: receipt_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: -
 --
 
-CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.receipt_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.insert_blocker();
+CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.receipt_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_internal.insert_blocker();
 
 
 --
 -- Name: state_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: -
 --
 
-CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.state_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.insert_blocker();
+CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.state_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_internal.insert_blocker();
 
 
 --
 -- Name: storage_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: -
 --
 
-CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.storage_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.insert_blocker();
+CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.storage_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_internal.insert_blocker();
 
 
 --
 -- Name: transaction_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: -
 --
 
-CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.transaction_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.insert_blocker();
+CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.transaction_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_internal.insert_blocker();
 
 
 --
 -- Name: uncle_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: -
 --
 
-CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.uncle_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.insert_blocker();
+CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.uncle_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_internal.insert_blocker();
-
-
---
--- Name: withdrawal_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: -
---
-
-CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.withdrawal_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.insert_blocker();
 
 
 --
 -- Name: blocks ts_insert_blocker; Type: TRIGGER; Schema: ipld; Owner: -
 --
 
-CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON ipld.blocks FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.insert_blocker();
+CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON ipld.blocks FOR EACH ROW EXECUTE FUNCTION _timescaledb_internal.insert_blocker();
 
 
 --

scripts/startup_script.sh

@@ -1,4 +1,5 @@
 #!/bin/sh
+# Runs the db migrations
 set -e
 
 # Default command is "goose up"
@@ -10,5 +11,14 @@ fi
 VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable
 
 # Run the DB migrations
-set -x
+echo "Connecting with: $VDB_PG_CONNECT"
-exec ./goose -dir migrations postgres "$VDB_PG_CONNECT" "$@"
+echo "Running database migrations"
+./goose -dir migrations postgres "$VDB_PG_CONNECT" "$@"
+
+# If the db migrations ran without err
+if [[ $? -eq 0 ]]; then
+    echo "Migration process ran successfully"
+else
+    echo "Could not run migrations. Are the database details correct?"
+    exit 1
+fi