7 changed files with 321 additions and 74 deletions
--- a/.gitea/workflows/on-pr.yaml
+++ b/.gitea/workflows/on-pr.yaml
@ -1,22 +1,78 @@
-name: Basic test
+name: Docker Build
+
 on: [pull_request]

 jobs:
-  basic-test:
-    name: Build and sanity check
+  build:
+    name: Run docker build
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/checkout@v4
-      - name: Build Docker images
+      - uses: actions/checkout@v2
+      - name: Run docker build
        run: docker compose build
-      - name: Run Docker containers
-        run: docker compose up -d
-      - name: Check migration version
-        timeout-minutes: 1
-        run: |
-          while
-            version=$(docker compose run --rm migrations version 2>&1 | tail -1 | awk '{print $(NF)}')
-            [[ $version != 21 ]]; do
-            echo "Incorrect version: $version"
-            echo "Retrying..."
-          done
+  test:
+    # Add a dummy job to avoid failing GitHub CI checks.
+    # Other checks to be added later.
+    name: Dummy job
+    runs-on: ubuntu-latest
+    steps:
+      - name: Run dummy job
+        run: echo "Empty dummy job"
+  # concise_migration_diff:
+  #   name: Verify concise migration and generated schema
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - uses: actions/checkout@v2
+  #     - name: Run docker concise migration build
+  #       run: make docker-concise-migration-build
+  #     - name: Run database
+  #       run: docker-compose -f docker-compose.test.yml up -d test-db
+  #     - name: Test concise migration
+  #       run: |
+  #         sleep 10
+  #         docker run --rm --network host -e DATABASE_USER=vdbm -e DATABASE_PASSWORD=password \
+  #         -e DATABASE_HOSTNAME=127.0.0.1 -e DATABASE_PORT=8066 -e DATABASE_NAME=vulcanize_testing \
+  #         vulcanize/concise-migration-build
+  #     - name: Verify schema is latest
+  #       run: |
+  #         PGPASSWORD="password" pg_dump -h localhost -p 8066 -U vdbm vulcanize_testing --no-owner --schema-only > ./db/migration_schema.sql
+  #         ./scripts/check_diff.sh ./db/migration_schema.sql db/schema.sql
+
+  # incremental_migration_diff:
+  #   name: Compare conscise migration schema with incremental migration.
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - uses: actions/checkout@v2
+  #     - name: Run database
+  #       run: docker-compose -f docker-compose.test.yml up -d test-db statediff-migrations
+  #     - name: Test incremental migration
+  #       run: |
+  #         sleep 10
+  #         docker run --rm --network host -e DATABASE_USER=vdbm -e DATABASE_PASSWORD=password \
+  #         -e DATABASE_HOSTNAME=127.0.0.1 -e DATABASE_PORT=8066 -e DATABASE_NAME=vulcanize_testing \
+  #         vulcanize/statediff-migrations:v0.9.0
+  #     - name: Verify schema is latest
+  #       run: |
+  #         PGPASSWORD="password" pg_dump -h localhost -p 8066 -U vdbm vulcanize_testing --no-owner --schema-only > ./db/migration_schema.sql
+  #         ./scripts/check_diff.sh db/schema.sql ./db/migration_schema.sql
+
+  # migration:
+  #   name: Compare up and down migration
+  #   env:
+  #     GOPATH: /tmp/go
+  #   strategy:
+  #     matrix:
+  #       go-version: [ 1.16.x ]
+  #       os: [ ubuntu-latest ]
+  #   runs-on: ${{ matrix.os }}
+  #   steps:
+  #     - name: Create GOPATH
+  #       run: mkdir -p /tmp/go
+  #     - name: Install Go
+  #       uses: actions/setup-go@v2
+  #       with:
+  #         go-version: ${{ matrix.go-version }}
+  #     - uses: actions/checkout@v2
+  #     - name: Test migration
+  #       run: |
+  #         timeout 5m make test-migrations
--- a/.github/workflows/on-pr.yaml
+++ b/.github/workflows/on-pr.yaml
@ -0,0 +1,78 @@
+name: Docker Build
+
+on: [pull_request]
+
+jobs:
+  build:
+    name: Run docker build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Run docker build
+        run: docker compose build
+  test:
+    # Add a dummy job to avoid failing GitHub CI checks.
+    # Other checks to be added later.
+    name: Dummy job
+    runs-on: ubuntu-latest
+    steps:
+      - name: Run dummy job
+        run: echo "Empty dummy job"
+  # concise_migration_diff:
+  #   name: Verify concise migration and generated schema
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - uses: actions/checkout@v2
+  #     - name: Run docker concise migration build
+  #       run: make docker-concise-migration-build
+  #     - name: Run database
+  #       run: docker-compose -f docker-compose.test.yml up -d test-db
+  #     - name: Test concise migration
+  #       run: |
+  #         sleep 10
+  #         docker run --rm --network host -e DATABASE_USER=vdbm -e DATABASE_PASSWORD=password \
+  #         -e DATABASE_HOSTNAME=127.0.0.1 -e DATABASE_PORT=8066 -e DATABASE_NAME=vulcanize_testing \
+  #         vulcanize/concise-migration-build
+  #     - name: Verify schema is latest
+  #       run: |
+  #         PGPASSWORD="password" pg_dump -h localhost -p 8066 -U vdbm vulcanize_testing --no-owner --schema-only > ./db/migration_schema.sql
+  #         ./scripts/check_diff.sh ./db/migration_schema.sql db/schema.sql
+
+  # incremental_migration_diff:
+  #   name: Compare conscise migration schema with incremental migration.
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - uses: actions/checkout@v2
+  #     - name: Run database
+  #       run: docker-compose -f docker-compose.test.yml up -d test-db statediff-migrations
+  #     - name: Test incremental migration
+  #       run: |
+  #         sleep 10
+  #         docker run --rm --network host -e DATABASE_USER=vdbm -e DATABASE_PASSWORD=password \
+  #         -e DATABASE_HOSTNAME=127.0.0.1 -e DATABASE_PORT=8066 -e DATABASE_NAME=vulcanize_testing \
+  #         vulcanize/statediff-migrations:v0.9.0
+  #     - name: Verify schema is latest
+  #       run: |
+  #         PGPASSWORD="password" pg_dump -h localhost -p 8066 -U vdbm vulcanize_testing --no-owner --schema-only > ./db/migration_schema.sql
+  #         ./scripts/check_diff.sh db/schema.sql ./db/migration_schema.sql
+
+  # migration:
+  #   name: Compare up and down migration
+  #   env:
+  #     GOPATH: /tmp/go
+  #   strategy:
+  #     matrix:
+  #       go-version: [ 1.16.x ]
+  #       os: [ ubuntu-latest ]
+  #   runs-on: ${{ matrix.os }}
+  #   steps:
+  #     - name: Create GOPATH
+  #       run: mkdir -p /tmp/go
+  #     - name: Install Go
+  #       uses: actions/setup-go@v2
+  #       with:
+  #         go-version: ${{ matrix.go-version }}
+  #     - uses: actions/checkout@v2
+  #     - name: Test migration
+  #       run: |
+  #         timeout 5m make test-migrations
--- a/.github/workflows/publish.yaml
+++ b/.github/workflows/publish.yaml
@ -0,0 +1,26 @@
+name: Publish Docker image
+on:
+  release:
+    types: [published, edited]
+jobs:
+  build:
+    name: Build and publish image
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - id: vars
+        name: Output SHA and version tag
+        run: |
+          echo "sha=${GITHUB_SHA:0:7}" >> $GITHUB_OUTPUT
+          echo "tag=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
+      - name: Build and tag image
+        run: |
+          docker build . \
+          -t cerc-io/ipld-eth-db \
+          -t git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.sha}} \
+          -t git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.tag}}
+      - name: Push image tags
+        run: |
+          echo ${{ secrets.GITEA_PUBLISH_TOKEN }} | docker login https://git.vdb.to -u cerccicd --password-stdin
+          docker push git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.sha}}
+          docker push git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:${{steps.vars.outputs.tag}}
--- a/121
+++ b/121
@ -0,0 +1,121 @@
+ifndef GOPATH
+override GOPATH = $(HOME)/go
+endif
+
+BIN = $(GOPATH)/bin
+
+# Tools
+## Migration tool
+GOOSE = $(BIN)/goose
+$(BIN)/goose:
+	go get -u github.com/pressly/goose/cmd/goose
+
+.PHONY: installtools
+installtools: | $(GOOSE)
+	echo "Installing tools"
+
+#Database
+HOST_NAME = localhost
+PORT = 5432
+NAME =
+USER = postgres
+PASSWORD = password
+CONNECT_STRING=postgresql://$(USER):$(PASSWORD)@$(HOST_NAME):$(PORT)/$(NAME)?sslmode=disable
+
+# Parameter checks
+## Check that DB variables are provided
+.PHONY: checkdbvars
+checkdbvars:
+	test -n "$(HOST_NAME)" # $$HOST_NAME
+	test -n "$(PORT)" # $$PORT
+	test -n "$(NAME)" # $$NAME
+	@echo $(CONNECT_STRING)
+
+## Check that the migration variable (id/timestamp) is provided
+.PHONY: checkmigration
+checkmigration:
+	test -n "$(MIGRATION)" # $$MIGRATION
+
+# Check that the migration name is provided
+.PHONY: checkmigname
+checkmigname:
+	test -n "$(NAME)" # $$NAME
+
+# Migration operations
+## Rollback the last migration
+.PHONY: rollback
+rollback: $(GOOSE) checkdbvars
+	$(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" down
+	pg_dump -O -s $(CONNECT_STRING) > schema.sql
+
+## Rollback to a select migration (id/timestamp)
+.PHONY: rollback_to
+rollback_to: $(GOOSE) checkmigration checkdbvars
+	$(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" down-to "$(MIGRATION)"
+
+## Rollback pre_batch_set
+.PHONY: `rollback_pre_batch_set`
+rollback_pre_batch_set: $(GOOSE) checkdbvars
+	$(GOOSE) -dir db/pre_batch_processing_migrations postgres "$(CONNECT_STRING)" down
+
+## Rollback post_batch_set
+.PHONY: rollback_post_batch_set
+rollback_post_batch_set: $(GOOSE) checkdbvars
+	$(GOOSE) -dir db/post_batch_processing_migrations postgres "$(CONNECT_STRING)" down
+
+## Apply the next up migration
+.PHONY: migrate_up_by_one
+migrate_up_by_one: $(GOOSE) checkdbvars
+	$(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" up-by-one
+
+## Apply all migrations not already run
+.PHONY: migrate
+migrate: $(GOOSE) checkdbvars
+	$(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" up
+	pg_dump -O -s $(CONNECT_STRING) > schema.sql
+
+## Apply all the migrations used to generate a UML diagram (containing FKs)
+.PHONY: migrate_for_uml
+migrate_for_uml: $(GOOSE) checkdbvars
+	$(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" up-to 00018
+
+## Apply migrations to be ran before a batch processing
+.PHONY: migrate_pre_batch_set
+migrate_pre_batch_set: $(GOOSE) checkdbvars
+	$(GOOSE) -dir db/pre_batch_processing_migrations postgres "$(CONNECT_STRING)" up
+
+## Apply migrations to be ran after a batch processing, one-by-one
+.PHONY: migrate_post_batch_set_up_by_one
+migrate_post_batch_set_up_by_one: $(GOOSE) checkdbvars
+	$(GOOSE) -dir db/post_batch_processing_migrations postgres "$(CONNECT_STRING)" up-by-one
+
+## Apply migrations to be ran after a batch processing
+.PHONY: migrate_post_batch_set
+migrate_post_batch_set: $(GOOSE) checkdbvars
+	$(GOOSE) -dir db/post_batch_processing_migrations postgres "$(CONNECT_STRING)" up
+
+## Create a new migration file
+.PHONY: new_migration
+new_migration: $(GOOSE) checkmigname
+	$(GOOSE) -dir db/migrations create $(NAME) sql
+
+## Check which migrations are applied at the moment
+.PHONY: migration_status
+migration_status: $(GOOSE) checkdbvars
+	$(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" status
+
+# Convert timestamped migrations to versioned (to be run in CI);
+# merge timestamped files to prevent conflict
+.PHONY: version_migrations
+version_migrations:
+	$(GOOSE) -dir db/migrations fix
+
+# Import a psql schema to the database
+.PHONY: import
+import:
+	test -n "$(NAME)" # $$NAME
+	psql $(NAME) < schema.sql
+
+.PHONY: test-migrations
+test-migrations: $(GOOSE)
+	./scripts/test_migration.sh
--- a/compose.yml
+++ b/compose.yml
@ -2,14 +2,13 @@ services:
  migrations:
    restart: on-failure
    depends_on:
-      ipld-eth-db:
-        condition: service_healthy
+      - ipld-eth-db
    # Use local build
    build:
      context: .
      dockerfile: Dockerfile
    # Use an existing image
-    image: cerc/ipld-eth-db:local
+    image: cerc/ipld-eth-db
    environment:
      DATABASE_USER: "vdbm"
      DATABASE_NAME: "cerc_testing"
@ -27,8 +26,3 @@ services:
      POSTGRES_PASSWORD: "password"
    ports:
      - "127.0.0.1:8077:5432"
-    healthcheck:
-      test: ["CMD", "pg_isready", "-U", "vdbm"]
-      interval: 2s
-      timeout: 1s
-      retries: 3
--- a/schema.sql
+++ b/schema.sql
@ -2,8 +2,8 @@
 -- PostgreSQL database dump
 --

-- Dumped from database version 14.12
-- Dumped by pg_dump version 14.12
+-- Dumped from database version 14.8
+-- Dumped by pg_dump version 14.8 (Ubuntu 14.8-0ubuntu0.22.04.1)

 SET statement_timeout = 0;
 SET lock_timeout = 0;
@ -209,8 +209,7 @@ CREATE TABLE eth.header_cids (
    bloom bytea NOT NULL,
    "timestamp" bigint NOT NULL,
    coinbase character varying(66) NOT NULL,
-    canonical boolean DEFAULT true NOT NULL,
-    withdrawals_root character varying(66) NOT NULL
+    canonical boolean DEFAULT true NOT NULL
 );


@ -334,21 +333,6 @@ CREATE TABLE eth.uncle_cids (
 );


--
-- Name: withdrawal_cids; Type: TABLE; Schema: eth; Owner: -
--
-
-CREATE TABLE eth.withdrawal_cids (
-    block_number bigint NOT NULL,
-    header_id character varying(66) NOT NULL,
-    cid text NOT NULL,
-    index integer NOT NULL,
-    validator integer NOT NULL,
-    address character varying(66) NOT NULL,
-    amount numeric NOT NULL
-);
-
-
 --
 -- Name: watched_addresses; Type: TABLE; Schema: eth_meta; Owner: -
 --
@ -506,14 +490,6 @@ ALTER TABLE ONLY eth.uncle_cids
    ADD CONSTRAINT uncle_cids_pkey PRIMARY KEY (block_hash, block_number);


--
-- Name: withdrawal_cids withdrawal_cids_pkey; Type: CONSTRAINT; Schema: eth; Owner: -
--
-
-ALTER TABLE ONLY eth.withdrawal_cids
-    ADD CONSTRAINT withdrawal_cids_pkey PRIMARY KEY (index, header_id, block_number);
-
-
 --
 -- Name: watched_addresses watched_addresses_pkey; Type: CONSTRAINT; Schema: eth_meta; Owner: -
 --
@ -806,13 +782,6 @@ CREATE UNIQUE INDEX uncle_cid_block_number_index ON eth.uncle_cids USING btree (
 CREATE INDEX uncle_header_id_index ON eth.uncle_cids USING btree (header_id);


--
-- Name: withdrawal_cids_block_number_idx; Type: INDEX; Schema: eth; Owner: -
--
-
-CREATE INDEX withdrawal_cids_block_number_idx ON eth.withdrawal_cids USING btree (block_number DESC);
-
-
 --
 -- Name: blocks_block_number_idx; Type: INDEX; Schema: ipld; Owner: -
 --
@ -824,56 +793,49 @@ CREATE INDEX blocks_block_number_idx ON ipld.blocks USING btree (block_number DE
 -- Name: log_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: -
 --

-CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.log_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.insert_blocker();
+CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.log_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_internal.insert_blocker();


 --
 -- Name: receipt_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: -
 --

-CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.receipt_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.insert_blocker();
+CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.receipt_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_internal.insert_blocker();


 --
 -- Name: state_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: -
 --

-CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.state_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.insert_blocker();
+CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.state_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_internal.insert_blocker();


 --
 -- Name: storage_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: -
 --

-CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.storage_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.insert_blocker();
+CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.storage_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_internal.insert_blocker();


 --
 -- Name: transaction_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: -
 --

-CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.transaction_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.insert_blocker();
+CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.transaction_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_internal.insert_blocker();


 --
 -- Name: uncle_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: -
 --

-CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.uncle_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.insert_blocker();
-
-
--
-- Name: withdrawal_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: -
--
-
-CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.withdrawal_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.insert_blocker();
+CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.uncle_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_internal.insert_blocker();


 --
 -- Name: blocks ts_insert_blocker; Type: TRIGGER; Schema: ipld; Owner: -
 --

-CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON ipld.blocks FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.insert_blocker();
+CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON ipld.blocks FOR EACH ROW EXECUTE FUNCTION _timescaledb_internal.insert_blocker();


 --
--- a/scripts/startup_script.sh
+++ b/scripts/startup_script.sh
@ -1,4 +1,5 @@
 #!/bin/sh
+# Runs the db migrations
 set -e

 # Default command is "goose up"
@ -10,5 +11,14 @@ fi
 VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable

 # Run the DB migrations
-set -x
-exec ./goose -dir migrations postgres "$VDB_PG_CONNECT" "$@"
+echo "Connecting with: $VDB_PG_CONNECT"
+echo "Running database migrations"
+./goose -dir migrations postgres "$VDB_PG_CONNECT" "$@"
+
+# If the db migrations ran without err
+if [[ $? -eq 0 ]]; then
+    echo "Migration process ran successfully"
+else
+    echo "Could not run migrations. Are the database details correct?"
+    exit 1
+fi