Fix stored functions for in-place snapshot (#87)

Update startup_script.sh

Dockerfile

@@ -17,9 +17,6 @@ WORKDIR /app
 
 COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-db/scripts/startup_script.sh .
 
-# copy over file for TimescaleDB setup
-COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-db/docker-compose.test.yml docker-tsdb/docker-compose.test.yml
-
 COPY --from=builder /go/src/github.com/pressly/goose/cmd/goose/goose goose
 COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-db/db/migrations migrations/vulcanizedb
 

README.md

@@ -12,28 +12,24 @@ Schemas and utils for IPLD ETH Postgres database
   docker-compose down -v --remove-orphans
   ```
 
-* Spin up a TimescaleDB instance using [docker-compose.test.yml](./docker-compose.test.yml):
+* Spin up `ipld-eth-db` using an existing image:
 
-  ```bash
+  * Update image source used for running the migrations in [docker-compose.yml](./docker-compose.yml) (if required).
-  docker-compose -f docker-compose.test.yml up
-  ```
 
-  Following final output should be seen:
+  * Run:
 
     ```
-    LOG:  TimescaleDB background worker launcher connected to shared catalogs
+    docker-compose -f docker-compose.yml up
     ```
 
-* In another `ipld-eth-db` terminal window, build an image `migrations-test` using [Dockerfile](./db/Dockerfile):
+* Spin up `ipld-eth-db` using a locally built image:
 
-  ```bash
+  * Update [Dockerfile](./Dockerfile) (if required).
-  docker build -t migrations-test -f ./db/Dockerfile .
-  ```
 
-* Start a container using `migrations-test` image to run the db migrations:
+  * Update build context used for running the migrations in [docker-compose.test.yml](./docker-compose.test.yml) (if required).
 
-  ```bash
+  * Run:
-  # Here, we are running the container using host network.
+
-  # So connect to TimescaleDB on 127.0.0.1:8066
+    ```
-  docker run --rm --network host -e DATABASE_USER=vdbm -e DATABASE_PASSWORD=password -e DATABASE_HOSTNAME=127.0.0.1 -e DATABASE_PORT=8066 -e DATABASE_NAME=vulcanize_testing_v4 migrations-test
+    docker-compose -f docker-compose.test.yml up
-  ```
+    ```

db/migrations/00022_create_stored_functions.sql

@@ -155,26 +155,26 @@ BEGIN
     END IF;
 
     -- select all of the state nodes for this snapshot: the latest state node record at every unique path
-    SELECT DISTINCT ON (state_path) blocks.data, state_cids.state_leaf_key, state_cids.cid, state_cids.state_path,
+    SELECT ARRAY (SELECT DISTINCT ON (state_path) ROW (blocks.data, state_cids.state_leaf_key, state_cids.cid, state_cids.state_path,
-        state_cids.node_type, state_cids.mh_key
+        state_cids.node_type, state_cids.mh_key)
-    INTO results
     FROM eth.state_cids
         INNER JOIN public.blocks
             ON (state_cids.mh_key, state_cids.block_number) = (blocks.key, blocks.block_number)
     WHERE state_cids.block_number BETWEEN starting_height AND ending_height
-    ORDER BY state_path, block_number DESC;
+    ORDER BY state_path, state_cids.block_number DESC)
+    INTO results;
 
     -- from the set returned above, insert public.block records at the ending_height block number
     INSERT INTO public.blocks (block_number, key, data)
     SELECT ending_height, r.mh_key, r.data
-    FROM results r;
+    FROM unnest(results) r;
 
     -- from the set returned above, insert eth.state_cids records at the ending_height block number
     -- anchoring all the records to the canonical header found at ending_height
     INSERT INTO eth.state_cids (block_number, header_id, state_leaf_key, cid, state_path, node_type, diff, mh_key)
     SELECT ending_height, canonical_hash, r.state_leaf_key, r.cid, r.state_path, r.node_type, false, r.mh_key
-    FROM results r
+    FROM unnest(results) r
-    ON CONFLICT (state_path, header_id) DO NOTHING;
+    ON CONFLICT (state_path, header_id, block_number) DO NOTHING;
 END
 $BODY$
 LANGUAGE 'plpgsql';
@@ -208,29 +208,29 @@ BEGIN
     END IF;
 
     -- select all of the storage nodes for this snapshot: the latest storage node record at every unique state leaf key
-    SELECT DISTINCT ON (state_leaf_key, storage_path) block.data, storage_cids.state_path, storage_cids.storage_leaf_key,
+    SELECT ARRAY (SELECT DISTINCT ON (state_leaf_key, storage_path) ROW (blocks.data, storage_cids.state_path, storage_cids.storage_leaf_key,
-     storage_cids.cid, storage_cids.storage_path, storage_cids.node_type, storage_cids.mh_key
+     storage_cids.cid, storage_cids.storage_path, storage_cids.node_type, storage_cids.mh_key)
-    INTO results
     FROM eth.storage_cids
         INNER JOIN public.blocks
         ON (storage_cids.mh_key, storage_cids.block_number) = (blocks.key, blocks.block_number)
         INNER JOIN eth.state_cids
         ON (storage_cids.state_path, storage_cids.header_id) = (state_cids.state_path, state_cids.header_id)
     WHERE storage_cids.block_number BETWEEN starting_height AND ending_height
-    ORDER BY state_path, storage_path, block_number DESC;
+    ORDER BY state_leaf_key, storage_path, storage_cids.state_path, storage_cids.block_number DESC)
+    INTO results;
 
     -- from the set returned above, insert public.block records at the ending_height block number
     INSERT INTO public.blocks (block_number, key, data)
     SELECT ending_height, r.mh_key, r.data
-    FROM results r;
+    FROM unnest(results) r;
 
     -- from the set returned above, insert eth.state_cids records at the ending_height block number
     -- anchoring all the records to the canonical header found at ending_height
     INSERT INTO eth.storage_cids (block_number, header_id, state_path, storage_leaf_key, cid, storage_path,
                               node_type, diff, mh_key)
     SELECT ending_height, canonical_hash, r.state_path, r.storage_leaf_key, r.cid, r.storage_path, r.node_type, false, r.mh_key
-    FROM results r
+    FROM unnest(results) r
-    ON CONFLICT (storage_path, state_path, header_id) DO NOTHING;
+    ON CONFLICT (storage_path, state_path, header_id, block_number) DO NOTHING;
 END
 $BODY$
 LANGUAGE 'plpgsql';

docker-compose.test.yml

@@ -1,14 +1,28 @@
 version: '3.2'
 
 services:
-  access-node:
+  migrations:
+    restart: on-failure
+    depends_on:
+      - ipld-eth-db
+    # Build image using local context
+    build:
+      context: .
+      dockerfile: Dockerfile
+    environment:
+      DATABASE_USER: "vdbm"
+      DATABASE_NAME: "vulcanize_testing"
+      DATABASE_PASSWORD: "password"
+      DATABASE_HOSTNAME: "ipld-eth-db"
+      DATABASE_PORT: 5432
+
+  ipld-eth-db:
     image: timescale/timescaledb:latest-pg14
     restart: always
-    container_name: access-node
     command: ["postgres", "-c", "log_statement=all"]
     environment:
       POSTGRES_USER: "vdbm"
-      POSTGRES_DB: "vulcanize_testing_v4"
+      POSTGRES_DB: "vulcanize_testing"
       POSTGRES_PASSWORD: "password"
     ports:
-      - "127.0.0.1:8066:5432"
+      - "127.0.0.1:8077:5432"

docker-compose.yml

@@ -1,14 +1,26 @@
 version: '3.2'
 
 services:
+  migrations:
+    restart: on-failure
+    depends_on:
+      - ipld-eth-db
+    # Use an existing image
+    image: vulcanize/ipld-eth-db:v4.1.1-alpha
+    environment:
+      DATABASE_USER: "vdbm"
+      DATABASE_NAME: "vulcanize_testing"
+      DATABASE_PASSWORD: "password"
+      DATABASE_HOSTNAME: "ipld-eth-db"
+      DATABASE_PORT: 5432
+
   ipld-eth-db:
+    image: timescale/timescaledb:latest-pg14
     restart: always
-    image: vulcanize/ipld-eth-db
+    command: ["postgres", "-c", "log_statement=all"]
-    build: .
     environment:
       POSTGRES_USER: "vdbm"
       POSTGRES_DB: "vulcanize_testing"
       POSTGRES_PASSWORD: "password"
-      hostname: db
     ports:
-      - "127.0.0.1:8077:5432"
+      - "127.0.0.1:8077:5432"

scripts/startup_script.sh

@@ -13,6 +13,7 @@ echo "Running database migrations"
 # If the db migrations ran without err
 if [[ $? -eq 0 ]]; then
     echo "Migration process ran successfully"
+    tail -f /dev/null
 else
     echo "Could not run migrations. Are the database details correct?"
     exit 1