cerc-io/stack-orchestrator
17
1
Fork 3
Code Issues 113 Pull Requests 19 Actions Packages Projects Releases 480 Wiki Activity

Add deployed/error status output to the state file. #719

Merged
telackey merged 7 commits from telackey/sostatus into main 2024-01-31 04:13:45 +00:00
Conversation 0 Commits 7 Files Changed 3 +44 -76
13 changed files with 44 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 25f4ffb22b - Show all commits

2
.gitea/workflows/fixturenet-eth-plugeth-test.yml
View File

@ -6,6 +6,8 @@ on:
paths:
- '!**'
- '.gitea/workflows/triggers/fixturenet-eth-plugeth-test'
schedule: # Note: coordinate with other tests to not overload runners at the same time of day
- cron: '2 14 * * *'
# Needed until we can incorporate docker startup into the executor container
env:

11
.gitea/workflows/fixturenet-laconicd-test.yml
View File

@ -6,11 +6,8 @@ on:
paths:
- '!**'
- '.gitea/workflows/triggers/fixturenet-laconicd-test'
# Needed until we can incorporate docker startup into the executor container
env:
DOCKER_HOST: unix:///var/run/dind.sock
schedule:
- cron: '1 13 * * *'
jobs:
test:
@ -47,9 +44,5 @@ jobs:
run: ./scripts/create_build_tag_file.sh
- name: "Build local shiv package"
run: ./scripts/build_shiv_package.sh
- name: Start dockerd # Also needed until we can incorporate into the executor
run: |
dockerd -H $DOCKER_HOST --userland-proxy=false &
sleep 5
- name: "Run fixturenet-laconicd tests"
run: ./tests/fixturenet-laconicd/run-test.sh

32
.gitea/workflows/test-k8s-deploy.yml
View File

@ -1,23 +1,18 @@
name: K8s Deploy Test
on:
pull_request:
branches: '*'
push:
branches:
- main
- ci-test
paths-ignore:
- '.gitea/workflows/triggers/*'
# Needed until we can incorporate docker startup into the executor container
env:
DOCKER_HOST: unix:///var/run/dind.sock
branches: '*'
paths:
- '!**'
- '.gitea/workflows/triggers/test-k8s-deploy'
- '.gitea/workflows/test-k8s-deploy.yml'
- 'tests/k8s-deploy/run-deploy-test.sh'
jobs:
test:
name: "Run deploy test suite on kind/k8s"
runs-on: ubuntu-22.04-with-syn-ethdb
runs-on: ubuntu-22.04
steps:
- name: "Clone project repository"
uses: actions/checkout@v3
@ -41,14 +36,15 @@ jobs:
run: ./scripts/create_build_tag_file.sh
- name: "Build local shiv package"
run: ./scripts/build_shiv_package.sh
- name: Start dockerd # Also needed until we can incorporate into the executor
run: |
dockerd -H $DOCKER_HOST --userland-proxy=false &
- name: "Check container"
run: cat /usr/local/bin/entrypoint.sh
- name: "Check cgroups version"
run: mount | grep cgroup
- name: "Install kind"
run: ./tests/scripts/install-kind.sh
- name: "Install Kubectl"
run: ./tests/scripts/install-kubectl.sh
- name: "Run k8s deployment test"
run: ./tests/k8s-deploy/run-deploy-test.sh
run: |
source /opt/bash-utils/cgroup-helper.sh
join_cgroup
./tests/k8s-deploy/run-deploy-test.sh

1
.gitea/workflows/triggers/fixturenet-eth-plugeth-test
View File

@ -1,2 +1,3 @@
Change this file to trigger running the fixturenet-eth-plugeth-test CI job
trigger
trigger

3
.gitea/workflows/triggers/fixturenet-laconicd-test
View File

@ -1,2 +1,3 @@
Change this file to trigger running the fixturenet-laconicd-test CI job
Trigger
Trigger

1
.gitea/workflows/triggers/test-k8s-deploy Normal file
View File

@ -0,0 +1 @@
Change this file to trigger running the test-k8s-deploy CI job

4
stack_orchestrator/data/compose/docker-compose-grafana.yml
View File

@ -2,8 +2,10 @@ version: "3.7"
services:
grafana:
image: grafana/grafana
image: grafana/grafana:10.2.2
restart: always
environment:
GF_SERVER_ROOT_URL: ${GF_SERVER_ROOT_URL}
volumes:
- ../config/monitoring/grafana/provisioning:/etc/grafana/provisioning
- ../config/monitoring/grafana/dashboards:/etc/grafana/dashboards

2
stack_orchestrator/data/compose/docker-compose-prom-server.yml
View File

@ -2,7 +2,7 @@ version: "3.7"
services:
prometheus:
image: prom/prometheus
image: prom/prometheus:v2.49.1
restart: always
volumes:
- ../config/monitoring/prometheus:/etc/prometheus

40
stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml
View File

@ -95,14 +95,10 @@ groups:
maxDataPoints: 43200
refId: condition
type: math
dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
panelId: 24
noDataState: Alerting
execErrState: Alerting
for: 15m
annotations:
__dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
__panelId__: "24"
summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
isPaused: false
- uid: censures_diff_external
@ -191,14 +187,10 @@ groups:
maxDataPoints: 43200
refId: condition
type: math
dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
panelId: 24
noDataState: Alerting
execErrState: Alerting
for: 15m
annotations:
__dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
__panelId__: "24"
summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
isPaused: false
- uid: claims_diff_external
@ -287,14 +279,10 @@ groups:
maxDataPoints: 43200
refId: condition
type: math
dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
panelId: 24
noDataState: Alerting
execErrState: Alerting
for: 15m
annotations:
__dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
__panelId__: "24"
summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
isPaused: false
- uid: conditional_star_release_diff_external
@ -383,14 +371,10 @@ groups:
maxDataPoints: 43200
refId: condition
type: math
dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
panelId: 24
noDataState: Alerting
execErrState: Alerting
for: 15m
annotations:
__dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
__panelId__: "24"
summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
isPaused: false
- uid: delegated_sending_diff_external
@ -479,14 +463,10 @@ groups:
maxDataPoints: 43200
refId: condition
type: math
dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
panelId: 24
noDataState: Alerting
execErrState: Alerting
for: 15m
annotations:
__dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
__panelId__: "24"
summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
isPaused: false
- uid: ecliptic_diff_external
@ -575,14 +555,10 @@ groups:
maxDataPoints: 43200
refId: condition
type: math
dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
panelId: 24
noDataState: Alerting
execErrState: Alerting
for: 15m
annotations:
__dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
__panelId__: "24"
summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
isPaused: false
- uid: linear_star_release_diff_external
@ -671,14 +647,10 @@ groups:
maxDataPoints: 43200
refId: condition
type: math
dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
panelId: 24
noDataState: Alerting
execErrState: Alerting
for: 15m
annotations:
__dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
__panelId__: "24"
summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
isPaused: false
- uid: polls_diff_external
@ -767,14 +739,10 @@ groups:
maxDataPoints: 43200
refId: condition
type: math
dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
panelId: 24
noDataState: Alerting
execErrState: Alerting
for: 15m
annotations:
__dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
__panelId__: "24"
summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
isPaused: false
@ -865,14 +833,10 @@ groups:
maxDataPoints: 43200
refId: condition
type: math
dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
panelId: 24
noDataState: Alerting
execErrState: Alerting
for: 15m
annotations:
__dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
__panelId__: "24"
summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
isPaused: false
- uid: merkl_sushiswap_diff_external
@ -961,13 +925,9 @@ groups:
maxDataPoints: 43200
refId: condition
type: math
dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
panelId: 24
noDataState: Alerting
execErrState: Alerting
for: 15m
annotations:
__dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
__panelId__: "24"
summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
isPaused: false

5
stack_orchestrator/data/container-build/cerc-fixturenet-eth-genesis/Dockerfile
View File

@ -10,9 +10,10 @@ COPY genesis /opt/genesis
COPY --from=ethgen /usr/local/bin/eth2-testnet-genesis /usr/local/bin/
COPY --from=ethgen /usr/local/bin/eth2-val-tools /usr/local/bin/
COPY --from=ethgen /apps /apps
RUN cd /apps/el-gen && pip3 install -r requirements.txt
RUN cd /apps/el-gen && pip3 install --break-system-packages -r requirements.txt
# web3==5.24.0 used by el-gen is broken on python 3.11
RUN pip3 install --upgrade "web3==6.5.0"
RUN pip3 install --break-system-packages --upgrade "web3==6.5.0"
RUN pip3 install --break-system-packages --upgrade "typing-extensions"
# Build genesis config
RUN apk add --no-cache make bash envsubst jq

4
stack_orchestrator/data/stacks/monitoring/README.md
View File

@ -140,6 +140,10 @@ Set the following env variables in the deployment env config file (`monitoring-d
# External ETH RPC endpoint (filecoin)
# (Optional, default: https://api.node.glif.io/rpc/v1)
CERC_FIL_RPC_ENDPOINT=
# Grafana server host URL (used in various links in alerts, etc.)
# (Optional, default: http://localhost:3000)
GF_SERVER_ROOT_URL=
```
## Start the stack

4
stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md
View File

@ -130,6 +130,10 @@ Set the following env variables in the deployment env config file (`monitoring-w
```bash
# Infura key to be used
CERC_INFURA_KEY=
# Grafana server host URL to be used
# (Optional, default: http://localhost:3000)
GF_SERVER_ROOT_URL=
```
## Start the stack

11
tests/k8s-deploy/run-deploy-test.sh
View File

@ -9,7 +9,7 @@ fi
# Helper functions: TODO move into a separate file
wait_for_pods_started () {
for i in {1..5}
for i in {1..50}
do
local ps_output=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir ps )
@ -27,7 +27,7 @@ wait_for_pods_started () {
}
wait_for_log_output () {
for i in {1..5}
for i in {1..50}
do
local log_output=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
@ -63,7 +63,9 @@ echo "Version reported is: ${reported_version_string}"
echo "Cloning repositories into: $CERC_REPO_BASE_DIR"
rm -rf $CERC_REPO_BASE_DIR
mkdir -p $CERC_REPO_BASE_DIR
# Test basic stack-orchestrator deploy
$TEST_TARGET_SO --stack test setup-repositories
$TEST_TARGET_SO --stack test build-containers
# Test basic stack-orchestrator deploy to k8s
test_deployment_dir=$CERC_REPO_BASE_DIR/test-deployment-dir
test_deployment_spec=$CERC_REPO_BASE_DIR/test-deployment-spec.yml
$TEST_TARGET_SO --stack test deploy --deploy-to k8s-kind init --output $test_deployment_spec --config CERC_TEST_PARAM_1=PASSED
@ -118,7 +120,8 @@ fi
# Stop then start again and check the volume was preserved
$TEST_TARGET_SO deployment --dir $test_deployment_dir stop
# Sleep a bit just in case
sleep 2
# sleep for longer to check if that's why the subsequent create cluster fails
sleep 20
$TEST_TARGET_SO deployment --dir $test_deployment_dir start
wait_for_pods_started
wait_for_log_output