launch panel server from docker

.gitignore

@@ -1,2 +1,4 @@
 lotus
 venv/
+__pycache__/
+.ipynb_checkpoints/

composer/Dockerfile

@@ -2,18 +2,18 @@ FROM python:3.8-buster
 
 WORKDIR /usr/src/app
 
-RUN mkdir /jupyter && chmod 777 /jupyter
+RUN mkdir /composer && chmod 777 /composer
+RUN mkdir /testground && chmod 777 /testground
+
+ENV HOME /composer
+ENV TESTGROUND_HOME /testground
+ENV LISTEN_PORT 5006
+
+VOLUME /testground/plans
+
 
 COPY requirements.txt ./
 RUN pip install -r requirements.txt
-
 COPY . .
 
-VOLUME /testground/plans
-VOLUME /jupyter/runtime
-
-ENV TESTGROUND_HOME /testground
-ENV HOME /jupyter
-ENV JUPYTER_RUNTIME_DIR /jupyter/runtime
-
-CMD ["jupyter", "notebook", "--allow-root", "--ip", "0.0.0.0"]
+CMD panel serve --address 0.0.0.0 --port $LISTEN_PORT composer.ipynb

composer/composer.sh

@@ -10,6 +10,7 @@ image_full_name="$image_name:$image_tag"
 tg_home=${TESTGROUND_HOME:-$HOME/testground}
 container_plans_dir="/testground/plans"
 jupyter_port=${JUPYTER_PORT:-8888}
+panel_port=${PANEL_PORT:-5006}
 
 poll_interval=30
 
@@ -73,10 +74,6 @@ cleanup () {
   if [[ -d "$temp_manifest_dir" ]]; then
     rm -rf ${temp_manifest_dir}
   fi
-
-  if [[ -d "$temp_jupyter_runtime_dir" ]]; then
-    rm -rf ${temp_jupyter_runtime_dir}
-  fi
 }
 
 # run cleanup on exit
@@ -92,31 +89,24 @@ if [[ "$TEMP" != "" ]]; then
 fi
 
 temp_manifest_dir="$(mktemp -d ${temp_base}/testground-composer-XXXX)"
-temp_jupyter_runtime_dir="$(mktemp -d ${temp_base}/testground-composer-jupyter-XXXX)"
 echo "temp manifest dir: $temp_manifest_dir"
-echo "temp jupyter dir: $temp_jupyter_runtime_dir"
 
 # copy the manifests to the temp dir
 update_manifests ${temp_manifest_dir}
 
 # run the container in detached mode and grab the id
-container_id=$(docker run -d --user $(id -u):$(id -g) -p ${jupyter_port}:8888 -v ${temp_manifest_dir}:${container_plans_dir}:ro -v ${temp_jupyter_runtime_dir}:/jupyter/runtime $image_full_name)
+container_id=$(docker run -d --user $(id -u):$(id -g) -p ${panel_port}:5006 -v ${temp_manifest_dir}:${container_plans_dir}:ro $image_full_name)
 
 echo "container $container_id started"
 # print the log output
 docker logs -f ${container_id} &
 
+# sleep for a couple seconds to let the server start up
+sleep 2
 
-while [[ ! -f ${temp_jupyter_runtime_dir}/nbserver-1.json ]]; do
-  sleep 1
-done
-
-token=$(jq -r '.token' ${temp_jupyter_runtime_dir}/nbserver-1.json)
-jupyter_url="http://localhost:${jupyter_port}/?token=${token}"
-
-echo "Jupyter url: $jupyter_url"
-open_url $jupyter_url
-
+# open a browser to the app url
+panel_url="http://localhost:${panel_port}"
+open_url $panel_url
 
 # poll & check for manifest changes every few seconds
 watch_manifests ${temp_manifest_dir}

composer/fixtures/all-both-k8s.toml

@@ -0,0 +1,214 @@
+[metadata]
+  name = "all-both"
+  author = "adin"
+
+[global]
+  plan = "dht"
+  case = "all"
+  total_instances = 1000
+  builder = "docker:go"
+  runner = "cluster:k8s"
+  [global.build_config]
+    push_registry = true
+    registry_type = "aws"
+
+[[groups]]
+  id = "balsam-undialable-provider"
+  [groups.instances]
+    count = 5
+    percentage = 0.0
+  [groups.build]
+    selectors = ["balsam"]
+  [groups.run]
+    artifact = "909427826938.dkr.ecr.us-east-1.amazonaws.com/testground-us-east-1-dht:701251a63b92"
+    [groups.run.test_params]
+      bs_strategy = "7"
+      bucket_size = "10"
+      expect_dht = "false"
+      group_order = "4"
+      latency = "100"
+      record_count = "1"
+      timeout_secs = "600"
+      undialable = "true"
+
+[[groups]]
+  id = "balsam-undialable-searcher"
+  [groups.instances]
+    count = 5
+    percentage = 0.0
+  [groups.build]
+    selectors = ["balsam"]
+  [groups.run]
+    artifact = "909427826938.dkr.ecr.us-east-1.amazonaws.com/testground-us-east-1-dht:701251a63b92"
+    [groups.run.test_params]
+      bs_strategy = "7"
+      bucket_size = "10"
+      expect_dht = "false"
+      group_order = "5"
+      latency = "100"
+      search_records = "true"
+      timeout_secs = "600"
+      undialable = "true"
+
+[[groups]]
+  id = "balsam-dialable-passive"
+  [groups.instances]
+    count = 780
+    percentage = 0.0
+  [groups.build]
+    selectors = ["balsam"]
+  [groups.run]
+    artifact = "909427826938.dkr.ecr.us-east-1.amazonaws.com/testground-us-east-1-dht:701251a63b92"
+    [groups.run.test_params]
+      bs_strategy = "7"
+      bucket_size = "10"
+      expect_dht = "false"
+      group_order = "6"
+      latency = "100"
+      timeout_secs = "600"
+      undialable = "false"
+
+[[groups]]
+  id = "balsam-dialable-provider"
+  [groups.instances]
+    count = 5
+    percentage = 0.0
+  [groups.build]
+    selectors = ["balsam"]
+  [groups.run]
+    artifact = "909427826938.dkr.ecr.us-east-1.amazonaws.com/testground-us-east-1-dht:701251a63b92"
+    [groups.run.test_params]
+      bs_strategy = "7"
+      bucket_size = "10"
+      expect_dht = "false"
+      group_order = "7"
+      latency = "100"
+      record_count = "1"
+      timeout_secs = "600"
+      undialable = "false"
+
+[[groups]]
+  id = "balsam-dialable-searcher"
+  [groups.instances]
+    count = 5
+    percentage = 0.0
+  [groups.build]
+    selectors = ["balsam"]
+  [groups.run]
+    artifact = "909427826938.dkr.ecr.us-east-1.amazonaws.com/testground-us-east-1-dht:701251a63b92"
+    [groups.run.test_params]
+      bs_strategy = "7"
+      bucket_size = "10"
+      expect_dht = "false"
+      group_order = "8"
+      latency = "100"
+      search_records = "true"
+      timeout_secs = "600"
+      undialable = "false"
+
+[[groups]]
+  id = "cypress-passive"
+  [groups.instances]
+    count = 185
+    percentage = 0.0
+  [groups.build]
+    selectors = ["cypress"]
+
+    [[groups.build.dependencies]]
+      module = "github.com/libp2p/go-libp2p-kad-dht"
+      version = "180be07b8303d536e39809bc39c58be5407fedd9"
+
+    [[groups.build.dependencies]]
+      module = "github.com/libp2p/go-libp2p-xor"
+      version = "df24f5b04bcbdc0059b27989163a6090f4f6dc7a"
+  [groups.run]
+    artifact = "909427826938.dkr.ecr.us-east-1.amazonaws.com/testground-us-east-1-dht:ca78473d669d"
+    [groups.run.test_params]
+      alpha = "6"
+      beta = "3"
+      bs_strategy = "7"
+      bucket_size = "10"
+      group_order = "1"
+      latency = "100"
+      timeout_secs = "600"
+
+[[groups]]
+  id = "cypress-provider"
+  [groups.instances]
+    count = 5
+    percentage = 0.0
+  [groups.build]
+    selectors = ["cypress"]
+
+    [[groups.build.dependencies]]
+      module = "github.com/libp2p/go-libp2p-kad-dht"
+      version = "180be07b8303d536e39809bc39c58be5407fedd9"
+
+    [[groups.build.dependencies]]
+      module = "github.com/libp2p/go-libp2p-xor"
+      version = "df24f5b04bcbdc0059b27989163a6090f4f6dc7a"
+  [groups.run]
+    artifact = "909427826938.dkr.ecr.us-east-1.amazonaws.com/testground-us-east-1-dht:ca78473d669d"
+    [groups.run.test_params]
+      alpha = "6"
+      beta = "3"
+      bs_strategy = "7"
+      bucket_size = "10"
+      group_order = "2"
+      latency = "100"
+      record_count = "1"
+      timeout_secs = "600"
+
+[[groups]]
+  id = "cypress-searcher"
+  [groups.instances]
+    count = 5
+    percentage = 0.0
+  [groups.build]
+    selectors = ["cypress"]
+
+    [[groups.build.dependencies]]
+      module = "github.com/libp2p/go-libp2p-kad-dht"
+      version = "180be07b8303d536e39809bc39c58be5407fedd9"
+
+    [[groups.build.dependencies]]
+      module = "github.com/libp2p/go-libp2p-xor"
+      version = "df24f5b04bcbdc0059b27989163a6090f4f6dc7a"
+  [groups.run]
+    artifact = "909427826938.dkr.ecr.us-east-1.amazonaws.com/testground-us-east-1-dht:ca78473d669d"
+    [groups.run.test_params]
+      alpha = "6"
+      beta = "3"
+      bs_strategy = "7"
+      bucket_size = "10"
+      group_order = "3"
+      latency = "100"
+      search_records = "true"
+      timeout_secs = "600"
+
+[[groups]]
+  id = "cypress-bs"
+  [groups.instances]
+    count = 5
+    percentage = 0.0
+  [groups.build]
+    selectors = ["cypress"]
+
+    [[groups.build.dependencies]]
+      module = "github.com/libp2p/go-libp2p-kad-dht"
+      version = "180be07b8303d536e39809bc39c58be5407fedd9"
+
+    [[groups.build.dependencies]]
+      module = "github.com/libp2p/go-libp2p-xor"
+      version = "df24f5b04bcbdc0059b27989163a6090f4f6dc7a"
+  [groups.run]
+    artifact = "909427826938.dkr.ecr.us-east-1.amazonaws.com/testground-us-east-1-dht:ca78473d669d"
+    [groups.run.test_params]
+      alpha = "6"
+      beta = "3"
+      bootstrapper = "true"
+      bs_strategy = "7"
+      bucket_size = "10"
+      group_order = "0"
+      latency = "100"
+      timeout_secs = "600"

composer/fixtures/example-manifest.toml

@@ -0,0 +1,63 @@
+name = "example"
+
+[defaults]
+builder = "exec:go"
+runner = "local:exec"
+
+[builders."docker:generic"]
+  enabled = true
+  
+[builders."docker:generic".build_args]
+build_image = "golang:alpine"
+run_image = "scratch"
+
+[builders."docker:go"]
+enabled = true
+
+[builders."exec:go"]
+enabled = true
+
+[runners."local:docker"]
+enabled = true
+
+[runners."local:exec"]
+enabled = true
+
+[runners."cluster:swarm"]
+enabled = true
+
+[runners."cluster:k8s"]
+enabled = true
+
+[[testcases]]
+name = "output"
+instances = { min = 1, max = 200, default = 1 }
+
+[[testcases]]
+name = "failure"
+instances = { min = 1, max = 200, default = 1 }
+
+[[testcases]]
+name = "panic"
+instances = { min = 1, max = 200, default = 1 }
+
+[[testcases]]
+name = "params"
+instances = { min = 1, max = 200, default = 1 }
+  [testcases.params]
+  param1 = { type = "int", desc = "some param 1", unit = "widgets", default=1 }
+  param2 = { type = "int", desc = "some param 2", unit = "widgets", default=2 }
+  param3 = { type = "int", desc = "some param 3", unit = "widgets", default=3 }
+
+[[testcases]]
+name = "sync"
+instances = { min = 2, max = 200, default = 5 }
+
+[[testcases]]
+name = "metrics"
+instances = { min = 1, max = 200, default = 5 }
+
+[[testcases]]
+name = "artifact"
+instances = { min = 1, max = 200, default = 5 }
+