From 6eae66728b3f40a76c09bfd891124b0ba4eef490 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Mon, 22 Jun 2020 10:26:59 -0400 Subject: [PATCH] create new composition or load existing --- composer/app/app.py | 90 ++++++++ composer/app/composition.py | 31 +-- composer/app/util.py | 26 +++ composer/composer.ipynb | 439 +++--------------------------------- composer/composer.sh | 3 +- 5 files changed, 151 insertions(+), 438 deletions(-) create mode 100644 composer/app/app.py create mode 100644 composer/app/util.py diff --git a/composer/app/app.py b/composer/app/app.py new file mode 100644 index 000000000..23adfa516 --- /dev/null +++ b/composer/app/app.py @@ -0,0 +1,90 @@ +import param +import panel as pn +import toml +from .util import get_plans, get_manifest +from .composition import Composition + +STAGE_WELCOME = 'Welcome' +STAGE_CONFIG_COMPOSITION = 'Configure' + + +class Welcome(param.Parameterized): + composition = param.Parameter() + composition_picker = pn.widgets.FileInput(accept='.toml') + plan_picker = param.Selector() + ready = param.Boolean() + + def __init__(self, **params): + super().__init__(**params) + self.composition_picker.param.watch(self._composition_updated, 'value') + self.param.watch(self._plan_selected, 'plan_picker') + self.param['plan_picker'].objects = ['Select a Plan'] + get_plans() + + def panel(self): + tabs = pn.Tabs( + ('New Compostion', self.param['plan_picker']), + ('Existing Composition', self.composition_picker), + ) + + return pn.Column( + "Either choose an existing composition or select a plan to create a new composition:", + tabs, + ) + + def _composition_updated(self, *args): + print('composition updated') + content = self.composition_picker.value.decode('utf8') + comp_toml = toml.loads(content) + manifest = get_manifest(comp_toml['global']['plan']) + self.composition = Composition.from_dict(comp_toml, manifest=manifest) + print('existing composition: {}'.format(self.composition)) + self.ready = True + + def _plan_selected(self, evt): + if evt.new == 'Select a Plan': + return + print('plan selected: {}'.format(evt.new)) + manifest = get_manifest(evt.new) + self.composition = Composition(manifest=manifest) + print('new composition: ', self.composition) + self.ready = True + + +class ConfigureComposition(param.Parameterized): + composition = param.Parameter() + + @param.depends('composition') + def panel(self): + if self.composition is None: + return pn.Pane("no composition :(") + print('composition: ', self.composition) + return self.composition.panel() + + +class WorkflowPipeline(object): + def __init__(self): + stages = [ + (STAGE_WELCOME, Welcome(), dict(ready_parameter='ready')), + (STAGE_CONFIG_COMPOSITION, ConfigureComposition()) + ] + + self.pipeline = pn.pipeline.Pipeline(debug=True, stages=stages) + + def panel(self): + return pn.Column( + pn.Row( + self.pipeline.title, + self.pipeline.network, + self.pipeline.prev_button, + self.pipeline.next_button, + ), + self.pipeline.stage, + ) + + +class App(object): + def __init__(self): + self.workflow = WorkflowPipeline() + + def ui(self): + return self.workflow.panel().servable("Testground Composer") diff --git a/composer/app/composition.py b/composer/app/composition.py index ccd50483a..35b39a3f7 100644 --- a/composer/app/composition.py +++ b/composer/app/composition.py @@ -1,31 +1,7 @@ import param import panel as pn import toml -import os -import sys - - -def parse_manifest(manifest_path): - with open(manifest_path, 'rt') as f: - return toml.load(f) - - -def tg_home(): - return os.environ.get('TESTGROUND_HOME', - os.path.join(os.environ['HOME'], 'testground')) - - -def get_plans(): - return list(os.listdir(os.path.join(tg_home(), 'plans'))) - - -def get_manifest(plan_name): - manifest_path = os.path.join(tg_home(), 'plans', plan_name, 'manifest.toml') - return parse_manifest(manifest_path) - - -def print_err(*args): - print(*args, file=sys.stderr) +from .util import get_manifest, print_err def value_dict(parameterized, renames=None): @@ -241,7 +217,10 @@ class Composition(param.Parameterized): add_group_button = pn.widgets.Button(name='Add Group') add_group_button.on_click(self._add_group) - group_panel = self.groups.panel() + if self.groups is None: + group_panel = pn.Column() + else: + group_panel = self.groups.panel() if self.groups_ui is None: self.groups_ui = pn.Column( add_group_button, diff --git a/composer/app/util.py b/composer/app/util.py new file mode 100644 index 000000000..5321a95e8 --- /dev/null +++ b/composer/app/util.py @@ -0,0 +1,26 @@ +import toml +import os +import sys + + +def parse_manifest(manifest_path): + with open(manifest_path, 'rt') as f: + return toml.load(f) + + +def tg_home(): + return os.environ.get('TESTGROUND_HOME', + os.path.join(os.environ['HOME'], 'testground')) + + +def get_plans(): + return list(os.listdir(os.path.join(tg_home(), 'plans'))) + + +def get_manifest(plan_name): + manifest_path = os.path.join(tg_home(), 'plans', plan_name, 'manifest.toml') + return parse_manifest(manifest_path) + + +def print_err(*args): + print(*args, file=sys.stderr) diff --git a/composer/composer.ipynb b/composer/composer.ipynb index 768b8cefc..269d325cd 100644 --- a/composer/composer.ipynb +++ b/composer/composer.ipynb @@ -1439,6 +1439,16 @@ "metadata": {}, "output_type": "display_data" }, + { + "data": { + "text/html": [ + "plan selected: dht
\n", + "new composition:
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, { "data": { "application/vnd.holoviews_exec.v0+json": "", @@ -1449,12 +1459,12 @@ "\n", "\n", "\n", - "
\n", + "
\n", "\n", "" ], "text/plain": [ - "Row\n", - " [0] Column\n", - " [0] Column(margin=5, name='Param00228', width=300)\n", - " [0] StaticText(value='Metadata')\n", - " [1] TextInput(name='Composition name', value='all-both')\n", - " [2] TextInput(name='Author', value='adin')\n", - " [1] Column(margin=5, name='Param00236', width=300)\n", - " [0] StaticText(value='Global')\n", - " [1] TextInput(name='Plan', value='dht')\n", - " [2] TextInput(name='Case', value='all')\n", - " [3] TextInput(name='Builder', value='docker:go')\n", - " [4] TextInput(name='Runner', value='cluster:k8s')\n", - " [5] Spinner(name='Total instances', value=1000)\n", - " [6] DictInput(name='Build config', type=, value={'push_registry': True, ...})\n", - " [7] DictInput(name='Run config', type=)\n", - " [1] Column\n", - " [0] Button(name='Add Group')\n", - " [1] Param(Composition, expand_button=False, parameters=['groups'], show_name=False)\n", - " [2] Column\n", + "Column\n", + " [0] Row\n", + " [0] Markdown(str, margin=(0, 0, 0, 5))\n", + " [1] HoloViews(Overlay, backend='bokeh')\n", + " [2] Button(disabled=True, name='Previous', width=125)\n", + " [3] Button(disabled=True, name='Next', width=125)\n", + " [1] Row\n", + " [0] Column\n", " [0] Markdown(str)\n", - " [1] Column(margin=5, name='Param00161', width=300)\n", - " [0] StaticText(value='Instances')\n", - " [1] Spinner(name='Count', value=5)\n", - " [2] Spinner(name='Percentage')\n", - " [2] Column(margin=5, name='Param00169', width=300)\n", - " [0] StaticText(value='Resources')\n", - " [1] TextInput(name='Memory')\n", - " [2] TextInput(name='Cpu')\n", - " [3] Column(margin=5, name='Param00177', width=300)\n", - " [0] StaticText(value='Build')\n", - " [1] ListInput(name='Selectors', type=, value=['balsam'])\n", - " [2] ListInput(name='Dependencies', type=)\n", - " [4] Column(margin=5, name='Param00185', width=300)\n", - " [0] StaticText(value='Test Params f...)\n", - " [1] Spinner(name='Timeout secs', value=300)\n", - " [2] Spinner(name='Latency', value=100)\n", - " [3] Checkbox(name='Auto refresh', value=True)\n", - " [4] Checkbox(name='Random walk')\n", - " [5] Spinner(name='Bucket size', value=2)\n", - " [6] Spinner(name='Alpha', value=3)\n", - " [7] Spinner(name='Beta', value=3)\n", - " [8] Checkbox(name='Client mode')\n", - " [9] Spinner(name='Datastore')\n", - " [10] Spinner(name='Peer id seed')\n", - " [11] Checkbox(name='Bootstrapper')\n", - " [12] Spinner(name='Bs strategy')\n", - " [13] Checkbox(name='Undialable')\n", - " [14] Spinner(name='Group order')\n", - " [15] Checkbox(name='Expect dht', value=True)\n", - " [16] Spinner(name='Record seed')\n", - " [17] Spinner(name='Record count')\n", - " [18] Checkbox(name='Search records')\n", - " [19] Spinner(name='N find peers')" + " [1] Tabs\n", + " [0] Select(name='Plan picker', options=OrderedDict([('Select a Pl...]), value='Select a Plan')\n", + " [1] FileInput(accept='.toml')" ] }, "execution_count": 1, @@ -1544,366 +1513,14 @@ "source": [ "import param\n", "import panel as pn\n", - "import app.composition as model\n", + "import app.app as app\n", "import importlib\n", - "importlib.reload(model)\n", + "importlib.reload(app)\n", "\n", "pn.extension()\n", "\n", - "c = model.Composition.from_toml_file('fixtures/all-both-k8s.toml')\n", - "ui = c.panel()\n", - "ui.servable()\n", - "ui" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[[groups]]\n", - "id = \"balsam-undialable-provider\"\n", - "\n", - "[groups.instances]\n", - "count = 5\n", - "percentage = 0.0\n", - "[groups.resources]\n", - "memory = \"\"\n", - "cpu = \"\"\n", - "[groups.build]\n", - "selectors = [ \"balsam\",]\n", - "dependencies = []\n", - "[groups.params]\n", - "timeout_secs = 300\n", - "latency = 100\n", - "auto_refresh = true\n", - "random_walk = false\n", - "bucket_size = 2\n", - "alpha = 3\n", - "beta = 3\n", - "client_mode = false\n", - "datastore = 0\n", - "peer_id_seed = 0\n", - "bootstrapper = false\n", - "bs_strategy = 0\n", - "undialable = false\n", - "group_order = 0\n", - "expect_dht = true\n", - "record_seed = 0\n", - "record_count = 0\n", - "search_records = false\n", - "n_find_peers = 0\n", - "[[groups]]\n", - "id = \"balsam-undialable-searcher\"\n", - "\n", - "[groups.instances]\n", - "count = 5\n", - "percentage = 0.0\n", - "[groups.resources]\n", - "memory = \"\"\n", - "cpu = \"\"\n", - "[groups.build]\n", - "selectors = [ \"balsam\",]\n", - "dependencies = []\n", - "[groups.params]\n", - "timeout_secs = 300\n", - "latency = 100\n", - "auto_refresh = true\n", - "random_walk = false\n", - "bucket_size = 2\n", - "alpha = 3\n", - "beta = 3\n", - "client_mode = false\n", - "datastore = 0\n", - "peer_id_seed = 0\n", - "bootstrapper = false\n", - "bs_strategy = 0\n", - "undialable = false\n", - "group_order = 0\n", - "expect_dht = true\n", - "record_seed = 0\n", - "record_count = 0\n", - "search_records = false\n", - "n_find_peers = 0\n", - "[[groups]]\n", - "id = \"balsam-dialable-passive\"\n", - "\n", - "[groups.instances]\n", - "count = 780\n", - "percentage = 0.0\n", - "[groups.resources]\n", - "memory = \"\"\n", - "cpu = \"\"\n", - "[groups.build]\n", - "selectors = [ \"balsam\",]\n", - "dependencies = []\n", - "[groups.params]\n", - "timeout_secs = 300\n", - "latency = 100\n", - "auto_refresh = true\n", - "random_walk = false\n", - "bucket_size = 2\n", - "alpha = 3\n", - "beta = 3\n", - "client_mode = false\n", - "datastore = 0\n", - "peer_id_seed = 0\n", - "bootstrapper = false\n", - "bs_strategy = 0\n", - "undialable = false\n", - "group_order = 0\n", - "expect_dht = true\n", - "record_seed = 0\n", - "record_count = 0\n", - "search_records = false\n", - "n_find_peers = 0\n", - "[[groups]]\n", - "id = \"balsam-dialable-provider\"\n", - "\n", - "[groups.instances]\n", - "count = 5\n", - "percentage = 0.0\n", - "[groups.resources]\n", - "memory = \"\"\n", - "cpu = \"\"\n", - "[groups.build]\n", - "selectors = [ \"balsam\",]\n", - "dependencies = []\n", - "[groups.params]\n", - "timeout_secs = 300\n", - "latency = 100\n", - "auto_refresh = true\n", - "random_walk = false\n", - "bucket_size = 2\n", - "alpha = 3\n", - "beta = 3\n", - "client_mode = false\n", - "datastore = 0\n", - "peer_id_seed = 0\n", - "bootstrapper = false\n", - "bs_strategy = 0\n", - "undialable = false\n", - "group_order = 0\n", - "expect_dht = true\n", - "record_seed = 0\n", - "record_count = 0\n", - "search_records = false\n", - "n_find_peers = 0\n", - "[[groups]]\n", - "id = \"balsam-dialable-searcher\"\n", - "\n", - "[groups.instances]\n", - "count = 5\n", - "percentage = 0.0\n", - "[groups.resources]\n", - "memory = \"\"\n", - "cpu = \"\"\n", - "[groups.build]\n", - "selectors = [ \"balsam\",]\n", - "dependencies = []\n", - "[groups.params]\n", - "timeout_secs = 300\n", - "latency = 100\n", - "auto_refresh = true\n", - "random_walk = false\n", - "bucket_size = 2\n", - "alpha = 3\n", - "beta = 3\n", - "client_mode = false\n", - "datastore = 0\n", - "peer_id_seed = 0\n", - "bootstrapper = false\n", - "bs_strategy = 0\n", - "undialable = false\n", - "group_order = 0\n", - "expect_dht = true\n", - "record_seed = 0\n", - "record_count = 0\n", - "search_records = false\n", - "n_find_peers = 0\n", - "[[groups]]\n", - "id = \"cypress-passive\"\n", - "\n", - "[groups.instances]\n", - "count = 185\n", - "percentage = 0.0\n", - "[groups.resources]\n", - "memory = \"\"\n", - "cpu = \"\"\n", - "[groups.build]\n", - "selectors = [ \"cypress\",]\n", - "[[groups.build.dependencies]]\n", - "module = \"github.com/libp2p/go-libp2p-kad-dht\"\n", - "version = \"180be07b8303d536e39809bc39c58be5407fedd9\"\n", - "\n", - "[[groups.build.dependencies]]\n", - "module = \"github.com/libp2p/go-libp2p-xor\"\n", - "version = \"df24f5b04bcbdc0059b27989163a6090f4f6dc7a\"\n", - "\n", - "[groups.params]\n", - "timeout_secs = 300\n", - "latency = 100\n", - "auto_refresh = true\n", - "random_walk = false\n", - "bucket_size = 2\n", - "alpha = 3\n", - "beta = 3\n", - "client_mode = false\n", - "datastore = 0\n", - "peer_id_seed = 0\n", - "bootstrapper = false\n", - "bs_strategy = 0\n", - "undialable = false\n", - "group_order = 0\n", - "expect_dht = true\n", - "record_seed = 0\n", - "record_count = 0\n", - "search_records = false\n", - "n_find_peers = 0\n", - "[[groups]]\n", - "id = \"cypress-provider\"\n", - "\n", - "[groups.instances]\n", - "count = 5\n", - "percentage = 0.0\n", - "[groups.resources]\n", - "memory = \"\"\n", - "cpu = \"\"\n", - "[groups.build]\n", - "selectors = [ \"cypress\",]\n", - "[[groups.build.dependencies]]\n", - "module = \"github.com/libp2p/go-libp2p-kad-dht\"\n", - "version = \"180be07b8303d536e39809bc39c58be5407fedd9\"\n", - "\n", - "[[groups.build.dependencies]]\n", - "module = \"github.com/libp2p/go-libp2p-xor\"\n", - "version = \"df24f5b04bcbdc0059b27989163a6090f4f6dc7a\"\n", - "\n", - "[groups.params]\n", - "timeout_secs = 300\n", - "latency = 100\n", - "auto_refresh = true\n", - "random_walk = false\n", - "bucket_size = 2\n", - "alpha = 3\n", - "beta = 3\n", - "client_mode = false\n", - "datastore = 0\n", - "peer_id_seed = 0\n", - "bootstrapper = false\n", - "bs_strategy = 0\n", - "undialable = false\n", - "group_order = 0\n", - "expect_dht = true\n", - "record_seed = 0\n", - "record_count = 0\n", - "search_records = false\n", - "n_find_peers = 0\n", - "[[groups]]\n", - "id = \"cypress-searcher\"\n", - "\n", - "[groups.instances]\n", - "count = 5\n", - "percentage = 0.0\n", - "[groups.resources]\n", - "memory = \"\"\n", - "cpu = \"\"\n", - "[groups.build]\n", - "selectors = [ \"cypress\",]\n", - "[[groups.build.dependencies]]\n", - "module = \"github.com/libp2p/go-libp2p-kad-dht\"\n", - "version = \"180be07b8303d536e39809bc39c58be5407fedd9\"\n", - "\n", - "[[groups.build.dependencies]]\n", - "module = \"github.com/libp2p/go-libp2p-xor\"\n", - "version = \"df24f5b04bcbdc0059b27989163a6090f4f6dc7a\"\n", - "\n", - "[groups.params]\n", - "timeout_secs = 300\n", - "latency = 100\n", - "auto_refresh = true\n", - "random_walk = false\n", - "bucket_size = 2\n", - "alpha = 3\n", - "beta = 3\n", - "client_mode = false\n", - "datastore = 0\n", - "peer_id_seed = 0\n", - "bootstrapper = false\n", - "bs_strategy = 0\n", - "undialable = false\n", - "group_order = 0\n", - "expect_dht = true\n", - "record_seed = 0\n", - "record_count = 0\n", - "search_records = false\n", - "n_find_peers = 0\n", - "[[groups]]\n", - "id = \"cypress-bs\"\n", - "\n", - "[groups.instances]\n", - "count = 5\n", - "percentage = 0.0\n", - "[groups.resources]\n", - "memory = \"\"\n", - "cpu = \"\"\n", - "[groups.build]\n", - "selectors = [ \"cypress\",]\n", - "[[groups.build.dependencies]]\n", - "module = \"github.com/libp2p/go-libp2p-kad-dht\"\n", - "version = \"180be07b8303d536e39809bc39c58be5407fedd9\"\n", - "\n", - "[[groups.build.dependencies]]\n", - "module = \"github.com/libp2p/go-libp2p-xor\"\n", - "version = \"df24f5b04bcbdc0059b27989163a6090f4f6dc7a\"\n", - "\n", - "[groups.params]\n", - "timeout_secs = 300\n", - "latency = 100\n", - "auto_refresh = true\n", - "random_walk = false\n", - "bucket_size = 2\n", - "alpha = 3\n", - "beta = 3\n", - "client_mode = false\n", - "datastore = 0\n", - "peer_id_seed = 0\n", - "bootstrapper = false\n", - "bs_strategy = 0\n", - "undialable = false\n", - "group_order = 0\n", - "expect_dht = true\n", - "record_seed = 0\n", - "record_count = 0\n", - "search_records = false\n", - "n_find_peers = 0\n", - "\n", - "[metadata]\n", - "name = \"all-both\"\n", - "author = \"adin\"\n", - "\n", - "[global]\n", - "plan = \"dht\"\n", - "case = \"all\"\n", - "builder = \"docker:go\"\n", - "runner = \"cluster:k8s\"\n", - "total_instances = 1000\n", - "\n", - "[global.build_config]\n", - "push_registry = true\n", - "registry_type = \"aws\"\n", - "\n", - "[global.run_config]\n", - "\n" - ] - } - ], - "source": [ - "print(c.to_toml())" + "a = app.App()\n", + "a.ui()" ] } ], diff --git a/composer/composer.sh b/composer/composer.sh index 72dd1f984..f7d1a1f2b 100755 --- a/composer/composer.sh +++ b/composer/composer.sh @@ -80,7 +80,8 @@ cleanup () { trap "{ cleanup; }" EXIT # make sure we have the commands we need -require_cmds jq docker +require_cmds jq dockerimport param +import panel as pn # make temp dir for manifests temp_base="/tmp"