Compare commits

..
Author SHA1 Message Date
telackey 9043a67c7c Skip checks on requests we've already seen (#805)
Reviewed-on: cerc-io/stack-orchestrator#805
Co-authored-by: Thomas E Lackey <telackey@bozemanpass.com>
Co-committed-by: Thomas E Lackey <telackey@bozemanpass.com>
2024-04-15 20:27:35 +00:00
telackey 7f84a45cfd Switch repo to cerc-io org. (#804)
Update stack to track moved repo.

Reviewed-on: cerc-io/stack-orchestrator#804
Reviewed-by: David Boreham <dboreham@noreply.git.vdb.to>
2024-04-15 18:59:08 +00:00
telackey 4126f2fc43 Add --fqdn-policy option to deploy-webapp-from-registry. (#802)
This add a new option `--fqdn-policy` to the `deploy-webapp-from-registry`.

The default policy, `prohibit` means that `ApplicationDeploymentRequests` which specify a FQDN will be rejected.  The `allow` policy will cause them to be processed.  The `preexisting` policy will only process them if an existing `DnsRecord` exists in the registry with the correct ownership.

The latter would be useful in conjunction with a pre-checking scheme in the UI (eg, that the DNS entry is properly configured, the domain is under the control of the requestor, etc.)  Only after all the checks were successful would the `DnsRecord` be created, allowing for `ApplicationDeploymentRequests` to use it.

Reviewed-on: cerc-io/stack-orchestrator#802
Reviewed-by: David Boreham <dboreham@noreply.git.vdb.to>
Co-authored-by: Thomas E Lackey <telackey@bozemanpass.com>
Co-committed-by: Thomas E Lackey <telackey@bozemanpass.com>
2024-04-15 12:20:35 +00:00
8 changed files with 138 additions and 281 deletions

No files matched your search

@@ -1,147 +0,0 @@
apiVersion: 1
groups:
- orgId: 1
name: blackbox
folder: BlackboxAlerts
interval: 30s
rules:
# Azimuth Gateway endpoint
- uid: azimuth_gateway
title: azimuth_gateway_endpoint_tracking
condition: condition
data:
- refId: probe
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
editorMode: code
expr: probe_success{destination="azimuth_gateway"}
instant: true
intervalMs: 1000
legendFormat: __auto
maxDataPoints: 43200
range: false
refId: probe
- refId: http_status_code
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
editorMode: code
expr: probe_http_status_code{destination="azimuth_gateway"}
instant: true
intervalMs: 1000
legendFormat: __auto
maxDataPoints: 43200
range: false
refId: http_status_code
- refId: condition
relativeTimeRange:
from: 600
to: 0
datasourceUid: __expr__
model:
conditions:
- evaluator:
params:
- 0
- 0
type: gt
operator:
type: and
query:
params: []
reducer:
params: []
type: avg
type: query
datasource:
name: Expression
type: __expr__
uid: __expr__
expression: ${probe} != 1 || ${http_status_code} != 200
intervalMs: 1000
maxDataPoints: 43200
refId: condition
type: math
noDataState: Alerting
execErrState: Alerting
for: 5m
annotations:
summary: Probe failed for Azimuth gateway endpoint
labels:
probe_success: '{{ index $values "probe" }}'
isPaused: false
# Laconicd GQL endpoint
- uid: laconicd_gql
title: laconicd_gql_endpoint_tracking
condition: condition
data:
- refId: probe
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
editorMode: code
expr: probe_success{destination="laconicd_gql"}
instant: true
intervalMs: 1000
legendFormat: __auto
maxDataPoints: 43200
range: false
refId: probe
- refId: http_status_code
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
editorMode: code
expr: probe_http_status_code{destination="laconicd_gql"}
instant: true
intervalMs: 1000
legendFormat: __auto
maxDataPoints: 43200
range: false
refId: http_status_code
- refId: condition
relativeTimeRange:
from: 600
to: 0
datasourceUid: __expr__
model:
conditions:
- evaluator:
params:
- 0
- 0
type: gt
operator:
type: and
query:
params: []
reducer:
params: []
type: avg
type: query
datasource:
name: Expression
type: __expr__
uid: __expr__
expression: ${probe} != 1 || ${http_status_code} != 200
intervalMs: 1000
maxDataPoints: 43200
refId: condition
type: math
noDataState: Alerting
execErrState: Alerting
for: 5m
annotations:
summary: Probe failed for Laconicd GQL endpoint
labels:
probe_success: '{{ index $values "probe" }}'
isPaused: false
@@ -49,7 +49,7 @@
},
"gridPos": {
"h": 3,
"w": 4,
"w": 3,
"x": 0,
"y": 0
},
@@ -24,10 +24,9 @@ scrape_configs:
params:
module: [http_2xx]
static_configs:
# Add URLs for targets to be monitored below
# - targets: [https://github.com]
# labels:
# destination: 'github'
# Add URLs to be monitored below
- targets:
# - https://github.com
relabel_configs:
- source_labels: [__address__]
regex: (.*)(:80)?
@@ -59,29 +59,29 @@ groups:
conditions:
- evaluator:
params:
- 15
- 0
- 0
type: gt
operator:
type: when
type: and
query:
params:
- diff
params: []
reducer:
params: []
type: last
type: avg
type: query
datasource:
name: Expression
type: __expr__
uid: __expr__
expression: ""
hide: false
expression: ${diff} >= 16
intervalMs: 1000
maxDataPoints: 43200
refId: condition
type: classic_conditions
type: math
noDataState: Alerting
execErrState: Alerting
for: 5m
for: 15m
annotations:
summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
isPaused: false
@@ -135,29 +135,29 @@ groups:
conditions:
- evaluator:
params:
- 15
- 0
- 0
type: gt
operator:
type: when
type: and
query:
params:
- diff
params: []
reducer:
params: []
type: last
type: avg
type: query
datasource:
name: Expression
type: __expr__
uid: __expr__
expression: ""
hide: false
expression: ${diff} >= 16
intervalMs: 1000
maxDataPoints: 43200
refId: condition
type: classic_conditions
type: math
noDataState: Alerting
execErrState: Alerting
for: 5m
for: 15m
annotations:
summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
isPaused: false
@@ -202,7 +202,7 @@ groups:
legendFormat: __auto
range: false
refId: latest_external
- refId: condition
- refId: condition
relativeTimeRange:
from: 600
to: 0
@@ -211,29 +211,29 @@ groups:
conditions:
- evaluator:
params:
- 15
- 0
- 0
type: gt
operator:
type: when
type: and
query:
params:
- diff
params: []
reducer:
params: []
type: last
type: avg
type: query
datasource:
name: Expression
type: __expr__
uid: __expr__
expression: ""
hide: false
expression: ${diff} >= 16
intervalMs: 1000
maxDataPoints: 43200
refId: condition
type: classic_conditions
type: math
noDataState: Alerting
execErrState: Alerting
for: 5m
for: 15m
annotations:
summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
isPaused: false
@@ -278,7 +278,7 @@ groups:
legendFormat: __auto
range: false
refId: latest_external
- refId: condition
- refId: condition
relativeTimeRange:
from: 600
to: 0
@@ -287,29 +287,29 @@ groups:
conditions:
- evaluator:
params:
- 15
- 0
- 0
type: gt
operator:
type: when
type: and
query:
params:
- diff
params: []
reducer:
params: []
type: last
type: avg
type: query
datasource:
name: Expression
type: __expr__
uid: __expr__
expression: ""
hide: false
expression: ${diff} >= 16
intervalMs: 1000
maxDataPoints: 43200
refId: condition
type: classic_conditions
type: math
noDataState: Alerting
execErrState: Alerting
for: 5m
for: 15m
annotations:
summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
isPaused: false
@@ -363,29 +363,29 @@ groups:
conditions:
- evaluator:
params:
- 15
- 0
- 0
type: gt
operator:
type: when
type: and
query:
params:
- diff
params: []
reducer:
params: []
type: last
type: avg
type: query
datasource:
name: Expression
type: __expr__
uid: __expr__
expression: ""
hide: false
expression: ${diff} >= 16
intervalMs: 1000
maxDataPoints: 43200
refId: condition
type: classic_conditions
type: math
noDataState: Alerting
execErrState: Alerting
for: 5m
for: 15m
annotations:
summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
isPaused: false
@@ -439,29 +439,29 @@ groups:
conditions:
- evaluator:
params:
- 15
- 0
- 0
type: gt
operator:
type: when
type: and
query:
params:
- diff
params: []
reducer:
params: []
type: last
type: avg
type: query
datasource:
name: Expression
type: __expr__
uid: __expr__
expression: ""
hide: false
expression: ${diff} >= 16
intervalMs: 1000
maxDataPoints: 43200
refId: condition
type: classic_conditions
type: math
noDataState: Alerting
execErrState: Alerting
for: 5m
for: 15m
annotations:
summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
isPaused: false
@@ -515,29 +515,29 @@ groups:
conditions:
- evaluator:
params:
- 15
- 0
- 0
type: gt
operator:
type: when
type: and
query:
params:
- diff
params: []
reducer:
params: []
type: last
type: avg
type: query
datasource:
name: Expression
type: __expr__
uid: __expr__
expression: ""
hide: false
expression: ${diff} >= 16
intervalMs: 1000
maxDataPoints: 43200
refId: condition
type: classic_conditions
type: math
noDataState: Alerting
execErrState: Alerting
for: 5m
for: 15m
annotations:
summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
isPaused: false
@@ -591,29 +591,29 @@ groups:
conditions:
- evaluator:
params:
- 15
- 0
- 0
type: gt
operator:
type: when
type: and
query:
params:
- diff
params: []
reducer:
params: []
type: last
type: avg
type: query
datasource:
name: Expression
type: __expr__
uid: __expr__
expression: ""
hide: false
expression: ${diff} >= 16
intervalMs: 1000
maxDataPoints: 43200
refId: condition
type: classic_conditions
type: math
noDataState: Alerting
execErrState: Alerting
for: 5m
for: 15m
annotations:
summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
isPaused: false
@@ -669,29 +669,29 @@ groups:
conditions:
- evaluator:
params:
- 15
- 0
- 0
type: gt
operator:
type: when
type: and
query:
params:
- diff
params: []
reducer:
params: []
type: last
type: avg
type: query
datasource:
name: Expression
type: __expr__
uid: __expr__
expression: ""
hide: false
expression: ${diff} >= 16
intervalMs: 1000
maxDataPoints: 43200
refId: condition
type: classic_conditions
type: math
noDataState: Alerting
execErrState: Alerting
for: 5m
for: 15m
annotations:
summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
isPaused: false
@@ -745,29 +745,29 @@ groups:
conditions:
- evaluator:
params:
- 15
- 0
- 0
type: gt
operator:
type: when
type: and
query:
params:
- diff
params: []
reducer:
params: []
type: last
type: avg
type: query
datasource:
name: Expression
type: __expr__
uid: __expr__
expression: ""
hide: false
expression: ${diff} >= 16
intervalMs: 1000
maxDataPoints: 43200
refId: condition
type: classic_conditions
type: math
noDataState: Alerting
execErrState: Alerting
for: 5m
for: 15m
annotations:
summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
isPaused: false
@@ -823,29 +823,29 @@ groups:
conditions:
- evaluator:
params:
- 15
- 0
- 0
type: gt
operator:
type: when
type: and
query:
params:
- diff
params: []
reducer:
params: []
type: last
type: avg
type: query
datasource:
name: Expression
type: __expr__
uid: __expr__
expression: ""
hide: false
expression: ${diff} >= 16
intervalMs: 1000
maxDataPoints: 43200
refId: condition
type: classic_conditions
type: math
noDataState: Alerting
execErrState: Alerting
for: 5m
for: 15m
annotations:
summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
isPaused: false
@@ -123,9 +123,8 @@ laconic-so --stack monitoring deploy create --spec-file monitoring-spec.yml --de
```yml
...
- job_name: laconicd
...
static_configs:
- targets: ['example-host:1317']
static_configs:
- targets: ['example-host:1317']
...
```
@@ -44,18 +44,13 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers
- job_name: 'blackbox'
...
static_configs:
- targets: [<AZIMUTH_GATEWAY_GQL_ENDPOINT>]
labels:
# Add destination label for pre-configured alerts
destination: 'azimuth_gateway'
- targets: [<LACONICD_GQL_ENDPOINT>]
labels:
destination: 'laconicd_gql'
- targets:
- <AZIMUTH_GATEWAY_GQL_ENDPOINT>
- <LACONICD_GQL_ENDPOINT>
...
- job_name: laconicd
...
static_configs:
- targets: ['LACONICD_REST_HOST:LACONICD_REST_PORT']
static_configs:
- targets: ['LACONICD_REST_HOST:LACONICD_REST_PORT']
...
- job_name: azimuth
scrape_interval: 10s
@@ -124,12 +119,10 @@ Add scrape config as done above for any additional watcher to add it to the Watc
### Grafana alerts config
Place the pre-configured watcher and blackbox endpoint alerts rules in Grafana provisioning directory:
Place the pre-configured watcher alerts rules in Grafana provisioning directory:
```bash
cp monitoring-watchers-deployment/config/monitoring/watcher-alert-rules.yml monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/
cp monitoring-watchers-deployment/config/monitoring/blackbox-alert-rules.yml monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/
```
Update the alerting contact points config (`monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/contactpoints.yml`) with desired contact points
@@ -142,7 +135,7 @@ Add corresponding routes to the notification policies config (`monitoring-watche
- receiver: SlackNotifier
object_matchers:
# Add matchers below
- ['grafana_folder', '=~', 'WatcherAlerts|BlackboxAlerts']
- ['grafana_folder', '=', 'WatcherAlerts']
```
### Env
@@ -2,10 +2,10 @@ version: "1.0"
name: webapp-deployer-backend
description: "Deployer for webapps"
repos:
- git.vdb.to/telackey/webapp-deployment-status-api
- git.vdb.to/cerc-io/webapp-deployment-status-api
containers:
- cerc/webapp-deployer-backend
pods:
- name: webapp-deployer-backend
repository: git.vdb.to/telackey/webapp-deployment-status-api
repository: git.vdb.to/cerc-io/webapp-deployment-status-api
path: ./
@@ -39,11 +39,12 @@ def process_app_deployment_request(
app_deployment_request,
deployment_record_namespace,
dns_record_namespace,
dns_suffix,
default_dns_suffix,
deployment_parent_dir,
kube_config,
image_registry,
force_rebuild,
fqdn_policy,
logger
):
logger.log("BEGIN - process_app_deployment_request")
@@ -56,14 +57,15 @@ def process_app_deployment_request(
requested_name = hostname_for_deployment_request(app_deployment_request, laconic)
logger.log(f"Determined requested name: {requested_name}")
# HACK
if "." in requested_name:
raise Exception("Only unqualified hostnames allowed at this time.")
fqdn = f"{requested_name}.{dns_suffix}"
if "allow" == fqdn_policy or "preexisting" == fqdn_policy:
fqdn = requested_name
else:
raise Exception(f"{requested_name} is invalid: only unqualified hostnames are allowed.")
else:
fqdn = f"{requested_name}.{default_dns_suffix}"
# 3. check ownership of existing dnsrecord vs this request
# TODO: Support foreign DNS
dns_crn = f"{dns_record_namespace}/{fqdn}"
dns_record = laconic.get_record(dns_crn)
if dns_record:
@@ -75,7 +77,9 @@ def process_app_deployment_request(
logger.log(f"Matched DnsRecord ownership: {matched_owner}")
else:
raise Exception("Unable to confirm ownership of DnsRecord %s for request %s" %
(dns_record.id, app_deployment_request.id))
(dns_crn, app_deployment_request.id))
elif "preexisting" == fqdn_policy:
raise Exception(f"No pre-existing DnsRecord {dns_crn} could be found for request {app_deployment_request.id}.")
# 4. get build and runtime config from request
env_filename = None
@@ -191,6 +195,7 @@ def dump_known_requests(filename, requests, status="SEEN"):
@click.option("--state-file", help="File to store state about previously seen requests.")
@click.option("--only-update-state", help="Only update the state file, don't process any requests anything.", is_flag=True)
@click.option("--dns-suffix", help="DNS domain to use eg, laconic.servesthe.world")
@click.option("--fqdn-policy", help="How to handle requests with an FQDN: prohibit, allow, preexisting", default="prohibit")
@click.option("--record-namespace-dns", help="eg, crn://laconic/dns")
@click.option("--record-namespace-deployments", help="eg, crn://laconic/deployments")
@click.option("--dry-run", help="Don't do anything, just report what would be done.", is_flag=True)
@@ -201,7 +206,7 @@ def dump_known_requests(filename, requests, status="SEEN"):
@click.pass_context
def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_dir, # noqa: C901
request_id, discover, state_file, only_update_state,
dns_suffix, record_namespace_dns, record_namespace_deployments, dry_run,
dns_suffix, fqdn_policy, record_namespace_dns, record_namespace_deployments, dry_run,
include_tags, exclude_tags, force_rebuild, log_dir):
if request_id and discover:
print("Cannot specify both --request-id and --discover", file=sys.stderr)
@@ -220,6 +225,10 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
print("--dns-suffix, --record-namespace-dns, and --record-namespace-deployments are all required", file=sys.stderr)
sys.exit(2)
if fqdn_policy not in ["prohibit", "allow", "preexisting"]:
print("--fqdn-policy must be one of 'prohibit', 'allow', or 'preexisting'", file=sys.stderr)
sys.exit(2)
# Split CSV and clean up values.
include_tags = [tag.strip() for tag in include_tags.split(",") if tag]
exclude_tags = [tag.strip() for tag in exclude_tags.split(",") if tag]
@@ -247,7 +256,10 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
requests_by_name = {}
skipped_by_name = {}
for r in requests:
# TODO: Do this _after_ filtering deployments and cancellations to minimize round trips.
if r.id in previous_requests and previous_requests[r.id].get("status", "") != "RETRY":
print(f"Skipping request {r.id}, we've already seen it.")
continue
app = laconic.get_record(r.attributes.application)
if not app:
print("Skipping request %s, cannot locate app." % r.id)
@@ -334,6 +346,7 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
kube_config,
image_registry,
force_rebuild,
fqdn_policy,
logger
)
status = "DEPLOYED"